[Gelistirici] fetcher at PiSi
S.Çağlar Onur
caglar at pardus.org.tr
1 Şub 2008 Cum 03:04:19 EET
Selamlar;
Aşağıdaki yama fetcher modülünü değiştirerek;
1. URLGrabber'ın Mirrors özelliğinin kullanılmasını sağlıyor,
2. _test_range_support fonksiyonunu URLGrabber'a yüklüyor (downloadlara resume desteği),
3. İndirilen dosyanın önce dosyaadı.part sonra dosyaadı haline gelmesi işini bırakıyor, urlgrabber zaten eksik download edilmiş arkadaşlara resume desteğini sağlıyor
4. Arada raise edilmemiş bir exception'ı raise ediyor, ufak tefek kod stili temizliği yapıyor
Buradan sonra UIHandler arkadaşı da urlgrabber TextMeter arkadaş ile değiştirmek niyetindeyim (fakat dışardan kullanlar var), fetcher ile ilgili olası tüm işleri urlgrabber'a atıp, kendi yazdığımız kısımları minimuma çekmek niyetindeyim.
Bu hali remote buildleri kırıyor fakat daha fazla ilerlemeden fikri destekleyip desteklemediğiniz öğreneyim istedim, özetle review lütfen :)
Index: pisi/sourcearchive.py
===================================================================
--- pisi/sourcearchive.py (revision 16916)
+++ pisi/sourcearchive.py (working copy)
@@ -45,10 +45,7 @@
self.progress = None
try:
- if self.url.get_uri().startswith("mirrors://"):
- self.fetch_from_mirror()
- else:
- pisi.fetcher.fetch_url(self.url, ctx.config.archives_dir(), self.progress)
+ pisi.fetcher.fetch_url(self.url, ctx.config.archives_dir(), self.progress)
except pisi.fetcher.FetchError:
if ctx.config.values.build.fallback:
self.fetch_from_fallback()
@@ -58,30 +55,12 @@
def fetch_from_fallback(self):
archive = os.path.basename(self.url.get_uri())
src = os.path.join(ctx.config.values.build.fallback, archive)
- ctx.ui.warning(_('Trying fallback address: %s') % src)
- pisi.fetcher.fetch_url(src, ctx.config.archives_dir(), self.progress)
+ ctx.ui.warning(_('\nTrying fallback address: %s') % src)
+ try:
+ pisi.fetcher.fetch_url(src, ctx.config.archives_dir(), self.progress)
+ except pisi.fetcher.FetchError, e:
+ raise pisi.fetcher.FetchError(_('Could not fetch source from %s fallback.') % src);
- def fetch_from_mirror(self):
- uri = self.url.get_uri()
- sep = uri[len("mirrors://"):].split("/")
- name = sep.pop(0)
- archive = "/".join(sep)
-
- mirrors = pisi.mirrors.Mirrors().get_mirrors(name)
- if not mirrors:
- raise Error(_("%s mirrors are not defined.") % name)
-
- for mirror in mirrors:
- try:
- url = os.path.join(mirror, archive)
- ctx.ui.warning(_('Fetching source from mirror: %s') % url)
- pisi.fetcher.fetch_url(url, ctx.config.archives_dir(), self.progress)
- return
- except pisi.fetcher.FetchError:
- pass
-
- raise pisi.fetcher.FetchError(_('Could not fetch source from %s mirrors.') % name);
-
def is_cached(self, interactive=True):
if not os.access(self.archiveFile, os.R_OK):
return False
@@ -95,10 +74,9 @@
return False
def unpack(self, clean_dir=True):
-
# check archive file's integrity
if not util.check_file_hash(self.archiveFile, self.archive.sha1sum):
- raise Error, _("unpack: check_file_hash failed")
+ raise Error, _('unpack: check_file_hash failed')
archive = pisi.archive.Archive(self.archiveFile, self.archive.type)
archive.unpack(self.pkg_work_dir, clean_dir)
Index: pisi/fetcher.py
===================================================================
--- pisi/fetcher.py (revision 16916)
+++ pisi/fetcher.py (working copy)
@@ -103,24 +103,23 @@
if not isinstance(url, pisi.uri.URI):
url = pisi.uri.URI(url)
- if ctx.config.get_option("authinfo"):
- url.set_auth_info(ctx.config.get_option("authinfo"))
+ if ctx.config.get_option('authinfo'):
+ url.set_auth_info(ctx.config.get_option('authinfo'))
self.url = url
self.destdir = destdir
self.archive_file = os.path.join(self.destdir, self.url.filename())
- self.partial_file = self.archive_file + '.part'
self.progress = None
util.check_dir(self.destdir)
-
def fetch (self):
"""Return value: Fetched file's full path.."""
# import urlgrabber module
try:
- import urlgrabber
+ from urlgrabber.grabber import URLGrabber, URLGrabError
+ from urlgrabber.mirror import MirrorGroup
except ImportError:
raise FetchError(_('Urlgrabber needs to be installed to run this command'))
@@ -134,36 +133,51 @@
raise FetchError(_('Access denied to destination file: "%s"') % (self.archive_file))
try:
- urlgrabber.urlgrab(self.url.get_uri(),
- self.partial_file,
- progress_obj = UIHandler(self.progress),
- http_headers = self._get_http_headers(),
- ftp_headers = self._get_ftp_headers(),
- proxies = self._get_proxies(),
- throttle = self._get_bandwith_limit(),
- reget = self._test_range_support(),
- user_agent = 'PiSi Fetcher/' + pisi.__version__)
- except urlgrabber.grabber.URLGrabError, e:
- raise FetchError(_('Could not fetch destination file "%s": %s') % (self.archive_file, e))
+ if self.url.get_uri().startswith('mirrors://'):
+ uri = self.url.get_uri()
+ sep = uri[len('mirrors://'):].split('/')
+ name = sep.pop(0)
+ archive = '/'.join(sep)
- if os.stat(self.partial_file).st_size == 0:
- os.remove(self.partial_file)
- FetchError(_('A problem occurred. Please check the archive address and/or permissions again.'))
+ mirrors = pisi.mirrors.Mirrors().get_mirrors(name)
+ if not mirrors:
+ raise Error(_('%s mirrors are not defined.') % name)
+ uri = archive
+ else:
+ uri = self.url.get_uri()
+ mirrors = []
- shutil.move(self.partial_file, self.archive_file)
+ fetcher = URLGrabber(
+ progress_obj = UIHandler(self.progress),
+ http_headers = self._get_http_headers(),
+ ftp_headers = self._get_ftp_headers(),
+ proxies = self._get_proxies(),
+ throttle = self._get_bandwith_limit(),
+ reget = 'simple',
+ keepalive = 1,
+ user_agent = 'PiSi Fetcher/' + pisi.__version__)
+ mg = MirrorGroup(fetcher, mirrors)
+ mg.urlgrab(uri, self.archive_file)
+ except URLGrabError, e:
+ raise FetchError(_('Could not fetch destination file "%s": %s') % (self.archive_file, e))
+
+ if os.stat(self.archive_file).st_size == 0:
+ os.remove(self.archive_file)
+ raise FetchError(_('A problem occurred. Please check the archive address and/or permissions again.'))
+
return self.archive_file
def _get_http_headers(self):
headers = []
- if self.url.auth_info() and (self.url.scheme() == "http" or self.url.scheme() == "https"):
+ if self.url.auth_info() and (self.url.scheme() == 'http' or self.url.scheme() == 'https'):
enc = base64.encodestring('%s:%s' % self.url.auth_info())
headers.append(('Authorization', 'Basic %s' % enc),)
return tuple(headers)
def _get_ftp_headers(self):
headers = []
- if self.url.auth_info() and self.url.scheme() == "ftp":
+ if self.url.auth_info() and self.url.scheme() == 'ftp':
enc = base64.encodestring('%s:%s' % self.url.auth_info())
headers.append(('Authorization', 'Basic %s' % enc),)
return tuple(headers)
@@ -171,44 +185,28 @@
def _get_proxies(self):
proxies = {}
- if ctx.config.values.general.http_proxy and self.url.scheme() == "http":
+ if ctx.config.values.general.http_proxy and self.url.scheme() == 'http':
proxies[pisi.uri.URI(http_proxy).scheme()] = ctx.config.values.general.http_proxy
- if ctx.config.values.general.https_proxy and self.url.scheme() == "https":
+ if ctx.config.values.general.https_proxy and self.url.scheme() == 'https':
proxies[pisi.uri.URI(https_proxy).scheme()] = ctx.config.values.general.https_proxy
- if ctx.config.values.general.ftp_proxy and self.url.scheme() == "ftp":
+ if ctx.config.values.general.ftp_proxy and self.url.scheme() == 'ftp':
proxies[pisi.uri.URI(ftp_proxy).scheme()] = ctx.config.values.general.ftp_proxy
if self.url.scheme() in proxies:
- ctx.ui.info(_("Proxy configuration has been found for '%s' protocol") % self.url.scheme())
+ ctx.ui.info(_('Proxy configuration has been found for "%s" protocol') % self.url.scheme())
return proxies
def _get_bandwith_limit(self):
bandwidth_limit = ctx.config.options.bandwidth_limit or ctx.config.values.general.bandwidth_limit
if bandwidth_limit:
- ctx.ui.warning(_("Bandwidth usage is limited to %s KB/s") % bandwidth_limit)
+ ctx.ui.warning(_('Bandwidth usage is limited to %s KB/s') % bandwidth_limit)
return 1024 * int(bandwidth_limit)
else:
return 0
- def _test_range_support(self):
- if not os.path.exists(self.partial_file):
- return None
-
- import urllib2
- file_obj = urllib2.urlopen(urllib2.Request(self.url.get_uri()))
- headers = file_obj.info()
- file_obj.close()
- if headers.has_key('Content-Length'):
- return 'check_timestamp'
- else:
- ctx.ui.debug(_("Server doesn't support partial downloads. Previously downloaded part of the file will be over-written."))
- os.remove(self.partial_file)
- return None
-
-
# helper function
def fetch_url(url, destdir, progress=None):
fetch = Fetcher(url, destdir)
Saygılar
--
S.Çağlar Onur <caglar at pardus.org.tr>
http://cekirdek.pardus.org.tr/~caglar/
Linux is like living in a teepee. No Windows, no Gates and an Apache in house!
-------------- sonraki bölüm --------------
A non-text attachment was scrubbed...
Name: kullanılamıyor
Type: application/pgp-signature
Size: 189 bytes
Desc: This is a digitally signed message part.
URL: <http://liste.pardus.org.tr/gelistirici/attachments/20080201/f9fd2ab5/attachment-0002.pgp>
Gelistirici mesaj listesiyle ilgili
daha fazla bilgi