diff --git a/pisi/fetcher.py b/pisi/fetcher.py index 3e3c65f8..01d8d26c 100644 --- a/pisi/fetcher.py +++ b/pisi/fetcher.py @@ -16,12 +16,14 @@ all kinds of things: source tarballs, index files, packages, and God knows what.""" # python standard library modules +import contextlib import os import time import base64 import shutil import gettext + __trans = gettext.translation('pisi', fallback=True) _ = __trans.gettext @@ -30,74 +32,72 @@ import pisi import pisi.util as util import pisi.context as ctx import pisi.uri +import pisi.request class FetchError(pisi.Error): pass -class UIHandler: - def __init__(self, progress): - self.filename = None - self.url = None - self.basename = None - self.downloaded_size = 0 - self.percent = None - self.rate = 0.0 - self.size = 0 - self.eta = '--:--:--' - self.symbol = '--/-' - self.last_updated = 0 - self.exist_size = 0 - - def start(self, archive, url, basename, size, text): +class FetchHandler: + def __init__(self, url, archive, bandwidth_limit): + self.url = url + self.percent = None + self.rate = 0.0 + self.size = 0 + self.eta = '--:--:--' + self.symbol = '--/-' + self.last_updated = 0 + self.filename = url.filename() + self.total_size = 0 + self.exist_size = 0 + self.bandwidth_limit = bandwidth_limit if os.path.exists(archive): self.exist_size = os.path.getsize(archive) - self.filename = util.remove_suffix(ctx.const.partial_suffix, basename) - self.url = url - self.basename = basename - self.total_size = size or 0 - self.text = text - self.now = lambda: time.time() + self.now = lambda: time.time() self.t_diff = lambda: self.now() - self.s_time self.s_time = self.now() - def update(self, size): - - if self.size == size: - return - - self.size = size + def update(self, blocknum, bs, size): + self.total_size = size + self.exist_size + self.size = blocknum * bs + self.exist_size if self.total_size: - self.percent = (size * 100.0) / self.total_size + self.percent = self.size * 100 / self.total_size else: self.percent = 0 - if int(self.now()) != int(self.last_updated) and size > 0: + if int(self.now()) != int(self.last_updated) and self.size > 0: try: - self.rate, self.symbol = util.human_readable_rate((size - self.exist_size) / (self.now() - self.s_time)) + self.rate, self.symbol = util.human_readable_rate(self.size / (self.now() - self.s_time)) except ZeroDivisionError: return if self.total_size: - self.eta = '%02d:%02d:%02d' %\ + self.eta = '%02d:%02d:%02d' %\ tuple([i for i in time.gmtime((self.t_diff() * (100 - self.percent)) / self.percent)[3:6]]) self._update_ui() + self._limit_bandwidth() - def end(self, read): - pass + def _limit_bandwidth(self): + if self.bandwidth_limit: + expected_time = (self.size - self.exist_size) / self.bandwidth_limit + sleep_time = expected_time - self.t_diff() + if sleep_time > 0: + time.sleep(sleep_time) def _update_ui(self): - ctx.ui.display_progress(operation = "fetching", - percent = self.percent, - filename = self.filename, - total_size = self.total_size or self.size, - downloaded_size = self.size, - rate = self.rate, - eta = self.eta, - symbol = self.symbol) + ctx.ui.display_progress( + operation="fetching", + percent=self.percent, + filename=self.filename, + total_size=self.total_size, + downloaded_size=self.size, + rate=self.rate, + eta=self.eta, + symbol=self.symbol + ) self.last_updated = self.now() @@ -115,58 +115,67 @@ class Fetcher: self.url = url self.destdir = destdir self.destfile = destfile - self.progress = None self.archive_file = os.path.join(destdir, destfile or url.filename()) self.partial_file = os.path.join(self.destdir, self.url.filename()) + ctx.const.partial_suffix util.ensure_dirs(self.destdir) - def test(self, timeout=3): - import urlgrabber - - try: - urlgrabber.urlopen(self.url.get_uri(), - http_headers = self._get_http_headers(), - ftp_headers = self._get_ftp_headers(), - proxies = self._get_proxies(), - timeout = timeout, - user_agent = 'PiSi Fetcher/' + pisi.__version__) - except urlgrabber.grabber.URLGrabError: - return False - - return True - - def fetch (self): + def fetch(self): """Return value: Fetched file's full path..""" - # import urlgrabber module - try: - import urlgrabber - except ImportError: - raise FetchError(_('Urlgrabber needs to be installed to run this command')) - if not self.url.filename(): raise FetchError(_('Filename error')) if not os.access(self.destdir, os.W_OK): - raise FetchError(_('Access denied to write to destination directory: "%s"') % (self.destdir)) + raise FetchError(_('Access denied to write to destination directory: "%s"') % self.destdir) if os.path.exists(self.archive_file) and not os.access(self.archive_file, os.W_OK): - raise FetchError(_('Access denied to destination file: "%s"') % (self.archive_file)) + raise FetchError(_('Access denied to destination file: "%s"') % self.archive_file) + + import urllib.request try: - urlgrabber.urlgrab(self.url.get_uri(), - self.partial_file, - progress_obj = UIHandler(self.progress), - http_headers = self._get_http_headers(), - ftp_headers = self._get_ftp_headers(), - proxies = self._get_proxies(), - throttle = self._get_bandwith_limit(), - reget = self._test_range_support(), - copy_local = 1, - user_agent = 'PiSi Fetcher/' + pisi.__version__) - except urlgrabber.grabber.URLGrabError as e: + fetch_handler = FetchHandler(self.url, self.partial_file, self._get_bandwidth_limit()) + + proxy = urllib.request.ProxyHandler(self._get_proxies()) + opener = urllib.request.build_opener(proxy) + opener.addheaders = self._get_headers() + urllib.request.install_opener(opener) + has_range_support = self._test_range_support() + + if has_range_support and os.path.exists(self.partial_file): + partial_file_size = os.path.getsize(self.partial_file) + opener.addheaders.append(('Range', 'bytes=%s-' % partial_file_size)) + + with contextlib.closing(urllib.request.urlopen(self.url.get_uri())) as fp: + headers = fp.info() + + if self.url.is_local_file(): + return os.path.normpath(self.url.path()) + + if has_range_support: + tfp = open(self.partial_file, 'ab') + else: + tfp = open(self.partial_file, 'wb') + + with tfp: + bs = 1024 * 8 + size = -1 + read = 0 + blocknum = 0 + if "content-length" in headers: + size = int(headers["Content-Length"]) + fetch_handler.update(blocknum, bs, size) + while True: + block = fp.read(bs) + if not block: + break + read += len(block) + tfp.write(block) + blocknum += 1 + fetch_handler.update(blocknum, bs, size) + except urllib.request.URLError as e: raise FetchError(_('Could not fetch destination file "%s": %s') % (self.url.get_uri(), e)) if os.stat(self.partial_file).st_size == 0: @@ -177,19 +186,13 @@ class Fetcher: return self.archive_file - def _get_http_headers(self): + def _get_headers(self): headers = [] - if self.url.auth_info() and (self.url.scheme() == "http" or self.url.scheme() == "https"): + if self.url.auth_info(): enc = base64.encodestring('%s:%s' % self.url.auth_info()) - headers.append(('Authorization', 'Basic %s' % enc),) - return tuple(headers) - - def _get_ftp_headers(self): - headers = [] - if self.url.auth_info() and self.url.scheme() == "ftp": - enc = base64.encodestring('%s:%s' % self.url.auth_info()) - headers.append(('Authorization', 'Basic %s' % enc),) - return tuple(headers) + headers.append(('Authorization', 'Basic %s' % enc)) + headers.append(('User-Agent', 'PiSi Fetcher/' + pisi.__version__)) + return headers def _get_proxies(self): proxies = {} @@ -208,7 +211,7 @@ class Fetcher: return proxies - def _get_bandwith_limit(self): + def _get_bandwidth_limit(self): bandwidth_limit = ctx.config.options.bandwidth_limit or ctx.config.values.general.bandwidth_limit if bandwidth_limit and bandwidth_limit != "0": ctx.ui.warning(_("Bandwidth usage is limited to %s KB/s") % bandwidth_limit) @@ -218,24 +221,26 @@ class Fetcher: def _test_range_support(self): if not os.path.exists(self.partial_file): - return None + return False + + import urllib.request + import urllib.error - import urllib.request, urllib.error, urllib.parse try: file_obj = urllib.request.urlopen(urllib.request.Request(self.url.get_uri())) except urllib.error.URLError: ctx.ui.debug(_("Remote file can not be reached. Previously downloaded part of the file will be removed.")) os.remove(self.partial_file) - return None + return False headers = file_obj.info() file_obj.close() if 'Content-Length' in headers: - return 'simple' + return True else: ctx.ui.debug(_("Server doesn't support partial downloads. Previously downloaded part of the file will be over-written.")) os.remove(self.partial_file) - return None + return False # helper function diff --git a/pisi/ui.py b/pisi/ui.py index 9645b01b..b5d5045b 100644 --- a/pisi/ui.py +++ b/pisi/ui.py @@ -13,7 +13,8 @@ # (installed, upgraded, removed, installing, removing, configuring, configured, extracting, - downloading, packagestogo, updatingrepo, cached, desktopfile) = list(range(13)) + downloading, packagestogo, updatingrepo, cached, desktopfile) = list(range(13)) + class UI(object): "Abstract class for UI operations, derive from this." diff --git a/po/de.po b/po/de.po index 66b1a4f5..c5d71df3 100644 --- a/po/de.po +++ b/po/de.po @@ -2485,10 +2485,6 @@ msgstr "Fehler beim analysieren der Index-Datei des Depots. Index-Datei fehlerha msgid "Repository '%s' is not compatible with your distribution. Repository is disabled." msgstr "Depot '%s' ist nicht kompatibel zu Ihrer Distribution. Depot ist desaktiviert." -#: pisi/fetcher.py:147 -msgid "Urlgrabber needs to be installed to run this command" -msgstr "Urlgrabber muss installiert sein um diesen Befehl auszuführen zu können" - #: pisi/fetcher.py:150 msgid "Filename error" msgstr "Dateinamen Fehler" diff --git a/po/es.po b/po/es.po index 688589a8..f059605c 100644 --- a/po/es.po +++ b/po/es.po @@ -2495,10 +2495,6 @@ msgstr "Error al analizar la información del repositorio. Archivo index no exis msgid "Repository '%s' is not compatible with your distribution. Repository is disabled." msgstr "Repositorio '%s' no es compatible con su distribución. Repositor deshabilitado." -#: pisi/fetcher.py:147 -msgid "Urlgrabber needs to be installed to run this command" -msgstr "Se requiere la instalación de Urlgrabber para ejecutar este comando" - #: pisi/fetcher.py:150 msgid "Filename error" msgstr "Error de nombre de archivo" diff --git a/po/fr.po b/po/fr.po index f18b1e79..ea7c9dcc 100644 --- a/po/fr.po +++ b/po/fr.po @@ -2429,10 +2429,6 @@ msgstr "Erreur d'analyse de l'index du dépôt. Le fichier d'index n'existe pas msgid "Repository '%s' is not compatible with your distribution. Repository is disabled." msgstr "Le dépôt '%s' n'est pas compatible avec votre version de distribution. Le dépôt est désactivé." -#: pisi/fetcher.py:147 -msgid "Urlgrabber needs to be installed to run this command" -msgstr "L'utilisation de cette commande nécessite l'installation de Urlgrabber" - #: pisi/fetcher.py:150 msgid "Filename error" msgstr "Erreur de nom de fichier" diff --git a/po/hr.po b/po/hr.po index e0252651..e2a8ddb5 100644 --- a/po/hr.po +++ b/po/hr.po @@ -2242,10 +2242,6 @@ msgstr "" msgid "Repository '%s' is not compatible with your distribution. Repository is disabled." msgstr "Repozitorij %s nije kompatibilan sa vašom distribucijom.Repozitorij je onemogućen. " -#: pisi/fetcher.py:147 -msgid "Urlgrabber needs to be installed to run this command" -msgstr "" - #: pisi/fetcher.py:150 msgid "Filename error" msgstr "Greška imena datoteke" diff --git a/po/hu.po b/po/hu.po index b895e4fe..f98f1d6a 100644 --- a/po/hu.po +++ b/po/hu.po @@ -2447,10 +2447,6 @@ msgstr "Hiba a tároló indexének feldolgozása során. Az index-fájl nem lét msgid "Repository '%s' is not compatible with your distribution. Repository is disabled." msgstr "A(z) tároló nem kompatíbilis az ön által használt disztribúcióval. A tárolót kikapcsoltam." -#: pisi/fetcher.py:147 -msgid "Urlgrabber needs to be installed to run this command" -msgstr "E parancs futtatásához az urlgrabber telepítése szükséges." - #: pisi/fetcher.py:150 msgid "Filename error" msgstr "Fájlnév hiba" diff --git a/po/it.po b/po/it.po index 3e3c3cde..d0c6e9a9 100644 --- a/po/it.po +++ b/po/it.po @@ -2427,10 +2427,6 @@ msgstr "" msgid "Repository '%s' is not compatible with your distribution. Repository is disabled." msgstr "Il pacchetto %s non è disponibile nei repositorio. Repositorio è disattivato." -#: pisi/fetcher.py:147 -msgid "Urlgrabber needs to be installed to run this command" -msgstr "" - #: pisi/fetcher.py:150 msgid "Filename error" msgstr "Errore nel nome del file" diff --git a/po/nl.po b/po/nl.po index 002a054f..ab4c218b 100644 --- a/po/nl.po +++ b/po/nl.po @@ -2545,10 +2545,6 @@ msgid "" msgstr "" "Softwarebron %s is niet compatibel met de distributie. Bron is uitgeschakeld." -#: pisi/fetcher.py:147 -msgid "Urlgrabber needs to be installed to run this command" -msgstr "Urlgrabber dient geïnstalleerd te zijn om dit commando uit te voeren" - #: pisi/fetcher.py:150 msgid "Filename error" msgstr "Bestandsnaamfout" diff --git a/po/pisi.pot b/po/pisi.pot index 3583c6c0..43ab006a 100644 --- a/po/pisi.pot +++ b/po/pisi.pot @@ -2211,10 +2211,6 @@ msgid "" "disabled." msgstr "" -#: pisi/fetcher.py:147 -msgid "Urlgrabber needs to be installed to run this command" -msgstr "" - #: pisi/fetcher.py:150 msgid "Filename error" msgstr "" diff --git a/po/pl.po b/po/pl.po index 31afe838..ec2f426e 100644 --- a/po/pl.po +++ b/po/pl.po @@ -2220,10 +2220,6 @@ msgstr "" msgid "Repository '%s' is not compatible with your distribution. Repository is disabled." msgstr "" -#: pisi/fetcher.py:147 -msgid "Urlgrabber needs to be installed to run this command" -msgstr "" - #: pisi/fetcher.py:150 msgid "Filename error" msgstr "Błąd nazwy pliku" diff --git a/po/ru.po b/po/ru.po index d7cb683e..3c117238 100644 --- a/po/ru.po +++ b/po/ru.po @@ -2438,10 +2438,6 @@ msgstr "Ошибка разбора информации индекса репо msgid "Repository '%s' is not compatible with your distribution. Repository is disabled." msgstr "Репозиторий '%s' не совместим с Вашим дистрибутивом. Репозиторий отключен." -#: pisi/fetcher.py:147 -msgid "Urlgrabber needs to be installed to run this command" -msgstr "Urlgrabber должен быть установлен для запуска этой команды" - #: pisi/fetcher.py:150 msgid "Filename error" msgstr "Ошибка имени файла" diff --git a/po/sv.po b/po/sv.po index eb37b268..7d64022c 100644 --- a/po/sv.po +++ b/po/sv.po @@ -2202,10 +2202,6 @@ msgstr "" msgid "Repository '%s' is not compatible with your distribution. Repository is disabled." msgstr "" -#: pisi/fetcher.py:147 -msgid "Urlgrabber needs to be installed to run this command" -msgstr "" - #: pisi/fetcher.py:150 msgid "Filename error" msgstr "Filnamnsfel" diff --git a/po/tr.po b/po/tr.po index c9b96592..a243d3fa 100644 --- a/po/tr.po +++ b/po/tr.po @@ -2460,10 +2460,6 @@ msgstr "Depo bilgisi ayrıştırılırken hata oluştu. Katalog dosyası mevcut msgid "Repository '%s' is not compatible with your distribution. Repository is disabled." msgstr "'%s' adlı depo dağıtımınız ile uyumlu değil. Depo etkisiz hale getirildi." -#: pisi/fetcher.py:147 -msgid "Urlgrabber needs to be installed to run this command" -msgstr "Bu komutun çalıştırılabilmesi için Urlgrabber kurulu olmalı." - #: pisi/fetcher.py:150 msgid "Filename error" msgstr "Dosya adı hatası" diff --git a/po/uk.po b/po/uk.po index 6df4de6b..28748b77 100644 --- a/po/uk.po +++ b/po/uk.po @@ -2192,10 +2192,6 @@ msgstr "" msgid "Repository '%s' is not compatible with your distribution. Repository is disabled." msgstr "" -#: pisi/fetcher.py:147 -msgid "Urlgrabber needs to be installed to run this command" -msgstr "" - #: pisi/fetcher.py:150 msgid "Filename error" msgstr ""