remove urlgrabber dependency

pisi/fetcher.py

@@ -16,12 +16,14 @@ all kinds of things: source tarballs, index files, packages, and God
 knows what."""
 
 # python standard library modules
+import contextlib
 import os
 import time
 import base64
 import shutil
 
 import gettext
+
 __trans = gettext.translation('pisi', fallback=True)
 _ = __trans.gettext
 
@@ -30,54 +32,45 @@ import pisi
 import pisi.util as util
 import pisi.context as ctx
 import pisi.uri
+import pisi.request
 
 
 class FetchError(pisi.Error):
     pass
 
 
-class UIHandler:
-    def __init__(self, progress):
-        self.filename        = None
-        self.url             = None
-        self.basename        = None
-        self.downloaded_size = 0
+class FetchHandler:
+    def __init__(self, url, archive, bandwidth_limit):
+        self.url = url
         self.percent = None
         self.rate = 0.0
         self.size = 0
         self.eta = '--:--:--'
         self.symbol = '--/-'
         self.last_updated = 0
+        self.filename = url.filename()
+        self.total_size = 0
         self.exist_size = 0
-
-    def start(self, archive, url, basename, size, text):
+        self.bandwidth_limit = bandwidth_limit
         if os.path.exists(archive):
             self.exist_size = os.path.getsize(archive)
-        self.filename   = util.remove_suffix(ctx.const.partial_suffix, basename)
-        self.url        = url
-        self.basename   = basename
-        self.total_size = size or 0
-        self.text       = text
 
         self.now = lambda: time.time()
         self.t_diff = lambda: self.now() - self.s_time
 
         self.s_time = self.now()
 
-    def update(self, size):
-
-        if self.size == size:
-            return
-
-        self.size = size
+    def update(self, blocknum, bs, size):
+        self.total_size = size + self.exist_size
+        self.size = blocknum * bs + self.exist_size
         if self.total_size:
-            self.percent = (size * 100.0) / self.total_size
+            self.percent = self.size * 100 / self.total_size
         else:
             self.percent = 0
 
-        if int(self.now()) != int(self.last_updated) and size > 0:
+        if int(self.now()) != int(self.last_updated) and self.size > 0:
             try:
-                self.rate, self.symbol = util.human_readable_rate((size - self.exist_size) / (self.now() - self.s_time))
+                self.rate, self.symbol = util.human_readable_rate(self.size / (self.now() - self.s_time))
             except ZeroDivisionError:
                 return
             if self.total_size:
@@ -85,19 +78,26 @@ class UIHandler:
                     tuple([i for i in time.gmtime((self.t_diff() * (100 - self.percent)) / self.percent)[3:6]])
 
         self._update_ui()
+        self._limit_bandwidth()
 
-    def end(self, read):
-        pass
+    def _limit_bandwidth(self):
+        if self.bandwidth_limit:
+            expected_time = (self.size - self.exist_size) / self.bandwidth_limit
+            sleep_time = expected_time - self.t_diff()
+            if sleep_time > 0:
+                time.sleep(sleep_time)
 
     def _update_ui(self):
-        ctx.ui.display_progress(operation       = "fetching",
+        ctx.ui.display_progress(
+            operation="fetching",
             percent=self.percent,
             filename=self.filename,
-                                total_size      = self.total_size or self.size,
+            total_size=self.total_size,
             downloaded_size=self.size,
             rate=self.rate,
             eta=self.eta,
-                                symbol          = self.symbol)
+            symbol=self.symbol
+        )
 
         self.last_updated = self.now()
 
@@ -115,58 +115,67 @@ class Fetcher:
         self.url = url
         self.destdir = destdir
         self.destfile = destfile
-        self.progress = None
 
         self.archive_file = os.path.join(destdir, destfile or url.filename())
         self.partial_file = os.path.join(self.destdir, self.url.filename()) + ctx.const.partial_suffix
 
         util.ensure_dirs(self.destdir)
 
-    def test(self, timeout=3):
-        import urlgrabber
-
-        try:
-            urlgrabber.urlopen(self.url.get_uri(),
-                           http_headers = self._get_http_headers(),
-                           ftp_headers  = self._get_ftp_headers(),
-                           proxies      = self._get_proxies(),
-                           timeout      = timeout,
-                           user_agent   = 'PiSi Fetcher/' + pisi.__version__)
-        except urlgrabber.grabber.URLGrabError:
-            return False
-
-        return True
-
     def fetch(self):
         """Return value: Fetched file's full path.."""
 
-        # import urlgrabber module
-        try:
-            import urlgrabber
-        except ImportError:
-            raise FetchError(_('Urlgrabber needs to be installed to run this command'))
-
         if not self.url.filename():
             raise FetchError(_('Filename error'))
 
         if not os.access(self.destdir, os.W_OK):
-            raise FetchError(_('Access denied to write to destination directory: "%s"') % (self.destdir))
+            raise FetchError(_('Access denied to write to destination directory: "%s"') % self.destdir)
 
         if os.path.exists(self.archive_file) and not os.access(self.archive_file, os.W_OK):
-            raise FetchError(_('Access denied to destination file: "%s"') % (self.archive_file))
+            raise FetchError(_('Access denied to destination file: "%s"') % self.archive_file)
+
+        import urllib.request
 
         try:
-            urlgrabber.urlgrab(self.url.get_uri(),
-                           self.partial_file,
-                           progress_obj = UIHandler(self.progress),
-                           http_headers = self._get_http_headers(),
-                           ftp_headers  = self._get_ftp_headers(),
-                           proxies      = self._get_proxies(),
-                           throttle     = self._get_bandwith_limit(),
-                           reget        = self._test_range_support(),
-                           copy_local   = 1,
-                           user_agent   = 'PiSi Fetcher/' + pisi.__version__)
-        except urlgrabber.grabber.URLGrabError as e:
+            fetch_handler = FetchHandler(self.url, self.partial_file, self._get_bandwidth_limit())
+
+            proxy = urllib.request.ProxyHandler(self._get_proxies())
+            opener = urllib.request.build_opener(proxy)
+            opener.addheaders = self._get_headers()
+            urllib.request.install_opener(opener)
+            has_range_support = self._test_range_support()
+
+            if has_range_support and os.path.exists(self.partial_file):
+                partial_file_size = os.path.getsize(self.partial_file)
+                opener.addheaders.append(('Range', 'bytes=%s-' % partial_file_size))
+
+            with contextlib.closing(urllib.request.urlopen(self.url.get_uri())) as fp:
+                headers = fp.info()
+
+                if self.url.is_local_file():
+                    return os.path.normpath(self.url.path())
+
+                if has_range_support:
+                    tfp = open(self.partial_file, 'ab')
+                else:
+                    tfp = open(self.partial_file, 'wb')
+
+                with tfp:
+                    bs = 1024 * 8
+                    size = -1
+                    read = 0
+                    blocknum = 0
+                    if "content-length" in headers:
+                        size = int(headers["Content-Length"])
+                    fetch_handler.update(blocknum, bs, size)
+                    while True:
+                        block = fp.read(bs)
+                        if not block:
+                            break
+                        read += len(block)
+                        tfp.write(block)
+                        blocknum += 1
+                        fetch_handler.update(blocknum, bs, size)
+        except urllib.request.URLError as e:
             raise FetchError(_('Could not fetch destination file "%s": %s') % (self.url.get_uri(), e))
 
         if os.stat(self.partial_file).st_size == 0:
@@ -177,19 +186,13 @@ class Fetcher:
 
         return self.archive_file
 
-    def _get_http_headers(self):
+    def _get_headers(self):
         headers = []
-        if self.url.auth_info() and (self.url.scheme() == "http" or self.url.scheme() == "https"):
+        if self.url.auth_info():
             enc = base64.encodestring('%s:%s' % self.url.auth_info())
-            headers.append(('Authorization', 'Basic %s' % enc),)
-        return tuple(headers)
-
-    def _get_ftp_headers(self):
-        headers = []
-        if self.url.auth_info() and self.url.scheme() == "ftp":
-            enc = base64.encodestring('%s:%s' % self.url.auth_info())
-            headers.append(('Authorization', 'Basic %s' % enc),)
-        return tuple(headers)
+            headers.append(('Authorization', 'Basic %s' % enc))
+        headers.append(('User-Agent', 'PiSi Fetcher/' + pisi.__version__))
+        return headers
 
     def _get_proxies(self):
         proxies = {}
@@ -208,7 +211,7 @@ class Fetcher:
 
         return proxies
 
-    def _get_bandwith_limit(self):
+    def _get_bandwidth_limit(self):
         bandwidth_limit = ctx.config.options.bandwidth_limit or ctx.config.values.general.bandwidth_limit
         if bandwidth_limit and bandwidth_limit != "0":
             ctx.ui.warning(_("Bandwidth usage is limited to %s KB/s") % bandwidth_limit)
@@ -218,24 +221,26 @@ class Fetcher:
 
     def _test_range_support(self):
         if not os.path.exists(self.partial_file):
-            return None
+            return False
+
+        import urllib.request
+        import urllib.error
 
-        import urllib.request, urllib.error, urllib.parse
         try:
             file_obj = urllib.request.urlopen(urllib.request.Request(self.url.get_uri()))
         except urllib.error.URLError:
             ctx.ui.debug(_("Remote file can not be reached. Previously downloaded part of the file will be removed."))
             os.remove(self.partial_file)
-            return None
+            return False
 
         headers = file_obj.info()
         file_obj.close()
         if 'Content-Length' in headers:
-            return 'simple'
+            return True
         else:
             ctx.ui.debug(_("Server doesn't support partial downloads. Previously downloaded part of the file will be over-written."))
             os.remove(self.partial_file)
-            return None
+            return False
 
 
 # helper function

pisi/ui.py

@@ -15,6 +15,7 @@
 (installed, upgraded, removed, installing, removing, configuring, configured, extracting,
  downloading, packagestogo, updatingrepo, cached, desktopfile) = list(range(13))
 
+
 class UI(object):
     "Abstract class for UI operations, derive from this."
 

po/de.po

@@ -2485,10 +2485,6 @@ msgstr "Fehler beim analysieren der Index-Datei des Depots. Index-Datei fehlerha
 msgid "Repository '%s' is not compatible with your distribution. Repository is disabled."
 msgstr "Depot '%s' ist nicht kompatibel zu Ihrer Distribution. Depot ist desaktiviert."
 
-#: pisi/fetcher.py:147
-msgid "Urlgrabber needs to be installed to run this command"
-msgstr "Urlgrabber muss installiert sein um diesen Befehl auszuführen zu können"
-
 #: pisi/fetcher.py:150
 msgid "Filename error"
 msgstr "Dateinamen Fehler"

po/es.po

@@ -2495,10 +2495,6 @@ msgstr "Error al analizar la información del repositorio. Archivo index no exis
 msgid "Repository '%s' is not compatible with your distribution. Repository is disabled."
 msgstr "Repositorio '%s' no es compatible con su distribución. Repositor deshabilitado."
 
-#: pisi/fetcher.py:147
-msgid "Urlgrabber needs to be installed to run this command"
-msgstr "Se requiere la instalación de Urlgrabber para ejecutar este comando"
-
 #: pisi/fetcher.py:150
 msgid "Filename error"
 msgstr "Error de nombre de archivo"

po/fr.po

@@ -2429,10 +2429,6 @@ msgstr "Erreur d'analyse de l'index du dépôt. Le fichier d'index n'existe pas
 msgid "Repository '%s' is not compatible with your distribution. Repository is disabled."
 msgstr "Le dépôt '%s' n'est pas compatible avec votre version de distribution. Le dépôt est désactivé."
 
-#: pisi/fetcher.py:147
-msgid "Urlgrabber needs to be installed to run this command"
-msgstr "L'utilisation de cette commande nécessite l'installation de Urlgrabber"
-
 #: pisi/fetcher.py:150
 msgid "Filename error"
 msgstr "Erreur de nom de fichier"

po/hr.po

@@ -2242,10 +2242,6 @@ msgstr ""
 msgid "Repository '%s' is not compatible with your distribution. Repository is disabled."
 msgstr "Repozitorij %s nije kompatibilan sa vašom distribucijom.Repozitorij je onemogućen. "
 
-#: pisi/fetcher.py:147
-msgid "Urlgrabber needs to be installed to run this command"
-msgstr ""
-
 #: pisi/fetcher.py:150
 msgid "Filename error"
 msgstr "Greška imena datoteke"

po/hu.po

@@ -2447,10 +2447,6 @@ msgstr "Hiba a tároló indexének feldolgozása során. Az index-fájl nem lét
 msgid "Repository '%s' is not compatible with your distribution. Repository is disabled."
 msgstr "A(z) tároló nem kompatíbilis az ön által használt disztribúcióval. A tárolót kikapcsoltam."
 
-#: pisi/fetcher.py:147
-msgid "Urlgrabber needs to be installed to run this command"
-msgstr "E parancs futtatásához az urlgrabber telepítése szükséges."
-
 #: pisi/fetcher.py:150
 msgid "Filename error"
 msgstr "Fájlnév hiba"

po/it.po

@@ -2427,10 +2427,6 @@ msgstr ""
 msgid "Repository '%s' is not compatible with your distribution. Repository is disabled."
 msgstr "Il pacchetto %s non è disponibile nei repositorio. Repositorio è disattivato."
 
-#: pisi/fetcher.py:147
-msgid "Urlgrabber needs to be installed to run this command"
-msgstr ""
-
 #: pisi/fetcher.py:150
 msgid "Filename error"
 msgstr "Errore nel nome del file"

po/nl.po

@@ -2545,10 +2545,6 @@ msgid ""
 msgstr ""
 "Softwarebron %s is niet compatibel met de distributie. Bron is uitgeschakeld."
 
-#: pisi/fetcher.py:147
-msgid "Urlgrabber needs to be installed to run this command"
-msgstr "Urlgrabber dient geïnstalleerd te zijn om dit commando uit te voeren"
-
 #: pisi/fetcher.py:150
 msgid "Filename error"
 msgstr "Bestandsnaamfout"

po/pisi.pot

@@ -2211,10 +2211,6 @@ msgid ""
 "disabled."
 msgstr ""
 
-#: pisi/fetcher.py:147
-msgid "Urlgrabber needs to be installed to run this command"
-msgstr ""
-
 #: pisi/fetcher.py:150
 msgid "Filename error"
 msgstr ""

po/pl.po

@@ -2220,10 +2220,6 @@ msgstr ""
 msgid "Repository '%s' is not compatible with your distribution. Repository is disabled."
 msgstr ""
 
-#: pisi/fetcher.py:147
-msgid "Urlgrabber needs to be installed to run this command"
-msgstr ""
-
 #: pisi/fetcher.py:150
 msgid "Filename error"
 msgstr "Błąd nazwy pliku"

po/ru.po

@@ -2438,10 +2438,6 @@ msgstr "Ошибка разбора информации индекса репо
 msgid "Repository '%s' is not compatible with your distribution. Repository is disabled."
 msgstr "Репозиторий '%s' не совместим с Вашим дистрибутивом. Репозиторий отключен."
 
-#: pisi/fetcher.py:147
-msgid "Urlgrabber needs to be installed to run this command"
-msgstr "Urlgrabber должен быть установлен для запуска этой команды"
-
 #: pisi/fetcher.py:150
 msgid "Filename error"
 msgstr "Ошибка имени файла"

po/sv.po

@@ -2202,10 +2202,6 @@ msgstr ""
 msgid "Repository '%s' is not compatible with your distribution. Repository is disabled."
 msgstr ""
 
-#: pisi/fetcher.py:147
-msgid "Urlgrabber needs to be installed to run this command"
-msgstr ""
-
 #: pisi/fetcher.py:150
 msgid "Filename error"
 msgstr "Filnamnsfel"

po/tr.po

@@ -2460,10 +2460,6 @@ msgstr "Depo bilgisi ayrıştırılırken hata oluştu. Katalog dosyası mevcut
 msgid "Repository '%s' is not compatible with your distribution. Repository is disabled."
 msgstr "'%s' adlı depo dağıtımınız ile uyumlu değil. Depo etkisiz hale getirildi."
 
-#: pisi/fetcher.py:147
-msgid "Urlgrabber needs to be installed to run this command"
-msgstr "Bu komutun çalıştırılabilmesi için Urlgrabber kurulu olmalı."
-
 #: pisi/fetcher.py:150
 msgid "Filename error"
 msgstr "Dosya adı hatası"

po/uk.po

@@ -2192,10 +2192,6 @@ msgstr ""
 msgid "Repository '%s' is not compatible with your distribution. Repository is disabled."
 msgstr ""
 
-#: pisi/fetcher.py:147
-msgid "Urlgrabber needs to be installed to run this command"
-msgstr ""
-
 #: pisi/fetcher.py:150
 msgid "Filename error"
 msgstr ""