Use urlgrabber library to fetch stuff.. Spiffy!
This commit is contained in:
A. Murat Eren
+103
-289
@@ -16,18 +16,11 @@ all kinds of things: source tarballs, index files, packages, and God
|
||||
knows what."""
|
||||
|
||||
# python standard library modules
|
||||
import urllib2
|
||||
import urllib
|
||||
import ftplib
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import mimetypes
|
||||
import mimetools
|
||||
import time
|
||||
import base64
|
||||
import shutil
|
||||
import time
|
||||
import httplib
|
||||
|
||||
import gettext
|
||||
__trans = gettext.translation('pisi', fallback=True)
|
||||
@@ -42,330 +35,151 @@ import pisi.uri
|
||||
class FetchError(pisi.Error):
|
||||
pass
|
||||
|
||||
class RangeError(pisi.Error):
|
||||
pass
|
||||
|
||||
# helper functions
|
||||
def fetch_url(url, destdir, progress=None, resume=True):
|
||||
fetch = Fetcher(url, destdir)
|
||||
if not resume:
|
||||
fetch.resume = False
|
||||
fetch.progress = progress
|
||||
fetch.fetch()
|
||||
class UIHandler:
|
||||
def __init__(self, progress):
|
||||
self.filename = None
|
||||
self.url = None
|
||||
self.basename = None
|
||||
self.downloaded_size = 0
|
||||
self.percent = None
|
||||
self.rate = 0.0
|
||||
self.eta = '--:--:--'
|
||||
self.symbol = None
|
||||
self.last_updated = 0
|
||||
self.exist_size = 0
|
||||
|
||||
def start(self, archive, url, basename, total_size, text):
|
||||
if os.path.exists(archive):
|
||||
self.exist_size = os.path.getsize(archive)
|
||||
self.filename = basename
|
||||
self.url = url
|
||||
self.basename = basename
|
||||
self.total_size = total_size
|
||||
self.text = text
|
||||
|
||||
self.now = lambda: time.time()
|
||||
self.Tdiff = lambda: self.now() - self.s_time
|
||||
|
||||
self.s_time = self.now()
|
||||
|
||||
def update(self, size):
|
||||
self.size = size
|
||||
self.percent = (size * 100.0) / self.total_size
|
||||
|
||||
if int(self.now()) != int(self.last_updated) and size > 0:
|
||||
self.rate, self.symbol = util.human_readable_rate((size - self.exist_size) / (self.now() - self.s_time))
|
||||
self.eta = '%02d:%02d:%02d' %\
|
||||
tuple([i for i in time.gmtime((self.Tdiff() * (100 - self.percent)) / self.percent)[3:6]])
|
||||
|
||||
self._update_ui()
|
||||
|
||||
def end(self, read):
|
||||
pass
|
||||
|
||||
def _update_ui(self):
|
||||
ctx.ui.display_progress(operation = "fetching",
|
||||
percent = self.percent,
|
||||
filename = self.filename,
|
||||
total_size = self.total_size,
|
||||
downloaded_size = self.size,
|
||||
rate = self.rate,
|
||||
eta = self.eta,
|
||||
symbol = self.symbol)
|
||||
|
||||
self.last_updated = self.now()
|
||||
|
||||
|
||||
class Fetcher:
|
||||
"""Fetcher can fetch a file from various sources using various
|
||||
protocols."""
|
||||
def __init__(self, url, destdir, resume = True):
|
||||
def __init__(self, url, destdir):
|
||||
if not isinstance(url, pisi.uri.URI):
|
||||
url = pisi.uri.URI(url)
|
||||
|
||||
if ctx.config.get_option("authinfo"):
|
||||
url.set_auth_info(ctx.config.get_option("authinfo"))
|
||||
|
||||
self.resume = resume
|
||||
self.scheme = url.scheme()
|
||||
self.url = url
|
||||
self.destdir = destdir
|
||||
util.check_dir(self.destdir)
|
||||
self.eta = '??:??:??'
|
||||
self.percent = 0
|
||||
self.rate = 0.0
|
||||
self.url = url
|
||||
self.destdir = destdir
|
||||
self.progress = None
|
||||
self.exist_size = 0
|
||||
|
||||
util.check_dir(self.destdir)
|
||||
|
||||
|
||||
def fetch (self):
|
||||
"""Return value: Fetched file's full path.."""
|
||||
|
||||
# import urlgrabber module
|
||||
try:
|
||||
import urlgrabber
|
||||
except ImportError:
|
||||
raise FetchError(_('Urlgrabber needs to be installed to run this command'))
|
||||
|
||||
if not self.url.filename():
|
||||
self.err(_('Filename error'))
|
||||
FetchError(_('Filename error'))
|
||||
|
||||
if not os.access(self.destdir, os.W_OK):
|
||||
self.err(_('Access denied to write to destination directory: "%s"') % (self.destdir))
|
||||
FetchError(_('Access denied to write to destination directory: "%s"') % (self.destdir))
|
||||
|
||||
archive_file = os.path.join(self.destdir, self.url.filename())
|
||||
|
||||
if os.path.exists(archive_file) and not os.access(archive_file, os.W_OK):
|
||||
self.err(_('Access denied to destination file: "%s"') % (archive_file))
|
||||
FetchError(_('Access denied to destination file: "%s"') % (archive_file))
|
||||
|
||||
partial_file = archive_file + '.part'
|
||||
|
||||
if self.url.is_local_file():
|
||||
self.fetchLocalFile(partial_file)
|
||||
else:
|
||||
self.fetchRemoteFile(partial_file)
|
||||
urlgrabber.urlgrab(self.url.get_uri(),
|
||||
partial_file,
|
||||
progress_obj = UIHandler(self.progress),
|
||||
http_headers = self._get_http_headers(),
|
||||
ftp_headers = self._get_ftp_headers(),
|
||||
proxies = self._get_proxies(),
|
||||
user_agent = 'PiSi Fetcher/' + pisi.__version__,
|
||||
reget = 'check_timestamp')
|
||||
|
||||
if os.stat(partial_file).st_size == 0:
|
||||
os.remove(partial_file)
|
||||
self.err(_('A problem occured. Please check the archive address and/or permissions again.'))
|
||||
FetchError(_('A problem occurred. Please check the archive address and/or permissions again.'))
|
||||
|
||||
shutil.move(partial_file, archive_file)
|
||||
|
||||
return archive_file
|
||||
|
||||
def _do_grab(self, fileURI, dest, total_size):
|
||||
bs, tt, = 1024, int(time.time())
|
||||
s_time = time.time()
|
||||
Tdiff = lambda: time.time() - s_time
|
||||
downloaded_size = exist_size = self.exist_size
|
||||
symbol = 'B/s'
|
||||
st = time.time()
|
||||
chunk = fileURI.read(bs)
|
||||
downloaded_size += len(chunk)
|
||||
|
||||
if self.progress:
|
||||
p = self.progress(total_size, exist_size)
|
||||
self.percent = p.update(downloaded_size)
|
||||
self.complete = False
|
||||
|
||||
while chunk:
|
||||
dest.write(chunk)
|
||||
chunk = fileURI.read(bs)
|
||||
downloaded_size += len(chunk)
|
||||
ct = time.time()
|
||||
if int(tt) != int(ct):
|
||||
self.rate = (downloaded_size - exist_size) / (ct - st)
|
||||
|
||||
if self.percent:
|
||||
self.eta = '%02d:%02d:%02d' %\
|
||||
tuple([i for i in time.gmtime((Tdiff() * (100 - self.percent)) / self.percent)[3:6]])
|
||||
|
||||
self.rate, symbol = util.human_readable_rate(self.rate)
|
||||
|
||||
tt = time.time()
|
||||
|
||||
if self.progress:
|
||||
if p.update(downloaded_size):
|
||||
self.percent = p.percent
|
||||
if not self.complete:
|
||||
ctx.ui.display_progress(operation = "fetching",
|
||||
percent = self.percent,
|
||||
filename = self.url.filename(),
|
||||
total_size = total_size,
|
||||
downloaded_size = downloaded_size,
|
||||
rate = self.rate,
|
||||
eta = self.eta,
|
||||
symbol = symbol)
|
||||
if self.percent == 100: #FIXME: will be superseded by a
|
||||
self.complete = True # working progress interface
|
||||
|
||||
dest.close()
|
||||
|
||||
def fetchLocalFile (self, archive_file):
|
||||
url = self.url
|
||||
|
||||
if not os.access(url.path(), os.F_OK):
|
||||
self.err(_('No such file or no permission to read'))
|
||||
|
||||
dest = open(archive_file, 'w')
|
||||
total_size = os.path.getsize(url.path())
|
||||
fileObj = open(url.path())
|
||||
self._do_grab(fileObj, dest, total_size)
|
||||
|
||||
def fetchRemoteFile (self, archive_file):
|
||||
|
||||
if os.path.exists(archive_file) and self.resume:
|
||||
if self.scheme == 'http' or self.scheme == 'https' or self.scheme == 'ftp':
|
||||
self.exist_size = os.path.getsize(archive_file)
|
||||
dest = open(archive_file, 'ab')
|
||||
else:
|
||||
dest = open(archive_file, 'wb')
|
||||
|
||||
uri = self.url.get_uri()
|
||||
|
||||
flag = 1
|
||||
try:
|
||||
try:
|
||||
try:
|
||||
fileObj = urllib2.urlopen(self.formatRequest(urllib2.Request(uri)))
|
||||
except RangeError:
|
||||
ctx.ui.info(_('Requested range not satisfiable, starting again.'))
|
||||
dest = open(archive_file, 'wb')
|
||||
self.exist_size = 0
|
||||
fileObj = urllib2.urlopen(self.formatRequest(urllib2.Request(uri)))
|
||||
headers = fileObj.info()
|
||||
flag = 0
|
||||
except ValueError, e:
|
||||
self.err(_('Cannot fetch %s; value error: %s') % (uri, e))
|
||||
except urllib2.HTTPError, e:
|
||||
self.err(_('Cannot fetch %s; %s') % (uri, e))
|
||||
except urllib2.URLError, e:
|
||||
self.err(_('Please check your network connections and try again. (%s)') % e[-1][-1])
|
||||
except OSError, e:
|
||||
self.err(_('Cannot fetch %s; %s') % (uri, e))
|
||||
except httplib.HTTPException, e:
|
||||
self.err(_('Cannot fetch %s; (%s): %s') % (uri, e.__class__.__name__, e))
|
||||
finally:
|
||||
if flag:
|
||||
if os.stat(archive_file).st_size == 0:
|
||||
os.remove(archive_file)
|
||||
|
||||
try:
|
||||
total_size = int(headers['Content-Length']) + self.exist_size
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except Exception, e: #FIXME: what exception could we catch here, replace with that.
|
||||
total_size = 0
|
||||
|
||||
self._do_grab(fileObj, dest, total_size)
|
||||
|
||||
def formatRequest(self, request):
|
||||
if self.url.auth_info():
|
||||
def _get_http_headers(self):
|
||||
headers = []
|
||||
if self.url.auth_info() and (self.url.scheme() == "http" or self.url.scheme() == "https"):
|
||||
enc = base64.encodestring('%s:%s' % self.url.auth_info())
|
||||
request.add_header('Authorization', 'Basic %s' % enc)
|
||||
headers.append(('Authorization', 'Basic %s' % enc),)
|
||||
return tuple(headers)
|
||||
|
||||
range_handlers = {
|
||||
'http' : HTTPRangeHandler,
|
||||
'https': HTTPRangeHandler,
|
||||
'ftp' : FTPRangeHandler
|
||||
}
|
||||
def _get_ftp_headers(self):
|
||||
headers = []
|
||||
if self.url.auth_info() and self.url.scheme() == "ftp":
|
||||
enc = base64.encodestring('%s:%s' % self.url.auth_info())
|
||||
headers.append(('Authorization', 'Basic %s' % enc),)
|
||||
return tuple(headers)
|
||||
|
||||
if self.exist_size and range_handlers.has_key(self.scheme):
|
||||
opener = urllib2.build_opener(range_handlers.get(self.scheme)())
|
||||
urllib2.install_opener(opener)
|
||||
request.add_header('Range', 'bytes=%d-' % self.exist_size)
|
||||
|
||||
proxy_handler = None
|
||||
def _get_proxies(self):
|
||||
proxies = {}
|
||||
|
||||
if ctx.config.values.general.http_proxy and self.url.scheme() == "http":
|
||||
http_proxy = ctx.config.values.general.http_proxy
|
||||
proxy_handler = urllib2.ProxyHandler({pisi.uri.URI(http_proxy).scheme(): http_proxy})
|
||||
proxies[pisi.uri.URI(http_proxy).scheme()] = ctx.config.values.general.http_proxy
|
||||
|
||||
elif ctx.config.values.general.https_proxy and self.url.scheme() == "https":
|
||||
https_proxy = ctx.config.values.general.https_proxy
|
||||
proxy_handler = urllib2.ProxyHandler({pisi.uri.URI(https_proxy): https_proxy})
|
||||
if ctx.config.values.general.https_proxy and self.url.scheme() == "https":
|
||||
proxies[pisi.uri.URI(https_proxy).scheme()] = ctx.config.values.general.https_proxy
|
||||
|
||||
elif ctx.config.values.general.ftp_proxy and self.url.scheme() == "ftp":
|
||||
ftp_proxy = ctx.config.values.general.ftp_proxy
|
||||
proxy_handler = urllib2.ProxyHandler({pisi.uri.URI(http_proxy): ftp_proxy})
|
||||
if ctx.config.values.general.ftp_proxy and self.url.scheme() == "ftp":
|
||||
proxies[pisi.uri.URI(ftp_proxy).scheme()] = ctx.config.values.general.ftp_proxy
|
||||
|
||||
if proxy_handler:
|
||||
if self.url.scheme() in proxies:
|
||||
ctx.ui.info(_("Proxy configuration has been found for '%s' protocol") % self.url.scheme())
|
||||
opener = urllib2.build_opener(proxy_handler)
|
||||
urllib2.install_opener(opener)
|
||||
|
||||
return request
|
||||
|
||||
def err (self, error):
|
||||
raise FetchError(error)
|
||||
|
||||
class HTTPRangeHandler(urllib2.BaseHandler):
|
||||
"""
|
||||
to override the urllib2 error: 'Error 206: Partial Content'
|
||||
this reponse from the HTTP server is already what we expected to get.
|
||||
Don't give up, resume downloading..
|
||||
"""
|
||||
|
||||
def http_error_206(self, request, fp, errcode, msg, headers):
|
||||
return urllib.addinfourl(fp, headers, request.get_full_url())
|
||||
|
||||
def http_error_416(self, request, fp, errcode, msg, headers):
|
||||
# HTTP 1.1's 'Range Not Satisfiable' error..
|
||||
raise RangeError
|
||||
return proxies
|
||||
|
||||
|
||||
class FTPRangeHandler(urllib2.FTPHandler):
|
||||
"""
|
||||
FTP Range support..
|
||||
"""
|
||||
def ftp_open(self, req):
|
||||
host = req.get_host()
|
||||
host, port = urllib.splitport(host)
|
||||
if port is None:
|
||||
port = ftplib.FTP_PORT
|
||||
|
||||
try:
|
||||
host = socket.gethostbyname(host)
|
||||
except socket.error, msg:
|
||||
raise FetchError(msg)
|
||||
|
||||
path, attrs = urllib.splitattr(req.get_selector())
|
||||
dirs = path.split('/')
|
||||
dirs = map(urllib.unquote, dirs)
|
||||
dirs, f = dirs[:-1], dirs[-1]
|
||||
if dirs and not dirs[0]:
|
||||
dirs = dirs[1:]
|
||||
try:
|
||||
fw = self.connect_ftp('', '', host, port, dirs)
|
||||
t = f and 'I' or 'D'
|
||||
for attr in attrs:
|
||||
attr, value = urllib.splitattr(attr)
|
||||
if attr.lower() == 'type' and \
|
||||
value in ('a', 'A', 'i', 'I', 'd', 'D'):
|
||||
t = value.upper()
|
||||
|
||||
rawr = req.headers.get('Range', None)
|
||||
if rawr:
|
||||
rest = int(rawr.split("=")[1].rstrip("-"))
|
||||
else:
|
||||
rest = 0
|
||||
|
||||
fp, retrlen = fw.retrfile(f, t, rest)
|
||||
|
||||
fb, lb = rest, retrlen
|
||||
if retrlen is None or retrlen == 0:
|
||||
raise RangeError
|
||||
retrlen = lb - fb
|
||||
if retrlen < 0:
|
||||
# beginning of range is larger than file
|
||||
raise RangeError
|
||||
|
||||
headers = ''
|
||||
mtype = mimetypes.guess_type(req.get_full_url())[0]
|
||||
if mtype:
|
||||
headers += 'Content-Type: %s\n' % mtype
|
||||
if retrlen is not None and retrlen >= 0:
|
||||
headers += 'Content-Length: %d\n' % retrlen
|
||||
|
||||
try:
|
||||
import cStringIO as StringIO
|
||||
except ImportError, msg:
|
||||
import StringIO
|
||||
|
||||
return urllib.addinfourl(fp, mimetools.Message(StringIO.StringIO(headers)), req.get_full_url())
|
||||
except ftplib.all_errors, msg:
|
||||
raise IOError, (_('ftp error'), msg), sys.exc_info()[2]
|
||||
|
||||
def connect_ftp(self, user, passwd, host, port, dirs):
|
||||
fw = ftpwrapper(user, passwd, host, port, dirs)
|
||||
return fw
|
||||
|
||||
class ftpwrapper(urllib.ftpwrapper):
|
||||
def retrfile(self, file, type, rest=None):
|
||||
self.endtransfer()
|
||||
if type in ('d', 'D'): cmd = 'TYPE A'; isdir = 1
|
||||
else: cmd = 'TYPE ' + type; isdir = 0
|
||||
try:
|
||||
self.ftp.voidcmd(cmd)
|
||||
except ftplib.all_errors:
|
||||
self.init()
|
||||
self.ftp.voidcmd(cmd)
|
||||
conn = None
|
||||
if file and not isdir:
|
||||
try:
|
||||
self.ftp.nlst(file)
|
||||
except ftplib.error_perm, reason:
|
||||
raise IOError, (_('ftp error'), reason), sys.exc_info()[2]
|
||||
# Restore the transfer mode!
|
||||
self.ftp.voidcmd(cmd)
|
||||
try:
|
||||
cmd = 'RETR ' + file
|
||||
conn = self.ftp.ntransfercmd(cmd, rest)
|
||||
except ftplib.error_perm, reason:
|
||||
if str(reason)[:3] == '501':
|
||||
# workaround for REST not suported error
|
||||
fp, retrlen = self.retrfile(file, type)
|
||||
# WTF? No global (RangeableFileObject) found. RangeableFileObject only defined in urlgrabber / caglar
|
||||
fp = RangeableFileObject(fp, (rest,''))
|
||||
return (fp, retrlen)
|
||||
elif str(reason)[:3] != '550':
|
||||
raise IOError, (_('ftp error'), reason), sys.exc_info()[2]
|
||||
if not conn:
|
||||
self.ftp.voidcmd('TYPE A')
|
||||
if file: cmd = 'LIST ' + file
|
||||
else: cmd = 'LIST'
|
||||
conn = self.ftp.ntransfercmd(cmd)
|
||||
self.busy = 1
|
||||
return (urllib.addclosehook(conn[0].makefile('rb'),
|
||||
self.endtransfer), conn[1])
|
||||
|
||||
# helper function
|
||||
def fetch_url(url, destdir, progress=None):
|
||||
fetch = Fetcher(url, destdir)
|
||||
fetch.progress = progress
|
||||
fetch.fetch()
|
||||
|
||||
Reference in New Issue
Block a user