revert 7247.

this again breaks pisi and we don't want it while tring to build
the packages for a new release. Experimental code and new features
_will_ go to repos/pisi-new-features!

pisi/archive.py

@@ -130,7 +130,7 @@ class ArchiveZip(ArchiveBase):
                 attr.external_attr = self.symmagic 
                 self.zip_obj.writestr(attr, dest)
             else:
-                self.zip_obj.write(file_name, file_name, zipfileext.ZIP_LZMA_BOGUS)
+                self.zip_obj.write(file_name, file_name, zipfileext.ZIP_LZMA)
                 zinfo = self.zip_obj.getinfo(file_name)
                 zinfo.create_system = 3
 

pisi/zipfileext.py

@@ -9,11 +9,10 @@
 #
 # Please read the COPYING file.
 #
-# Authors:  Eray Ozkural <eray at pardus.org.tr>
+# Authors:  Eray Ozkural <eray@pardus.org.tr>
-#                Faik Uygur   <faik at pardus.org.tr>
+#           Faik Uygur   <faik@pardus.org.tr>
 #
-
+# Extends zipfile module with lzma and bzip2 support
-"""Extends zipfile module with lzma and bzip2 support"""
 
 # python standard library modules
 import os
@@ -21,174 +20,92 @@ import struct
 import time
 import binascii
 
-# we are really extending the zipfile module, not rewriting it.
 from zipfile import *
 
 try:
     import zlib
 except ImportError:
     zlib = None
+
 try:
-    import bz2
+    import bzip2
 except ImportError:
-    bz2 = None
+    bzip2 = None
+
 try:
     import pylzma
 except ImportError:
     pylzma = None
 
 ZIP_BZIP2 = 12
-ZIP_LZMA_BOGUS = 255  # FIXME: we are going to add the official ID when PKWARE gives it to us, and keep this one for a while
+ZIP_LZMA = 255
 
-compression_methods = [ZIP_STORED, ZIP_DEFLATED, ZIP_BZIP2, ZIP_LZMA_BOGUS]
+compression_methods = [ZIP_STORED, ZIP_DEFLATED, ZIP_BZIP2, ZIP_LZMA]
 
-
+class FileEntry:
-class ZipFileEntry:
+    """File-like object used to access entries in a ZipFile"""
-    """File-like object used to access entries in a ZipFile, we're using a factory 
-    design pattern that's a little better than switch blocks thrown about"""
     
-    def __init__(self, fp):
+    def __init__(self, fp, length):
         self.fp = fp
         self.readBytes = 0
-        self.returnedBytes = 0 #FIXME: we don't use studlyCaps in python
+        self.returnedBytes = 0
+        self.length = length
         self.finished = 0
-        self.buffer = ""
 
+class LzmaFileEntry(FileEntry):
+    """File-like object used to read a LZMA entry in a ZipFile"""
+
+    def __init__(self, fp, length):
+        FileEntry.__init__(self, fp, length)
+        self.decomp = pylzma.decompressobj()
+        self.buffer = ""
+        
     def tell(self):
         return self.returnedBytes
-
+    
-    def read(self, compress_size, n=None):
+    def read(self, n=None):
-        """read the whole file, or n bytes and return the decompressed stuff
-        does it in a buffered fashion"""
-        self.length = compress_size
         if self.finished:
             return ""
         if n is None:
-            compr_data = self.fp.read(self.length)
+            result = [self.buffer,]
-            data = self.decompress(compr_data)
+            result.append(self.decomp.decompress(self.fp.read(self.length - self.readBytes)))
+            result.append(self.decomp.flush())
+            self.buffer = ""
             self.finished = 1
-            self.returnedBytes += len(data)
+            result = "".join(result)
-            return data
+            self.returnedBytes += len(result)
+            return result
         else:
-            # FIXME: must always decompress in streaming mode, not just when n is given
+            while len(self.buffer) < n:
-            while len(self.buffer) < n:  
+                data = self.fp.read(min(n, 1024, self.length - self.readBytes))
-                compr_data = self.fp.read(min(n, 1024 * 8, self.length - self.readBytes))
+                self.readBytes += len(data)
-                self.readBytes += len(compr_data)
                 if not data:
-                    result = self.buffer # + self.flush_decompressor()
+                    result = self.buffer + self.decomp.decompress() + self.decomp.flush()
                     self.finished = 1
                     self.buffer = ""
                     self.returnedBytes += len(result)
                     return result
                 else:
-                    self.buffer += self.decompress(data)
+                    self.buffer += self.decomp.decompress(data)
             result = self.buffer[:n]
             self.buffer = self.buffer[n:]
             self.returnedBytes += len(result)
             return result
-
+    
-    #TODO: use this instead of bulk compression
-    def write(self, infile):
-        self.CRC = 0
-        self.file_size = 0
-        self.compress_size = 0
-        while 1:
-            buf = infile.read(1024 * 8)
-            if not buf:
-                break
-            self.file_size = self.file_size + len(buf)
-            self.CRC = binascii.crc32(buf, self.CRC)
-            if compressor:
-                buf = self.compress(buf)
-            self.compress_size = self.compress_size + len(buf)
-            self.fp.write(buf)
-        return self.compress_size
-#            if cmpr:
-  #              buf = cmpr.flush()
-    #            compress_size = compress_size + len(buf)
-      #          self.fp.write(buf)
-
     def close(self):
         self.finished = 1
         del self.fp
 
-
+class ZipFileEntry(FileEntry):
-#TODO: test deflate and bzip2 support thoroughly
-class DeflatedZipFileEntry(ZipFileEntry):
-    """File-like object used to read a deflated entry in a ZipFile"""
-    
-    def __init__(self, fp):
-        ZipFileEntry.__init__(self, fp)
-        self.decomp = zlib.decompressobj(-15)
-        self.compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
-                                                                   zlib.DEFLATED, -15)
-        
-    def decompress(self, compr_data):
-        return self.decomp.decompress(compr_data) + self.decomp.decompress("Z") + self.decomp.flush()
-
-    def compress(self, data):
-        # big deal with flush....
-        return self.compressor.compress(data)  + compressor.flush()
-
-    def write(self, infile):
-        data = infile.read()
-        compressed = self.compress(data)
-        self.fp.write(compressed)
-        return len(compressed)
-
-
-class Bzip2ZipFileEntry(ZipFileEntry):
-    """File-like object used to read a BZIP2 entry in a ZipFile"""
-    
-    def __init__(self, fp):
-        ZipFileEntry.__init__(self, fp)
-        self.decomp = bz2.Decompressor()
-        
-    def decompress(self, compr_data):
-        return self.decomp.decompress(compr_data)
-
-    def write(self, infile):
-        data = infile.read()
-        self.file_size = len(data)
-        self.CRC = binascii.crc32(data)
-        compressed = bz2.compress(data)
-        self.fp.write(compressed)
-        self.compress_size = len(compressed)
-        return self.compress_size
-
-
-class LzmaZipFileEntry(ZipFileEntry):
-    """File-like object used to read a LZMA entry in a ZipFile"""
-
-    def __init__(self, fp):
-        ZipFileEntry.__init__(self, fp)
-        self.decompressor = pylzma.decompressobj()
-    
-    def decompress(self, compr_data):
-        return self.decompressor.decompress(compr_data) + self.decompressor.flush()
-
-    def write(self, infile):
-        #TODO: use the buffered write in superclass
-        data = infile.read()
-        self.file_size = len(data)
-        self.CRC = binascii.crc32(data)
-        compressed = pylzma.compress(data, eos=1)
-        self.fp.write(compressed)
-        self.compress_size = len(compressed)
-        return len(compressed)
-
-
-class StoredZipFileEntry(ZipFileEntry):
     """File-like object used to read an uncompressed entry in a ZipFile"""
     
-    def __init__(self, fp):
+    def __init__(self, fp, length):
-        ZipFileEntry.__init__(self, fp)
+        FileEntry.__init__(self, fp, length)
         
     def tell(self):
         return self.readBytes
     
-    def read(self, length, n=None):
+    def read(self, n=None):
-        self.length = length
         if n is None:
             n = self.length - self.readBytes
         if n == 0 or self.finished:
@@ -205,31 +122,52 @@ class StoredZipFileEntry(ZipFileEntry):
         del self.fp
 
 
-class ZipFileExt(ZipFile):
+class DeflatedZipFileEntry(FileEntry):
-
+    """File-like object used to read a deflated entry in a ZipFile"""
-    def build_file_entry(self, compress_type):
+    
-        "a small factory method"
+    def __init__(self, fp, length):
-        if compress_type == ZIP_STORED:
+        FileEntry.__init__(self, fp, length)
-            return StoredZipFileEntry(self.fp)
+        self.decomp = zlib.decompressobj(-15)
-        elif compress_type == ZIP_DEFLATED:
+        self.buffer = ""
-            if not zlib:
+        
-                raise RuntimeError, \
+    def tell(self):
-                      "Compression requires the missing %s module" % "zlib"
+        return self.returnedBytes
-            return DeflatedZipFileEntry(self.fp)
+    
-        elif compress_type == ZIP_BZIP2:
+    def read(self, n=None):
-            if not bz2:
+        if self.finished:
-                raise RuntimeError, \
+            return ""
-                      "Compression method requires the missing %s module" % "bz2"
+        if n is None:
-            return Bzip2ZipFileEntry(self.fp)
+            result = [self.buffer,]
-        elif compress_type == ZIP_LZMA_BOGUS:
+            result.append(self.decomp.decompress(self.fp.read(self.length - self.readBytes)))
-            if not pylzma:
+            result.append(self.decomp.decompress("Z"))
-                raise RuntimeError, \
+            result.append(self.decomp.flush())
-                      "Compression method requires the missing %s module" % "pylzma"
+            self.buffer = ""
-            return LzmaZipFileEntry(self.fp)
+            self.finished = 1
+            result = "".join(result)
+            self.returnedBytes += len(result)
+            return result
         else:
-            raise BadZipfile, \
+            while len(self.buffer) < n:
-                  "Unsupported compression method %d for file %s" % \
+                data = self.fp.read(min(n, 1024, self.length - self.readBytes))
-            (compress_type, name)
+                self.readBytes += len(data)
+                if not data:
+                    result = self.buffer + self.decomp.decompress("Z") + self.decomp.flush()
+                    self.finished = 1
+                    self.buffer = ""
+                    self.returnedBytes += len(result)
+                    return result
+                else:
+                    self.buffer += self.decomp.decompress(data)
+            result = self.buffer[:n]
+            self.buffer = self.buffer[n:]
+            self.returnedBytes += len(result)
+            return result
+    
+    def close(self):
+        self.finished = 1
+        del self.fp
+
+class ZipFileExt(ZipFile):
 
     def _writecheck(self, zinfo):
         """Check for errors before writing a file to the archive."""
@@ -241,8 +179,20 @@ class ZipFileExt(ZipFile):
         if not self.fp:
             raise RuntimeError, \
                   "Attempt to write ZIP archive that was already closed"
+        if zinfo.compress_type not in compression_methods:
+            raise RuntimeError, \
+                  "Compression method %s is not supported" % zinfo.compress_type
+        if zinfo.compress_type == ZIP_DEFLATED and not zlib:
+            raise RuntimeError, \
+                  "Compression requires the (missing) zlib module"
+        if zinfo.compress_type == ZIP_BZIP2 and not bzip2:
+            raise RuntimeError, \
+                  "Compression requires the (missing) bzip2 module"
+        if zinfo.compress_type == ZIP_LZMA and not pylzma:
+            raise RuntimeError, \
+                  "Compression requires the (missing) pylzma module"
 
-    def write(self, filename, arcname=None, compress_type=ZIP_DEFLATED):
+    def write(self, filename, arcname=None, compress_type=None):
         """Put the bytes from filename into the archive under the name
         arcname."""
         st = os.stat(filename)
@@ -269,38 +219,90 @@ class ZipFileExt(ZipFile):
         self.fp.write(zinfo.FileHeader())
         zinfo.file_offset = self.fp.tell()      # Start of file bytes
 
-        # build a zipfileentry object from factory, and write compressed data
+        # really compress
-        fileentry = self.build_file_entry(zinfo.compress_type)
+        if zinfo.compress_type == ZIP_DEFLATED or zinfo.compress_type == ZIP_STORED:  
-        fileentry.write(fp)
+            if zinfo.compress_type == ZIP_DEFLATED:
-        # update zinfo
+                cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
-        zinfo.CRC = fileentry.CRC
+                                        zlib.DEFLATED, -15)            
-        zinfo.file_size = fileentry.file_size
+            else:
-        zinfo.compress_size = fileentry.compress_size
+                cmpr = None
 
+            while 1:
+                buf = fp.read(1024 * 8)
+                if not buf:
+                    break
+                file_size = file_size + len(buf)
+                CRC = binascii.crc32(buf, CRC)
+                if cmpr:
+                    buf = cmpr.compress(buf)
+                    compress_size = compress_size + len(buf)
+                self.fp.write(buf)
+            fp.close()
+            if cmpr:
+                buf = cmpr.flush()
+                compress_size = compress_size + len(buf)
+                self.fp.write(buf)
+                zinfo.compress_size = compress_size
+            else:
+                zinfo.compress_size = file_size
+
+        elif zinfo.compress_type == ZIP_LZMA:
+            # unfortunately pylzma.compressobj is not implemented yet.
+            # So in order to calculate the CRC, we are going to read
+            # all the file at once until it is implemented.
+            buf = fp.read()
+            CRC = binascii.crc32(buf, CRC)
+            compressed = pylzma.compress(buf, eos=1)
+            self.fp.write(compressed)
+            zinfo.compress_size = len(compressed)
+
+        zinfo.CRC = CRC
+        zinfo.file_size = file_size
         # Seek backwards and write CRC and file sizes
         position = self.fp.tell()       # Preserve current position in file
         self.fp.seek(zinfo.header_offset + 14, 0)
         self.fp.write(struct.pack("<lLL", zinfo.CRC, zinfo.compress_size,
-                                                zinfo.file_size))
+              zinfo.file_size))
         self.fp.seek(position, 0)
         self.filelist.append(zinfo)
         self.NameToInfo[zinfo.filename] = zinfo
 
-    def _readcheck(self, zinfo):
+    def readfile(self, name):
         """Return file-like object for name."""
         if self.mode not in ("r", "a"):
             raise RuntimeError, 'read() requires mode "r" or "a"'
         if not self.fp:
             raise RuntimeError, \
                   "Attempt to read ZIP archive that was already closed"
- 
+        zinfo = self.getinfo(name)
+        self.fp.seek(zinfo.file_offset, 0)
+        if zinfo.compress_type == ZIP_STORED:
+            return ZipFileEntry(self.fp, zinfo.compress_size)
+        elif zinfo.compress_type == ZIP_DEFLATED:
+            if not zlib:
+                raise RuntimeError, \
+                      "De-compression requires the (missing) zlib module"
+            return DeflatedZipFileEntry(self.fp, zinfo.compress_size)
+##        elif zinfo.compress_type == ZIP_BZIP2:
+##            if not bzip2:
+##                raise RuntimeError, \
+##                      "De-compression requires the (missing) bzip2 module"
+##	    return LZMAFileEntry(self.fp, zinfo.compress_size)
+        elif zinfo.compress_type == ZIP_LZMA:
+            if not pylzma:
+                raise RuntimeError, \
+                      "De-compression requires the (missing) pylzma module"
+            return LzmaFileEntry(self.fp, zinfo.compress_size)
+        else:
+            raise BadZipfile, \
+                  "Unsupported compression method %d for file %s" % \
+            (zinfo.compress_type, name)
+
     def read(self, name):
         """Return file bytes (as a string) for name."""
+        f = self.readfile(name)
         zinfo = self.getinfo(name)
-        self._readcheck(zinfo)
+        bytes = f.read()
-        self.fp.seek(zinfo.file_offset, 0)
-        f = self.build_file_entry(zinfo.compress_type)
-        bytes = f.read(zinfo.compress_size)
         crc = binascii.crc32(bytes)
         if crc != zinfo.CRC:
             raise BadZipfile, "Bad CRC-32 for file %s" % name