[DELTA] Delta merge from pisi-delta branch

pisi/atomicoperations.py

@@ -18,6 +18,7 @@ _ = __trans.ugettext
 import sys
 import os
 import bsddb3.db as db
+import shutil
 
 import pisi
 import pisi.context as ctx
@@ -31,6 +32,7 @@ from pisi.files import Files
 from pisi.uri import URI
 import pisi.ui
 from pisi.version import Version
+from pisi.delta import find_relocations
 
 class Error(pisi.Error):
     pass
@@ -66,14 +68,25 @@ class Install(AtomicOperation):
             ctx.ui.info(_("Package %s found in repository %s") % (name, repo))
             repo = ctx.repodb.get_repo(repo)
             pkg = ctx.packagedb.get_package(name)
+            delta = None
 
-            # FIXME: let pkg.packageURI be stored as URI type rather than string
+            # Package is installed. This is an upgrade. Check delta.
-            pkg_uri = URI(pkg.packageURI)
+            if ctx.installdb.is_installed(pkg.name):
-            if pkg_uri.is_absolute_path():
+                (version, release, build) = ctx.installdb.get_version(pkg.name)
-                pkg_path = str(pkg.packageURI)
+                delta = pkg.get_delta(buildFrom=build)
+
+            # If delta exists than use the delta uri.
+            if delta:
+                pkg_uri = delta.packageURI
+            else:
+                pkg_uri = pkg.packageURI
+
+            uri = URI(pkg_uri)
+            if uri.is_absolute_path():
+                pkg_path = str(pkg_uri)
             else:
                 pkg_path = os.path.join(os.path.dirname(repo.indexuri.get_uri()),
-                                        str(pkg_uri.path()))
+                                        str(uri.path()))
 
             ctx.ui.info(_("Package URI: %s") % pkg_path, verbose=True)
 
@@ -287,9 +300,42 @@ class Install(AtomicOperation):
                 oldconfig = path + '.old'
                 if os.path.exists(newconfig):
                     os.unlink(newconfig)
-                os.rename(path, newconfig)
+
+                # In the case of delta packages: the old package and the new package
+                # may contain same config typed files with same hashes, so the delta
+                # package will not have that config file. In order to protect user
+                # changed config files, they are renamed with ".old" prefix in case
+                # of the hashes of these files on the filesystem and the new config 
+                # file that is coming from the new package. But in delta package case
+                # with the given scenario there wont be any, so we can pass this one.
+                # If the config files were not be the same between these packages the
+                # delta package would have it and extract it and the path would point
+                # to that new config file. If they are same and the user had changed 
+                # that file and using the changed config file, there is no problem 
+                # here.
+                if os.path.exists(path):
+                    os.rename(path, newconfig)
+
                 os.rename(oldconfig, path)
 
+        # Delta package does not contain the files that have the same hash as in 
+        # the old package's. Because it means the file has not changed. But some 
+        # of these files may be relocated to some other directory in the new package. 
+        # We handle these cases here.
+        def relocate_files():
+            for old_file, new_file in find_relocations(self.old_files, self.files):
+                old_path, new_path = ("/" + old_file.path, "/" + new_file.path)
+
+                destdir = os.path.dirname(new_path)
+                if not os.path.exists(destdir):
+                    os.makedirs(destdir)
+
+                if os.path.islink(old_path): 
+                    if not os.path.lexists(new_path):
+                        os.symlink(os.readlink(old_path), new_path)
+                else:
+                    shutil.copy(old_path, new_path)
+
         # remove left over files from the old package.
         def clean_leftovers():
             new = set(map(lambda x: str(x.path), self.files.list))
@@ -321,6 +367,9 @@ class Install(AtomicOperation):
                     # there may be left over config files
                     check_config_changed(file)
 
+        if self.package_fname.endswith(ctx.const.delta_package_suffix):
+            relocate_files()
+
         self.package.extract_install(ctx.config.dest_dir())
 
         if config_changed:

pisi/cli/commands.py

@@ -494,6 +494,55 @@ to be downloaded from a repository containing sources.
                 pisi.api.build(x)
         self.finalize()
 
+class Delta(Command):
+    """Creates delta PiSi packages
+
+Usage: delta oldpackage newpackage
+
+Delta command finds the changed files between the given packages by comparing the sha1sum of the files
+and creates a delta pisi package with the changed files between two releases.
+
+"""
+    __metaclass__ = autocommand
+
+    def __init__(self, args):
+        super(Delta, self).__init__(args)
+
+    name = ("delta", "dt")
+
+    def options(self):
+
+        group = OptionGroup(self.parser, _("delta options"))
+        self.add_options(group)
+        self.parser.add_option_group(group)
+
+    def add_options(self, group):
+        group.add_option("-O", "--output-dir", action="store", default=None,
+                               help=_("Output directory for produced packages"))
+
+    def run(self):
+
+        from pisi.delta import create_delta_package 
+
+        self.init(database=False, write=False)
+
+        if len(self.args) is not 2:
+            self.help()
+            return
+
+        if ctx.get_option('output_dir'):
+            ctx.ui.info(_('Output directory: %s') % ctx.config.options.output_dir)
+        else:
+            ctx.ui.info(_('Outputting packages in the working directory.'))
+            ctx.config.options.output_dir = '.'
+
+        oldpackage = self.args[0]
+        newpackage = self.args[1]
+
+        create_delta_package(oldpackage, newpackage)
+
+        self.finalize()
+
 class Emerge(Build):
     """Build and install PiSi source packages from repository
 

pisi/delta.py

@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2007, TUBITAK/UEKAE
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# Please read the COPYING file.
+
+import os
+from sets import Set as set
+
+import gettext
+__trans = gettext.translation("pisi", fallback=True)
+_ = __trans.ugettext
+
+import pisi.context as ctx
+from pisi.package import Package
+import pisi.util as util
+import pisi.archive as archive
+
+def create_delta_package(old_package, new_package):
+
+    if old_package == new_package:
+        ctx.ui.error(_("Cannot create delta for same package!"))
+        return
+
+    oldpkg = Package(old_package, "r")
+    newpkg = Package(new_package, "r")
+
+    newmd = newpkg.get_metadata()
+    oldmd = oldpkg.get_metadata()
+
+    oldfiles = oldpkg.get_files()
+    newfiles = newpkg.get_files()
+
+    files_delta = find_delta(oldfiles, newfiles)
+
+    ctx.ui.info(_("Creating delta PiSi package between %s %s") % (old_package, new_package))
+
+    # Unpack new package to temp
+    newpkg_name = util.package_name(newmd.package.name, newmd.package.version, newmd.package.release, newmd.package.build, False)
+    newpkg_path = util.join_path(ctx.config.tmp_dir(), newpkg_name)
+    newpkg.extract_to(newpkg_path, True)
+
+    tar = archive.ArchiveTar(util.join_path(newpkg_path, ctx.const.install_tar_lzma), "tarlzma", False, False)
+    tar.unpack_dir(newpkg_path)
+
+    # Create delta package
+    deltaname = "%s-%s-%s%s" % (oldmd.package.name, oldmd.package.build, newmd.package.build, ctx.const.delta_package_suffix)
+
+    outdir = ctx.get_option("output_dir")
+    if outdir:
+        deltaname = util.join_path(outdir, deltaname)
+
+    deltapkg = Package(deltaname, "w")
+
+    c = os.getcwd()
+    os.chdir(newpkg_path)
+
+    # add comar files to package
+    for pcomar in newmd.package.providesComar:
+        fname = util.join_path(ctx.const.comar_dir, pcomar.script)
+        deltapkg.add_to_package(fname)
+
+    # add xmls and files
+    deltapkg.add_to_package(ctx.const.metadata_xml)
+    deltapkg.add_to_package(ctx.const.files_xml)
+
+    # only metadata information may change in a package, so no install.tar.lzma added to delta package
+    if files_delta:
+        ctx.build_leftover = util.join_path(ctx.config.tmp_dir(), ctx.const.install_tar_lzma)
+
+        tar = archive.ArchiveTar(util.join_path(ctx.config.tmp_dir(), ctx.const.install_tar_lzma), "tarlzma")
+        for file in files_delta:
+            tar.add_to_archive(file.path)
+        tar.close()
+
+        os.chdir(ctx.config.tmp_dir())
+        deltapkg.add_to_package(ctx.const.install_tar_lzma)
+
+    deltapkg.close()
+
+    tmp_file = util.join_path(ctx.config.tmp_dir(), ctx.const.install_tar_lzma)
+    if os.path.exists(tmp_file):
+        os.unlink(tmp_file)
+
+    ctx.build_leftover = None
+    os.chdir(c)
+
+    ctx.ui.info(_("Done."))
+
+#  Hash not equals                      (these are the deltas)
+#  Hash equal but path different ones   (these are the relocations)
+#  Hash and also path equal ones        (do nothing)
+
+def find_delta(oldfiles, newfiles):
+
+    hashto_files = {}
+    for file in newfiles.list:
+        hashto_files.setdefault(file.hash, []).append(file)
+
+    files_new = set(map(lambda x:x.hash, newfiles.list))
+    files_old = set(map(lambda x:x.hash, oldfiles.list))
+    files_delta = files_new - files_old
+
+    deltas = []
+    for hash in files_delta:
+        deltas.extend(hashto_files[hash])
+
+    return deltas
+
+def find_relocations(oldfiles, newfiles):
+
+    files_new = {}
+    for file in newfiles.list:
+        files_new.setdefault(file.hash, []).append(file)
+
+    files_old = {}
+    for file in oldfiles.list:
+        files_old.setdefault(file.hash, []).append(file)
+
+    relocations = []
+    for hash in files_new.keys():
+        if hash and hash in files_old:
+            for i in range(len(files_new[hash])):
+                if files_old[hash][0].path != files_new[hash][i].path:
+                    relocations.append((files_old[hash][0], files_new[hash][i]))
+
+    return relocations

pisi/index.py

@@ -88,10 +88,16 @@ class Index(XmlFile):
         self.repo_dir = repo_uri
 
         packages = []
+        deltas = {}
         for root, dirs, files in os.walk(repo_uri):
             for fn in files:
-                if fn.endswith(ctx.const.package_suffix):
+
+                if fn.endswith(ctx.const.delta_package_suffix):
+                    name, version = util.parse_package_name(fn)
+                    deltas.setdefault(name, []).append(os.path.join(root, fn))
+                elif fn.endswith(ctx.const.package_suffix):
                     packages.append(os.path.join(root, fn))
+
                 if fn == 'component.xml':
                     ctx.ui.info(_('Adding %s to component index') % fn)
                     self.add_component(os.path.join(root, fn))
@@ -106,7 +112,7 @@ class Index(XmlFile):
             pkg_name = util.parse_package_name(os.path.basename(pkg))[0]
             if pkg_name not in obsoletes_list:
                 ctx.ui.info(_('Adding %s to package index') % pkg)
-                self.add_package(pkg, repo_uri)
+                self.add_package(pkg, deltas, repo_uri)
 
     def update_db(self, repo, txn = None):
         # FIXME: updating db takes too much time. So a notify mechanism is used to inform the status
@@ -133,7 +139,7 @@ class Index(XmlFile):
             ctx.sourcedb.add_spec(sf, repo, txn=txn)
             update_progress()
 
-    def add_package(self, path, repo_uri):
+    def add_package(self, path, deltas, repo_uri):
         package = Package(path, 'r')
         md = package.get_metadata()
         md.package.packageSize = os.path.getsize(path)
@@ -153,6 +159,17 @@ class Index(XmlFile):
             # No need to carry these with index (#3965)
             md.package.files = None
             md.package.additionalFiles = None
+
+            if md.package.name in deltas:
+                for delta_path in deltas[md.package.name]:
+                    delta = metadata.Delta()
+                    delta.packageURI = util.removepathprefix(repo_uri, delta_path)
+                    delta.packageSize = os.path.getsize(delta_path)
+                    delta.packageHash = util.sha1_file(delta_path)
+                    name, buildFrom, buildTo = util.parse_delta_package_name(delta_path)
+                    delta.buildFrom = buildFrom
+                    md.package.deltaPackages.append(delta)
+
             self.packages.append(md.package)
 
     def add_component(self, path):

pisi/metadata.py

@@ -27,6 +27,14 @@ import pisi.pxml.xmlfile as xmlfile
 import pisi.pxml.autoxml as autoxml
 import pisi.util as util
 
+class Delta:
+    __metaclass__ = autoxml.autoxml
+    
+    t_PackageURI = [ autoxml.String, autoxml.optional]
+    t_PackageSize = [ autoxml.Long, autoxml.optional]
+    t_PackageHash = [ autoxml.String, autoxml.optional, "SHA1Sum" ]
+    a_buildFrom = [autoxml.String, autoxml.optional]
+
 class Source:
     __metaclass__ = autoxml.autoxml
 
@@ -45,10 +53,18 @@ class Package(specfile.Package):
     t_PackageSize = [ autoxml.Long, autoxml.optional]
     t_PackageHash = [ autoxml.String, autoxml.optional, "SHA1Sum" ]
     t_PackageURI = [ autoxml.String, autoxml.optional]
+    t_DeltaPackages = [ [Delta], autoxml.optional]
     t_PackageFormat = [ autoxml.String, autoxml.optional]
 
     t_Source = [ Source, autoxml.optional]
 
+    def get_delta(self, buildFrom):
+        for delta in self.deltaPackages:
+            if delta.buildFrom == str(buildFrom):
+                return delta
+        else:
+            return None
+
     def decode_hook(self, node, errs, where):
         self.version = self.history[0].version
         self.release = self.history[0].release

pisi/operations.py

@@ -799,15 +799,27 @@ def calculate_download_sizes(order):
 
     for pkg in [ctx.packagedb.get_package(name) for name in order]:
 
-        # get the cached package's path
+        delta = None
-        fn = util.package_name(pkg.name, pkg.version, pkg.release, pkg.build, True)
+        if ctx.installdb.is_installed(pkg.name):
+            (version, release, build) = ctx.installdb.get_version(pkg.name)
+            delta = pkg.get_delta(buildFrom=build)
+
+        if delta:
+            fn = os.path.basename(delta.packageURI)
+            pkg_hash = delta.packageHash
+            pkg_size = delta.packageSize
+        else:
+            fn = os.path.basename(pkg.packageURI)
+            pkg_hash = pkg.packageHash
+            pkg_size = pkg.packageSize
+
         path = util.join_path(ctx.config.packages_dir(), fn)
 
         # check the file and sha1sum to be sure it _is_ the cached package
-        if os.path.exists(path) and util.sha1_file(path) == pkg.packageHash:
+        if os.path.exists(path) and util.sha1_file(path) == pkg_hash:
-            cached_size += pkg.packageSize
+            cached_size += pkg_size
 
-        total_size += pkg.packageSize
+        total_size += pkg_size
 
     ctx.ui.notify(ui.cached, total=total_size, cached=cached_size)
     return total_size, cached_size

pisi/package.py

@@ -98,6 +98,11 @@ class Package:
         this is the function used by the installer"""
         self.impl.unpack_dir_flat(dir, outdir)
 
+    def extract_to(self, outdir, clean_dir = False):
+        """Extracts contents of the archive to outdir. Before extracting if clean_dir 
+        is set, outdir is deleted with its contents"""
+        self.impl.unpack(outdir, clean_dir)
+
     def extract_pisi_files(self, outdir):
         """Extract PiSi control files: metadata.xml, files.xml,
         action scripts, etc."""

pisi/util.py

@@ -644,6 +644,18 @@ def parse_package_name(package_name):
 
     return (name, version)
 
+def parse_delta_package_name(package_name):
+    """Separate delta package name and release infos
+    
+    example: tasma-5-7.pisi.delta -> (tasma, 5, 7)
+    
+    """
+    name, build = parse_package_name(package_name)
+    build = build[:-len(ctx.const.delta_package_suffix)]
+    buildFrom, buildTo = build.split("-")
+    
+    return name, buildFrom, buildTo
+
 def filter_latest_packages(package_paths):
     """ For a given pisi package paths list where there may also be multiple versions
         of the same package, filters only the latest versioned ones """