From c836f33d735f74a24845027b053a4d258f00056b Mon Sep 17 00:00:00 2001 From: Faik Uygur Date: Fri, 23 Mar 2007 14:35:16 +0000 Subject: [PATCH] [DELTA] Delta merge from pisi-delta branch --- pisi/atomicoperations.py | 61 ++++++++++++++++-- pisi/cli/commands.py | 49 +++++++++++++++ pisi/delta.py | 132 +++++++++++++++++++++++++++++++++++++++ pisi/index.py | 23 ++++++- pisi/metadata.py | 16 +++++ pisi/operations.py | 22 +++++-- pisi/package.py | 5 ++ pisi/util.py | 12 ++++ 8 files changed, 306 insertions(+), 14 deletions(-) create mode 100644 pisi/delta.py diff --git a/pisi/atomicoperations.py b/pisi/atomicoperations.py index 2cbfb564..86bea8a2 100644 --- a/pisi/atomicoperations.py +++ b/pisi/atomicoperations.py @@ -18,6 +18,7 @@ _ = __trans.ugettext import sys import os import bsddb3.db as db +import shutil import pisi import pisi.context as ctx @@ -31,6 +32,7 @@ from pisi.files import Files from pisi.uri import URI import pisi.ui from pisi.version import Version +from pisi.delta import find_relocations class Error(pisi.Error): pass @@ -66,14 +68,25 @@ class Install(AtomicOperation): ctx.ui.info(_("Package %s found in repository %s") % (name, repo)) repo = ctx.repodb.get_repo(repo) pkg = ctx.packagedb.get_package(name) + delta = None - # FIXME: let pkg.packageURI be stored as URI type rather than string - pkg_uri = URI(pkg.packageURI) - if pkg_uri.is_absolute_path(): - pkg_path = str(pkg.packageURI) + # Package is installed. This is an upgrade. Check delta. + if ctx.installdb.is_installed(pkg.name): + (version, release, build) = ctx.installdb.get_version(pkg.name) + delta = pkg.get_delta(buildFrom=build) + + # If delta exists than use the delta uri. + if delta: + pkg_uri = delta.packageURI + else: + pkg_uri = pkg.packageURI + + uri = URI(pkg_uri) + if uri.is_absolute_path(): + pkg_path = str(pkg_uri) else: pkg_path = os.path.join(os.path.dirname(repo.indexuri.get_uri()), - str(pkg_uri.path())) + str(uri.path())) ctx.ui.info(_("Package URI: %s") % pkg_path, verbose=True) @@ -287,9 +300,42 @@ class Install(AtomicOperation): oldconfig = path + '.old' if os.path.exists(newconfig): os.unlink(newconfig) - os.rename(path, newconfig) + + # In the case of delta packages: the old package and the new package + # may contain same config typed files with same hashes, so the delta + # package will not have that config file. In order to protect user + # changed config files, they are renamed with ".old" prefix in case + # of the hashes of these files on the filesystem and the new config + # file that is coming from the new package. But in delta package case + # with the given scenario there wont be any, so we can pass this one. + # If the config files were not be the same between these packages the + # delta package would have it and extract it and the path would point + # to that new config file. If they are same and the user had changed + # that file and using the changed config file, there is no problem + # here. + if os.path.exists(path): + os.rename(path, newconfig) + os.rename(oldconfig, path) + # Delta package does not contain the files that have the same hash as in + # the old package's. Because it means the file has not changed. But some + # of these files may be relocated to some other directory in the new package. + # We handle these cases here. + def relocate_files(): + for old_file, new_file in find_relocations(self.old_files, self.files): + old_path, new_path = ("/" + old_file.path, "/" + new_file.path) + + destdir = os.path.dirname(new_path) + if not os.path.exists(destdir): + os.makedirs(destdir) + + if os.path.islink(old_path): + if not os.path.lexists(new_path): + os.symlink(os.readlink(old_path), new_path) + else: + shutil.copy(old_path, new_path) + # remove left over files from the old package. def clean_leftovers(): new = set(map(lambda x: str(x.path), self.files.list)) @@ -321,6 +367,9 @@ class Install(AtomicOperation): # there may be left over config files check_config_changed(file) + if self.package_fname.endswith(ctx.const.delta_package_suffix): + relocate_files() + self.package.extract_install(ctx.config.dest_dir()) if config_changed: diff --git a/pisi/cli/commands.py b/pisi/cli/commands.py index f469a6e7..1e5fd2d1 100644 --- a/pisi/cli/commands.py +++ b/pisi/cli/commands.py @@ -494,6 +494,55 @@ to be downloaded from a repository containing sources. pisi.api.build(x) self.finalize() +class Delta(Command): + """Creates delta PiSi packages + +Usage: delta oldpackage newpackage + +Delta command finds the changed files between the given packages by comparing the sha1sum of the files +and creates a delta pisi package with the changed files between two releases. + +""" + __metaclass__ = autocommand + + def __init__(self, args): + super(Delta, self).__init__(args) + + name = ("delta", "dt") + + def options(self): + + group = OptionGroup(self.parser, _("delta options")) + self.add_options(group) + self.parser.add_option_group(group) + + def add_options(self, group): + group.add_option("-O", "--output-dir", action="store", default=None, + help=_("Output directory for produced packages")) + + def run(self): + + from pisi.delta import create_delta_package + + self.init(database=False, write=False) + + if len(self.args) is not 2: + self.help() + return + + if ctx.get_option('output_dir'): + ctx.ui.info(_('Output directory: %s') % ctx.config.options.output_dir) + else: + ctx.ui.info(_('Outputting packages in the working directory.')) + ctx.config.options.output_dir = '.' + + oldpackage = self.args[0] + newpackage = self.args[1] + + create_delta_package(oldpackage, newpackage) + + self.finalize() + class Emerge(Build): """Build and install PiSi source packages from repository diff --git a/pisi/delta.py b/pisi/delta.py new file mode 100644 index 00000000..e9dd29fb --- /dev/null +++ b/pisi/delta.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2007, TUBITAK/UEKAE +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# Please read the COPYING file. + +import os +from sets import Set as set + +import gettext +__trans = gettext.translation("pisi", fallback=True) +_ = __trans.ugettext + +import pisi.context as ctx +from pisi.package import Package +import pisi.util as util +import pisi.archive as archive + +def create_delta_package(old_package, new_package): + + if old_package == new_package: + ctx.ui.error(_("Cannot create delta for same package!")) + return + + oldpkg = Package(old_package, "r") + newpkg = Package(new_package, "r") + + newmd = newpkg.get_metadata() + oldmd = oldpkg.get_metadata() + + oldfiles = oldpkg.get_files() + newfiles = newpkg.get_files() + + files_delta = find_delta(oldfiles, newfiles) + + ctx.ui.info(_("Creating delta PiSi package between %s %s") % (old_package, new_package)) + + # Unpack new package to temp + newpkg_name = util.package_name(newmd.package.name, newmd.package.version, newmd.package.release, newmd.package.build, False) + newpkg_path = util.join_path(ctx.config.tmp_dir(), newpkg_name) + newpkg.extract_to(newpkg_path, True) + + tar = archive.ArchiveTar(util.join_path(newpkg_path, ctx.const.install_tar_lzma), "tarlzma", False, False) + tar.unpack_dir(newpkg_path) + + # Create delta package + deltaname = "%s-%s-%s%s" % (oldmd.package.name, oldmd.package.build, newmd.package.build, ctx.const.delta_package_suffix) + + outdir = ctx.get_option("output_dir") + if outdir: + deltaname = util.join_path(outdir, deltaname) + + deltapkg = Package(deltaname, "w") + + c = os.getcwd() + os.chdir(newpkg_path) + + # add comar files to package + for pcomar in newmd.package.providesComar: + fname = util.join_path(ctx.const.comar_dir, pcomar.script) + deltapkg.add_to_package(fname) + + # add xmls and files + deltapkg.add_to_package(ctx.const.metadata_xml) + deltapkg.add_to_package(ctx.const.files_xml) + + # only metadata information may change in a package, so no install.tar.lzma added to delta package + if files_delta: + ctx.build_leftover = util.join_path(ctx.config.tmp_dir(), ctx.const.install_tar_lzma) + + tar = archive.ArchiveTar(util.join_path(ctx.config.tmp_dir(), ctx.const.install_tar_lzma), "tarlzma") + for file in files_delta: + tar.add_to_archive(file.path) + tar.close() + + os.chdir(ctx.config.tmp_dir()) + deltapkg.add_to_package(ctx.const.install_tar_lzma) + + deltapkg.close() + + tmp_file = util.join_path(ctx.config.tmp_dir(), ctx.const.install_tar_lzma) + if os.path.exists(tmp_file): + os.unlink(tmp_file) + + ctx.build_leftover = None + os.chdir(c) + + ctx.ui.info(_("Done.")) + +# Hash not equals (these are the deltas) +# Hash equal but path different ones (these are the relocations) +# Hash and also path equal ones (do nothing) + +def find_delta(oldfiles, newfiles): + + hashto_files = {} + for file in newfiles.list: + hashto_files.setdefault(file.hash, []).append(file) + + files_new = set(map(lambda x:x.hash, newfiles.list)) + files_old = set(map(lambda x:x.hash, oldfiles.list)) + files_delta = files_new - files_old + + deltas = [] + for hash in files_delta: + deltas.extend(hashto_files[hash]) + + return deltas + +def find_relocations(oldfiles, newfiles): + + files_new = {} + for file in newfiles.list: + files_new.setdefault(file.hash, []).append(file) + + files_old = {} + for file in oldfiles.list: + files_old.setdefault(file.hash, []).append(file) + + relocations = [] + for hash in files_new.keys(): + if hash and hash in files_old: + for i in range(len(files_new[hash])): + if files_old[hash][0].path != files_new[hash][i].path: + relocations.append((files_old[hash][0], files_new[hash][i])) + + return relocations diff --git a/pisi/index.py b/pisi/index.py index 7ac82a48..8d3d38c3 100644 --- a/pisi/index.py +++ b/pisi/index.py @@ -88,10 +88,16 @@ class Index(XmlFile): self.repo_dir = repo_uri packages = [] + deltas = {} for root, dirs, files in os.walk(repo_uri): for fn in files: - if fn.endswith(ctx.const.package_suffix): + + if fn.endswith(ctx.const.delta_package_suffix): + name, version = util.parse_package_name(fn) + deltas.setdefault(name, []).append(os.path.join(root, fn)) + elif fn.endswith(ctx.const.package_suffix): packages.append(os.path.join(root, fn)) + if fn == 'component.xml': ctx.ui.info(_('Adding %s to component index') % fn) self.add_component(os.path.join(root, fn)) @@ -106,7 +112,7 @@ class Index(XmlFile): pkg_name = util.parse_package_name(os.path.basename(pkg))[0] if pkg_name not in obsoletes_list: ctx.ui.info(_('Adding %s to package index') % pkg) - self.add_package(pkg, repo_uri) + self.add_package(pkg, deltas, repo_uri) def update_db(self, repo, txn = None): # FIXME: updating db takes too much time. So a notify mechanism is used to inform the status @@ -133,7 +139,7 @@ class Index(XmlFile): ctx.sourcedb.add_spec(sf, repo, txn=txn) update_progress() - def add_package(self, path, repo_uri): + def add_package(self, path, deltas, repo_uri): package = Package(path, 'r') md = package.get_metadata() md.package.packageSize = os.path.getsize(path) @@ -153,6 +159,17 @@ class Index(XmlFile): # No need to carry these with index (#3965) md.package.files = None md.package.additionalFiles = None + + if md.package.name in deltas: + for delta_path in deltas[md.package.name]: + delta = metadata.Delta() + delta.packageURI = util.removepathprefix(repo_uri, delta_path) + delta.packageSize = os.path.getsize(delta_path) + delta.packageHash = util.sha1_file(delta_path) + name, buildFrom, buildTo = util.parse_delta_package_name(delta_path) + delta.buildFrom = buildFrom + md.package.deltaPackages.append(delta) + self.packages.append(md.package) def add_component(self, path): diff --git a/pisi/metadata.py b/pisi/metadata.py index f26e5ac1..2229221f 100644 --- a/pisi/metadata.py +++ b/pisi/metadata.py @@ -27,6 +27,14 @@ import pisi.pxml.xmlfile as xmlfile import pisi.pxml.autoxml as autoxml import pisi.util as util +class Delta: + __metaclass__ = autoxml.autoxml + + t_PackageURI = [ autoxml.String, autoxml.optional] + t_PackageSize = [ autoxml.Long, autoxml.optional] + t_PackageHash = [ autoxml.String, autoxml.optional, "SHA1Sum" ] + a_buildFrom = [autoxml.String, autoxml.optional] + class Source: __metaclass__ = autoxml.autoxml @@ -45,10 +53,18 @@ class Package(specfile.Package): t_PackageSize = [ autoxml.Long, autoxml.optional] t_PackageHash = [ autoxml.String, autoxml.optional, "SHA1Sum" ] t_PackageURI = [ autoxml.String, autoxml.optional] + t_DeltaPackages = [ [Delta], autoxml.optional] t_PackageFormat = [ autoxml.String, autoxml.optional] t_Source = [ Source, autoxml.optional] + def get_delta(self, buildFrom): + for delta in self.deltaPackages: + if delta.buildFrom == str(buildFrom): + return delta + else: + return None + def decode_hook(self, node, errs, where): self.version = self.history[0].version self.release = self.history[0].release diff --git a/pisi/operations.py b/pisi/operations.py index 1aff30ac..7aff23a6 100644 --- a/pisi/operations.py +++ b/pisi/operations.py @@ -799,15 +799,27 @@ def calculate_download_sizes(order): for pkg in [ctx.packagedb.get_package(name) for name in order]: - # get the cached package's path - fn = util.package_name(pkg.name, pkg.version, pkg.release, pkg.build, True) + delta = None + if ctx.installdb.is_installed(pkg.name): + (version, release, build) = ctx.installdb.get_version(pkg.name) + delta = pkg.get_delta(buildFrom=build) + + if delta: + fn = os.path.basename(delta.packageURI) + pkg_hash = delta.packageHash + pkg_size = delta.packageSize + else: + fn = os.path.basename(pkg.packageURI) + pkg_hash = pkg.packageHash + pkg_size = pkg.packageSize + path = util.join_path(ctx.config.packages_dir(), fn) # check the file and sha1sum to be sure it _is_ the cached package - if os.path.exists(path) and util.sha1_file(path) == pkg.packageHash: - cached_size += pkg.packageSize + if os.path.exists(path) and util.sha1_file(path) == pkg_hash: + cached_size += pkg_size - total_size += pkg.packageSize + total_size += pkg_size ctx.ui.notify(ui.cached, total=total_size, cached=cached_size) return total_size, cached_size diff --git a/pisi/package.py b/pisi/package.py index 42807766..ef309d1e 100644 --- a/pisi/package.py +++ b/pisi/package.py @@ -98,6 +98,11 @@ class Package: this is the function used by the installer""" self.impl.unpack_dir_flat(dir, outdir) + def extract_to(self, outdir, clean_dir = False): + """Extracts contents of the archive to outdir. Before extracting if clean_dir + is set, outdir is deleted with its contents""" + self.impl.unpack(outdir, clean_dir) + def extract_pisi_files(self, outdir): """Extract PiSi control files: metadata.xml, files.xml, action scripts, etc.""" diff --git a/pisi/util.py b/pisi/util.py index 9e90e02e..ec811b90 100644 --- a/pisi/util.py +++ b/pisi/util.py @@ -644,6 +644,18 @@ def parse_package_name(package_name): return (name, version) +def parse_delta_package_name(package_name): + """Separate delta package name and release infos + + example: tasma-5-7.pisi.delta -> (tasma, 5, 7) + + """ + name, build = parse_package_name(package_name) + build = build[:-len(ctx.const.delta_package_suffix)] + buildFrom, buildTo = build.split("-") + + return name, buildFrom, buildTo + def filter_latest_packages(package_paths): """ For a given pisi package paths list where there may also be multiple versions of the same package, filters only the latest versioned ones """