Pisilinux3/pisi
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[DELTA] Delta merge from pisi-delta branch

Browse Source
This commit is contained in:
Faik Uygur
2007-03-23 14:35:16 +00:00
parent 35847b93df
commit c836f33d73
8 changed files with 306 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pisi/atomicoperations.py
+55 -6
View File
@@ -18,6 +18,7 @@ _ = __trans.ugettext
import sys
import os
import bsddb3.db as db
import shutil
import pisi
import pisi.context as ctx
@@ -31,6 +32,7 @@ from pisi.files import Files
from pisi.uri import URI
import pisi.ui
from pisi.version import Version
from pisi.delta import find_relocations
class Error(pisi.Error):
pass
@@ -66,14 +68,25 @@ class Install(AtomicOperation):
ctx.ui.info(_("Package %s found in repository %s") % (name, repo))
repo = ctx.repodb.get_repo(repo)
pkg = ctx.packagedb.get_package(name)
delta = None
# FIXME: let pkg.packageURI be stored as URI type rather than string
pkg_uri = URI(pkg.packageURI)
if pkg_uri.is_absolute_path():
pkg_path = str(pkg.packageURI)
# Package is installed. This is an upgrade. Check delta.
if ctx.installdb.is_installed(pkg.name):
(version, release, build) = ctx.installdb.get_version(pkg.name)
delta = pkg.get_delta(buildFrom=build)
# If delta exists than use the delta uri.
if delta:
pkg_uri = delta.packageURI
else:
pkg_uri = pkg.packageURI
uri = URI(pkg_uri)
if uri.is_absolute_path():
pkg_path = str(pkg_uri)
else:
pkg_path = os.path.join(os.path.dirname(repo.indexuri.get_uri()),
str(pkg_uri.path()))
str(uri.path()))
ctx.ui.info(_("Package URI: %s") % pkg_path, verbose=True)
@@ -287,9 +300,42 @@ class Install(AtomicOperation):
oldconfig = path + '.old'
if os.path.exists(newconfig):
os.unlink(newconfig)
os.rename(path, newconfig)
# In the case of delta packages: the old package and the new package
# may contain same config typed files with same hashes, so the delta
# package will not have that config file. In order to protect user
# changed config files, they are renamed with ".old" prefix in case
# of the hashes of these files on the filesystem and the new config
# file that is coming from the new package. But in delta package case
# with the given scenario there wont be any, so we can pass this one.
# If the config files were not be the same between these packages the
# delta package would have it and extract it and the path would point
# to that new config file. If they are same and the user had changed
# that file and using the changed config file, there is no problem
# here.
if os.path.exists(path):
os.rename(path, newconfig)
os.rename(oldconfig, path)
# Delta package does not contain the files that have the same hash as in
# the old package's. Because it means the file has not changed. But some
# of these files may be relocated to some other directory in the new package.
# We handle these cases here.
def relocate_files():
for old_file, new_file in find_relocations(self.old_files, self.files):
old_path, new_path = ("/" + old_file.path, "/" + new_file.path)
destdir = os.path.dirname(new_path)
if not os.path.exists(destdir):
os.makedirs(destdir)
if os.path.islink(old_path):
if not os.path.lexists(new_path):
os.symlink(os.readlink(old_path), new_path)
else:
shutil.copy(old_path, new_path)
# remove left over files from the old package.
def clean_leftovers():
new = set(map(lambda x: str(x.path), self.files.list))
@@ -321,6 +367,9 @@ class Install(AtomicOperation):
# there may be left over config files
check_config_changed(file)
if self.package_fname.endswith(ctx.const.delta_package_suffix):
relocate_files()
self.package.extract_install(ctx.config.dest_dir())
if config_changed:
pisi/cli/commands.py
+49
View File
@@ -494,6 +494,55 @@ to be downloaded from a repository containing sources.
pisi.api.build(x)
self.finalize()
class Delta(Command):
"""Creates delta PiSi packages
Usage: delta oldpackage newpackage
Delta command finds the changed files between the given packages by comparing the sha1sum of the files
and creates a delta pisi package with the changed files between two releases.
"""
__metaclass__ = autocommand
def __init__(self, args):
super(Delta, self).__init__(args)
name = ("delta", "dt")
def options(self):
group = OptionGroup(self.parser, _("delta options"))
self.add_options(group)
self.parser.add_option_group(group)
def add_options(self, group):
group.add_option("-O", "--output-dir", action="store", default=None,
help=_("Output directory for produced packages"))
def run(self):
from pisi.delta import create_delta_package
self.init(database=False, write=False)
if len(self.args) is not 2:
self.help()
return
if ctx.get_option('output_dir'):
ctx.ui.info(_('Output directory: %s') % ctx.config.options.output_dir)
else:
ctx.ui.info(_('Outputting packages in the working directory.'))
ctx.config.options.output_dir = '.'
oldpackage = self.args[0]
newpackage = self.args[1]
create_delta_package(oldpackage, newpackage)
self.finalize()
class Emerge(Build):
"""Build and install PiSi source packages from repository
pisi/delta.py
+132
View File
@@ -0,0 +1,132 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2007, TUBITAK/UEKAE
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.
import os
from sets import Set as set
import gettext
__trans = gettext.translation("pisi", fallback=True)
_ = __trans.ugettext
import pisi.context as ctx
from pisi.package import Package
import pisi.util as util
import pisi.archive as archive
def create_delta_package(old_package, new_package):
if old_package == new_package:
ctx.ui.error(_("Cannot create delta for same package!"))
return
oldpkg = Package(old_package, "r")
newpkg = Package(new_package, "r")
newmd = newpkg.get_metadata()
oldmd = oldpkg.get_metadata()
oldfiles = oldpkg.get_files()
newfiles = newpkg.get_files()
files_delta = find_delta(oldfiles, newfiles)
ctx.ui.info(_("Creating delta PiSi package between %s %s") % (old_package, new_package))
# Unpack new package to temp
newpkg_name = util.package_name(newmd.package.name, newmd.package.version, newmd.package.release, newmd.package.build, False)
newpkg_path = util.join_path(ctx.config.tmp_dir(), newpkg_name)
newpkg.extract_to(newpkg_path, True)
tar = archive.ArchiveTar(util.join_path(newpkg_path, ctx.const.install_tar_lzma), "tarlzma", False, False)
tar.unpack_dir(newpkg_path)
# Create delta package
deltaname = "%s-%s-%s%s" % (oldmd.package.name, oldmd.package.build, newmd.package.build, ctx.const.delta_package_suffix)
outdir = ctx.get_option("output_dir")
if outdir:
deltaname = util.join_path(outdir, deltaname)
deltapkg = Package(deltaname, "w")
c = os.getcwd()
os.chdir(newpkg_path)
# add comar files to package
for pcomar in newmd.package.providesComar:
fname = util.join_path(ctx.const.comar_dir, pcomar.script)
deltapkg.add_to_package(fname)
# add xmls and files
deltapkg.add_to_package(ctx.const.metadata_xml)
deltapkg.add_to_package(ctx.const.files_xml)
# only metadata information may change in a package, so no install.tar.lzma added to delta package
if files_delta:
ctx.build_leftover = util.join_path(ctx.config.tmp_dir(), ctx.const.install_tar_lzma)
tar = archive.ArchiveTar(util.join_path(ctx.config.tmp_dir(), ctx.const.install_tar_lzma), "tarlzma")
for file in files_delta:
tar.add_to_archive(file.path)
tar.close()
os.chdir(ctx.config.tmp_dir())
deltapkg.add_to_package(ctx.const.install_tar_lzma)
deltapkg.close()
tmp_file = util.join_path(ctx.config.tmp_dir(), ctx.const.install_tar_lzma)
if os.path.exists(tmp_file):
os.unlink(tmp_file)
ctx.build_leftover = None
os.chdir(c)
ctx.ui.info(_("Done."))
# Hash not equals (these are the deltas)
# Hash equal but path different ones (these are the relocations)
# Hash and also path equal ones (do nothing)
def find_delta(oldfiles, newfiles):
hashto_files = {}
for file in newfiles.list:
hashto_files.setdefault(file.hash, []).append(file)
files_new = set(map(lambda x:x.hash, newfiles.list))
files_old = set(map(lambda x:x.hash, oldfiles.list))
files_delta = files_new - files_old
deltas = []
for hash in files_delta:
deltas.extend(hashto_files[hash])
return deltas
def find_relocations(oldfiles, newfiles):
files_new = {}
for file in newfiles.list:
files_new.setdefault(file.hash, []).append(file)
files_old = {}
for file in oldfiles.list:
files_old.setdefault(file.hash, []).append(file)
relocations = []
for hash in files_new.keys():
if hash and hash in files_old:
for i in range(len(files_new[hash])):
if files_old[hash][0].path != files_new[hash][i].path:
relocations.append((files_old[hash][0], files_new[hash][i]))
return relocations
pisi/index.py
+20 -3
View File
@@ -88,10 +88,16 @@ class Index(XmlFile):
self.repo_dir = repo_uri
packages = []
deltas = {}
for root, dirs, files in os.walk(repo_uri):
for fn in files:
if fn.endswith(ctx.const.package_suffix):
if fn.endswith(ctx.const.delta_package_suffix):
name, version = util.parse_package_name(fn)
deltas.setdefault(name, []).append(os.path.join(root, fn))
elif fn.endswith(ctx.const.package_suffix):
packages.append(os.path.join(root, fn))
if fn == 'component.xml':
ctx.ui.info(_('Adding %s to component index') % fn)
self.add_component(os.path.join(root, fn))
@@ -106,7 +112,7 @@ class Index(XmlFile):
pkg_name = util.parse_package_name(os.path.basename(pkg))[0]
if pkg_name not in obsoletes_list:
ctx.ui.info(_('Adding %s to package index') % pkg)
self.add_package(pkg, repo_uri)
self.add_package(pkg, deltas, repo_uri)
def update_db(self, repo, txn = None):
# FIXME: updating db takes too much time. So a notify mechanism is used to inform the status
@@ -133,7 +139,7 @@ class Index(XmlFile):
ctx.sourcedb.add_spec(sf, repo, txn=txn)
update_progress()
def add_package(self, path, repo_uri):
def add_package(self, path, deltas, repo_uri):
package = Package(path, 'r')
md = package.get_metadata()
md.package.packageSize = os.path.getsize(path)
@@ -153,6 +159,17 @@ class Index(XmlFile):
# No need to carry these with index (#3965)
md.package.files = None
md.package.additionalFiles = None
if md.package.name in deltas:
for delta_path in deltas[md.package.name]:
delta = metadata.Delta()
delta.packageURI = util.removepathprefix(repo_uri, delta_path)
delta.packageSize = os.path.getsize(delta_path)
delta.packageHash = util.sha1_file(delta_path)
name, buildFrom, buildTo = util.parse_delta_package_name(delta_path)
delta.buildFrom = buildFrom
md.package.deltaPackages.append(delta)
self.packages.append(md.package)
def add_component(self, path):
pisi/metadata.py
+16
View File
@@ -27,6 +27,14 @@ import pisi.pxml.xmlfile as xmlfile
import pisi.pxml.autoxml as autoxml
import pisi.util as util
class Delta:
__metaclass__ = autoxml.autoxml
t_PackageURI = [ autoxml.String, autoxml.optional]
t_PackageSize = [ autoxml.Long, autoxml.optional]
t_PackageHash = [ autoxml.String, autoxml.optional, "SHA1Sum" ]
a_buildFrom = [autoxml.String, autoxml.optional]
class Source:
__metaclass__ = autoxml.autoxml
@@ -45,10 +53,18 @@ class Package(specfile.Package):
t_PackageSize = [ autoxml.Long, autoxml.optional]
t_PackageHash = [ autoxml.String, autoxml.optional, "SHA1Sum" ]
t_PackageURI = [ autoxml.String, autoxml.optional]
t_DeltaPackages = [ [Delta], autoxml.optional]
t_PackageFormat = [ autoxml.String, autoxml.optional]
t_Source = [ Source, autoxml.optional]
def get_delta(self, buildFrom):
for delta in self.deltaPackages:
if delta.buildFrom == str(buildFrom):
return delta
else:
return None
def decode_hook(self, node, errs, where):
self.version = self.history[0].version
self.release = self.history[0].release
pisi/operations.py
+17 -5
View File
@@ -799,15 +799,27 @@ def calculate_download_sizes(order):
for pkg in [ctx.packagedb.get_package(name) for name in order]:
# get the cached package's path
fn = util.package_name(pkg.name, pkg.version, pkg.release, pkg.build, True)
delta = None
if ctx.installdb.is_installed(pkg.name):
(version, release, build) = ctx.installdb.get_version(pkg.name)
delta = pkg.get_delta(buildFrom=build)
if delta:
fn = os.path.basename(delta.packageURI)
pkg_hash = delta.packageHash
pkg_size = delta.packageSize
else:
fn = os.path.basename(pkg.packageURI)
pkg_hash = pkg.packageHash
pkg_size = pkg.packageSize
path = util.join_path(ctx.config.packages_dir(), fn)
# check the file and sha1sum to be sure it _is_ the cached package
if os.path.exists(path) and util.sha1_file(path) == pkg.packageHash:
cached_size += pkg.packageSize
if os.path.exists(path) and util.sha1_file(path) == pkg_hash:
cached_size += pkg_size
total_size += pkg.packageSize
total_size += pkg_size
ctx.ui.notify(ui.cached, total=total_size, cached=cached_size)
return total_size, cached_size
pisi/package.py
+5
View File
@@ -98,6 +98,11 @@ class Package:
this is the function used by the installer"""
self.impl.unpack_dir_flat(dir, outdir)
def extract_to(self, outdir, clean_dir = False):
"""Extracts contents of the archive to outdir. Before extracting if clean_dir
is set, outdir is deleted with its contents"""
self.impl.unpack(outdir, clean_dir)
def extract_pisi_files(self, outdir):
"""Extract PiSi control files: metadata.xml, files.xml,
action scripts, etc."""
pisi/util.py
+12
View File
@@ -644,6 +644,18 @@ def parse_package_name(package_name):
return (name, version)
def parse_delta_package_name(package_name):
"""Separate delta package name and release infos
example: tasma-5-7.pisi.delta -> (tasma, 5, 7)
"""
name, build = parse_package_name(package_name)
build = build[:-len(ctx.const.delta_package_suffix)]
buildFrom, buildTo = build.split("-")
return name, buildFrom, buildTo
def filter_latest_packages(package_paths):
""" For a given pisi package paths list where there may also be multiple versions
of the same package, filters only the latest versioned ones """