From d7055cb33e9a39f16729f2e0bd669bca551d4a22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrer=20=C3=96zen?= Date: Sun, 9 Oct 2005 17:39:14 +0000 Subject: [PATCH] =?UTF-8?q?Evet,=20yeni=20repostats=20haz=C4=B1r.=20Kullan?= =?UTF-8?q?=C4=B1m=20i=C3=A7in:?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tools/repostats.py ~/pardus/pardus-devel Önce bulduğu hataları (varsa, ki var!) sonra da genel istatistikleri ekrana basacak. tools/repostats.py ~/pardus/pardus-devel packager "Gürer Özen" tools/repostats.py ~/pardus/pardus-devel package k3b gibi iki özel kullanımı daha var. Bir de tools/repostats.py ~/pardus/pardus-devel html ile paksite adlı bir dizin açıp, içine paketler, paketçiler vs herbiri için bilgilerin olduğu bir site oluşturuyor. firefox paksite/index.html Eğer bulunduğunuz dizinde programın başındaki def_repo_html gibi templatelere karşılık repo.html gibi dosyalar varsa, onlara göre sayfalar oluşturuluyor. Eksikleri var, daha gelişecek. --- tools/repostats.py | 626 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 504 insertions(+), 122 deletions(-) diff --git a/tools/repostats.py b/tools/repostats.py index c464562b..7b8be51d 100755 --- a/tools/repostats.py +++ b/tools/repostats.py @@ -11,6 +11,7 @@ import sys import os +import codecs import gettext __trans = gettext.translation('pisi', fallback=True) @@ -20,8 +21,134 @@ sys.path.append('.') import pisi.specfile import pisi.uri -def echo(string): - print string.encode("utf8") +# default html templates + +def_table_html = u""" +%s%s +""" + +def_repo_html = u""" + + Depo istatistikleri + + + +

+Depoda toplam %(nr_source)d kaynak paket, ve bu paketlerden oluşturulacak +%(nr_packages)d ikili paket bulunmaktadır. Toplam %(nr_patches)d yama mevcuttur. +

+ +

+%(errors)s +

+ +

En fazla yamalanmış 5 kaynak paket:

+%(most_patched)s +

+ +

Eksik paketler

+%(missing)s +

+ +

Paketçiler:

+%(packagers)s +

+ + + +""" + +def_packager_html = u""" + + Paketçi %(name)s + + + +

%(name)s <%(email)s>

+

Paketler:

+

%(sources)s

+ + +""" + +def_package_html = u""" + + İkili paket %(name)s + + + +

İkili paket: %(name)s

+

Versiyon %(version)s, sürüm %(release)s

+ +

Derlemek için bağımlılar:

+

%(buildDeps)s

+ +

Çalıştırmak için bağımlılar:

+

%(runtimeDeps)s

+ +

Pakete derlenmek için bağımlılar:

+

%(revBuildDeps)s

+ +

Pakete çalışmak için bağımlılar:

+

%(revRuntimeDeps)s

+ + +""" + +def_source_html = u""" + + Kaynak paket %(name)s + + + +

Kaynak paket: %(name)s

+

Versiyon %(version)s, sürüm %(release)s

+ +

Bu kaynaktan derlenen ikili paketler:

+

%(packages)s

+ + +""" + +def_missing_html = u""" + + Eksik ikili paket %(name)s + + + +

Eksik ikili paket: %(name)s

+ +

Pakete derlenmek için bağımlılar:

+

%(revBuildDeps)s

+ +

Pakete çalışmak için bağımlılar:

+

%(revRuntimeDeps)s

+ + +""" + + +# FIXME: This check should be in specfile +valid_filetypes = [ + "executable", + "library", + "data", + "config", + "doc", + "man", + "info", + "localedata", + "header", + "all" +] + +def printu(obj): + if isinstance(obj, unicode): + print obj.encode('utf-8') + elif isinstance(obj, str): + print obj.decode('utf-8') + else: + print str(obj) def valuesort(x, y): if x[1] > y[1]: @@ -31,11 +158,41 @@ def valuesort(x, y): else: return 1 +def find_pspecs(folder): + paks = [] + for root, dirs, files in os.walk(folder): + if "pspec.xml" in files: + paks.append(root) + # dont walk into the versioned stuff + if ".svn" in dirs: + dirs.remove(".svn") + return paks + +def template_get(tmpl_name): + try: + f = codecs.open(tmpl_name + ".html", "r", "utf-8") + data = f.read() + f.close() + return data + except: + return globals()["def_" + tmpl_name + "_html"] + +def template_write(filename, tmpl_name, dict): + f = codecs.open(filename, "w", "utf-8") + f.write(template_get(tmpl_name) % dict) + f.close() + +def template_table(tmpl_name, list): + tmpl = template_get(tmpl_name) + data = "" + for item in list: + data += (tmpl % item) + return data + class Histogram: - def __init__(self, title): + def __init__(self): self.list = {} - self.title = title def add(self, name, value=None): if value: @@ -46,138 +203,363 @@ class Histogram: else: self.list[name] = 1 - def get_list(self): + def get_list(self, max=0): items = self.list.items() items.sort(valuesort) - return items - - def html_out(self, max=0): - echo("

%s

" % self.title) - list = self.get_list() if max != 0: - list = list[:max] - for name,cnt in list: - echo("" % (name, cnt)) - echo("
%s%s
") + return items[:max] + else: + return items -def scan_pspec(folder): - paks = [] - for root, dirs, files in os.walk(folder): - if "pspec.xml" in files: - paks.append(root) - # dont walk into the versioned stuff - if ".svn" in dirs: - dirs.remove(".svn") - return paks +# Dictionary of all source packages keyed by the source name +sources = {} -def add_deps(deps, spec): - a = {} - for d in spec.source.buildDeps: - p = d.package - if not a.has_key(p): - a[p] = 1 - for pak in spec.packages: - for d in pak.runtimeDeps: - p = d.package - if not a.has_key(p): - a[p] = 1 - for p in a: - deps.add(p) +# Dictionary of all binary packages keyed by the package name +packages = {} -hosts = Histogram(_("Source hosts")) -people = Histogram(_("Packagers")) -licenses = Histogram(_("Licenses")) -categories = Histogram(_("Categories")) -components = Histogram(_("Components")) -dependencies = Histogram(_("Dependencies")) -releases = Histogram(_("Releases")) -types = Histogram(_("File types")) -mostpatched = Histogram(_("Top five most patched source")) -longpy = Histogram(_("Top five longest action.py scripts")) -nr_binpaks = 0 -nr_patches = 0 -paknames = {} +# Dictionary of all packagers keyed by the packager name +packagers = {} +# Dictionary of missing depended binary packages keyed by the package name +missing = {} + +# List of all repository problems errors = [] -paks = scan_pspec(sys.argv[1]) -for pak in paks: - spec = pisi.specfile.SpecFile() - try: - spec.read(os.path.join(pak, "pspec.xml")) - except Exception, inst: - errors.append([pak, str(inst)]) - continue - try: - f = file(os.path.join(pak, "actions.py")) - L = len(f.readlines()) - longpy.add(spec.source.name, L) - f.close() - except: - pass - nr_binpaks += len(spec.packages) - nr_patches += len(spec.source.patches) - mostpatched.add(spec.source.name, len(spec.source.patches)) - releases.add(str(spec.source.release)) - for p in spec.packages: - if p.partof: - components.add(p.partof) - for x in p.isa: - categories.add(x) - for x in p.paths: - types.add(x.fileType) - paknames[spec.source.name] = 1 - add_deps(dependencies, spec) - name = spec.source.packager.name - lices = spec.source.license - people.add(name) - for lice in lices: - licenses.add(lice) - host = pisi.uri.URI(spec.source.archiveUri).location() - if host == "": - print pisi.uri.URI(spec.source.archiveUri) - hosts.add(host) -print "%s İstatistikleri" % (sys.argv[1]) -print '' -echo("") +class Missing: + def __init__(self, name): + missing[name] = self + self.name = name + self.revBuildDeps = [] + self.revRuntimeDeps = [] + + def report_html(self): + bDeps = map(lambda x: "%s" % (x, x), self.revBuildDeps) + rDeps = map(lambda x: "%s" % (x, x), self.revRuntimeDeps) + dict = { + "name": self.name, + "revBuildDeps": ", ".join(bDeps), + "revRuntimeDeps": ", ".join(rDeps) + } + template_write("paksite/package-%s.html" % self.name, "missing", dict) -print "

Toplam %d kod paketi, ve bunlardan oluşturulacak %d ikili paket var.

" % (len(paks), nr_binpaks) -echo(u"

Toplam peç sayısı %d

" % nr_patches) -mostpatched.html_out(5) -longpy.html_out(5) +class Package: + def __init__(self, source, pakspec): + name = pakspec.name + if packages.has_key(name): + errors.append(_("Duplicate binary packages:\n%s\n%s\n") % ( + source.name, packages[name].source.name)) + return + for p in pakspec.paths: + if p.fileType not in valid_filetypes: + e = _("Unknown file type '%s' in package '%s'") % ( + p.fileType, source.name) + errors.append(e) + packages[name] = self + self.name = name + self.source = source + self.pakspec = pakspec + self.revBuildDeps = [] + self.revRuntimeDeps = [] + + def markDeps(self): + # mark reverse build dependencies + for d in self.source.spec.source.buildDeps: + p = d.package + if packages.has_key(p): + packages[p].revBuildDeps.append(self.name) + else: + if not missing.has_key(p): + Missing(p) + missing[p].revBuildDeps.append(self.name) + # mark reverse runtime dependencies + for d in self.pakspec.runtimeDeps: + p = d.package + if packages.has_key(p): + packages[p].revRuntimeDeps.append(self.name) + else: + if not missing.has_key(p): + Missing(p) + missing[p].revRuntimeDeps.append(self.name) + + def report(self): + source = self.source.spec.source + printu(_("Binary package: %s") % self.name) + printu(_(" Version %s release %s") % (source.version, source.release)) + printu(_(" Packager: %s <%s>") % ( + source.packager.name, source.packager.email)) + printu(_(" Build dependencies:")) + for d in source.buildDeps: + printu(" %s" % d.package) + printu(_(" Runtime dependencies:")) + for d in self.pakspec.runtimeDeps: + printu(" %s" % d.package) + if self.revBuildDeps: + printu(_(" Reverse build dependencies:")) + for d in self.revBuildDeps: + printu(" %s" % d) + if self.revRuntimeDeps: + printu(_(" Reverse runtime dependencies:")) + for d in self.revRuntimeDeps: + printu(" %s" % d) + + def report_html(self): + source = self.source.spec.source + bDeps = map(lambda x: "%s" % (x, x), + (map(lambda x: x.package, source.buildDeps))) + rDeps = map(lambda x: "%s" % (x, x), + (map(lambda x: x.package, self.pakspec.runtimeDeps))) + rbDeps = map(lambda x: "%s" % (x, x), self.revBuildDeps) + rrDeps = map(lambda x: "%s" % (x, x), self.revRuntimeDeps) + dict = { + "name": self.name, + "version": source.version, + "release": source.release, + "buildDeps": ", ".join(bDeps), + "runtimeDeps": ", ".join(rDeps), + "revBuildDeps": ", ".join(rbDeps), + "revRuntimeDeps": ", ".join(rrDeps) + } + template_write("paksite/package-%s.html" % self.name, "package", dict) -if errors != []: - print "

Hatalar

" + +class Source: + def __init__(self, path, spec): + name = spec.source.name + if sources.has_key(name): + errors.append(_("Duplicate source packages:\n%s\n%s\n") % ( + path, sources[name].path)) + return + sources[name] = self + self.spec = spec + self.name = name + self.path = path + for p in spec.packages: + Package(self, p) + self.checkRelease() + + def checkRelease(self): + # FIXME: this check also belongs to specfile + prev = None + for h in self.spec.source.history: + if prev: + prev -= 1 + if prev <= 0: + e = _("Source package '%s' has wrong release numbers") % self.name + errors.append(e) + return + if int(h.release) != prev: + e = _("Source package '%s' lacks release %d") % (self.name, prev) + errors.append(e) + return + else: + prev = int(h.release) + if prev != 1: + e = _("Source package '%s' has no first release") % self.name + errors.append(e) + + def report_html(self): + source = self.spec.source + paks = map(lambda x: "%s" % (x, x), + (map(lambda x: x.name, self.spec.packages))) + dict = { + "name": self.name, + "version": source.version, + "release": source.release, + "packages": ", ".join(paks) + } + template_write("paksite/source-%s.html" % self.name, "source", dict) + + +class Packager: + def __init__(self, spec): + name = spec.source.packager.name + email = spec.source.packager.email + if packagers.has_key(name): + if email != packagers[name].email: + e = _("Developer '%s <%s>' has another mail address '%s' in source package '%s'") % ( + name, packagers[name].email, email, spec.source.name) + packagers[name].errors.append(e) + errors.append(e) + packagers[name].sources.append(spec.source.name) + else: + packagers[name] = self + self.name = name + self.email = email + self.sources = [spec.source.name] + self.errors = [] + + def report(self): + printu(_("Packager: %s <%s>") % (self.name, self.email)) + if self.errors: + printu(_(" Errors:")) + for e in self.errors: + printu(" %s" % e) + printu(_(" Source packages (%d):") % len(self.sources)) + for s in self.sources: + printu(" %s" % s) + + def report_html(self): + srcs = map(lambda x: "%s" % (x, x), self.sources) + dict = { + "name": self.name, + "email": self.email, + "sources": ", ".join(srcs) + } + template_write("paksite/%s.html" % self.name, "packager", dict) + + +class Repository: + def __init__(self, path): + self.path = path + self.nr_sources = 0 + self.nr_packages = 0 + self.nr_patches = 0 + self.people = Histogram() + self.licenses = Histogram() + self.mostpatched = Histogram() + self.longpy = Histogram() + self.cscripts = Histogram() + + def processPspec(self, path, spec): + # new classes + Packager(spec) + Source(path, spec) + # update global stats + self.nr_sources += 1 + self.nr_packages += len(spec.packages) + self.nr_patches += len(spec.source.patches) + # update top fives + self.people.add(spec.source.packager.name) + for p in spec.packages: + for cs in p.providesComar: + self.cscripts.add(cs.om) + for L in spec.source.license: + self.licenses.add(L) + self.mostpatched.add(spec.source.name, len(spec.source.patches)) + try: + f = file(os.path.join(path, "actions.py")) + L = len(f.readlines()) + self.longpy.add(spec.source.name, L) + f.close() + except: + pass + + def scan(self): + for pak in find_pspecs(self.path): + spec = pisi.specfile.SpecFile() + try: + spec.read(os.path.join(pak, "pspec.xml")) + except Exception, inst: + errors.append(_("Cannot parse '%s':\n%s\n") % (pak, inst.args[0])) + continue + self.processPspec(pak, spec) + for p in packages.values(): + p.markDeps() + + def report(self): + printu(_("Repository Statistics:")) + # general stats + printu(_(" Total of %d source packages, and %d binary packages.") % ( + self.nr_sources, self.nr_packages)) + printu(_(" There are %d patches applied.") % self.nr_patches) + # problems + if missing: + printu(_(" Missing packages (%d):") % len(missing)) + for m in missing.values(): + printu(" %s" % m.name) + # trivia + printu(_(" Top five most patched source:")) + for p in self.mostpatched.get_list(5): + printu(" %4d %s" % (p[1], p[0])) + printu(_(" Top five longest action.py scripts:")) + for p in self.longpy.get_list(5): + printu(" %4d %s" % (p[1], p[0])) + # lists + printu(_(" COMAR scripts:")) + for cs in self.cscripts.get_list(): + printu(" %4d %s" % (cs[1], cs[0])) + people = self.people.get_list() + printu(_(" Packagers (%d):") % len(people)) + for p in people: + printu(" %4d %s" % (p[1], p[0])) + licenses = self.licenses.get_list() + printu(_(" Licenses (%d):") % len(licenses)) + for p in licenses: + printu(" %4d %s" % (p[1], p[0])) + + def report_html(self): + miss = map(lambda x: "" % (x, x), missing.keys()) + people = self.people.get_list() + upeople = {} + for p in people: + upeople["%s" % (p[0], p[0])] = p[1] + if errors: + e = "
".join(errors) + else: + e = "" + dict = { + "nr_source": self.nr_sources, + "nr_packages": self.nr_packages, + "nr_patches": self.nr_patches, + "most_patched": template_table("table", self.mostpatched.get_list(5)), + "packagers": template_table("table", upeople.items()), + "missing": "\n".join(miss), + "errors": e + } + template_write("paksite/index.html", "repo", dict) + + +# command line driver + +if len(sys.argv) < 2: + printu(_("Usage: repostats.py source-repo-path [ command [arg] ]")) + sys.exit(0) + +repo = Repository(sys.argv[1]) +printu(_("Scanning source repository...")) +repo.scan() + +if errors: + printu("***") + printu(_("Encountered %d errors! Fix them immediately!") % len(errors)) for e in errors: - print "" % (e[0], e[1]) - print "
%s
%s%s
" -else: - print "

Hata yok, tebrikler!

" + printu(e) + printu("***") -components.html_out() -categories.html_out() -people.html_out() -licenses.html_out() -releases.html_out() -types.html_out() +if len(sys.argv) > 2: + if sys.argv[2] == "packager": + if len(sys.argv) > 3: + packagers[unicode(sys.argv[3].decode('utf-8'))].report() + sys.exit(0) + else: + for p in packagers.values(): + printu(_("%s <%s>, %s source package") % (p.name, p.email, len(p.sources))) + sys.exit(0) + elif sys.argv[2] == "package": + if len(sys.argv) > 3: + packages[unicode(sys.argv[3].decode('utf-8'))].report() + sys.exit(0) + else: + printu(_("Which package?")) + sys.exit(0) + elif sys.argv[2] == "html": + try: + os.mkdir("paksite") + except: + pass + repo.report_html() + for p in packagers.values(): + p.report_html() + for p in missing.values(): + p.report_html() + for p in packages.values(): + p.report_html() + for p in sources.values(): + p.report_html() + sys.exit(0) -echo(_("

Dependencies
")) -echo(_("package name with italics are not available in the repository

")) -print "" -notavail = 0 -for name,cnt in dependencies.get_list(): - if paknames.has_key(name): - print "" % (name, cnt) - else: - notavail += 1 - print "" % (name, cnt) -print "
%s%d
%s%d
" - -print "

%d packages are not available in repository.

" % notavail - -hosts.html_out() - -print "" +repo.report()