diff options
Diffstat (limited to 'scripts/reporead.py')
| -rwxr-xr-x | scripts/reporead.py | 327 |
1 files changed, 327 insertions, 0 deletions
diff --git a/scripts/reporead.py b/scripts/reporead.py new file mode 100755 index 00000000..bbf0251a --- /dev/null +++ b/scripts/reporead.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python +""" +reporead.py + +Parses a repo.db.tar.gz file and updates the Arch database with the relevant +changes. + +Usage: reporead.py ARCH PATH + ARCH: architecture to update, and can be one of: i686, x86_64 + PATH: full path to the repo.db.tar.gz file. + +Example: + reporead.py i686 /tmp/core.db.tar.gz + +""" + +### +### User Variables +### + +# multi value blocks +REPOVARS = ['arch', 'backup', 'builddate', 'conflicts', 'csize', + 'deltas', 'depends', 'desc', 'filename', 'files', 'force', + 'groups', 'installdate', 'isize', 'license', 'md5sum', + 'name', 'optdepends', 'packager', 'provides', 'reason', + 'replaces', 'size', 'url', 'version'] + +### +### Imports +### + +import os +import re +import sys +import gzip +import tarfile +import logging +from datetime import datetime +from django.core.management import setup_environ +# mung the sys path to get to django root dir, no matter +# where we are called from +archweb_app_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +os.chdir(archweb_app_path) +sys.path[0] = archweb_app_path +import settings +setup_environ(settings) +from pprint import pprint as pp +from cStringIO import StringIO +from logging import CRITICAL,ERROR,WARNING,INFO,DEBUG +from main.models import Package, PackageFile, PackageDepend + +### +### Initialization +### + +logging.basicConfig( + level=WARNING, + format='%(asctime)s -> %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + stream=sys.stderr) +logger = logging.getLogger() + + +### +### function and class definitions +### + +class Pkg(object): + """An interim 'container' object for holding Arch package data.""" + + def __init__(self, val): + selfdict = {} + squash = ['arch', 'builddate', 'csize', 'desc', 'filename', + 'installdate', 'isize', 'license', 'md5sum', + 'packager', 'size', 'url'] + + selfdict['name'] = val['name'][0] + del val['name'] + if 'url' not in val: + val['url'] = '' + for x in val.keys(): + if x in squash: + if len(val[x]) == 0: + logger.warning("Package %s has no %s" % (selfdict['name'],x)) + selfdict[x] = ''.join(val[x]) + elif x == 'force': + selfdict[x] = True + elif x == 'version': + version = val[x][0].rsplit('-') + selfdict['ver'] = version[0] + selfdict['rel'] = version[1] + elif x == 'reason': + selfdict[x] = int(val[x][0]) + else: + selfdict[x] = val[x] + self.__dict__ = selfdict + + def __getattr__(self,name): + if name == 'force': + return False + else: + return None + + +def usage(): + """Print the usage of this application.""" + print __doc__.strip() + + +def fetchiter_dict(cursor): + """ + Given a DB API 2.0 cursor object that has been executed, returns a + dictionary that maps each field to a column index + """ + rows = cursor.fetchmany(size=30) + while rows: + for row in rows: + #pp(rows) + #for row in rows: + yield dictize(cursor,row) + rows = cursor.fetchmany(size=30) + + +def fetchone_dict(cursor): + """ + Given a DB API 2.0 cursor object that has been executed, returns a + dictionary that maps each field to a column index + """ + results = {} + row = cursor.fetchone() + return dictize(cursor,row) + + +def dictize(cursor,row): + result = {} + for column,desc in enumerate(cursor.description): + result[desc[0]] = row[column] + return result + + +def db_update(archname, pkgs): + """ + Parses a list and updates the Arch dev database accordingly. + + Arguments: + pkgs -- A list of Pkg objects. + + """ + logger.debug('Starting database updates.') + repository = Package.REPOS[pkgs[0].repo.lower()] + architecture = Package.ARCHES[archname.lower()] + dbpkgs = Package.objects.filter(arch=architecture, repo=repository) + now = datetime.now() + + # go go set theory! + # thank you python for having a set class <3 + dbset = set([pkg.pkgname for pkg in dbpkgs]) + syncset = set([pkg.name for pkg in pkgs]) + + # packages in syncdb and not in database (add to database) + in_sync_not_db = syncset - dbset + for p in [x for x in pkgs if x.name in in_sync_not_db]: + logger.info("Adding package %s to database", p.name) + ## note: maintainer is being set to orphan for now + ## maybe later we can add logic to match pkgbuild maintainers + ## to db maintainer ids + pkg = Package( + repo = repository, arch=architecture, maintainer_id = 0, + needupdate = False, url = p.url, last_update = now, + pkgname = p.name, pkgver = p.ver, pkgrel = p.rel, + pkgdesc = p.desc) + pkg.save() + # files are not in the repo.db.tar.gz + #for x in p.files: + # pkg.packagefile_set.create(path=x) + if 'depends' in p.__dict__: + for y in p.depends: + # make sure we aren't adding self depends.. + # yes *sigh* i have seen them in pkgbuilds + dpname,dpvcmp = re.match(r"([a-z0-9-]+)(.*)", y).groups() + if dpname == p.name: + logger.warning('Package %s has a depend on itself' % p.name) + continue + pkg.packagedepend_set.create(depname=dpname, depvcmp=dpvcmp) + logger.debug('Added %s as dep for pkg %s' % (dpname,p.name)) + + # packages in database and not in syncdb (remove from database) + in_db_not_sync = dbset - syncset + for p in in_db_not_sync: + logger.info("Removing package %s from database", p) + Package.objects.get( + pkgname=p, arch=architecture, repo=repository).delete() + + # packages in both database and in syncdb (update in database) + pkg_in_both = syncset & dbset + for p in [x for x in pkgs if x.name in pkg_in_both]: + dbp = dbpkgs.get(pkgname=p.name) + if ''.join((p.ver,p.rel)) == ''.join((dbp.pkgver,dbp.pkgrel)): + continue + logger.info("Updating package %s in database", p.name) + pkg = Package.objects.get( + pkgname=p.name,arch=architecture, repo=repository) + pkg.pkgver = p.ver + pkg.pkgrel = p.rel + pkg.pkgdesc = p.desc + pkg.needupdate = False + pkg.last_update = now + pkg.save() + + # files are not in the repo.db.tar.gz + #pkg.packagefile_set.all().delete() + #for x in p.files: + # pkg.packagefile_set.create(path=x) + pkg.packagedepend_set.all().delete() + if 'depends' in p.__dict__: + for y in p.depends: + dpname,dpvcmp = re.match(r"([a-z0-9-]+)(.*)", y).groups() + pkg.packagedepend_set.create(depname=dpname, depvcmp=dpvcmp) + + +def parse_inf(iofile): + """ + Parses an Arch repo db information file, and returns variables as a list. + + Arguments: + iofile -- A StringIO, FileType, or other object with readlines method. + + """ + store = {} + lines = iofile.readlines() + blockname = None + max = len(lines) + i = 0 + while i < max: + line = lines[i].strip() + if len(line) > 0 and line[0] == '%' and line[1:-1].lower() in REPOVARS: + blockname = line[1:-1].lower() + logger.debug("Parsing package block %s",blockname) + store[blockname] = [] + i += 1 + while i < max and len(lines[i].strip()) > 0: + store[blockname].append(lines[i].strip()) + i += 1 + # here is where i would convert arrays to strings + # based on count and type, but i dont think it is needed now + i += 1 + return store + + +def parse_repo(repopath): + """ + Parses an Arch repo db file, and returns a list of Pkg objects. + + Arguments: + repopath -- The path of a repository db file. + + """ + logger.debug("Starting repo parsing") + if not os.path.exists(repopath): + logger.error("Could not read file %s", repopath) + + logger.debug("Reading repo tarfile") + filename = os.path.split(repopath)[1] + rindex = filename.rindex('.db.tar.gz') + reponame = filename[:rindex] + + repodb = tarfile.open(repopath,"r:gz") + ## assuming well formed tar, with dir first then files after + ## repo-add enforces this + logger.debug("Starting package parsing") + pkgs = [] + tpkg = None + while True: + tarinfo = repodb.next() + if tarinfo == None or tarinfo.isdir(): + if tpkg != None: + tpkg.reset() + data = parse_inf(tpkg) + p = Pkg(data) + p.repo = reponame + logger.debug("Done parsing package %s", p.name) + pkgs.append(p) + if tarinfo == None: + break + # set new tpkg + tpkg = StringIO() + if tarinfo.isreg(): + if os.path.split(tarinfo.name)[1] in ('desc','depends'): + tpkg.write(repodb.extractfile(tarinfo).read()) + tpkg.write('\n') # just in case + repodb.close() + logger.debug("Finished repo parsing") + return pkgs + + +def main(argv=None): + """ + Parses repo.db.tar.gz file and returns exit status. + + Keyword Arguments: + argv -- A list/array simulating a sys.argv (default None) + If left empty, sys.argv is used + + """ + if argv == None: + argv = sys.argv + if len(argv) != 3: + usage() + return 0 + # check if arch is valid + if argv[1] not in Package.ARCHES: + usage() + return 0 + repo_file = os.path.normpath(argv[2]) + packages = parse_repo(repo_file) + db_update(argv[1], packages) + return 0 + + +### +### Main eval +### + +if __name__ == '__main__': + logger.level = WARNING + sys.exit(main()) + |