diff options
author | Luke Shumaker <LukeShu@sbcglobal.net> | 2013-04-21 02:22:44 -0400 |
---|---|---|
committer | Luke Shumaker <LukeShu@sbcglobal.net> | 2013-04-21 02:22:44 -0400 |
commit | 03fa7e4f27bdb39a8f8f5ed91a87d18bf8357b47 (patch) | |
tree | c67eafcbda55706f18400b3115a2b8a5be318394 /devel/management/commands/reporead.py | |
parent | 91c451821ce7000cbc268cec8427d208a6cedd7e (diff) | |
parent | b8ee7b1ee281b45b245fb454228b8ad847c56200 (diff) |
Merge branch 'archweb' into archweb-generic2
Conflicts:
devel/views.py
feeds.py
public/views.py
settings.py
sitestatic/archweb.js
templates/base.html
templates/devel/profile.html
templates/mirrors/status.html
templates/news/view.html
templates/packages/flaghelp.html
templates/packages/opensearch.xml
templates/public/download.html
templates/public/feeds.html
templates/public/index.html
templates/registration/login.html
templates/releng/results.html
templates/todolists/public_list.html
Diffstat (limited to 'devel/management/commands/reporead.py')
-rw-r--r-- | devel/management/commands/reporead.py | 102 |
1 files changed, 64 insertions, 38 deletions
diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index cf98f004..1e456c8c 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -14,6 +14,7 @@ Example: """ from collections import defaultdict +from copy import copy import io import os import re @@ -27,11 +28,12 @@ from pytz import utc from django.core.management.base import BaseCommand, CommandError from django.db import connections, router, transaction from django.db.utils import IntegrityError +from django.utils.timezone import now from devel.utils import UserFinder -from main.models import Arch, Package, PackageDepend, PackageFile, Repo -from main.utils import utc_now -from packages.models import Conflict, Provision, Replacement +from main.models import Arch, Package, PackageFile, Repo +from packages.models import Depend, Conflict, Provision, Replacement, Update +from packages.utils import parse_version logging.basicConfig( @@ -78,10 +80,9 @@ class RepoPackage(object): bare = ( 'name', 'base', 'arch', 'filename', 'md5sum', 'sha256sum', 'url', 'packager' ) number = ( 'csize', 'isize' ) - collections = ( 'depends', 'optdepends', 'conflicts', - 'provides', 'replaces', 'groups', 'license', 'files' ) - - version_re = re.compile(r'^((\d+):)?(.+)-([^-]+)$') + collections = ( 'depends', 'optdepends', 'makedepends', 'checkdepends', + 'conflicts', 'provides', 'replaces', 'groups', 'license', + 'files' ) def __init__(self, repo): self.repo = repo @@ -111,11 +112,7 @@ class RepoPackage(object): v[0] = 'missing' setattr(self, k, v[0]) elif k == 'version': - match = self.version_re.match(v[0]) - self.ver = match.group(3) - self.rel = match.group(4) - if match.group(2): - self.epoch = int(match.group(2)) + self.ver, self.rel, self.epoch = parse_version(v[0]) elif k == 'builddate': try: builddate = datetime.utcfromtimestamp(int(v[0])) @@ -143,19 +140,21 @@ class RepoPackage(object): return u'%s-%s' % (self.ver, self.rel) -DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.*))?$") +DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.+))?$") -def create_depend(package, dep_str, optional=False): - depend = PackageDepend(pkg=package, optional=optional) +def create_depend(package, dep_str, deptype='D'): + depend = Depend(pkg=package, deptype=deptype) # lop off any description first parts = dep_str.split(':', 1) if len(parts) > 1: depend.description = parts[1].strip() match = DEPEND_RE.match(parts[0].strip()) if match: - depend.depname = match.group(1) - if match.group(2): - depend.depvcmp = match.group(2) + depend.name = match.group(1) + if match.group(3): + depend.comparison = match.group(3) + if match.group(4): + depend.version = match.group(4) else: logger.warning('Package %s had unparsable depend string %s', package.pkgname, dep_str) @@ -183,6 +182,7 @@ def create_related(model, package, rel_str, equals_only=False): return None return related + def create_multivalued(dbpkg, repopkg, db_attr, repo_attr): '''Populate the simplest of multivalued attributes. These are those that only deal with a 'name' attribute, such as licenses, groups, etc. The input @@ -236,8 +236,10 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): dbpkg.depends.all().delete() deps = [create_depend(dbpkg, y) for y in repopkg.depends] - deps += [create_depend(dbpkg, y, True) for y in repopkg.optdepends] - PackageDepend.objects.bulk_create(deps) + deps += [create_depend(dbpkg, y, 'O') for y in repopkg.optdepends] + deps += [create_depend(dbpkg, y, 'M') for y in repopkg.makedepends] + deps += [create_depend(dbpkg, y, 'C') for y in repopkg.checkdepends] + Depend.objects.bulk_create(deps) dbpkg.conflicts.all().delete() conflicts = [create_related(Conflict, dbpkg, y) for y in repopkg.conflicts] @@ -275,7 +277,7 @@ def populate_files(dbpkg, repopkg, force=False): return if not dbpkg.files_last_update or not dbpkg.last_update: pass - elif dbpkg.files_last_update > dbpkg.last_update: + elif dbpkg.files_last_update >= dbpkg.last_update: return # only delete files if we are reading a DB that contains them @@ -284,17 +286,24 @@ def populate_files(dbpkg, repopkg, force=False): logger.info("adding %d files for package %s", len(repopkg.files), dbpkg.pkgname) pkg_files = [] - for f in repopkg.files: - dirname, filename = f.rsplit('/', 1) + # sort in normal alpha-order that pacman uses, rather than makepkg's + # default breadth-first, directory-first ordering + files = sorted(repopkg.files) + for f in files: + if '/' in f: + dirname, filename = f.rsplit('/', 1) + dirname += '/' + else: + dirname, filename = '', f if filename == '': filename = None pkgfile = PackageFile(pkg=dbpkg, is_directory=(filename is None), - directory=dirname + '/', + directory=dirname, filename=filename) pkg_files.append(pkgfile) PackageFile.objects.bulk_create(pkg_files) - dbpkg.files_last_update = utc_now() + dbpkg.files_last_update = now() dbpkg.save() @@ -307,7 +316,7 @@ def update_common(archname, reponame, pkgs, sanity_check=True): transaction.set_dirty() repository = Repo.objects.get(name__iexact=reponame) - architecture = Arch.objects.get(name__iexact=archname) + architecture = Arch.objects.get(name=archname) # no-arg order_by() removes even the default ordering; we don't need it dbpkgs = Package.objects.filter( arch=architecture, repo=repository).order_by() @@ -346,35 +355,42 @@ def db_update(archname, reponame, pkgs, force=False): logger.info('Updating %s (%s)', reponame, archname) dbpkgs = update_common(archname, reponame, pkgs, True) repository = Repo.objects.get(name__iexact=reponame) - architecture = Arch.objects.get(name__iexact=archname) + architecture = Arch.objects.get(name=archname) # This makes our inner loop where we find packages by name *way* more # efficient by not having to go to the database for each package to # SELECT them by name. - dbdict = dict((dbpkg.pkgname, dbpkg) for dbpkg in dbpkgs) + dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs} dbset = set(dbdict.keys()) - syncset = set([pkg.name for pkg in pkgs]) + syncset = {pkg.name for pkg in pkgs} in_sync_not_db = syncset - dbset logger.info("%d packages in sync not db", len(in_sync_not_db)) # packages in syncdb and not in database (add to database) for pkg in (pkg for pkg in pkgs if pkg.name in in_sync_not_db): logger.info("Adding package %s", pkg.name) - dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository) + timestamp = now() + dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository, + created=timestamp) try: with transaction.commit_on_success(): - populate_pkg(dbpkg, pkg, timestamp=utc_now()) + populate_pkg(dbpkg, pkg, timestamp=timestamp) + Update.objects.log_update(None, dbpkg) except IntegrityError: - logger.warning("Could not add package %s; " - "not fatal if another thread beat us to it.", - pkg.name, exc_info=True) + if architecture.agnostic: + logger.warning("Could not add package %s; " + "not fatal if another thread beat us to it.", + pkg.name) + else: + logger.exception("Could not add package %s", pkg.name) # packages in database and not in syncdb (remove from database) for pkgname in (dbset - syncset): logger.info("Removing package %s", pkgname) dbpkg = dbdict[pkgname] with transaction.commit_on_success(): + Update.objects.log_update(dbpkg, None) # no race condition here as long as simultaneous threads both # issue deletes; second delete will be a no-op delete_pkg_files(dbpkg) @@ -391,7 +407,7 @@ def db_update(archname, reponame, pkgs, force=False): if not force and pkg_same_version(pkg, dbpkg): continue elif not force: - timestamp = utc_now() + timestamp = now() # The odd select_for_update song and dance here are to ensure # simultaneous updates don't happen on a package, causing @@ -402,7 +418,9 @@ def db_update(archname, reponame, pkgs, force=False): logger.debug("Package %s was already updated", pkg.name) continue logger.info("Updating package %s", pkg.name) + prevpkg = copy(dbpkg) populate_pkg(dbpkg, pkg, force=force, timestamp=timestamp) + Update.objects.log_update(prevpkg, dbpkg) logger.info('Finished updating arch: %s', archname) @@ -413,7 +431,7 @@ def filesonly_update(archname, reponame, pkgs, force=False): """ logger.info('Updating files for %s (%s)', reponame, archname) dbpkgs = update_common(archname, reponame, pkgs, False) - dbdict = dict((dbpkg.pkgname, dbpkg) for dbpkg in dbpkgs) + dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs} dbset = set(dbdict.keys()) for pkg in (pkg for pkg in pkgs if pkg.name in dbset): @@ -425,7 +443,7 @@ def filesonly_update(archname, reponame, pkgs, force=False): with transaction.commit_on_success(): if not dbpkg.files_last_update or not dbpkg.last_update: pass - elif not force and dbpkg.files_last_update > dbpkg.last_update: + elif not force and dbpkg.files_last_update >= dbpkg.last_update: logger.debug("Files for %s are up to date", pkg.name) continue dbpkg = Package.objects.select_for_update().get(id=dbpkg.id) @@ -509,7 +527,7 @@ def locate_arch(arch): if isinstance(arch, Arch): return arch try: - return Arch.objects.get(name__iexact=arch) + return Arch.objects.get(name=arch) except Arch.DoesNotExist: raise CommandError( 'Specified architecture %s is not currently known.' % arch) @@ -541,6 +559,12 @@ def read_repo(primary_arch, repo_file, options): package.name, repo_file, package.arch)) del packages + database = router.db_for_write(Package) + connection = connections[database] + if connection.vendor == 'sqlite': + cursor = connection.cursor() + cursor.execute('PRAGMA synchronous = NORMAL') + logger.info('Starting database updates for %s.', repo_file) for arch in sorted(packages_arches.keys()): if filesonly: @@ -548,6 +572,8 @@ def read_repo(primary_arch, repo_file, options): else: db_update(arch, repo, packages_arches[arch], force) logger.info('Finished database updates for %s.', repo_file) + connection.commit() + connection.close() return 0 # vim: set ts=4 sw=4 et: |