diff options
Diffstat (limited to 'devel/management')
| -rw-r--r-- | devel/management/commands/generate_keyring.py | 6 | ||||
| -rw-r--r-- | devel/management/commands/import_signatures.py | 123 | ||||
| -rw-r--r-- | devel/management/commands/pgp_import.py | 241 | ||||
| -rw-r--r-- | devel/management/commands/rematch_developers.py | 63 | ||||
| -rw-r--r-- | devel/management/commands/reporead.py | 102 | ||||
| -rw-r--r--[-rwxr-xr-x] | devel/management/commands/reporead_inotify.py | 19 |
6 files changed, 353 insertions, 201 deletions
diff --git a/devel/management/commands/generate_keyring.py b/devel/management/commands/generate_keyring.py index b9117c84..34bcd2f8 100644 --- a/devel/management/commands/generate_keyring.py +++ b/devel/management/commands/generate_keyring.py @@ -48,13 +48,17 @@ def generate_keyring(keyserver, keyring): logger.info("getting all known key IDs") # Screw you Django, for not letting one natively do value != <empty string> - key_ids = UserProfile.objects.filter(user__is_active=True, + key_ids = UserProfile.objects.filter( pgp_key__isnull=False).extra(where=["pgp_key != ''"]).values_list( "pgp_key", flat=True) logger.info("%d keys fetched from user profiles", len(key_ids)) master_key_ids = MasterKey.objects.values_list("pgp_key", flat=True) logger.info("%d keys fetched from master keys", len(master_key_ids)) + # GPG is stupid and interprets any filename without path portion as being + # in ~/.gnupg/. Fake it out if we just get a bare filename. + if '/' not in keyring: + keyring = './%s' % keyring gpg_cmd = ["gpg", "--no-default-keyring", "--keyring", keyring, "--keyserver", keyserver, "--recv-keys"] logger.info("running command: %r", gpg_cmd) diff --git a/devel/management/commands/import_signatures.py b/devel/management/commands/import_signatures.py deleted file mode 100644 index ce1aba90..00000000 --- a/devel/management/commands/import_signatures.py +++ /dev/null @@ -1,123 +0,0 @@ -# -*- coding: utf-8 -*- -""" -import_signatures command - -Import signatures from a given GPG keyring. - -Usage: ./manage.py generate_keyring <keyring_path> -""" - -from collections import namedtuple -from datetime import datetime -import logging -import subprocess -import sys - -from django.core.management.base import BaseCommand, CommandError -from django.db import transaction - -from devel.models import PGPSignature - -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s -> %(levelname)s: %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', - stream=sys.stderr) -logger = logging.getLogger() - -class Command(BaseCommand): - args = "<keyring_path>" - help = "Import signatures from a given GPG keyring." - - def handle(self, *args, **options): - v = int(options.get('verbosity', None)) - if v == 0: - logger.level = logging.ERROR - elif v == 1: - logger.level = logging.INFO - elif v == 2: - logger.level = logging.DEBUG - - if len(args) < 1: - raise CommandError("keyring_path must be provided") - - import_signatures(args[0]) - - -SignatureData = namedtuple('SignatureData', - ('signer', 'signee', 'created', 'expires', 'valid')) - - -def get_date(epoch_string): - '''Convert a epoch string into a python 'date' object (not datetime).''' - return datetime.utcfromtimestamp(int(epoch_string)).date() - - -def parse_sigdata(data): - nodes = {} - edges = [] - current_pubkey = None - - # parse all of the output from our successful GPG command - logger.info("parsing command output") - for line in data.split('\n'): - parts = line.split(':') - if parts[0] == 'pub': - current_pubkey = parts[4] - nodes[current_pubkey] = None - if parts[0] == 'uid': - uid = parts[9] - # only set uid if this is the first one encountered - if nodes[current_pubkey] is None: - nodes[current_pubkey] = uid - if parts[0] == 'sig': - signer = parts[4] - created = get_date(parts[5]) - expires = None - if parts[6]: - expires = get_date(parts[6]) - valid = parts[1] != '-' - edge = SignatureData(signer, current_pubkey, - created, expires, valid) - edges.append(edge) - - return nodes, edges - - -def import_signatures(keyring): - gpg_cmd = ["gpg", "--no-default-keyring", "--keyring", keyring, - "--list-sigs", "--with-colons", "--fixed-list-mode"] - logger.info("running command: %r", gpg_cmd) - proc = subprocess.Popen(gpg_cmd, stdout=subprocess.PIPE) - outdata, errdata = proc.communicate() - if proc.returncode != 0: - logger.error(errdata) - raise subprocess.CalledProcessError(proc.returncode, gpg_cmd) - - nodes, edges = parse_sigdata(outdata) - - # now prune the data down to what we actually want. - # prune edges not in nodes, remove duplicates, and self-sigs - pruned_edges = set(edge for edge in edges - if edge.signer in nodes and edge.signer != edge.signee) - - logger.info("creating or finding %d signatures", len(pruned_edges)) - created_ct = updated_ct = 0 - with transaction.commit_on_success(): - for edge in pruned_edges: - sig, created = PGPSignature.objects.get_or_create( - signer=edge.signer, signee=edge.signee, - created=edge.created, expires=edge.expires, - defaults={ 'valid': edge.valid }) - if sig.valid != edge.valid: - sig.valid = edge.valid - sig.save() - updated_ct = 1 - if created: - created_ct += 1 - - sig_ct = PGPSignature.objects.all().count() - logger.info("%d total signatures in database", sig_ct) - logger.info("created %d, updated %d signatures", created_ct, updated_ct) - -# vim: set ts=4 sw=4 et: diff --git a/devel/management/commands/pgp_import.py b/devel/management/commands/pgp_import.py new file mode 100644 index 00000000..10e6cfcb --- /dev/null +++ b/devel/management/commands/pgp_import.py @@ -0,0 +1,241 @@ +# -*- coding: utf-8 -*- +""" +pgp_import command + +Import keys and signatures from a given GPG keyring. + +Usage: ./manage.py pgp_import <keyring_path> +""" + +from collections import namedtuple, OrderedDict +from datetime import datetime +import logging +from pytz import utc +import subprocess +import sys + +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction + +from devel.models import DeveloperKey, PGPSignature +from devel.utils import UserFinder + + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s -> %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + stream=sys.stderr) +logger = logging.getLogger() + +class Command(BaseCommand): + args = "<keyring_path>" + help = "Import keys and signatures from a given GPG keyring." + + def handle(self, *args, **options): + v = int(options.get('verbosity', None)) + if v == 0: + logger.level = logging.ERROR + elif v == 1: + logger.level = logging.INFO + elif v == 2: + logger.level = logging.DEBUG + + if len(args) < 1: + raise CommandError("keyring_path must be provided") + + import_keys(args[0]) + import_signatures(args[0]) + + +def get_date(epoch_string): + '''Convert a epoch string into a python 'date' object (not datetime).''' + if not epoch_string: + return None + return datetime.utcfromtimestamp(int(epoch_string)).date() + + +def get_datetime(epoch_string): + '''Convert a epoch string into a python 'datetime' object.''' + if not epoch_string: + return None + return datetime.utcfromtimestamp(int(epoch_string)).replace(tzinfo=utc) + + +def call_gpg(keyring, *args): + # GPG is stupid and interprets any filename without path portion as being + # in ~/.gnupg/. Fake it out if we just get a bare filename. + if '/' not in keyring: + keyring = './%s' % keyring + gpg_cmd = ["gpg2", "--no-default-keyring", "--keyring", keyring, + "--with-colons", "--fixed-list-mode"] + gpg_cmd.extend(args) + logger.info("running command: %s", ' '.join(gpg_cmd)) + proc = subprocess.Popen(gpg_cmd, stdout=subprocess.PIPE) + outdata, errdata = proc.communicate() + if proc.returncode != 0: + logger.error(errdata) + raise subprocess.CalledProcessError(proc.returncode, gpg_cmd) + return outdata + + +class KeyData(object): + def __init__(self, key, created, expires): + self.key = key + self.created = get_datetime(created) + self.expires = get_datetime(expires) + self.parent = None + self.revoked = None + self.db_id = None + + +def parse_keydata(data): + keys = OrderedDict() + current_pubkey = None + + # parse all of the output from our successful GPG command + logger.info("parsing command output") + for line in data.split('\n'): + parts = line.split(':') + if parts[0] == 'pub': + key = parts[4] + current_pubkey = key + keys[key] = KeyData(key, parts[5], parts[6]) + node = parts[0] + elif parts[0] == 'sub': + key = parts[4] + keys[key] = KeyData(key, parts[5], parts[6]) + keys[key].parent = current_pubkey + node = parts[0] + elif parts[0] == 'uid': + node = parts[0] + elif parts[0] == 'rev' and node in ('pub', 'sub'): + keys[current_pubkey].revoked = get_datetime(parts[5]) + + return keys + + +def find_key_owner(key, keys, finder): + '''Recurse up the chain, looking for an owner.''' + if key is None: + return None + owner = finder.find_by_pgp_key(key.key) + if owner: + return owner + if key.parent: + return find_key_owner(keys[key.parent], keys, finder) + return None + + +def import_keys(keyring): + outdata = call_gpg(keyring, "--list-sigs") + keydata = parse_keydata(outdata) + + logger.info("creating or finding %d keys", len(keydata)) + created_ct = updated_ct = 0 + with transaction.commit_on_success(): + finder = UserFinder() + # we are dependent on parents coming before children; parse_keydata + # uses an OrderedDict to ensure this is the case. + for data in keydata.values(): + parent_id = None + if data.parent: + parent_data = keydata.get(data.parent, None) + if parent_data: + parent_id = parent_data.db_id + other = { + 'expires': data.expires, + 'revoked': data.revoked, + 'parent_id': parent_id, + } + dkey, created = DeveloperKey.objects.get_or_create( + key=data.key, created=data.created, defaults=other) + data.db_id = dkey.id + + # set or update any additional data we might need to + needs_save = False + if created: + created_ct += 1 + else: + for k, v in other.items(): + if getattr(dkey, k) != v: + setattr(dkey, k, v) + needs_save = True + if dkey.owner_id is None: + owner = find_key_owner(data, keydata, finder) + if owner is not None: + dkey.owner = owner + needs_save = True + if needs_save: + dkey.save() + updated_ct += 1 + + key_ct = DeveloperKey.objects.all().count() + logger.info("%d total keys in database", key_ct) + logger.info("created %d, updated %d keys", created_ct, updated_ct) + + +SignatureData = namedtuple('SignatureData', + ('signer', 'signee', 'created', 'expires', 'valid')) + + +def parse_sigdata(data): + nodes = {} + edges = [] + current_pubkey = None + + # parse all of the output from our successful GPG command + logger.info("parsing command output") + for line in data.split('\n'): + parts = line.split(':') + if parts[0] == 'pub': + current_pubkey = parts[4] + nodes[current_pubkey] = None + if parts[0] == 'uid': + uid = parts[9] + # only set uid if this is the first one encountered + if nodes[current_pubkey] is None: + nodes[current_pubkey] = uid + if parts[0] == 'sig': + signer = parts[4] + created = get_date(parts[5]) + expires = None + if parts[6]: + expires = get_date(parts[6]) + valid = parts[1] != '-' + edge = SignatureData(signer, current_pubkey, + created, expires, valid) + edges.append(edge) + + return nodes, edges + + +def import_signatures(keyring): + outdata = call_gpg(keyring, "--list-sigs") + nodes, edges = parse_sigdata(outdata) + + # now prune the data down to what we actually want. + # prune edges not in nodes, remove duplicates, and self-sigs + pruned_edges = {edge for edge in edges + if edge.signer in nodes and edge.signer != edge.signee} + + logger.info("creating or finding %d signatures", len(pruned_edges)) + created_ct = updated_ct = 0 + with transaction.commit_on_success(): + for edge in pruned_edges: + sig, created = PGPSignature.objects.get_or_create( + signer=edge.signer, signee=edge.signee, + created=edge.created, expires=edge.expires, + defaults={ 'valid': edge.valid }) + if sig.valid != edge.valid: + sig.valid = edge.valid + sig.save() + updated_ct = 1 + if created: + created_ct += 1 + + sig_ct = PGPSignature.objects.all().count() + logger.info("%d total signatures in database", sig_ct) + logger.info("created %d, updated %d signatures", created_ct, updated_ct) + +# vim: set ts=4 sw=4 et: diff --git a/devel/management/commands/rematch_developers.py b/devel/management/commands/rematch_developers.py index 8383cc8d..2b379588 100644 --- a/devel/management/commands/rematch_developers.py +++ b/devel/management/commands/rematch_developers.py @@ -46,52 +46,53 @@ class Command(NoArgsCommand): @transaction.commit_on_success def match_packager(finder): - logger.info("getting all unmatched packages") + logger.info("getting all unmatched packager strings") package_count = matched_count = 0 - unknown = set() - - for package in Package.objects.filter(packager__isnull=True): - if package.packager_str in unknown: - continue - logger.debug("package %s, packager string %s", - package.pkgname, package.packager_str) - package_count += 1 - user = finder.find(package.packager_str) + mapping = {} + + unmatched = Package.objects.filter(packager__isnull=True).values_list( + 'packager_str', flat=True).order_by().distinct() + + logger.info("%d packager strings retrieved", len(unmatched)) + for packager in unmatched: + logger.debug("packager string %s", packager) + user = finder.find(packager) if user: - package.packager = user + mapping[packager] = user logger.debug(" found user %s" % user.username) - package.save() matched_count += 1 - else: - unknown.add(package.packager_str) - logger.info("%d packager strings checked, %d newly matched", + for packager_str, user in mapping.items(): + package_count += Package.objects.filter(packager__isnull=True, + packager_str=packager_str).update(packager=user) + + logger.info("%d packages updated, %d packager strings matched", package_count, matched_count) - logger.debug("unknown packagers:\n%s", - "\n".join(unknown)) @transaction.commit_on_success def match_flagrequest(finder): - logger.info("getting all non-user flag requests") + logger.info("getting all flag request email addresses from unknown users") req_count = matched_count = 0 - unknown = set() - - for request in FlagRequest.objects.filter(user__isnull=True): - if request.user_email in unknown: - continue - logger.debug("email %s", request.user_email) - req_count += 1 - user = finder.find_by_email(request.user_email) + mapping = {} + + unmatched = FlagRequest.objects.filter(user__isnull=True).values_list( + 'user_email', flat=True).order_by().distinct() + + logger.info("%d email addresses retrieved", len(unmatched)) + for user_email in unmatched: + logger.debug("email %s", user_email) + user = finder.find_by_email(user_email) if user: - request.user = user + mapping[user_email] = user logger.debug(" found user %s" % user.username) - request.save() matched_count += 1 - else: - unknown.add(request.user_email) - logger.info("%d request emails checked, %d newly matched", + for user_email, user in mapping.items(): + req_count += FlagRequest.objects.filter(user__isnull=True, + user_email=user_email).update(user=user) + + logger.info("%d request emails updated, %d emails matched", req_count, matched_count) # vim: set ts=4 sw=4 et: diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index cf98f004..1e456c8c 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -14,6 +14,7 @@ Example: """ from collections import defaultdict +from copy import copy import io import os import re @@ -27,11 +28,12 @@ from pytz import utc from django.core.management.base import BaseCommand, CommandError from django.db import connections, router, transaction from django.db.utils import IntegrityError +from django.utils.timezone import now from devel.utils import UserFinder -from main.models import Arch, Package, PackageDepend, PackageFile, Repo -from main.utils import utc_now -from packages.models import Conflict, Provision, Replacement +from main.models import Arch, Package, PackageFile, Repo +from packages.models import Depend, Conflict, Provision, Replacement, Update +from packages.utils import parse_version logging.basicConfig( @@ -78,10 +80,9 @@ class RepoPackage(object): bare = ( 'name', 'base', 'arch', 'filename', 'md5sum', 'sha256sum', 'url', 'packager' ) number = ( 'csize', 'isize' ) - collections = ( 'depends', 'optdepends', 'conflicts', - 'provides', 'replaces', 'groups', 'license', 'files' ) - - version_re = re.compile(r'^((\d+):)?(.+)-([^-]+)$') + collections = ( 'depends', 'optdepends', 'makedepends', 'checkdepends', + 'conflicts', 'provides', 'replaces', 'groups', 'license', + 'files' ) def __init__(self, repo): self.repo = repo @@ -111,11 +112,7 @@ class RepoPackage(object): v[0] = 'missing' setattr(self, k, v[0]) elif k == 'version': - match = self.version_re.match(v[0]) - self.ver = match.group(3) - self.rel = match.group(4) - if match.group(2): - self.epoch = int(match.group(2)) + self.ver, self.rel, self.epoch = parse_version(v[0]) elif k == 'builddate': try: builddate = datetime.utcfromtimestamp(int(v[0])) @@ -143,19 +140,21 @@ class RepoPackage(object): return u'%s-%s' % (self.ver, self.rel) -DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.*))?$") +DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.+))?$") -def create_depend(package, dep_str, optional=False): - depend = PackageDepend(pkg=package, optional=optional) +def create_depend(package, dep_str, deptype='D'): + depend = Depend(pkg=package, deptype=deptype) # lop off any description first parts = dep_str.split(':', 1) if len(parts) > 1: depend.description = parts[1].strip() match = DEPEND_RE.match(parts[0].strip()) if match: - depend.depname = match.group(1) - if match.group(2): - depend.depvcmp = match.group(2) + depend.name = match.group(1) + if match.group(3): + depend.comparison = match.group(3) + if match.group(4): + depend.version = match.group(4) else: logger.warning('Package %s had unparsable depend string %s', package.pkgname, dep_str) @@ -183,6 +182,7 @@ def create_related(model, package, rel_str, equals_only=False): return None return related + def create_multivalued(dbpkg, repopkg, db_attr, repo_attr): '''Populate the simplest of multivalued attributes. These are those that only deal with a 'name' attribute, such as licenses, groups, etc. The input @@ -236,8 +236,10 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): dbpkg.depends.all().delete() deps = [create_depend(dbpkg, y) for y in repopkg.depends] - deps += [create_depend(dbpkg, y, True) for y in repopkg.optdepends] - PackageDepend.objects.bulk_create(deps) + deps += [create_depend(dbpkg, y, 'O') for y in repopkg.optdepends] + deps += [create_depend(dbpkg, y, 'M') for y in repopkg.makedepends] + deps += [create_depend(dbpkg, y, 'C') for y in repopkg.checkdepends] + Depend.objects.bulk_create(deps) dbpkg.conflicts.all().delete() conflicts = [create_related(Conflict, dbpkg, y) for y in repopkg.conflicts] @@ -275,7 +277,7 @@ def populate_files(dbpkg, repopkg, force=False): return if not dbpkg.files_last_update or not dbpkg.last_update: pass - elif dbpkg.files_last_update > dbpkg.last_update: + elif dbpkg.files_last_update >= dbpkg.last_update: return # only delete files if we are reading a DB that contains them @@ -284,17 +286,24 @@ def populate_files(dbpkg, repopkg, force=False): logger.info("adding %d files for package %s", len(repopkg.files), dbpkg.pkgname) pkg_files = [] - for f in repopkg.files: - dirname, filename = f.rsplit('/', 1) + # sort in normal alpha-order that pacman uses, rather than makepkg's + # default breadth-first, directory-first ordering + files = sorted(repopkg.files) + for f in files: + if '/' in f: + dirname, filename = f.rsplit('/', 1) + dirname += '/' + else: + dirname, filename = '', f if filename == '': filename = None pkgfile = PackageFile(pkg=dbpkg, is_directory=(filename is None), - directory=dirname + '/', + directory=dirname, filename=filename) pkg_files.append(pkgfile) PackageFile.objects.bulk_create(pkg_files) - dbpkg.files_last_update = utc_now() + dbpkg.files_last_update = now() dbpkg.save() @@ -307,7 +316,7 @@ def update_common(archname, reponame, pkgs, sanity_check=True): transaction.set_dirty() repository = Repo.objects.get(name__iexact=reponame) - architecture = Arch.objects.get(name__iexact=archname) + architecture = Arch.objects.get(name=archname) # no-arg order_by() removes even the default ordering; we don't need it dbpkgs = Package.objects.filter( arch=architecture, repo=repository).order_by() @@ -346,35 +355,42 @@ def db_update(archname, reponame, pkgs, force=False): logger.info('Updating %s (%s)', reponame, archname) dbpkgs = update_common(archname, reponame, pkgs, True) repository = Repo.objects.get(name__iexact=reponame) - architecture = Arch.objects.get(name__iexact=archname) + architecture = Arch.objects.get(name=archname) # This makes our inner loop where we find packages by name *way* more # efficient by not having to go to the database for each package to # SELECT them by name. - dbdict = dict((dbpkg.pkgname, dbpkg) for dbpkg in dbpkgs) + dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs} dbset = set(dbdict.keys()) - syncset = set([pkg.name for pkg in pkgs]) + syncset = {pkg.name for pkg in pkgs} in_sync_not_db = syncset - dbset logger.info("%d packages in sync not db", len(in_sync_not_db)) # packages in syncdb and not in database (add to database) for pkg in (pkg for pkg in pkgs if pkg.name in in_sync_not_db): logger.info("Adding package %s", pkg.name) - dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository) + timestamp = now() + dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository, + created=timestamp) try: with transaction.commit_on_success(): - populate_pkg(dbpkg, pkg, timestamp=utc_now()) + populate_pkg(dbpkg, pkg, timestamp=timestamp) + Update.objects.log_update(None, dbpkg) except IntegrityError: - logger.warning("Could not add package %s; " - "not fatal if another thread beat us to it.", - pkg.name, exc_info=True) + if architecture.agnostic: + logger.warning("Could not add package %s; " + "not fatal if another thread beat us to it.", + pkg.name) + else: + logger.exception("Could not add package %s", pkg.name) # packages in database and not in syncdb (remove from database) for pkgname in (dbset - syncset): logger.info("Removing package %s", pkgname) dbpkg = dbdict[pkgname] with transaction.commit_on_success(): + Update.objects.log_update(dbpkg, None) # no race condition here as long as simultaneous threads both # issue deletes; second delete will be a no-op delete_pkg_files(dbpkg) @@ -391,7 +407,7 @@ def db_update(archname, reponame, pkgs, force=False): if not force and pkg_same_version(pkg, dbpkg): continue elif not force: - timestamp = utc_now() + timestamp = now() # The odd select_for_update song and dance here are to ensure # simultaneous updates don't happen on a package, causing @@ -402,7 +418,9 @@ def db_update(archname, reponame, pkgs, force=False): logger.debug("Package %s was already updated", pkg.name) continue logger.info("Updating package %s", pkg.name) + prevpkg = copy(dbpkg) populate_pkg(dbpkg, pkg, force=force, timestamp=timestamp) + Update.objects.log_update(prevpkg, dbpkg) logger.info('Finished updating arch: %s', archname) @@ -413,7 +431,7 @@ def filesonly_update(archname, reponame, pkgs, force=False): """ logger.info('Updating files for %s (%s)', reponame, archname) dbpkgs = update_common(archname, reponame, pkgs, False) - dbdict = dict((dbpkg.pkgname, dbpkg) for dbpkg in dbpkgs) + dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs} dbset = set(dbdict.keys()) for pkg in (pkg for pkg in pkgs if pkg.name in dbset): @@ -425,7 +443,7 @@ def filesonly_update(archname, reponame, pkgs, force=False): with transaction.commit_on_success(): if not dbpkg.files_last_update or not dbpkg.last_update: pass - elif not force and dbpkg.files_last_update > dbpkg.last_update: + elif not force and dbpkg.files_last_update >= dbpkg.last_update: logger.debug("Files for %s are up to date", pkg.name) continue dbpkg = Package.objects.select_for_update().get(id=dbpkg.id) @@ -509,7 +527,7 @@ def locate_arch(arch): if isinstance(arch, Arch): return arch try: - return Arch.objects.get(name__iexact=arch) + return Arch.objects.get(name=arch) except Arch.DoesNotExist: raise CommandError( 'Specified architecture %s is not currently known.' % arch) @@ -541,6 +559,12 @@ def read_repo(primary_arch, repo_file, options): package.name, repo_file, package.arch)) del packages + database = router.db_for_write(Package) + connection = connections[database] + if connection.vendor == 'sqlite': + cursor = connection.cursor() + cursor.execute('PRAGMA synchronous = NORMAL') + logger.info('Starting database updates for %s.', repo_file) for arch in sorted(packages_arches.keys()): if filesonly: @@ -548,6 +572,8 @@ def read_repo(primary_arch, repo_file, options): else: db_update(arch, repo, packages_arches[arch], force) logger.info('Finished database updates for %s.', repo_file) + connection.commit() + connection.close() return 0 # vim: set ts=4 sw=4 et: diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py index c74762eb..8c1e47bf 100755..100644 --- a/devel/management/commands/reporead_inotify.py +++ b/devel/management/commands/reporead_inotify.py @@ -23,7 +23,7 @@ import threading import time from django.core.management.base import BaseCommand, CommandError -from django.db import connection +from django.db import connection, transaction from main.models import Arch, Repo from .reporead import read_repo @@ -53,6 +53,11 @@ class Command(BaseCommand): self.path_template = path_template notifier = self.setup_notifier() + # this thread is done using the database; all future access is done in + # the spawned read_repo() processes, so close the otherwise completely + # idle connection. + connection.close() + logger.info('Entering notifier loop') notifier.loop() @@ -61,15 +66,18 @@ class Command(BaseCommand): if hasattr(thread, 'cancel'): thread.cancel() + @transaction.commit_on_success def setup_notifier(self): '''Set up and configure the inotify machinery and logic. This takes the provided or default path_template and builds a list of directories we need to watch for database updates. It then validates and passes these on to the various pyinotify pieces as necessary and finally builds and returns a notifier object.''' + transaction.commit_manually() arches = Arch.objects.filter(agnostic=False) repos = Repo.objects.all() - arch_path_map = dict((arch, None) for arch in arches) + transaction.set_dirty() + arch_path_map = {arch: None for arch in arches} all_paths = set() total_paths = 0 for arch in arches: @@ -77,7 +85,7 @@ class Command(BaseCommand): for repo in repos) # take a python format string and generate all unique combinations # of directories from it; using set() ensures we filter it down - paths = set(self.path_template % values for values in combos) + paths = {self.path_template % values for values in combos} total_paths += len(paths) all_paths |= paths arch_path_map[arch] = paths @@ -91,11 +99,6 @@ class Command(BaseCommand): raise CommandError('path template did not uniquely ' 'determine architecture for each file') - # this thread is done using the database; all future access is done in - # the spawned read_repo() processes, so close the otherwise completely - # idle connection. - connection.close() - # A proper atomic replacement of the database as done by rsync is type # IN_MOVED_TO. repo-add/remove will finish with a IN_CLOSE_WRITE. mask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO |