Merge branch 'archweb' into archweb-generic2

Conflicts:
	devel/views.py
	feeds.py
	public/views.py
	settings.py
	sitestatic/archweb.js
	templates/base.html
	templates/devel/profile.html
	templates/mirrors/status.html
	templates/news/view.html
	templates/packages/flaghelp.html
	templates/packages/opensearch.xml
	templates/public/download.html
	templates/public/feeds.html
	templates/public/index.html
	templates/registration/login.html
	templates/releng/results.html
	templates/todolists/public_list.html

6 files changed, 353 insertions, 201 deletions

diff --git a/devel/management/commands/generate_keyring.py b/devel/management/commands/generate_keyring.py
index b9117c84..34bcd2f8 100644
--- a/devel/management/commands/generate_keyring.py
+++ b/devel/management/commands/generate_keyring.py
@@ -48,13 +48,17 @@ def generate_keyring(keyserver, keyring):
     logger.info("getting all known key IDs")
 
     # Screw you Django, for not letting one natively do value != <empty string>
-    key_ids = UserProfile.objects.filter(user__is_active=True,
+    key_ids = UserProfile.objects.filter(
             pgp_key__isnull=False).extra(where=["pgp_key != ''"]).values_list(
             "pgp_key", flat=True)
     logger.info("%d keys fetched from user profiles", len(key_ids))
     master_key_ids = MasterKey.objects.values_list("pgp_key", flat=True)
     logger.info("%d keys fetched from master keys", len(master_key_ids))
 
+    # GPG is stupid and interprets any filename without path portion as being
+    # in ~/.gnupg/. Fake it out if we just get a bare filename.
+    if '/' not in keyring:
+        keyring = './%s' % keyring
     gpg_cmd = ["gpg", "--no-default-keyring", "--keyring", keyring,
             "--keyserver", keyserver, "--recv-keys"]
     logger.info("running command: %r", gpg_cmd)
diff --git a/devel/management/commands/import_signatures.py b/devel/management/commands/import_signatures.py
deleted file mode 100644
index ce1aba90..00000000
--- a/devel/management/commands/import_signatures.py
+++ /dev/null
@@ -1,123 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-import_signatures command
-
-Import signatures from a given GPG keyring.
-
-Usage: ./manage.py generate_keyring <keyring_path>
-"""
-
-from collections import namedtuple
-from datetime import datetime
-import logging
-import subprocess
-import sys
-
-from django.core.management.base import BaseCommand, CommandError
-from django.db import transaction
-
-from devel.models import PGPSignature
-
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s -> %(levelname)s: %(message)s',
-    datefmt='%Y-%m-%d %H:%M:%S',
-    stream=sys.stderr)
-logger = logging.getLogger()
-
-class Command(BaseCommand):
-    args = "<keyring_path>"
-    help = "Import signatures from a given GPG keyring."
-
-    def handle(self, *args, **options):
-        v = int(options.get('verbosity', None))
-        if v == 0:
-            logger.level = logging.ERROR
-        elif v == 1:
-            logger.level = logging.INFO
-        elif v == 2:
-            logger.level = logging.DEBUG
-
-        if len(args) < 1:
-            raise CommandError("keyring_path must be provided")
-
-        import_signatures(args[0])
-
-
-SignatureData = namedtuple('SignatureData',
-        ('signer', 'signee', 'created', 'expires', 'valid'))
-
-
-def get_date(epoch_string):
-    '''Convert a epoch string into a python 'date' object (not datetime).'''
-    return datetime.utcfromtimestamp(int(epoch_string)).date()
-
-
-def parse_sigdata(data):
-    nodes = {}
-    edges = []
-    current_pubkey = None
-
-    # parse all of the output from our successful GPG command
-    logger.info("parsing command output")
-    for line in data.split('\n'):
-        parts = line.split(':')
-        if parts[0] == 'pub':
-            current_pubkey = parts[4]
-            nodes[current_pubkey] = None
-        if parts[0] == 'uid':
-            uid = parts[9]
-            # only set uid if this is the first one encountered
-            if nodes[current_pubkey] is None:
-                nodes[current_pubkey] = uid
-        if parts[0] == 'sig':
-            signer = parts[4]
-            created = get_date(parts[5])
-            expires = None
-            if parts[6]:
-                expires = get_date(parts[6])
-            valid = parts[1] != '-'
-            edge = SignatureData(signer, current_pubkey,
-                    created, expires, valid)
-            edges.append(edge)
-
-    return nodes, edges
-
-
-def import_signatures(keyring):
-    gpg_cmd = ["gpg", "--no-default-keyring", "--keyring", keyring,
-            "--list-sigs", "--with-colons", "--fixed-list-mode"]
-    logger.info("running command: %r", gpg_cmd)
-    proc = subprocess.Popen(gpg_cmd, stdout=subprocess.PIPE)
-    outdata, errdata = proc.communicate()
-    if proc.returncode != 0:
-        logger.error(errdata)
-        raise subprocess.CalledProcessError(proc.returncode, gpg_cmd)
-
-    nodes, edges = parse_sigdata(outdata)
-
-    # now prune the data down to what we actually want.
-    # prune edges not in nodes, remove duplicates, and self-sigs
-    pruned_edges = set(edge for edge in edges
-            if edge.signer in nodes and edge.signer != edge.signee)
-
-    logger.info("creating or finding %d signatures", len(pruned_edges))
-    created_ct = updated_ct = 0
-    with transaction.commit_on_success():
-        for edge in pruned_edges:
-            sig, created = PGPSignature.objects.get_or_create(
-                    signer=edge.signer, signee=edge.signee,
-                    created=edge.created, expires=edge.expires,
-                    defaults={ 'valid': edge.valid })
-            if sig.valid != edge.valid:
-                sig.valid = edge.valid
-                sig.save()
-                updated_ct = 1
-            if created:
-                created_ct += 1
-
-    sig_ct = PGPSignature.objects.all().count()
-    logger.info("%d total signatures in database", sig_ct)
-    logger.info("created %d, updated %d signatures", created_ct, updated_ct)
-
-# vim: set ts=4 sw=4 et:
diff --git a/devel/management/commands/pgp_import.py b/devel/management/commands/pgp_import.py
new file mode 100644
index 00000000..10e6cfcb
--- /dev/null
+++ b/devel/management/commands/pgp_import.py
@@ -0,0 +1,241 @@
+# -*- coding: utf-8 -*-
+"""
+pgp_import command
+
+Import keys and signatures from a given GPG keyring.
+
+Usage: ./manage.py pgp_import <keyring_path>
+"""
+
+from collections import namedtuple, OrderedDict
+from datetime import datetime
+import logging
+from pytz import utc
+import subprocess
+import sys
+
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+
+from devel.models import DeveloperKey, PGPSignature
+from devel.utils import UserFinder
+
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s -> %(levelname)s: %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S',
+    stream=sys.stderr)
+logger = logging.getLogger()
+
+class Command(BaseCommand):
+    args = "<keyring_path>"
+    help = "Import keys and signatures from a given GPG keyring."
+
+    def handle(self, *args, **options):
+        v = int(options.get('verbosity', None))
+        if v == 0:
+            logger.level = logging.ERROR
+        elif v == 1:
+            logger.level = logging.INFO
+        elif v == 2:
+            logger.level = logging.DEBUG
+
+        if len(args) < 1:
+            raise CommandError("keyring_path must be provided")
+
+        import_keys(args[0])
+        import_signatures(args[0])
+
+
+def get_date(epoch_string):
+    '''Convert a epoch string into a python 'date' object (not datetime).'''
+    if not epoch_string:
+        return None
+    return datetime.utcfromtimestamp(int(epoch_string)).date()
+
+
+def get_datetime(epoch_string):
+    '''Convert a epoch string into a python 'datetime' object.'''
+    if not epoch_string:
+        return None
+    return datetime.utcfromtimestamp(int(epoch_string)).replace(tzinfo=utc)
+
+
+def call_gpg(keyring, *args):
+    # GPG is stupid and interprets any filename without path portion as being
+    # in ~/.gnupg/. Fake it out if we just get a bare filename.
+    if '/' not in keyring:
+        keyring = './%s' % keyring
+    gpg_cmd = ["gpg2", "--no-default-keyring", "--keyring", keyring,
+            "--with-colons", "--fixed-list-mode"]
+    gpg_cmd.extend(args)
+    logger.info("running command: %s", ' '.join(gpg_cmd))
+    proc = subprocess.Popen(gpg_cmd, stdout=subprocess.PIPE)
+    outdata, errdata = proc.communicate()
+    if proc.returncode != 0:
+        logger.error(errdata)
+        raise subprocess.CalledProcessError(proc.returncode, gpg_cmd)
+    return outdata
+
+
+class KeyData(object):
+    def __init__(self, key, created, expires):
+        self.key = key
+        self.created = get_datetime(created)
+        self.expires = get_datetime(expires)
+        self.parent = None
+        self.revoked = None
+        self.db_id = None
+
+
+def parse_keydata(data):
+    keys = OrderedDict()
+    current_pubkey = None
+
+    # parse all of the output from our successful GPG command
+    logger.info("parsing command output")
+    for line in data.split('\n'):
+        parts = line.split(':')
+        if parts[0] == 'pub':
+            key = parts[4]
+            current_pubkey = key
+            keys[key] = KeyData(key, parts[5], parts[6])
+            node = parts[0]
+        elif parts[0] == 'sub':
+            key = parts[4]
+            keys[key] = KeyData(key, parts[5], parts[6])
+            keys[key].parent = current_pubkey
+            node = parts[0]
+        elif parts[0] == 'uid':
+            node = parts[0]
+        elif parts[0] == 'rev' and node in ('pub', 'sub'):
+            keys[current_pubkey].revoked = get_datetime(parts[5])
+
+    return keys
+
+
+def find_key_owner(key, keys, finder):
+    '''Recurse up the chain, looking for an owner.'''
+    if key is None:
+        return None
+    owner = finder.find_by_pgp_key(key.key)
+    if owner:
+        return owner
+    if key.parent:
+        return find_key_owner(keys[key.parent], keys, finder)
+    return None
+
+
+def import_keys(keyring):
+    outdata = call_gpg(keyring, "--list-sigs")
+    keydata = parse_keydata(outdata)
+
+    logger.info("creating or finding %d keys", len(keydata))
+    created_ct = updated_ct = 0
+    with transaction.commit_on_success():
+        finder = UserFinder()
+        # we are dependent on parents coming before children; parse_keydata
+        # uses an OrderedDict to ensure this is the case.
+        for data in keydata.values():
+            parent_id = None
+            if data.parent:
+                parent_data = keydata.get(data.parent, None)
+                if parent_data:
+                    parent_id = parent_data.db_id
+            other = {
+                'expires': data.expires,
+                'revoked': data.revoked,
+                'parent_id': parent_id,
+            }
+            dkey, created = DeveloperKey.objects.get_or_create(
+                    key=data.key, created=data.created, defaults=other)
+            data.db_id = dkey.id
+
+            # set or update any additional data we might need to
+            needs_save = False
+            if created:
+                created_ct += 1
+            else:
+                for k, v in other.items():
+                    if getattr(dkey, k) != v:
+                        setattr(dkey, k, v)
+                        needs_save = True
+            if dkey.owner_id is None:
+                owner = find_key_owner(data, keydata, finder)
+                if owner is not None:
+                    dkey.owner = owner
+                    needs_save = True
+            if needs_save:
+                dkey.save()
+                updated_ct += 1
+
+    key_ct = DeveloperKey.objects.all().count()
+    logger.info("%d total keys in database", key_ct)
+    logger.info("created %d, updated %d keys", created_ct, updated_ct)
+
+
+SignatureData = namedtuple('SignatureData',
+        ('signer', 'signee', 'created', 'expires', 'valid'))
+
+
+def parse_sigdata(data):
+    nodes = {}
+    edges = []
+    current_pubkey = None
+
+    # parse all of the output from our successful GPG command
+    logger.info("parsing command output")
+    for line in data.split('\n'):
+        parts = line.split(':')
+        if parts[0] == 'pub':
+            current_pubkey = parts[4]
+            nodes[current_pubkey] = None
+        if parts[0] == 'uid':
+            uid = parts[9]
+            # only set uid if this is the first one encountered
+            if nodes[current_pubkey] is None:
+                nodes[current_pubkey] = uid
+        if parts[0] == 'sig':
+            signer = parts[4]
+            created = get_date(parts[5])
+            expires = None
+            if parts[6]:
+                expires = get_date(parts[6])
+            valid = parts[1] != '-'
+            edge = SignatureData(signer, current_pubkey,
+                    created, expires, valid)
+            edges.append(edge)
+
+    return nodes, edges
+
+
+def import_signatures(keyring):
+    outdata = call_gpg(keyring, "--list-sigs")
+    nodes, edges = parse_sigdata(outdata)
+
+    # now prune the data down to what we actually want.
+    # prune edges not in nodes, remove duplicates, and self-sigs
+    pruned_edges = {edge for edge in edges
+            if edge.signer in nodes and edge.signer != edge.signee}
+
+    logger.info("creating or finding %d signatures", len(pruned_edges))
+    created_ct = updated_ct = 0
+    with transaction.commit_on_success():
+        for edge in pruned_edges:
+            sig, created = PGPSignature.objects.get_or_create(
+                    signer=edge.signer, signee=edge.signee,
+                    created=edge.created, expires=edge.expires,
+                    defaults={ 'valid': edge.valid })
+            if sig.valid != edge.valid:
+                sig.valid = edge.valid
+                sig.save()
+                updated_ct = 1
+            if created:
+                created_ct += 1
+
+    sig_ct = PGPSignature.objects.all().count()
+    logger.info("%d total signatures in database", sig_ct)
+    logger.info("created %d, updated %d signatures", created_ct, updated_ct)
+
+# vim: set ts=4 sw=4 et:
diff --git a/devel/management/commands/rematch_developers.py b/devel/management/commands/rematch_developers.py
index 8383cc8d..2b379588 100644
--- a/devel/management/commands/rematch_developers.py
+++ b/devel/management/commands/rematch_developers.py
@@ -46,52 +46,53 @@ class Command(NoArgsCommand):
 
 @transaction.commit_on_success
 def match_packager(finder):
-    logger.info("getting all unmatched packages")
+    logger.info("getting all unmatched packager strings")
     package_count = matched_count = 0
-    unknown = set()
-
-    for package in Package.objects.filter(packager__isnull=True):
-        if package.packager_str in unknown:
-            continue
-        logger.debug("package %s, packager string %s",
-                package.pkgname, package.packager_str)
-        package_count += 1
-        user = finder.find(package.packager_str)
+    mapping = {}
+
+    unmatched = Package.objects.filter(packager__isnull=True).values_list(
+            'packager_str', flat=True).order_by().distinct()
+
+    logger.info("%d packager strings retrieved", len(unmatched))
+    for packager in unmatched:
+        logger.debug("packager string %s", packager)
+        user = finder.find(packager)
         if user:
-            package.packager = user
+            mapping[packager] = user
             logger.debug("  found user %s" % user.username)
-            package.save()
             matched_count += 1
-        else:
-            unknown.add(package.packager_str)
 
-    logger.info("%d packager strings checked, %d newly matched",
+    for packager_str, user in mapping.items():
+        package_count += Package.objects.filter(packager__isnull=True,
+                packager_str=packager_str).update(packager=user)
+
+    logger.info("%d packages updated, %d packager strings matched",
             package_count, matched_count)
-    logger.debug("unknown packagers:\n%s",
-            "\n".join(unknown))
 
 
 @transaction.commit_on_success
 def match_flagrequest(finder):
-    logger.info("getting all non-user flag requests")
+    logger.info("getting all flag request email addresses from unknown users")
     req_count = matched_count = 0
-    unknown = set()
-
-    for request in FlagRequest.objects.filter(user__isnull=True):
-        if request.user_email in unknown:
-            continue
-        logger.debug("email %s", request.user_email)
-        req_count += 1
-        user = finder.find_by_email(request.user_email)
+    mapping = {}
+
+    unmatched = FlagRequest.objects.filter(user__isnull=True).values_list(
+            'user_email', flat=True).order_by().distinct()
+    
+    logger.info("%d email addresses retrieved", len(unmatched))
+    for user_email in unmatched:
+        logger.debug("email %s", user_email)
+        user = finder.find_by_email(user_email)
         if user:
-            request.user = user
+            mapping[user_email] = user
             logger.debug("  found user %s" % user.username)
-            request.save()
             matched_count += 1
-        else:
-            unknown.add(request.user_email)
 
-    logger.info("%d request emails checked, %d newly matched",
+    for user_email, user in mapping.items():
+        req_count += FlagRequest.objects.filter(user__isnull=True,
+                user_email=user_email).update(user=user)
+
+    logger.info("%d request emails updated, %d emails matched",
             req_count, matched_count)
 
 # vim: set ts=4 sw=4 et:
diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index cf98f004..1e456c8c 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -14,6 +14,7 @@ Example:
 """
 
 from collections import defaultdict
+from copy import copy
 import io
 import os
 import re
@@ -27,11 +28,12 @@ from pytz import utc
 from django.core.management.base import BaseCommand, CommandError
 from django.db import connections, router, transaction
 from django.db.utils import IntegrityError
+from django.utils.timezone import now
 
 from devel.utils import UserFinder
-from main.models import Arch, Package, PackageDepend, PackageFile, Repo
-from main.utils import utc_now
-from packages.models import Conflict, Provision, Replacement
+from main.models import Arch, Package, PackageFile, Repo
+from packages.models import Depend, Conflict, Provision, Replacement, Update
+from packages.utils import parse_version
 
 
 logging.basicConfig(
@@ -78,10 +80,9 @@ class RepoPackage(object):
     bare = ( 'name', 'base', 'arch', 'filename',
             'md5sum', 'sha256sum', 'url', 'packager' )
     number = ( 'csize', 'isize' )
-    collections = ( 'depends', 'optdepends', 'conflicts',
-            'provides', 'replaces', 'groups', 'license', 'files' )
-
-    version_re = re.compile(r'^((\d+):)?(.+)-([^-]+)$')
+    collections = ( 'depends', 'optdepends', 'makedepends', 'checkdepends',
+            'conflicts', 'provides', 'replaces', 'groups', 'license',
+            'files' )
 
     def __init__(self, repo):
         self.repo = repo
@@ -111,11 +112,7 @@ class RepoPackage(object):
                     v[0] = 'missing'
                 setattr(self, k, v[0])
             elif k == 'version':
-                match = self.version_re.match(v[0])
-                self.ver = match.group(3)
-                self.rel = match.group(4)
-                if match.group(2):
-                    self.epoch = int(match.group(2))
+                self.ver, self.rel, self.epoch = parse_version(v[0])
             elif k == 'builddate':
                 try:
                     builddate = datetime.utcfromtimestamp(int(v[0]))
@@ -143,19 +140,21 @@ class RepoPackage(object):
         return u'%s-%s' % (self.ver, self.rel)
 
 
-DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.*))?$")
+DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.+))?$")
 
-def create_depend(package, dep_str, optional=False):
-    depend = PackageDepend(pkg=package, optional=optional)
+def create_depend(package, dep_str, deptype='D'):
+    depend = Depend(pkg=package, deptype=deptype)
     # lop off any description first
     parts = dep_str.split(':', 1)
     if len(parts) > 1:
         depend.description = parts[1].strip()
     match = DEPEND_RE.match(parts[0].strip())
     if match:
-        depend.depname = match.group(1)
-        if match.group(2):
-            depend.depvcmp = match.group(2)
+        depend.name = match.group(1)
+        if match.group(3):
+            depend.comparison = match.group(3)
+        if match.group(4):
+            depend.version = match.group(4)
     else:
         logger.warning('Package %s had unparsable depend string %s',
                 package.pkgname, dep_str)
@@ -183,6 +182,7 @@ def create_related(model, package, rel_str, equals_only=False):
         return None
     return related
 
+
 def create_multivalued(dbpkg, repopkg, db_attr, repo_attr):
     '''Populate the simplest of multivalued attributes. These are those that
     only deal with a 'name' attribute, such as licenses, groups, etc. The input
@@ -236,8 +236,10 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None):
 
     dbpkg.depends.all().delete()
     deps = [create_depend(dbpkg, y) for y in repopkg.depends]
-    deps += [create_depend(dbpkg, y, True) for y in repopkg.optdepends]
-    PackageDepend.objects.bulk_create(deps)
+    deps += [create_depend(dbpkg, y, 'O') for y in repopkg.optdepends]
+    deps += [create_depend(dbpkg, y, 'M') for y in repopkg.makedepends]
+    deps += [create_depend(dbpkg, y, 'C') for y in repopkg.checkdepends]
+    Depend.objects.bulk_create(deps)
 
     dbpkg.conflicts.all().delete()
     conflicts = [create_related(Conflict, dbpkg, y) for y in repopkg.conflicts]
@@ -275,7 +277,7 @@ def populate_files(dbpkg, repopkg, force=False):
             return
         if not dbpkg.files_last_update or not dbpkg.last_update:
             pass
-        elif dbpkg.files_last_update > dbpkg.last_update:
+        elif dbpkg.files_last_update >= dbpkg.last_update:
             return
 
     # only delete files if we are reading a DB that contains them
@@ -284,17 +286,24 @@ def populate_files(dbpkg, repopkg, force=False):
         logger.info("adding %d files for package %s",
                 len(repopkg.files), dbpkg.pkgname)
         pkg_files = []
-        for f in repopkg.files:
-            dirname, filename = f.rsplit('/', 1)
+        # sort in normal alpha-order that pacman uses, rather than makepkg's
+        # default breadth-first, directory-first ordering
+        files = sorted(repopkg.files)
+        for f in files:
+            if '/' in f:
+                dirname, filename = f.rsplit('/', 1)
+                dirname += '/'
+            else:
+                dirname, filename = '', f
             if filename == '':
                 filename = None
             pkgfile = PackageFile(pkg=dbpkg,
                     is_directory=(filename is None),
-                    directory=dirname + '/',
+                    directory=dirname,
                     filename=filename)
             pkg_files.append(pkgfile)
         PackageFile.objects.bulk_create(pkg_files)
-        dbpkg.files_last_update = utc_now()
+        dbpkg.files_last_update = now()
         dbpkg.save()
 
 
@@ -307,7 +316,7 @@ def update_common(archname, reponame, pkgs, sanity_check=True):
         transaction.set_dirty()
 
         repository = Repo.objects.get(name__iexact=reponame)
-        architecture = Arch.objects.get(name__iexact=archname)
+        architecture = Arch.objects.get(name=archname)
         # no-arg order_by() removes even the default ordering; we don't need it
         dbpkgs = Package.objects.filter(
                 arch=architecture, repo=repository).order_by()
@@ -346,35 +355,42 @@ def db_update(archname, reponame, pkgs, force=False):
     logger.info('Updating %s (%s)', reponame, archname)
     dbpkgs = update_common(archname, reponame, pkgs, True)
     repository = Repo.objects.get(name__iexact=reponame)
-    architecture = Arch.objects.get(name__iexact=archname)
+    architecture = Arch.objects.get(name=archname)
 
     # This makes our inner loop where we find packages by name *way* more
     # efficient by not having to go to the database for each package to
     # SELECT them by name.
-    dbdict = dict((dbpkg.pkgname, dbpkg) for dbpkg in dbpkgs)
+    dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs}
 
     dbset = set(dbdict.keys())
-    syncset = set([pkg.name for pkg in pkgs])
+    syncset = {pkg.name for pkg in pkgs}
 
     in_sync_not_db = syncset - dbset
     logger.info("%d packages in sync not db", len(in_sync_not_db))
     # packages in syncdb and not in database (add to database)
     for pkg in (pkg for pkg in pkgs if pkg.name in in_sync_not_db):
         logger.info("Adding package %s", pkg.name)
-        dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository)
+        timestamp = now()
+        dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository,
+                created=timestamp)
         try:
             with transaction.commit_on_success():
-                populate_pkg(dbpkg, pkg, timestamp=utc_now())
+                populate_pkg(dbpkg, pkg, timestamp=timestamp)
+                Update.objects.log_update(None, dbpkg)
         except IntegrityError:
-            logger.warning("Could not add package %s; "
-                    "not fatal if another thread beat us to it.",
-                    pkg.name, exc_info=True)
+            if architecture.agnostic:
+                logger.warning("Could not add package %s; "
+                        "not fatal if another thread beat us to it.",
+                        pkg.name)
+            else:
+                logger.exception("Could not add package %s", pkg.name)
 
     # packages in database and not in syncdb (remove from database)
     for pkgname in (dbset - syncset):
         logger.info("Removing package %s", pkgname)
         dbpkg = dbdict[pkgname]
         with transaction.commit_on_success():
+            Update.objects.log_update(dbpkg, None)
             # no race condition here as long as simultaneous threads both
             # issue deletes; second delete will be a no-op
             delete_pkg_files(dbpkg)
@@ -391,7 +407,7 @@ def db_update(archname, reponame, pkgs, force=False):
         if not force and pkg_same_version(pkg, dbpkg):
             continue
         elif not force:
-            timestamp = utc_now()
+            timestamp = now()
 
         # The odd select_for_update song and dance here are to ensure
         # simultaneous updates don't happen on a package, causing
@@ -402,7 +418,9 @@ def db_update(archname, reponame, pkgs, force=False):
                 logger.debug("Package %s was already updated", pkg.name)
                 continue
             logger.info("Updating package %s", pkg.name)
+            prevpkg = copy(dbpkg)
             populate_pkg(dbpkg, pkg, force=force, timestamp=timestamp)
+            Update.objects.log_update(prevpkg, dbpkg)
 
     logger.info('Finished updating arch: %s', archname)
 
@@ -413,7 +431,7 @@ def filesonly_update(archname, reponame, pkgs, force=False):
     """
     logger.info('Updating files for %s (%s)', reponame, archname)
     dbpkgs = update_common(archname, reponame, pkgs, False)
-    dbdict = dict((dbpkg.pkgname, dbpkg) for dbpkg in dbpkgs)
+    dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs}
     dbset = set(dbdict.keys())
 
     for pkg in (pkg for pkg in pkgs if pkg.name in dbset):
@@ -425,7 +443,7 @@ def filesonly_update(archname, reponame, pkgs, force=False):
         with transaction.commit_on_success():
             if not dbpkg.files_last_update or not dbpkg.last_update:
                 pass
-            elif not force and dbpkg.files_last_update > dbpkg.last_update:
+            elif not force and dbpkg.files_last_update >= dbpkg.last_update:
                 logger.debug("Files for %s are up to date", pkg.name)
                 continue
             dbpkg = Package.objects.select_for_update().get(id=dbpkg.id)
@@ -509,7 +527,7 @@ def locate_arch(arch):
     if isinstance(arch, Arch):
         return arch
     try:
-        return Arch.objects.get(name__iexact=arch)
+        return Arch.objects.get(name=arch)
     except Arch.DoesNotExist:
         raise CommandError(
                 'Specified architecture %s is not currently known.' % arch)
@@ -541,6 +559,12 @@ def read_repo(primary_arch, repo_file, options):
                     package.name, repo_file, package.arch))
     del packages
 
+    database = router.db_for_write(Package)
+    connection = connections[database]
+    if connection.vendor == 'sqlite':
+        cursor = connection.cursor()
+        cursor.execute('PRAGMA synchronous = NORMAL')
+
     logger.info('Starting database updates for %s.', repo_file)
     for arch in sorted(packages_arches.keys()):
         if filesonly:
@@ -548,6 +572,8 @@ def read_repo(primary_arch, repo_file, options):
         else:
             db_update(arch, repo, packages_arches[arch], force)
     logger.info('Finished database updates for %s.', repo_file)
+    connection.commit()
+    connection.close()
     return 0
 
 # vim: set ts=4 sw=4 et:
diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py
index c74762eb..8c1e47bf 100755..100644
--- a/devel/management/commands/reporead_inotify.py
+++ b/devel/management/commands/reporead_inotify.py
@@ -23,7 +23,7 @@ import threading
 import time
 
 from django.core.management.base import BaseCommand, CommandError
-from django.db import connection
+from django.db import connection, transaction
 
 from main.models import Arch, Repo
 from .reporead import read_repo
@@ -53,6 +53,11 @@ class Command(BaseCommand):
         self.path_template = path_template
 
         notifier = self.setup_notifier()
+        # this thread is done using the database; all future access is done in
+        # the spawned read_repo() processes, so close the otherwise completely
+        # idle connection.
+        connection.close()
+
         logger.info('Entering notifier loop')
         notifier.loop()
 
@@ -61,15 +66,18 @@ class Command(BaseCommand):
             if hasattr(thread, 'cancel'):
                 thread.cancel()
 
+    @transaction.commit_on_success
     def setup_notifier(self):
         '''Set up and configure the inotify machinery and logic.
         This takes the provided or default path_template and builds a list of
         directories we need to watch for database updates. It then validates
         and passes these on to the various pyinotify pieces as necessary and
         finally builds and returns a notifier object.'''
+        transaction.commit_manually()
         arches = Arch.objects.filter(agnostic=False)
         repos = Repo.objects.all()
-        arch_path_map = dict((arch, None) for arch in arches)
+        transaction.set_dirty()
+        arch_path_map = {arch: None for arch in arches}
         all_paths = set()
         total_paths = 0
         for arch in arches:
@@ -77,7 +85,7 @@ class Command(BaseCommand):
                     for repo in repos)
             # take a python format string and generate all unique combinations
             # of directories from it; using set() ensures we filter it down
-            paths = set(self.path_template % values for values in combos)
+            paths = {self.path_template % values for values in combos}
             total_paths += len(paths)
             all_paths |= paths
             arch_path_map[arch] = paths
@@ -91,11 +99,6 @@ class Command(BaseCommand):
             raise CommandError('path template did not uniquely '
                     'determine architecture for each file')
 
-        # this thread is done using the database; all future access is done in
-        # the spawned read_repo() processes, so close the otherwise completely
-        # idle connection.
-        connection.close()
-
         # A proper atomic replacement of the database as done by rsync is type
         # IN_MOVED_TO. repo-add/remove will finish with a IN_CLOSE_WRITE.
         mask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO