Merge branch 'archweb' into archweb-generic2

Conflicts:
	devel/views.py
	feeds.py
	public/views.py
	settings.py
	sitestatic/archweb.js
	templates/base.html
	templates/devel/profile.html
	templates/mirrors/status.html
	templates/news/view.html
	templates/packages/flaghelp.html
	templates/packages/opensearch.xml
	templates/public/download.html
	templates/public/feeds.html
	templates/public/index.html
	templates/registration/login.html
	templates/releng/results.html
	templates/todolists/public_list.html

1 files changed, 64 insertions, 38 deletions

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index cf98f004..1e456c8c 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -14,6 +14,7 @@ Example:
 """
 
 from collections import defaultdict
+from copy import copy
 import io
 import os
 import re
@@ -27,11 +28,12 @@ from pytz import utc
 from django.core.management.base import BaseCommand, CommandError
 from django.db import connections, router, transaction
 from django.db.utils import IntegrityError
+from django.utils.timezone import now
 
 from devel.utils import UserFinder
-from main.models import Arch, Package, PackageDepend, PackageFile, Repo
-from main.utils import utc_now
-from packages.models import Conflict, Provision, Replacement
+from main.models import Arch, Package, PackageFile, Repo
+from packages.models import Depend, Conflict, Provision, Replacement, Update
+from packages.utils import parse_version
 
 
 logging.basicConfig(
@@ -78,10 +80,9 @@ class RepoPackage(object):
     bare = ( 'name', 'base', 'arch', 'filename',
             'md5sum', 'sha256sum', 'url', 'packager' )
     number = ( 'csize', 'isize' )
-    collections = ( 'depends', 'optdepends', 'conflicts',
-            'provides', 'replaces', 'groups', 'license', 'files' )
-
-    version_re = re.compile(r'^((\d+):)?(.+)-([^-]+)$')
+    collections = ( 'depends', 'optdepends', 'makedepends', 'checkdepends',
+            'conflicts', 'provides', 'replaces', 'groups', 'license',
+            'files' )
 
     def __init__(self, repo):
         self.repo = repo
@@ -111,11 +112,7 @@ class RepoPackage(object):
                     v[0] = 'missing'
                 setattr(self, k, v[0])
             elif k == 'version':
-                match = self.version_re.match(v[0])
-                self.ver = match.group(3)
-                self.rel = match.group(4)
-                if match.group(2):
-                    self.epoch = int(match.group(2))
+                self.ver, self.rel, self.epoch = parse_version(v[0])
             elif k == 'builddate':
                 try:
                     builddate = datetime.utcfromtimestamp(int(v[0]))
@@ -143,19 +140,21 @@ class RepoPackage(object):
         return u'%s-%s' % (self.ver, self.rel)
 
 
-DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.*))?$")
+DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.+))?$")
 
-def create_depend(package, dep_str, optional=False):
-    depend = PackageDepend(pkg=package, optional=optional)
+def create_depend(package, dep_str, deptype='D'):
+    depend = Depend(pkg=package, deptype=deptype)
     # lop off any description first
     parts = dep_str.split(':', 1)
     if len(parts) > 1:
         depend.description = parts[1].strip()
     match = DEPEND_RE.match(parts[0].strip())
     if match:
-        depend.depname = match.group(1)
-        if match.group(2):
-            depend.depvcmp = match.group(2)
+        depend.name = match.group(1)
+        if match.group(3):
+            depend.comparison = match.group(3)
+        if match.group(4):
+            depend.version = match.group(4)
     else:
         logger.warning('Package %s had unparsable depend string %s',
                 package.pkgname, dep_str)
@@ -183,6 +182,7 @@ def create_related(model, package, rel_str, equals_only=False):
         return None
     return related
 
+
 def create_multivalued(dbpkg, repopkg, db_attr, repo_attr):
     '''Populate the simplest of multivalued attributes. These are those that
     only deal with a 'name' attribute, such as licenses, groups, etc. The input
@@ -236,8 +236,10 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None):
 
     dbpkg.depends.all().delete()
     deps = [create_depend(dbpkg, y) for y in repopkg.depends]
-    deps += [create_depend(dbpkg, y, True) for y in repopkg.optdepends]
-    PackageDepend.objects.bulk_create(deps)
+    deps += [create_depend(dbpkg, y, 'O') for y in repopkg.optdepends]
+    deps += [create_depend(dbpkg, y, 'M') for y in repopkg.makedepends]
+    deps += [create_depend(dbpkg, y, 'C') for y in repopkg.checkdepends]
+    Depend.objects.bulk_create(deps)
 
     dbpkg.conflicts.all().delete()
     conflicts = [create_related(Conflict, dbpkg, y) for y in repopkg.conflicts]
@@ -275,7 +277,7 @@ def populate_files(dbpkg, repopkg, force=False):
             return
         if not dbpkg.files_last_update or not dbpkg.last_update:
             pass
-        elif dbpkg.files_last_update > dbpkg.last_update:
+        elif dbpkg.files_last_update >= dbpkg.last_update:
             return
 
     # only delete files if we are reading a DB that contains them
@@ -284,17 +286,24 @@ def populate_files(dbpkg, repopkg, force=False):
         logger.info("adding %d files for package %s",
                 len(repopkg.files), dbpkg.pkgname)
         pkg_files = []
-        for f in repopkg.files:
-            dirname, filename = f.rsplit('/', 1)
+        # sort in normal alpha-order that pacman uses, rather than makepkg's
+        # default breadth-first, directory-first ordering
+        files = sorted(repopkg.files)
+        for f in files:
+            if '/' in f:
+                dirname, filename = f.rsplit('/', 1)
+                dirname += '/'
+            else:
+                dirname, filename = '', f
             if filename == '':
                 filename = None
             pkgfile = PackageFile(pkg=dbpkg,
                     is_directory=(filename is None),
-                    directory=dirname + '/',
+                    directory=dirname,
                     filename=filename)
             pkg_files.append(pkgfile)
         PackageFile.objects.bulk_create(pkg_files)
-        dbpkg.files_last_update = utc_now()
+        dbpkg.files_last_update = now()
         dbpkg.save()
 
 
@@ -307,7 +316,7 @@ def update_common(archname, reponame, pkgs, sanity_check=True):
         transaction.set_dirty()
 
         repository = Repo.objects.get(name__iexact=reponame)
-        architecture = Arch.objects.get(name__iexact=archname)
+        architecture = Arch.objects.get(name=archname)
         # no-arg order_by() removes even the default ordering; we don't need it
         dbpkgs = Package.objects.filter(
                 arch=architecture, repo=repository).order_by()
@@ -346,35 +355,42 @@ def db_update(archname, reponame, pkgs, force=False):
     logger.info('Updating %s (%s)', reponame, archname)
     dbpkgs = update_common(archname, reponame, pkgs, True)
     repository = Repo.objects.get(name__iexact=reponame)
-    architecture = Arch.objects.get(name__iexact=archname)
+    architecture = Arch.objects.get(name=archname)
 
     # This makes our inner loop where we find packages by name *way* more
     # efficient by not having to go to the database for each package to
     # SELECT them by name.
-    dbdict = dict((dbpkg.pkgname, dbpkg) for dbpkg in dbpkgs)
+    dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs}
 
     dbset = set(dbdict.keys())
-    syncset = set([pkg.name for pkg in pkgs])
+    syncset = {pkg.name for pkg in pkgs}
 
     in_sync_not_db = syncset - dbset
     logger.info("%d packages in sync not db", len(in_sync_not_db))
     # packages in syncdb and not in database (add to database)
     for pkg in (pkg for pkg in pkgs if pkg.name in in_sync_not_db):
         logger.info("Adding package %s", pkg.name)
-        dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository)
+        timestamp = now()
+        dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository,
+                created=timestamp)
         try:
             with transaction.commit_on_success():
-                populate_pkg(dbpkg, pkg, timestamp=utc_now())
+                populate_pkg(dbpkg, pkg, timestamp=timestamp)
+                Update.objects.log_update(None, dbpkg)
         except IntegrityError:
-            logger.warning("Could not add package %s; "
-                    "not fatal if another thread beat us to it.",
-                    pkg.name, exc_info=True)
+            if architecture.agnostic:
+                logger.warning("Could not add package %s; "
+                        "not fatal if another thread beat us to it.",
+                        pkg.name)
+            else:
+                logger.exception("Could not add package %s", pkg.name)
 
     # packages in database and not in syncdb (remove from database)
     for pkgname in (dbset - syncset):
         logger.info("Removing package %s", pkgname)
         dbpkg = dbdict[pkgname]
         with transaction.commit_on_success():
+            Update.objects.log_update(dbpkg, None)
             # no race condition here as long as simultaneous threads both
             # issue deletes; second delete will be a no-op
             delete_pkg_files(dbpkg)
@@ -391,7 +407,7 @@ def db_update(archname, reponame, pkgs, force=False):
         if not force and pkg_same_version(pkg, dbpkg):
             continue
         elif not force:
-            timestamp = utc_now()
+            timestamp = now()
 
         # The odd select_for_update song and dance here are to ensure
         # simultaneous updates don't happen on a package, causing
@@ -402,7 +418,9 @@ def db_update(archname, reponame, pkgs, force=False):
                 logger.debug("Package %s was already updated", pkg.name)
                 continue
             logger.info("Updating package %s", pkg.name)
+            prevpkg = copy(dbpkg)
             populate_pkg(dbpkg, pkg, force=force, timestamp=timestamp)
+            Update.objects.log_update(prevpkg, dbpkg)
 
     logger.info('Finished updating arch: %s', archname)
 
@@ -413,7 +431,7 @@ def filesonly_update(archname, reponame, pkgs, force=False):
     """
     logger.info('Updating files for %s (%s)', reponame, archname)
     dbpkgs = update_common(archname, reponame, pkgs, False)
-    dbdict = dict((dbpkg.pkgname, dbpkg) for dbpkg in dbpkgs)
+    dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs}
     dbset = set(dbdict.keys())
 
     for pkg in (pkg for pkg in pkgs if pkg.name in dbset):
@@ -425,7 +443,7 @@ def filesonly_update(archname, reponame, pkgs, force=False):
         with transaction.commit_on_success():
             if not dbpkg.files_last_update or not dbpkg.last_update:
                 pass
-            elif not force and dbpkg.files_last_update > dbpkg.last_update:
+            elif not force and dbpkg.files_last_update >= dbpkg.last_update:
                 logger.debug("Files for %s are up to date", pkg.name)
                 continue
             dbpkg = Package.objects.select_for_update().get(id=dbpkg.id)
@@ -509,7 +527,7 @@ def locate_arch(arch):
     if isinstance(arch, Arch):
         return arch
     try:
-        return Arch.objects.get(name__iexact=arch)
+        return Arch.objects.get(name=arch)
     except Arch.DoesNotExist:
         raise CommandError(
                 'Specified architecture %s is not currently known.' % arch)
@@ -541,6 +559,12 @@ def read_repo(primary_arch, repo_file, options):
                     package.name, repo_file, package.arch))
     del packages
 
+    database = router.db_for_write(Package)
+    connection = connections[database]
+    if connection.vendor == 'sqlite':
+        cursor = connection.cursor()
+        cursor.execute('PRAGMA synchronous = NORMAL')
+
     logger.info('Starting database updates for %s.', repo_file)
     for arch in sorted(packages_arches.keys()):
         if filesonly:
@@ -548,6 +572,8 @@ def read_repo(primary_arch, repo_file, options):
         else:
             db_update(arch, repo, packages_arches[arch], force)
     logger.info('Finished database updates for %s.', repo_file)
+    connection.commit()
+    connection.close()
     return 0
 
 # vim: set ts=4 sw=4 et: