Merge branch 'archweb-generic2'

Conflicts:
	README.BRANDING
	local_settings.py.example
	packages/templatetags/package_extras.py
	public/views.py
	releng/views.py
	settings.py
	sitestatic/archnavbar/archnavbar.css
	sitestatic/silhouette.png
	templates/base.html
	templates/packages/differences.html
	templates/packages/opensearch.xml
	templates/packages/search.html
	templates/public/donate.html
	templates/public/download.html
	templates/public/feeds.html
	templates/public/index.html
	urls.py

1 files changed, 110 insertions, 67 deletions

diff --git a/packages/utils.py b/packages/utils.py
index 469dc213..e77dbace 100644
--- a/packages/utils.py
+++ b/packages/utils.py
@@ -1,23 +1,42 @@
 from collections import defaultdict
 from itertools import chain
-from operator import itemgetter
+from operator import attrgetter, itemgetter
+import re
 
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db import connection
 from django.db.models import Count, Max, F
+from django.db.models.query import QuerySet
 from django.contrib.auth.models import User
 
-from main.models import Package, PackageDepend, PackageFile, Arch, Repo
-from main.utils import cache_function, groupby_preserve_order, PackageStandin
+from main.models import Package, PackageFile, Arch, Repo
+from main.utils import (database_vendor,
+        groupby_preserve_order, PackageStandin)
 from .models import (PackageGroup, PackageRelation,
-        License, Conflict, Provision, Replacement,
+        License, Depend, Conflict, Provision, Replacement,
         SignoffSpecification, Signoff, DEFAULT_SIGNOFF_SPEC)
 
-@cache_function(127)
+
+VERSION_RE = re.compile(r'^((\d+):)?(.+)-([^-]+)$')
+
+
+def parse_version(version):
+    match = VERSION_RE.match(version)
+    if not match:
+        return None, None, 0
+    ver = match.group(3)
+    rel = match.group(4)
+    if match.group(2):
+        epoch = int(match.group(2))
+    else:
+        epoch = 0
+    return ver, rel, epoch
+
+
 def get_group_info(include_arches=None):
     raw_groups = PackageGroup.objects.values_list(
             'name', 'pkg__arch__name').order_by('name').annotate(
-             cnt=Count('pkg'), last_update=Max('pkg__last_update'))
+            cnt=Count('pkg'), last_update=Max('pkg__last_update'))
     # now for post_processing. we need to seperate things out and add
     # the count in for 'any' to all of the other architectures.
     group_mapping = {}
@@ -52,15 +71,16 @@ def get_group_info(include_arches=None):
             groups.extend(val.itervalues())
     return sorted(groups, key=itemgetter('name', 'arch'))
 
+
 def get_split_packages_info():
     '''Return info on split packages that do not have an actual package name
     matching the split pkgbase.'''
     pkgnames = Package.objects.values('pkgname')
     split_pkgs = Package.objects.exclude(pkgname=F('pkgbase')).exclude(
             pkgbase__in=pkgnames).values('pkgbase', 'repo', 'arch').annotate(
-            last_update=Max('last_update'))
-    all_arches = Arch.objects.in_bulk(set(s['arch'] for s in split_pkgs))
-    all_repos = Repo.objects.in_bulk(set(s['repo'] for s in split_pkgs))
+            last_update=Max('last_update')).order_by().distinct()
+    all_arches = Arch.objects.in_bulk({s['arch'] for s in split_pkgs})
+    all_repos = Repo.objects.in_bulk({s['repo'] for s in split_pkgs})
     for split in split_pkgs:
         split['arch'] = all_arches[split['arch']]
         split['repo'] = all_repos[split['repo']]
@@ -88,13 +108,17 @@ class Difference(object):
             css_classes.append(self.pkg_b.arch.name)
         return ' '.join(css_classes)
 
-    def __cmp__(self, other):
-        if isinstance(other, Difference):
-            return cmp(self.__dict__, other.__dict__)
-        return False
+    def __key(self):
+        return (self.pkgname, hash(self.repo),
+                hash(self.pkg_a), hash(self.pkg_b))
+
+    def __eq__(self, other):
+        return self.__key() == other.__key()
+
+    def __hash__(self):
+        return hash(self.__key())
 
 
-@cache_function(127)
 def get_differences_info(arch_a, arch_b):
     # This is a monster. Join packages against itself, looking for packages in
     # our non-'any' architectures only, and not having a corresponding package
@@ -127,11 +151,11 @@ SELECT p.id, q.id
     cursor.execute(sql, [arch_a.id, arch_b.id, arch_a.id, arch_b.id])
     results = cursor.fetchall()
     # column A will always have a value, column B might be NULL
-    to_fetch = [row[0] for row in results]
+    to_fetch = {row[0] for row in results}
     # fetch all of the necessary packages
     pkgs = Package.objects.normal().in_bulk(to_fetch)
-    # now build a list of tuples containing differences
-    differences = []
+    # now build a set containing differences
+    differences = set()
     for row in results:
         pkg_a = pkgs.get(row[0])
         pkg_b = pkgs.get(row[1])
@@ -144,22 +168,28 @@ SELECT p.id, q.id
             name = pkg_a.pkgname if pkg_a else pkg_b.pkgname
             repo = pkg_a.repo if pkg_a else pkg_b.repo
             item = Difference(name, repo, pkg_b, pkg_a)
-        if item not in differences:
-            differences.append(item)
+        differences.add(item)
 
     # now sort our list by repository, package name
-    differences.sort(key=lambda a: (a.repo.name, a.pkgname))
+    key_func = attrgetter('repo.name', 'pkgname')
+    differences = sorted(differences, key=key_func)
     return differences
 
 
 def multilib_differences():
     # Query for checking multilib out of date-ness
-    sql = """
-SELECT ml.id, reg.id
-    FROM packages ml
-    JOIN packages reg
-    ON (
-        reg.pkgname = (
+    if database_vendor(Package) == 'sqlite':
+        pkgname_sql = """
+            CASE WHEN ml.pkgname LIKE %s
+                THEN SUBSTR(ml.pkgname, 7)
+            WHEN ml.pkgname LIKE %s
+                THEN SUBSTR(ml.pkgname, 1, LENGTH(ml.pkgname) - 9)
+            ELSE
+                ml.pkgname
+            END
+        """
+    else:
+        pkgname_sql = """
             CASE WHEN ml.pkgname LIKE %s
                 THEN SUBSTRING(ml.pkgname, 7)
             WHEN ml.pkgname LIKE %s
@@ -167,7 +197,13 @@ SELECT ml.id, reg.id
             ELSE
                 ml.pkgname
             END
-        )
+        """
+    sql = """
+SELECT ml.id, reg.id
+    FROM packages ml
+    JOIN packages reg
+    ON (
+        reg.pkgname = (""" + pkgname_sql + """)
         AND reg.pkgver != ml.pkgver
     )
     JOIN repos r ON reg.repo_id = r.id
@@ -176,7 +212,7 @@ SELECT ml.id, reg.id
     AND r.staging = %s
     AND reg.arch_id = %s
     ORDER BY ml.last_update
-"""
+    """
     multilib = Repo.objects.get(name__iexact='multilib')
     i686 = Arch.objects.get(name='i686')
     params = ['lib32-%', '%-multilib', multilib.id, False, False, i686.id]
@@ -200,12 +236,13 @@ SELECT DISTINCT id
         FROM packages p
         JOIN packages_packagerelation pr ON p.pkgbase = pr.pkgbase
         WHERE pr.type = %s
-        ) pkgs
-    WHERE pkgs.repo_id NOT IN (
-        SELECT repo_id FROM user_profiles_allowed_repos ar
+    ) mp
+    LEFT JOIN (
+        SELECT user_id, repo_id FROM user_profiles_allowed_repos ar
         INNER JOIN user_profiles up ON ar.userprofile_id = up.id
-        WHERE up.user_id = pkgs.user_id
-    )
+    ) ur
+    ON mp.user_id = ur.user_id AND mp.repo_id = ur.repo_id
+    WHERE ur.user_id IS NULL;
 """
     cursor = connection.cursor()
     cursor.execute(sql, [PackageRelation.MAINTAINER])
@@ -219,13 +256,17 @@ def attach_maintainers(packages):
     '''Given a queryset or something resembling it of package objects, find all
     the maintainers and attach them to the packages to prevent N+1 query
     cascading.'''
-    packages = list(packages)
-    pkgbases = set(p.pkgbase for p in packages)
+    if isinstance(packages, QuerySet):
+        pkgbases = packages.values('pkgbase')
+    else:
+        packages = list(packages)
+        pkgbases = {p.pkgbase for p in packages if p is not None}
     rels = PackageRelation.objects.filter(type=PackageRelation.MAINTAINER,
-            pkgbase__in=pkgbases).values_list('pkgbase', 'user_id').distinct()
+            pkgbase__in=pkgbases).values_list(
+            'pkgbase', 'user_id').order_by().distinct()
 
     # get all the user objects we will need
-    user_ids = set(rel[1] for rel in rels)
+    user_ids = {rel[1] for rel in rels}
     users = User.objects.in_bulk(user_ids)
 
     # now build a pkgbase -> [maintainers...] map
@@ -236,6 +277,8 @@ def attach_maintainers(packages):
     annotated = []
     # and finally, attach the maintainer lists on the original packages
     for package in packages:
+        if package is None:
+            continue
         package.maintainers = maintainers[package.pkgbase]
         annotated.append(package)
 
@@ -248,6 +291,7 @@ def approved_by_signoffs(signoffs, spec):
         return good_signoffs >= spec.required
     return False
 
+
 class PackageSignoffGroup(object):
     '''Encompasses all packages in testing with the same pkgbase.'''
     def __init__(self, packages):
@@ -333,7 +377,9 @@ class PackageSignoffGroup(object):
         return u'%s-%s (%s): %d' % (
                 self.pkgbase, self.version, self.arch, len(self.signoffs))
 
-_SQL_SPEC_OR_SIGNOFF = """
+
+def signoffs_id_query(model, repos):
+    sql = """
 SELECT DISTINCT s.id
     FROM %s s
     JOIN packages p ON (
@@ -344,35 +390,33 @@ SELECT DISTINCT s.id
         AND s.arch_id = p.arch_id
         AND s.repo_id = p.repo_id
     )
-    AND p.repo_id IN (%s)
-"""
-
-def get_current_signoffs(repos):
-    '''Returns a mapping of pkgbase -> signoff objects for the given repos.'''
+    WHERE p.repo_id IN (%s)
+    AND s.repo_id IN (%s)
+    """
     cursor = connection.cursor()
     # query pre-process- fill in table name and placeholders for IN
-    sql = _SQL_SPEC_OR_SIGNOFF % ('packages_signoff',
-            ','.join(['%s' for r in repos]))
-    cursor.execute(sql, [r.pk for r in repos])
+    repo_sql = ','.join(['%s' for _ in repos])
+    sql = sql % (model._meta.db_table, repo_sql, repo_sql)
+    repo_ids = [r.pk for r in repos]
+    # repo_ids are needed twice, so double the array
+    cursor.execute(sql, repo_ids * 2)
 
     results = cursor.fetchall()
-    # fetch all of the returned signoffs by ID
-    to_fetch = [row[0] for row in results]
-    signoffs = Signoff.objects.select_related('user').in_bulk(to_fetch)
-    return signoffs.values()
+    return [row[0] for row in results]
 
-def get_current_specifications(repos):
-    '''Returns a mapping of pkgbase -> signoff specification objects for the
-    given repos.'''
-    cursor = connection.cursor()
-    sql = _SQL_SPEC_OR_SIGNOFF % ('packages_signoffspecification',
-            ','.join(['%s' for r in repos]))
-    cursor.execute(sql, [r.pk for r in repos])
 
-    results = cursor.fetchall()
-    to_fetch = [row[0] for row in results]
+def get_current_signoffs(repos):
+    '''Returns a list of signoff objects for the given repos.'''
+    to_fetch = signoffs_id_query(Signoff, repos)
+    return Signoff.objects.select_related('user').in_bulk(to_fetch).values()
+
+
+def get_current_specifications(repos):
+    '''Returns a list of signoff specification objects for the given repos.'''
+    to_fetch = signoffs_id_query(SignoffSpecification, repos)
     return SignoffSpecification.objects.in_bulk(to_fetch).values()
 
+
 def get_target_repo_map(repos):
     sql = """
 SELECT DISTINCT p1.pkgbase, r.name
@@ -393,6 +437,7 @@ SELECT DISTINCT p1.pkgbase, r.name
     cursor.execute(sql, params)
     return dict(cursor.fetchall())
 
+
 def get_signoff_groups(repos=None, user=None):
     if repos is None:
         repos = Repo.objects.filter(testing=True)
@@ -430,20 +475,19 @@ def get_signoff_groups(repos=None, user=None):
 
 
 class PackageJSONEncoder(DjangoJSONEncoder):
-    pkg_attributes = [ 'pkgname', 'pkgbase', 'repo', 'arch', 'pkgver',
+    pkg_attributes = ['pkgname', 'pkgbase', 'repo', 'arch', 'pkgver',
             'pkgrel', 'epoch', 'pkgdesc', 'url', 'filename', 'compressed_size',
             'installed_size', 'build_date', 'last_update', 'flag_date',
-            'maintainers', 'packager' ]
-    pkg_list_attributes = [ 'groups', 'licenses', 'conflicts',
-            'provides', 'replaces', 'depends' ]
+            'maintainers', 'packager']
+    pkg_list_attributes = ['groups', 'licenses', 'conflicts',
+            'provides', 'replaces', 'depends']
 
     def default(self, obj):
         if hasattr(obj, '__iter__'):
             # mainly for queryset serialization
             return list(obj)
         if isinstance(obj, Package):
-            data = dict((attr, getattr(obj, attr))
-                    for attr in self.pkg_attributes)
+            data = {attr: getattr(obj, attr) for attr in self.pkg_attributes}
             for attr in self.pkg_list_attributes:
                 data[attr] = getattr(obj, attr).all()
             return data
@@ -454,11 +498,10 @@ class PackageJSONEncoder(DjangoJSONEncoder):
             return obj.name.lower()
         if isinstance(obj, (PackageGroup, License)):
             return obj.name
-        if isinstance(obj, (Conflict, Provision, Replacement, PackageDepend)):
+        if isinstance(obj, (Depend, Conflict, Provision, Replacement)):
             return unicode(obj)
         elif isinstance(obj, User):
             return obj.username
         return super(PackageJSONEncoder, self).default(obj)
 
-
 # vim: set ts=4 sw=4 et: