summaryrefslogtreecommitdiff
path: root/packages/utils.py
diff options
context:
space:
mode:
authorLuke Shumaker <LukeShu@sbcglobal.net>2013-04-22 00:36:57 -0400
committerLuke Shumaker <LukeShu@sbcglobal.net>2013-04-22 00:36:57 -0400
commitdf7a6aa620af6a165bdacd755757f8cb1179331c (patch)
tree384b4c62d1f50d8effb733d81d2a810666807624 /packages/utils.py
parent94f972bb892dbf9a86f089f1872ae6d849c0cd0e (diff)
parenta22557811a24b68ef85d4271787c48d8d1e4fc99 (diff)
Merge branch 'archweb-generic2'
Conflicts: README.BRANDING local_settings.py.example packages/templatetags/package_extras.py public/views.py releng/views.py settings.py sitestatic/archnavbar/archnavbar.css sitestatic/silhouette.png templates/base.html templates/packages/differences.html templates/packages/opensearch.xml templates/packages/search.html templates/public/donate.html templates/public/download.html templates/public/feeds.html templates/public/index.html urls.py
Diffstat (limited to 'packages/utils.py')
-rw-r--r--packages/utils.py177
1 files changed, 110 insertions, 67 deletions
diff --git a/packages/utils.py b/packages/utils.py
index 469dc213..e77dbace 100644
--- a/packages/utils.py
+++ b/packages/utils.py
@@ -1,23 +1,42 @@
from collections import defaultdict
from itertools import chain
-from operator import itemgetter
+from operator import attrgetter, itemgetter
+import re
from django.core.serializers.json import DjangoJSONEncoder
from django.db import connection
from django.db.models import Count, Max, F
+from django.db.models.query import QuerySet
from django.contrib.auth.models import User
-from main.models import Package, PackageDepend, PackageFile, Arch, Repo
-from main.utils import cache_function, groupby_preserve_order, PackageStandin
+from main.models import Package, PackageFile, Arch, Repo
+from main.utils import (database_vendor,
+ groupby_preserve_order, PackageStandin)
from .models import (PackageGroup, PackageRelation,
- License, Conflict, Provision, Replacement,
+ License, Depend, Conflict, Provision, Replacement,
SignoffSpecification, Signoff, DEFAULT_SIGNOFF_SPEC)
-@cache_function(127)
+
+VERSION_RE = re.compile(r'^((\d+):)?(.+)-([^-]+)$')
+
+
+def parse_version(version):
+ match = VERSION_RE.match(version)
+ if not match:
+ return None, None, 0
+ ver = match.group(3)
+ rel = match.group(4)
+ if match.group(2):
+ epoch = int(match.group(2))
+ else:
+ epoch = 0
+ return ver, rel, epoch
+
+
def get_group_info(include_arches=None):
raw_groups = PackageGroup.objects.values_list(
'name', 'pkg__arch__name').order_by('name').annotate(
- cnt=Count('pkg'), last_update=Max('pkg__last_update'))
+ cnt=Count('pkg'), last_update=Max('pkg__last_update'))
# now for post_processing. we need to seperate things out and add
# the count in for 'any' to all of the other architectures.
group_mapping = {}
@@ -52,15 +71,16 @@ def get_group_info(include_arches=None):
groups.extend(val.itervalues())
return sorted(groups, key=itemgetter('name', 'arch'))
+
def get_split_packages_info():
'''Return info on split packages that do not have an actual package name
matching the split pkgbase.'''
pkgnames = Package.objects.values('pkgname')
split_pkgs = Package.objects.exclude(pkgname=F('pkgbase')).exclude(
pkgbase__in=pkgnames).values('pkgbase', 'repo', 'arch').annotate(
- last_update=Max('last_update'))
- all_arches = Arch.objects.in_bulk(set(s['arch'] for s in split_pkgs))
- all_repos = Repo.objects.in_bulk(set(s['repo'] for s in split_pkgs))
+ last_update=Max('last_update')).order_by().distinct()
+ all_arches = Arch.objects.in_bulk({s['arch'] for s in split_pkgs})
+ all_repos = Repo.objects.in_bulk({s['repo'] for s in split_pkgs})
for split in split_pkgs:
split['arch'] = all_arches[split['arch']]
split['repo'] = all_repos[split['repo']]
@@ -88,13 +108,17 @@ class Difference(object):
css_classes.append(self.pkg_b.arch.name)
return ' '.join(css_classes)
- def __cmp__(self, other):
- if isinstance(other, Difference):
- return cmp(self.__dict__, other.__dict__)
- return False
+ def __key(self):
+ return (self.pkgname, hash(self.repo),
+ hash(self.pkg_a), hash(self.pkg_b))
+
+ def __eq__(self, other):
+ return self.__key() == other.__key()
+
+ def __hash__(self):
+ return hash(self.__key())
-@cache_function(127)
def get_differences_info(arch_a, arch_b):
# This is a monster. Join packages against itself, looking for packages in
# our non-'any' architectures only, and not having a corresponding package
@@ -127,11 +151,11 @@ SELECT p.id, q.id
cursor.execute(sql, [arch_a.id, arch_b.id, arch_a.id, arch_b.id])
results = cursor.fetchall()
# column A will always have a value, column B might be NULL
- to_fetch = [row[0] for row in results]
+ to_fetch = {row[0] for row in results}
# fetch all of the necessary packages
pkgs = Package.objects.normal().in_bulk(to_fetch)
- # now build a list of tuples containing differences
- differences = []
+ # now build a set containing differences
+ differences = set()
for row in results:
pkg_a = pkgs.get(row[0])
pkg_b = pkgs.get(row[1])
@@ -144,22 +168,28 @@ SELECT p.id, q.id
name = pkg_a.pkgname if pkg_a else pkg_b.pkgname
repo = pkg_a.repo if pkg_a else pkg_b.repo
item = Difference(name, repo, pkg_b, pkg_a)
- if item not in differences:
- differences.append(item)
+ differences.add(item)
# now sort our list by repository, package name
- differences.sort(key=lambda a: (a.repo.name, a.pkgname))
+ key_func = attrgetter('repo.name', 'pkgname')
+ differences = sorted(differences, key=key_func)
return differences
def multilib_differences():
# Query for checking multilib out of date-ness
- sql = """
-SELECT ml.id, reg.id
- FROM packages ml
- JOIN packages reg
- ON (
- reg.pkgname = (
+ if database_vendor(Package) == 'sqlite':
+ pkgname_sql = """
+ CASE WHEN ml.pkgname LIKE %s
+ THEN SUBSTR(ml.pkgname, 7)
+ WHEN ml.pkgname LIKE %s
+ THEN SUBSTR(ml.pkgname, 1, LENGTH(ml.pkgname) - 9)
+ ELSE
+ ml.pkgname
+ END
+ """
+ else:
+ pkgname_sql = """
CASE WHEN ml.pkgname LIKE %s
THEN SUBSTRING(ml.pkgname, 7)
WHEN ml.pkgname LIKE %s
@@ -167,7 +197,13 @@ SELECT ml.id, reg.id
ELSE
ml.pkgname
END
- )
+ """
+ sql = """
+SELECT ml.id, reg.id
+ FROM packages ml
+ JOIN packages reg
+ ON (
+ reg.pkgname = (""" + pkgname_sql + """)
AND reg.pkgver != ml.pkgver
)
JOIN repos r ON reg.repo_id = r.id
@@ -176,7 +212,7 @@ SELECT ml.id, reg.id
AND r.staging = %s
AND reg.arch_id = %s
ORDER BY ml.last_update
-"""
+ """
multilib = Repo.objects.get(name__iexact='multilib')
i686 = Arch.objects.get(name='i686')
params = ['lib32-%', '%-multilib', multilib.id, False, False, i686.id]
@@ -200,12 +236,13 @@ SELECT DISTINCT id
FROM packages p
JOIN packages_packagerelation pr ON p.pkgbase = pr.pkgbase
WHERE pr.type = %s
- ) pkgs
- WHERE pkgs.repo_id NOT IN (
- SELECT repo_id FROM user_profiles_allowed_repos ar
+ ) mp
+ LEFT JOIN (
+ SELECT user_id, repo_id FROM user_profiles_allowed_repos ar
INNER JOIN user_profiles up ON ar.userprofile_id = up.id
- WHERE up.user_id = pkgs.user_id
- )
+ ) ur
+ ON mp.user_id = ur.user_id AND mp.repo_id = ur.repo_id
+ WHERE ur.user_id IS NULL;
"""
cursor = connection.cursor()
cursor.execute(sql, [PackageRelation.MAINTAINER])
@@ -219,13 +256,17 @@ def attach_maintainers(packages):
'''Given a queryset or something resembling it of package objects, find all
the maintainers and attach them to the packages to prevent N+1 query
cascading.'''
- packages = list(packages)
- pkgbases = set(p.pkgbase for p in packages)
+ if isinstance(packages, QuerySet):
+ pkgbases = packages.values('pkgbase')
+ else:
+ packages = list(packages)
+ pkgbases = {p.pkgbase for p in packages if p is not None}
rels = PackageRelation.objects.filter(type=PackageRelation.MAINTAINER,
- pkgbase__in=pkgbases).values_list('pkgbase', 'user_id').distinct()
+ pkgbase__in=pkgbases).values_list(
+ 'pkgbase', 'user_id').order_by().distinct()
# get all the user objects we will need
- user_ids = set(rel[1] for rel in rels)
+ user_ids = {rel[1] for rel in rels}
users = User.objects.in_bulk(user_ids)
# now build a pkgbase -> [maintainers...] map
@@ -236,6 +277,8 @@ def attach_maintainers(packages):
annotated = []
# and finally, attach the maintainer lists on the original packages
for package in packages:
+ if package is None:
+ continue
package.maintainers = maintainers[package.pkgbase]
annotated.append(package)
@@ -248,6 +291,7 @@ def approved_by_signoffs(signoffs, spec):
return good_signoffs >= spec.required
return False
+
class PackageSignoffGroup(object):
'''Encompasses all packages in testing with the same pkgbase.'''
def __init__(self, packages):
@@ -333,7 +377,9 @@ class PackageSignoffGroup(object):
return u'%s-%s (%s): %d' % (
self.pkgbase, self.version, self.arch, len(self.signoffs))
-_SQL_SPEC_OR_SIGNOFF = """
+
+def signoffs_id_query(model, repos):
+ sql = """
SELECT DISTINCT s.id
FROM %s s
JOIN packages p ON (
@@ -344,35 +390,33 @@ SELECT DISTINCT s.id
AND s.arch_id = p.arch_id
AND s.repo_id = p.repo_id
)
- AND p.repo_id IN (%s)
-"""
-
-def get_current_signoffs(repos):
- '''Returns a mapping of pkgbase -> signoff objects for the given repos.'''
+ WHERE p.repo_id IN (%s)
+ AND s.repo_id IN (%s)
+ """
cursor = connection.cursor()
# query pre-process- fill in table name and placeholders for IN
- sql = _SQL_SPEC_OR_SIGNOFF % ('packages_signoff',
- ','.join(['%s' for r in repos]))
- cursor.execute(sql, [r.pk for r in repos])
+ repo_sql = ','.join(['%s' for _ in repos])
+ sql = sql % (model._meta.db_table, repo_sql, repo_sql)
+ repo_ids = [r.pk for r in repos]
+ # repo_ids are needed twice, so double the array
+ cursor.execute(sql, repo_ids * 2)
results = cursor.fetchall()
- # fetch all of the returned signoffs by ID
- to_fetch = [row[0] for row in results]
- signoffs = Signoff.objects.select_related('user').in_bulk(to_fetch)
- return signoffs.values()
+ return [row[0] for row in results]
-def get_current_specifications(repos):
- '''Returns a mapping of pkgbase -> signoff specification objects for the
- given repos.'''
- cursor = connection.cursor()
- sql = _SQL_SPEC_OR_SIGNOFF % ('packages_signoffspecification',
- ','.join(['%s' for r in repos]))
- cursor.execute(sql, [r.pk for r in repos])
- results = cursor.fetchall()
- to_fetch = [row[0] for row in results]
+def get_current_signoffs(repos):
+ '''Returns a list of signoff objects for the given repos.'''
+ to_fetch = signoffs_id_query(Signoff, repos)
+ return Signoff.objects.select_related('user').in_bulk(to_fetch).values()
+
+
+def get_current_specifications(repos):
+ '''Returns a list of signoff specification objects for the given repos.'''
+ to_fetch = signoffs_id_query(SignoffSpecification, repos)
return SignoffSpecification.objects.in_bulk(to_fetch).values()
+
def get_target_repo_map(repos):
sql = """
SELECT DISTINCT p1.pkgbase, r.name
@@ -393,6 +437,7 @@ SELECT DISTINCT p1.pkgbase, r.name
cursor.execute(sql, params)
return dict(cursor.fetchall())
+
def get_signoff_groups(repos=None, user=None):
if repos is None:
repos = Repo.objects.filter(testing=True)
@@ -430,20 +475,19 @@ def get_signoff_groups(repos=None, user=None):
class PackageJSONEncoder(DjangoJSONEncoder):
- pkg_attributes = [ 'pkgname', 'pkgbase', 'repo', 'arch', 'pkgver',
+ pkg_attributes = ['pkgname', 'pkgbase', 'repo', 'arch', 'pkgver',
'pkgrel', 'epoch', 'pkgdesc', 'url', 'filename', 'compressed_size',
'installed_size', 'build_date', 'last_update', 'flag_date',
- 'maintainers', 'packager' ]
- pkg_list_attributes = [ 'groups', 'licenses', 'conflicts',
- 'provides', 'replaces', 'depends' ]
+ 'maintainers', 'packager']
+ pkg_list_attributes = ['groups', 'licenses', 'conflicts',
+ 'provides', 'replaces', 'depends']
def default(self, obj):
if hasattr(obj, '__iter__'):
# mainly for queryset serialization
return list(obj)
if isinstance(obj, Package):
- data = dict((attr, getattr(obj, attr))
- for attr in self.pkg_attributes)
+ data = {attr: getattr(obj, attr) for attr in self.pkg_attributes}
for attr in self.pkg_list_attributes:
data[attr] = getattr(obj, attr).all()
return data
@@ -454,11 +498,10 @@ class PackageJSONEncoder(DjangoJSONEncoder):
return obj.name.lower()
if isinstance(obj, (PackageGroup, License)):
return obj.name
- if isinstance(obj, (Conflict, Provision, Replacement, PackageDepend)):
+ if isinstance(obj, (Depend, Conflict, Provision, Replacement)):
return unicode(obj)
elif isinstance(obj, User):
return obj.username
return super(PackageJSONEncoder, self).default(obj)
-
# vim: set ts=4 sw=4 et: