diff options
Diffstat (limited to 'packages/utils.py')
| -rw-r--r-- | packages/utils.py | 177 |
1 files changed, 110 insertions, 67 deletions
diff --git a/packages/utils.py b/packages/utils.py index a3c13b17..a4217fbd 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -1,23 +1,42 @@ from collections import defaultdict from itertools import chain -from operator import itemgetter +from operator import attrgetter, itemgetter +import re from django.core.serializers.json import DjangoJSONEncoder from django.db import connection from django.db.models import Count, Max, F +from django.db.models.query import QuerySet from django.contrib.auth.models import User -from main.models import Package, PackageDepend, PackageFile, Arch, Repo -from main.utils import cache_function, groupby_preserve_order, PackageStandin +from main.models import Package, PackageFile, Arch, Repo +from main.utils import (database_vendor, + groupby_preserve_order, PackageStandin) from .models import (PackageGroup, PackageRelation, - License, Conflict, Provision, Replacement, + License, Depend, Conflict, Provision, Replacement, SignoffSpecification, Signoff, DEFAULT_SIGNOFF_SPEC) -@cache_function(127) + +VERSION_RE = re.compile(r'^((\d+):)?(.+)-([^-]+)$') + + +def parse_version(version): + match = VERSION_RE.match(version) + if not match: + return None, None, 0 + ver = match.group(3) + rel = match.group(4) + if match.group(2): + epoch = int(match.group(2)) + else: + epoch = 0 + return ver, rel, epoch + + def get_group_info(include_arches=None): raw_groups = PackageGroup.objects.values_list( 'name', 'pkg__arch__name').order_by('name').annotate( - cnt=Count('pkg'), last_update=Max('pkg__last_update')) + cnt=Count('pkg'), last_update=Max('pkg__last_update')) # now for post_processing. we need to seperate things out and add # the count in for 'any' to all of the other architectures. group_mapping = {} @@ -52,15 +71,16 @@ def get_group_info(include_arches=None): groups.extend(val.itervalues()) return sorted(groups, key=itemgetter('name', 'arch')) + def get_split_packages_info(): '''Return info on split packages that do not have an actual package name matching the split pkgbase.''' pkgnames = Package.objects.values('pkgname') split_pkgs = Package.objects.exclude(pkgname=F('pkgbase')).exclude( pkgbase__in=pkgnames).values('pkgbase', 'repo', 'arch').annotate( - last_update=Max('last_update')) - all_arches = Arch.objects.in_bulk(set(s['arch'] for s in split_pkgs)) - all_repos = Repo.objects.in_bulk(set(s['repo'] for s in split_pkgs)) + last_update=Max('last_update')).order_by().distinct() + all_arches = Arch.objects.in_bulk({s['arch'] for s in split_pkgs}) + all_repos = Repo.objects.in_bulk({s['repo'] for s in split_pkgs}) for split in split_pkgs: split['arch'] = all_arches[split['arch']] split['repo'] = all_repos[split['repo']] @@ -88,13 +108,17 @@ class Difference(object): css_classes.append(self.pkg_b.arch.name) return ' '.join(css_classes) - def __cmp__(self, other): - if isinstance(other, Difference): - return cmp(self.__dict__, other.__dict__) - return False + def __key(self): + return (self.pkgname, hash(self.repo), + hash(self.pkg_a), hash(self.pkg_b)) + + def __eq__(self, other): + return self.__key() == other.__key() + + def __hash__(self): + return hash(self.__key()) -@cache_function(127) def get_differences_info(arch_a, arch_b): # This is a monster. Join packages against itself, looking for packages in # our non-'any' architectures only, and not having a corresponding package @@ -123,11 +147,11 @@ SELECT p.id, q.id cursor.execute(sql, [arch_a.id, arch_b.id]) results = cursor.fetchall() # column A will always have a value, column B might be NULL - to_fetch = [row[0] for row in results] + to_fetch = {row[0] for row in results} # fetch all of the necessary packages pkgs = Package.objects.normal().in_bulk(to_fetch) - # now build a list of tuples containing differences - differences = [] + # now build a set containing differences + differences = set() for row in results: pkg_a = pkgs.get(row[0]) pkg_b = pkgs.get(row[1]) @@ -140,22 +164,28 @@ SELECT p.id, q.id name = pkg_a.pkgname if pkg_a else pkg_b.pkgname repo = pkg_a.repo if pkg_a else pkg_b.repo item = Difference(name, repo, pkg_b, pkg_a) - if item not in differences: - differences.append(item) + differences.add(item) # now sort our list by repository, package name - differences.sort(key=lambda a: (a.repo.name, a.pkgname)) + key_func = attrgetter('repo.name', 'pkgname') + differences = sorted(differences, key=key_func) return differences def multilib_differences(): # Query for checking multilib out of date-ness - sql = """ -SELECT ml.id, reg.id - FROM packages ml - JOIN packages reg - ON ( - reg.pkgname = ( + if database_vendor(Package) == 'sqlite': + pkgname_sql = """ + CASE WHEN ml.pkgname LIKE %s + THEN SUBSTR(ml.pkgname, 7) + WHEN ml.pkgname LIKE %s + THEN SUBSTR(ml.pkgname, 1, LENGTH(ml.pkgname) - 9) + ELSE + ml.pkgname + END + """ + else: + pkgname_sql = """ CASE WHEN ml.pkgname LIKE %s THEN SUBSTRING(ml.pkgname, 7) WHEN ml.pkgname LIKE %s @@ -163,7 +193,13 @@ SELECT ml.id, reg.id ELSE ml.pkgname END - ) + """ + sql = """ +SELECT ml.id, reg.id + FROM packages ml + JOIN packages reg + ON ( + reg.pkgname = (""" + pkgname_sql + """) AND reg.pkgver != ml.pkgver ) JOIN repos r ON reg.repo_id = r.id @@ -172,7 +208,7 @@ SELECT ml.id, reg.id AND r.staging = %s AND reg.arch_id = %s ORDER BY ml.last_update -""" + """ multilib = Repo.objects.get(name__iexact='multilib') i686 = Arch.objects.get(name='i686') params = ['lib32-%', '%-multilib', multilib.id, False, False, i686.id] @@ -196,12 +232,13 @@ SELECT DISTINCT id FROM packages p JOIN packages_packagerelation pr ON p.pkgbase = pr.pkgbase WHERE pr.type = %s - ) pkgs - WHERE pkgs.repo_id NOT IN ( - SELECT repo_id FROM user_profiles_allowed_repos ar + ) mp + LEFT JOIN ( + SELECT user_id, repo_id FROM user_profiles_allowed_repos ar INNER JOIN user_profiles up ON ar.userprofile_id = up.id - WHERE up.user_id = pkgs.user_id - ) + ) ur + ON mp.user_id = ur.user_id AND mp.repo_id = ur.repo_id + WHERE ur.user_id IS NULL; """ cursor = connection.cursor() cursor.execute(sql, [PackageRelation.MAINTAINER]) @@ -215,13 +252,17 @@ def attach_maintainers(packages): '''Given a queryset or something resembling it of package objects, find all the maintainers and attach them to the packages to prevent N+1 query cascading.''' - packages = list(packages) - pkgbases = set(p.pkgbase for p in packages) + if isinstance(packages, QuerySet): + pkgbases = packages.values('pkgbase') + else: + packages = list(packages) + pkgbases = {p.pkgbase for p in packages if p is not None} rels = PackageRelation.objects.filter(type=PackageRelation.MAINTAINER, - pkgbase__in=pkgbases).values_list('pkgbase', 'user_id').distinct() + pkgbase__in=pkgbases).values_list( + 'pkgbase', 'user_id').order_by().distinct() # get all the user objects we will need - user_ids = set(rel[1] for rel in rels) + user_ids = {rel[1] for rel in rels} users = User.objects.in_bulk(user_ids) # now build a pkgbase -> [maintainers...] map @@ -232,6 +273,8 @@ def attach_maintainers(packages): annotated = [] # and finally, attach the maintainer lists on the original packages for package in packages: + if package is None: + continue package.maintainers = maintainers[package.pkgbase] annotated.append(package) @@ -244,6 +287,7 @@ def approved_by_signoffs(signoffs, spec): return good_signoffs >= spec.required return False + class PackageSignoffGroup(object): '''Encompasses all packages in testing with the same pkgbase.''' def __init__(self, packages): @@ -329,7 +373,9 @@ class PackageSignoffGroup(object): return u'%s-%s (%s): %d' % ( self.pkgbase, self.version, self.arch, len(self.signoffs)) -_SQL_SPEC_OR_SIGNOFF = """ + +def signoffs_id_query(model, repos): + sql = """ SELECT DISTINCT s.id FROM %s s JOIN packages p ON ( @@ -340,35 +386,33 @@ SELECT DISTINCT s.id AND s.arch_id = p.arch_id AND s.repo_id = p.repo_id ) - AND p.repo_id IN (%s) -""" - -def get_current_signoffs(repos): - '''Returns a mapping of pkgbase -> signoff objects for the given repos.''' + WHERE p.repo_id IN (%s) + AND s.repo_id IN (%s) + """ cursor = connection.cursor() # query pre-process- fill in table name and placeholders for IN - sql = _SQL_SPEC_OR_SIGNOFF % ('packages_signoff', - ','.join(['%s' for r in repos])) - cursor.execute(sql, [r.pk for r in repos]) + repo_sql = ','.join(['%s' for r in repos]) + sql = sql % (model._meta.db_table, repo_sql, repo_sql) + repo_ids = [r.pk for r in repos] + # repo_ids are needed twice, so double the array + cursor.execute(sql, repo_ids * 2) results = cursor.fetchall() - # fetch all of the returned signoffs by ID - to_fetch = [row[0] for row in results] - signoffs = Signoff.objects.select_related('user').in_bulk(to_fetch) - return signoffs.values() + return [row[0] for row in results] -def get_current_specifications(repos): - '''Returns a mapping of pkgbase -> signoff specification objects for the - given repos.''' - cursor = connection.cursor() - sql = _SQL_SPEC_OR_SIGNOFF % ('packages_signoffspecification', - ','.join(['%s' for r in repos])) - cursor.execute(sql, [r.pk for r in repos]) - results = cursor.fetchall() - to_fetch = [row[0] for row in results] +def get_current_signoffs(repos): + '''Returns a list of signoff objects for the given repos.''' + to_fetch = signoffs_id_query(Signoff, repos) + return Signoff.objects.select_related('user').in_bulk(to_fetch).values() + + +def get_current_specifications(repos): + '''Returns a list of signoff specification objects for the given repos.''' + to_fetch = signoffs_id_query(SignoffSpecification, repos) return SignoffSpecification.objects.in_bulk(to_fetch).values() + def get_target_repo_map(repos): sql = """ SELECT DISTINCT p1.pkgbase, r.name @@ -389,6 +433,7 @@ SELECT DISTINCT p1.pkgbase, r.name cursor.execute(sql, params) return dict(cursor.fetchall()) + def get_signoff_groups(repos=None, user=None): if repos is None: repos = Repo.objects.filter(testing=True) @@ -426,20 +471,19 @@ def get_signoff_groups(repos=None, user=None): class PackageJSONEncoder(DjangoJSONEncoder): - pkg_attributes = [ 'pkgname', 'pkgbase', 'repo', 'arch', 'pkgver', + pkg_attributes = ['pkgname', 'pkgbase', 'repo', 'arch', 'pkgver', 'pkgrel', 'epoch', 'pkgdesc', 'url', 'filename', 'compressed_size', 'installed_size', 'build_date', 'last_update', 'flag_date', - 'maintainers', 'packager' ] - pkg_list_attributes = [ 'groups', 'licenses', 'conflicts', - 'provides', 'replaces', 'depends' ] + 'maintainers', 'packager'] + pkg_list_attributes = ['groups', 'licenses', 'conflicts', + 'provides', 'replaces', 'depends'] def default(self, obj): if hasattr(obj, '__iter__'): # mainly for queryset serialization return list(obj) if isinstance(obj, Package): - data = dict((attr, getattr(obj, attr)) - for attr in self.pkg_attributes) + data = {attr: getattr(obj, attr) for attr in self.pkg_attributes} for attr in self.pkg_list_attributes: data[attr] = getattr(obj, attr).all() return data @@ -450,11 +494,10 @@ class PackageJSONEncoder(DjangoJSONEncoder): return obj.name.lower() if isinstance(obj, (PackageGroup, License)): return obj.name - if isinstance(obj, (Conflict, Provision, Replacement, PackageDepend)): + if isinstance(obj, (Depend, Conflict, Provision, Replacement)): return unicode(obj) elif isinstance(obj, User): return obj.username return super(PackageJSONEncoder, self).default(obj) - # vim: set ts=4 sw=4 et: |