From cf8ecdf9fce0573ad207d024708f21a5dbbbb120 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 2 May 2012 09:46:52 -0500 Subject: rematch_developers: do mass updates instead of single saves When updating a lot of objects, it makes much more sense to perform targeted update queries rather than one-row-at-a-time saves. Signed-off-by: Dan McGee --- devel/management/commands/rematch_developers.py | 61 ++++++++++++------------- 1 file changed, 30 insertions(+), 31 deletions(-) (limited to 'devel/management') diff --git a/devel/management/commands/rematch_developers.py b/devel/management/commands/rematch_developers.py index 8383cc8d..ab2f0f4b 100644 --- a/devel/management/commands/rematch_developers.py +++ b/devel/management/commands/rematch_developers.py @@ -46,52 +46,51 @@ def handle_noargs(self, **options): @transaction.commit_on_success def match_packager(finder): - logger.info("getting all unmatched packages") + logger.info("getting all unmatched packager strings") package_count = matched_count = 0 - unknown = set() - - for package in Package.objects.filter(packager__isnull=True): - if package.packager_str in unknown: - continue - logger.debug("package %s, packager string %s", - package.pkgname, package.packager_str) - package_count += 1 - user = finder.find(package.packager_str) + mapping = {} + + unmatched = Package.objects.filter(packager__isnull=True).values_list( + 'packager_str', flat=True).order_by().distinct() + + for packager in unmatched: + logger.debug("packager string %s", packager) + user = finder.find(packager) if user: - package.packager = user + mapping[packager] = user logger.debug(" found user %s" % user.username) - package.save() matched_count += 1 - else: - unknown.add(package.packager_str) - logger.info("%d packager strings checked, %d newly matched", + for packager_str, user in mapping.items(): + package_count += Package.objects.filter(packager__isnull=True, + packager_str=packager_str).update(packager=user) + + logger.info("%d packages updated, %d packager strings matched", package_count, matched_count) - logger.debug("unknown packagers:\n%s", - "\n".join(unknown)) @transaction.commit_on_success def match_flagrequest(finder): - logger.info("getting all non-user flag requests") + logger.info("getting all flag requests emails from unknown users") req_count = matched_count = 0 - unknown = set() - - for request in FlagRequest.objects.filter(user__isnull=True): - if request.user_email in unknown: - continue - logger.debug("email %s", request.user_email) - req_count += 1 - user = finder.find_by_email(request.user_email) + mapping = {} + + unmatched = FlagRequest.objects.filter(user__isnull=True).values_list( + 'user_email', flat=True).order_by().distinct() + + for user_email in unmatched: + logger.debug("email %s", user_email) + user = finder.find_by_email(user_email) if user: - request.user = user + mapping[user_email] = user logger.debug(" found user %s" % user.username) - request.save() matched_count += 1 - else: - unknown.add(request.user_email) - logger.info("%d request emails checked, %d newly matched", + for user_email, user in mapping.items(): + req_count += FlagRequest.objects.filter(user__isnull=True, + user_email=user_email).update(user=user) + + logger.info("%d request emails updated, %d emails matched", req_count, matched_count) # vim: set ts=4 sw=4 et: -- cgit v1.2.3-54-g00ecf From 72a92102df4999dbcc370064707c9026d51c4fe7 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 18 May 2012 21:29:03 -0500 Subject: Switch to usage of new Depend object Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 18 +++++++++------- devel/views.py | 6 +++--- main/models.py | 7 +++--- packages/models.py | 34 ++++++++++++++++++++++++++---- packages/utils.py | 6 +++--- templates/packages/details_depend.html | 6 +++--- templates/packages/details_requiredby.html | 2 +- 7 files changed, 54 insertions(+), 25 deletions(-) (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index fd8e3979..47294d9a 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -29,9 +29,9 @@ from django.db.utils import IntegrityError from devel.utils import UserFinder -from main.models import Arch, Package, PackageDepend, PackageFile, Repo +from main.models import Arch, Package, PackageFile, Repo from main.utils import utc_now -from packages.models import Conflict, Provision, Replacement +from packages.models import Depend, Conflict, Provision, Replacement logging.basicConfig( @@ -141,19 +141,21 @@ def full_version(self): return u'%s-%s' % (self.ver, self.rel) -DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.*))?$") +DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.+))?$") def create_depend(package, dep_str, optional=False): - depend = PackageDepend(pkg=package, optional=optional) + depend = Depend(pkg=package, optional=optional) # lop off any description first parts = dep_str.split(':', 1) if len(parts) > 1: depend.description = parts[1].strip() match = DEPEND_RE.match(parts[0].strip()) if match: - depend.depname = match.group(1) - if match.group(2): - depend.depvcmp = match.group(2) + depend.name = match.group(1) + if match.group(3): + depend.comparison = match.group(3) + if match.group(4): + related.version = match.group(4) else: logger.warning('Package %s had unparsable depend string %s', package.pkgname, dep_str) @@ -232,7 +234,7 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): dbpkg.depends.all().delete() deps = [create_depend(dbpkg, y) for y in repopkg.depends] deps += [create_depend(dbpkg, y, True) for y in repopkg.optdepends] - PackageDepend.objects.bulk_create(deps) + Depend.objects.bulk_create(deps) dbpkg.conflicts.all().delete() conflicts = [create_related(Conflict, dbpkg, y) for y in repopkg.conflicts] diff --git a/devel/views.py b/devel/views.py index 0f1c8d15..16b6acc6 100644 --- a/devel/views.py +++ b/devel/views.py @@ -26,11 +26,11 @@ from django.utils.http import http_date from .models import UserProfile -from main.models import Package, PackageDepend, PackageFile, TodolistPkg +from main.models import Package, PackageFile, TodolistPkg from main.models import Arch, Repo from main.utils import utc_now from news.models import News -from packages.models import PackageRelation, Signoff +from packages.models import PackageRelation, Signoff, Depend from packages.utils import get_signoff_groups from todolists.utils import get_annotated_todolists from .utils import get_annotated_maintainers, UserFinder @@ -267,7 +267,7 @@ def report(request, report_name, username=None): elif report_name == 'unneeded-orphans': title = 'Orphan packages required by no other packages' owned = PackageRelation.objects.all().values('pkgbase') - required = PackageDepend.objects.all().values('depname') + required = Depend.objects.all().values('name') # The two separate calls to exclude is required to do the right thing packages = packages.exclude(pkgbase__in=owned).exclude( pkgname__in=required) diff --git a/main/models.py b/main/models.py index 4b445dd0..f17d4a4d 100644 --- a/main/models.py +++ b/main/models.py @@ -180,11 +180,12 @@ def get_requiredby(self): list slim by including the corresponding package in the same testing category as this package if that check makes sense. """ + from packages.models import Depend provides = set(self.provides.values_list('name', flat=True)) provides.add(self.pkgname) - requiredby = PackageDepend.objects.select_related('pkg', + requiredby = Depend.objects.select_related('pkg', 'pkg__arch', 'pkg__repo').filter( - depname__in=provides).order_by( + name__in=provides).order_by( 'pkg__pkgname', 'pkg__arch__name', 'pkg__repo__name') if not self.arch.agnostic: # make sure we match architectures if possible @@ -232,7 +233,7 @@ def get_depends(self): deps = [] arches = None # TODO: we can use list comprehension and an 'in' query to make this more effective - for dep in self.depends.order_by('optional', 'depname'): + for dep in self.depends.order_by('optional', 'name'): pkg = dep.get_best_satisfier() providers = None if not pkg: diff --git a/packages/models.py b/packages/models.py index c7b1cab4..cb65f1f1 100644 --- a/packages/models.py +++ b/packages/models.py @@ -228,10 +228,6 @@ def get_best_satisfier(self): '''Find a satisfier for this related package that best matches the given criteria. It will not search provisions, but will find packages named and matching repo characteristics if possible.''' - # NOTE: this is cribbed directly from the PackageDepend method of the - # same name. Really, all of these things could use the same method if - # the PackageDepend class was moved here and field names were changed - # to match the layout we use here. pkgs = Package.objects.normal().filter(pkgname=self.name) if not self.pkg.arch.agnostic: # make sure we match architectures if possible @@ -258,6 +254,36 @@ def get_best_satisfier(self): return pkg + def get_providers(self): + '''Return providers of this related package. Does *not* include exact + matches as it checks the Provision names only, use get_best_satisfier() + instead for exact matches.''' + pkgs = Package.objects.normal().filter( + provides__name=self.name).order_by().distinct() + if not self.pkg.arch.agnostic: + # make sure we match architectures if possible + arches = self.pkg.applicable_arches() + pkgs = pkgs.filter(arch__in=arches) + + # Logic here is to filter out packages that are in multiple repos if + # they are not requested. For example, if testing is False, only show a + # testing package if it doesn't exist in a non-testing repo. + filtered = {} + for package in pkgs: + if package.pkgname not in filtered or \ + package.repo.staging == self.pkg.repo.staging: + filtered[package.pkgname] = package + pkgs = filtered.values() + + filtered = {} + for package in pkgs: + if package.pkgname not in filtered or \ + package.repo.testing == self.pkg.repo.testing: + filtered[package.pkgname] = package + pkgs = filtered.values() + + return pkgs + def __unicode__(self): if self.version: return u'%s%s%s' % (self.name, self.comparison, self.version) diff --git a/packages/utils.py b/packages/utils.py index 8d00bd68..82313472 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -7,10 +7,10 @@ from django.db.models import Count, Max, F from django.contrib.auth.models import User -from main.models import Package, PackageDepend, PackageFile, Arch, Repo +from main.models import Package, PackageFile, Arch, Repo from main.utils import cache_function, groupby_preserve_order, PackageStandin from .models import (PackageGroup, PackageRelation, - License, Conflict, Provision, Replacement, + License, Depend, Conflict, Provision, Replacement, SignoffSpecification, Signoff, DEFAULT_SIGNOFF_SPEC) @cache_function(127) @@ -451,7 +451,7 @@ def default(self, obj): return obj.name.lower() if isinstance(obj, (PackageGroup, License)): return obj.name - if isinstance(obj, (Conflict, Provision, Replacement, PackageDepend)): + if isinstance(obj, (Depend, Conflict, Provision, Replacement)): return unicode(obj) elif isinstance(obj, User): return obj.username diff --git a/templates/packages/details_depend.html b/templates/packages/details_depend.html index 8b6e85c9..0cf2c36a 100644 --- a/templates/packages/details_depend.html +++ b/templates/packages/details_depend.html @@ -2,12 +2,12 @@
  • {% ifequal depend.pkg None %} {% if depend.providers %} -{{ depend.dep.depname }} ({% multi_pkg_details depend.providers %}) +{{ depend.dep.name }}{{ depend.dep.comparison|default:"" }}{{ depend.dep.version|default:"" }} ({% multi_pkg_details depend.providers %}) {% else %} -{{ depend.dep.depname }} (virtual) +{{ depend.dep.name }}{{ depend.dep.comparison|default:"" }}{{ depend.dep.version|default:"" }} (virtual) {% endif %} {% else %} -{% pkg_details_link depend.pkg %}{{ depend.dep.depvcmp|default:"" }} +{% pkg_details_link depend.pkg %}{{ depend.dep.comparison|default:"" }}{{ depend.dep.version|default:"" }} {% if depend.pkg.repo.testing %} (testing){% endif %} {% if depend.pkg.repo.staging %} (staging){% endif %} {% endifequal %} diff --git a/templates/packages/details_requiredby.html b/templates/packages/details_requiredby.html index c7697289..ecc92b29 100644 --- a/templates/packages/details_requiredby.html +++ b/templates/packages/details_requiredby.html @@ -1,6 +1,6 @@ {% load package_extras %}
  • {% pkg_details_link req.pkg %} -{% if req.depname != pkg.pkgname %}(requires {{ req.depname }}){% endif %} +{% if req.name != pkg.pkgname %}(requires {{ req.name }}){% endif %} {% if req.pkg.repo.testing %}(testing){% endif %} {% if req.pkg.repo.staging %}(staging){% endif %} {% if req.optional %}(optional){% endif %} -- cgit v1.2.3-54-g00ecf From 3f9aeb45c2a2b498a389bacc92b5f56d9feb4329 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 19 May 2012 09:54:15 -0500 Subject: reporead: fix copy/paste issue Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 47294d9a..2e8c4625 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -155,7 +155,7 @@ def create_depend(package, dep_str, optional=False): if match.group(3): depend.comparison = match.group(3) if match.group(4): - related.version = match.group(4) + depend.version = match.group(4) else: logger.warning('Package %s had unparsable depend string %s', package.pkgname, dep_str) -- cgit v1.2.3-54-g00ecf From a87fe016d1a1bf7fdcd2b19f515aa72a5b93db2b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 1 Jul 2012 20:21:34 -0500 Subject: Log package updates during reporead invocation This adds a Manager and log_update method to help log all updates made to the packages table during reporead runs. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 7 ++++++- packages/models.py | 36 +++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 2e8c4625..4e242af1 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -14,6 +14,7 @@ """ from collections import defaultdict +from copy import copy import io import os import re @@ -31,7 +32,7 @@ from devel.utils import UserFinder from main.models import Arch, Package, PackageFile, Repo from main.utils import utc_now -from packages.models import Depend, Conflict, Provision, Replacement +from packages.models import Depend, Conflict, Provision, Replacement, Update logging.basicConfig( @@ -362,6 +363,7 @@ def db_update(archname, reponame, pkgs, force=False): try: with transaction.commit_on_success(): populate_pkg(dbpkg, pkg, timestamp=utc_now()) + Update.objects.log_update(None, dbpkg) except IntegrityError: logger.warning("Could not add package %s; " "not fatal if another thread beat us to it.", @@ -372,6 +374,7 @@ def db_update(archname, reponame, pkgs, force=False): logger.info("Removing package %s", pkgname) dbpkg = dbdict[pkgname] with transaction.commit_on_success(): + Update.objects.log_update(dbpkg, None) # no race condition here as long as simultaneous threads both # issue deletes; second delete will be a no-op delete_pkg_files(dbpkg) @@ -399,7 +402,9 @@ def db_update(archname, reponame, pkgs, force=False): logger.debug("Package %s was already updated", pkg.name) continue logger.info("Updating package %s", pkg.name) + prevpkg = copy(dbpkg) populate_pkg(dbpkg, pkg, force=force, timestamp=timestamp) + Update.objects.log_update(prevpkg, dbpkg) logger.info('Finished updating arch: %s', archname) diff --git a/packages/models.py b/packages/models.py index 5ee06575..04f35f9d 100644 --- a/packages/models.py +++ b/packages/models.py @@ -203,6 +203,40 @@ def __unicode__(self): ) +class UpdateManager(models.Manager): + def log_update(self, old_pkg, new_pkg): + '''Utility method to help log an update. This will determine the type + based on how many packages are passed in, and will pull the relevant + necesary fields off the given packages.''' + update = Update() + if new_pkg: + update.action_flag = ADDITION + update.package = new_pkg + update.arch = new_pkg.arch + update.repo = new_pkg.repo + update.pkgname = new_pkg.pkgname + update.pkgbase = new_pkg.pkgbase + update.new_pkgver = new_pkg.pkgver + update.new_pkgrel = new_pkg.pkgrel + update.new_epoch = new_pkg.epoch + if old_pkg: + if new_pkg: + update.action_flag = CHANGE + else: + update.action_flag = DELETION + update.arch = old_pkg.arch + update.repo = old_pkg.repo + update.pkgname = old_pkg.pkgname + update.pkgbase = old_pkg.pkgbase + + update.old_pkgver = old_pkg.pkgver + update.old_pkgrel = old_pkg.pkgrel + update.old_epoch = old_pkg.epoch + + update.save(force_insert=True) + return update + + class Update(models.Model): package = models.ForeignKey(Package, related_name="updates", null=True, on_delete=models.SET_NULL) @@ -222,6 +256,8 @@ class Update(models.Model): new_pkgrel = models.CharField(max_length=255, null=True) new_epoch = models.PositiveIntegerField(null=True) + objects = UpdateManager() + class Meta: get_latest_by = 'created' -- cgit v1.2.3-54-g00ecf From daf011b67a338f26ead8058a9f9caedfe251c62c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 5 Jul 2012 11:25:00 -0400 Subject: reporead: properly handle cases where last_update == files_last_update We should assume the filelists are up to date in this case, not out of date. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 4e242af1..51c73c02 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -273,7 +273,7 @@ def populate_files(dbpkg, repopkg, force=False): return if not dbpkg.files_last_update or not dbpkg.last_update: pass - elif dbpkg.files_last_update > dbpkg.last_update: + elif dbpkg.files_last_update >= dbpkg.last_update: return # only delete files if we are reading a DB that contains them @@ -427,7 +427,7 @@ def filesonly_update(archname, reponame, pkgs, force=False): with transaction.commit_on_success(): if not dbpkg.files_last_update or not dbpkg.last_update: pass - elif not force and dbpkg.files_last_update > dbpkg.last_update: + elif not force and dbpkg.files_last_update >= dbpkg.last_update: logger.debug("Files for %s are up to date", pkg.name) continue dbpkg = Package.objects.select_for_update().get(id=dbpkg.id) -- cgit v1.2.3-54-g00ecf From 9d91cad678133e97345111fab2c103fcda9b9f28 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 5 Jul 2012 11:25:40 -0400 Subject: reporead: handle files in root directory properly Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 51c73c02..df29a8a7 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -283,7 +283,10 @@ def populate_files(dbpkg, repopkg, force=False): len(repopkg.files), dbpkg.pkgname) pkg_files = [] for f in repopkg.files: - dirname, filename = f.rsplit('/', 1) + if '/' in f: + dirname, filename = f.rsplit('/', 1) + else: + dirname, filename = '', f if filename == '': filename = None pkgfile = PackageFile(pkg=dbpkg, -- cgit v1.2.3-54-g00ecf From 909cb9a209b4a4db00232b3a62656f95c4b88d45 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 5 Jul 2012 17:09:55 -0500 Subject: reporead: don't append slash to empty (root) directory Add the slash only if we have a directory name, and not otherwise. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index df29a8a7..43578d4a 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -285,13 +285,14 @@ def populate_files(dbpkg, repopkg, force=False): for f in repopkg.files: if '/' in f: dirname, filename = f.rsplit('/', 1) + dirname += '/' else: dirname, filename = '', f if filename == '': filename = None pkgfile = PackageFile(pkg=dbpkg, is_directory=(filename is None), - directory=dirname + '/', + directory=dirname, filename=filename) pkg_files.append(pkgfile) PackageFile.objects.bulk_create(pkg_files) -- cgit v1.2.3-54-g00ecf From a1ec14fc68282d67c00c79b5aa6aab60461f056a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 12 Mar 2012 13:14:49 -0400 Subject: reporead: disable FULL synchronous writes for sqlite3 At least on Linux, we hit a huge bottleneck waiting for the FULL commit to happen for each added package during reporead operations. It makes much more sense to back this off to FULL level instead, which trades some possible loss of durability for speedier operation. Additionally, no one would possibly be running their production version of this site on sqlite3, right? Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 43578d4a..e50686b1 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -547,6 +547,12 @@ def read_repo(primary_arch, repo_file, options): package.name, repo_file, package.arch)) del packages + database = router.db_for_write(Package) + connection = connections[database] + if connection.vendor == 'sqlite': + cursor = connection.cursor() + cursor.execute('PRAGMA synchronous = NORMAL') + logger.info('Starting database updates for %s.', repo_file) for arch in sorted(packages_arches.keys()): if filesonly: -- cgit v1.2.3-54-g00ecf From 88ee61a39ac3690267f2b7903f3646972e8f055d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 8 Jul 2012 21:24:48 -0500 Subject: Work around bulk_create limitations in sqlite3 in reporead Given the 999 SQL statement variable limit, we can easily hit it when updating a package with thousands of files or a few hundred depends. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index e50686b1..2d9b68b2 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -31,7 +31,7 @@ from devel.utils import UserFinder from main.models import Arch, Package, PackageFile, Repo -from main.utils import utc_now +from main.utils import utc_now, database_vendor from packages.models import Depend, Conflict, Provision, Replacement, Update @@ -184,6 +184,28 @@ def create_related(model, package, rel_str, equals_only=False): return None return related + +def batched_bulk_create(model, all_objects): + # for short lists, just bulk_create as we should be fine + if len(all_objects) < 20: + return model.objects.bulk_create(all_objects) + + if database_vendor(model, mode='write') == 'sqlite': + # 999 max variables in each SQL statement + incr = 999 // len(model._meta.fields) + else: + incr = 1000 + + def chunks(): + offset = 0 + while offset < len(all_objects): + yield all_objects[offset:offset + incr] + offset += incr + + for items in chunks(): + model.objects.bulk_create(items) + + def create_multivalued(dbpkg, repopkg, db_attr, repo_attr): '''Populate the simplest of multivalued attributes. These are those that only deal with a 'name' attribute, such as licenses, groups, etc. The input @@ -235,20 +257,20 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): dbpkg.depends.all().delete() deps = [create_depend(dbpkg, y) for y in repopkg.depends] deps += [create_depend(dbpkg, y, True) for y in repopkg.optdepends] - Depend.objects.bulk_create(deps) + batched_bulk_create(Depend, deps) dbpkg.conflicts.all().delete() conflicts = [create_related(Conflict, dbpkg, y) for y in repopkg.conflicts] - Conflict.objects.bulk_create(conflicts) + batched_bulk_create(Conflict, conflicts) dbpkg.provides.all().delete() provides = [create_related(Provision, dbpkg, y, equals_only=True) for y in repopkg.provides] - Provision.objects.bulk_create(provides) + batched_bulk_create(Provision, provides) dbpkg.replaces.all().delete() replaces = [create_related(Replacement, dbpkg, y) for y in repopkg.replaces] - Replacement.objects.bulk_create(replaces) + batched_bulk_create(Replacement, replaces) create_multivalued(dbpkg, repopkg, 'groups', 'groups') create_multivalued(dbpkg, repopkg, 'licenses', 'license') @@ -295,7 +317,7 @@ def populate_files(dbpkg, repopkg, force=False): directory=dirname, filename=filename) pkg_files.append(pkgfile) - PackageFile.objects.bulk_create(pkg_files) + batched_bulk_create(PackageFile, pkg_files) dbpkg.files_last_update = utc_now() dbpkg.save() -- cgit v1.2.3-54-g00ecf From c0bf9e20660cfae7ea8994472555bba23398b598 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 24 Jul 2012 09:19:48 -0500 Subject: Remove custom utc_now() function, use django.utils.timezone.now() This was around from the time when we handled timezones sanely and Django did not; now that we are on 1.4 we no longer need our own code to handle this. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 9 +++++---- devel/models.py | 5 +++-- devel/views.py | 12 ++++++------ main/models.py | 5 +++-- main/utils.py | 9 ++------- mirrors/management/commands/mirrorcheck.py | 7 ++++--- mirrors/utils.py | 17 +++++++++-------- news/models.py | 11 +++++------ packages/management/commands/signoff_report.py | 8 ++++---- packages/views/flag.py | 8 ++++---- packages/views/signoff.py | 4 ++-- releng/management/commands/syncisos.py | 5 ++--- 12 files changed, 49 insertions(+), 51 deletions(-) (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 2d9b68b2..e69691db 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -28,10 +28,11 @@ from django.core.management.base import BaseCommand, CommandError from django.db import connections, router, transaction from django.db.utils import IntegrityError +from django.utils.timezone import now from devel.utils import UserFinder from main.models import Arch, Package, PackageFile, Repo -from main.utils import utc_now, database_vendor +from main.utils import database_vendor from packages.models import Depend, Conflict, Provision, Replacement, Update @@ -318,7 +319,7 @@ def populate_files(dbpkg, repopkg, force=False): filename=filename) pkg_files.append(pkgfile) batched_bulk_create(PackageFile, pkg_files) - dbpkg.files_last_update = utc_now() + dbpkg.files_last_update = now() dbpkg.save() @@ -388,7 +389,7 @@ def db_update(archname, reponame, pkgs, force=False): dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository) try: with transaction.commit_on_success(): - populate_pkg(dbpkg, pkg, timestamp=utc_now()) + populate_pkg(dbpkg, pkg, timestamp=now()) Update.objects.log_update(None, dbpkg) except IntegrityError: logger.warning("Could not add package %s; " @@ -417,7 +418,7 @@ def db_update(archname, reponame, pkgs, force=False): if not force and pkg_same_version(pkg, dbpkg): continue elif not force: - timestamp = utc_now() + timestamp = now() # The odd select_for_update song and dance here are to ensure # simultaneous updates don't happen on a package, causing diff --git a/devel/models.py b/devel/models.py index fd5df00a..9b6f07a7 100644 --- a/devel/models.py +++ b/devel/models.py @@ -4,10 +4,11 @@ from django.db import models from django.db.models.signals import pre_save from django.contrib.auth.models import User +from django.utils.timezone import now from django_countries import CountryField from .fields import PGPKeyField -from main.utils import make_choice, utc_now +from main.utils import make_choice class UserProfile(models.Model): @@ -105,7 +106,7 @@ def set_last_modified(sender, **kwargs): signal handler.''' obj = kwargs['instance'] if hasattr(obj, 'last_modified'): - obj.last_modified = utc_now() + obj.last_modified = now() # connect signals needed to keep cache in line with reality diff --git a/devel/views.py b/devel/views.py index 78ed26f2..143b12bf 100644 --- a/devel/views.py +++ b/devel/views.py @@ -24,11 +24,11 @@ from django.views.generic.simple import direct_to_template from django.utils.encoding import force_unicode from django.utils.http import http_date +from django.utils.timezone import now from .models import UserProfile from main.models import Package, PackageFile, TodolistPkg from main.models import Arch, Repo -from main.utils import utc_now from news.models import News from packages.models import PackageRelation, Signoff, Depend from packages.utils import get_signoff_groups @@ -122,15 +122,15 @@ def clock(request): else: dev.last_action = None - now = utc_now() + current_time = now() page_dict = { 'developers': devs, - 'utc_now': now, + 'utc_now': current_time, } response = direct_to_template(request, 'devel/clock.html', page_dict) if not response.has_header('Expires'): - expire_time = now.replace(second=0, microsecond=0) + expire_time = current_time.replace(second=0, microsecond=0) expire_time += timedelta(minutes=1) expire_time = time.mktime(expire_time.timetuple()) response['Expires'] = http_date(expire_time) @@ -198,12 +198,12 @@ def report(request, report_name, username=None): if report_name == 'old': title = 'Packages last built more than one year ago' - cutoff = utc_now() - timedelta(days=365) + cutoff = now() - timedelta(days=365) packages = packages.filter( build_date__lt=cutoff).order_by('build_date') elif report_name == 'long-out-of-date': title = 'Packages marked out-of-date more than 90 days ago' - cutoff = utc_now() - timedelta(days=90) + cutoff = now() - timedelta(days=90) packages = packages.filter( flag_date__lt=cutoff).order_by('flag_date') elif report_name == 'big': diff --git a/main/models.py b/main/models.py index 04d8da8f..6c9dfe4d 100644 --- a/main/models.py +++ b/main/models.py @@ -6,9 +6,10 @@ from django.db import models from django.contrib.auth.models import User from django.contrib.sites.models import Site +from django.utils.timezone import now from .fields import PositiveBigIntegerField -from .utils import cache_function, set_created_field, utc_now +from .utils import cache_function, set_created_field class TodolistManager(models.Manager): @@ -385,7 +386,7 @@ class Meta: def set_todolist_fields(sender, **kwargs): todolist = kwargs['instance'] if not todolist.date_added: - todolist.date_added = utc_now() + todolist.date_added = now() # connect signals needed to keep cache in line with reality from main.utils import refresh_latest diff --git a/main/utils.py b/main/utils.py index 879abfb9..0b6849a4 100644 --- a/main/utils.py +++ b/main/utils.py @@ -5,10 +5,10 @@ from datetime import datetime import hashlib -from pytz import utc from django.core.cache import cache from django.db import connections, router +from django.utils.timezone import now CACHE_TIMEOUT = 1800 @@ -94,17 +94,12 @@ def retrieve_latest(sender, latest_by=None): return None -def utc_now(): - '''Returns a timezone-aware UTC date representing now.''' - return datetime.utcnow().replace(tzinfo=utc) - - def set_created_field(sender, **kwargs): '''This will set the 'created' field on any object to the current UTC time if it is unset. For use as a pre_save signal handler.''' obj = kwargs['instance'] if hasattr(obj, 'created') and not obj.created: - obj.created = utc_now() + obj.created = now() def database_vendor(model, mode='read'): diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 7a133cbf..e09ea680 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -29,8 +29,9 @@ from django.core.management.base import NoArgsCommand from django.db import transaction +from django.utils.timezone import now -from main.utils import utc_now, database_vendor +from main.utils import database_vendor from mirrors.models import MirrorUrl, MirrorLog logging.basicConfig( @@ -83,7 +84,7 @@ def parse_lastsync(log, data): def check_mirror_url(mirror_url, timeout): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) - log = MirrorLog(url=mirror_url, check_time=utc_now()) + log = MirrorLog(url=mirror_url, check_time=now()) headers = {'User-Agent': 'archweb/1.0'} req = urllib2.Request(url, None, headers) try: @@ -136,7 +137,7 @@ def check_mirror_url(mirror_url, timeout): def check_rsync_url(mirror_url, timeout): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) - log = MirrorLog(url=mirror_url, check_time=utc_now()) + log = MirrorLog(url=mirror_url, check_time=now()) tempdir = tempfile.mkdtemp() lastsync_path = os.path.join(tempdir, 'lastsync') diff --git a/mirrors/utils.py b/mirrors/utils.py index f2c98ee0..bf030d39 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,13 +1,14 @@ from datetime import timedelta from django.db.models import Avg, Count, Max, Min, StdDev +from django.utils.timezone import now from django_countries.fields import Country -from main.utils import cache_function, utc_now, database_vendor +from main.utils import cache_function, database_vendor from .models import MirrorLog, MirrorProtocol, MirrorUrl -default_cutoff = timedelta(hours=24) +DEFAULT_CUTOFF = timedelta(hours=24) def annotate_url(url, delays): '''Given a MirrorURL object, add a few more attributes to it regarding @@ -30,8 +31,8 @@ def annotate_url(url, delays): @cache_function(123) -def get_mirror_statuses(cutoff=default_cutoff): - cutoff_time = utc_now() - cutoff +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): + cutoff_time = now() - cutoff # I swear, this actually has decent performance... urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( mirror__active=True, mirror__public=True, @@ -88,8 +89,8 @@ def get_mirror_statuses(cutoff=default_cutoff): @cache_function(117) -def get_mirror_errors(cutoff=default_cutoff): - cutoff_time = utc_now() - cutoff +def get_mirror_errors(cutoff=DEFAULT_CUTOFF): + cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, url__mirror__active=True, url__mirror__public=True).values( @@ -105,11 +106,11 @@ def get_mirror_errors(cutoff=default_cutoff): @cache_function(295) -def get_mirror_url_for_download(cutoff=default_cutoff): +def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF): '''Find a good mirror URL to use for package downloads. If we have mirror status data available, it is used to determine a good choice by looking at the last batch of status rows.''' - cutoff_time = utc_now() - cutoff + cutoff_time = now() - cutoff status_data = MirrorLog.objects.filter( check_time__gte=cutoff_time).aggregate( Max('check_time'), Max('last_sync')) diff --git a/news/models.py b/news/models.py index 95026e1d..2efea579 100644 --- a/news/models.py +++ b/news/models.py @@ -1,8 +1,7 @@ from django.db import models from django.contrib.auth.models import User from django.contrib.sites.models import Site - -from main.utils import utc_now +from django.utils.timezone import now class News(models.Model): @@ -29,13 +28,13 @@ class Meta: def set_news_fields(sender, **kwargs): news = kwargs['instance'] - now = utc_now() - news.last_modified = now + current_time = now() + news.last_modified = current_time if not news.postdate: - news.postdate = now + news.postdate = current_time # http://diveintomark.org/archives/2004/05/28/howto-atom-id news.guid = 'tag:%s,%s:%s' % (Site.objects.get_current(), - now.strftime('%Y-%m-%d'), news.get_absolute_url()) + current_time.strftime('%Y-%m-%d'), news.get_absolute_url()) # connect signals needed to keep cache in line with reality from main.utils import refresh_latest diff --git a/packages/management/commands/signoff_report.py b/packages/management/commands/signoff_report.py index ddf930db..72fcbe1e 100644 --- a/packages/management/commands/signoff_report.py +++ b/packages/management/commands/signoff_report.py @@ -15,6 +15,7 @@ from django.contrib.sites.models import Site from django.db.models import Count from django.template import loader, Context +from django.utils.timezone import now from collections import namedtuple from datetime import timedelta @@ -23,7 +24,6 @@ import sys from main.models import Repo -from main.utils import utc_now from packages.models import Signoff from packages.utils import get_signoff_groups @@ -66,9 +66,9 @@ def generate_report(email, repo_name): new_hours = 24 old_days = 14 - now = utc_now() - new_cutoff = now - timedelta(hours=new_hours) - old_cutoff = now - timedelta(days=old_days) + current_time = now() + new_cutoff = current_time - timedelta(hours=new_hours) + old_cutoff = current_time - timedelta(days=old_days) if len(signoff_groups) == 0: # no need to send an email at all diff --git a/packages/views/flag.py b/packages/views/flag.py index 7fa2d508..f3db93b3 100644 --- a/packages/views/flag.py +++ b/packages/views/flag.py @@ -5,12 +5,12 @@ from django.db import transaction from django.shortcuts import get_object_or_404, redirect from django.template import loader, Context +from django.utils.timezone import now from django.views.generic.simple import direct_to_template from django.views.decorators.cache import cache_page, never_cache from ..models import FlagRequest from main.models import Package -from main.utils import utc_now class FlagForm(forms.Form): @@ -76,10 +76,10 @@ def flag(request, name, repo, arch): @transaction.commit_on_success def perform_updates(): - now = utc_now() - pkgs.update(flag_date=now) + current_time = now() + pkgs.update(flag_date=current_time) # store our flag request - flag_request = FlagRequest(created=now, + flag_request = FlagRequest(created=current_time, user_email=email, message=message, ip_address=ip_addr, pkgbase=pkg.pkgbase, version=version, repo=pkg.repo, diff --git a/packages/views/signoff.py b/packages/views/signoff.py index 61d949fc..7aa39106 100644 --- a/packages/views/signoff.py +++ b/packages/views/signoff.py @@ -8,11 +8,11 @@ from django.db import transaction from django.http import HttpResponse, Http404 from django.shortcuts import get_list_or_404, redirect, render +from django.utils.timezone import now from django.views.decorators.cache import never_cache from django.views.generic.simple import direct_to_template from main.models import Package, Arch, Repo -from main.utils import utc_now from ..models import SignoffSpecification, Signoff from ..utils import (get_signoff_groups, approved_by_signoffs, PackageSignoffGroup) @@ -45,7 +45,7 @@ def signoff_package(request, name, repo, arch, revoke=False): package, request.user, False) except Signoff.DoesNotExist: raise Http404 - signoff.revoked = utc_now() + signoff.revoked = now() signoff.save() created = False else: diff --git a/releng/management/commands/syncisos.py b/releng/management/commands/syncisos.py index 62f005ff..223c771b 100644 --- a/releng/management/commands/syncisos.py +++ b/releng/management/commands/syncisos.py @@ -4,8 +4,8 @@ from django.conf import settings from django.core.management.base import BaseCommand, CommandError +from django.utils.timezone import now -from main.utils import utc_now from releng.models import Iso @@ -54,9 +54,8 @@ def handle(self, *args, **options): existing.active = True existing.removed = None existing.save() - now = utc_now() # and then mark all other names as no longer active Iso.objects.filter(active=True).exclude(name__in=active_isos).update( - active=False, removed=now) + active=False, removed=now()) # vim: set ts=4 sw=4 et: -- cgit v1.2.3-54-g00ecf From 280c53eec5661252b5692fa374292c4d421e3bd8 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 28 Jul 2012 11:45:15 -0500 Subject: reporead: don't use iexact lookup on arch name We don't do this anywhere else, so we shouldn't do this here either. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index e69691db..aaa9812e 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -332,7 +332,7 @@ def update_common(archname, reponame, pkgs, sanity_check=True): transaction.set_dirty() repository = Repo.objects.get(name__iexact=reponame) - architecture = Arch.objects.get(name__iexact=archname) + architecture = Arch.objects.get(name=archname) # no-arg order_by() removes even the default ordering; we don't need it dbpkgs = Package.objects.filter( arch=architecture, repo=repository).order_by() @@ -371,7 +371,7 @@ def db_update(archname, reponame, pkgs, force=False): logger.info('Updating %s (%s)', reponame, archname) dbpkgs = update_common(archname, reponame, pkgs, True) repository = Repo.objects.get(name__iexact=reponame) - architecture = Arch.objects.get(name__iexact=archname) + architecture = Arch.objects.get(name=archname) # This makes our inner loop where we find packages by name *way* more # efficient by not having to go to the database for each package to @@ -538,7 +538,7 @@ def locate_arch(arch): if isinstance(arch, Arch): return arch try: - return Arch.objects.get(name__iexact=arch) + return Arch.objects.get(name=arch) except Arch.DoesNotExist: raise CommandError( 'Specified architecture %s is not currently known.' % arch) -- cgit v1.2.3-54-g00ecf From 3f0c024754047d92e8ce4aa4ecf93a06865f8448 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 31 Jul 2012 18:37:30 -0500 Subject: PGP key handling updates * Import signatures for all known keys, not just active developers * Ensure we are only showing and accounting for active developers on the master keys page * Add a new table showing signatures between developers Signed-off-by: Dan McGee --- devel/management/commands/generate_keyring.py | 2 +- main/templatetags/pgp.py | 8 +++++++ public/views.py | 18 +++++++++++----- templates/public/keys.html | 30 +++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 6 deletions(-) (limited to 'devel/management') diff --git a/devel/management/commands/generate_keyring.py b/devel/management/commands/generate_keyring.py index b9117c84..15ae488d 100644 --- a/devel/management/commands/generate_keyring.py +++ b/devel/management/commands/generate_keyring.py @@ -48,7 +48,7 @@ def generate_keyring(keyserver, keyring): logger.info("getting all known key IDs") # Screw you Django, for not letting one natively do value != - key_ids = UserProfile.objects.filter(user__is_active=True, + key_ids = UserProfile.objects.filter( pgp_key__isnull=False).extra(where=["pgp_key != ''"]).values_list( "pgp_key", flat=True) logger.info("%d keys fetched from user profiles", len(key_ids)) diff --git a/main/templatetags/pgp.py b/main/templatetags/pgp.py index 50b1aa17..5c9fe511 100644 --- a/main/templatetags/pgp.py +++ b/main/templatetags/pgp.py @@ -39,6 +39,14 @@ def pgp_key_link(key_id, link_text=None): values = (url, format_key(key_id), link_text) return '%s' % values +@register.simple_tag +def user_pgp_key_link(users, key_id): + matched = [user for user in users if user.userprofile.pgp_key and + user.userprofile.pgp_key[-16:] == key_id[-16:]] + if matched and len(matched) == 1: + return pgp_key_link(key_id, matched[0].get_full_name()) + return pgp_key_link(key_id) + @register.filter def pgp_fingerprint(key_id, autoescape=True): if not key_id: diff --git a/public/views.py b/public/views.py index c8854b72..312cb3b2 100644 --- a/public/views.py +++ b/public/views.py @@ -91,30 +91,38 @@ def feeds(request): @cache_control(max_age=300) def keys(request): + users = User.objects.filter(is_active=True).select_related( + 'userprofile__pgp_key').order_by('first_name', 'last_name') + user_key_ids = frozenset(user.userprofile.pgp_key[-16:] for user in users + if user.userprofile.pgp_key) + not_expired = Q(expires__gt=datetime.utcnow) | Q(expires__isnull=True) master_keys = MasterKey.objects.select_related('owner', 'revoker', 'owner__userprofile', 'revoker__userprofile').filter( revoked__isnull=True) + master_key_ids = frozenset(key.pgp_key[-16:] for key in master_keys) - sig_counts = PGPSignature.objects.filter( - not_expired, valid=True).values_list('signer').annotate( + sig_counts = PGPSignature.objects.filter(not_expired, valid=True, + signee__in=user_key_ids).values_list('signer').annotate( Count('signer')) sig_counts = dict((key_id[-16:], ct) for key_id, ct in sig_counts) for key in master_keys: key.signature_count = sig_counts.get(key.pgp_key[-16:], 0) - users = User.objects.filter(is_active=True).select_related( - 'userprofile__pgp_key').order_by('first_name', 'last_name') - # frozenset because we are going to do lots of __contains__ lookups signatures = frozenset(PGPSignature.objects.filter( not_expired, valid=True).values_list('signer', 'signee')) + restrict = Q(signer__in=user_key_ids) & Q(signee__in=user_key_ids) + cross_signatures = PGPSignature.objects.filter(restrict, + not_expired, valid=True).order_by('created') + context = { 'keys': master_keys, 'active_users': users, 'signatures': signatures, + 'cross_signatures': cross_signatures, } return render(request, 'public/keys.html', context) diff --git a/templates/public/keys.html b/templates/public/keys.html index 1fed3c15..1b027202 100644 --- a/templates/public/keys.html +++ b/templates/public/keys.html @@ -89,6 +89,33 @@

    Master Signing Keys

    + +
    +

    Developer Cross-Signatures

    + +

    This table lists signatures directly between developer keys.

    + + + + + + + + + + + + {% for sig in cross_signatures %} + + + + + + + {% endfor %} + +
    SignerSigneeCreatedExpires
    {% user_pgp_key_link active_users sig.signer %}{% user_pgp_key_link active_users sig.signee %}{{ sig.created }}{{ sig.expires|default:"" }}
    +
    {% load cdn %}{% jquery %}{% jquery_tablesorter %} {% endblock %} -- cgit v1.2.3-54-g00ecf From a64bbbd4139d91cbbca10d804067cbd87a95872d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 31 Jul 2012 20:27:43 -0500 Subject: Make adjustments for optional -> deptype conversion Very little dealt directly with this field. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 6 +- main/models.py | 9 +- packages/migrations/0021_migrate_optional_deps.py | 210 ++++++++++++++++++++++ templates/packages/details_depend.html | 2 +- templates/packages/details_requiredby.html | 2 +- 5 files changed, 222 insertions(+), 7 deletions(-) create mode 100644 packages/migrations/0021_migrate_optional_deps.py (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index aaa9812e..a3bf3e0c 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -145,8 +145,8 @@ def full_version(self): DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.+))?$") -def create_depend(package, dep_str, optional=False): - depend = Depend(pkg=package, optional=optional) +def create_depend(package, dep_str, deptype='D'): + depend = Depend(pkg=package, deptype=deptype) # lop off any description first parts = dep_str.split(':', 1) if len(parts) > 1: @@ -257,7 +257,7 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): dbpkg.depends.all().delete() deps = [create_depend(dbpkg, y) for y in repopkg.depends] - deps += [create_depend(dbpkg, y, True) for y in repopkg.optdepends] + deps += [create_depend(dbpkg, y, 'O') for y in repopkg.optdepends] batched_bulk_create(Depend, deps) dbpkg.conflicts.all().delete() diff --git a/main/models.py b/main/models.py index 577f11c6..f4ced350 100644 --- a/main/models.py +++ b/main/models.py @@ -245,13 +245,18 @@ def get_depends(self): deps = [] arches = None # TODO: we can use list comprehension and an 'in' query to make this more effective - for dep in self.depends.order_by('optional', 'name'): + for dep in self.depends.all(): pkg = dep.get_best_satisfier() providers = None if not pkg: providers = dep.get_providers() deps.append({'dep': dep, 'pkg': pkg, 'providers': providers}) - return deps + # sort the list; deptype sorting makes this tricker than expected + sort_order = {'D': 0, 'O': 1, 'M': 2, 'C': 3} + def sort_key(val): + dep = val['dep'] + return (sort_order.get(dep.deptype, 1000), dep.name) + return sorted(deps, key=sort_key) @cache_function(125) def base_package(self): diff --git a/packages/migrations/0021_migrate_optional_deps.py b/packages/migrations/0021_migrate_optional_deps.py new file mode 100644 index 00000000..f6652ce1 --- /dev/null +++ b/packages/migrations/0021_migrate_optional_deps.py @@ -0,0 +1,210 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import DataMigration +from django.db import models + +class Migration(DataMigration): + + def forwards(self, orm): + orm['packages.Depend'].objects.filter(optional=False).update(deptype='D') + orm['packages.Depend'].objects.filter(optional=True).update(deptype='O') + + def backwards(self, orm): + orm['packages.Depend'].objects.all().update(deptype='D') + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'main.arch': { + 'Meta': {'ordering': "['name']", 'object_name': 'Arch', 'db_table': "'arches'"}, + 'agnostic': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'main.package': { + 'Meta': {'ordering': "('pkgname',)", 'unique_together': "(('pkgname', 'repo', 'arch'),)", 'object_name': 'Package', 'db_table': "'packages'"}, + 'arch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'on_delete': 'models.PROTECT', 'to': "orm['main.Arch']"}), + 'build_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'compressed_size': ('main.fields.PositiveBigIntegerField', [], {}), + 'epoch': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'files_last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'flag_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'installed_size': ('main.fields.PositiveBigIntegerField', [], {}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'packager': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'on_delete': 'models.SET_NULL'}), + 'packager_str': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pgp_signature': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkgdesc': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'pkgname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'repo': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'on_delete': 'models.PROTECT', 'to': "orm['main.Repo']"}), + 'url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}) + }, + 'main.repo': { + 'Meta': {'ordering': "['name']", 'object_name': 'Repo', 'db_table': "'repos'"}, + 'bugs_category': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'bugs_project': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'staging': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'svn_root': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + 'packages.conflict': { + 'Meta': {'ordering': "['name']", 'object_name': 'Conflict'}, + 'comparison': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'conflicts'", 'to': "orm['main.Package']"}), + 'version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}) + }, + 'packages.depend': { + 'Meta': {'ordering': "['name']", 'object_name': 'Depend'}, + 'comparison': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}), + 'deptype': ('django.db.models.fields.CharField', [], {'default': "'D'", 'max_length': '1'}), + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'optional': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'depends'", 'to': "orm['main.Package']"}), + 'version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}) + }, + 'packages.flagrequest': { + 'Meta': {'object_name': 'FlagRequest'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip_address': ('django.db.models.fields.GenericIPAddressField', [], {'max_length': '39'}), + 'is_legitimate': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_spam': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'message': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'num_packages': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'repo': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Repo']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'}), + 'user_email': ('django.db.models.fields.EmailField', [], {'max_length': '75'}), + 'version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}) + }, + 'packages.license': { + 'Meta': {'ordering': "['name']", 'object_name': 'License'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'licenses'", 'to': "orm['main.Package']"}) + }, + 'packages.packagegroup': { + 'Meta': {'object_name': 'PackageGroup'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'groups'", 'to': "orm['main.Package']"}) + }, + 'packages.packagerelation': { + 'Meta': {'unique_together': "(('pkgbase', 'user', 'type'),)", 'object_name': 'PackageRelation'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'type': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'package_relations'", 'to': "orm['auth.User']"}) + }, + 'packages.provision': { + 'Meta': {'ordering': "['name']", 'object_name': 'Provision'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'provides'", 'to': "orm['main.Package']"}), + 'version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}) + }, + 'packages.replacement': { + 'Meta': {'ordering': "['name']", 'object_name': 'Replacement'}, + 'comparison': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'replaces'", 'to': "orm['main.Package']"}), + 'version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}) + }, + 'packages.signoff': { + 'Meta': {'object_name': 'Signoff'}, + 'arch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Arch']"}), + 'comments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'epoch': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'repo': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Repo']"}), + 'revoked': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'package_signoffs'", 'to': "orm['auth.User']"}) + }, + 'packages.signoffspecification': { + 'Meta': {'object_name': 'SignoffSpecification'}, + 'arch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Arch']"}), + 'comments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'epoch': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'known_bad': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'repo': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Repo']"}), + 'required': ('django.db.models.fields.PositiveIntegerField', [], {'default': '2'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True'}) + }, + 'packages.update': { + 'Meta': {'object_name': 'Update'}, + 'action_flag': ('django.db.models.fields.PositiveSmallIntegerField', [], {}), + 'arch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'updates'", 'to': "orm['main.Arch']"}), + 'created': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'new_epoch': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True'}), + 'new_pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), + 'new_pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), + 'old_epoch': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True'}), + 'old_pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), + 'old_pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'updates'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['main.Package']"}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgname': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'repo': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'updates'", 'to': "orm['main.Repo']"}) + } + } + + complete_apps = ['packages'] + symmetrical = True diff --git a/templates/packages/details_depend.html b/templates/packages/details_depend.html index 0cf2c36a..1eb35474 100644 --- a/templates/packages/details_depend.html +++ b/templates/packages/details_depend.html @@ -11,6 +11,6 @@ {% if depend.pkg.repo.testing %} (testing){% endif %} {% if depend.pkg.repo.staging %} (staging){% endif %} {% endifequal %} -{% if depend.dep.optional %} (optional){% endif %} +{% if depend.dep.deptype == 'O' %} (optional){% endif %} {% if depend.dep.description %}- {{ depend.dep.description }}{% endif %}
    • diff --git a/templates/packages/details_requiredby.html b/templates/packages/details_requiredby.html index ecc92b29..15c62c61 100644 --- a/templates/packages/details_requiredby.html +++ b/templates/packages/details_requiredby.html @@ -3,5 +3,5 @@ {% if req.name != pkg.pkgname %}(requires {{ req.name }}){% endif %} {% if req.pkg.repo.testing %}(testing){% endif %} {% if req.pkg.repo.staging %}(staging){% endif %} -{% if req.optional %}(optional){% endif %} +{% if req.deptype == 'O' %}(optional){% endif %} -- cgit v1.2.3-54-g00ecf From 1f2466fffceafebfaca34e3ed2d34de6b622768b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 31 Jul 2012 20:35:50 -0500 Subject: reporead: import make and check depends We don't have these in the database yet, but future verisons of repo-add will put this information in the sync databases. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index a3bf3e0c..8b55b09a 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -80,8 +80,9 @@ class RepoPackage(object): bare = ( 'name', 'base', 'arch', 'filename', 'md5sum', 'sha256sum', 'url', 'packager' ) number = ( 'csize', 'isize' ) - collections = ( 'depends', 'optdepends', 'conflicts', - 'provides', 'replaces', 'groups', 'license', 'files' ) + collections = ( 'depends', 'optdepends', 'makedepends', 'checkdepends', + 'conflicts', 'provides', 'replaces', 'groups', 'license', + 'files' ) version_re = re.compile(r'^((\d+):)?(.+)-([^-]+)$') @@ -258,6 +259,8 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): dbpkg.depends.all().delete() deps = [create_depend(dbpkg, y) for y in repopkg.depends] deps += [create_depend(dbpkg, y, 'O') for y in repopkg.optdepends] + deps += [create_depend(dbpkg, y, 'M') for y in repopkg.makedepends] + deps += [create_depend(dbpkg, y, 'C') for y in repopkg.checkdepends] batched_bulk_create(Depend, deps) dbpkg.conflicts.all().delete() -- cgit v1.2.3-54-g00ecf From 241ff8fbd79f9f17cd326a34eb39096851f630ba Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 8 Aug 2012 22:07:06 -0500 Subject: Extract parse_version function from reporead logic Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 9 ++------- packages/utils.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 7 deletions(-) (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 8b55b09a..af0a2dc0 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -34,6 +34,7 @@ from main.models import Arch, Package, PackageFile, Repo from main.utils import database_vendor from packages.models import Depend, Conflict, Provision, Replacement, Update +from packages.utils import parse_version logging.basicConfig( @@ -84,8 +85,6 @@ class RepoPackage(object): 'conflicts', 'provides', 'replaces', 'groups', 'license', 'files' ) - version_re = re.compile(r'^((\d+):)?(.+)-([^-]+)$') - def __init__(self, repo): self.repo = repo self.ver = None @@ -112,11 +111,7 @@ def populate(self, values): # do NOT prune these values at all setattr(self, k, v[0]) elif k == 'version': - match = self.version_re.match(v[0]) - self.ver = match.group(3) - self.rel = match.group(4) - if match.group(2): - self.epoch = int(match.group(2)) + self.ver, self.rel, self.epoch = parse_version(v[0]) elif k == 'builddate': try: builddate = datetime.utcfromtimestamp(int(v[0])) diff --git a/packages/utils.py b/packages/utils.py index 6d54d71a..d4b4e611 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -1,6 +1,7 @@ from collections import defaultdict from itertools import chain from operator import itemgetter +import re from django.core.serializers.json import DjangoJSONEncoder from django.db import connection @@ -14,6 +15,23 @@ License, Depend, Conflict, Provision, Replacement, SignoffSpecification, Signoff, DEFAULT_SIGNOFF_SPEC) + +VERSION_RE = re.compile(r'^((\d+):)?(.+)-([^-]+)$') + + +def parse_version(version): + match = VERSION_RE.match(version) + if not match: + return None, None, 0 + ver = match.group(3) + rel = match.group(4) + if match.group(2): + epoch = int(match.group(2)) + else: + epoch = 0 + return ver, rel, epoch + + @cache_function(127) def get_group_info(include_arches=None): raw_groups = PackageGroup.objects.values_list( -- cgit v1.2.3-54-g00ecf From ca0011c585ec28f9dde0f400a77fd6f859d520b0 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 15 Aug 2012 08:11:00 -0500 Subject: Add ability to rematch developers on @archlinux.org addresses This makes this matcher catch a bit more with the wide net we were already casting. Signed-off-by: Dan McGee --- devel/management/commands/rematch_developers.py | 4 +- devel/utils.py | 50 +++++++++++++++++-------- 2 files changed, 38 insertions(+), 16 deletions(-) (limited to 'devel/management') diff --git a/devel/management/commands/rematch_developers.py b/devel/management/commands/rematch_developers.py index ab2f0f4b..2b379588 100644 --- a/devel/management/commands/rematch_developers.py +++ b/devel/management/commands/rematch_developers.py @@ -53,6 +53,7 @@ def match_packager(finder): unmatched = Package.objects.filter(packager__isnull=True).values_list( 'packager_str', flat=True).order_by().distinct() + logger.info("%d packager strings retrieved", len(unmatched)) for packager in unmatched: logger.debug("packager string %s", packager) user = finder.find(packager) @@ -71,13 +72,14 @@ def match_packager(finder): @transaction.commit_on_success def match_flagrequest(finder): - logger.info("getting all flag requests emails from unknown users") + logger.info("getting all flag request email addresses from unknown users") req_count = matched_count = 0 mapping = {} unmatched = FlagRequest.objects.filter(user__isnull=True).values_list( 'user_email', flat=True).order_by().distinct() + logger.info("%d email addresses retrieved", len(unmatched)) for user_email in unmatched: logger.debug("email %s", user_email) user = finder.find_by_email(user_email) diff --git a/devel/utils.py b/devel/utils.py index 85b4e42f..e8e3a6c4 100644 --- a/devel/utils.py +++ b/devel/utils.py @@ -1,6 +1,7 @@ import re from django.contrib.auth.models import User +from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned from django.db import connection from django.db.models import Count, Q @@ -44,6 +45,15 @@ def get_annotated_maintainers(): return maintainers +def ignore_does_not_exist(func): + def new_func(*args, **kwargs): + try: + return func(*args, **kwargs) + except (ObjectDoesNotExist, MultipleObjectsReturned): + return None + return new_func + + class UserFinder(object): def __init__(self): self.cache = {} @@ -52,18 +62,33 @@ def __init__(self): self.pgp_cache = {} @staticmethod + @ignore_does_not_exist def user_email(name, email): if email: return User.objects.get(email=email) return None @staticmethod + @ignore_does_not_exist + def username_email(name, email): + if email and '@' in email: + # split email addr at '@' symbol, ensure domain matches + # or is a subdomain of archlinux.org + # TODO: configurable domain/regex somewhere? + username, domain = email.split('@', 1) + if re.match(r'^(.+\.)?archlinux.org$', domain): + return User.objects.get(username=username) + return None + + @staticmethod + @ignore_does_not_exist def profile_email(name, email): if email: return User.objects.get(userprofile__public_email=email) return None @staticmethod + @ignore_does_not_exist def user_name(name, email): # yes, a bit odd but this is the easiest way since we can't always be # sure how to split the name. Ensure every 'token' appears in at least @@ -102,14 +127,12 @@ def find(self, userstring): email = matches.group(2) user = None - find_methods = (self.user_email, self.profile_email, self.user_name) + find_methods = (self.user_email, self.profile_email, + self.username_email, self.user_name) for matcher in find_methods: - try: - user = matcher(name, email) - if user != None: - break - except (User.DoesNotExist, User.MultipleObjectsReturned): - pass + user = matcher(name, email) + if user != None: + break self.cache[userstring] = user self.email_cache[email] = user @@ -135,14 +158,11 @@ def find_by_email(self, email): if email in self.email_cache: return self.email_cache[email] - user = None - try: - user = self.user_email(None, email) - except User.DoesNotExist: - try: - user = self.profile_email(None, email) - except User.DoesNotExist: - pass + user = self.user_email(None, email) + if user is None: + user = self.profile_email(None, email) + if user is None: + user = self.username_email(None, email) self.email_cache[email] = user return user -- cgit v1.2.3-54-g00ecf From c76e5c768687394b8022883d01edf85dc3c30e7f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 17 Sep 2012 21:42:48 -0500 Subject: Sort package list before inserting it into the database FS#30323. This will take some time to propagate to all existing packages, but all new and updated packages will start getting filelists in the right order. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index af0a2dc0..ac745092 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -303,7 +303,10 @@ def populate_files(dbpkg, repopkg, force=False): logger.info("adding %d files for package %s", len(repopkg.files), dbpkg.pkgname) pkg_files = [] - for f in repopkg.files: + # sort in normal alpha-order that pacman uses, rather than makepkg's + # default breadth-first, directory-first ordering + files = sorted(repopkg.files) + for f in files: if '/' in f: dirname, filename = f.rsplit('/', 1) dirname += '/' -- cgit v1.2.3-54-g00ecf From 7d5cfe45d52c4dbd2f431f0edcafc9936b740ab2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 20 Sep 2012 09:32:42 -0500 Subject: Explicitly close the database connection in reporead This is the cause of these warnings showing up in the PostgreSQL log: LOG: unexpected EOF on client connection with an open transaction All management commands are guilty of this as they do not clean up and close the connection when they exit, unlike the standard web request cycle. Other commands should probably be updated as well, but for now, this is the biggest culprit. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 1 + 1 file changed, 1 insertion(+) (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index ac745092..ce5c8cb7 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -584,6 +584,7 @@ def read_repo(primary_arch, repo_file, options): else: db_update(arch, repo, packages_arches[arch], force) logger.info('Finished database updates for %s.', repo_file) + connection.close() return 0 # vim: set ts=4 sw=4 et: -- cgit v1.2.3-54-g00ecf From 89c6ffc95cb1d5fe4bd2534562ca732d727a8686 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 20 Sep 2012 09:34:46 -0500 Subject: chmod -x reporead_inotify.py Signed-off-by: Dan McGee --- devel/management/commands/reporead_inotify.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 devel/management/commands/reporead_inotify.py (limited to 'devel/management') diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py old mode 100755 new mode 100644 -- cgit v1.2.3-54-g00ecf From 3530303c9a7d017bdfec40d9dc7c38bd3fb2c09b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 25 Sep 2012 18:21:10 -0500 Subject: Only watch non-staging repos in inotify reporead This is temporary until we do more work to ensure staging packages don't show up and confuse regular users of the web interface. Signed-off-by: Dan McGee --- devel/management/commands/reporead_inotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'devel/management') diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py index c74762eb..043e13fe 100644 --- a/devel/management/commands/reporead_inotify.py +++ b/devel/management/commands/reporead_inotify.py @@ -68,7 +68,7 @@ def setup_notifier(self): and passes these on to the various pyinotify pieces as necessary and finally builds and returns a notifier object.''' arches = Arch.objects.filter(agnostic=False) - repos = Repo.objects.all() + repos = Repo.objects.filter(staging=False) arch_path_map = dict((arch, None) for arch in arches) all_paths = set() total_paths = 0 -- cgit v1.2.3-54-g00ecf From ed1adeb1254c4d5754260bbe1ae2fbbc2a88debb Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 30 Sep 2012 02:02:04 -0500 Subject: Begin importing staging repos This reverts 3530303c9a7d now that we have reasonably hidden most staging package confusion on the site for normal end users. Signed-off-by: Dan McGee --- devel/management/commands/reporead_inotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'devel/management') diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py index 043e13fe..c74762eb 100644 --- a/devel/management/commands/reporead_inotify.py +++ b/devel/management/commands/reporead_inotify.py @@ -68,7 +68,7 @@ def setup_notifier(self): and passes these on to the various pyinotify pieces as necessary and finally builds and returns a notifier object.''' arches = Arch.objects.filter(agnostic=False) - repos = Repo.objects.filter(staging=False) + repos = Repo.objects.all() arch_path_map = dict((arch, None) for arch in arches) all_paths = set() total_paths = 0 -- cgit v1.2.3-54-g00ecf From 5228cb5f584f076e547e1d0af695c08975801d2f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 12 Oct 2012 11:39:16 -0500 Subject: reporead: don't print full backtrace if unnecessary In the architecture agnostic case, this error is much more likely to happen, so printing it like an error message is deceiving. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index ce5c8cb7..a1e77b49 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -393,9 +393,12 @@ def db_update(archname, reponame, pkgs, force=False): populate_pkg(dbpkg, pkg, timestamp=now()) Update.objects.log_update(None, dbpkg) except IntegrityError: - logger.warning("Could not add package %s; " - "not fatal if another thread beat us to it.", - pkg.name, exc_info=True) + if architecture.agnostic: + logger.warning("Could not add package %s; " + "not fatal if another thread beat us to it.", + pkg.name) + else: + logger.exception("Could not add package %s", pkg.name) # packages in database and not in syncdb (remove from database) for pkgname in (dbset - syncset): -- cgit v1.2.3-54-g00ecf From 6dd4d54bb0adbbb0f8c2b1beaa92b7a58971cf88 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 16 Nov 2012 16:20:11 -0600 Subject: Use Python 2.7 dictionary comprehension syntax Rather than the old idiom of dict((k, v) for <> in <>). Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 4 ++-- devel/management/commands/reporead_inotify.py | 2 +- mirrors/views.py | 11 ++++------- packages/templatetags/package_extras.py | 2 +- packages/utils.py | 5 ++--- packages/views/signoff.py | 8 +++----- public/views.py | 2 +- visualize/views.py | 4 ++-- 8 files changed, 16 insertions(+), 22 deletions(-) (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index a1e77b49..3d4e6375 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -377,7 +377,7 @@ def db_update(archname, reponame, pkgs, force=False): # This makes our inner loop where we find packages by name *way* more # efficient by not having to go to the database for each package to # SELECT them by name. - dbdict = dict((dbpkg.pkgname, dbpkg) for dbpkg in dbpkgs) + dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs} dbset = set(dbdict.keys()) syncset = set([pkg.name for pkg in pkgs]) @@ -446,7 +446,7 @@ def filesonly_update(archname, reponame, pkgs, force=False): """ logger.info('Updating files for %s (%s)', reponame, archname) dbpkgs = update_common(archname, reponame, pkgs, False) - dbdict = dict((dbpkg.pkgname, dbpkg) for dbpkg in dbpkgs) + dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs} dbset = set(dbdict.keys()) for pkg in (pkg for pkg in pkgs if pkg.name in dbset): diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py index c74762eb..16b3869c 100644 --- a/devel/management/commands/reporead_inotify.py +++ b/devel/management/commands/reporead_inotify.py @@ -69,7 +69,7 @@ def setup_notifier(self): finally builds and returns a notifier object.''' arches = Arch.objects.filter(agnostic=False) repos = Repo.objects.all() - arch_path_map = dict((arch, None) for arch in arches) + arch_path_map = {arch: None for arch in arches} all_paths = set() total_paths = 0 for arch in arches: diff --git a/mirrors/views.py b/mirrors/views.py index 2e1e83b6..d0ce0a97 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -94,7 +94,7 @@ def default_protocol_filter(original_urls): def status_filter(original_urls): status_info = get_mirror_statuses() - scores = dict((u.id, u.score) for u in status_info['urls']) + scores = {u.id: u.score for u in status_info['urls']} urls = [] for u in original_urls: u.score = scores.get(u.id, None) @@ -165,7 +165,7 @@ def mirrors(request): if not request.user.is_authenticated(): mirror_list = mirror_list.filter(public=True, active=True) protos = protos.filter(mirror__public=True, mirror__active=True) - protos = dict((k, list(v)) for k, v in groupby(protos, key=itemgetter(0))) + protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))} for mirror in mirror_list: items = protos.get(mirror.id, []) mirror.protocols = [item[1] for item in items] @@ -253,8 +253,7 @@ def default(self, obj): # mainly for queryset serialization return list(obj) if isinstance(obj, MirrorUrl): - data = dict((attr, getattr(obj, attr)) - for attr in self.url_attributes) + data = {attr: getattr(obj, attr) for attr in self.url_attributes} # get any override on the country attribute first country = obj.real_country data['country'] = unicode(country.name) @@ -277,9 +276,7 @@ def default(self, obj): check_time__gte=cutoff).order_by('check_time') return data if isinstance(obj, MirrorLog): - data = dict((attr, getattr(obj, attr)) - for attr in self.log_attributes) - return data + return {attr: getattr(obj, attr) for attr in self.log_attributes} return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) diff --git a/packages/templatetags/package_extras.py b/packages/templatetags/package_extras.py index 994265d8..f3613e69 100644 --- a/packages/templatetags/package_extras.py +++ b/packages/templatetags/package_extras.py @@ -13,7 +13,7 @@ def link_encode(url, query): # massage the data into all utf-8 encoded strings first, so urlencode # doesn't barf at the data we pass it - query = dict((k, unicode(v).encode('utf-8')) for k, v in query.items()) + query = {k: unicode(v).encode('utf-8') for k, v in query.items()} data = urlencode(query).replace('&', '&') return "%s?%s" % (url, data) diff --git a/packages/utils.py b/packages/utils.py index 051fed8e..199e141d 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -385,7 +385,7 @@ def signoffs_id_query(model, repos): repo_sql = ','.join(['%s' for r in repos]) sql = sql % (model._meta.db_table, repo_sql, repo_sql) repo_ids = [r.pk for r in repos] - # repo_ids are needed twice, so double the array + # repo_ids are needed twice, so double the array cursor.execute(sql, repo_ids * 2) results = cursor.fetchall() @@ -474,8 +474,7 @@ def default(self, obj): # mainly for queryset serialization return list(obj) if isinstance(obj, Package): - data = dict((attr, getattr(obj, attr)) - for attr in self.pkg_attributes) + data = {attr: getattr(obj, attr) for attr in self.pkg_attributes} for attr in self.pkg_list_attributes: data[attr] = getattr(obj, attr).all() return data diff --git a/packages/views/signoff.py b/packages/views/signoff.py index 56eb060c..824a9922 100644 --- a/packages/views/signoff.py +++ b/packages/views/signoff.py @@ -155,8 +155,8 @@ class SignoffJSONEncoder(DjangoJSONEncoder): def default(self, obj): if isinstance(obj, PackageSignoffGroup): - data = dict((attr, getattr(obj, attr)) - for attr in self.signoff_group_attrs) + data = {attr: getattr(obj, attr) + for attr in self.signoff_group_attrs} data['pkgnames'] = [p.pkgname for p in obj.packages] data['package_count'] = len(obj.packages) data['approved'] = obj.approved() @@ -164,9 +164,7 @@ def default(self, obj): for attr in self.signoff_spec_attrs) return data elif isinstance(obj, Signoff): - data = dict((attr, getattr(obj, attr)) - for attr in self.signoff_attrs) - return data + return {attr: getattr(obj, attr) for attr in self.signoff_attrs} elif isinstance(obj, Arch) or isinstance(obj, Repo): return unicode(obj) elif isinstance(obj, User): diff --git a/public/views.py b/public/views.py index 96120761..3e15f9df 100644 --- a/public/views.py +++ b/public/views.py @@ -118,7 +118,7 @@ def keys(request): sig_counts = PGPSignature.objects.filter(not_expired, valid=True, signee__in=user_key_ids).values_list('signer').annotate( Count('signer')) - sig_counts = dict((key_id[-16:], ct) for key_id, ct in sig_counts) + sig_counts = {key_id[-16:]: ct for key_id, ct in sig_counts} for key in master_keys: key.signature_count = sig_counts.get(key.pgp_key[-16:], 0) diff --git a/visualize/views.py b/visualize/views.py index 8d878937..48e8f86b 100644 --- a/visualize/views.py +++ b/visualize/views.py @@ -33,8 +33,8 @@ def build_map(name, arch, repo): # now transform these results into two mappings: one ordered (repo, arch), # and one ordered (arch, repo). - arch_groups = dict((a, build_map(a, a, None)) for a in arches) - repo_groups = dict((r, build_map(r, None, r)) for r in repos) + arch_groups = {a: build_map(a, a, None) for a in arches} + repo_groups = {r: build_map(r, None, r) for r in repos} for row in qs: arch = row['arch__name'] repo = row['repo__name'] -- cgit v1.2.3-54-g00ecf From 9e9157d0a8cbf9ea076231e438fb30f58bff8e29 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 16 Nov 2012 16:37:31 -0600 Subject: Use python set comprehension syntax supported in 2.7 Signed-off-by: Dan McGee --- devel/management/commands/import_signatures.py | 4 ++-- devel/management/commands/reporead.py | 2 +- devel/management/commands/reporead_inotify.py | 2 +- devel/views.py | 4 ++-- main/models.py | 2 +- packages/models.py | 2 +- packages/utils.py | 10 +++++----- packages/views/signoff.py | 2 +- todolists/views.py | 8 ++++---- 9 files changed, 18 insertions(+), 18 deletions(-) (limited to 'devel/management') diff --git a/devel/management/commands/import_signatures.py b/devel/management/commands/import_signatures.py index ce1aba90..da1397ca 100644 --- a/devel/management/commands/import_signatures.py +++ b/devel/management/commands/import_signatures.py @@ -98,8 +98,8 @@ def import_signatures(keyring): # now prune the data down to what we actually want. # prune edges not in nodes, remove duplicates, and self-sigs - pruned_edges = set(edge for edge in edges - if edge.signer in nodes and edge.signer != edge.signee) + pruned_edges = {edge for edge in edges + if edge.signer in nodes and edge.signer != edge.signee} logger.info("creating or finding %d signatures", len(pruned_edges)) created_ct = updated_ct = 0 diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 3d4e6375..981c4dce 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -380,7 +380,7 @@ def db_update(archname, reponame, pkgs, force=False): dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs} dbset = set(dbdict.keys()) - syncset = set([pkg.name for pkg in pkgs]) + syncset = {pkg.name for pkg in pkgs} in_sync_not_db = syncset - dbset logger.info("%d packages in sync not db", len(in_sync_not_db)) diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py index 16b3869c..04f65764 100644 --- a/devel/management/commands/reporead_inotify.py +++ b/devel/management/commands/reporead_inotify.py @@ -77,7 +77,7 @@ def setup_notifier(self): for repo in repos) # take a python format string and generate all unique combinations # of directories from it; using set() ensures we filter it down - paths = set(self.path_template % values for values in combos) + paths = {self.path_template % values for values in combos} total_paths += len(paths) all_paths |= paths arch_path_map[arch] = paths diff --git a/devel/views.py b/devel/views.py index 083665d9..7d5947d1 100644 --- a/devel/views.py +++ b/devel/views.py @@ -277,8 +277,8 @@ def report(request, report_name, username=None): else: raise Http404 - arches = set(pkg.arch for pkg in packages) - repos = set(pkg.repo for pkg in packages) + arches = {pkg.arch for pkg in packages} + repos = {pkg.repo for pkg in packages} context = { 'all_maintainers': maints, 'title': title, diff --git a/main/models.py b/main/models.py index 5700cdf1..cc81637c 100644 --- a/main/models.py +++ b/main/models.py @@ -197,7 +197,7 @@ def get_requiredby(self): """ from packages.models import Depend provides = self.provides.all() - provide_names = set(provide.name for provide in provides) + provide_names = {provide.name for provide in provides} provide_names.add(self.pkgname) requiredby = Depend.objects.select_related('pkg', 'pkg__arch', 'pkg__repo').filter( diff --git a/packages/models.py b/packages/models.py index 0d0fbdf2..ede8c275 100644 --- a/packages/models.py +++ b/packages/models.py @@ -33,7 +33,7 @@ def get_associated_packages(self): def repositories(self): packages = self.get_associated_packages() - return sorted(set([p.repo for p in packages])) + return sorted({p.repo for p in packages}) def __unicode__(self): return u'%s: %s (%s)' % ( diff --git a/packages/utils.py b/packages/utils.py index 199e141d..5adc8637 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -79,8 +79,8 @@ def get_split_packages_info(): split_pkgs = Package.objects.exclude(pkgname=F('pkgbase')).exclude( pkgbase__in=pkgnames).values('pkgbase', 'repo', 'arch').annotate( last_update=Max('last_update')) - all_arches = Arch.objects.in_bulk(set(s['arch'] for s in split_pkgs)) - all_repos = Repo.objects.in_bulk(set(s['repo'] for s in split_pkgs)) + all_arches = Arch.objects.in_bulk({s['arch'] for s in split_pkgs}) + all_repos = Repo.objects.in_bulk({s['repo'] for s in split_pkgs}) for split in split_pkgs: split['arch'] = all_arches[split['arch']] split['repo'] = all_repos[split['repo']] @@ -143,7 +143,7 @@ def get_differences_info(arch_a, arch_b): cursor.execute(sql, [arch_a.id, arch_b.id]) results = cursor.fetchall() # column A will always have a value, column B might be NULL - to_fetch = set(row[0] for row in results) + to_fetch = {row[0] for row in results} # fetch all of the necessary packages pkgs = Package.objects.normal().in_bulk(to_fetch) # now build a list of tuples containing differences @@ -249,13 +249,13 @@ def attach_maintainers(packages): the maintainers and attach them to the packages to prevent N+1 query cascading.''' packages = list(packages) - pkgbases = set(p.pkgbase for p in packages) + pkgbases = {p.pkgbase for p in packages} rels = PackageRelation.objects.filter(type=PackageRelation.MAINTAINER, pkgbase__in=pkgbases).values_list( 'pkgbase', 'user_id').order_by().distinct() # get all the user objects we will need - user_ids = set(rel[1] for rel in rels) + user_ids = {rel[1] for rel in rels} users = User.objects.in_bulk(user_ids) # now build a pkgbase -> [maintainers...] map diff --git a/packages/views/signoff.py b/packages/views/signoff.py index 824a9922..340b2311 100644 --- a/packages/views/signoff.py +++ b/packages/views/signoff.py @@ -25,7 +25,7 @@ def signoffs(request): context = { 'signoff_groups': signoff_groups, 'arches': Arch.objects.all(), - 'repo_names': sorted(set(g.target_repo for g in signoff_groups)), + 'repo_names': sorted({g.target_repo for g in signoff_groups}), } return render(request, 'packages/signoffs.html', context) diff --git a/todolists/views.py b/todolists/views.py index b8d1dae1..9984ef9a 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -53,8 +53,8 @@ def view(request, list_id): # we don't hold onto the result, but the objects are the same here, # so accessing maintainers in the template is now cheap attach_maintainers(tp.pkg for tp in todolist.packages) - arches = set(tp.pkg.arch for tp in todolist.packages) - repos = set(tp.pkg.repo for tp in todolist.packages) + arches = {tp.pkg.arch for tp in todolist.packages} + repos = {tp.pkg.repo for tp in todolist.packages} return render(request, 'todolists/view.html', { 'list': todolist, 'svn_roots': svn_roots, @@ -67,8 +67,8 @@ def list_pkgbases(request, list_id, svn_root): '''Used to make bulk moves of packages a lot easier.''' todolist = get_object_or_404(Todolist, id=list_id) repos = get_list_or_404(Repo, svn_root=svn_root) - pkgbases = set(tp.pkg.pkgbase for tp in todolist.packages - if tp.pkg.repo in repos) + pkgbases = {tp.pkg.pkgbase for tp in todolist.packages + if tp.pkg.repo in repos} return HttpResponse('\n'.join(sorted(pkgbases)), mimetype='text/plain') -- cgit v1.2.3-54-g00ecf From 04f23a040a839f4989fdc83afe0f5ad4f72224be Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 31 Dec 2012 10:17:22 -0600 Subject: Add 'created' field to packages model This will be used to eventually implement the UI side of FS#13441, but to do that, we first need the data. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 6 +- .../0063_auto__add_field_package_created.py | 116 +++++++++++++++++++++ main/models.py | 3 + 3 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 main/migrations/0063_auto__add_field_package_created.py (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 981c4dce..e00e54c3 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -387,10 +387,12 @@ def db_update(archname, reponame, pkgs, force=False): # packages in syncdb and not in database (add to database) for pkg in (pkg for pkg in pkgs if pkg.name in in_sync_not_db): logger.info("Adding package %s", pkg.name) - dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository) + timestamp = now() + dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository, + created=timestamp) try: with transaction.commit_on_success(): - populate_pkg(dbpkg, pkg, timestamp=now()) + populate_pkg(dbpkg, pkg, timestamp=timestamp) Update.objects.log_update(None, dbpkg) except IntegrityError: if architecture.agnostic: diff --git a/main/migrations/0063_auto__add_field_package_created.py b/main/migrations/0063_auto__add_field_package_created.py new file mode 100644 index 00000000..e5a990c3 --- /dev/null +++ b/main/migrations/0063_auto__add_field_package_created.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +import datetime +from pytz import utc +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + old_date = datetime.datetime(2000, 1, 1) + old_date = old_date.replace(tzinfo=utc) + db.add_column('packages', 'created', + self.gf('django.db.models.fields.DateTimeField')(default=old_date), keep_default=False) + + + def backwards(self, orm): + db.delete_column('packages', 'created') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'main.arch': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Arch', 'db_table': "'arches'"}, + 'agnostic': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'main.donor': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Donor', 'db_table': "'donors'"}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'visible': ('django.db.models.fields.BooleanField', [], {'default': 'True'}) + }, + 'main.package': { + 'Meta': {'ordering': "('pkgname',)", 'unique_together': "(('pkgname', 'repo', 'arch'),)", 'object_name': 'Package', 'db_table': "'packages'"}, + 'arch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'on_delete': 'models.PROTECT', 'to': "orm['main.Arch']"}), + 'build_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'compressed_size': ('main.fields.PositiveBigIntegerField', [], {}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'epoch': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'files_last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'flag_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'installed_size': ('main.fields.PositiveBigIntegerField', [], {}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'packager': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}), + 'packager_str': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pgp_signature': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkgdesc': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'pkgname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'repo': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'on_delete': 'models.PROTECT', 'to': "orm['main.Repo']"}), + 'url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}) + }, + 'main.packagefile': { + 'Meta': {'object_name': 'PackageFile', 'db_table': "'package_files'"}, + 'directory': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_directory': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}) + }, + 'main.repo': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Repo', 'db_table': "'repos'"}, + 'bugs_category': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'bugs_project': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'staging': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'svn_root': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + } + } + + complete_apps = ['main'] diff --git a/main/models.py b/main/models.py index 7ec04ad7..39bc555e 100644 --- a/main/models.py +++ b/main/models.py @@ -103,6 +103,7 @@ class Package(models.Model): build_date = models.DateTimeField(null=True) last_update = models.DateTimeField(db_index=True) files_last_update = models.DateTimeField(null=True, blank=True) + created = models.DateTimeField() packager_str = models.CharField(max_length=255) packager = models.ForeignKey(User, null=True, blank=True, on_delete=models.SET_NULL) @@ -424,6 +425,8 @@ class Meta: post_save.connect(refresh_latest, sender=Package, dispatch_uid="main.models") +# note: reporead sets the 'created' field on Package objects, so no signal +# listener is set up here to do so pre_save.connect(set_created_field, sender=Donor, dispatch_uid="main.models") -- cgit v1.2.3-54-g00ecf From af32c23768c7537f19e0613525579208b4f44eb4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 15 Jan 2013 20:49:56 -0600 Subject: Handle connection and transaction more properly in reporead A few minor things are fixed here. One is PostgreSQL, and more specifically pgbouncer, don't like it when the connection is closed after psycopg2 has started an implicit transaction even for read-only queries. Ensure we call commit as our last database action in all cases. The other is related- Django in management commands doesn't ever call close on any database connection you may have been using, so PostgreSQL gets mad about this fact and logs a message saying such. Close the connection explicitly when we are done with it to play nice. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 1 + devel/management/commands/reporead_inotify.py | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index e00e54c3..ab0efeed 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -589,6 +589,7 @@ def read_repo(primary_arch, repo_file, options): else: db_update(arch, repo, packages_arches[arch], force) logger.info('Finished database updates for %s.', repo_file) + connection.commit() connection.close() return 0 diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py index 04f65764..8c1e47bf 100644 --- a/devel/management/commands/reporead_inotify.py +++ b/devel/management/commands/reporead_inotify.py @@ -23,7 +23,7 @@ import time from django.core.management.base import BaseCommand, CommandError -from django.db import connection +from django.db import connection, transaction from main.models import Arch, Repo from .reporead import read_repo @@ -53,6 +53,11 @@ def handle(self, path_template=None, **options): self.path_template = path_template notifier = self.setup_notifier() + # this thread is done using the database; all future access is done in + # the spawned read_repo() processes, so close the otherwise completely + # idle connection. + connection.close() + logger.info('Entering notifier loop') notifier.loop() @@ -61,14 +66,17 @@ def handle(self, path_template=None, **options): if hasattr(thread, 'cancel'): thread.cancel() + @transaction.commit_on_success def setup_notifier(self): '''Set up and configure the inotify machinery and logic. This takes the provided or default path_template and builds a list of directories we need to watch for database updates. It then validates and passes these on to the various pyinotify pieces as necessary and finally builds and returns a notifier object.''' + transaction.commit_manually() arches = Arch.objects.filter(agnostic=False) repos = Repo.objects.all() + transaction.set_dirty() arch_path_map = {arch: None for arch in arches} all_paths = set() total_paths = 0 @@ -91,11 +99,6 @@ def setup_notifier(self): raise CommandError('path template did not uniquely ' 'determine architecture for each file') - # this thread is done using the database; all future access is done in - # the spawned read_repo() processes, so close the otherwise completely - # idle connection. - connection.close() - # A proper atomic replacement of the database as done by rsync is type # IN_MOVED_TO. repo-add/remove will finish with a IN_CLOSE_WRITE. mask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO -- cgit v1.2.3-54-g00ecf From 7e6279057a57ef44c11349e594ad392fbfce0098 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 3 Feb 2013 14:26:10 -0600 Subject: Add new pgp_import command; replaces import_signatures This command now imports keys, subkeys, and signatures of those keys & subkeys. This will allow us to actually match developers with their packages signed by subkeys rather than the primary key. Signed-off-by: Dan McGee --- devel/management/commands/import_signatures.py | 123 ------------- devel/management/commands/pgp_import.py | 241 +++++++++++++++++++++++++ 2 files changed, 241 insertions(+), 123 deletions(-) delete mode 100644 devel/management/commands/import_signatures.py create mode 100644 devel/management/commands/pgp_import.py (limited to 'devel/management') diff --git a/devel/management/commands/import_signatures.py b/devel/management/commands/import_signatures.py deleted file mode 100644 index da1397ca..00000000 --- a/devel/management/commands/import_signatures.py +++ /dev/null @@ -1,123 +0,0 @@ -# -*- coding: utf-8 -*- -""" -import_signatures command - -Import signatures from a given GPG keyring. - -Usage: ./manage.py generate_keyring -""" - -from collections import namedtuple -from datetime import datetime -import logging -import subprocess -import sys - -from django.core.management.base import BaseCommand, CommandError -from django.db import transaction - -from devel.models import PGPSignature - -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s -> %(levelname)s: %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', - stream=sys.stderr) -logger = logging.getLogger() - -class Command(BaseCommand): - args = "" - help = "Import signatures from a given GPG keyring." - - def handle(self, *args, **options): - v = int(options.get('verbosity', None)) - if v == 0: - logger.level = logging.ERROR - elif v == 1: - logger.level = logging.INFO - elif v == 2: - logger.level = logging.DEBUG - - if len(args) < 1: - raise CommandError("keyring_path must be provided") - - import_signatures(args[0]) - - -SignatureData = namedtuple('SignatureData', - ('signer', 'signee', 'created', 'expires', 'valid')) - - -def get_date(epoch_string): - '''Convert a epoch string into a python 'date' object (not datetime).''' - return datetime.utcfromtimestamp(int(epoch_string)).date() - - -def parse_sigdata(data): - nodes = {} - edges = [] - current_pubkey = None - - # parse all of the output from our successful GPG command - logger.info("parsing command output") - for line in data.split('\n'): - parts = line.split(':') - if parts[0] == 'pub': - current_pubkey = parts[4] - nodes[current_pubkey] = None - if parts[0] == 'uid': - uid = parts[9] - # only set uid if this is the first one encountered - if nodes[current_pubkey] is None: - nodes[current_pubkey] = uid - if parts[0] == 'sig': - signer = parts[4] - created = get_date(parts[5]) - expires = None - if parts[6]: - expires = get_date(parts[6]) - valid = parts[1] != '-' - edge = SignatureData(signer, current_pubkey, - created, expires, valid) - edges.append(edge) - - return nodes, edges - - -def import_signatures(keyring): - gpg_cmd = ["gpg", "--no-default-keyring", "--keyring", keyring, - "--list-sigs", "--with-colons", "--fixed-list-mode"] - logger.info("running command: %r", gpg_cmd) - proc = subprocess.Popen(gpg_cmd, stdout=subprocess.PIPE) - outdata, errdata = proc.communicate() - if proc.returncode != 0: - logger.error(errdata) - raise subprocess.CalledProcessError(proc.returncode, gpg_cmd) - - nodes, edges = parse_sigdata(outdata) - - # now prune the data down to what we actually want. - # prune edges not in nodes, remove duplicates, and self-sigs - pruned_edges = {edge for edge in edges - if edge.signer in nodes and edge.signer != edge.signee} - - logger.info("creating or finding %d signatures", len(pruned_edges)) - created_ct = updated_ct = 0 - with transaction.commit_on_success(): - for edge in pruned_edges: - sig, created = PGPSignature.objects.get_or_create( - signer=edge.signer, signee=edge.signee, - created=edge.created, expires=edge.expires, - defaults={ 'valid': edge.valid }) - if sig.valid != edge.valid: - sig.valid = edge.valid - sig.save() - updated_ct = 1 - if created: - created_ct += 1 - - sig_ct = PGPSignature.objects.all().count() - logger.info("%d total signatures in database", sig_ct) - logger.info("created %d, updated %d signatures", created_ct, updated_ct) - -# vim: set ts=4 sw=4 et: diff --git a/devel/management/commands/pgp_import.py b/devel/management/commands/pgp_import.py new file mode 100644 index 00000000..10e6cfcb --- /dev/null +++ b/devel/management/commands/pgp_import.py @@ -0,0 +1,241 @@ +# -*- coding: utf-8 -*- +""" +pgp_import command + +Import keys and signatures from a given GPG keyring. + +Usage: ./manage.py pgp_import +""" + +from collections import namedtuple, OrderedDict +from datetime import datetime +import logging +from pytz import utc +import subprocess +import sys + +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction + +from devel.models import DeveloperKey, PGPSignature +from devel.utils import UserFinder + + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s -> %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + stream=sys.stderr) +logger = logging.getLogger() + +class Command(BaseCommand): + args = "" + help = "Import keys and signatures from a given GPG keyring." + + def handle(self, *args, **options): + v = int(options.get('verbosity', None)) + if v == 0: + logger.level = logging.ERROR + elif v == 1: + logger.level = logging.INFO + elif v == 2: + logger.level = logging.DEBUG + + if len(args) < 1: + raise CommandError("keyring_path must be provided") + + import_keys(args[0]) + import_signatures(args[0]) + + +def get_date(epoch_string): + '''Convert a epoch string into a python 'date' object (not datetime).''' + if not epoch_string: + return None + return datetime.utcfromtimestamp(int(epoch_string)).date() + + +def get_datetime(epoch_string): + '''Convert a epoch string into a python 'datetime' object.''' + if not epoch_string: + return None + return datetime.utcfromtimestamp(int(epoch_string)).replace(tzinfo=utc) + + +def call_gpg(keyring, *args): + # GPG is stupid and interprets any filename without path portion as being + # in ~/.gnupg/. Fake it out if we just get a bare filename. + if '/' not in keyring: + keyring = './%s' % keyring + gpg_cmd = ["gpg2", "--no-default-keyring", "--keyring", keyring, + "--with-colons", "--fixed-list-mode"] + gpg_cmd.extend(args) + logger.info("running command: %s", ' '.join(gpg_cmd)) + proc = subprocess.Popen(gpg_cmd, stdout=subprocess.PIPE) + outdata, errdata = proc.communicate() + if proc.returncode != 0: + logger.error(errdata) + raise subprocess.CalledProcessError(proc.returncode, gpg_cmd) + return outdata + + +class KeyData(object): + def __init__(self, key, created, expires): + self.key = key + self.created = get_datetime(created) + self.expires = get_datetime(expires) + self.parent = None + self.revoked = None + self.db_id = None + + +def parse_keydata(data): + keys = OrderedDict() + current_pubkey = None + + # parse all of the output from our successful GPG command + logger.info("parsing command output") + for line in data.split('\n'): + parts = line.split(':') + if parts[0] == 'pub': + key = parts[4] + current_pubkey = key + keys[key] = KeyData(key, parts[5], parts[6]) + node = parts[0] + elif parts[0] == 'sub': + key = parts[4] + keys[key] = KeyData(key, parts[5], parts[6]) + keys[key].parent = current_pubkey + node = parts[0] + elif parts[0] == 'uid': + node = parts[0] + elif parts[0] == 'rev' and node in ('pub', 'sub'): + keys[current_pubkey].revoked = get_datetime(parts[5]) + + return keys + + +def find_key_owner(key, keys, finder): + '''Recurse up the chain, looking for an owner.''' + if key is None: + return None + owner = finder.find_by_pgp_key(key.key) + if owner: + return owner + if key.parent: + return find_key_owner(keys[key.parent], keys, finder) + return None + + +def import_keys(keyring): + outdata = call_gpg(keyring, "--list-sigs") + keydata = parse_keydata(outdata) + + logger.info("creating or finding %d keys", len(keydata)) + created_ct = updated_ct = 0 + with transaction.commit_on_success(): + finder = UserFinder() + # we are dependent on parents coming before children; parse_keydata + # uses an OrderedDict to ensure this is the case. + for data in keydata.values(): + parent_id = None + if data.parent: + parent_data = keydata.get(data.parent, None) + if parent_data: + parent_id = parent_data.db_id + other = { + 'expires': data.expires, + 'revoked': data.revoked, + 'parent_id': parent_id, + } + dkey, created = DeveloperKey.objects.get_or_create( + key=data.key, created=data.created, defaults=other) + data.db_id = dkey.id + + # set or update any additional data we might need to + needs_save = False + if created: + created_ct += 1 + else: + for k, v in other.items(): + if getattr(dkey, k) != v: + setattr(dkey, k, v) + needs_save = True + if dkey.owner_id is None: + owner = find_key_owner(data, keydata, finder) + if owner is not None: + dkey.owner = owner + needs_save = True + if needs_save: + dkey.save() + updated_ct += 1 + + key_ct = DeveloperKey.objects.all().count() + logger.info("%d total keys in database", key_ct) + logger.info("created %d, updated %d keys", created_ct, updated_ct) + + +SignatureData = namedtuple('SignatureData', + ('signer', 'signee', 'created', 'expires', 'valid')) + + +def parse_sigdata(data): + nodes = {} + edges = [] + current_pubkey = None + + # parse all of the output from our successful GPG command + logger.info("parsing command output") + for line in data.split('\n'): + parts = line.split(':') + if parts[0] == 'pub': + current_pubkey = parts[4] + nodes[current_pubkey] = None + if parts[0] == 'uid': + uid = parts[9] + # only set uid if this is the first one encountered + if nodes[current_pubkey] is None: + nodes[current_pubkey] = uid + if parts[0] == 'sig': + signer = parts[4] + created = get_date(parts[5]) + expires = None + if parts[6]: + expires = get_date(parts[6]) + valid = parts[1] != '-' + edge = SignatureData(signer, current_pubkey, + created, expires, valid) + edges.append(edge) + + return nodes, edges + + +def import_signatures(keyring): + outdata = call_gpg(keyring, "--list-sigs") + nodes, edges = parse_sigdata(outdata) + + # now prune the data down to what we actually want. + # prune edges not in nodes, remove duplicates, and self-sigs + pruned_edges = {edge for edge in edges + if edge.signer in nodes and edge.signer != edge.signee} + + logger.info("creating or finding %d signatures", len(pruned_edges)) + created_ct = updated_ct = 0 + with transaction.commit_on_success(): + for edge in pruned_edges: + sig, created = PGPSignature.objects.get_or_create( + signer=edge.signer, signee=edge.signee, + created=edge.created, expires=edge.expires, + defaults={ 'valid': edge.valid }) + if sig.valid != edge.valid: + sig.valid = edge.valid + sig.save() + updated_ct = 1 + if created: + created_ct += 1 + + sig_ct = PGPSignature.objects.all().count() + logger.info("%d total signatures in database", sig_ct) + logger.info("created %d, updated %d signatures", created_ct, updated_ct) + +# vim: set ts=4 sw=4 et: -- cgit v1.2.3-54-g00ecf From f98ff8cd22185c11dccdbe19b5bb7ed849b38e6b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 4 Feb 2013 20:02:00 -0600 Subject: Add './' hack to generate_keyring as well If you specify a relative path to gpg without a slash character, it interprets as relative to ~/.gnupg, which is stupid. Signed-off-by: Dan McGee --- devel/management/commands/generate_keyring.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'devel/management') diff --git a/devel/management/commands/generate_keyring.py b/devel/management/commands/generate_keyring.py index 15ae488d..34bcd2f8 100644 --- a/devel/management/commands/generate_keyring.py +++ b/devel/management/commands/generate_keyring.py @@ -55,6 +55,10 @@ def generate_keyring(keyserver, keyring): master_key_ids = MasterKey.objects.values_list("pgp_key", flat=True) logger.info("%d keys fetched from master keys", len(master_key_ids)) + # GPG is stupid and interprets any filename without path portion as being + # in ~/.gnupg/. Fake it out if we just get a bare filename. + if '/' not in keyring: + keyring = './%s' % keyring gpg_cmd = ["gpg", "--no-default-keyring", "--keyring", keyring, "--keyserver", keyserver, "--recv-keys"] logger.info("running command: %r", gpg_cmd) -- cgit v1.2.3-54-g00ecf From 55179a4f9e8b80d515bae7032af8aefc33ae0192 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 16 Jan 2013 16:07:26 -0600 Subject: reporead: remove batched_bulk_create Now that Django 1.5 is out and realized SQLite3 only allows for 999 parameters per SQL call, we don't need to manually batch things up anymore and can let the underlying bulk_create code do it for us. This basically reverts commit 88ee61a39ac3. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 32 +++++--------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index ab0efeed..ccac55f2 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -32,7 +32,6 @@ from devel.utils import UserFinder from main.models import Arch, Package, PackageFile, Repo -from main.utils import database_vendor from packages.models import Depend, Conflict, Provision, Replacement, Update from packages.utils import parse_version @@ -182,27 +181,6 @@ def create_related(model, package, rel_str, equals_only=False): return related -def batched_bulk_create(model, all_objects): - # for short lists, just bulk_create as we should be fine - if len(all_objects) < 20: - return model.objects.bulk_create(all_objects) - - if database_vendor(model, mode='write') == 'sqlite': - # 999 max variables in each SQL statement - incr = 999 // len(model._meta.fields) - else: - incr = 1000 - - def chunks(): - offset = 0 - while offset < len(all_objects): - yield all_objects[offset:offset + incr] - offset += incr - - for items in chunks(): - model.objects.bulk_create(items) - - def create_multivalued(dbpkg, repopkg, db_attr, repo_attr): '''Populate the simplest of multivalued attributes. These are those that only deal with a 'name' attribute, such as licenses, groups, etc. The input @@ -256,20 +234,20 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): deps += [create_depend(dbpkg, y, 'O') for y in repopkg.optdepends] deps += [create_depend(dbpkg, y, 'M') for y in repopkg.makedepends] deps += [create_depend(dbpkg, y, 'C') for y in repopkg.checkdepends] - batched_bulk_create(Depend, deps) + Depend.objects.bulk_create(deps) dbpkg.conflicts.all().delete() conflicts = [create_related(Conflict, dbpkg, y) for y in repopkg.conflicts] - batched_bulk_create(Conflict, conflicts) + Conflict.objects.bulk_create(conflicts) dbpkg.provides.all().delete() provides = [create_related(Provision, dbpkg, y, equals_only=True) for y in repopkg.provides] - batched_bulk_create(Provision, provides) + Provision.objects.bulk_create(provides) dbpkg.replaces.all().delete() replaces = [create_related(Replacement, dbpkg, y) for y in repopkg.replaces] - batched_bulk_create(Replacement, replaces) + Replacement.objects.bulk_create(replaces) create_multivalued(dbpkg, repopkg, 'groups', 'groups') create_multivalued(dbpkg, repopkg, 'licenses', 'license') @@ -319,7 +297,7 @@ def populate_files(dbpkg, repopkg, force=False): directory=dirname, filename=filename) pkg_files.append(pkgfile) - batched_bulk_create(PackageFile, pkg_files) + PackageFile.objects.bulk_create(pkg_files) dbpkg.files_last_update = now() dbpkg.save() -- cgit v1.2.3-54-g00ecf From 25aa164823a88f13761c750057833b6724808675 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 19 Mar 2013 00:10:52 -0500 Subject: Remove old-style build date parsing This was added in 2010 in commit e95c4563e32 as a short-term fix. The short-term is up. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'devel/management') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index ccac55f2..a0e77dc7 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -116,13 +116,9 @@ def populate(self, values): builddate = datetime.utcfromtimestamp(int(v[0])) self.builddate = builddate.replace(tzinfo=utc) except ValueError: - try: - self.builddate = datetime.strptime(v[0], - '%a %b %d %H:%M:%S %Y') - except ValueError: - logger.warning( - 'Package %s had unparsable build date %s', - self.name, v[0]) + logger.warning( + 'Package %s had unparsable build date %s', + self.name, v[0]) elif k == 'files': self.files = tuple(v) self.has_files = True -- cgit v1.2.3-54-g00ecf From b7b24740640e24883cd17fd683e1d465fbb343f8 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 16 Apr 2013 22:12:01 -0500 Subject: Various minor code cleanups and fixes Most of these were suggested by PyCharm, and include everything from little syntax issues and other bad smells to dead or bad code. Signed-off-by: Dan McGee --- devel/management/commands/pgp_import.py | 1 + devel/models.py | 1 - devel/utils.py | 2 +- devel/views.py | 2 +- main/log.py | 1 - main/migrations/0029_fill_in_repo_data.py | 1 - main/models.py | 12 +++++------- main/utils.py | 1 - mirrors/management/commands/mirrorcheck.py | 12 +++--------- mirrors/models.py | 2 +- mirrors/utils.py | 6 +++--- packages/migrations/0002_populate_package_relation.py | 2 -- packages/templatetags/package_extras.py | 4 ++-- packages/utils.py | 2 +- packages/views/display.py | 2 -- packages/views/flag.py | 3 +-- public/views.py | 1 - releng/management/commands/syncisos.py | 2 +- releng/models.py | 2 +- releng/views.py | 2 +- retro/templates/retro/index-20030330.html | 1 - sitestatic/archweb.js | 1 - todolists/utils.py | 1 - todolists/views.py | 1 - visualize/static/visualize.js | 2 +- 25 files changed, 23 insertions(+), 44 deletions(-) (limited to 'devel/management') diff --git a/devel/management/commands/pgp_import.py b/devel/management/commands/pgp_import.py index 10e6cfcb..b1f29d77 100644 --- a/devel/management/commands/pgp_import.py +++ b/devel/management/commands/pgp_import.py @@ -95,6 +95,7 @@ def parse_keydata(data): # parse all of the output from our successful GPG command logger.info("parsing command output") + node = None for line in data.split('\n'): parts = line.split(':') if parts[0] == 'pub': diff --git a/devel/models.py b/devel/models.py index 67de40a6..4354e0f2 100644 --- a/devel/models.py +++ b/devel/models.py @@ -4,7 +4,6 @@ from django.db import models from django.db.models.signals import pre_save from django.contrib.auth.models import User -from django.utils.timezone import now from django_countries import CountryField from .fields import PGPKeyField diff --git a/devel/utils.py b/devel/utils.py index e8e3a6c4..340841f5 100644 --- a/devel/utils.py +++ b/devel/utils.py @@ -131,7 +131,7 @@ def find(self, userstring): self.username_email, self.user_name) for matcher in find_methods: user = matcher(name, email) - if user != None: + if user is not None: break self.cache[userstring] = user diff --git a/devel/views.py b/devel/views.py index 61c1e568..4258ea7f 100644 --- a/devel/views.py +++ b/devel/views.py @@ -34,7 +34,7 @@ @login_required def index(request): '''the developer dashboard''' - if(request.user.is_authenticated()): + if request.user.is_authenticated(): inner_q = PackageRelation.objects.filter(user=request.user) else: inner_q = PackageRelation.objects.none() diff --git a/main/log.py b/main/log.py index 63634874..5c745cc8 100644 --- a/main/log.py +++ b/main/log.py @@ -46,7 +46,6 @@ def filter(self, record): trace = '\n'.join(traceback.format_exception(*record.exc_info)) key = md5(trace).hexdigest() - duplicate = False cache = self.cache_module.cache # Test if the cache works diff --git a/main/migrations/0029_fill_in_repo_data.py b/main/migrations/0029_fill_in_repo_data.py index 0887b28c..7da6b1c4 100644 --- a/main/migrations/0029_fill_in_repo_data.py +++ b/main/migrations/0029_fill_in_repo_data.py @@ -7,7 +7,6 @@ class Migration(DataMigration): def forwards(self, orm): - "Write your forwards methods here." orm.Repo.objects.filter(name__istartswith='community').update(bugs_project=5, svn_root='community') orm.Repo.objects.filter(name__iexact='multilib').update(bugs_project=5, svn_root='community') diff --git a/main/models.py b/main/models.py index 89215f05..24aeed89 100644 --- a/main/models.py +++ b/main/models.py @@ -7,7 +7,6 @@ from django.db.models import Q from django.contrib.auth.models import User from django.contrib.sites.models import Site -from django.utils.timezone import now from .fields import PositiveBigIntegerField from .utils import set_created_field @@ -140,7 +139,7 @@ def get_full_url(self, proto='https'): @property def signature(self): try: - data = b64decode(self.pgp_signature) + data = b64decode(self.pgp_signature.encode('utf-8')) except TypeError: return None if not data: @@ -274,7 +273,6 @@ def get_depends(self): Packages will match the testing status of this package if possible. """ deps = [] - arches = None # TODO: we can use list comprehension and an 'in' query to make this # more effective for dep in self.depends.all(): @@ -400,13 +398,13 @@ def elsewhere(self): '''attempt to locate this package anywhere else, regardless of architecture or repository. Excludes this package from the list.''' names = [self.pkgname] - if self.pkgname.startswith('lib32-'): + if self.pkgname.startswith(u'lib32-'): names.append(self.pkgname[6:]) - elif self.pkgname.endswith('-multilib'): + elif self.pkgname.endswith(u'-multilib'): names.append(self.pkgname[:-9]) else: - names.append('lib32-' + self.pkgname) - names.append(self.pkgname + '-multilib') + names.append(u'lib32-' + self.pkgname) + names.append(self.pkgname + u'-multilib') return Package.objects.normal().filter( pkgname__in=names).exclude(id=self.id).order_by( 'arch__name', 'repo__name') diff --git a/main/utils.py b/main/utils.py index 8394e5cd..9ee8db58 100644 --- a/main/utils.py +++ b/main/utils.py @@ -3,7 +3,6 @@ except ImportError: import pickle -from datetime import datetime import hashlib from django.core.cache import cache diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index d6de8f22..e7dd7b49 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -106,19 +106,13 @@ def parse_lastsync(log, data): def check_mirror_url(mirror_url, location, timeout): - if location: - if location.family == socket.AF_INET6: - ipopt = '--ipv6' - elif location.family == socket.AF_INET: - ipopt = '--ipv4' - url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) log = MirrorLog(url=mirror_url, check_time=now(), location=location) headers = {'User-Agent': 'archweb/1.0'} req = urllib2.Request(url, None, headers) + start = time.time() try: - start = time.time() result = urllib2.urlopen(req, timeout=timeout) data = result.read() result.close() @@ -147,12 +141,12 @@ def check_mirror_url(mirror_url, location, timeout): elif isinstance(e.reason, socket.error): log.error = e.reason.args[1] logger.debug("failed: %s, %s", url, log.error) - except HTTPException as e: + except HTTPException: # e.g., BadStatusLine log.is_success = False log.error = "Exception in processing HTTP request." logger.debug("failed: %s, %s", url, log.error) - except socket.timeout as e: + except socket.timeout: log.is_success = False log.error = "Connection timed out." logger.debug("failed: %s, %s", url, log.error) diff --git a/mirrors/models.py b/mirrors/models.py index 791b0078..d8ac7952 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -92,7 +92,7 @@ def clean(self): families = self.address_families() self.has_ipv4 = socket.AF_INET in families self.has_ipv6 = socket.AF_INET6 in families - except socket.error as e: + except socket.error: # We don't fail in this case; we'll just set both to False self.has_ipv4 = False self.has_ipv6 = False diff --git a/mirrors/utils.py b/mirrors/utils.py index 5a8bbf5d..531cf005 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,13 +1,13 @@ from datetime import timedelta from django.db import connection -from django.db.models import Avg, Count, Max, Min, StdDev +from django.db.models import Count, Max, Min from django.utils.dateparse import parse_datetime from django.utils.timezone import now from django_countries.fields import Country from main.utils import cache_function, database_vendor -from .models import MirrorLog, MirrorProtocol, MirrorUrl +from .models import MirrorLog, MirrorUrl DEFAULT_CUTOFF = timedelta(hours=24) @@ -165,7 +165,7 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): ).order_by('-last_occurred', '-error_count') if mirror_id: - urls = urls.filter(mirror_id=mirror_id) + errors = errors.filter(url__mirror_id=mirror_id) errors = list(errors) for err in errors: diff --git a/packages/migrations/0002_populate_package_relation.py b/packages/migrations/0002_populate_package_relation.py index 738e068f..b0d32c7a 100644 --- a/packages/migrations/0002_populate_package_relation.py +++ b/packages/migrations/0002_populate_package_relation.py @@ -11,7 +11,6 @@ class Migration(DataMigration): ) def forwards(self, orm): - "Write your forwards methods here." # search by pkgbase first and insert those records qs = orm['main.Package'].objects.exclude(maintainer=None).exclude( pkgbase=None).distinct().values('pkgbase', 'maintainer_id') @@ -29,7 +28,6 @@ def forwards(self, orm): defaults={'user_id': row['maintainer_id']}) def backwards(self, orm): - "Write your backwards methods here." if not db.dry_run: orm.PackageRelation.objects.all().delete() pass diff --git a/packages/templatetags/package_extras.py b/packages/templatetags/package_extras.py index f14fab1e..ef0e1aea 100644 --- a/packages/templatetags/package_extras.py +++ b/packages/templatetags/package_extras.py @@ -53,10 +53,10 @@ def do_buildsortqs(parser, token): tagname, sortfield = token.split_contents() except ValueError: raise template.TemplateSyntaxError( - "%r tag requires a single argument" % tagname) + "%r tag requires a single argument" % token) if not (sortfield[0] == sortfield[-1] and sortfield[0] in ('"', "'")): raise template.TemplateSyntaxError( - "%r tag's argument should be in quotes" % tagname) + "%r tag's argument should be in quotes" % token) return BuildQueryStringNode(sortfield[1:-1]) diff --git a/packages/utils.py b/packages/utils.py index a4217fbd..4f3b8665 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -391,7 +391,7 @@ def signoffs_id_query(model, repos): """ cursor = connection.cursor() # query pre-process- fill in table name and placeholders for IN - repo_sql = ','.join(['%s' for r in repos]) + repo_sql = ','.join(['%s' for _ in repos]) sql = sql % (model._meta.db_table, repo_sql, repo_sql) repo_ids = [r.pk for r in repos] # repo_ids are needed twice, so double the array diff --git a/packages/views/display.py b/packages/views/display.py index 87424483..021c7ed8 100644 --- a/packages/views/display.py +++ b/packages/views/display.py @@ -228,8 +228,6 @@ def download(request, name, repo, arch): if pkg.arch.agnostic: # grab the first non-any arch to fake the download path arch = Arch.objects.exclude(agnostic=True)[0].name - values = { - } url = '{host}{repo}/os/{arch}/{filename}'.format(host=url.url, repo=pkg.repo.name.lower(), arch=arch, filename=pkg.filename) return redirect(url) diff --git a/packages/views/flag.py b/packages/views/flag.py index 5c76e1d5..39cdcef8 100644 --- a/packages/views/flag.py +++ b/packages/views/flag.py @@ -110,7 +110,7 @@ def perform_updates(): subject = '%s package [%s] marked out-of-date' % \ (pkg.repo.name, pkg.pkgname) for maint in maints: - if maint.userprofile.notify == True: + if maint.userprofile.notify is True: toemail.append(maint.email) if toemail: @@ -133,7 +133,6 @@ def perform_updates(): return redirect('package-flag-confirmed', name=name, repo=repo, arch=arch) else: - initial = {} form = FlagForm(authenticated=authenticated) context = { diff --git a/public/views.py b/public/views.py index 22cb8759..39273396 100644 --- a/public/views.py +++ b/public/views.py @@ -125,7 +125,6 @@ def keys(request): master_keys = MasterKey.objects.select_related('owner', 'revoker', 'owner__userprofile', 'revoker__userprofile').filter( revoked__isnull=True) - master_key_ids = frozenset(key.pgp_key[-16:] for key in master_keys) sig_counts = PGPSignature.objects.filter(not_expired, valid=True, signee__in=user_key_ids).order_by().values_list('signer').annotate( diff --git a/releng/management/commands/syncisos.py b/releng/management/commands/syncisos.py index c9f61964..f182cc33 100644 --- a/releng/management/commands/syncisos.py +++ b/releng/management/commands/syncisos.py @@ -20,7 +20,7 @@ def handle_starttag(self, tag, attrs): if tag == 'a': for name, value in attrs: if name == "href": - if value != '../' and self.url_re.search(value) != None: + if value != '../' and self.url_re.search(value) is not None: self.hyperlinks.append(value[:-1]) def parse(self, url): diff --git a/releng/models.py b/releng/models.py index b95f7d52..5ee2f325 100644 --- a/releng/models.py +++ b/releng/models.py @@ -160,7 +160,7 @@ def info_html(self): def torrent(self): try: - data = b64decode(self.torrent_data) + data = b64decode(self.torrent_data.encode('utf-8')) except TypeError: return None if not data: diff --git a/releng/views.py b/releng/views.py index ad4b07d1..b1c76a4a 100644 --- a/releng/views.py +++ b/releng/views.py @@ -231,7 +231,7 @@ def release_torrent(request, version): release = get_object_or_404(Release, version=version) if not release.torrent_data: raise Http404 - data = b64decode(release.torrent_data) + data = b64decode(release.torrent_data.encode('utf-8')) response = HttpResponse(data, content_type='application/x-bittorrent') # TODO: this is duplicated from Release.iso_url() filename = 'archlinux-%s-dual.iso.torrent' % release.version diff --git a/retro/templates/retro/index-20030330.html b/retro/templates/retro/index-20030330.html index 449731af..51cc8ba3 100644 --- a/retro/templates/retro/index-20030330.html +++ b/retro/templates/retro/index-20030330.html @@ -232,7 +232,6 @@
    [ Older News ]

    -


    diff --git a/sitestatic/archweb.js b/sitestatic/archweb.js index dda22d9e..aa225f5f 100644 --- a/sitestatic/archweb.js +++ b/sitestatic/archweb.js @@ -146,7 +146,6 @@ if (typeof $ !== 'undefined' && typeof $.tablesorter !== 'undefined') { (function($) { $.fn.enableCheckboxRangeSelection = function() { var lastCheckbox = null, - lastElement = null, spec = this; spec.unbind("click.checkboxrange"); diff --git a/todolists/utils.py b/todolists/utils.py index 51a75a3c..7b98c887 100644 --- a/todolists/utils.py +++ b/todolists/utils.py @@ -1,5 +1,4 @@ from django.db import connections, router -from django.db.models import Count from .models import Todolist, TodolistPackage from packages.models import Package diff --git a/todolists/views.py b/todolists/views.py index 7636d38e..d5b39934 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -9,7 +9,6 @@ from django.views.decorators.cache import never_cache from django.views.generic import DeleteView from django.template import Context, loader -from django.template.defaultfilters import slugify from django.utils.timezone import now from main.models import Package, Repo diff --git a/visualize/static/visualize.js b/visualize/static/visualize.js index 7e240d44..5004fe6c 100644 --- a/visualize/static/visualize.js +++ b/visualize/static/visualize.js @@ -55,7 +55,7 @@ function packages_treemap(chart_id, orderings, default_order) { var nodes = d3_div.data([json]).selectAll("div") .data(treemap.nodes, key_func); /* start out new nodes in the center of the picture area */ - var w_center = jq_div.width() / 2; + var w_center = jq_div.width() / 2, h_center = jq_div.height() / 2; nodes.enter().append("div") .attr("class", "treemap-cell") -- cgit v1.2.3-54-g00ecf