From dc94eade03022ce3a5286f5e781576321a5f1653 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 27 Apr 2012 08:59:00 -0500 Subject: Incomplete-only todolists optimization We can push this down to the database if we know in advance we only need the incomplete lists. This helps our call on the developer dashboard quite a bit; the time of the single query in question drops from >1300ms to around 40ms. Signed-off-by: Dan McGee --- devel/views.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index d2ce65db..39f28a65 100644 --- a/devel/views.py +++ b/devel/views.py @@ -49,8 +49,7 @@ def index(request): todopkgs = todopkgs.filter(pkg__pkgbase__in=inner_q).order_by( 'list__name', 'pkg__pkgname') - todolists = get_annotated_todolists() - todolists = [todolist for todolist in todolists if todolist.incomplete_count > 0] + todolists = get_annotated_todolists(incomplete_only=True) signoffs = sorted(get_signoff_groups(user=request.user), key=operator.attrgetter('pkgbase')) -- cgit v1.2.3-54-g00ecf From 80e7d19726a95b40727b7f35b9ad80b436b14b93 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 27 Apr 2012 09:24:34 -0500 Subject: Dev dashboard performance improvement Rather than one query per cell in the arches and repos statistics tables, we can group these together up front using Django annotations. This means we only need one query per table. In my local instance with all of the staging repos imported, this reduces the total query count on this page from 56 to 26, a rather marked improvement. Signed-off-by: Dan McGee --- devel/views.py | 15 ++++++++++----- templates/devel/index.html | 8 ++++---- 2 files changed, 14 insertions(+), 9 deletions(-) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index 39f28a65..cf0d8ad2 100644 --- a/devel/views.py +++ b/devel/views.py @@ -13,7 +13,7 @@ from django.contrib.sites.models import Site from django.core.mail import send_mail from django.db import transaction -from django.db.models import F +from django.db.models import F, Count from django.http import Http404 from django.shortcuts import get_object_or_404 from django.template import loader, Context @@ -54,6 +54,11 @@ def index(request): signoffs = sorted(get_signoff_groups(user=request.user), key=operator.attrgetter('pkgbase')) + arches = Arch.objects.all().annotate( + total_ct=Count('packages'), flagged_ct=Count('packages__flag_date')) + repos = Repo.objects.all().annotate( + total_ct=Count('packages'), flagged_ct=Count('packages__flag_date')) + maintainers = get_annotated_maintainers() maintained = PackageRelation.objects.filter( @@ -70,12 +75,12 @@ def index(request): page_dict = { 'todos': todolists, - 'repos': Repo.objects.all(), - 'arches': Arch.objects.all(), + 'arches': arches, + 'repos': repos, 'maintainers': maintainers, 'orphan': orphan, - 'flagged' : flagged, - 'todopkgs' : todopkgs, + 'flagged': flagged, + 'todopkgs': todopkgs, 'signoffs': signoffs } diff --git a/templates/devel/index.html b/templates/devel/index.html index a0ccb91f..c67cf277 100644 --- a/templates/devel/index.html +++ b/templates/devel/index.html @@ -194,10 +194,10 @@

Stats by Architecture

{{ arch.name }} - {{ arch.packages.count }} packages + {{ arch.total_ct }} packages - {{ arch.packages.flagged.count }} packages + {{ arch.flagged_ct }} packages {% endfor %} @@ -224,10 +224,10 @@

Stats by Repository

{{ repo.name }} - {{ repo.packages.count }} packages + {{ repo.total_ct }} packages - {{ repo.packages.flagged.count }} packages + {{ repo.flagged_ct }} packages {% endfor %} -- cgit v1.2.3-54-g00ecf From 25a2fbc7c1cb50fa80ed4de50830721507765a91 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 29 Apr 2012 19:15:19 -0500 Subject: Add a "last action" column to developer clocks page This allows people to easily see if a developer has done anything recently that we can easily grab a date for. Obviously this doesn't include all sources of activity, so the list of things checked is clearly stated at the top. Signed-off-by: Dan McGee --- devel/views.py | 43 ++++++++++++++++++++++++++++++++++++------- templates/devel/clock.html | 21 ++++++++++++++++++--- 2 files changed, 54 insertions(+), 10 deletions(-) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index cf0d8ad2..85acda74 100644 --- a/devel/views.py +++ b/devel/views.py @@ -1,4 +1,4 @@ -from datetime import date, datetime, timedelta +from datetime import timedelta import operator import pytz import random @@ -9,24 +9,28 @@ from django.http import HttpResponseRedirect from django.contrib.auth.decorators import \ login_required, permission_required, user_passes_test +from django.contrib.admin.models import LogEntry, ADDITION from django.contrib.auth.models import User, Group +from django.contrib.contenttypes.models import ContentType from django.contrib.sites.models import Site from django.core.mail import send_mail from django.db import transaction -from django.db.models import F, Count +from django.db.models import F, Count, Max from django.http import Http404 from django.shortcuts import get_object_or_404 from django.template import loader, Context from django.template.defaultfilters import filesizeformat from django.views.decorators.cache import never_cache from django.views.generic.simple import direct_to_template +from django.utils.encoding import force_unicode from django.utils.http import http_date from .models import UserProfile from main.models import Package, PackageDepend, PackageFile, TodolistPkg from main.models import Arch, Repo from main.utils import utc_now -from packages.models import PackageRelation +from news.models import News +from packages.models import PackageRelation, Signoff from packages.utils import get_signoff_groups from todolists.utils import get_annotated_todolists from .utils import get_annotated_maintainers, UserFinder @@ -91,6 +95,33 @@ def clock(request): devs = User.objects.filter(is_active=True).order_by( 'first_name', 'last_name').select_related('userprofile') + latest_news = dict(News.objects.filter( + author__is_active=True).values_list('author').order_by( + ).annotate(last_post=Max('postdate'))) + latest_package = dict(Package.objects.filter( + packager__is_active=True).values_list('packager').order_by( + ).annotate(last_build=Max('build_date'))) + latest_signoff = dict(Signoff.objects.filter( + user__is_active=True).values_list('user').order_by( + ).annotate(last_signoff=Max('created'))) + latest_log = dict(LogEntry.objects.filter( + user__is_active=True).values_list('user').order_by( + ).annotate(last_log=Max('action_time'))) + + for dev in devs: + dates = [ + latest_news.get(dev.id, None), + latest_package.get(dev.id, None), + latest_signoff.get(dev.id, None), + latest_log.get(dev.id, None), + dev.last_login, + ] + dates = [d for d in dates if d is not None] + if dates: + dev.last_action = max(dates) + else: + dev.last_action = None + now = utc_now() page_dict = { 'developers': devs, @@ -135,7 +166,8 @@ class Meta: def change_profile(request): if request.POST: form = ProfileForm(request.POST) - profile_form = UserProfileForm(request.POST, request.FILES, instance=request.user.get_profile()) + profile_form = UserProfileForm(request.POST, request.FILES, + instance=request.user.get_profile()) if form.is_valid() and profile_form.is_valid(): request.user.email = form.cleaned_data['email'] if form.cleaned_data['passwd1']: @@ -331,9 +363,6 @@ def save(self, commit=True): def log_addition(request, obj): """Cribbed from ModelAdmin.log_addition.""" - from django.contrib.admin.models import LogEntry, ADDITION - from django.contrib.contenttypes.models import ContentType - from django.utils.encoding import force_unicode LogEntry.objects.log_action( user_id = request.user.pk, content_type_id = ContentType.objects.get_for_model(obj).pk, diff --git a/templates/devel/clock.html b/templates/devel/clock.html index bdb7341d..6a3f0a69 100644 --- a/templates/devel/clock.html +++ b/templates/devel/clock.html @@ -11,8 +11,18 @@

Developer World Clocks

This page helps prevent you from waking a sleeping developer. It also depends on developers keeping the time zone information up to date, so if you see 'UTC' listed, pester them to update their settings.

+

The "Last Action" column shows the last time this developer has done + something we know about. Considered dates for each developer include:

+
    +
  • Build date of a package in the repositories
    • +
  • Last login to the developer side of this site
    • +
  • Admin log entry on this site (e.g., adding a donor, modifying a + mirror)
    • +
  • Post date of a news item
    • +
  • Signing off on a package
    • +

- UTC Time: {{ utc_now|date:"Y-m-d H:i T" }} + Current UTC Time: {{ utc_now|date:"Y-m-d H:i T" }}

@@ -21,6 +31,7 @@

Developer World Clocks

+ @@ -32,6 +43,7 @@

Developer World Clocks

+ @@ -45,8 +57,11 @@

Developer World Clocks

{% endblock %} -- cgit v1.2.3-54-g00ecf From badc535aeb1d310a9b8aa59aade07045e6eae653 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 18 Apr 2012 15:05:43 -0500 Subject: Ensure order_by default value is cleared when using distinct() Otherwise the queryset returns nonsensical results. I find the design of this less than obvious but so be it; we can ensure the results work regardless of a default ordering on the model. Signed-off-by: Dan McGee --- devel/views.py | 6 ++++-- main/models.py | 7 ++++--- news/views.py | 3 ++- packages/utils.py | 3 ++- todolists/views.py | 4 ++-- 5 files changed, 14 insertions(+), 9 deletions(-) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index 85acda74..7ef33362 100644 --- a/devel/views.py +++ b/devel/views.py @@ -249,7 +249,8 @@ def report(request, report_name, username=None): if username: pkg_ids = set(packages.values_list('id', flat=True)) bad_files = bad_files.filter(pkg__in=pkg_ids) - bad_files = bad_files.values_list('pkg_id', flat=True).distinct() + bad_files = bad_files.values_list( + 'pkg_id', flat=True).order_by().distinct() packages = packages.filter(id__in=set(bad_files)) elif report_name == 'uncompressed-info': title = 'Packages with uncompressed infopages' @@ -260,7 +261,8 @@ def report(request, report_name, username=None): if username: pkg_ids = set(packages.values_list('id', flat=True)) bad_files = bad_files.filter(pkg__in=pkg_ids) - bad_files = bad_files.values_list('pkg_id', flat=True).distinct() + bad_files = bad_files.values_list( + 'pkg_id', flat=True).order_by().distinct() packages = packages.filter(id__in=set(bad_files)) elif report_name == 'unneeded-orphans': title = 'Orphan packages required by no other packages' diff --git a/main/models.py b/main/models.py index c532ed56..398cb88b 100644 --- a/main/models.py +++ b/main/models.py @@ -13,7 +13,7 @@ class TodolistManager(models.Manager): def incomplete(self): - return self.filter(todolistpkg__complete=False).distinct() + return self.filter(todolistpkg__complete=False).order_by().distinct() class PackageManager(models.Manager): def flagged(self): @@ -378,7 +378,7 @@ def get_providers(self, arches=None, testing=None, staging=None): '''Return providers of this dep. Does *not* include exact matches as it checks the Provision names only, use get_best_satisfier() instead.''' pkgs = Package.objects.normal().filter( - provides__name=self.depname).distinct() + provides__name=self.depname).order_by().distinct() if arches is not None: pkgs = pkgs.filter(arch__in=arches) @@ -432,7 +432,8 @@ def packages(self): @property def package_names(self): # depends on packages property returning a queryset - return self.packages.values_list('pkg__pkgname', flat=True).distinct() + return self.packages.values_list( + 'pkg__pkgname', flat=True).order_by().distinct() class Meta: db_table = 'todolists' diff --git a/news/views.py b/news/views.py index 268f0523..03f3b0ac 100644 --- a/news/views.py +++ b/news/views.py @@ -13,7 +13,8 @@ def find_unique_slug(newsitem): '''Attempt to find a unique slug for this news item.''' - existing = list(News.objects.values_list('slug', flat=True).distinct()) + existing = list(News.objects.values_list( + 'slug', flat=True).order_by().distinct()) suffixed = slug = slugify(newsitem.title) suffix = 0 diff --git a/packages/utils.py b/packages/utils.py index a3c13b17..8d00bd68 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -218,7 +218,8 @@ def attach_maintainers(packages): packages = list(packages) pkgbases = set(p.pkgbase for p in packages) rels = PackageRelation.objects.filter(type=PackageRelation.MAINTAINER, - pkgbase__in=pkgbases).values_list('pkgbase', 'user_id').distinct() + pkgbase__in=pkgbases).values_list( + 'pkgbase', 'user_id').order_by().distinct() # get all the user objects we will need user_ids = set(rel[1] for rel in rels) diff --git a/todolists/views.py b/todolists/views.py index e5cc0823..70209b6d 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -49,8 +49,8 @@ def flag(request, list_id, pkg_id): @login_required def view(request, list_id): todolist = get_object_or_404(Todolist, id=list_id) - svn_roots = Repo.objects.order_by().values_list( - 'svn_root', flat=True).distinct() + svn_roots = Repo.objects.values_list( + 'svn_root', flat=True).order_by().distinct() # we don't hold onto the result, but the objects are the same here, # so accessing maintainers in the template is now cheap attach_maintainers(tp.pkg for tp in todolist.packages) -- cgit v1.2.3-54-g00ecf From cf8ecdf9fce0573ad207d024708f21a5dbbbb120 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 2 May 2012 09:46:52 -0500 Subject: rematch_developers: do mass updates instead of single saves When updating a lot of objects, it makes much more sense to perform targeted update queries rather than one-row-at-a-time saves. Signed-off-by: Dan McGee --- devel/management/commands/rematch_developers.py | 61 ++++++++++++------------- 1 file changed, 30 insertions(+), 31 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/rematch_developers.py b/devel/management/commands/rematch_developers.py index 8383cc8d..ab2f0f4b 100644 --- a/devel/management/commands/rematch_developers.py +++ b/devel/management/commands/rematch_developers.py @@ -46,52 +46,51 @@ def handle_noargs(self, **options): @transaction.commit_on_success def match_packager(finder): - logger.info("getting all unmatched packages") + logger.info("getting all unmatched packager strings") package_count = matched_count = 0 - unknown = set() - - for package in Package.objects.filter(packager__isnull=True): - if package.packager_str in unknown: - continue - logger.debug("package %s, packager string %s", - package.pkgname, package.packager_str) - package_count += 1 - user = finder.find(package.packager_str) + mapping = {} + + unmatched = Package.objects.filter(packager__isnull=True).values_list( + 'packager_str', flat=True).order_by().distinct() + + for packager in unmatched: + logger.debug("packager string %s", packager) + user = finder.find(packager) if user: - package.packager = user + mapping[packager] = user logger.debug(" found user %s" % user.username) - package.save() matched_count += 1 - else: - unknown.add(package.packager_str) - logger.info("%d packager strings checked, %d newly matched", + for packager_str, user in mapping.items(): + package_count += Package.objects.filter(packager__isnull=True, + packager_str=packager_str).update(packager=user) + + logger.info("%d packages updated, %d packager strings matched", package_count, matched_count) - logger.debug("unknown packagers:\n%s", - "\n".join(unknown)) @transaction.commit_on_success def match_flagrequest(finder): - logger.info("getting all non-user flag requests") + logger.info("getting all flag requests emails from unknown users") req_count = matched_count = 0 - unknown = set() - - for request in FlagRequest.objects.filter(user__isnull=True): - if request.user_email in unknown: - continue - logger.debug("email %s", request.user_email) - req_count += 1 - user = finder.find_by_email(request.user_email) + mapping = {} + + unmatched = FlagRequest.objects.filter(user__isnull=True).values_list( + 'user_email', flat=True).order_by().distinct() + + for user_email in unmatched: + logger.debug("email %s", user_email) + user = finder.find_by_email(user_email) if user: - request.user = user + mapping[user_email] = user logger.debug(" found user %s" % user.username) - request.save() matched_count += 1 - else: - unknown.add(request.user_email) - logger.info("%d request emails checked, %d newly matched", + for user_email, user in mapping.items(): + req_count += FlagRequest.objects.filter(user__isnull=True, + user_email=user_email).update(user=user) + + logger.info("%d request emails updated, %d emails matched", req_count, matched_count) # vim: set ts=4 sw=4 et: -- cgit v1.2.3-54-g00ecf From f36d876aca5571f09032d0d2a67c8b1f1a3258c8 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 12 May 2012 09:22:52 -0500 Subject: Change to new time access methods in pgpdump code Signed-off-by: Dan McGee --- devel/views.py | 2 +- templates/packages/details.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index 7ef33362..0f1c8d15 100644 --- a/devel/views.py +++ b/devel/views.py @@ -280,7 +280,7 @@ def report(request, report_name, username=None): filtered = [] packages = packages.filter(pgp_signature__isnull=False) for package in packages: - sig_date = package.signature.datetime.replace(tzinfo=pytz.utc) + sig_date = package.signature.creation_time.replace(tzinfo=pytz.utc) package.sig_date = sig_date.date() key_id = package.signature.key_id signer = finder.find_by_pgp_key(key_id) diff --git a/templates/packages/details.html b/templates/packages/details.html index 4ab55ef2..4cb6032e 100644 --- a/templates/packages/details.html +++ b/templates/packages/details.html @@ -165,7 +165,7 @@

Versions Elsewhere

 - + {% else %} -- cgit v1.2.3-54-g00ecf From 72a92102df4999dbcc370064707c9026d51c4fe7 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 18 May 2012 21:29:03 -0500 Subject: Switch to usage of new Depend object Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 18 +++++++++------- devel/views.py | 6 +++--- main/models.py | 7 +++--- packages/models.py | 34 ++++++++++++++++++++++++++---- packages/utils.py | 6 +++--- templates/packages/details_depend.html | 6 +++--- templates/packages/details_requiredby.html | 2 +- 7 files changed, 54 insertions(+), 25 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index fd8e3979..47294d9a 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -29,9 +29,9 @@ from django.db.utils import IntegrityError from devel.utils import UserFinder -from main.models import Arch, Package, PackageDepend, PackageFile, Repo +from main.models import Arch, Package, PackageFile, Repo from main.utils import utc_now -from packages.models import Conflict, Provision, Replacement +from packages.models import Depend, Conflict, Provision, Replacement logging.basicConfig( @@ -141,19 +141,21 @@ def full_version(self): return u'%s-%s' % (self.ver, self.rel) -DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.*))?$") +DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.+))?$") def create_depend(package, dep_str, optional=False): - depend = PackageDepend(pkg=package, optional=optional) + depend = Depend(pkg=package, optional=optional) # lop off any description first parts = dep_str.split(':', 1) if len(parts) > 1: depend.description = parts[1].strip() match = DEPEND_RE.match(parts[0].strip()) if match: - depend.depname = match.group(1) - if match.group(2): - depend.depvcmp = match.group(2) + depend.name = match.group(1) + if match.group(3): + depend.comparison = match.group(3) + if match.group(4): + related.version = match.group(4) else: logger.warning('Package %s had unparsable depend string %s', package.pkgname, dep_str) @@ -232,7 +234,7 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): dbpkg.depends.all().delete() deps = [create_depend(dbpkg, y) for y in repopkg.depends] deps += [create_depend(dbpkg, y, True) for y in repopkg.optdepends] - PackageDepend.objects.bulk_create(deps) + Depend.objects.bulk_create(deps) dbpkg.conflicts.all().delete() conflicts = [create_related(Conflict, dbpkg, y) for y in repopkg.conflicts] diff --git a/devel/views.py b/devel/views.py index 0f1c8d15..16b6acc6 100644 --- a/devel/views.py +++ b/devel/views.py @@ -26,11 +26,11 @@ from django.utils.http import http_date from .models import UserProfile -from main.models import Package, PackageDepend, PackageFile, TodolistPkg +from main.models import Package, PackageFile, TodolistPkg from main.models import Arch, Repo from main.utils import utc_now from news.models import News -from packages.models import PackageRelation, Signoff +from packages.models import PackageRelation, Signoff, Depend from packages.utils import get_signoff_groups from todolists.utils import get_annotated_todolists from .utils import get_annotated_maintainers, UserFinder @@ -267,7 +267,7 @@ def report(request, report_name, username=None): elif report_name == 'unneeded-orphans': title = 'Orphan packages required by no other packages' owned = PackageRelation.objects.all().values('pkgbase') - required = PackageDepend.objects.all().values('depname') + required = Depend.objects.all().values('name') # The two separate calls to exclude is required to do the right thing packages = packages.exclude(pkgbase__in=owned).exclude( pkgname__in=required) diff --git a/main/models.py b/main/models.py index 4b445dd0..f17d4a4d 100644 --- a/main/models.py +++ b/main/models.py @@ -180,11 +180,12 @@ def get_requiredby(self): list slim by including the corresponding package in the same testing category as this package if that check makes sense. """ + from packages.models import Depend provides = set(self.provides.values_list('name', flat=True)) provides.add(self.pkgname) - requiredby = PackageDepend.objects.select_related('pkg', + requiredby = Depend.objects.select_related('pkg', 'pkg__arch', 'pkg__repo').filter( - depname__in=provides).order_by( + name__in=provides).order_by( 'pkg__pkgname', 'pkg__arch__name', 'pkg__repo__name') if not self.arch.agnostic: # make sure we match architectures if possible @@ -232,7 +233,7 @@ def get_depends(self): deps = [] arches = None # TODO: we can use list comprehension and an 'in' query to make this more effective - for dep in self.depends.order_by('optional', 'depname'): + for dep in self.depends.order_by('optional', 'name'): pkg = dep.get_best_satisfier() providers = None if not pkg: diff --git a/packages/models.py b/packages/models.py index c7b1cab4..cb65f1f1 100644 --- a/packages/models.py +++ b/packages/models.py @@ -228,10 +228,6 @@ def get_best_satisfier(self): '''Find a satisfier for this related package that best matches the given criteria. It will not search provisions, but will find packages named and matching repo characteristics if possible.''' - # NOTE: this is cribbed directly from the PackageDepend method of the - # same name. Really, all of these things could use the same method if - # the PackageDepend class was moved here and field names were changed - # to match the layout we use here. pkgs = Package.objects.normal().filter(pkgname=self.name) if not self.pkg.arch.agnostic: # make sure we match architectures if possible @@ -258,6 +254,36 @@ def get_best_satisfier(self): return pkg + def get_providers(self): + '''Return providers of this related package. Does *not* include exact + matches as it checks the Provision names only, use get_best_satisfier() + instead for exact matches.''' + pkgs = Package.objects.normal().filter( + provides__name=self.name).order_by().distinct() + if not self.pkg.arch.agnostic: + # make sure we match architectures if possible + arches = self.pkg.applicable_arches() + pkgs = pkgs.filter(arch__in=arches) + + # Logic here is to filter out packages that are in multiple repos if + # they are not requested. For example, if testing is False, only show a + # testing package if it doesn't exist in a non-testing repo. + filtered = {} + for package in pkgs: + if package.pkgname not in filtered or \ + package.repo.staging == self.pkg.repo.staging: + filtered[package.pkgname] = package + pkgs = filtered.values() + + filtered = {} + for package in pkgs: + if package.pkgname not in filtered or \ + package.repo.testing == self.pkg.repo.testing: + filtered[package.pkgname] = package + pkgs = filtered.values() + + return pkgs + def __unicode__(self): if self.version: return u'%s%s%s' % (self.name, self.comparison, self.version) diff --git a/packages/utils.py b/packages/utils.py index 8d00bd68..82313472 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -7,10 +7,10 @@ from django.db.models import Count, Max, F from django.contrib.auth.models import User -from main.models import Package, PackageDepend, PackageFile, Arch, Repo +from main.models import Package, PackageFile, Arch, Repo from main.utils import cache_function, groupby_preserve_order, PackageStandin from .models import (PackageGroup, PackageRelation, - License, Conflict, Provision, Replacement, + License, Depend, Conflict, Provision, Replacement, SignoffSpecification, Signoff, DEFAULT_SIGNOFF_SPEC) @cache_function(127) @@ -451,7 +451,7 @@ def default(self, obj): return obj.name.lower() if isinstance(obj, (PackageGroup, License)): return obj.name - if isinstance(obj, (Conflict, Provision, Replacement, PackageDepend)): + if isinstance(obj, (Depend, Conflict, Provision, Replacement)): return unicode(obj) elif isinstance(obj, User): return obj.username diff --git a/templates/packages/details_depend.html b/templates/packages/details_depend.html index 8b6e85c9..0cf2c36a 100644 --- a/templates/packages/details_depend.html +++ b/templates/packages/details_depend.html @@ -2,12 +2,12 @@
  • {% ifequal depend.pkg None %} {% if depend.providers %} -{{ depend.dep.depname }} ({% multi_pkg_details depend.providers %}) +{{ depend.dep.name }}{{ depend.dep.comparison|default:"" }}{{ depend.dep.version|default:"" }} ({% multi_pkg_details depend.providers %}) {% else %} -{{ depend.dep.depname }} (virtual) +{{ depend.dep.name }}{{ depend.dep.comparison|default:"" }}{{ depend.dep.version|default:"" }} (virtual) {% endif %} {% else %} -{% pkg_details_link depend.pkg %}{{ depend.dep.depvcmp|default:"" }} +{% pkg_details_link depend.pkg %}{{ depend.dep.comparison|default:"" }}{{ depend.dep.version|default:"" }} {% if depend.pkg.repo.testing %} (testing){% endif %} {% if depend.pkg.repo.staging %} (staging){% endif %} {% endifequal %} diff --git a/templates/packages/details_requiredby.html b/templates/packages/details_requiredby.html index c7697289..ecc92b29 100644 --- a/templates/packages/details_requiredby.html +++ b/templates/packages/details_requiredby.html @@ -1,6 +1,6 @@ {% load package_extras %}
  • {% pkg_details_link req.pkg %} -{% if req.depname != pkg.pkgname %}(requires {{ req.depname }}){% endif %} +{% if req.name != pkg.pkgname %}(requires {{ req.name }}){% endif %} {% if req.pkg.repo.testing %}(testing){% endif %} {% if req.pkg.repo.staging %}(staging){% endif %} {% if req.optional %}(optional){% endif %} -- cgit v1.2.3-54-g00ecf From 3f9aeb45c2a2b498a389bacc92b5f56d9feb4329 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 19 May 2012 09:54:15 -0500 Subject: reporead: fix copy/paste issue Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 47294d9a..2e8c4625 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -155,7 +155,7 @@ def create_depend(package, dep_str, optional=False): if match.group(3): depend.comparison = match.group(3) if match.group(4): - related.version = match.group(4) + depend.version = match.group(4) else: logger.warning('Package %s had unparsable depend string %s', package.pkgname, dep_str) -- cgit v1.2.3-54-g00ecf From 1b03069f0d9e0397a7ff07404343c9400bbcfa1c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 7 Jun 2012 20:53:52 -0500 Subject: Use 3 decimal places for showing compression ratio Otherwise there are too many grouped under each value. Signed-off-by: Dan McGee --- devel/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index 16b6acc6..78ed26f2 100644 --- a/devel/views.py +++ b/devel/views.py @@ -234,7 +234,7 @@ def report(request, report_name, username=None): package.installed_size_pretty = filesizeformat( package.installed_size) ratio = package.compressed_size / float(package.installed_size) - package.ratio = '%.2f' % ratio + package.ratio = '%.3f' % ratio package.compress_type = package.filename.split('.')[-1] elif report_name == 'uncompressed-man': title = 'Packages with uncompressed manpages' -- cgit v1.2.3-54-g00ecf From bbcbde0197d4862b5acc595b17bc5051780dbc9e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 20 Jun 2012 17:03:26 -0500 Subject: Add a last_modified field to user profiles A behind the scenes field that might be slightly useful. Signed-off-by: Dan McGee --- ...08_auto__add_field_userprofile_last_modified.py | 108 +++++++++++++++++++++ devel/models.py | 18 +++- 2 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 devel/migrations/0008_auto__add_field_userprofile_last_modified.py (limited to 'devel') diff --git a/devel/migrations/0008_auto__add_field_userprofile_last_modified.py b/devel/migrations/0008_auto__add_field_userprofile_last_modified.py new file mode 100644 index 00000000..2695987a --- /dev/null +++ b/devel/migrations/0008_auto__add_field_userprofile_last_modified.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + def forwards(self, orm): + db.add_column('user_profiles', 'last_modified', + self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime(2000, 1, 1, 0, 0)), + keep_default=False) + + def backwards(self, orm): + db.delete_column('user_profiles', 'last_modified') + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'devel.masterkey': { + 'Meta': {'ordering': "('created',)", 'object_name': 'MasterKey'}, + 'created': ('django.db.models.fields.DateField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'owner': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'masterkey_owner'", 'to': "orm['auth.User']"}), + 'pgp_key': ('devel.fields.PGPKeyField', [], {'max_length': '40'}), + 'revoked': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'revoker': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'masterkey_revoker'", 'to': "orm['auth.User']"}) + }, + 'devel.pgpsignature': { + 'Meta': {'object_name': 'PGPSignature'}, + 'created': ('django.db.models.fields.DateField', [], {}), + 'expires': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'signee': ('devel.fields.PGPKeyField', [], {'max_length': '40'}), + 'signer': ('devel.fields.PGPKeyField', [], {'max_length': '40'}), + 'valid': ('django.db.models.fields.BooleanField', [], {'default': 'True'}) + }, + 'devel.userprofile': { + 'Meta': {'object_name': 'UserProfile', 'db_table': "'user_profiles'"}, + 'alias': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'allowed_repos': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['main.Repo']", 'symmetrical': 'False', 'blank': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2', 'blank': 'True'}), + 'favorite_distros': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interests': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'languages': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'last_modified': ('django.db.models.fields.DateTimeField', [], {}), + 'latin_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'location': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'notify': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'occupation': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'other_contact': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'pgp_key': ('devel.fields.PGPKeyField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}), + 'picture': ('django.db.models.fields.files.FileField', [], {'default': "'devs/silhouette.png'", 'max_length': '100'}), + 'public_email': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'roles': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'time_zone': ('django.db.models.fields.CharField', [], {'default': "'UTC'", 'max_length': '100'}), + 'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'userprofile'", 'unique': 'True', 'to': "orm['auth.User']"}), + 'website': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'yob': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) + }, + 'main.repo': { + 'Meta': {'ordering': "['name']", 'object_name': 'Repo', 'db_table': "'repos'"}, + 'bugs_category': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'bugs_project': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'staging': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'svn_root': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + } + } + + complete_apps = ['devel'] diff --git a/devel/models.py b/devel/models.py index fd5a0347..fd5df00a 100644 --- a/devel/models.py +++ b/devel/models.py @@ -2,11 +2,12 @@ import pytz from django.db import models +from django.db.models.signals import pre_save from django.contrib.auth.models import User from django_countries import CountryField from .fields import PGPKeyField -from main.utils import make_choice +from main.utils import make_choice, utc_now class UserProfile(models.Model): @@ -44,6 +45,7 @@ class UserProfile(models.Model): allowed_repos = models.ManyToManyField('main.Repo', blank=True) latin_name = models.CharField(max_length=255, null=True, blank=True, help_text="Latin-form name; used only for non-Latin full names") + last_modified = models.DateTimeField(editable=False) class Meta: db_table = 'user_profiles' @@ -96,4 +98,18 @@ class Meta: def __unicode__(self): return u'%s → %s' % (self.signer, self.signee) + +def set_last_modified(sender, **kwargs): + '''This will set the 'last_modified' field on the user profile to the + current UTC time when either the profile is updated. For use as a pre_save + signal handler.''' + obj = kwargs['instance'] + if hasattr(obj, 'last_modified'): + obj.last_modified = utc_now() + + +# connect signals needed to keep cache in line with reality +pre_save.connect(set_last_modified, sender=UserProfile, + dispatch_uid="devel.models") + # vim: set ts=4 sw=4 et: -- cgit v1.2.3-54-g00ecf From a87fe016d1a1bf7fdcd2b19f515aa72a5b93db2b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 1 Jul 2012 20:21:34 -0500 Subject: Log package updates during reporead invocation This adds a Manager and log_update method to help log all updates made to the packages table during reporead runs. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 7 ++++++- packages/models.py | 36 +++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 2e8c4625..4e242af1 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -14,6 +14,7 @@ """ from collections import defaultdict +from copy import copy import io import os import re @@ -31,7 +32,7 @@ from devel.utils import UserFinder from main.models import Arch, Package, PackageFile, Repo from main.utils import utc_now -from packages.models import Depend, Conflict, Provision, Replacement +from packages.models import Depend, Conflict, Provision, Replacement, Update logging.basicConfig( @@ -362,6 +363,7 @@ def db_update(archname, reponame, pkgs, force=False): try: with transaction.commit_on_success(): populate_pkg(dbpkg, pkg, timestamp=utc_now()) + Update.objects.log_update(None, dbpkg) except IntegrityError: logger.warning("Could not add package %s; " "not fatal if another thread beat us to it.", @@ -372,6 +374,7 @@ def db_update(archname, reponame, pkgs, force=False): logger.info("Removing package %s", pkgname) dbpkg = dbdict[pkgname] with transaction.commit_on_success(): + Update.objects.log_update(dbpkg, None) # no race condition here as long as simultaneous threads both # issue deletes; second delete will be a no-op delete_pkg_files(dbpkg) @@ -399,7 +402,9 @@ def db_update(archname, reponame, pkgs, force=False): logger.debug("Package %s was already updated", pkg.name) continue logger.info("Updating package %s", pkg.name) + prevpkg = copy(dbpkg) populate_pkg(dbpkg, pkg, force=force, timestamp=timestamp) + Update.objects.log_update(prevpkg, dbpkg) logger.info('Finished updating arch: %s', archname) diff --git a/packages/models.py b/packages/models.py index 5ee06575..04f35f9d 100644 --- a/packages/models.py +++ b/packages/models.py @@ -203,6 +203,40 @@ def __unicode__(self): ) +class UpdateManager(models.Manager): + def log_update(self, old_pkg, new_pkg): + '''Utility method to help log an update. This will determine the type + based on how many packages are passed in, and will pull the relevant + necesary fields off the given packages.''' + update = Update() + if new_pkg: + update.action_flag = ADDITION + update.package = new_pkg + update.arch = new_pkg.arch + update.repo = new_pkg.repo + update.pkgname = new_pkg.pkgname + update.pkgbase = new_pkg.pkgbase + update.new_pkgver = new_pkg.pkgver + update.new_pkgrel = new_pkg.pkgrel + update.new_epoch = new_pkg.epoch + if old_pkg: + if new_pkg: + update.action_flag = CHANGE + else: + update.action_flag = DELETION + update.arch = old_pkg.arch + update.repo = old_pkg.repo + update.pkgname = old_pkg.pkgname + update.pkgbase = old_pkg.pkgbase + + update.old_pkgver = old_pkg.pkgver + update.old_pkgrel = old_pkg.pkgrel + update.old_epoch = old_pkg.epoch + + update.save(force_insert=True) + return update + + class Update(models.Model): package = models.ForeignKey(Package, related_name="updates", null=True, on_delete=models.SET_NULL) @@ -222,6 +256,8 @@ class Update(models.Model): new_pkgrel = models.CharField(max_length=255, null=True) new_epoch = models.PositiveIntegerField(null=True) + objects = UpdateManager() + class Meta: get_latest_by = 'created' -- cgit v1.2.3-54-g00ecf From daf011b67a338f26ead8058a9f9caedfe251c62c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 5 Jul 2012 11:25:00 -0400 Subject: reporead: properly handle cases where last_update == files_last_update We should assume the filelists are up to date in this case, not out of date. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 4e242af1..51c73c02 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -273,7 +273,7 @@ def populate_files(dbpkg, repopkg, force=False): return if not dbpkg.files_last_update or not dbpkg.last_update: pass - elif dbpkg.files_last_update > dbpkg.last_update: + elif dbpkg.files_last_update >= dbpkg.last_update: return # only delete files if we are reading a DB that contains them @@ -427,7 +427,7 @@ def filesonly_update(archname, reponame, pkgs, force=False): with transaction.commit_on_success(): if not dbpkg.files_last_update or not dbpkg.last_update: pass - elif not force and dbpkg.files_last_update > dbpkg.last_update: + elif not force and dbpkg.files_last_update >= dbpkg.last_update: logger.debug("Files for %s are up to date", pkg.name) continue dbpkg = Package.objects.select_for_update().get(id=dbpkg.id) -- cgit v1.2.3-54-g00ecf From 9d91cad678133e97345111fab2c103fcda9b9f28 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 5 Jul 2012 11:25:40 -0400 Subject: reporead: handle files in root directory properly Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 51c73c02..df29a8a7 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -283,7 +283,10 @@ def populate_files(dbpkg, repopkg, force=False): len(repopkg.files), dbpkg.pkgname) pkg_files = [] for f in repopkg.files: - dirname, filename = f.rsplit('/', 1) + if '/' in f: + dirname, filename = f.rsplit('/', 1) + else: + dirname, filename = '', f if filename == '': filename = None pkgfile = PackageFile(pkg=dbpkg, -- cgit v1.2.3-54-g00ecf From 909cb9a209b4a4db00232b3a62656f95c4b88d45 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 5 Jul 2012 17:09:55 -0500 Subject: reporead: don't append slash to empty (root) directory Add the slash only if we have a directory name, and not otherwise. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index df29a8a7..43578d4a 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -285,13 +285,14 @@ def populate_files(dbpkg, repopkg, force=False): for f in repopkg.files: if '/' in f: dirname, filename = f.rsplit('/', 1) + dirname += '/' else: dirname, filename = '', f if filename == '': filename = None pkgfile = PackageFile(pkg=dbpkg, is_directory=(filename is None), - directory=dirname + '/', + directory=dirname, filename=filename) pkg_files.append(pkgfile) PackageFile.objects.bulk_create(pkg_files) -- cgit v1.2.3-54-g00ecf From a1ec14fc68282d67c00c79b5aa6aab60461f056a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 12 Mar 2012 13:14:49 -0400 Subject: reporead: disable FULL synchronous writes for sqlite3 At least on Linux, we hit a huge bottleneck waiting for the FULL commit to happen for each added package during reporead operations. It makes much more sense to back this off to FULL level instead, which trades some possible loss of durability for speedier operation. Additionally, no one would possibly be running their production version of this site on sqlite3, right? Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 43578d4a..e50686b1 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -547,6 +547,12 @@ def read_repo(primary_arch, repo_file, options): package.name, repo_file, package.arch)) del packages + database = router.db_for_write(Package) + connection = connections[database] + if connection.vendor == 'sqlite': + cursor = connection.cursor() + cursor.execute('PRAGMA synchronous = NORMAL') + logger.info('Starting database updates for %s.', repo_file) for arch in sorted(packages_arches.keys()): if filesonly: -- cgit v1.2.3-54-g00ecf From 88ee61a39ac3690267f2b7903f3646972e8f055d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 8 Jul 2012 21:24:48 -0500 Subject: Work around bulk_create limitations in sqlite3 in reporead Given the 999 SQL statement variable limit, we can easily hit it when updating a package with thousands of files or a few hundred depends. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index e50686b1..2d9b68b2 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -31,7 +31,7 @@ from devel.utils import UserFinder from main.models import Arch, Package, PackageFile, Repo -from main.utils import utc_now +from main.utils import utc_now, database_vendor from packages.models import Depend, Conflict, Provision, Replacement, Update @@ -184,6 +184,28 @@ def create_related(model, package, rel_str, equals_only=False): return None return related + +def batched_bulk_create(model, all_objects): + # for short lists, just bulk_create as we should be fine + if len(all_objects) < 20: + return model.objects.bulk_create(all_objects) + + if database_vendor(model, mode='write') == 'sqlite': + # 999 max variables in each SQL statement + incr = 999 // len(model._meta.fields) + else: + incr = 1000 + + def chunks(): + offset = 0 + while offset < len(all_objects): + yield all_objects[offset:offset + incr] + offset += incr + + for items in chunks(): + model.objects.bulk_create(items) + + def create_multivalued(dbpkg, repopkg, db_attr, repo_attr): '''Populate the simplest of multivalued attributes. These are those that only deal with a 'name' attribute, such as licenses, groups, etc. The input @@ -235,20 +257,20 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): dbpkg.depends.all().delete() deps = [create_depend(dbpkg, y) for y in repopkg.depends] deps += [create_depend(dbpkg, y, True) for y in repopkg.optdepends] - Depend.objects.bulk_create(deps) + batched_bulk_create(Depend, deps) dbpkg.conflicts.all().delete() conflicts = [create_related(Conflict, dbpkg, y) for y in repopkg.conflicts] - Conflict.objects.bulk_create(conflicts) + batched_bulk_create(Conflict, conflicts) dbpkg.provides.all().delete() provides = [create_related(Provision, dbpkg, y, equals_only=True) for y in repopkg.provides] - Provision.objects.bulk_create(provides) + batched_bulk_create(Provision, provides) dbpkg.replaces.all().delete() replaces = [create_related(Replacement, dbpkg, y) for y in repopkg.replaces] - Replacement.objects.bulk_create(replaces) + batched_bulk_create(Replacement, replaces) create_multivalued(dbpkg, repopkg, 'groups', 'groups') create_multivalued(dbpkg, repopkg, 'licenses', 'license') @@ -295,7 +317,7 @@ def populate_files(dbpkg, repopkg, force=False): directory=dirname, filename=filename) pkg_files.append(pkgfile) - PackageFile.objects.bulk_create(pkg_files) + batched_bulk_create(PackageFile, pkg_files) dbpkg.files_last_update = utc_now() dbpkg.save() -- cgit v1.2.3-54-g00ecf From c0bf9e20660cfae7ea8994472555bba23398b598 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 24 Jul 2012 09:19:48 -0500 Subject: Remove custom utc_now() function, use django.utils.timezone.now() This was around from the time when we handled timezones sanely and Django did not; now that we are on 1.4 we no longer need our own code to handle this. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 9 +++++---- devel/models.py | 5 +++-- devel/views.py | 12 ++++++------ main/models.py | 5 +++-- main/utils.py | 9 ++------- mirrors/management/commands/mirrorcheck.py | 7 ++++--- mirrors/utils.py | 17 +++++++++-------- news/models.py | 11 +++++------ packages/management/commands/signoff_report.py | 8 ++++---- packages/views/flag.py | 8 ++++---- packages/views/signoff.py | 4 ++-- releng/management/commands/syncisos.py | 5 ++--- 12 files changed, 49 insertions(+), 51 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 2d9b68b2..e69691db 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -28,10 +28,11 @@ from django.core.management.base import BaseCommand, CommandError from django.db import connections, router, transaction from django.db.utils import IntegrityError +from django.utils.timezone import now from devel.utils import UserFinder from main.models import Arch, Package, PackageFile, Repo -from main.utils import utc_now, database_vendor +from main.utils import database_vendor from packages.models import Depend, Conflict, Provision, Replacement, Update @@ -318,7 +319,7 @@ def populate_files(dbpkg, repopkg, force=False): filename=filename) pkg_files.append(pkgfile) batched_bulk_create(PackageFile, pkg_files) - dbpkg.files_last_update = utc_now() + dbpkg.files_last_update = now() dbpkg.save() @@ -388,7 +389,7 @@ def db_update(archname, reponame, pkgs, force=False): dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository) try: with transaction.commit_on_success(): - populate_pkg(dbpkg, pkg, timestamp=utc_now()) + populate_pkg(dbpkg, pkg, timestamp=now()) Update.objects.log_update(None, dbpkg) except IntegrityError: logger.warning("Could not add package %s; " @@ -417,7 +418,7 @@ def db_update(archname, reponame, pkgs, force=False): if not force and pkg_same_version(pkg, dbpkg): continue elif not force: - timestamp = utc_now() + timestamp = now() # The odd select_for_update song and dance here are to ensure # simultaneous updates don't happen on a package, causing diff --git a/devel/models.py b/devel/models.py index fd5df00a..9b6f07a7 100644 --- a/devel/models.py +++ b/devel/models.py @@ -4,10 +4,11 @@ from django.db import models from django.db.models.signals import pre_save from django.contrib.auth.models import User +from django.utils.timezone import now from django_countries import CountryField from .fields import PGPKeyField -from main.utils import make_choice, utc_now +from main.utils import make_choice class UserProfile(models.Model): @@ -105,7 +106,7 @@ def set_last_modified(sender, **kwargs): signal handler.''' obj = kwargs['instance'] if hasattr(obj, 'last_modified'): - obj.last_modified = utc_now() + obj.last_modified = now() # connect signals needed to keep cache in line with reality diff --git a/devel/views.py b/devel/views.py index 78ed26f2..143b12bf 100644 --- a/devel/views.py +++ b/devel/views.py @@ -24,11 +24,11 @@ from django.views.generic.simple import direct_to_template from django.utils.encoding import force_unicode from django.utils.http import http_date +from django.utils.timezone import now from .models import UserProfile from main.models import Package, PackageFile, TodolistPkg from main.models import Arch, Repo -from main.utils import utc_now from news.models import News from packages.models import PackageRelation, Signoff, Depend from packages.utils import get_signoff_groups @@ -122,15 +122,15 @@ def clock(request): else: dev.last_action = None - now = utc_now() + current_time = now() page_dict = { 'developers': devs, - 'utc_now': now, + 'utc_now': current_time, } response = direct_to_template(request, 'devel/clock.html', page_dict) if not response.has_header('Expires'): - expire_time = now.replace(second=0, microsecond=0) + expire_time = current_time.replace(second=0, microsecond=0) expire_time += timedelta(minutes=1) expire_time = time.mktime(expire_time.timetuple()) response['Expires'] = http_date(expire_time) @@ -198,12 +198,12 @@ def report(request, report_name, username=None): if report_name == 'old': title = 'Packages last built more than one year ago' - cutoff = utc_now() - timedelta(days=365) + cutoff = now() - timedelta(days=365) packages = packages.filter( build_date__lt=cutoff).order_by('build_date') elif report_name == 'long-out-of-date': title = 'Packages marked out-of-date more than 90 days ago' - cutoff = utc_now() - timedelta(days=90) + cutoff = now() - timedelta(days=90) packages = packages.filter( flag_date__lt=cutoff).order_by('flag_date') elif report_name == 'big': diff --git a/main/models.py b/main/models.py index 04d8da8f..6c9dfe4d 100644 --- a/main/models.py +++ b/main/models.py @@ -6,9 +6,10 @@ from django.db import models from django.contrib.auth.models import User from django.contrib.sites.models import Site +from django.utils.timezone import now from .fields import PositiveBigIntegerField -from .utils import cache_function, set_created_field, utc_now +from .utils import cache_function, set_created_field class TodolistManager(models.Manager): @@ -385,7 +386,7 @@ class Meta: def set_todolist_fields(sender, **kwargs): todolist = kwargs['instance'] if not todolist.date_added: - todolist.date_added = utc_now() + todolist.date_added = now() # connect signals needed to keep cache in line with reality from main.utils import refresh_latest diff --git a/main/utils.py b/main/utils.py index 879abfb9..0b6849a4 100644 --- a/main/utils.py +++ b/main/utils.py @@ -5,10 +5,10 @@ from datetime import datetime import hashlib -from pytz import utc from django.core.cache import cache from django.db import connections, router +from django.utils.timezone import now CACHE_TIMEOUT = 1800 @@ -94,17 +94,12 @@ def retrieve_latest(sender, latest_by=None): return None -def utc_now(): - '''Returns a timezone-aware UTC date representing now.''' - return datetime.utcnow().replace(tzinfo=utc) - - def set_created_field(sender, **kwargs): '''This will set the 'created' field on any object to the current UTC time if it is unset. For use as a pre_save signal handler.''' obj = kwargs['instance'] if hasattr(obj, 'created') and not obj.created: - obj.created = utc_now() + obj.created = now() def database_vendor(model, mode='read'): diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 7a133cbf..e09ea680 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -29,8 +29,9 @@ from django.core.management.base import NoArgsCommand from django.db import transaction +from django.utils.timezone import now -from main.utils import utc_now, database_vendor +from main.utils import database_vendor from mirrors.models import MirrorUrl, MirrorLog logging.basicConfig( @@ -83,7 +84,7 @@ def parse_lastsync(log, data): def check_mirror_url(mirror_url, timeout): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) - log = MirrorLog(url=mirror_url, check_time=utc_now()) + log = MirrorLog(url=mirror_url, check_time=now()) headers = {'User-Agent': 'archweb/1.0'} req = urllib2.Request(url, None, headers) try: @@ -136,7 +137,7 @@ def check_mirror_url(mirror_url, timeout): def check_rsync_url(mirror_url, timeout): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) - log = MirrorLog(url=mirror_url, check_time=utc_now()) + log = MirrorLog(url=mirror_url, check_time=now()) tempdir = tempfile.mkdtemp() lastsync_path = os.path.join(tempdir, 'lastsync') diff --git a/mirrors/utils.py b/mirrors/utils.py index f2c98ee0..bf030d39 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,13 +1,14 @@ from datetime import timedelta from django.db.models import Avg, Count, Max, Min, StdDev +from django.utils.timezone import now from django_countries.fields import Country -from main.utils import cache_function, utc_now, database_vendor +from main.utils import cache_function, database_vendor from .models import MirrorLog, MirrorProtocol, MirrorUrl -default_cutoff = timedelta(hours=24) +DEFAULT_CUTOFF = timedelta(hours=24) def annotate_url(url, delays): '''Given a MirrorURL object, add a few more attributes to it regarding @@ -30,8 +31,8 @@ def annotate_url(url, delays): @cache_function(123) -def get_mirror_statuses(cutoff=default_cutoff): - cutoff_time = utc_now() - cutoff +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): + cutoff_time = now() - cutoff # I swear, this actually has decent performance... urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( mirror__active=True, mirror__public=True, @@ -88,8 +89,8 @@ def get_mirror_statuses(cutoff=default_cutoff): @cache_function(117) -def get_mirror_errors(cutoff=default_cutoff): - cutoff_time = utc_now() - cutoff +def get_mirror_errors(cutoff=DEFAULT_CUTOFF): + cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, url__mirror__active=True, url__mirror__public=True).values( @@ -105,11 +106,11 @@ def get_mirror_errors(cutoff=default_cutoff): @cache_function(295) -def get_mirror_url_for_download(cutoff=default_cutoff): +def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF): '''Find a good mirror URL to use for package downloads. If we have mirror status data available, it is used to determine a good choice by looking at the last batch of status rows.''' - cutoff_time = utc_now() - cutoff + cutoff_time = now() - cutoff status_data = MirrorLog.objects.filter( check_time__gte=cutoff_time).aggregate( Max('check_time'), Max('last_sync')) diff --git a/news/models.py b/news/models.py index 95026e1d..2efea579 100644 --- a/news/models.py +++ b/news/models.py @@ -1,8 +1,7 @@ from django.db import models from django.contrib.auth.models import User from django.contrib.sites.models import Site - -from main.utils import utc_now +from django.utils.timezone import now class News(models.Model): @@ -29,13 +28,13 @@ class Meta: def set_news_fields(sender, **kwargs): news = kwargs['instance'] - now = utc_now() - news.last_modified = now + current_time = now() + news.last_modified = current_time if not news.postdate: - news.postdate = now + news.postdate = current_time # http://diveintomark.org/archives/2004/05/28/howto-atom-id news.guid = 'tag:%s,%s:%s' % (Site.objects.get_current(), - now.strftime('%Y-%m-%d'), news.get_absolute_url()) + current_time.strftime('%Y-%m-%d'), news.get_absolute_url()) # connect signals needed to keep cache in line with reality from main.utils import refresh_latest diff --git a/packages/management/commands/signoff_report.py b/packages/management/commands/signoff_report.py index ddf930db..72fcbe1e 100644 --- a/packages/management/commands/signoff_report.py +++ b/packages/management/commands/signoff_report.py @@ -15,6 +15,7 @@ from django.contrib.sites.models import Site from django.db.models import Count from django.template import loader, Context +from django.utils.timezone import now from collections import namedtuple from datetime import timedelta @@ -23,7 +24,6 @@ import sys from main.models import Repo -from main.utils import utc_now from packages.models import Signoff from packages.utils import get_signoff_groups @@ -66,9 +66,9 @@ def generate_report(email, repo_name): new_hours = 24 old_days = 14 - now = utc_now() - new_cutoff = now - timedelta(hours=new_hours) - old_cutoff = now - timedelta(days=old_days) + current_time = now() + new_cutoff = current_time - timedelta(hours=new_hours) + old_cutoff = current_time - timedelta(days=old_days) if len(signoff_groups) == 0: # no need to send an email at all diff --git a/packages/views/flag.py b/packages/views/flag.py index 7fa2d508..f3db93b3 100644 --- a/packages/views/flag.py +++ b/packages/views/flag.py @@ -5,12 +5,12 @@ from django.db import transaction from django.shortcuts import get_object_or_404, redirect from django.template import loader, Context +from django.utils.timezone import now from django.views.generic.simple import direct_to_template from django.views.decorators.cache import cache_page, never_cache from ..models import FlagRequest from main.models import Package -from main.utils import utc_now class FlagForm(forms.Form): @@ -76,10 +76,10 @@ def flag(request, name, repo, arch): @transaction.commit_on_success def perform_updates(): - now = utc_now() - pkgs.update(flag_date=now) + current_time = now() + pkgs.update(flag_date=current_time) # store our flag request - flag_request = FlagRequest(created=now, + flag_request = FlagRequest(created=current_time, user_email=email, message=message, ip_address=ip_addr, pkgbase=pkg.pkgbase, version=version, repo=pkg.repo, diff --git a/packages/views/signoff.py b/packages/views/signoff.py index 61d949fc..7aa39106 100644 --- a/packages/views/signoff.py +++ b/packages/views/signoff.py @@ -8,11 +8,11 @@ from django.db import transaction from django.http import HttpResponse, Http404 from django.shortcuts import get_list_or_404, redirect, render +from django.utils.timezone import now from django.views.decorators.cache import never_cache from django.views.generic.simple import direct_to_template from main.models import Package, Arch, Repo -from main.utils import utc_now from ..models import SignoffSpecification, Signoff from ..utils import (get_signoff_groups, approved_by_signoffs, PackageSignoffGroup) @@ -45,7 +45,7 @@ def signoff_package(request, name, repo, arch, revoke=False): package, request.user, False) except Signoff.DoesNotExist: raise Http404 - signoff.revoked = utc_now() + signoff.revoked = now() signoff.save() created = False else: diff --git a/releng/management/commands/syncisos.py b/releng/management/commands/syncisos.py index 62f005ff..223c771b 100644 --- a/releng/management/commands/syncisos.py +++ b/releng/management/commands/syncisos.py @@ -4,8 +4,8 @@ from django.conf import settings from django.core.management.base import BaseCommand, CommandError +from django.utils.timezone import now -from main.utils import utc_now from releng.models import Iso @@ -54,9 +54,8 @@ def handle(self, *args, **options): existing.active = True existing.removed = None existing.save() - now = utc_now() # and then mark all other names as no longer active Iso.objects.filter(active=True).exclude(name__in=active_isos).update( - active=False, removed=now) + active=False, removed=now()) # vim: set ts=4 sw=4 et: -- cgit v1.2.3-54-g00ecf From 76c37ce3acc7a4af0271c7535d4a33042f7749b5 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 24 Jul 2012 09:35:55 -0500 Subject: Replace deprecated direct_to_template() with render() shortcut Now that Django actually provides a concise way to use a RequestContext object without instantiating it, we can use that rather than the old function-based generic view that worked well to do the same. Additionally, these function-based generic views will be gone in Django 1.5, so might as well make the move now. Signed-off-by: Dan McGee --- devel/views.py | 15 +++++++-------- mirrors/views.py | 19 +++++++++---------- packages/views/__init__.py | 11 +++++------ packages/views/display.py | 13 +++++-------- packages/views/flag.py | 12 +++++------- packages/views/signoff.py | 5 ++--- public/views.py | 14 +++++++------- releng/views.py | 15 +++++++-------- retro/views.py | 4 ++-- todolists/views.py | 13 ++++++------- visualize/views.py | 4 ++-- 11 files changed, 57 insertions(+), 68 deletions(-) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index 143b12bf..f877bc84 100644 --- a/devel/views.py +++ b/devel/views.py @@ -17,11 +17,10 @@ from django.db import transaction from django.db.models import F, Count, Max from django.http import Http404 -from django.shortcuts import get_object_or_404 +from django.shortcuts import get_object_or_404, render from django.template import loader, Context from django.template.defaultfilters import filesizeformat from django.views.decorators.cache import never_cache -from django.views.generic.simple import direct_to_template from django.utils.encoding import force_unicode from django.utils.http import http_date from django.utils.timezone import now @@ -88,7 +87,7 @@ def index(request): 'signoffs': signoffs } - return direct_to_template(request, 'devel/index.html', page_dict) + return render(request, 'devel/index.html', page_dict) @login_required def clock(request): @@ -128,7 +127,7 @@ def clock(request): 'utc_now': current_time, } - response = direct_to_template(request, 'devel/clock.html', page_dict) + response = render(request, 'devel/clock.html', page_dict) if not response.has_header('Expires'): expire_time = current_time.replace(second=0, microsecond=0) expire_time += timedelta(minutes=1) @@ -178,7 +177,7 @@ def change_profile(request): else: form = ProfileForm(initial={'email': request.user.email}) profile_form = UserProfileForm(instance=request.user.get_profile()) - return direct_to_template(request, 'devel/profile.html', + return render(request, 'devel/profile.html', {'form': form, 'profile_form': profile_form}) @login_required @@ -301,7 +300,7 @@ def report(request, report_name, username=None): 'column_names': names, 'column_attrs': attrs, } - return direct_to_template(request, 'devel/packages.html', context) + return render(request, 'devel/packages.html', context) class NewUserForm(forms.ModelForm): @@ -399,7 +398,7 @@ def inner_save(): 'title': 'Create User', 'submit_text': 'Create User' } - return direct_to_template(request, 'general_form.html', context) + return render(request, 'general_form.html', context) @user_passes_test(lambda u: u.is_superuser) def admin_log(request, username=None): @@ -410,6 +409,6 @@ def admin_log(request, username=None): 'title': "Admin Action Log", 'log_user': user, } - return direct_to_template(request, 'devel/admin_log.html', context) + return render(request, 'devel/admin_log.html', context) # vim: set ts=4 sw=4 et: diff --git a/mirrors/views.py b/mirrors/views.py index 8f092be7..400c084d 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -8,9 +8,8 @@ from django.core.serializers.json import DjangoJSONEncoder from django.db.models import Q from django.http import Http404, HttpResponse -from django.shortcuts import get_object_or_404 +from django.shortcuts import get_object_or_404, render from django.views.decorators.csrf import csrf_exempt -from django.views.generic.simple import direct_to_template from django_countries.countries import COUNTRIES from .models import Mirror, MirrorUrl, MirrorProtocol, TIER_CHOICES @@ -76,7 +75,7 @@ def generate_mirrorlist(request): else: form = MirrorlistForm() - return direct_to_template(request, 'mirrors/mirrorlist_generate.html', + return render(request, 'mirrors/mirrorlist_generate.html', {'mirrorlist_form': form}) @@ -142,10 +141,10 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, urls = status_filter(urls) template = 'mirrors/mirrorlist_status.txt' - return direct_to_template(request, template, { - 'mirror_urls': urls, - }, - mimetype='text/plain') + context = { + 'mirror_urls': urls, + } + return render(request, template, context, content_type='text/plain') def find_mirrors_simple(request, protocol): @@ -161,7 +160,7 @@ def mirrors(request): mirror_list = Mirror.objects.select_related().order_by('tier', 'country') if not request.user.is_authenticated(): mirror_list = mirror_list.filter(public=True, active=True) - return direct_to_template(request, 'mirrors/mirrors.html', + return render(request, 'mirrors/mirrors.html', {'mirror_list': mirror_list}) @@ -180,7 +179,7 @@ def mirror_details(request, name): all_urls = set(checked_urls).union(all_urls) all_urls = sorted(all_urls, key=attrgetter('url')) - return direct_to_template(request, 'mirrors/mirror_details.html', + return render(request, 'mirrors/mirror_details.html', {'mirror': mirror, 'urls': all_urls}) @@ -217,7 +216,7 @@ def status(request, tier=None): 'error_logs': error_logs, 'tier': tier, }) - return direct_to_template(request, 'mirrors/status.html', context) + return render(request, 'mirrors/status.html', context) class MirrorStatusJSONEncoder(DjangoJSONEncoder): diff --git a/packages/views/__init__.py b/packages/views/__init__.py index fa67daa8..19ea9103 100644 --- a/packages/views/__init__.py +++ b/packages/views/__init__.py @@ -6,10 +6,9 @@ from django.contrib.auth.models import User from django.core.cache import cache from django.http import HttpResponse -from django.shortcuts import redirect +from django.shortcuts import redirect, render from django.views.decorators.cache import cache_control from django.views.decorators.http import require_GET, require_POST -from django.views.generic.simple import direct_to_template from main.models import Package, Arch from ..models import PackageRelation @@ -32,9 +31,9 @@ def opensearch(request): else: domain = "http://%s" % request.META['HTTP_HOST'] - return direct_to_template(request, 'packages/opensearch.xml', + return render(request, 'packages/opensearch.xml', {'domain': domain}, - mimetype='application/opensearchdescription+xml') + content_type='application/opensearchdescription+xml') @require_GET @@ -115,7 +114,7 @@ def arch_differences(request): 'differences': differences, 'multilib_differences': multilib_diffs } - return direct_to_template(request, 'packages/differences.html', context) + return render(request, 'packages/differences.html', context) @permission_required('main.change_package') def stale_relations(request): @@ -132,7 +131,7 @@ def stale_relations(request): 'missing_pkgbase': missing_pkgbase, 'wrong_permissions': wrong_permissions, } - return direct_to_template(request, 'packages/stale_relations.html', context) + return render(request, 'packages/stale_relations.html', context) @permission_required('packages.delete_packagerelation') @require_POST diff --git a/packages/views/display.py b/packages/views/display.py index 31f18c79..585e0e4e 100644 --- a/packages/views/display.py +++ b/packages/views/display.py @@ -6,7 +6,6 @@ from django.http import HttpResponse, Http404 from django.shortcuts import get_object_or_404, redirect, render from django.utils.timezone import now -from django.views.generic.simple import direct_to_template from main.models import Package, PackageFile, Arch, Repo from mirrors.utils import get_mirror_url_for_download @@ -33,8 +32,7 @@ def split_package_details(request, name, repo, arch): 'arch': arch, 'packages': pkgs, } - return direct_to_template(request, 'packages/packages_list.html', - context) + return render(request, 'packages/packages_list.html', context) CUTOFF = datetime.timedelta(days=60) @@ -67,8 +65,7 @@ def details(request, name='', repo='', arch=''): pkg = Package.objects.select_related( 'arch', 'repo', 'packager').get(pkgname=name, repo__name__iexact=repo, arch__name=arch) - return direct_to_template(request, 'packages/details.html', - {'pkg': pkg, }) + return render(request, 'packages/details.html', {'pkg': pkg}) except Package.DoesNotExist: arch_obj = get_object_or_404(Arch, name=arch) # for arch='any' packages, we can issue a redirect to them if we @@ -111,7 +108,7 @@ def groups(request, arch=None): 'groups': grps, 'arch': arch, } - return direct_to_template(request, 'packages/groups.html', context) + return render(request, 'packages/groups.html', context) def group_details(request, arch, name): @@ -128,7 +125,7 @@ def group_details(request, arch, name): 'arch': arch, 'packages': pkgs, } - return direct_to_template(request, 'packages/packages_list.html', context) + return render(request, 'packages/packages_list.html', context) def files(request, name, repo, arch): @@ -153,7 +150,7 @@ def files(request, name, repo, arch): 'dir_count': dir_count, } template = 'packages/files.html' - return direct_to_template(request, template, context) + return render(request, template, context) def details_json(request, name, repo, arch): diff --git a/packages/views/flag.py b/packages/views/flag.py index f3db93b3..b9542a62 100644 --- a/packages/views/flag.py +++ b/packages/views/flag.py @@ -3,10 +3,9 @@ from django.contrib.auth.decorators import permission_required from django.core.mail import send_mail from django.db import transaction -from django.shortcuts import get_object_or_404, redirect +from django.shortcuts import get_object_or_404, redirect, render from django.template import loader, Context from django.utils.timezone import now -from django.views.generic.simple import direct_to_template from django.views.decorators.cache import cache_page, never_cache from ..models import FlagRequest @@ -34,7 +33,7 @@ def __init__(self, *args, **kwargs): @cache_page(3600) def flaghelp(request): - return direct_to_template(request, 'packages/flaghelp.html') + return render(request, 'packages/flaghelp.html') @never_cache @@ -43,8 +42,7 @@ def flag(request, name, repo, arch): pkgname=name, repo__name__iexact=repo, arch__name=arch) if pkg.flag_date is not None: # already flagged. do nothing. - return direct_to_template(request, 'packages/flagged.html', - {'pkg': pkg}) + return render(request, 'packages/flagged.html', {'pkg': pkg}) # find all packages from (hopefully) the same PKGBUILD pkgs = Package.objects.normal().filter( pkgbase=pkg.pkgbase, flag_date__isnull=True, @@ -129,7 +127,7 @@ def perform_updates(): 'packages': pkgs, 'form': form } - return direct_to_template(request, 'packages/flag.html', context) + return render(request, 'packages/flag.html', context) def flag_confirmed(request, name, repo, arch): pkg = get_object_or_404(Package, @@ -142,7 +140,7 @@ def flag_confirmed(request, name, repo, arch): context = {'package': pkg, 'packages': pkgs} - return direct_to_template(request, 'packages/flag_confirmed.html', context) + return render(request, 'packages/flag_confirmed.html', context) @permission_required('main.change_package') def unflag(request, name, repo, arch): diff --git a/packages/views/signoff.py b/packages/views/signoff.py index 7aa39106..56eb060c 100644 --- a/packages/views/signoff.py +++ b/packages/views/signoff.py @@ -10,7 +10,6 @@ from django.shortcuts import get_list_or_404, redirect, render from django.utils.timezone import now from django.views.decorators.cache import never_cache -from django.views.generic.simple import direct_to_template from main.models import Package, Arch, Repo from ..models import SignoffSpecification, Signoff @@ -28,7 +27,7 @@ def signoffs(request): 'arches': Arch.objects.all(), 'repo_names': sorted(set(g.target_repo for g in signoff_groups)), } - return direct_to_template(request, 'packages/signoffs.html', context) + return render(request, 'packages/signoffs.html', context) @permission_required('main.change_package') @never_cache @@ -144,7 +143,7 @@ def signoff_options(request, name, repo, arch): 'package': package, 'form': form, } - return direct_to_template(request, 'packages/signoff_options.html', context) + return render(request, 'packages/signoff_options.html', context) class SignoffJSONEncoder(DjangoJSONEncoder): '''Base JSONEncoder extended to handle all serialization of all classes diff --git a/public/views.py b/public/views.py index 3ea8f841..3f68545c 100644 --- a/public/views.py +++ b/public/views.py @@ -5,8 +5,8 @@ from django.contrib.auth.models import User from django.db.models import Count, Q from django.http import Http404 +from django.shortcuts import render from django.views.decorators.cache import cache_control -from django.views.generic.simple import direct_to_template from devel.models import MasterKey, PGPSignature from main.models import Arch, Repo, Donor @@ -21,7 +21,7 @@ def index(request): 'news_updates': News.objects.order_by('-postdate', '-id')[:15], 'pkg_updates': pkgs, } - return direct_to_template(request, 'public/index.html', context) + return render(request, 'public/index.html', context) USER_LISTS = { 'devs': { @@ -55,14 +55,14 @@ def userlist(request, user_type='devs'): users = users.distinct() context = USER_LISTS[user_type].copy() context['users'] = users - return direct_to_template(request, 'public/userlist.html', context) + return render(request, 'public/userlist.html', context) @cache_control(max_age=300) def donate(request): context = { 'donors': Donor.objects.filter(visible=True).order_by('name'), } - return direct_to_template(request, 'public/donate.html', context) + return render(request, 'public/donate.html', context) @cache_control(max_age=300) def download(request): @@ -76,7 +76,7 @@ def download(request): 'releng_pxeboot_url': settings.PXEBOOT_URL, 'mirror_urls': mirror_urls, } - return direct_to_template(request, 'public/download.html', context) + return render(request, 'public/download.html', context) @cache_control(max_age=300) def feeds(request): @@ -84,7 +84,7 @@ def feeds(request): 'arches': Arch.objects.all(), 'repos': Repo.objects.all(), } - return direct_to_template(request, 'public/feeds.html', context) + return render(request, 'public/feeds.html', context) @cache_control(max_age=300) def keys(request): @@ -113,6 +113,6 @@ def keys(request): 'active_users': users, 'signatures': signatures, } - return direct_to_template(request, 'public/keys.html', context) + return render(request, 'public/keys.html', context) # vim: set ts=4 sw=4 et: diff --git a/releng/views.py b/releng/views.py index fc81d410..a1bc3b81 100644 --- a/releng/views.py +++ b/releng/views.py @@ -2,8 +2,7 @@ from django.conf import settings from django.db.models import Count, Max from django.http import Http404 -from django.shortcuts import get_object_or_404, redirect -from django.views.generic.simple import direct_to_template +from django.shortcuts import get_object_or_404, redirect, render from .models import (Architecture, BootType, Bootloader, ClockChoice, Filesystem, HardwareType, InstallType, Iso, IsoType, Module, Source, @@ -73,7 +72,7 @@ def submit_test_result(request): form = TestForm() context = {'form': form} - return direct_to_template(request, 'releng/add.html', context) + return render(request, 'releng/add.html', context) def calculate_option_overview(field_name): field = Test._meta.get_field(field_name) @@ -145,7 +144,7 @@ def test_results_overview(request): 'options': all_options, 'iso_url': settings.ISO_LIST_URL, } - return direct_to_template(request, 'releng/results.html', context) + return render(request, 'releng/results.html', context) def test_results_iso(request, iso_id): iso = get_object_or_404(Iso, pk=iso_id) @@ -154,7 +153,7 @@ def test_results_iso(request, iso_id): 'iso_name': iso.name, 'test_list': test_list } - return direct_to_template(request, 'releng/result_list.html', context) + return render(request, 'releng/result_list.html', context) def test_results_for(request, option, value): if option not in Test._meta.get_all_field_names(): @@ -170,10 +169,10 @@ def test_results_for(request, option, value): 'value_id': value, 'test_list': test_list } - return direct_to_template(request, 'releng/result_list.html', context) + return render(request, 'releng/result_list.html', context) def submit_test_thanks(request): - return direct_to_template(request, "releng/thanks.html", None) + return render(request, "releng/thanks.html", None) def iso_overview(request): isos = Iso.objects.all().order_by('-pk') @@ -192,6 +191,6 @@ def iso_overview(request): context = { 'isos': isos } - return direct_to_template(request, 'releng/iso_overview.html', context) + return render(request, 'releng/iso_overview.html', context) # vim: set ts=4 sw=4 et: diff --git a/retro/views.py b/retro/views.py index 3bc59e9f..31226deb 100644 --- a/retro/views.py +++ b/retro/views.py @@ -1,6 +1,6 @@ from django.http import Http404 +from django.shortcuts import render from django.views.decorators.cache import cache_page -from django.views.generic.simple import direct_to_template RETRO_YEAR_MAP = { @@ -26,6 +26,6 @@ def retro_homepage(request, year): context = { 'year': year, } - return direct_to_template(request, 'retro/%s' % template, context) + return render(request, 'retro/%s' % template, context) # vim: set ts=4 sw=4 et: diff --git a/todolists/views.py b/todolists/views.py index 580ec00f..c7ba2560 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -3,12 +3,11 @@ from django import forms from django.http import HttpResponse from django.core.mail import send_mail -from django.shortcuts import get_list_or_404, get_object_or_404, redirect +from django.shortcuts import get_list_or_404, get_object_or_404, redirect, render from django.contrib.auth.decorators import login_required, permission_required from django.db import transaction from django.views.decorators.cache import never_cache from django.views.generic import DeleteView -from django.views.generic.simple import direct_to_template from django.template import Context, loader from main.models import Todolist, TodolistPkg, Package, Repo @@ -54,7 +53,7 @@ def view(request, list_id): # we don't hold onto the result, but the objects are the same here, # so accessing maintainers in the template is now cheap attach_maintainers(tp.pkg for tp in todolist.packages) - return direct_to_template(request, 'todolists/view.html', { + return render(request, 'todolists/view.html', { 'list': todolist, 'svn_roots': svn_roots, }) @@ -72,7 +71,7 @@ def list_pkgbases(request, list_id, svn_root): @login_required def todolist_list(request): lists = get_annotated_todolists() - return direct_to_template(request, 'todolists/list.html', {'lists': lists}) + return render(request, 'todolists/list.html', {'lists': lists}) @permission_required('main.add_todolist') @never_cache @@ -92,7 +91,7 @@ def add(request): 'form': form, 'submit_text': 'Create List' } - return direct_to_template(request, 'general_form.html', page_dict) + return render(request, 'general_form.html', page_dict) # TODO: this calls for transaction management and async emailing @permission_required('main.change_todolist') @@ -115,7 +114,7 @@ def edit(request, list_id): 'form': form, 'submit_text': 'Save List' } - return direct_to_template(request, 'general_form.html', page_dict) + return render(request, 'general_form.html', page_dict) class DeleteTodolist(DeleteView): model = Todolist @@ -185,7 +184,7 @@ def public_list(request): # total hackjob, but it makes this a lot less query-intensive. all_pkgs = [tp for tl in todo_lists for tp in tl.packages] attach_maintainers([tp.pkg for tp in all_pkgs]) - return direct_to_template(request, "todolists/public_list.html", + return render(request, "todolists/public_list.html", {"todo_lists": todo_lists}) # vim: set ts=4 sw=4 et: diff --git a/visualize/views.py b/visualize/views.py index 95f8c314..44e60472 100644 --- a/visualize/views.py +++ b/visualize/views.py @@ -4,14 +4,14 @@ from django.contrib.auth.models import User from django.db.models import Count, Sum, Q from django.http import HttpResponse +from django.shortcuts import render from django.views.decorators.cache import cache_page -from django.views.generic.simple import direct_to_template from main.models import Package, Arch, Repo from devel.models import MasterKey, PGPSignature def index(request): - return direct_to_template(request, 'visualize/index.html', {}) + return render(request, 'visualize/index.html') def arch_repo_data(): qs = Package.objects.select_related().values( -- cgit v1.2.3-54-g00ecf From 280c53eec5661252b5692fa374292c4d421e3bd8 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 28 Jul 2012 11:45:15 -0500 Subject: reporead: don't use iexact lookup on arch name We don't do this anywhere else, so we shouldn't do this here either. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index e69691db..aaa9812e 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -332,7 +332,7 @@ def update_common(archname, reponame, pkgs, sanity_check=True): transaction.set_dirty() repository = Repo.objects.get(name__iexact=reponame) - architecture = Arch.objects.get(name__iexact=archname) + architecture = Arch.objects.get(name=archname) # no-arg order_by() removes even the default ordering; we don't need it dbpkgs = Package.objects.filter( arch=architecture, repo=repository).order_by() @@ -371,7 +371,7 @@ def db_update(archname, reponame, pkgs, force=False): logger.info('Updating %s (%s)', reponame, archname) dbpkgs = update_common(archname, reponame, pkgs, True) repository = Repo.objects.get(name__iexact=reponame) - architecture = Arch.objects.get(name__iexact=archname) + architecture = Arch.objects.get(name=archname) # This makes our inner loop where we find packages by name *way* more # efficient by not having to go to the database for each package to @@ -538,7 +538,7 @@ def locate_arch(arch): if isinstance(arch, Arch): return arch try: - return Arch.objects.get(name__iexact=arch) + return Arch.objects.get(name=arch) except Arch.DoesNotExist: raise CommandError( 'Specified architecture %s is not currently known.' % arch) -- cgit v1.2.3-54-g00ecf From 3f0c024754047d92e8ce4aa4ecf93a06865f8448 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 31 Jul 2012 18:37:30 -0500 Subject: PGP key handling updates * Import signatures for all known keys, not just active developers * Ensure we are only showing and accounting for active developers on the master keys page * Add a new table showing signatures between developers Signed-off-by: Dan McGee --- devel/management/commands/generate_keyring.py | 2 +- main/templatetags/pgp.py | 8 +++++++ public/views.py | 18 +++++++++++----- templates/public/keys.html | 30 +++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 6 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/generate_keyring.py b/devel/management/commands/generate_keyring.py index b9117c84..15ae488d 100644 --- a/devel/management/commands/generate_keyring.py +++ b/devel/management/commands/generate_keyring.py @@ -48,7 +48,7 @@ def generate_keyring(keyserver, keyring): logger.info("getting all known key IDs") # Screw you Django, for not letting one natively do value != - key_ids = UserProfile.objects.filter(user__is_active=True, + key_ids = UserProfile.objects.filter( pgp_key__isnull=False).extra(where=["pgp_key != ''"]).values_list( "pgp_key", flat=True) logger.info("%d keys fetched from user profiles", len(key_ids)) diff --git a/main/templatetags/pgp.py b/main/templatetags/pgp.py index 50b1aa17..5c9fe511 100644 --- a/main/templatetags/pgp.py +++ b/main/templatetags/pgp.py @@ -39,6 +39,14 @@ def pgp_key_link(key_id, link_text=None): values = (url, format_key(key_id), link_text) return '%s' % values +@register.simple_tag +def user_pgp_key_link(users, key_id): + matched = [user for user in users if user.userprofile.pgp_key and + user.userprofile.pgp_key[-16:] == key_id[-16:]] + if matched and len(matched) == 1: + return pgp_key_link(key_id, matched[0].get_full_name()) + return pgp_key_link(key_id) + @register.filter def pgp_fingerprint(key_id, autoescape=True): if not key_id: diff --git a/public/views.py b/public/views.py index c8854b72..312cb3b2 100644 --- a/public/views.py +++ b/public/views.py @@ -91,30 +91,38 @@ def feeds(request): @cache_control(max_age=300) def keys(request): + users = User.objects.filter(is_active=True).select_related( + 'userprofile__pgp_key').order_by('first_name', 'last_name') + user_key_ids = frozenset(user.userprofile.pgp_key[-16:] for user in users + if user.userprofile.pgp_key) + not_expired = Q(expires__gt=datetime.utcnow) | Q(expires__isnull=True) master_keys = MasterKey.objects.select_related('owner', 'revoker', 'owner__userprofile', 'revoker__userprofile').filter( revoked__isnull=True) + master_key_ids = frozenset(key.pgp_key[-16:] for key in master_keys) - sig_counts = PGPSignature.objects.filter( - not_expired, valid=True).values_list('signer').annotate( + sig_counts = PGPSignature.objects.filter(not_expired, valid=True, + signee__in=user_key_ids).values_list('signer').annotate( Count('signer')) sig_counts = dict((key_id[-16:], ct) for key_id, ct in sig_counts) for key in master_keys: key.signature_count = sig_counts.get(key.pgp_key[-16:], 0) - users = User.objects.filter(is_active=True).select_related( - 'userprofile__pgp_key').order_by('first_name', 'last_name') - # frozenset because we are going to do lots of __contains__ lookups signatures = frozenset(PGPSignature.objects.filter( not_expired, valid=True).values_list('signer', 'signee')) + restrict = Q(signer__in=user_key_ids) & Q(signee__in=user_key_ids) + cross_signatures = PGPSignature.objects.filter(restrict, + not_expired, valid=True).order_by('created') + context = { 'keys': master_keys, 'active_users': users, 'signatures': signatures, + 'cross_signatures': cross_signatures, } return render(request, 'public/keys.html', context) diff --git a/templates/public/keys.html b/templates/public/keys.html index 1fed3c15..1b027202 100644 --- a/templates/public/keys.html +++ b/templates/public/keys.html @@ -89,6 +89,33 @@

    Master Signing Keys

    • Developer Username AliasLast Action Location Time Zone Current Time {{ dev.get_full_name }} {{ dev.username }} {{ dev.userprofile.alias }}{{ dev.last_action }} {% if dev.userprofile.country %}{{ dev.userprofile.country.name }} {% endif %}{{ dev.userprofile.location }} {{ dev.userprofile.time_zone }} {{ utc_now|timezone:dev.userprofile.time_zone|date:"Y-m-d H:i T" }} {{ dev.userprofile.time_zone }} {% with pkg.signer as signer %}{% if signer %}{% pgp_key_link pkg.signature.key_id signer.get_full_name %}{% else %}Unknown ({% pgp_key_link pkg.signature.key_id %}){% endif %}{% endwith %}
    Signature Date:{{ pkg.signature.datetime|date:"DATETIME_FORMAT" }} UTC{{ pkg.signature.creation_time|date:"DATETIME_FORMAT" }} UTC
    Signed By: Unsigned
    + +
    +

    Developer Cross-Signatures

    + +

    This table lists signatures directly between developer keys.

    + + + + + + + + + + + + {% for sig in cross_signatures %} + + + + + + + {% endfor %} + +
    SignerSigneeCreatedExpires
    {% user_pgp_key_link active_users sig.signer %}{% user_pgp_key_link active_users sig.signee %}{{ sig.created }}{{ sig.expires|default:"" }}
    +
    {% load cdn %}{% jquery %}{% jquery_tablesorter %} {% endblock %} -- cgit v1.2.3-54-g00ecf From a64bbbd4139d91cbbca10d804067cbd87a95872d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 31 Jul 2012 20:27:43 -0500 Subject: Make adjustments for optional -> deptype conversion Very little dealt directly with this field. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 6 +- main/models.py | 9 +- packages/migrations/0021_migrate_optional_deps.py | 210 ++++++++++++++++++++++ templates/packages/details_depend.html | 2 +- templates/packages/details_requiredby.html | 2 +- 5 files changed, 222 insertions(+), 7 deletions(-) create mode 100644 packages/migrations/0021_migrate_optional_deps.py (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index aaa9812e..a3bf3e0c 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -145,8 +145,8 @@ def full_version(self): DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.+))?$") -def create_depend(package, dep_str, optional=False): - depend = Depend(pkg=package, optional=optional) +def create_depend(package, dep_str, deptype='D'): + depend = Depend(pkg=package, deptype=deptype) # lop off any description first parts = dep_str.split(':', 1) if len(parts) > 1: @@ -257,7 +257,7 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): dbpkg.depends.all().delete() deps = [create_depend(dbpkg, y) for y in repopkg.depends] - deps += [create_depend(dbpkg, y, True) for y in repopkg.optdepends] + deps += [create_depend(dbpkg, y, 'O') for y in repopkg.optdepends] batched_bulk_create(Depend, deps) dbpkg.conflicts.all().delete() diff --git a/main/models.py b/main/models.py index 577f11c6..f4ced350 100644 --- a/main/models.py +++ b/main/models.py @@ -245,13 +245,18 @@ def get_depends(self): deps = [] arches = None # TODO: we can use list comprehension and an 'in' query to make this more effective - for dep in self.depends.order_by('optional', 'name'): + for dep in self.depends.all(): pkg = dep.get_best_satisfier() providers = None if not pkg: providers = dep.get_providers() deps.append({'dep': dep, 'pkg': pkg, 'providers': providers}) - return deps + # sort the list; deptype sorting makes this tricker than expected + sort_order = {'D': 0, 'O': 1, 'M': 2, 'C': 3} + def sort_key(val): + dep = val['dep'] + return (sort_order.get(dep.deptype, 1000), dep.name) + return sorted(deps, key=sort_key) @cache_function(125) def base_package(self): diff --git a/packages/migrations/0021_migrate_optional_deps.py b/packages/migrations/0021_migrate_optional_deps.py new file mode 100644 index 00000000..f6652ce1 --- /dev/null +++ b/packages/migrations/0021_migrate_optional_deps.py @@ -0,0 +1,210 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import DataMigration +from django.db import models + +class Migration(DataMigration): + + def forwards(self, orm): + orm['packages.Depend'].objects.filter(optional=False).update(deptype='D') + orm['packages.Depend'].objects.filter(optional=True).update(deptype='O') + + def backwards(self, orm): + orm['packages.Depend'].objects.all().update(deptype='D') + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'main.arch': { + 'Meta': {'ordering': "['name']", 'object_name': 'Arch', 'db_table': "'arches'"}, + 'agnostic': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'main.package': { + 'Meta': {'ordering': "('pkgname',)", 'unique_together': "(('pkgname', 'repo', 'arch'),)", 'object_name': 'Package', 'db_table': "'packages'"}, + 'arch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'on_delete': 'models.PROTECT', 'to': "orm['main.Arch']"}), + 'build_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'compressed_size': ('main.fields.PositiveBigIntegerField', [], {}), + 'epoch': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'files_last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'flag_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'installed_size': ('main.fields.PositiveBigIntegerField', [], {}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'packager': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'on_delete': 'models.SET_NULL'}), + 'packager_str': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pgp_signature': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkgdesc': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'pkgname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'repo': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'on_delete': 'models.PROTECT', 'to': "orm['main.Repo']"}), + 'url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}) + }, + 'main.repo': { + 'Meta': {'ordering': "['name']", 'object_name': 'Repo', 'db_table': "'repos'"}, + 'bugs_category': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'bugs_project': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'staging': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'svn_root': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + 'packages.conflict': { + 'Meta': {'ordering': "['name']", 'object_name': 'Conflict'}, + 'comparison': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'conflicts'", 'to': "orm['main.Package']"}), + 'version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}) + }, + 'packages.depend': { + 'Meta': {'ordering': "['name']", 'object_name': 'Depend'}, + 'comparison': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}), + 'deptype': ('django.db.models.fields.CharField', [], {'default': "'D'", 'max_length': '1'}), + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'optional': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'depends'", 'to': "orm['main.Package']"}), + 'version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}) + }, + 'packages.flagrequest': { + 'Meta': {'object_name': 'FlagRequest'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip_address': ('django.db.models.fields.GenericIPAddressField', [], {'max_length': '39'}), + 'is_legitimate': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_spam': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'message': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'num_packages': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'repo': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Repo']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'}), + 'user_email': ('django.db.models.fields.EmailField', [], {'max_length': '75'}), + 'version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}) + }, + 'packages.license': { + 'Meta': {'ordering': "['name']", 'object_name': 'License'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'licenses'", 'to': "orm['main.Package']"}) + }, + 'packages.packagegroup': { + 'Meta': {'object_name': 'PackageGroup'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'groups'", 'to': "orm['main.Package']"}) + }, + 'packages.packagerelation': { + 'Meta': {'unique_together': "(('pkgbase', 'user', 'type'),)", 'object_name': 'PackageRelation'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'type': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'package_relations'", 'to': "orm['auth.User']"}) + }, + 'packages.provision': { + 'Meta': {'ordering': "['name']", 'object_name': 'Provision'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'provides'", 'to': "orm['main.Package']"}), + 'version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}) + }, + 'packages.replacement': { + 'Meta': {'ordering': "['name']", 'object_name': 'Replacement'}, + 'comparison': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'replaces'", 'to': "orm['main.Package']"}), + 'version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}) + }, + 'packages.signoff': { + 'Meta': {'object_name': 'Signoff'}, + 'arch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Arch']"}), + 'comments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'epoch': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'repo': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Repo']"}), + 'revoked': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'package_signoffs'", 'to': "orm['auth.User']"}) + }, + 'packages.signoffspecification': { + 'Meta': {'object_name': 'SignoffSpecification'}, + 'arch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Arch']"}), + 'comments': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'epoch': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'known_bad': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'repo': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Repo']"}), + 'required': ('django.db.models.fields.PositiveIntegerField', [], {'default': '2'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True'}) + }, + 'packages.update': { + 'Meta': {'object_name': 'Update'}, + 'action_flag': ('django.db.models.fields.PositiveSmallIntegerField', [], {}), + 'arch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'updates'", 'to': "orm['main.Arch']"}), + 'created': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'new_epoch': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True'}), + 'new_pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), + 'new_pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), + 'old_epoch': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True'}), + 'old_pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), + 'old_pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'updates'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['main.Package']"}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgname': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'repo': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'updates'", 'to': "orm['main.Repo']"}) + } + } + + complete_apps = ['packages'] + symmetrical = True diff --git a/templates/packages/details_depend.html b/templates/packages/details_depend.html index 0cf2c36a..1eb35474 100644 --- a/templates/packages/details_depend.html +++ b/templates/packages/details_depend.html @@ -11,6 +11,6 @@ {% if depend.pkg.repo.testing %} (testing){% endif %} {% if depend.pkg.repo.staging %} (staging){% endif %} {% endifequal %} -{% if depend.dep.optional %} (optional){% endif %} +{% if depend.dep.deptype == 'O' %} (optional){% endif %} {% if depend.dep.description %}- {{ depend.dep.description }}{% endif %} diff --git a/templates/packages/details_requiredby.html b/templates/packages/details_requiredby.html index ecc92b29..15c62c61 100644 --- a/templates/packages/details_requiredby.html +++ b/templates/packages/details_requiredby.html @@ -3,5 +3,5 @@ {% if req.name != pkg.pkgname %}(requires {{ req.name }}){% endif %} {% if req.pkg.repo.testing %}(testing){% endif %} {% if req.pkg.repo.staging %}(staging){% endif %} -{% if req.optional %}(optional){% endif %} +{% if req.deptype == 'O' %}(optional){% endif %} -- cgit v1.2.3-54-g00ecf From 1f2466fffceafebfaca34e3ed2d34de6b622768b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 31 Jul 2012 20:35:50 -0500 Subject: reporead: import make and check depends We don't have these in the database yet, but future verisons of repo-add will put this information in the sync databases. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index a3bf3e0c..8b55b09a 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -80,8 +80,9 @@ class RepoPackage(object): bare = ( 'name', 'base', 'arch', 'filename', 'md5sum', 'sha256sum', 'url', 'packager' ) number = ( 'csize', 'isize' ) - collections = ( 'depends', 'optdepends', 'conflicts', - 'provides', 'replaces', 'groups', 'license', 'files' ) + collections = ( 'depends', 'optdepends', 'makedepends', 'checkdepends', + 'conflicts', 'provides', 'replaces', 'groups', 'license', + 'files' ) version_re = re.compile(r'^((\d+):)?(.+)-([^-]+)$') @@ -258,6 +259,8 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): dbpkg.depends.all().delete() deps = [create_depend(dbpkg, y) for y in repopkg.depends] deps += [create_depend(dbpkg, y, 'O') for y in repopkg.optdepends] + deps += [create_depend(dbpkg, y, 'M') for y in repopkg.makedepends] + deps += [create_depend(dbpkg, y, 'C') for y in repopkg.checkdepends] batched_bulk_create(Depend, deps) dbpkg.conflicts.all().delete() -- cgit v1.2.3-54-g00ecf From 978a5c61a5412eeed054307d3e2979324ffcb64a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 8 Aug 2012 19:34:37 -0500 Subject: Add flag requests to developer last action calculation Signed-off-by: Dan McGee --- devel/views.py | 8 +++++++- templates/devel/clock.html | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index f877bc84..ad1f4deb 100644 --- a/devel/views.py +++ b/devel/views.py @@ -29,7 +29,7 @@ from main.models import Package, PackageFile, TodolistPkg from main.models import Arch, Repo from news.models import News -from packages.models import PackageRelation, Signoff, Depend +from packages.models import PackageRelation, Signoff, FlagRequest, Depend from packages.utils import get_signoff_groups from todolists.utils import get_annotated_todolists from .utils import get_annotated_maintainers, UserFinder @@ -103,6 +103,11 @@ def clock(request): latest_signoff = dict(Signoff.objects.filter( user__is_active=True).values_list('user').order_by( ).annotate(last_signoff=Max('created'))) + # The extra() bit ensures we can use our 'user_id IS NOT NULL' index + latest_flagreq = dict(FlagRequest.objects.filter( + user__is_active=True).extra( + where=['user_id IS NOT NULL']).values_list('user_id').order_by( + ).annotate(last_flagrequest=Max('created'))) latest_log = dict(LogEntry.objects.filter( user__is_active=True).values_list('user').order_by( ).annotate(last_log=Max('action_time'))) @@ -112,6 +117,7 @@ def clock(request): latest_news.get(dev.id, None), latest_package.get(dev.id, None), latest_signoff.get(dev.id, None), + latest_flagreq.get(dev.id, None), latest_log.get(dev.id, None), dev.last_login, ] diff --git a/templates/devel/clock.html b/templates/devel/clock.html index bf4614b5..02e42749 100644 --- a/templates/devel/clock.html +++ b/templates/devel/clock.html @@ -20,6 +20,7 @@

    Developer World Clocks

    mirror)
  • Post date of a news item
  • Signing off on a package
    • +
  • Flagging a package out-of-date

    • Current UTC Time: {{ utc_now|date:"Y-m-d H:i T" }} -- cgit v1.2.3-54-g00ecf From 241ff8fbd79f9f17cd326a34eb39096851f630ba Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 8 Aug 2012 22:07:06 -0500 Subject: Extract parse_version function from reporead logic Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 9 ++------- packages/utils.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 7 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 8b55b09a..af0a2dc0 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -34,6 +34,7 @@ from main.models import Arch, Package, PackageFile, Repo from main.utils import database_vendor from packages.models import Depend, Conflict, Provision, Replacement, Update +from packages.utils import parse_version logging.basicConfig( @@ -84,8 +85,6 @@ class RepoPackage(object): 'conflicts', 'provides', 'replaces', 'groups', 'license', 'files' ) - version_re = re.compile(r'^((\d+):)?(.+)-([^-]+)$') - def __init__(self, repo): self.repo = repo self.ver = None @@ -112,11 +111,7 @@ def populate(self, values): # do NOT prune these values at all setattr(self, k, v[0]) elif k == 'version': - match = self.version_re.match(v[0]) - self.ver = match.group(3) - self.rel = match.group(4) - if match.group(2): - self.epoch = int(match.group(2)) + self.ver, self.rel, self.epoch = parse_version(v[0]) elif k == 'builddate': try: builddate = datetime.utcfromtimestamp(int(v[0])) diff --git a/packages/utils.py b/packages/utils.py index 6d54d71a..d4b4e611 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -1,6 +1,7 @@ from collections import defaultdict from itertools import chain from operator import itemgetter +import re from django.core.serializers.json import DjangoJSONEncoder from django.db import connection @@ -14,6 +15,23 @@ License, Depend, Conflict, Provision, Replacement, SignoffSpecification, Signoff, DEFAULT_SIGNOFF_SPEC) + +VERSION_RE = re.compile(r'^((\d+):)?(.+)-([^-]+)$') + + +def parse_version(version): + match = VERSION_RE.match(version) + if not match: + return None, None, 0 + ver = match.group(3) + rel = match.group(4) + if match.group(2): + epoch = int(match.group(2)) + else: + epoch = 0 + return ver, rel, epoch + + @cache_function(127) def get_group_info(include_arches=None): raw_groups = PackageGroup.objects.values_list( -- cgit v1.2.3-54-g00ecf From ca0011c585ec28f9dde0f400a77fd6f859d520b0 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 15 Aug 2012 08:11:00 -0500 Subject: Add ability to rematch developers on @archlinux.org addresses This makes this matcher catch a bit more with the wide net we were already casting. Signed-off-by: Dan McGee --- devel/management/commands/rematch_developers.py | 4 +- devel/utils.py | 50 +++++++++++++++++-------- 2 files changed, 38 insertions(+), 16 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/rematch_developers.py b/devel/management/commands/rematch_developers.py index ab2f0f4b..2b379588 100644 --- a/devel/management/commands/rematch_developers.py +++ b/devel/management/commands/rematch_developers.py @@ -53,6 +53,7 @@ def match_packager(finder): unmatched = Package.objects.filter(packager__isnull=True).values_list( 'packager_str', flat=True).order_by().distinct() + logger.info("%d packager strings retrieved", len(unmatched)) for packager in unmatched: logger.debug("packager string %s", packager) user = finder.find(packager) @@ -71,13 +72,14 @@ def match_packager(finder): @transaction.commit_on_success def match_flagrequest(finder): - logger.info("getting all flag requests emails from unknown users") + logger.info("getting all flag request email addresses from unknown users") req_count = matched_count = 0 mapping = {} unmatched = FlagRequest.objects.filter(user__isnull=True).values_list( 'user_email', flat=True).order_by().distinct() + logger.info("%d email addresses retrieved", len(unmatched)) for user_email in unmatched: logger.debug("email %s", user_email) user = finder.find_by_email(user_email) diff --git a/devel/utils.py b/devel/utils.py index 85b4e42f..e8e3a6c4 100644 --- a/devel/utils.py +++ b/devel/utils.py @@ -1,6 +1,7 @@ import re from django.contrib.auth.models import User +from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned from django.db import connection from django.db.models import Count, Q @@ -44,6 +45,15 @@ def get_annotated_maintainers(): return maintainers +def ignore_does_not_exist(func): + def new_func(*args, **kwargs): + try: + return func(*args, **kwargs) + except (ObjectDoesNotExist, MultipleObjectsReturned): + return None + return new_func + + class UserFinder(object): def __init__(self): self.cache = {} @@ -52,18 +62,33 @@ def __init__(self): self.pgp_cache = {} @staticmethod + @ignore_does_not_exist def user_email(name, email): if email: return User.objects.get(email=email) return None @staticmethod + @ignore_does_not_exist + def username_email(name, email): + if email and '@' in email: + # split email addr at '@' symbol, ensure domain matches + # or is a subdomain of archlinux.org + # TODO: configurable domain/regex somewhere? + username, domain = email.split('@', 1) + if re.match(r'^(.+\.)?archlinux.org$', domain): + return User.objects.get(username=username) + return None + + @staticmethod + @ignore_does_not_exist def profile_email(name, email): if email: return User.objects.get(userprofile__public_email=email) return None @staticmethod + @ignore_does_not_exist def user_name(name, email): # yes, a bit odd but this is the easiest way since we can't always be # sure how to split the name. Ensure every 'token' appears in at least @@ -102,14 +127,12 @@ def find(self, userstring): email = matches.group(2) user = None - find_methods = (self.user_email, self.profile_email, self.user_name) + find_methods = (self.user_email, self.profile_email, + self.username_email, self.user_name) for matcher in find_methods: - try: - user = matcher(name, email) - if user != None: - break - except (User.DoesNotExist, User.MultipleObjectsReturned): - pass + user = matcher(name, email) + if user != None: + break self.cache[userstring] = user self.email_cache[email] = user @@ -135,14 +158,11 @@ def find_by_email(self, email): if email in self.email_cache: return self.email_cache[email] - user = None - try: - user = self.user_email(None, email) - except User.DoesNotExist: - try: - user = self.profile_email(None, email) - except User.DoesNotExist: - pass + user = self.user_email(None, email) + if user is None: + user = self.profile_email(None, email) + if user is None: + user = self.username_email(None, email) self.email_cache[email] = user return user -- cgit v1.2.3-54-g00ecf From e7e9b151643772f2bf9564d215ec8b90cd9b45c6 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 5 Sep 2012 08:44:33 -0500 Subject: Split devel forms out of devel/views.py Signed-off-by: Dan McGee --- devel/forms.py | 99 +++++++++++++++++++++++++++++++++++++++++++++++++++++ devel/views.py | 106 ++++++--------------------------------------------------- 2 files changed, 109 insertions(+), 96 deletions(-) create mode 100644 devel/forms.py (limited to 'devel') diff --git a/devel/forms.py b/devel/forms.py new file mode 100644 index 00000000..861a576c --- /dev/null +++ b/devel/forms.py @@ -0,0 +1,99 @@ +import random +from string import ascii_letters, digits + +from django import forms +from django.contrib.auth.models import User, Group +from django.contrib.sites.models import Site +from django.core.mail import send_mail +from django.template import loader, Context + +from .models import UserProfile + + +class ProfileForm(forms.Form): + email = forms.EmailField(label='Private email (not shown publicly):', + help_text="Used for out-of-date notifications, etc.") + passwd1 = forms.CharField(label='New Password', required=False, + widget=forms.PasswordInput) + passwd2 = forms.CharField(label='Confirm Password', required=False, + widget=forms.PasswordInput) + + def clean(self): + if self.cleaned_data['passwd1'] != self.cleaned_data['passwd2']: + raise forms.ValidationError('Passwords do not match.') + return self.cleaned_data + + +class UserProfileForm(forms.ModelForm): + def clean_pgp_key(self): + data = self.cleaned_data['pgp_key'] + # strip 0x prefix if provided; store uppercase + if data.startswith('0x'): + data = data[2:] + return data.upper() + + class Meta: + model = UserProfile + exclude = ('allowed_repos', 'user', 'latin_name') + + +class NewUserForm(forms.ModelForm): + username = forms.CharField(max_length=30) + private_email = forms.EmailField() + first_name = forms.CharField(required=False) + last_name = forms.CharField(required=False) + groups = forms.ModelMultipleChoiceField(required=False, + queryset=Group.objects.all()) + + class Meta: + model = UserProfile + exclude = ('picture', 'user') + + def __init__(self, *args, **kwargs): + super(NewUserForm, self).__init__(*args, **kwargs) + # Hack ourself so certain fields appear first. self.fields is a + # SortedDict object where we can manipulate the keyOrder list. + order = self.fields.keyOrder + keys = ('username', 'private_email', 'first_name', 'last_name') + for key in reversed(keys): + order.remove(key) + order.insert(0, key) + + def clean_username(self): + username = self.cleaned_data['username'] + if User.objects.filter(username=username).exists(): + raise forms.ValidationError( + "A user with that username already exists.") + return username + + def save(self, commit=True): + profile = super(NewUserForm, self).save(False) + pwletters = ascii_letters + digits + password = ''.join([random.choice(pwletters) for _ in xrange(8)]) + user = User.objects.create_user(username=self.cleaned_data['username'], + email=self.cleaned_data['private_email'], password=password) + user.first_name = self.cleaned_data['first_name'] + user.last_name = self.cleaned_data['last_name'] + user.save() + # sucks that the MRM.add() method can't take a list directly... we have + # to resort to dirty * magic. + user.groups.add(*self.cleaned_data['groups']) + profile.user = user + if commit: + profile.save() + self.save_m2m() + + template = loader.get_template('devel/new_account.txt') + ctx = Context({ + 'site': Site.objects.get_current(), + 'user': user, + 'password': password, + }) + + send_mail("Your new archweb account", + template.render(ctx), + 'Arch Website Notification ', + [user.email], + fail_silently=False) + +# vim: set ts=4 sw=4 et: diff --git a/devel/views.py b/devel/views.py index ad1f4deb..5406974e 100644 --- a/devel/views.py +++ b/devel/views.py @@ -1,31 +1,25 @@ from datetime import timedelta import operator import pytz -import random -from string import ascii_letters, digits import time -from django import forms from django.http import HttpResponseRedirect from django.contrib.auth.decorators import \ login_required, permission_required, user_passes_test from django.contrib.admin.models import LogEntry, ADDITION -from django.contrib.auth.models import User, Group +from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType -from django.contrib.sites.models import Site -from django.core.mail import send_mail from django.db import transaction from django.db.models import F, Count, Max from django.http import Http404 from django.shortcuts import get_object_or_404, render -from django.template import loader, Context from django.template.defaultfilters import filesizeformat from django.views.decorators.cache import never_cache from django.utils.encoding import force_unicode from django.utils.http import http_date from django.utils.timezone import now -from .models import UserProfile +from .forms import ProfileForm, UserProfileForm, NewUserForm from main.models import Package, PackageFile, TodolistPkg from main.models import Arch, Repo from news.models import News @@ -89,6 +83,7 @@ def index(request): return render(request, 'devel/index.html', page_dict) + @login_required def clock(request): devs = User.objects.filter(is_active=True).order_by( @@ -141,30 +136,6 @@ def clock(request): response['Expires'] = http_date(expire_time) return response -class ProfileForm(forms.Form): - email = forms.EmailField(label='Private email (not shown publicly):', - help_text="Used for out-of-date notifications, etc.") - passwd1 = forms.CharField(label='New Password', required=False, - widget=forms.PasswordInput) - passwd2 = forms.CharField(label='Confirm Password', required=False, - widget=forms.PasswordInput) - - def clean(self): - if self.cleaned_data['passwd1'] != self.cleaned_data['passwd2']: - raise forms.ValidationError('Passwords do not match.') - return self.cleaned_data - -class UserProfileForm(forms.ModelForm): - def clean_pgp_key(self): - data = self.cleaned_data['pgp_key'] - # strip 0x prefix if provided; store uppercase - if data.startswith('0x'): - data = data[2:] - return data.upper() - - class Meta: - model = UserProfile - exclude = ('allowed_repos', 'user', 'latin_name') @login_required @never_cache @@ -177,8 +148,9 @@ def change_profile(request): request.user.email = form.cleaned_data['email'] if form.cleaned_data['passwd1']: request.user.set_password(form.cleaned_data['passwd1']) - request.user.save() - profile_form.save() + with transaction.commit_on_success(): + request.user.save() + profile_form.save() return HttpResponseRedirect('/devel/') else: form = ProfileForm(initial={'email': request.user.email}) @@ -186,6 +158,7 @@ def change_profile(request): return render(request, 'devel/profile.html', {'form': form, 'profile_form': profile_form}) + @login_required def report(request, report_name, username=None): title = 'Developer Report' @@ -309,65 +282,6 @@ def report(request, report_name, username=None): return render(request, 'devel/packages.html', context) -class NewUserForm(forms.ModelForm): - username = forms.CharField(max_length=30) - private_email = forms.EmailField() - first_name = forms.CharField(required=False) - last_name = forms.CharField(required=False) - groups = forms.ModelMultipleChoiceField(required=False, - queryset=Group.objects.all()) - - class Meta: - model = UserProfile - exclude = ('picture', 'user') - - def __init__(self, *args, **kwargs): - super(NewUserForm, self).__init__(*args, **kwargs) - # Hack ourself so certain fields appear first. self.fields is a - # SortedDict object where we can manipulate the keyOrder list. - order = self.fields.keyOrder - keys = ('username', 'private_email', 'first_name', 'last_name') - for key in reversed(keys): - order.remove(key) - order.insert(0, key) - - def clean_username(self): - username = self.cleaned_data['username'] - if User.objects.filter(username=username).exists(): - raise forms.ValidationError( - "A user with that username already exists.") - return username - - def save(self, commit=True): - profile = super(NewUserForm, self).save(False) - pwletters = ascii_letters + digits - password = ''.join([random.choice(pwletters) for _ in xrange(8)]) - user = User.objects.create_user(username=self.cleaned_data['username'], - email=self.cleaned_data['private_email'], password=password) - user.first_name = self.cleaned_data['first_name'] - user.last_name = self.cleaned_data['last_name'] - user.save() - # sucks that the MRM.add() method can't take a list directly... we have - # to resort to dirty * magic. - user.groups.add(*self.cleaned_data['groups']) - profile.user = user - if commit: - profile.save() - self.save_m2m() - - template = loader.get_template('devel/new_account.txt') - ctx = Context({ - 'site': Site.objects.get_current(), - 'user': user, - 'password': password, - }) - - send_mail("Your new archweb account", - template.render(ctx), - 'Arch Website Notification ', - [user.email], - fail_silently=False) - def log_addition(request, obj): """Cribbed from ModelAdmin.log_addition.""" LogEntry.objects.log_action( @@ -379,17 +293,16 @@ def log_addition(request, obj): change_message = "Added via Create New User form." ) + @permission_required('auth.add_user') @never_cache def new_user_form(request): if request.POST: form = NewUserForm(request.POST) if form.is_valid(): - @transaction.commit_on_success - def inner_save(): + with transaction.commit_on_success(): form.save() log_addition(request, form.instance.user) - inner_save() return HttpResponseRedirect('/admin/auth/user/%d/' % \ form.instance.user.id) else: @@ -406,6 +319,7 @@ def inner_save(): } return render(request, 'general_form.html', context) + @user_passes_test(lambda u: u.is_superuser) def admin_log(request, username=None): user = None -- cgit v1.2.3-54-g00ecf From a2034fc80d4e73816502537f8dfe864ab4ef8db3 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 15 Sep 2012 09:14:36 -0500 Subject: Add JS-based filtering to the developer reports This can use the todolist filtering functions we made more generic in a previous commit. Signed-off-by: Dan McGee --- devel/views.py | 4 ++++ templates/devel/packages.html | 30 ++++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index 5406974e..23ff9f74 100644 --- a/devel/views.py +++ b/devel/views.py @@ -271,11 +271,15 @@ def report(request, report_name, username=None): else: raise Http404 + arches = set(pkg.arch for pkg in packages) + repos = set(pkg.repo for pkg in packages) context = { 'all_maintainers': maints, 'title': title, 'maintainer': user, 'packages': packages, + 'arches': sorted(arches), + 'repos': sorted(repos), 'column_names': names, 'column_attrs': attrs, } diff --git a/templates/devel/packages.html b/templates/devel/packages.html index ac368124..4e1381ab 100644 --- a/templates/devel/packages.html +++ b/templates/devel/packages.html @@ -13,7 +13,28 @@

    {{ title }}{% if maintainer %}, {% if maintainer %}This report only includes packages maintained by {{ maintainer.get_full_name }} ({{ maintainer.username }}).{% endif %}

    - + +
    +

    Filter Packages

    +
    +
    + Select filter criteria + {% for arch in arches %} +
    +
    + {% endfor %} + {% for repo in repos %} +
    +
    + {% endfor %} +
    +
    +
    {{ packages|length }} packages displayed.
    +
    + +
    + +
    @@ -31,7 +52,7 @@

    {{ title }}{% if maintainer %},

    {% for pkg in packages %} - + @@ -57,6 +78,11 @@

    {{ title }}{% if maintainer %}, {% endblock %} -- cgit v1.2.3-54-g00ecf From c76e5c768687394b8022883d01edf85dc3c30e7f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 17 Sep 2012 21:42:48 -0500 Subject: Sort package list before inserting it into the database FS#30323. This will take some time to propagate to all existing packages, but all new and updated packages will start getting filelists in the right order. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index af0a2dc0..ac745092 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -303,7 +303,10 @@ def populate_files(dbpkg, repopkg, force=False): logger.info("adding %d files for package %s", len(repopkg.files), dbpkg.pkgname) pkg_files = [] - for f in repopkg.files: + # sort in normal alpha-order that pacman uses, rather than makepkg's + # default breadth-first, directory-first ordering + files = sorted(repopkg.files) + for f in files: if '/' in f: dirname, filename = f.rsplit('/', 1) dirname += '/' -- cgit v1.2.3-54-g00ecf From 7d5cfe45d52c4dbd2f431f0edcafc9936b740ab2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 20 Sep 2012 09:32:42 -0500 Subject: Explicitly close the database connection in reporead This is the cause of these warnings showing up in the PostgreSQL log: LOG: unexpected EOF on client connection with an open transaction All management commands are guilty of this as they do not clean up and close the connection when they exit, unlike the standard web request cycle. Other commands should probably be updated as well, but for now, this is the biggest culprit. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 1 + 1 file changed, 1 insertion(+) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index ac745092..ce5c8cb7 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -584,6 +584,7 @@ def read_repo(primary_arch, repo_file, options): else: db_update(arch, repo, packages_arches[arch], force) logger.info('Finished database updates for %s.', repo_file) + connection.close() return 0 # vim: set ts=4 sw=4 et: -- cgit v1.2.3-54-g00ecf From 89c6ffc95cb1d5fe4bd2534562ca732d727a8686 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 20 Sep 2012 09:34:46 -0500 Subject: chmod -x reporead_inotify.py Signed-off-by: Dan McGee --- devel/management/commands/reporead_inotify.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 devel/management/commands/reporead_inotify.py (limited to 'devel') diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py old mode 100755 new mode 100644 -- cgit v1.2.3-54-g00ecf From 05f309d7e57a66d9309abbf19b4328bad514b978 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 24 Sep 2012 21:13:02 -0500 Subject: Add a new column to developer repo stats Signed-off-by: Dan McGee --- devel/views.py | 5 +++++ templates/devel/index.html | 3 +++ 2 files changed, 8 insertions(+) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index 23ff9f74..ea85a901 100644 --- a/devel/views.py +++ b/devel/views.py @@ -55,6 +55,11 @@ def index(request): total_ct=Count('packages'), flagged_ct=Count('packages__flag_date')) repos = Repo.objects.all().annotate( total_ct=Count('packages'), flagged_ct=Count('packages__flag_date')) + # the join is huge unless we do this separately, so merge the result here + repo_maintainers = dict(Repo.objects.all().values_list('id').annotate( + Count('userprofile'))) + for repo in repos: + repo.maintainer_ct = repo_maintainers.get(repo.id, 0) maintainers = get_annotated_maintainers() diff --git a/templates/devel/index.html b/templates/devel/index.html index 488b6755..a07a4190 100644 --- a/templates/devel/index.html +++ b/templates/devel/index.html @@ -216,6 +216,7 @@

    Stats by Repository

    + @@ -228,6 +229,8 @@

    Stats by Repository

    + + {% endfor %} -- cgit v1.2.3-54-g00ecf From a1c4d831c92cbb32cb8c34f95b8cd5eb541cdf00 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 24 Sep 2012 21:23:18 -0500 Subject: Exclude inactive developers in maintainer count Signed-off-by: Dan McGee --- devel/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index ea85a901..083665d9 100644 --- a/devel/views.py +++ b/devel/views.py @@ -56,7 +56,8 @@ def index(request): repos = Repo.objects.all().annotate( total_ct=Count('packages'), flagged_ct=Count('packages__flag_date')) # the join is huge unless we do this separately, so merge the result here - repo_maintainers = dict(Repo.objects.all().values_list('id').annotate( + repo_maintainers = dict(Repo.objects.order_by().filter( + userprofile__user__is_active=True).values_list('id').annotate( Count('userprofile'))) for repo in repos: repo.maintainer_ct = repo_maintainers.get(repo.id, 0) -- cgit v1.2.3-54-g00ecf From 3530303c9a7d017bdfec40d9dc7c38bd3fb2c09b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 25 Sep 2012 18:21:10 -0500 Subject: Only watch non-staging repos in inotify reporead This is temporary until we do more work to ensure staging packages don't show up and confuse regular users of the web interface. Signed-off-by: Dan McGee --- devel/management/commands/reporead_inotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'devel') diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py index c74762eb..043e13fe 100644 --- a/devel/management/commands/reporead_inotify.py +++ b/devel/management/commands/reporead_inotify.py @@ -68,7 +68,7 @@ def setup_notifier(self): and passes these on to the various pyinotify pieces as necessary and finally builds and returns a notifier object.''' arches = Arch.objects.filter(agnostic=False) - repos = Repo.objects.all() + repos = Repo.objects.filter(staging=False) arch_path_map = dict((arch, None) for arch in arches) all_paths = set() total_paths = 0 -- cgit v1.2.3-54-g00ecf From ed1adeb1254c4d5754260bbe1ae2fbbc2a88debb Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 30 Sep 2012 02:02:04 -0500 Subject: Begin importing staging repos This reverts 3530303c9a7d now that we have reasonably hidden most staging package confusion on the site for normal end users. Signed-off-by: Dan McGee --- devel/management/commands/reporead_inotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'devel') diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py index 043e13fe..c74762eb 100644 --- a/devel/management/commands/reporead_inotify.py +++ b/devel/management/commands/reporead_inotify.py @@ -68,7 +68,7 @@ def setup_notifier(self): and passes these on to the various pyinotify pieces as necessary and finally builds and returns a notifier object.''' arches = Arch.objects.filter(agnostic=False) - repos = Repo.objects.filter(staging=False) + repos = Repo.objects.all() arch_path_map = dict((arch, None) for arch in arches) all_paths = set() total_paths = 0 -- cgit v1.2.3-54-g00ecf From 5228cb5f584f076e547e1d0af695c08975801d2f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 12 Oct 2012 11:39:16 -0500 Subject: reporead: don't print full backtrace if unnecessary In the architecture agnostic case, this error is much more likely to happen, so printing it like an error message is deceiving. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index ce5c8cb7..a1e77b49 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -393,9 +393,12 @@ def db_update(archname, reponame, pkgs, force=False): populate_pkg(dbpkg, pkg, timestamp=now()) Update.objects.log_update(None, dbpkg) except IntegrityError: - logger.warning("Could not add package %s; " - "not fatal if another thread beat us to it.", - pkg.name, exc_info=True) + if architecture.agnostic: + logger.warning("Could not add package %s; " + "not fatal if another thread beat us to it.", + pkg.name) + else: + logger.exception("Could not add package %s", pkg.name) # packages in database and not in syncdb (remove from database) for pkgname in (dbset - syncset): -- cgit v1.2.3-54-g00ecf From 6dd4d54bb0adbbb0f8c2b1beaa92b7a58971cf88 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 16 Nov 2012 16:20:11 -0600 Subject: Use Python 2.7 dictionary comprehension syntax Rather than the old idiom of dict((k, v) for <> in <>). Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 4 ++-- devel/management/commands/reporead_inotify.py | 2 +- mirrors/views.py | 11 ++++------- packages/templatetags/package_extras.py | 2 +- packages/utils.py | 5 ++--- packages/views/signoff.py | 8 +++----- public/views.py | 2 +- visualize/views.py | 4 ++-- 8 files changed, 16 insertions(+), 22 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index a1e77b49..3d4e6375 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -377,7 +377,7 @@ def db_update(archname, reponame, pkgs, force=False): # This makes our inner loop where we find packages by name *way* more # efficient by not having to go to the database for each package to # SELECT them by name. - dbdict = dict((dbpkg.pkgname, dbpkg) for dbpkg in dbpkgs) + dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs} dbset = set(dbdict.keys()) syncset = set([pkg.name for pkg in pkgs]) @@ -446,7 +446,7 @@ def filesonly_update(archname, reponame, pkgs, force=False): """ logger.info('Updating files for %s (%s)', reponame, archname) dbpkgs = update_common(archname, reponame, pkgs, False) - dbdict = dict((dbpkg.pkgname, dbpkg) for dbpkg in dbpkgs) + dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs} dbset = set(dbdict.keys()) for pkg in (pkg for pkg in pkgs if pkg.name in dbset): diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py index c74762eb..16b3869c 100644 --- a/devel/management/commands/reporead_inotify.py +++ b/devel/management/commands/reporead_inotify.py @@ -69,7 +69,7 @@ def setup_notifier(self): finally builds and returns a notifier object.''' arches = Arch.objects.filter(agnostic=False) repos = Repo.objects.all() - arch_path_map = dict((arch, None) for arch in arches) + arch_path_map = {arch: None for arch in arches} all_paths = set() total_paths = 0 for arch in arches: diff --git a/mirrors/views.py b/mirrors/views.py index 2e1e83b6..d0ce0a97 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -94,7 +94,7 @@ def default_protocol_filter(original_urls): def status_filter(original_urls): status_info = get_mirror_statuses() - scores = dict((u.id, u.score) for u in status_info['urls']) + scores = {u.id: u.score for u in status_info['urls']} urls = [] for u in original_urls: u.score = scores.get(u.id, None) @@ -165,7 +165,7 @@ def mirrors(request): if not request.user.is_authenticated(): mirror_list = mirror_list.filter(public=True, active=True) protos = protos.filter(mirror__public=True, mirror__active=True) - protos = dict((k, list(v)) for k, v in groupby(protos, key=itemgetter(0))) + protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))} for mirror in mirror_list: items = protos.get(mirror.id, []) mirror.protocols = [item[1] for item in items] @@ -253,8 +253,7 @@ def default(self, obj): # mainly for queryset serialization return list(obj) if isinstance(obj, MirrorUrl): - data = dict((attr, getattr(obj, attr)) - for attr in self.url_attributes) + data = {attr: getattr(obj, attr) for attr in self.url_attributes} # get any override on the country attribute first country = obj.real_country data['country'] = unicode(country.name) @@ -277,9 +276,7 @@ def default(self, obj): check_time__gte=cutoff).order_by('check_time') return data if isinstance(obj, MirrorLog): - data = dict((attr, getattr(obj, attr)) - for attr in self.log_attributes) - return data + return {attr: getattr(obj, attr) for attr in self.log_attributes} return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) diff --git a/packages/templatetags/package_extras.py b/packages/templatetags/package_extras.py index 994265d8..f3613e69 100644 --- a/packages/templatetags/package_extras.py +++ b/packages/templatetags/package_extras.py @@ -13,7 +13,7 @@ def link_encode(url, query): # massage the data into all utf-8 encoded strings first, so urlencode # doesn't barf at the data we pass it - query = dict((k, unicode(v).encode('utf-8')) for k, v in query.items()) + query = {k: unicode(v).encode('utf-8') for k, v in query.items()} data = urlencode(query).replace('&', '&') return "%s?%s" % (url, data) diff --git a/packages/utils.py b/packages/utils.py index 051fed8e..199e141d 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -385,7 +385,7 @@ def signoffs_id_query(model, repos): repo_sql = ','.join(['%s' for r in repos]) sql = sql % (model._meta.db_table, repo_sql, repo_sql) repo_ids = [r.pk for r in repos] - # repo_ids are needed twice, so double the array + # repo_ids are needed twice, so double the array cursor.execute(sql, repo_ids * 2) results = cursor.fetchall() @@ -474,8 +474,7 @@ def default(self, obj): # mainly for queryset serialization return list(obj) if isinstance(obj, Package): - data = dict((attr, getattr(obj, attr)) - for attr in self.pkg_attributes) + data = {attr: getattr(obj, attr) for attr in self.pkg_attributes} for attr in self.pkg_list_attributes: data[attr] = getattr(obj, attr).all() return data diff --git a/packages/views/signoff.py b/packages/views/signoff.py index 56eb060c..824a9922 100644 --- a/packages/views/signoff.py +++ b/packages/views/signoff.py @@ -155,8 +155,8 @@ class SignoffJSONEncoder(DjangoJSONEncoder): def default(self, obj): if isinstance(obj, PackageSignoffGroup): - data = dict((attr, getattr(obj, attr)) - for attr in self.signoff_group_attrs) + data = {attr: getattr(obj, attr) + for attr in self.signoff_group_attrs} data['pkgnames'] = [p.pkgname for p in obj.packages] data['package_count'] = len(obj.packages) data['approved'] = obj.approved() @@ -164,9 +164,7 @@ def default(self, obj): for attr in self.signoff_spec_attrs) return data elif isinstance(obj, Signoff): - data = dict((attr, getattr(obj, attr)) - for attr in self.signoff_attrs) - return data + return {attr: getattr(obj, attr) for attr in self.signoff_attrs} elif isinstance(obj, Arch) or isinstance(obj, Repo): return unicode(obj) elif isinstance(obj, User): diff --git a/public/views.py b/public/views.py index 96120761..3e15f9df 100644 --- a/public/views.py +++ b/public/views.py @@ -118,7 +118,7 @@ def keys(request): sig_counts = PGPSignature.objects.filter(not_expired, valid=True, signee__in=user_key_ids).values_list('signer').annotate( Count('signer')) - sig_counts = dict((key_id[-16:], ct) for key_id, ct in sig_counts) + sig_counts = {key_id[-16:]: ct for key_id, ct in sig_counts} for key in master_keys: key.signature_count = sig_counts.get(key.pgp_key[-16:], 0) diff --git a/visualize/views.py b/visualize/views.py index 8d878937..48e8f86b 100644 --- a/visualize/views.py +++ b/visualize/views.py @@ -33,8 +33,8 @@ def build_map(name, arch, repo): # now transform these results into two mappings: one ordered (repo, arch), # and one ordered (arch, repo). - arch_groups = dict((a, build_map(a, a, None)) for a in arches) - repo_groups = dict((r, build_map(r, None, r)) for r in repos) + arch_groups = {a: build_map(a, a, None) for a in arches} + repo_groups = {r: build_map(r, None, r) for r in repos} for row in qs: arch = row['arch__name'] repo = row['repo__name'] -- cgit v1.2.3-54-g00ecf From 9e9157d0a8cbf9ea076231e438fb30f58bff8e29 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 16 Nov 2012 16:37:31 -0600 Subject: Use python set comprehension syntax supported in 2.7 Signed-off-by: Dan McGee --- devel/management/commands/import_signatures.py | 4 ++-- devel/management/commands/reporead.py | 2 +- devel/management/commands/reporead_inotify.py | 2 +- devel/views.py | 4 ++-- main/models.py | 2 +- packages/models.py | 2 +- packages/utils.py | 10 +++++----- packages/views/signoff.py | 2 +- todolists/views.py | 8 ++++---- 9 files changed, 18 insertions(+), 18 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/import_signatures.py b/devel/management/commands/import_signatures.py index ce1aba90..da1397ca 100644 --- a/devel/management/commands/import_signatures.py +++ b/devel/management/commands/import_signatures.py @@ -98,8 +98,8 @@ def import_signatures(keyring): # now prune the data down to what we actually want. # prune edges not in nodes, remove duplicates, and self-sigs - pruned_edges = set(edge for edge in edges - if edge.signer in nodes and edge.signer != edge.signee) + pruned_edges = {edge for edge in edges + if edge.signer in nodes and edge.signer != edge.signee} logger.info("creating or finding %d signatures", len(pruned_edges)) created_ct = updated_ct = 0 diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 3d4e6375..981c4dce 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -380,7 +380,7 @@ def db_update(archname, reponame, pkgs, force=False): dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs} dbset = set(dbdict.keys()) - syncset = set([pkg.name for pkg in pkgs]) + syncset = {pkg.name for pkg in pkgs} in_sync_not_db = syncset - dbset logger.info("%d packages in sync not db", len(in_sync_not_db)) diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py index 16b3869c..04f65764 100644 --- a/devel/management/commands/reporead_inotify.py +++ b/devel/management/commands/reporead_inotify.py @@ -77,7 +77,7 @@ def setup_notifier(self): for repo in repos) # take a python format string and generate all unique combinations # of directories from it; using set() ensures we filter it down - paths = set(self.path_template % values for values in combos) + paths = {self.path_template % values for values in combos} total_paths += len(paths) all_paths |= paths arch_path_map[arch] = paths diff --git a/devel/views.py b/devel/views.py index 083665d9..7d5947d1 100644 --- a/devel/views.py +++ b/devel/views.py @@ -277,8 +277,8 @@ def report(request, report_name, username=None): else: raise Http404 - arches = set(pkg.arch for pkg in packages) - repos = set(pkg.repo for pkg in packages) + arches = {pkg.arch for pkg in packages} + repos = {pkg.repo for pkg in packages} context = { 'all_maintainers': maints, 'title': title, diff --git a/main/models.py b/main/models.py index 5700cdf1..cc81637c 100644 --- a/main/models.py +++ b/main/models.py @@ -197,7 +197,7 @@ def get_requiredby(self): """ from packages.models import Depend provides = self.provides.all() - provide_names = set(provide.name for provide in provides) + provide_names = {provide.name for provide in provides} provide_names.add(self.pkgname) requiredby = Depend.objects.select_related('pkg', 'pkg__arch', 'pkg__repo').filter( diff --git a/packages/models.py b/packages/models.py index 0d0fbdf2..ede8c275 100644 --- a/packages/models.py +++ b/packages/models.py @@ -33,7 +33,7 @@ def get_associated_packages(self): def repositories(self): packages = self.get_associated_packages() - return sorted(set([p.repo for p in packages])) + return sorted({p.repo for p in packages}) def __unicode__(self): return u'%s: %s (%s)' % ( diff --git a/packages/utils.py b/packages/utils.py index 199e141d..5adc8637 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -79,8 +79,8 @@ def get_split_packages_info(): split_pkgs = Package.objects.exclude(pkgname=F('pkgbase')).exclude( pkgbase__in=pkgnames).values('pkgbase', 'repo', 'arch').annotate( last_update=Max('last_update')) - all_arches = Arch.objects.in_bulk(set(s['arch'] for s in split_pkgs)) - all_repos = Repo.objects.in_bulk(set(s['repo'] for s in split_pkgs)) + all_arches = Arch.objects.in_bulk({s['arch'] for s in split_pkgs}) + all_repos = Repo.objects.in_bulk({s['repo'] for s in split_pkgs}) for split in split_pkgs: split['arch'] = all_arches[split['arch']] split['repo'] = all_repos[split['repo']] @@ -143,7 +143,7 @@ def get_differences_info(arch_a, arch_b): cursor.execute(sql, [arch_a.id, arch_b.id]) results = cursor.fetchall() # column A will always have a value, column B might be NULL - to_fetch = set(row[0] for row in results) + to_fetch = {row[0] for row in results} # fetch all of the necessary packages pkgs = Package.objects.normal().in_bulk(to_fetch) # now build a list of tuples containing differences @@ -249,13 +249,13 @@ def attach_maintainers(packages): the maintainers and attach them to the packages to prevent N+1 query cascading.''' packages = list(packages) - pkgbases = set(p.pkgbase for p in packages) + pkgbases = {p.pkgbase for p in packages} rels = PackageRelation.objects.filter(type=PackageRelation.MAINTAINER, pkgbase__in=pkgbases).values_list( 'pkgbase', 'user_id').order_by().distinct() # get all the user objects we will need - user_ids = set(rel[1] for rel in rels) + user_ids = {rel[1] for rel in rels} users = User.objects.in_bulk(user_ids) # now build a pkgbase -> [maintainers...] map diff --git a/packages/views/signoff.py b/packages/views/signoff.py index 824a9922..340b2311 100644 --- a/packages/views/signoff.py +++ b/packages/views/signoff.py @@ -25,7 +25,7 @@ def signoffs(request): context = { 'signoff_groups': signoff_groups, 'arches': Arch.objects.all(), - 'repo_names': sorted(set(g.target_repo for g in signoff_groups)), + 'repo_names': sorted({g.target_repo for g in signoff_groups}), } return render(request, 'packages/signoffs.html', context) diff --git a/todolists/views.py b/todolists/views.py index b8d1dae1..9984ef9a 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -53,8 +53,8 @@ def view(request, list_id): # we don't hold onto the result, but the objects are the same here, # so accessing maintainers in the template is now cheap attach_maintainers(tp.pkg for tp in todolist.packages) - arches = set(tp.pkg.arch for tp in todolist.packages) - repos = set(tp.pkg.repo for tp in todolist.packages) + arches = {tp.pkg.arch for tp in todolist.packages} + repos = {tp.pkg.repo for tp in todolist.packages} return render(request, 'todolists/view.html', { 'list': todolist, 'svn_roots': svn_roots, @@ -67,8 +67,8 @@ def list_pkgbases(request, list_id, svn_root): '''Used to make bulk moves of packages a lot easier.''' todolist = get_object_or_404(Todolist, id=list_id) repos = get_list_or_404(Repo, svn_root=svn_root) - pkgbases = set(tp.pkg.pkgbase for tp in todolist.packages - if tp.pkg.repo in repos) + pkgbases = {tp.pkg.pkgbase for tp in todolist.packages + if tp.pkg.repo in repos} return HttpResponse('\n'.join(sorted(pkgbases)), mimetype='text/plain') -- cgit v1.2.3-54-g00ecf From 4c699119820dfd060de6a0385e549f3397053548 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 4 Dec 2012 21:59:29 -0600 Subject: get_latest_by cleanups Fix some that referenced non-existent attributes, and add the attribute to other models. Signed-off-by: Dan McGee --- devel/models.py | 4 ++++ main/models.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'devel') diff --git a/devel/models.py b/devel/models.py index 9b6f07a7..f30bba85 100644 --- a/devel/models.py +++ b/devel/models.py @@ -50,6 +50,7 @@ class UserProfile(models.Model): class Meta: db_table = 'user_profiles' + get_latest_by = 'last_modified' verbose_name = 'Additional Profile Data' verbose_name_plural = 'Additional Profile Data' @@ -80,6 +81,7 @@ class MasterKey(models.Model): class Meta: ordering = ('created',) + get_latest_by = 'created' def __unicode__(self): return u'%s, created %s' % ( @@ -94,6 +96,8 @@ class PGPSignature(models.Model): valid = models.BooleanField(default=True) class Meta: + ordering = ('signer', 'signee') + get_latest_by = 'created' verbose_name = 'PGP signature' def __unicode__(self): diff --git a/main/models.py b/main/models.py index 603d7ccc..8e705c54 100644 --- a/main/models.py +++ b/main/models.py @@ -46,7 +46,7 @@ def __unicode__(self): class Meta: db_table = 'donors' ordering = ('name',) - get_latest_by = 'when' + get_latest_by = 'created' class Arch(models.Model): -- cgit v1.2.3-54-g00ecf From e68a5073a6e8b9473f726734e0b51fdb0a42c14b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 27 Dec 2012 17:02:40 -0600 Subject: Fix "RuntimeWarning: DateTimeField received a naive datetime" warnings When running tests, we can find old migrations that didn't use datetime objects with timezones attached. Signed-off-by: Dan McGee --- devel/migrations/0008_auto__add_field_userprofile_last_modified.py | 4 +++- main/migrations/0024_set_initial_flag_date.py | 4 ++-- news/migrations/0003_new_date_columns_precision.py | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) (limited to 'devel') diff --git a/devel/migrations/0008_auto__add_field_userprofile_last_modified.py b/devel/migrations/0008_auto__add_field_userprofile_last_modified.py index 2695987a..08972e1b 100644 --- a/devel/migrations/0008_auto__add_field_userprofile_last_modified.py +++ b/devel/migrations/0008_auto__add_field_userprofile_last_modified.py @@ -3,12 +3,14 @@ from south.db import db from south.v2 import SchemaMigration from django.db import models +from pytz import utc class Migration(SchemaMigration): def forwards(self, orm): + default = datetime.datetime(2000, 1, 1, 0, 0).replace(tzinfo=utc) db.add_column('user_profiles', 'last_modified', - self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime(2000, 1, 1, 0, 0)), + self.gf('django.db.models.fields.DateTimeField')(default=default), keep_default=False) def backwards(self, orm): diff --git a/main/migrations/0024_set_initial_flag_date.py b/main/migrations/0024_set_initial_flag_date.py index 5026f721..bd008792 100644 --- a/main/migrations/0024_set_initial_flag_date.py +++ b/main/migrations/0024_set_initial_flag_date.py @@ -1,14 +1,14 @@ # encoding: utf-8 -import datetime from south.db import db from south.v2 import DataMigration from django.db import models +from django.utils.timezone import now class Migration(DataMigration): def forwards(self, orm): orm.Package.objects.filter(needupdate=False).update(flag_date=None) - orm.Package.objects.filter(needupdate=True).update(flag_date=datetime.datetime.now()) + orm.Package.objects.filter(needupdate=True).update(flag_date=now()) def backwards(self, orm): orm.Package.objects.filter(flag_date__isnull=True).update(needupdate=False) diff --git a/news/migrations/0003_new_date_columns_precision.py b/news/migrations/0003_new_date_columns_precision.py index 21b64443..1c97f488 100644 --- a/news/migrations/0003_new_date_columns_precision.py +++ b/news/migrations/0003_new_date_columns_precision.py @@ -1,14 +1,14 @@ # encoding: utf-8 -import datetime from south.db import db from south.v2 import SchemaMigration from django.db import models +from django.utils.timezone import now class Migration(SchemaMigration): def forwards(self, orm): # Adding field 'News.last_modified' - db.add_column('news', 'last_modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, default=datetime.datetime.now(), db_index=True, blank=True), keep_default=False) + db.add_column('news', 'last_modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, default=now(), db_index=True, blank=True), keep_default=False) # Changing field 'News.postdate' db.alter_column('news', 'postdate', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True)) # Adding index on 'News', fields ['postdate'] -- cgit v1.2.3-54-g00ecf From bf4385a26c1b6f07bf9bdcddf7160b5eb4a71d9a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 27 Dec 2012 21:13:56 -0600 Subject: Move the body of set_last_modified to main/utils Instead of having multiple methods, move this into our single 'created' setter method. If the 'last_modified' property is present, we now update it accordingly when saving any model with this signal attached. Signed-off-by: Dan McGee --- devel/models.py | 14 ++------------ main/utils.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 14 deletions(-) (limited to 'devel') diff --git a/devel/models.py b/devel/models.py index f30bba85..5f0a8318 100644 --- a/devel/models.py +++ b/devel/models.py @@ -8,7 +8,7 @@ from django_countries import CountryField from .fields import PGPKeyField -from main.utils import make_choice +from main.utils import make_choice, set_created_field class UserProfile(models.Model): @@ -104,17 +104,7 @@ def __unicode__(self): return u'%s → %s' % (self.signer, self.signee) -def set_last_modified(sender, **kwargs): - '''This will set the 'last_modified' field on the user profile to the - current UTC time when either the profile is updated. For use as a pre_save - signal handler.''' - obj = kwargs['instance'] - if hasattr(obj, 'last_modified'): - obj.last_modified = now() - - -# connect signals needed to keep cache in line with reality -pre_save.connect(set_last_modified, sender=UserProfile, +pre_save.connect(set_created_field, sender=UserProfile, dispatch_uid="devel.models") # vim: set ts=4 sw=4 et: diff --git a/main/utils.py b/main/utils.py index d12e5e1a..17ca386e 100644 --- a/main/utils.py +++ b/main/utils.py @@ -106,10 +106,16 @@ def retrieve_latest(sender, latest_by=None): def set_created_field(sender, **kwargs): '''This will set the 'created' field on any object to the current UTC time - if it is unset. For use as a pre_save signal handler.''' + if it is unset. + Additionally, this will set the 'last_modified' field on any object to the + current UTC time on any save of the object. + For use as a pre_save signal handler.''' obj = kwargs['instance'] + time = now() if hasattr(obj, 'created') and not obj.created: - obj.created = now() + obj.created = time + if hasattr(obj, 'last_modified'): + obj.last_modified = time def database_vendor(model, mode='read'): -- cgit v1.2.3-54-g00ecf From c8ece67cec9c421ac0c711554edd34f022623b45 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 28 Dec 2012 00:27:20 -0600 Subject: Convert to using new todolist models everywhere This is a rather widespread set of changes converting usage to the new todo list and todo list package model recently introduced. The data migration is not included in this commit. After this commit, the old model should no longer be referenced anywhere. Signed-off-by: Dan McGee --- devel/views.py | 12 +++-- sitestatic/archweb.css | 5 ++ sitestatic/archweb.js | 17 +++---- templates/devel/index.html | 10 ++-- templates/todolists/email_notification.txt | 2 +- templates/todolists/list.html | 4 +- templates/todolists/public_list.html | 18 +++---- templates/todolists/view.html | 27 +++++------ todolists/urls.py | 2 +- todolists/utils.py | 14 +++--- todolists/views.py | 76 +++++++++++++++++------------- 11 files changed, 97 insertions(+), 90 deletions(-) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index 7d5947d1..e01590a0 100644 --- a/devel/views.py +++ b/devel/views.py @@ -20,11 +20,12 @@ from django.utils.timezone import now from .forms import ProfileForm, UserProfileForm, NewUserForm -from main.models import Package, PackageFile, TodolistPkg +from main.models import Package, PackageFile from main.models import Arch, Repo from news.models import News from packages.models import PackageRelation, Signoff, FlagRequest, Depend from packages.utils import get_signoff_groups +from todolists.models import TodolistPackage from todolists.utils import get_annotated_todolists from .utils import get_annotated_maintainers, UserFinder @@ -41,10 +42,11 @@ def index(request): flagged = Package.objects.normal().filter( flag_date__isnull=False, pkgbase__in=inner_q).order_by('pkgname') - todopkgs = TodolistPkg.objects.select_related( - 'pkg', 'pkg__arch', 'pkg__repo').filter(complete=False) - todopkgs = todopkgs.filter(pkg__pkgbase__in=inner_q).order_by( - 'list__name', 'pkg__pkgname') + todopkgs = TodolistPackage.objects.select_related( + 'todolist', 'pkg', 'arch', 'repo').exclude( + status=TodolistPackage.COMPLETE) + todopkgs = todopkgs.filter(pkgbase__in=inner_q).order_by( + 'todolist__name', 'pkgname') todolists = get_annotated_todolists(incomplete_only=True) diff --git a/sitestatic/archweb.css b/sitestatic/archweb.css index 6d6e1569..cfa30f5e 100644 --- a/sitestatic/archweb.css +++ b/sitestatic/archweb.css @@ -999,6 +999,11 @@ ul.admin-actions { .todo-table .incomplete { color: red; } + +.todo-table .inprogress { + color: darkorange; +} + .todo-info { margin: 0; color: #999; } diff --git a/sitestatic/archweb.js b/sitestatic/archweb.js index 42efb3fa..4a02fb63 100644 --- a/sitestatic/archweb.js +++ b/sitestatic/archweb.js @@ -15,7 +15,12 @@ if (typeof $ !== 'undefined' && typeof $.tablesorter !== 'undefined') { id: 'todostatus', is: function(s) { return false; }, format: function(s) { - return s.match(/incomplete/i) ? 1 : 0; + if (s.match(/incomplete/i)) { + return 1; + } else if (s.match(/in-progress/i)) { + return 0.5; + } + return 0; }, type: 'numeric' }); @@ -304,13 +309,9 @@ function todolist_flag() { // TODO: fix usage of this var link = this; $.getJSON(link.href, function(data) { - if (data.complete) { - $(link).text('Complete').addClass( - 'complete').removeClass('incomplete'); - } else { - $(link).text('Incomplete').addClass( - 'incomplete').removeClass('complete'); - } + $(link).text(data.status).removeClass( + 'complete inprogress incomplete').addClass( + data.css_class.toLowerCase()); /* let tablesorter know the cell value has changed */ $('.results').trigger('updateCell', [$(link).closest('td')[0], false, null]); }); diff --git a/templates/devel/index.html b/templates/devel/index.html index a07a4190..57151041 100644 --- a/templates/devel/index.html +++ b/templates/devel/index.html @@ -59,11 +59,11 @@

    My Incomplete Todo List Packages

     {% for todopkg in todopkgs %} - + - - + + {% empty %} @@ -90,7 +90,7 @@

    Package Todo Lists

    - + diff --git a/templates/todolists/email_notification.txt b/templates/todolists/email_notification.txt index 8b22b465..e454ec79 100644 --- a/templates/todolists/email_notification.txt +++ b/templates/todolists/email_notification.txt @@ -1,7 +1,7 @@ {% autoescape off %}The todo list "{{ todolist.name }}" has had the following packages added to it for which you are a maintainer: {% for tpkg in todo_packages %} -* {{ tpkg.pkg.repo.name|lower }}/{{ tpkg.pkg.pkgname }} ({{ tpkg.pkg.arch.name }}) - {{ tpkg.pkg.get_full_url }}{% endfor %} +* {{ tpkg.repo.name|lower }}/{{ tpkg.pkgname }} ({{ tpkg.arch.name }}) - {{ tpkg.pkg.get_full_url }}{% endfor %} Todo list information: Name: {{ todolist.name }} diff --git a/templates/todolists/list.html b/templates/todolists/list.html index 51f9b570..4e456d28 100644 --- a/templates/todolists/list.html +++ b/templates/todolists/list.html @@ -8,7 +8,7 @@

    Package Todo Lists

    - {% if perms.main.add_todolist %} + {% if perms.todolists.add_todolist %} @@ -31,7 +31,7 @@

    Package Todo Lists

    - + diff --git a/templates/todolists/public_list.html b/templates/todolists/public_list.html index 9e0e5234..41caf5b0 100644 --- a/templates/todolists/public_list.html +++ b/templates/todolists/public_list.html @@ -30,7 +30,7 @@

    Open Developer Todo Lists

    {{ list.name }}

    -

    {{ list.date_added|date }} - {{ list.creator.get_full_name }}

    +

    {{ list.created|date }} - {{ list.creator.get_full_name }}

    {{ list.description|urlize|linebreaks }}
    Arch
    {{ pkg.arch.name }} {{ pkg.repo.name|capfirst }} {% pkg_details_link pkg %}Repository # Packages # Flagged# Maintainers
    {{ repo.flagged_ct }} packages{{ repo.maintainer_ct }} maintainers
    {{ todopkg.list.name }}{{ todopkg.todolist.name }} {% pkg_details_link todopkg.pkg %}{{ todopkg.pkg.repo.name }}{{ todopkg.pkg.arch.name }}{{ todopkg.repo.name }}{{ todopkg.arch.name }} {{ todopkg.pkg.maintainers|join:', ' }}
    {{ todo.name }}{{ todo.date_added|date }}{{ todo.created|date }} {{ todo.creator.get_full_name }} {{ todo.description|urlize }} {{ todo.pkg_count }}
    {{ list.name }}{{ list.date_added|date }}{{ list.created|date }} {{ list.creator.get_full_name }} {{ list.description|urlize }} {{ list.pkg_count }}
    @@ -45,17 +45,11 @@

    {{ list.name }}

    {% for pkg in list.packages %} - - - - - + + + + + {% endfor %} diff --git a/templates/todolists/view.html b/templates/todolists/view.html index b6f59704..0f3475a2 100644 --- a/templates/todolists/view.html +++ b/templates/todolists/view.html @@ -10,17 +10,17 @@

    Todo List: {{ list.name }}

      - {% if perms.main.delete_todolist %} + {% if perms.todolists.delete_todolist %}
    • Delete Todo List
      • {% endif %} - {% if perms.main.change_todolist %} + {% if perms.todolists.change_todolist %}
    • Edit Todo List
      • {% endif %}
    -

    {{ list.date_added|date }} - {{ list.creator.get_full_name }}

    +

    {{ list.created|date }} - {{ list.creator.get_full_name }}

    {{list.description|urlize|linebreaks}}
    @@ -68,27 +68,22 @@

    Filter Todo List Packages

    {% for pkg in list.packages %} - - - - + + + + {% if pkg.pkg.flag_date %} {% else %} {% endif %} - + diff --git a/todolists/urls.py b/todolists/urls.py index a379468f..81ac11f5 100644 --- a/todolists/urls.py +++ b/todolists/urls.py @@ -11,7 +11,7 @@ (r'^edit/(?P\d+)/$', 'edit'), (r'^flag/(\d+)/(\d+)/$', 'flag'), (r'^delete/(?P\d+)/$', - permission_required('main.delete_todolist')(DeleteTodolist.as_view())), + permission_required('todolists.delete_todolist')(DeleteTodolist.as_view())), ) # vim: set ts=4 sw=4 et: diff --git a/todolists/utils.py b/todolists/utils.py index 03c47931..d084c645 100644 --- a/todolists/utils.py +++ b/todolists/utils.py @@ -1,26 +1,26 @@ from django.db import connections, router from django.db.models import Count -from main.models import Todolist, TodolistPkg +from .models import Todolist, TodolistPackage def todo_counts(): sql = """ -SELECT list_id, count(*), sum(CASE WHEN complete THEN 1 ELSE 0 END) - FROM todolist_pkgs - GROUP BY list_id +SELECT todolist_id, count(*), sum(CASE WHEN status = %s THEN 1 ELSE 0 END) + FROM todolists_todolistpackage + GROUP BY todolist_id """ - database = router.db_for_write(TodolistPkg) + database = router.db_for_write(TodolistPackage) connection = connections[database] cursor = connection.cursor() - cursor.execute(sql) + cursor.execute(sql, [TodolistPackage.COMPLETE]) results = cursor.fetchall() return {row[0]: (row[1], row[2]) for row in results} def get_annotated_todolists(incomplete_only=False): lists = Todolist.objects.all().select_related( - 'creator').order_by('-date_added') + 'creator').order_by('-created') lookup = todo_counts() # tag each list with package counts diff --git a/todolists/views.py b/todolists/views.py index 9984ef9a..94164391 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -10,39 +10,45 @@ from django.views.generic import DeleteView from django.template import Context, loader -from main.models import Todolist, TodolistPkg, Package, Repo +from main.models import Package, Repo from packages.utils import attach_maintainers +from .models import Todolist, TodolistPackage from .utils import get_annotated_todolists + class TodoListForm(forms.ModelForm): - packages = forms.CharField(required=False, + raw = forms.CharField(label='Packages', required=False, help_text='(one per line)', widget=forms.Textarea(attrs={'rows': '20', 'cols': '60'})) - def clean_packages(self): - package_names = [s.strip() for s in - self.cleaned_data['packages'].split("\n")] - package_names = set(package_names) - packages = Package.objects.filter(pkgname__in=package_names).filter( - repo__testing=False, repo__staging=False).select_related( - 'arch', 'repo').order_by('arch') - return packages + def packages(self): + package_names = {s.strip() for s in + self.cleaned_data['raw'].split("\n")} + return Package.objects.normal().filter(pkgname__in=package_names, + repo__testing=False, repo__staging=False).order_by('arch') class Meta: model = Todolist - fields = ('name', 'description') + fields = ('name', 'description', 'raw') + -@permission_required('main.change_todolistpkg') +@permission_required('todolists.change_todolistpackage') @never_cache def flag(request, list_id, pkg_id): todolist = get_object_or_404(Todolist, id=list_id) - pkg = get_object_or_404(TodolistPkg, id=pkg_id) - pkg.complete = not pkg.complete - pkg.save() + tlpkg = get_object_or_404(TodolistPackage, id=pkg_id) + # TODO: none of this; require absolute value on submit + if tlpkg.status == TodolistPackage.INCOMPLETE: + tlpkg.status = TodolistPackage.COMPLETE + else: + tlpkg.status = TodolistPackage.INCOMPLETE + tlpkg.save() if request.is_ajax(): - return HttpResponse( - json.dumps({'complete': pkg.complete}), - mimetype='application/json') + data = { + 'status': tlpkg.get_status_display(), + 'css_class': tlpkg.status_css_class(), + } + return HttpResponse(json.dumps(data), mimetype='application/json') return redirect(todolist) @login_required @@ -52,9 +58,9 @@ def view(request, list_id): 'svn_root', flat=True).order_by().distinct() # we don't hold onto the result, but the objects are the same here, # so accessing maintainers in the template is now cheap - attach_maintainers(tp.pkg for tp in todolist.packages) - arches = {tp.pkg.arch for tp in todolist.packages} - repos = {tp.pkg.repo for tp in todolist.packages} + attach_maintainers(todolist.packages()) + arches = {tp.arch for tp in todolist.packages()} + repos = {tp.repo for tp in todolist.packages()} return render(request, 'todolists/view.html', { 'list': todolist, 'svn_roots': svn_roots, @@ -67,9 +73,9 @@ def list_pkgbases(request, list_id, svn_root): '''Used to make bulk moves of packages a lot easier.''' todolist = get_object_or_404(Todolist, id=list_id) repos = get_list_or_404(Repo, svn_root=svn_root) - pkgbases = {tp.pkg.pkgbase for tp in todolist.packages - if tp.pkg.repo in repos} - return HttpResponse('\n'.join(sorted(pkgbases)), + pkgbases = TodolistPackage.objects.values_list('pkgbase', flat=True).filter( + todolist=todolist, repo__in=repos).distinct().order_by('pkgbase') + return HttpResponse('\n'.join(pkgbases), mimetype='text/plain') @login_required @@ -77,7 +83,7 @@ def todolist_list(request): lists = get_annotated_todolists() return render(request, 'todolists/list.html', {'lists': lists}) -@permission_required('main.add_todolist') +@permission_required('todolists.add_todolist') @never_cache def add(request): if request.POST: @@ -98,7 +104,7 @@ def add(request): return render(request, 'general_form.html', page_dict) # TODO: this calls for transaction management and async emailing -@permission_required('main.change_todolist') +@permission_required('todolists.change_todolist') @never_cache def edit(request, list_id): todo_list = get_object_or_404(Todolist, id=list_id) @@ -110,7 +116,7 @@ def edit(request, list_id): return redirect(todo_list) else: form = TodoListForm(instance=todo_list, - initial={ 'packages': '\n'.join(todo_list.package_names) }) + initial={ 'packages': todo_list.raw }) page_dict = { 'title': 'Edit Todo List: %s' % todo_list.name, @@ -128,7 +134,7 @@ class DeleteTodolist(DeleteView): @transaction.commit_on_success def create_todolist_packages(form, creator=None): - packages = form.cleaned_data['packages'] + packages = form.packages() if creator: # todo list is new todolist = form.save(commit=False) @@ -140,18 +146,22 @@ def create_todolist_packages(form, creator=None): # todo list already existed form.save() todolist = form.instance + # first delete any packages not in the new list - for todo_pkg in todolist.packages: + for todo_pkg in todolist.packages(): if todo_pkg.pkg not in packages: todo_pkg.delete() # save the old package list so we know what to add - old_packages = [p.pkg for p in todolist.packages] + old_packages = [todo_pkg.pkg for todo_pkg in todolist.packages()] todo_pkgs = [] for package in packages: if package not in old_packages: - todo_pkg = TodolistPkg.objects.create(list=todolist, pkg=package) + todo_pkg = TodolistPackage.objects.create(todolist=todolist, + pkg=package, pkgname=package.pkgname, + pkgbase=package.pkgbase, + arch=package.arch, repo=package.repo) todo_pkgs.append(todo_pkg) return todo_pkgs @@ -186,8 +196,8 @@ def send_todolist_emails(todo_list, new_packages): def public_list(request): todo_lists = Todolist.objects.incomplete() # total hackjob, but it makes this a lot less query-intensive. - all_pkgs = [tp for tl in todo_lists for tp in tl.packages] - attach_maintainers([tp.pkg for tp in all_pkgs]) + all_pkgs = [tp for tl in todo_lists for tp in tl.packages()] + attach_maintainers(all_pkgs) return render(request, "todolists/public_list.html", {"todo_lists": todo_lists}) -- cgit v1.2.3-54-g00ecf From 04f23a040a839f4989fdc83afe0f5ad4f72224be Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 31 Dec 2012 10:17:22 -0600 Subject: Add 'created' field to packages model This will be used to eventually implement the UI side of FS#13441, but to do that, we first need the data. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 6 +- .../0063_auto__add_field_package_created.py | 116 +++++++++++++++++++++ main/models.py | 3 + 3 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 main/migrations/0063_auto__add_field_package_created.py (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 981c4dce..e00e54c3 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -387,10 +387,12 @@ def db_update(archname, reponame, pkgs, force=False): # packages in syncdb and not in database (add to database) for pkg in (pkg for pkg in pkgs if pkg.name in in_sync_not_db): logger.info("Adding package %s", pkg.name) - dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository) + timestamp = now() + dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository, + created=timestamp) try: with transaction.commit_on_success(): - populate_pkg(dbpkg, pkg, timestamp=now()) + populate_pkg(dbpkg, pkg, timestamp=timestamp) Update.objects.log_update(None, dbpkg) except IntegrityError: if architecture.agnostic: diff --git a/main/migrations/0063_auto__add_field_package_created.py b/main/migrations/0063_auto__add_field_package_created.py new file mode 100644 index 00000000..e5a990c3 --- /dev/null +++ b/main/migrations/0063_auto__add_field_package_created.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +import datetime +from pytz import utc +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + old_date = datetime.datetime(2000, 1, 1) + old_date = old_date.replace(tzinfo=utc) + db.add_column('packages', 'created', + self.gf('django.db.models.fields.DateTimeField')(default=old_date), keep_default=False) + + + def backwards(self, orm): + db.delete_column('packages', 'created') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'main.arch': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Arch', 'db_table': "'arches'"}, + 'agnostic': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'main.donor': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Donor', 'db_table': "'donors'"}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'visible': ('django.db.models.fields.BooleanField', [], {'default': 'True'}) + }, + 'main.package': { + 'Meta': {'ordering': "('pkgname',)", 'unique_together': "(('pkgname', 'repo', 'arch'),)", 'object_name': 'Package', 'db_table': "'packages'"}, + 'arch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'on_delete': 'models.PROTECT', 'to': "orm['main.Arch']"}), + 'build_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'compressed_size': ('main.fields.PositiveBigIntegerField', [], {}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'epoch': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'files_last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'flag_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'installed_size': ('main.fields.PositiveBigIntegerField', [], {}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'packager': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}), + 'packager_str': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pgp_signature': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkgdesc': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'pkgname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'repo': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'on_delete': 'models.PROTECT', 'to': "orm['main.Repo']"}), + 'url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}) + }, + 'main.packagefile': { + 'Meta': {'object_name': 'PackageFile', 'db_table': "'package_files'"}, + 'directory': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_directory': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}) + }, + 'main.repo': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Repo', 'db_table': "'repos'"}, + 'bugs_category': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'bugs_project': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'staging': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'svn_root': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + } + } + + complete_apps = ['main'] diff --git a/main/models.py b/main/models.py index 7ec04ad7..39bc555e 100644 --- a/main/models.py +++ b/main/models.py @@ -103,6 +103,7 @@ class Package(models.Model): build_date = models.DateTimeField(null=True) last_update = models.DateTimeField(db_index=True) files_last_update = models.DateTimeField(null=True, blank=True) + created = models.DateTimeField() packager_str = models.CharField(max_length=255) packager = models.ForeignKey(User, null=True, blank=True, on_delete=models.SET_NULL) @@ -424,6 +425,8 @@ class Meta: post_save.connect(refresh_latest, sender=Package, dispatch_uid="main.models") +# note: reporead sets the 'created' field on Package objects, so no signal +# listener is set up here to do so pre_save.connect(set_created_field, sender=Donor, dispatch_uid="main.models") -- cgit v1.2.3-54-g00ecf From 7952fe0ede3a5a68a64f05eccb180194394652f3 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 31 Dec 2012 11:31:35 -0600 Subject: Mark todolist packages as removed rather than deleting them This makes it easier to see the progression of a todolist and its contents easier since we are no longer losing the data. Signed-off-by: Dan McGee --- devel/views.py | 2 +- todolists/models.py | 8 +++++--- todolists/utils.py | 1 + todolists/views.py | 50 +++++++++++++++++++++++++++++++------------------- 4 files changed, 38 insertions(+), 23 deletions(-) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index e01590a0..90839847 100644 --- a/devel/views.py +++ b/devel/views.py @@ -44,7 +44,7 @@ def index(request): todopkgs = TodolistPackage.objects.select_related( 'todolist', 'pkg', 'arch', 'repo').exclude( - status=TodolistPackage.COMPLETE) + status=TodolistPackage.COMPLETE).filter(removed__isnull=True) todopkgs = todopkgs.filter(pkgbase__in=inner_q).order_by( 'todolist__name', 'pkgname') diff --git a/todolists/models.py b/todolists/models.py index 156b041d..040f8a29 100644 --- a/todolists/models.py +++ b/todolists/models.py @@ -10,8 +10,9 @@ class TodolistManager(models.Manager): def incomplete(self): - not_done = (Q(todolistpackage__status=TodolistPackage.INCOMPLETE) | - Q(todolistpackage__status=TodolistPackage.IN_PROGRESS)) + not_done = ((Q(todolistpackage__status=TodolistPackage.INCOMPLETE) | + Q(todolistpackage__status=TodolistPackage.IN_PROGRESS)) & + Q(todolistpackage__removed__isnull=True)) return self.order_by().filter(not_done).distinct() @@ -44,7 +45,8 @@ def get_full_url(self, proto='https'): def packages(self): if not hasattr(self, '_packages'): - self._packages = self.todolistpackage_set.select_related( + self._packages = self.todolistpackage_set.filter( + removed__isnull=True).select_related( 'pkg', 'repo', 'arch').order_by('pkgname', 'arch') return self._packages diff --git a/todolists/utils.py b/todolists/utils.py index 0daca3b6..e86d9054 100644 --- a/todolists/utils.py +++ b/todolists/utils.py @@ -8,6 +8,7 @@ def todo_counts(): sql = """ SELECT todolist_id, count(*), sum(CASE WHEN status = %s THEN 1 ELSE 0 END) FROM todolists_todolistpackage + WHERE removed IS NULL GROUP BY todolist_id """ database = router.db_for_write(TodolistPackage) diff --git a/todolists/views.py b/todolists/views.py index 113b27e7..32ee7cc5 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -10,6 +10,7 @@ from django.views.generic import DeleteView from django.template import Context, loader from django.template.defaultfilters import slugify +from django.utils.timezone import now from main.models import Package, Repo from main.utils import find_unique_slug @@ -23,10 +24,12 @@ class TodoListForm(forms.ModelForm): help_text='(one per line)', widget=forms.Textarea(attrs={'rows': '20', 'cols': '60'})) + def package_names(self): + return {s.strip() for s in self.cleaned_data['raw'].split("\n")} + def packages(self): - package_names = {s.strip() for s in - self.cleaned_data['raw'].split("\n")} - return Package.objects.normal().filter(pkgname__in=package_names, + return Package.objects.normal().filter( + pkgname__in=self.package_names(), repo__testing=False, repo__staging=False).order_by('arch') class Meta: @@ -37,7 +40,7 @@ class Meta: @never_cache def flag(request, slug, pkg_id): todolist = get_object_or_404(Todolist, slug=slug) - tlpkg = get_object_or_404(TodolistPackage, id=pkg_id) + tlpkg = get_object_or_404(TodolistPackage, id=pkg_id, removed__isnull=True) # TODO: none of this; require absolute value on submit if tlpkg.status == TodolistPackage.INCOMPLETE: tlpkg.status = TodolistPackage.COMPLETE @@ -75,13 +78,14 @@ def view(request, slug): }) -# really no need for login_required on this one... def list_pkgbases(request, slug, svn_root): '''Used to make bulk moves of packages a lot easier.''' todolist = get_object_or_404(Todolist, slug=slug) repos = get_list_or_404(Repo, svn_root=svn_root) - pkgbases = TodolistPackage.objects.values_list('pkgbase', flat=True).filter( - todolist=todolist, repo__in=repos).distinct().order_by('pkgbase') + pkgbases = TodolistPackage.objects.values_list( + 'pkgbase', flat=True).filter( + todolist=todolist, repo__in=repos, removed__isnull=True).order_by( + 'pkgbase').distinct() return HttpResponse('\n'.join(pkgbases), mimetype='text/plain') @@ -143,36 +147,44 @@ class DeleteTodolist(DeleteView): @transaction.commit_on_success def create_todolist_packages(form, creator=None): + package_names = form.package_names() packages = form.packages() + timestamp = now() if creator: # todo list is new, populate creator and slug fields todolist = form.save(commit=False) todolist.creator = creator todolist.slug = find_unique_slug(Todolist, todolist.name) todolist.save() - - old_packages = [] else: # todo list already existed form.save() todolist = form.instance - # first delete any packages not in the new list + # first mark removed any packages not in the new list + to_remove = set() for todo_pkg in todolist.packages(): - if todo_pkg.pkg not in packages: - todo_pkg.delete() + if todo_pkg.pkg and todo_pkg.pkg not in packages: + to_remove.add(todo_pkg.pk) + elif todo_pkg.pkgname not in package_names: + to_remove.add(todo_pkg.pk) - # save the old package list so we know what to add - old_packages = [todo_pkg.pkg for todo_pkg in todolist.packages()] + TodolistPackage.objects.filter( + pk__in=to_remove).update(removed=timestamp) + # Add (or mark unremoved) any packages in the new packages list todo_pkgs = [] for package in packages: - if package not in old_packages: - todo_pkg = TodolistPackage.objects.create(todolist=todolist, - pkg=package, pkgname=package.pkgname, - pkgbase=package.pkgbase, - arch=package.arch, repo=package.repo) + todo_pkg, created = TodolistPackage.objects.get_or_create( + todolist=todolist, + pkg=package, pkgname=package.pkgname, + pkgbase=package.pkgbase, + arch=package.arch, repo=package.repo) + if created: todo_pkgs.append(todo_pkg) + elif todo_pkg.removed is not None: + todo_pkg.removed = None + todo_pkg.save() return todo_pkgs -- cgit v1.2.3-54-g00ecf From 3d23df0b661de5ccd5096dda22abcfb161e8b1b4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 31 Dec 2012 11:47:14 -0600 Subject: Fix case of devel user profile verbose name Signed-off-by: Dan McGee --- devel/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'devel') diff --git a/devel/models.py b/devel/models.py index 5f0a8318..6689ca3d 100644 --- a/devel/models.py +++ b/devel/models.py @@ -51,8 +51,8 @@ class UserProfile(models.Model): class Meta: db_table = 'user_profiles' get_latest_by = 'last_modified' - verbose_name = 'Additional Profile Data' - verbose_name_plural = 'Additional Profile Data' + verbose_name = 'additional profile data' + verbose_name_plural = 'additional profile data' def get_absolute_url(self): # TODO: this is disgusting. find a way to consolidate this logic with -- cgit v1.2.3-54-g00ecf From af32c23768c7537f19e0613525579208b4f44eb4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 15 Jan 2013 20:49:56 -0600 Subject: Handle connection and transaction more properly in reporead A few minor things are fixed here. One is PostgreSQL, and more specifically pgbouncer, don't like it when the connection is closed after psycopg2 has started an implicit transaction even for read-only queries. Ensure we call commit as our last database action in all cases. The other is related- Django in management commands doesn't ever call close on any database connection you may have been using, so PostgreSQL gets mad about this fact and logs a message saying such. Close the connection explicitly when we are done with it to play nice. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 1 + devel/management/commands/reporead_inotify.py | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index e00e54c3..ab0efeed 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -589,6 +589,7 @@ def read_repo(primary_arch, repo_file, options): else: db_update(arch, repo, packages_arches[arch], force) logger.info('Finished database updates for %s.', repo_file) + connection.commit() connection.close() return 0 diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py index 04f65764..8c1e47bf 100644 --- a/devel/management/commands/reporead_inotify.py +++ b/devel/management/commands/reporead_inotify.py @@ -23,7 +23,7 @@ import time from django.core.management.base import BaseCommand, CommandError -from django.db import connection +from django.db import connection, transaction from main.models import Arch, Repo from .reporead import read_repo @@ -53,6 +53,11 @@ def handle(self, path_template=None, **options): self.path_template = path_template notifier = self.setup_notifier() + # this thread is done using the database; all future access is done in + # the spawned read_repo() processes, so close the otherwise completely + # idle connection. + connection.close() + logger.info('Entering notifier loop') notifier.loop() @@ -61,14 +66,17 @@ def handle(self, path_template=None, **options): if hasattr(thread, 'cancel'): thread.cancel() + @transaction.commit_on_success def setup_notifier(self): '''Set up and configure the inotify machinery and logic. This takes the provided or default path_template and builds a list of directories we need to watch for database updates. It then validates and passes these on to the various pyinotify pieces as necessary and finally builds and returns a notifier object.''' + transaction.commit_manually() arches = Arch.objects.filter(agnostic=False) repos = Repo.objects.all() + transaction.set_dirty() arch_path_map = {arch: None for arch in arches} all_paths = set() total_paths = 0 @@ -91,11 +99,6 @@ def setup_notifier(self): raise CommandError('path template did not uniquely ' 'determine architecture for each file') - # this thread is done using the database; all future access is done in - # the spawned read_repo() processes, so close the otherwise completely - # idle connection. - connection.close() - # A proper atomic replacement of the database as done by rsync is type # IN_MOVED_TO. repo-add/remove will finish with a IN_CLOSE_WRITE. mask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO -- cgit v1.2.3-54-g00ecf From 9da8a63dd476fe3607a68a028655c9f9d0fee163 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 3 Feb 2013 13:55:25 -0600 Subject: Add DeveloperKey model We're starting to see developers use subkeys of their primary key to sign packages, which we aren't handling well in the web interface. These subkeys show up as unknown, which isn't strictly true. Start the process of being able to handle these keys by adding a model that will store all known keys and subkeys and the relationships among them, as well as which developer owns each. Signed-off-by: Dan McGee --- devel/admin.py | 12 ++- devel/migrations/0009_auto__add_developerkey.py | 126 ++++++++++++++++++++++++ devel/models.py | 15 ++- 3 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 devel/migrations/0009_auto__add_developerkey.py (limited to 'devel') diff --git a/devel/admin.py b/devel/admin.py index 5a704c0b..971933b7 100644 --- a/devel/admin.py +++ b/devel/admin.py @@ -2,7 +2,7 @@ from django.contrib.auth.admin import UserAdmin from django.contrib.auth.models import User -from .models import UserProfile, MasterKey, PGPSignature +from .models import UserProfile, MasterKey, DeveloperKey, PGPSignature class UserProfileInline(admin.StackedInline): @@ -17,7 +17,14 @@ class UserProfileAdmin(UserAdmin): class MasterKeyAdmin(admin.ModelAdmin): list_display = ('pgp_key', 'owner', 'created', 'revoker', 'revoked') - search_fields = ('pgp_key', 'owner', 'revoker') + search_fields = ('pgp_key', 'owner__username', 'revoker__username') + date_hierarchy = 'created' + + +class DeveloperKeyAdmin(admin.ModelAdmin): + list_display = ('key', 'parent', 'owner', 'created', 'expires', 'revoked') + search_fields = ('key', 'owner__username') + list_filter = ('owner',) date_hierarchy = 'created' @@ -32,6 +39,7 @@ class PGPSignatureAdmin(admin.ModelAdmin): admin.site.register(User, UserProfileAdmin) admin.site.register(MasterKey, MasterKeyAdmin) +admin.site.register(DeveloperKey, DeveloperKeyAdmin) admin.site.register(PGPSignature, PGPSignatureAdmin) # vim: set ts=4 sw=4 et: diff --git a/devel/migrations/0009_auto__add_developerkey.py b/devel/migrations/0009_auto__add_developerkey.py new file mode 100644 index 00000000..60d3f7b8 --- /dev/null +++ b/devel/migrations/0009_auto__add_developerkey.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.create_table('devel_developerkey', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('owner', self.gf('django.db.models.fields.related.ForeignKey')(related_name='all_keys', null=True, to=orm['auth.User'])), + ('key', self.gf('devel.fields.PGPKeyField')(unique=True, max_length=40)), + ('created', self.gf('django.db.models.fields.DateTimeField')()), + ('expires', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)), + ('revoked', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)), + ('parent', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['devel.DeveloperKey'], null=True, on_delete=models.SET_NULL)), + )) + db.send_create_signal('devel', ['DeveloperKey']) + + def backwards(self, orm): + db.delete_table('devel_developerkey') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'devel.developerkey': { + 'Meta': {'object_name': 'DeveloperKey'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'expires': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'key': ('devel.fields.PGPKeyField', [], {'unique': 'True', 'max_length': '40'}), + 'owner': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'all_keys'", 'null': 'True', 'to': "orm['auth.User']"}), + 'parent': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['devel.DeveloperKey']", 'null': 'True', 'on_delete': 'models.SET_NULL'}), + 'revoked': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}) + }, + 'devel.masterkey': { + 'Meta': {'ordering': "('created',)", 'object_name': 'MasterKey'}, + 'created': ('django.db.models.fields.DateField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'owner': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'masterkey_owner'", 'to': "orm['auth.User']"}), + 'pgp_key': ('devel.fields.PGPKeyField', [], {'max_length': '40'}), + 'revoked': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'revoker': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'masterkey_revoker'", 'to': "orm['auth.User']"}) + }, + 'devel.pgpsignature': { + 'Meta': {'ordering': "('signer', 'signee')", 'object_name': 'PGPSignature'}, + 'created': ('django.db.models.fields.DateField', [], {}), + 'expires': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'signee': ('devel.fields.PGPKeyField', [], {'max_length': '40'}), + 'signer': ('devel.fields.PGPKeyField', [], {'max_length': '40'}), + 'valid': ('django.db.models.fields.BooleanField', [], {'default': 'True'}) + }, + 'devel.userprofile': { + 'Meta': {'object_name': 'UserProfile', 'db_table': "'user_profiles'"}, + 'alias': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'allowed_repos': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['main.Repo']", 'symmetrical': 'False', 'blank': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2', 'blank': 'True'}), + 'favorite_distros': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interests': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'languages': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'last_modified': ('django.db.models.fields.DateTimeField', [], {}), + 'latin_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'location': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'notify': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'occupation': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'other_contact': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'pgp_key': ('devel.fields.PGPKeyField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}), + 'picture': ('django.db.models.fields.files.FileField', [], {'default': "'devs/silhouette.png'", 'max_length': '100'}), + 'public_email': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'roles': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'time_zone': ('django.db.models.fields.CharField', [], {'default': "'UTC'", 'max_length': '100'}), + 'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'userprofile'", 'unique': 'True', 'to': "orm['auth.User']"}), + 'website': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'yob': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) + }, + 'main.repo': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Repo', 'db_table': "'repos'"}, + 'bugs_category': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'bugs_project': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'staging': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'svn_root': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + } + } + + complete_apps = ['devel'] diff --git a/devel/models.py b/devel/models.py index 6689ca3d..67de40a6 100644 --- a/devel/models.py +++ b/devel/models.py @@ -68,7 +68,6 @@ def get_absolute_url(self): return '/%s/#%s' % (prefix, self.user.username) - class MasterKey(models.Model): owner = models.ForeignKey(User, related_name='masterkey_owner', help_text="The developer holding this master key") @@ -88,6 +87,20 @@ def __unicode__(self): self.owner.get_full_name(), self.created) +class DeveloperKey(models.Model): + owner = models.ForeignKey(User, related_name='all_keys', null=True, + help_text="The developer this key belongs to") + key = PGPKeyField(max_length=40, verbose_name="PGP key fingerprint", + unique=True) + created = models.DateTimeField() + expires = models.DateTimeField(null=True, blank=True) + revoked = models.DateTimeField(null=True, blank=True) + parent = models.ForeignKey('self', null=True, on_delete=models.SET_NULL) + + def __unicode__(self): + return self.key + + class PGPSignature(models.Model): signer = PGPKeyField(max_length=40, verbose_name="Signer key fingerprint") signee = PGPKeyField(max_length=40, verbose_name="Signee key fingerprint") -- cgit v1.2.3-54-g00ecf From 7e6279057a57ef44c11349e594ad392fbfce0098 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 3 Feb 2013 14:26:10 -0600 Subject: Add new pgp_import command; replaces import_signatures This command now imports keys, subkeys, and signatures of those keys & subkeys. This will allow us to actually match developers with their packages signed by subkeys rather than the primary key. Signed-off-by: Dan McGee --- devel/management/commands/import_signatures.py | 123 ------------- devel/management/commands/pgp_import.py | 241 +++++++++++++++++++++++++ 2 files changed, 241 insertions(+), 123 deletions(-) delete mode 100644 devel/management/commands/import_signatures.py create mode 100644 devel/management/commands/pgp_import.py (limited to 'devel') diff --git a/devel/management/commands/import_signatures.py b/devel/management/commands/import_signatures.py deleted file mode 100644 index da1397ca..00000000 --- a/devel/management/commands/import_signatures.py +++ /dev/null @@ -1,123 +0,0 @@ -# -*- coding: utf-8 -*- -""" -import_signatures command - -Import signatures from a given GPG keyring. - -Usage: ./manage.py generate_keyring -""" - -from collections import namedtuple -from datetime import datetime -import logging -import subprocess -import sys - -from django.core.management.base import BaseCommand, CommandError -from django.db import transaction - -from devel.models import PGPSignature - -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s -> %(levelname)s: %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', - stream=sys.stderr) -logger = logging.getLogger() - -class Command(BaseCommand): - args = "" - help = "Import signatures from a given GPG keyring." - - def handle(self, *args, **options): - v = int(options.get('verbosity', None)) - if v == 0: - logger.level = logging.ERROR - elif v == 1: - logger.level = logging.INFO - elif v == 2: - logger.level = logging.DEBUG - - if len(args) < 1: - raise CommandError("keyring_path must be provided") - - import_signatures(args[0]) - - -SignatureData = namedtuple('SignatureData', - ('signer', 'signee', 'created', 'expires', 'valid')) - - -def get_date(epoch_string): - '''Convert a epoch string into a python 'date' object (not datetime).''' - return datetime.utcfromtimestamp(int(epoch_string)).date() - - -def parse_sigdata(data): - nodes = {} - edges = [] - current_pubkey = None - - # parse all of the output from our successful GPG command - logger.info("parsing command output") - for line in data.split('\n'): - parts = line.split(':') - if parts[0] == 'pub': - current_pubkey = parts[4] - nodes[current_pubkey] = None - if parts[0] == 'uid': - uid = parts[9] - # only set uid if this is the first one encountered - if nodes[current_pubkey] is None: - nodes[current_pubkey] = uid - if parts[0] == 'sig': - signer = parts[4] - created = get_date(parts[5]) - expires = None - if parts[6]: - expires = get_date(parts[6]) - valid = parts[1] != '-' - edge = SignatureData(signer, current_pubkey, - created, expires, valid) - edges.append(edge) - - return nodes, edges - - -def import_signatures(keyring): - gpg_cmd = ["gpg", "--no-default-keyring", "--keyring", keyring, - "--list-sigs", "--with-colons", "--fixed-list-mode"] - logger.info("running command: %r", gpg_cmd) - proc = subprocess.Popen(gpg_cmd, stdout=subprocess.PIPE) - outdata, errdata = proc.communicate() - if proc.returncode != 0: - logger.error(errdata) - raise subprocess.CalledProcessError(proc.returncode, gpg_cmd) - - nodes, edges = parse_sigdata(outdata) - - # now prune the data down to what we actually want. - # prune edges not in nodes, remove duplicates, and self-sigs - pruned_edges = {edge for edge in edges - if edge.signer in nodes and edge.signer != edge.signee} - - logger.info("creating or finding %d signatures", len(pruned_edges)) - created_ct = updated_ct = 0 - with transaction.commit_on_success(): - for edge in pruned_edges: - sig, created = PGPSignature.objects.get_or_create( - signer=edge.signer, signee=edge.signee, - created=edge.created, expires=edge.expires, - defaults={ 'valid': edge.valid }) - if sig.valid != edge.valid: - sig.valid = edge.valid - sig.save() - updated_ct = 1 - if created: - created_ct += 1 - - sig_ct = PGPSignature.objects.all().count() - logger.info("%d total signatures in database", sig_ct) - logger.info("created %d, updated %d signatures", created_ct, updated_ct) - -# vim: set ts=4 sw=4 et: diff --git a/devel/management/commands/pgp_import.py b/devel/management/commands/pgp_import.py new file mode 100644 index 00000000..10e6cfcb --- /dev/null +++ b/devel/management/commands/pgp_import.py @@ -0,0 +1,241 @@ +# -*- coding: utf-8 -*- +""" +pgp_import command + +Import keys and signatures from a given GPG keyring. + +Usage: ./manage.py pgp_import +""" + +from collections import namedtuple, OrderedDict +from datetime import datetime +import logging +from pytz import utc +import subprocess +import sys + +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction + +from devel.models import DeveloperKey, PGPSignature +from devel.utils import UserFinder + + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s -> %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + stream=sys.stderr) +logger = logging.getLogger() + +class Command(BaseCommand): + args = "" + help = "Import keys and signatures from a given GPG keyring." + + def handle(self, *args, **options): + v = int(options.get('verbosity', None)) + if v == 0: + logger.level = logging.ERROR + elif v == 1: + logger.level = logging.INFO + elif v == 2: + logger.level = logging.DEBUG + + if len(args) < 1: + raise CommandError("keyring_path must be provided") + + import_keys(args[0]) + import_signatures(args[0]) + + +def get_date(epoch_string): + '''Convert a epoch string into a python 'date' object (not datetime).''' + if not epoch_string: + return None + return datetime.utcfromtimestamp(int(epoch_string)).date() + + +def get_datetime(epoch_string): + '''Convert a epoch string into a python 'datetime' object.''' + if not epoch_string: + return None + return datetime.utcfromtimestamp(int(epoch_string)).replace(tzinfo=utc) + + +def call_gpg(keyring, *args): + # GPG is stupid and interprets any filename without path portion as being + # in ~/.gnupg/. Fake it out if we just get a bare filename. + if '/' not in keyring: + keyring = './%s' % keyring + gpg_cmd = ["gpg2", "--no-default-keyring", "--keyring", keyring, + "--with-colons", "--fixed-list-mode"] + gpg_cmd.extend(args) + logger.info("running command: %s", ' '.join(gpg_cmd)) + proc = subprocess.Popen(gpg_cmd, stdout=subprocess.PIPE) + outdata, errdata = proc.communicate() + if proc.returncode != 0: + logger.error(errdata) + raise subprocess.CalledProcessError(proc.returncode, gpg_cmd) + return outdata + + +class KeyData(object): + def __init__(self, key, created, expires): + self.key = key + self.created = get_datetime(created) + self.expires = get_datetime(expires) + self.parent = None + self.revoked = None + self.db_id = None + + +def parse_keydata(data): + keys = OrderedDict() + current_pubkey = None + + # parse all of the output from our successful GPG command + logger.info("parsing command output") + for line in data.split('\n'): + parts = line.split(':') + if parts[0] == 'pub': + key = parts[4] + current_pubkey = key + keys[key] = KeyData(key, parts[5], parts[6]) + node = parts[0] + elif parts[0] == 'sub': + key = parts[4] + keys[key] = KeyData(key, parts[5], parts[6]) + keys[key].parent = current_pubkey + node = parts[0] + elif parts[0] == 'uid': + node = parts[0] + elif parts[0] == 'rev' and node in ('pub', 'sub'): + keys[current_pubkey].revoked = get_datetime(parts[5]) + + return keys + + +def find_key_owner(key, keys, finder): + '''Recurse up the chain, looking for an owner.''' + if key is None: + return None + owner = finder.find_by_pgp_key(key.key) + if owner: + return owner + if key.parent: + return find_key_owner(keys[key.parent], keys, finder) + return None + + +def import_keys(keyring): + outdata = call_gpg(keyring, "--list-sigs") + keydata = parse_keydata(outdata) + + logger.info("creating or finding %d keys", len(keydata)) + created_ct = updated_ct = 0 + with transaction.commit_on_success(): + finder = UserFinder() + # we are dependent on parents coming before children; parse_keydata + # uses an OrderedDict to ensure this is the case. + for data in keydata.values(): + parent_id = None + if data.parent: + parent_data = keydata.get(data.parent, None) + if parent_data: + parent_id = parent_data.db_id + other = { + 'expires': data.expires, + 'revoked': data.revoked, + 'parent_id': parent_id, + } + dkey, created = DeveloperKey.objects.get_or_create( + key=data.key, created=data.created, defaults=other) + data.db_id = dkey.id + + # set or update any additional data we might need to + needs_save = False + if created: + created_ct += 1 + else: + for k, v in other.items(): + if getattr(dkey, k) != v: + setattr(dkey, k, v) + needs_save = True + if dkey.owner_id is None: + owner = find_key_owner(data, keydata, finder) + if owner is not None: + dkey.owner = owner + needs_save = True + if needs_save: + dkey.save() + updated_ct += 1 + + key_ct = DeveloperKey.objects.all().count() + logger.info("%d total keys in database", key_ct) + logger.info("created %d, updated %d keys", created_ct, updated_ct) + + +SignatureData = namedtuple('SignatureData', + ('signer', 'signee', 'created', 'expires', 'valid')) + + +def parse_sigdata(data): + nodes = {} + edges = [] + current_pubkey = None + + # parse all of the output from our successful GPG command + logger.info("parsing command output") + for line in data.split('\n'): + parts = line.split(':') + if parts[0] == 'pub': + current_pubkey = parts[4] + nodes[current_pubkey] = None + if parts[0] == 'uid': + uid = parts[9] + # only set uid if this is the first one encountered + if nodes[current_pubkey] is None: + nodes[current_pubkey] = uid + if parts[0] == 'sig': + signer = parts[4] + created = get_date(parts[5]) + expires = None + if parts[6]: + expires = get_date(parts[6]) + valid = parts[1] != '-' + edge = SignatureData(signer, current_pubkey, + created, expires, valid) + edges.append(edge) + + return nodes, edges + + +def import_signatures(keyring): + outdata = call_gpg(keyring, "--list-sigs") + nodes, edges = parse_sigdata(outdata) + + # now prune the data down to what we actually want. + # prune edges not in nodes, remove duplicates, and self-sigs + pruned_edges = {edge for edge in edges + if edge.signer in nodes and edge.signer != edge.signee} + + logger.info("creating or finding %d signatures", len(pruned_edges)) + created_ct = updated_ct = 0 + with transaction.commit_on_success(): + for edge in pruned_edges: + sig, created = PGPSignature.objects.get_or_create( + signer=edge.signer, signee=edge.signee, + created=edge.created, expires=edge.expires, + defaults={ 'valid': edge.valid }) + if sig.valid != edge.valid: + sig.valid = edge.valid + sig.save() + updated_ct = 1 + if created: + created_ct += 1 + + sig_ct = PGPSignature.objects.all().count() + logger.info("%d total signatures in database", sig_ct) + logger.info("created %d, updated %d signatures", created_ct, updated_ct) + +# vim: set ts=4 sw=4 et: -- cgit v1.2.3-54-g00ecf From 508d06af810c787b2644331444279407ccfa27af Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 3 Feb 2013 15:08:21 -0600 Subject: Use DeveloperKey model on package page and reports This introduces the new model to the package page so subkey signings show up as attributed to the original developer. We also teach the mismatched signatures report to recognize all keys and subkeys of a given developer, cutting down on some of the bogus results. Signed-off-by: Dan McGee --- devel/views.py | 33 +++++++++++++++++++++++---------- main/models.py | 6 ++++-- 2 files changed, 27 insertions(+), 12 deletions(-) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index 90839847..ff1dec12 100644 --- a/devel/views.py +++ b/devel/views.py @@ -20,6 +20,7 @@ from django.utils.timezone import now from .forms import ProfileForm, UserProfileForm, NewUserForm +from .models import DeveloperKey from main.models import Package, PackageFile from main.models import Arch, Repo from news.models import News @@ -27,7 +28,7 @@ from packages.utils import get_signoff_groups from todolists.models import TodolistPackage from todolists.utils import get_annotated_todolists -from .utils import get_annotated_maintainers, UserFinder +from .utils import get_annotated_maintainers @login_required @@ -262,18 +263,30 @@ def report(request, report_name, username=None): names = [ 'Signature Date', 'Signed By', 'Packager' ] attrs = [ 'sig_date', 'sig_by', 'packager' ] cutoff = timedelta(hours=24) - finder = UserFinder() filtered = [] - packages = packages.filter(pgp_signature__isnull=False) + packages = packages.select_related( + 'arch', 'repo', 'packager').filter(pgp_signature__isnull=False) + known_keys = DeveloperKey.objects.select_related( + 'owner').filter(owner__isnull=False) + known_keys = {dk.key: dk for dk in known_keys} for package in packages: - sig_date = package.signature.creation_time.replace(tzinfo=pytz.utc) + bad = False + sig = package.signature + sig_date = sig.creation_time.replace(tzinfo=pytz.utc) package.sig_date = sig_date.date() - key_id = package.signature.key_id - signer = finder.find_by_pgp_key(key_id) - package.sig_by = signer or key_id - if signer is None or signer.id != package.packager_id: - filtered.append(package) - elif sig_date > package.build_date + cutoff: + dev_key = known_keys.get(sig.key_id, None) + if dev_key: + package.sig_by = dev_key.owner + if dev_key.owner_id != package.packager_id: + bad = True + else: + package.sig_by = sig.key_id + bad = True + + if sig_date > package.build_date + cutoff: + bad = True + + if bad: filtered.append(package) packages = filtered else: diff --git a/main/models.py b/main/models.py index 40466d65..53a24ffc 100644 --- a/main/models.py +++ b/main/models.py @@ -11,6 +11,7 @@ from .fields import PositiveBigIntegerField from .utils import cache_function, set_created_field +from devel.models import DeveloperKey from packages.alpm import AlpmAPI @@ -153,8 +154,9 @@ def signer(self): sig = self.signature if sig and sig.key_id: try: - user = User.objects.get( - userprofile__pgp_key__endswith=sig.key_id) + matching_key = DeveloperKey.objects.select_related( + 'owner').get(key=sig.key_id, owner_id__isnull=False) + user = matching_key.owner except User.DoesNotExist: user = None return user -- cgit v1.2.3-54-g00ecf From f98ff8cd22185c11dccdbe19b5bb7ed849b38e6b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 4 Feb 2013 20:02:00 -0600 Subject: Add './' hack to generate_keyring as well If you specify a relative path to gpg without a slash character, it interprets as relative to ~/.gnupg, which is stupid. Signed-off-by: Dan McGee --- devel/management/commands/generate_keyring.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'devel') diff --git a/devel/management/commands/generate_keyring.py b/devel/management/commands/generate_keyring.py index 15ae488d..34bcd2f8 100644 --- a/devel/management/commands/generate_keyring.py +++ b/devel/management/commands/generate_keyring.py @@ -55,6 +55,10 @@ def generate_keyring(keyserver, keyring): master_key_ids = MasterKey.objects.values_list("pgp_key", flat=True) logger.info("%d keys fetched from master keys", len(master_key_ids)) + # GPG is stupid and interprets any filename without path portion as being + # in ~/.gnupg/. Fake it out if we just get a bare filename. + if '/' not in keyring: + keyring = './%s' % keyring gpg_cmd = ["gpg", "--no-default-keyring", "--keyring", keyring, "--keyserver", keyserver, "--recv-keys"] logger.info("running command: %r", gpg_cmd) -- cgit v1.2.3-54-g00ecf From 55179a4f9e8b80d515bae7032af8aefc33ae0192 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 16 Jan 2013 16:07:26 -0600 Subject: reporead: remove batched_bulk_create Now that Django 1.5 is out and realized SQLite3 only allows for 999 parameters per SQL call, we don't need to manually batch things up anymore and can let the underlying bulk_create code do it for us. This basically reverts commit 88ee61a39ac3. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 32 +++++--------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index ab0efeed..ccac55f2 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -32,7 +32,6 @@ from devel.utils import UserFinder from main.models import Arch, Package, PackageFile, Repo -from main.utils import database_vendor from packages.models import Depend, Conflict, Provision, Replacement, Update from packages.utils import parse_version @@ -182,27 +181,6 @@ def create_related(model, package, rel_str, equals_only=False): return related -def batched_bulk_create(model, all_objects): - # for short lists, just bulk_create as we should be fine - if len(all_objects) < 20: - return model.objects.bulk_create(all_objects) - - if database_vendor(model, mode='write') == 'sqlite': - # 999 max variables in each SQL statement - incr = 999 // len(model._meta.fields) - else: - incr = 1000 - - def chunks(): - offset = 0 - while offset < len(all_objects): - yield all_objects[offset:offset + incr] - offset += incr - - for items in chunks(): - model.objects.bulk_create(items) - - def create_multivalued(dbpkg, repopkg, db_attr, repo_attr): '''Populate the simplest of multivalued attributes. These are those that only deal with a 'name' attribute, such as licenses, groups, etc. The input @@ -256,20 +234,20 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): deps += [create_depend(dbpkg, y, 'O') for y in repopkg.optdepends] deps += [create_depend(dbpkg, y, 'M') for y in repopkg.makedepends] deps += [create_depend(dbpkg, y, 'C') for y in repopkg.checkdepends] - batched_bulk_create(Depend, deps) + Depend.objects.bulk_create(deps) dbpkg.conflicts.all().delete() conflicts = [create_related(Conflict, dbpkg, y) for y in repopkg.conflicts] - batched_bulk_create(Conflict, conflicts) + Conflict.objects.bulk_create(conflicts) dbpkg.provides.all().delete() provides = [create_related(Provision, dbpkg, y, equals_only=True) for y in repopkg.provides] - batched_bulk_create(Provision, provides) + Provision.objects.bulk_create(provides) dbpkg.replaces.all().delete() replaces = [create_related(Replacement, dbpkg, y) for y in repopkg.replaces] - batched_bulk_create(Replacement, replaces) + Replacement.objects.bulk_create(replaces) create_multivalued(dbpkg, repopkg, 'groups', 'groups') create_multivalued(dbpkg, repopkg, 'licenses', 'license') @@ -319,7 +297,7 @@ def populate_files(dbpkg, repopkg, force=False): directory=dirname, filename=filename) pkg_files.append(pkgfile) - batched_bulk_create(PackageFile, pkg_files) + PackageFile.objects.bulk_create(pkg_files) dbpkg.files_last_update = now() dbpkg.save() -- cgit v1.2.3-54-g00ecf From dd0ecfaeaceb1e1b8a185800de35f0f6e741feac Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 26 Feb 2013 19:51:40 -0600 Subject: Use user.userprofile rather than user.get_profile() The get_profile() function is deprecated as of Django 1.5. Signed-off-by: Dan McGee --- devel/views.py | 4 ++-- packages/views/flag.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index ff1dec12..61c1e568 100644 --- a/devel/views.py +++ b/devel/views.py @@ -152,7 +152,7 @@ def change_profile(request): if request.POST: form = ProfileForm(request.POST) profile_form = UserProfileForm(request.POST, request.FILES, - instance=request.user.get_profile()) + instance=request.user.userprofile) if form.is_valid() and profile_form.is_valid(): request.user.email = form.cleaned_data['email'] if form.cleaned_data['passwd1']: @@ -163,7 +163,7 @@ def change_profile(request): return HttpResponseRedirect('/devel/') else: form = ProfileForm(initial={'email': request.user.email}) - profile_form = UserProfileForm(instance=request.user.get_profile()) + profile_form = UserProfileForm(instance=request.user.userprofile) return render(request, 'devel/profile.html', {'form': form, 'profile_form': profile_form}) diff --git a/packages/views/flag.py b/packages/views/flag.py index edb3f092..5c76e1d5 100644 --- a/packages/views/flag.py +++ b/packages/views/flag.py @@ -110,7 +110,7 @@ def perform_updates(): subject = '%s package [%s] marked out-of-date' % \ (pkg.repo.name, pkg.pkgname) for maint in maints: - if maint.get_profile().notify == True: + if maint.userprofile.notify == True: toemail.append(maint.email) if toemail: -- cgit v1.2.3-54-g00ecf
    {% pkg_details_link pkg.pkg %}{{ pkg.pkg.arch.name }}{{ pkg.pkg.repo.name|capfirst }}{{ pkg.pkg.maintainers|join:', ' }} - {% if pkg.complete %} - Complete - {% else %} - Incomplete - {% endif %} - {% pkg_details_link pkg.pkg pkg.pkgname %}{{ pkg.arch.name }}{{ pkg.repo.name|capfirst }}{{ pkg.maintainers|join:', ' }}{{ pkg.get_status_display }}
    {{ pkg.pkg.arch.name }}{{ pkg.pkg.repo.name|capfirst }}{% pkg_details_link pkg.pkg %}
    {{ pkg.arch.name }}{{ pkg.repo.name|capfirst }}{% pkg_details_link pkg.pkg pkg.pkgname %}{{ pkg.pkg.full_version }}{{ pkg.pkg.full_version }}{{ pkg.pkg.maintainers|join:', ' }}{{ pkg.maintainers|join:', ' }} - {% if perms.main.change_todolistpkg %} - {% if pkg.complete %} + {% if perms.todolist.change_todolistpackage %} Complete + class="status-link {{ pkg.status_css_class }}" title="Toggle completion status">{{ pkg.get_status_display }} {% else %} - Incomplete - {% endif %} - {% else %} - {% if pkg.complete %}Complete{% else %}Incomplete{% endif %} + {{ pkg.get_status_display }} {% endif %}