From b6f86d9ab0d6910f0f70398b07e965d337bd9e78 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Tue, 16 Aug 2011 16:04:16 -0500
Subject: Add two new DB fields to reporead

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index 470b785d..97fdbb73 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -73,7 +73,7 @@ def handle(self, arch=None, filename=None, **options):
 class Pkg(object):
     """An interim 'container' object for holding Arch package data."""
     bare = ( 'name', 'base', 'arch', 'desc', 'filename',
-            'md5sum', 'url', 'packager' )
+            'md5sum', 'sha256sum', 'pgpsig', 'url', 'packager' )
     number = ( 'csize', 'isize' )
     collections = ( 'depends', 'optdepends', 'conflicts',
             'provides', 'replaces', 'groups', 'license', 'files' )
-- 
cgit v1.2.3-54-g00ecf


From e5d09fb7e9003b7f96685af9c0a722b45746448e Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Wed, 17 Aug 2011 16:18:12 -0500
Subject: Add PGP signature package field

And add eventual display code for it to the details template, but don't
show it yet as no packages will have it.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py              |   1 +
 .../0053_auto__add_field_package_pgp_signature.py  | 152 +++++++++++++++++++++
 main/models.py                                     |   6 +-
 templates/packages/details.html                    |   5 +
 4 files changed, 163 insertions(+), 1 deletion(-)
 create mode 100644 main/migrations/0053_auto__add_field_package_pgp_signature.py

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index 97fdbb73..cf597577 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -203,6 +203,7 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None):
     dbpkg.packager_str = repopkg.packager
     # attempt to find the corresponding django user for this string
     dbpkg.packager = finder.find(repopkg.packager)
+    dbpkg.pgp_signature = repopkg.pgpsig
 
     if timestamp:
         dbpkg.flag_date = None
diff --git a/main/migrations/0053_auto__add_field_package_pgp_signature.py b/main/migrations/0053_auto__add_field_package_pgp_signature.py
new file mode 100644
index 00000000..a828d1ef
--- /dev/null
+++ b/main/migrations/0053_auto__add_field_package_pgp_signature.py
@@ -0,0 +1,152 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        db.add_column('packages', 'pgp_signature', self.gf('django.db.models.fields.TextField')(null=True, blank=True), keep_default=False)
+
+    def backwards(self, orm):
+        db.delete_column('packages', 'pgp_signature')
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'main.arch': {
+            'Meta': {'ordering': "['name']", 'object_name': 'Arch', 'db_table': "'arches'"},
+            'agnostic': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+        },
+        'main.donor': {
+            'Meta': {'ordering': "['name']", 'object_name': 'Donor', 'db_table': "'donors'"},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+            'visible': ('django.db.models.fields.BooleanField', [], {'default': 'True'})
+        },
+        'main.package': {
+            'Meta': {'ordering': "('pkgname',)", 'object_name': 'Package', 'db_table': "'packages'"},
+            'arch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'to': "orm['main.Arch']"}),
+            'build_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+            'compressed_size': ('main.models.PositiveBigIntegerField', [], {}),
+            'epoch': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}),
+            'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'files_last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'flag_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'installed_size': ('main.models.PositiveBigIntegerField', [], {}),
+            'last_update': ('django.db.models.fields.DateTimeField', [], {}),
+            'packager': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True'}),
+            'packager_str': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'pgp_signature': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'pkgdesc': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}),
+            'pkgname': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'repo': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'to': "orm['main.Repo']"}),
+            'url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'})
+        },
+        'main.packagedepend': {
+            'Meta': {'object_name': 'PackageDepend', 'db_table': "'package_depends'"},
+            'depname': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'depvcmp': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}),
+            'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'optional': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"})
+        },
+        'main.packagefile': {
+            'Meta': {'object_name': 'PackageFile', 'db_table': "'package_files'"},
+            'directory': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'filename': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_directory': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"})
+        },
+        'main.repo': {
+            'Meta': {'ordering': "['name']", 'object_name': 'Repo', 'db_table': "'repos'"},
+            'bugs_category': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'bugs_project': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+            'staging': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'svn_root': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False'})
+        },
+        'main.todolist': {
+            'Meta': {'object_name': 'Todolist', 'db_table': "'todolists'"},
+            'creator': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}),
+            'date_added': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'})
+        },
+        'main.todolistpkg': {
+            'Meta': {'unique_together': "(('list', 'pkg'),)", 'object_name': 'TodolistPkg', 'db_table': "'todolist_pkgs'"},
+            'complete': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'list': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Todolist']"}),
+            'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"})
+        },
+        'main.userprofile': {
+            'Meta': {'object_name': 'UserProfile', 'db_table': "'user_profiles'"},
+            'alias': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'allowed_repos': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['main.Repo']", 'symmetrical': 'False', 'blank': 'True'}),
+            'favorite_distros': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'interests': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'languages': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'location': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'notify': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'occupation': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'other_contact': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'pgp_key': ('main.models.PGPKeyField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}),
+            'picture': ('django.db.models.fields.files.FileField', [], {'default': "'devs/silhouette.png'", 'max_length': '100'}),
+            'public_email': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'roles': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'time_zone': ('django.db.models.fields.CharField', [], {'default': "'UTC'", 'max_length': '100'}),
+            'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'userprofile'", 'unique': 'True', 'to': "orm['auth.User']"}),
+            'website': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'yob': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['main']
diff --git a/main/models.py b/main/models.py
index 0125cc0e..e0681abf 100644
--- a/main/models.py
+++ b/main/models.py
@@ -82,6 +82,7 @@ class UserProfile(models.Model):
         help_text="Ideally 125px by 125px")
     user = models.OneToOneField(User, related_name='userprofile')
     allowed_repos = models.ManyToManyField('Repo', blank=True)
+
     class Meta:
         db_table = 'user_profiles'
         verbose_name = 'Additional Profile Data'
@@ -173,6 +174,7 @@ class Package(models.Model):
     packager_str = models.CharField(max_length=255)
     packager = models.ForeignKey(User, null=True,
             on_delete=models.SET_NULL)
+    pgp_signature = models.TextField(null=True, blank=True)
     flag_date = models.DateTimeField(null=True)
 
     objects = PackageManager()
@@ -199,6 +201,9 @@ def get_full_url(self, proto='http'):
         domain = Site.objects.get_current().domain
         return '%s://%s%s' % (proto, domain, self.get_absolute_url())
 
+    def is_signed(self):
+        return bool(self.pgp_signature)
+
     @property
     def maintainers(self):
         return User.objects.filter(
@@ -288,7 +293,6 @@ def get_depends(self):
             if not pkg:
                 providers = dep.get_providers(arches,
                         testing=self.repo.testing, staging=self.repo.staging)
-                print providers
             deps.append({'dep': dep, 'pkg': pkg, 'providers': providers})
         return deps
 
diff --git a/templates/packages/details.html b/templates/packages/details.html
index bec4bdff..2f1031a6 100644
--- a/templates/packages/details.html
+++ b/templates/packages/details.html
@@ -138,6 +138,11 @@ <h4>Versions Elsewhere</h4>
         </tr><tr>
             <th>Last Packager:</th>
             <td>{% with pkg.packager as pkgr %}{% if pkgr %}{% userpkgs pkgr %}{% else %}{{ pkg.packager_str }}{% endif %}{% endwith %}</td>
+        </tr><tr>
+        {% comment %}
+            <th>Signed:</th>
+            <td>{{ pkg.is_signed|yesno|capfirst }}</td>
+        {% endcomment %}
         </tr><tr>
             <th>Build Date:</th>
             <td>{{ pkg.build_date|date:"DATETIME_FORMAT" }} UTC</td>
-- 
cgit v1.2.3-54-g00ecf


From 81884005d8496e12f9636e872eaedff33af77e30 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Wed, 5 Oct 2011 10:33:54 -0500
Subject: Allow developer index to work with a non-authenticated user

This is not the normal case given the decorator on the view, but during
testing and development it is sometimes useful so others don't have to
log in over a non-secure connection to check things out.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/views.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'devel')

diff --git a/devel/views.py b/devel/views.py
index 79eef318..ebae3b32 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -31,7 +31,12 @@
 @never_cache
 def index(request):
     '''the developer dashboard'''
-    inner_q = PackageRelation.objects.filter(user=request.user).values('pkgbase')
+    if(request.user.is_authenticated()):
+        inner_q = PackageRelation.objects.filter(user=request.user)
+    else:
+        inner_q = PackageRelation.objects.none()
+    inner_q = inner_q.values('pkgbase')
+
     flagged = Package.objects.normal().filter(
             flag_date__isnull=False, pkgbase__in=inner_q).order_by('pkgname')
 
-- 
cgit v1.2.3-54-g00ecf


From 2d5777b11d229d115a31a6c82236570002c2dd57 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Fri, 21 Oct 2011 18:49:00 -0500
Subject: Add a generate_keyring command

This grabs all the PGP keys from the developer profiles and adds them to
the keyrings. Obviously we may want to do more in the future such as
filter by groups, active status, etc. but this is just a first
iteration.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/generate_keyring.py | 59 +++++++++++++++++++++++++++
 devel/management/commands/rematch_packager.py |  2 +-
 main/models.py                                |  2 +-
 3 files changed, 61 insertions(+), 2 deletions(-)
 create mode 100644 devel/management/commands/generate_keyring.py

(limited to 'devel')

diff --git a/devel/management/commands/generate_keyring.py b/devel/management/commands/generate_keyring.py
new file mode 100644
index 00000000..b95d5a8e
--- /dev/null
+++ b/devel/management/commands/generate_keyring.py
@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+"""
+generate_keyring command
+
+Assemble a GPG keyring with all known developer keys.
+
+Usage: ./manage.py generate_keyring <keyserver> <keyring_path>
+"""
+
+from django.core.management.base import BaseCommand, CommandError
+from django.db.models import Q
+
+import logging
+import subprocess
+import sys
+
+from main.models import UserProfile
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s -> %(levelname)s: %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S',
+    stream=sys.stderr)
+logger = logging.getLogger()
+
+class Command(BaseCommand):
+    args = "<keyserver> <keyring_path>"
+    help = "Assemble a GPG keyring with all known developer keys."
+
+    def handle(self, *args, **options):
+        v = int(options.get('verbosity', None))
+        if v == 0:
+            logger.level = logging.ERROR
+        elif v == 1:
+            logger.level = logging.INFO
+        elif v == 2:
+            logger.level = logging.DEBUG
+
+        if len(args) != 2:
+            raise CommandError("keyserver and keyring_path must be provided")
+
+        return generate_keyring(args[0], args[1])
+
+def generate_keyring(keyserver, keyring):
+    logger.info("getting all known key IDs")
+
+    exclude = Q(pgp_key__isnull=True) & Q(pgp_key__exact="")
+    key_ids = UserProfile.objects.exclude(
+            exclude).values_list("pgp_key", flat=True)
+    logger.info("%d keys fetched from user profiles", len(key_ids))
+
+    gpg_cmd = ["gpg", "--no-default-keyring", "--keyring", keyring,
+            "--keyserver", keyserver, "--recv-keys"]
+    logger.info("running command: %r", gpg_cmd)
+    gpg_cmd.extend(key_ids)
+    subprocess.check_call(gpg_cmd)
+    logger.info("keyring at %s successfully updated", keyring)
+
+# vim: set ts=4 sw=4 et:
diff --git a/devel/management/commands/rematch_packager.py b/devel/management/commands/rematch_packager.py
index ba6e6a54..461d83ab 100644
--- a/devel/management/commands/rematch_packager.py
+++ b/devel/management/commands/rematch_packager.py
@@ -24,7 +24,7 @@
 logger = logging.getLogger()
 
 class Command(NoArgsCommand):
-    help = "Runs a check on all active mirror URLs to determine if they are reachable via IPv4 and/or v6."
+    help = "Match all packages with a packager_str but NULL packager_id to a packager if we can find one."
 
     def handle_noargs(self, **options):
         v = int(options.get('verbosity', None))
diff --git a/main/models.py b/main/models.py
index bdb84624..440201da 100644
--- a/main/models.py
+++ b/main/models.py
@@ -27,7 +27,7 @@ class PGPKeyField(models.CharField):
     _south_introspects = True
 
     def to_python(self, value):
-        if value == '':
+        if value == '' or value is None:
             return None
         value = super(PGPKeyField, self).to_python(value)
         # remove all spaces
-- 
cgit v1.2.3-54-g00ecf


From 1b83844b30d3271b8fb50757d827c7b8fe8b5585 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Wed, 26 Oct 2011 03:25:15 -0500
Subject: Ensure PGP signature values are not trimmed

This makes them totally unusable for any real purpose down the road.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index cf597577..a8e3219e 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -73,7 +73,7 @@ def handle(self, arch=None, filename=None, **options):
 class Pkg(object):
     """An interim 'container' object for holding Arch package data."""
     bare = ( 'name', 'base', 'arch', 'desc', 'filename',
-            'md5sum', 'sha256sum', 'pgpsig', 'url', 'packager' )
+            'md5sum', 'sha256sum', 'url', 'packager' )
     number = ( 'csize', 'isize' )
     collections = ( 'depends', 'optdepends', 'conflicts',
             'provides', 'replaces', 'groups', 'license', 'files' )
@@ -85,6 +85,7 @@ def __init__(self, repo):
         self.ver = None
         self.rel = None
         self.epoch = 0
+        self.pgpsig = None
         for k in self.bare + self.number:
             setattr(self, k, None)
         for k in self.collections:
@@ -99,6 +100,9 @@ def populate(self, values):
                 setattr(self, k, v[0][:254])
             elif k in self.number:
                 setattr(self, k, long(v[0]))
+            elif k == 'pgpsig':
+                # do NOT prune this value at all
+                setattr(self, k, v[0])
             elif k == 'version':
                 match = self.version_re.match(v[0])
                 self.ver = match.group(3)
-- 
cgit v1.2.3-54-g00ecf


From ade2c08899abf77f6d836432c9988ad3f9652a95 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Tue, 1 Nov 2011 16:47:37 -0500
Subject: Really ensure we don't catch any NULL or blank values

Fuck you too, Django.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/generate_keyring.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/generate_keyring.py b/devel/management/commands/generate_keyring.py
index b95d5a8e..35ab8874 100644
--- a/devel/management/commands/generate_keyring.py
+++ b/devel/management/commands/generate_keyring.py
@@ -8,7 +8,6 @@
 """
 
 from django.core.management.base import BaseCommand, CommandError
-from django.db.models import Q
 
 import logging
 import subprocess
@@ -44,9 +43,10 @@ def handle(self, *args, **options):
 def generate_keyring(keyserver, keyring):
     logger.info("getting all known key IDs")
 
-    exclude = Q(pgp_key__isnull=True) & Q(pgp_key__exact="")
-    key_ids = UserProfile.objects.exclude(
-            exclude).values_list("pgp_key", flat=True)
+    # Screw you Django, for not letting one natively do value != <empty string>
+    key_ids = UserProfile.objects.filter(user__is_active=True,
+            pgp_key__isnull=False).extra(where=["pgp_key != ''"]).values_list(
+            "pgp_key", flat=True)
     logger.info("%d keys fetched from user profiles", len(key_ids))
 
     gpg_cmd = ["gpg", "--no-default-keyring", "--keyring", keyring,
-- 
cgit v1.2.3-54-g00ecf


From d8e34919811728149a12e30d438318a3c1036a83 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Thu, 3 Nov 2011 15:50:06 -0500
Subject: Use UTC now everywhere

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/views.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'devel')

diff --git a/devel/views.py b/devel/views.py
index ebae3b32..694ed6dc 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -80,9 +80,9 @@ def clock(request):
     devs = User.objects.filter(is_active=True).order_by(
             'username').select_related('userprofile')
 
-    # now annotate each dev object with their current time
     now = datetime.now()
     utc_now = datetime.utcnow().replace(tzinfo=pytz.utc)
+    # now annotate each dev object with their current time
     for dev in devs:
         tz = pytz.timezone(dev.userprofile.time_zone)
         dev.current_time = utc_now.astimezone(tz)
@@ -147,12 +147,12 @@ def report(request, report, username=None):
 
     if report == 'old':
         title = 'Packages last built more than two years ago'
-        cutoff = datetime.now() - timedelta(days=365 * 2)
+        cutoff = datetime.utcnow() - timedelta(days=365 * 2)
         packages = packages.filter(
                 build_date__lt=cutoff).order_by('build_date')
     elif report == 'long-out-of-date':
         title = 'Packages marked out-of-date more than 90 days ago'
-        cutoff = datetime.now() - timedelta(days=90)
+        cutoff = datetime.utcnow() - timedelta(days=90)
         packages = packages.filter(
                 flag_date__lt=cutoff).order_by('flag_date')
     elif report == 'big':
-- 
cgit v1.2.3-54-g00ecf


From 022692b3f33de8c45741d3cb27fa95f9f6facdea Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Fri, 11 Nov 2011 10:43:18 -0600
Subject: Show relevant signoffs on dashboard

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/views.py             |  5 +++++
 packages/utils.py          |  7 ++++++-
 templates/devel/index.html | 50 ++++++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 59 insertions(+), 3 deletions(-)

(limited to 'devel')

diff --git a/devel/views.py b/devel/views.py
index 694ed6dc..0002df04 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -18,6 +18,7 @@
 from main.models import Arch, Repo
 from main.models import UserProfile
 from packages.models import PackageRelation
+from packages.utils import get_signoff_groups
 from todolists.utils import get_annotated_todolists
 from .utils import get_annotated_maintainers
 
@@ -48,6 +49,9 @@ def index(request):
     todolists = get_annotated_todolists()
     todolists = [todolist for todolist in todolists if todolist.incomplete_count > 0]
 
+    signoffs = sorted(get_signoff_groups(user=request.user),
+            key=operator.attrgetter('pkgbase'))
+
     maintainers = get_annotated_maintainers()
 
     maintained = PackageRelation.objects.filter(
@@ -70,6 +74,7 @@ def index(request):
             'orphan': orphan,
             'flagged' : flagged,
             'todopkgs' : todopkgs,
+            'signoffs': signoffs
     }
 
     return direct_to_template(request, 'devel/index.html', page_dict)
diff --git a/packages/utils.py b/packages/utils.py
index b21ac557..0df0e382 100644
--- a/packages/utils.py
+++ b/packages/utils.py
@@ -330,7 +330,7 @@ def get_target_repo_map(repos):
     cursor.execute(sql, params)
     return dict(cursor.fetchall())
 
-def get_signoff_groups(repos=None):
+def get_signoff_groups(repos=None, user=None):
     if repos is None:
         repos = Repo.objects.filter(testing=True)
     repo_ids = [r.pk for r in repos]
@@ -340,6 +340,11 @@ def get_signoff_groups(repos=None):
     packages = test_pkgs.order_by('pkgname')
     packages = attach_maintainers(packages)
 
+    # Filter by user if asked to do so
+    if user is not None:
+        packages = [p for p in packages if user == p.packager
+                or user in p.maintainers]
+
     # Collect all pkgbase values in testing repos
     pkgtorepo = get_target_repo_map(repos)
 
diff --git a/templates/devel/index.html b/templates/devel/index.html
index d3f7ec3b..06cf10ab 100644
--- a/templates/devel/index.html
+++ b/templates/devel/index.html
@@ -15,8 +15,8 @@ <h3>My Flagged Packages</h3>
         <thead>
             <tr>
                 <th>Name</th>
-                <th>Repo</th>
                 <th>Version</th>
+                <th>Repo</th>
                 <th>Arch</th>
                 <th>Flagged</th>
                 <th>Last Updated</th>
@@ -26,8 +26,8 @@ <h3>My Flagged Packages</h3>
             {% for pkg in flagged %}
                 <tr class="{% cycle 'odd' 'even' %}">
                     <td>{% pkg_details_link pkg %}</td>
-                    <td>{{ pkg.repo.name }}</td>
                     <td>{{ pkg.full_version }}</td>
+                    <td>{{ pkg.repo.name }}</td>
                     <td>{{ pkg.arch.name }}</td>
                     <td>{{ pkg.flag_date|date }}</td>
                     <td>{{ pkg.last_update|date }}</td>
@@ -96,6 +96,47 @@ <h3>Package Todo Lists</h3>
         </tbody>
     </table>
 
+    <h3>Signoff Status</h3>
+
+    <table id="dash-signoffs" class="results">
+        <thead>
+            <tr>
+                <th>Name</th>
+                <th>Version</th>
+                <th>Arch</th>
+                <th>Target Repo</th>
+                <th>Last Updated</th>
+                <th>Approved</th>
+                <th>Signoffs</th>
+            </tr>
+        </thead>
+        <tbody>
+            {% for group in signoffs %}
+            <tr class="{% cycle 'odd' 'even' %}">
+                <td>{% pkg_details_link group.package %}</td>
+                <td>{{ group.version }}</td>
+                <td>{{ group.arch.name }}</td>
+                <td>{{ group.target_repo }}</td>
+                <td>{{ group.last_update|date }}</td>
+                {% if group.specification.known_bad %}
+                <td class="approval signoff-bad">Bad</td>
+                {% else %}
+                {% if not group.specification.enabled %}
+                <td class="approval signoff-disabled">Disabled</td>
+                {% else %}
+                <td class="approval signoff-{{ group.approved|yesno }}">{{ group.approved|yesno|capfirst }}</td>
+                {% endif %}
+                {% endif %}
+                <td><ul class="signoff-list">
+                    {% for signoff in group.signoffs %}
+                    <li class="signed-username" title="Signed off by {{ signoff.user }}">{{ signoff.user }}{% if signoff.revoked %} (revoked){% endif %}</li>
+                    {% endfor %}
+                </ul></td>
+            </tr>
+            {% endfor %}
+        </tbody>
+    </table>
+
     <h3>Developer Reports</h3>
     <ul>
         <li><a href="reports/big/">Big</a>:
@@ -255,6 +296,11 @@ <h2>Stats by Developer</h2>
         {widgets: ['zebra'], sortList: [[0,0], [1,0]]});
     $("#dash-todo:not(:has(tbody tr.empty))").tablesorter(
             {widgets: ['zebra'], sortList: [[1,1]]});
+    $("#dash-signoffs:not(:has(tbody tr.empty))").tablesorter({
+            widgets: ['zebra'],
+            sortList: [[0,0]],
+            headers: { 6: {sorter: false } }
+    });
     $(".dash-stats").tablesorter({
         widgets: ['zebra'],
         sortList: [[0,0]],
-- 
cgit v1.2.3-54-g00ecf


From 0344f8ad564644c50203985255fab1d053aed463 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Tue, 15 Nov 2011 15:04:33 -0600
Subject: Add ability to cache users by username on the UserFinder

This is very useful in the signoff message population script where we
are very likely to encounter the same users over and over.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/utils.py                                    | 16 ++++++++++++++++
 packages/management/commands/populate_signoffs.py | 13 ++++++-------
 2 files changed, 22 insertions(+), 7 deletions(-)

(limited to 'devel')

diff --git a/devel/utils.py b/devel/utils.py
index d7a154a8..62b12cd5 100644
--- a/devel/utils.py
+++ b/devel/utils.py
@@ -47,6 +47,7 @@ def get_annotated_maintainers():
 class UserFinder(object):
     def __init__(self):
         self.cache = {}
+        self.username_cache = {}
 
     @staticmethod
     def user_email(name, email):
@@ -111,7 +112,22 @@ def find(self, userstring):
         self.cache[userstring] = user
         return user
 
+    def find_by_username(self, username):
+        if not username:
+            return None
+        if username in self.username_cache:
+            return self.username_cache[username]
+
+        try:
+            user = User.objects.get(username=username)
+        except User.DoesNotExist:
+            user = None
+
+        self.username_cache[username] = user
+        return user
+
     def clear_cache(self):
         self.cache = {}
+        self.username_cache = {}
 
 # vim: set ts=4 sw=4 et:
diff --git a/packages/management/commands/populate_signoffs.py b/packages/management/commands/populate_signoffs.py
index 5b5acbaf..ce5ec734 100644
--- a/packages/management/commands/populate_signoffs.py
+++ b/packages/management/commands/populate_signoffs.py
@@ -20,6 +20,7 @@
 
 from ...models import SignoffSpecification
 from ...utils import get_signoff_groups
+from devel.utils import UserFinder
 
 logging.basicConfig(
     level=logging.INFO,
@@ -58,17 +59,13 @@ def svn_log(pkgbase, repo):
         'message': xml.findtext('logentry/msg'),
     }
 
-def create_specification(package, log):
+def create_specification(package, log, finder):
     trimmed_message = log['message'].strip()
     spec = SignoffSpecification(pkgbase=package.pkgbase,
             pkgver=package.pkgver, pkgrel=package.pkgrel,
             epoch=package.epoch, arch=package.arch, repo=package.repo,
             comments=trimmed_message)
-    try:
-        spec.user = User.objects.get(username=log['author'])
-    except User.DoesNotExist:
-        pass
-
+    spec.user = finder.find_by_username(log['author'])
     return spec
 
 def add_signoff_comments():
@@ -76,6 +73,8 @@ def add_signoff_comments():
     groups = get_signoff_groups()
     logger.info("%d signoff groups found", len(groups))
 
+    finder = UserFinder()
+
     for group in groups:
         if not group.default_spec:
             continue
@@ -83,7 +82,7 @@ def add_signoff_comments():
         logger.debug("getting SVN log for %s (%s)", group.pkgbase, group.repo)
         log = svn_log(group.pkgbase, group.repo)
         logger.info("creating spec with SVN message for %s", group.pkgbase)
-        spec = create_specification(group.packages[0], log)
+        spec = create_specification(group.packages[0], log, finder)
         spec.save()
 
 # vim: set ts=4 sw=4 et:
-- 
cgit v1.2.3-54-g00ecf


From 9550236a87fc65827e994bea108350a43d3f161f Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Tue, 15 Nov 2011 21:26:57 -0600
Subject: Improve primary arch validation

Ensure we can accept either a Arch object or an architecture name when
passed to read_repo() by moving the validation there and being a bit
more careful about typechecking and object lookup.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index a8e3219e..b4966834 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -51,8 +51,6 @@ class Command(BaseCommand):
     def handle(self, arch=None, filename=None, **options):
         if not arch:
             raise CommandError('Architecture is required.')
-        if not validate_arch(arch):
-            raise CommandError('Specified architecture %s is not currently known.' % arch)
         if not filename:
             raise CommandError('Package database file is required.')
         filename = os.path.normpath(filename)
@@ -464,29 +462,38 @@ def parse_repo(repopath):
     logger.info("Finished repo parsing, %d total packages", len(pkgs))
     return (reponame, pkgs.values())
 
-def validate_arch(archname):
+def locate_arch(arch):
     "Check if arch is valid."
-    return Arch.objects.filter(name__iexact=archname).exists()
+    if isinstance(arch, Arch):
+        return arch
+    try:
+        return Arch.objects.get(name__iexact=arch)
+    except Arch.DoesNotExist:
+        raise CommandError(
+                'Specified architecture %s is not currently known.' % arch)
+
 
 def read_repo(primary_arch, repo_file, options):
     """
     Parses repo.db.tar.gz file and returns exit status.
     """
+    # always returns an Arch object, regardless of what is passed in
+    primary_arch = locate_arch(primary_arch)
+
     repo, packages = parse_repo(repo_file)
 
     # group packages by arch -- to handle noarch stuff
     packages_arches = {}
     for arch in Arch.objects.filter(agnostic=True):
         packages_arches[arch.name] = []
-    packages_arches[primary_arch] = []
+    packages_arches[primary_arch.name] = []
 
     for package in packages:
         if package.arch in packages_arches:
             packages_arches[package.arch].append(package)
         else:
             # we don't include mis-arched packages
-            logger.warning("Package %s arch = %s",
-                package.name, package.arch)
+            logger.warning("Package %s arch = %s", package.name, package.arch)
     del packages
 
     logger.info('Starting database updates.')
-- 
cgit v1.2.3-54-g00ecf


From 2a2df0074e39a797a0a4b5f7db7cfc9097301328 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Tue, 15 Nov 2011 21:28:42 -0600
Subject: Add new reporead_inotify management command

This is the new on-the-fly updates hotness. Rather than continue to
schedule reporead to run once an hour in cron or however else you ran
it, this command can be run once and left running, and will
automagically pick up on any database file changes and run an import.

It operates on the files databases only; this will keep both the
packages and files always in sync and remove the delay in updating,
especially helpful for new testing packages.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead_inotify.py | 188 ++++++++++++++++++++++++++
 1 file changed, 188 insertions(+)
 create mode 100755 devel/management/commands/reporead_inotify.py

(limited to 'devel')

diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py
new file mode 100755
index 00000000..135c0367
--- /dev/null
+++ b/devel/management/commands/reporead_inotify.py
@@ -0,0 +1,188 @@
+# -*- coding: utf-8 -*-
+"""
+reporead_inotify command
+
+Watches repo.files.tar.gz files for updates and parses them after a short delay
+in order to catch all updates in a single bulk update.
+
+Usage: ./manage.py reporead_inotify [path_template]
+
+Where 'path_template' is an optional path_template for finding the
+repo.files.tar.gz files. The form is '/srv/ftp/%(repo)s/os/%(arch)s/', which is
+also the default template if none is specified. While 'repo' is not required to
+be present in the path_template, note that 'arch' is so reporead can function
+correctly.
+"""
+
+import logging
+import os.path
+import pyinotify
+import sys
+import threading
+import time
+
+from django.core.management.base import BaseCommand, CommandError
+
+from main.models import Arch, Repo
+from .reporead import read_repo
+
+logging.basicConfig(
+    level=logging.WARNING,
+    format='%(asctime)s -> %(levelname)s: %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S',
+    stream=sys.stderr)
+logger = logging.getLogger()
+
+class Command(BaseCommand):
+    help = "Watch database files and run an update when necessary."
+    args = "[path_template]"
+
+    def handle(self, path_template=None, **options):
+        v = int(options.get('verbosity', 0))
+        if v == 0:
+            logger.level = logging.ERROR
+        elif v == 1:
+            logger.level = logging.INFO
+        elif v == 2:
+            logger.level = logging.DEBUG
+
+        if not path_template:
+            path_template = '/srv/ftp/%(repo)s/os/%(arch)s/'
+        self.path_template = path_template
+
+        notifier = self.setup_notifier()
+        logger.info('Entering notifier loop')
+        notifier.loop()
+
+    def setup_notifier(self):
+        '''Set up and configure the inotify machinery and logic.
+        This takes the provided or default path_template and builds a list of
+        directories we need to watch for database updates. It then validates
+        and passes these on to the various pyinotify pieces as necessary and
+        finally builds and returns a notifier object.'''
+        arches = Arch.objects.filter(agnostic=False)
+        repos = Repo.objects.all()
+        arch_path_map = dict((arch, None) for arch in arches)
+        all_paths = set()
+        total_paths = 0
+        for arch in arches:
+            combos = ({ 'repo': repo.name.lower(), 'arch': arch.name }
+                    for repo in repos)
+            # take a python format string and generate all unique combinations
+            # of directories from it; using set() ensures we filter it down
+            paths = set(self.path_template % values for values in combos)
+            total_paths += len(paths)
+            all_paths |= paths
+            arch_path_map[arch] = paths
+
+        logger.info('Watching %d total paths', total_paths)
+        logger.debug(all_paths)
+
+        # sanity check- basically ensure every path we created from the
+        # template mapped to only one architecture
+        if total_paths != len(all_paths):
+            raise CommandError('path template did not uniquely '
+                    'determine architecture for each file')
+
+        # A proper atomic replacement of the database as done by rsync is type
+        # IN_MOVED_TO. repo-add/remove will finish with a IN_CLOSE_WRITE.
+        mask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO
+
+        manager = pyinotify.WatchManager()
+        for name in all_paths:
+            manager.add_watch(name, mask)
+
+        handler = EventHandler(arch_paths=arch_path_map)
+        return pyinotify.Notifier(manager, handler)
+
+
+class Database(object):
+    '''A object representing a pacman database on the filesystem. It stores
+    various bits of metadata and state representing the file path, when we last
+    updated, how long our delay is before performing the update, whether we are
+    updating now, etc.'''
+    def __init__(self, arch, path, delay=60.0):
+        self.arch = arch
+        self.path = path
+        self.delay = delay
+        self.mtime = None
+        self.last_import = None
+        self.update_thread = None
+        self.updating = False
+        self.run_again = False
+        self.lock = threading.Lock()
+
+    def _start_update_countdown(self):
+        self.update_thread = threading.Timer(self.delay, self.update)
+        logger.info('Starting %.1f second countdown to update %s',
+                self.delay, self.path)
+        self.update_thread.start()
+
+    def queue_for_update(self, mtime):
+        logger.debug('Queueing database %s...', self.path)
+        with self.lock:
+            self.mtime = mtime
+            if self.updating:
+                # store the fact that we will need to run it again
+                self.run_again = True
+                return
+            if self.update_thread:
+                self.update_thread.cancel()
+            self._start_update_countdown()
+
+    def update(self):
+        logger.debug('Updating database %s...', self.path)
+        with self.lock:
+            self.last_import = time.time()
+            self.updating = True
+
+        try:
+            # invoke reporead's primary method
+            read_repo(self.arch, self.path, {})
+        finally:
+            logger.debug('Done updating database %s.', self.path)
+            with self.lock:
+                self.update_thread = None
+                self.updating = False
+                if self.run_again:
+                    self.run_again = False
+                    self._start_update_countdown()
+
+
+class EventHandler(pyinotify.ProcessEvent):
+    '''Our main event handler which listens for database change events. Because
+    we are watching the whole directory, we filter down and only look at those
+    events dealing with files databases.'''
+
+    def my_init(self, **kwargs):
+        self.databases = {}
+        self.arch_lookup = {}
+
+        # we really want a single path to arch mapping, so massage the data
+        arch_paths = kwargs['arch_paths']
+        for arch, paths in arch_paths.items():
+            self.arch_lookup.update((path.rstrip('/'), arch) for path in paths)
+
+    def process_default(self, event):
+        '''Primary event processing function which kicks off reporead timer
+        threads if a files database was updated.'''
+        if not event.name:
+            return
+        # screen to only the files we care about
+        if event.name.endswith('.files.tar.gz'):
+            path = event.pathname
+            stat = os.stat(path)
+            database = self.databases.get(path, None)
+            if database is None:
+                arch = self.arch_lookup.get(event.path, None)
+                if arch is None:
+                    logger.warning(
+                            'Could not determine arch for %s, skipping update',
+                            path)
+                    return
+                database = Database(arch, path)
+                self.databases[path] = database
+            database.queue_for_update(stat.st_mtime)
+
+
+# vim: set ts=4 sw=4 et:
-- 
cgit v1.2.3-54-g00ecf


From d3b36e1ce992a8b70f4fe8fe9e8df74e835fb865 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Tue, 15 Nov 2011 22:03:01 -0600
Subject: reporead_inotify: cancel threads that haven't started yet on shutdown

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead_inotify.py | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'devel')

diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py
index 135c0367..4c865ce1 100755
--- a/devel/management/commands/reporead_inotify.py
+++ b/devel/management/commands/reporead_inotify.py
@@ -54,6 +54,11 @@ def handle(self, path_template=None, **options):
         logger.info('Entering notifier loop')
         notifier.loop()
 
+        logger.info('Cancelling remaining threads...')
+        for thread in threading.enumerate():
+            if hasattr(thread, 'cancel'):
+                thread.cancel()
+
     def setup_notifier(self):
         '''Set up and configure the inotify machinery and logic.
         This takes the provided or default path_template and builds a list of
-- 
cgit v1.2.3-54-g00ecf


From c00e7e84045613ee2aa80f66b9972db971ab3f26 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Tue, 15 Nov 2011 23:43:42 -0600
Subject: reporead: clean up some debug logging

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index b4966834..45229524 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -36,6 +36,8 @@
     format='%(asctime)s -> %(levelname)s: %(message)s',
     datefmt='%Y-%m-%d %H:%M:%S',
     stream=sys.stderr)
+TRACE = 5
+logging.addLevelName(TRACE, 'TRACE')
 logger = logging.getLogger()
 
 class Command(BaseCommand):
@@ -368,7 +370,7 @@ def db_update(archname, reponame, pkgs, options):
     # packages in both database and in syncdb (update in database)
     pkg_in_both = syncset & dbset
     for p in [x for x in pkgs if x.name in pkg_in_both]:
-        logger.debug("Looking for package updates")
+        logger.debug("Checking package %s", p.name)
         dbp = dbdict[p.name]
         timestamp = None
         # for a force, we don't want to update the timestamp.
@@ -406,7 +408,7 @@ def parse_info(iofile):
             continue
         elif line.startswith('%') and line.endswith('%'):
             blockname = line[1:-1].lower()
-            logger.debug("Parsing package block %s", blockname)
+            logger.log(TRACE, "Parsing package block %s", blockname)
             store[blockname] = []
         elif blockname:
             store[blockname].append(line)
@@ -456,7 +458,7 @@ def parse_repo(repopath):
                         tarinfo.name)
             data_file.close()
 
-            logger.debug("Done parsing file %s", fname)
+            logger.debug("Done parsing file %s/%s", pkgid, fname)
 
     repodb.close()
     logger.info("Finished repo parsing, %d total packages", len(pkgs))
-- 
cgit v1.2.3-54-g00ecf


From 404c4b400b2bd2a14e0363e33d66505c51903fe7 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Wed, 16 Nov 2011 12:48:36 -0600
Subject: reporead: a few small tweaks

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index 45229524..ad76db4d 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -450,13 +450,14 @@ def parse_repo(repopath):
                 continue
             data_file = repodb.extractfile(tarinfo)
             data_file = io.TextIOWrapper(io.BytesIO(data_file.read()),
-                    encoding='utf=8')
+                    encoding='UTF-8')
             try:
                 pkgs[pkgid].populate(parse_info(data_file))
             except UnicodeDecodeError:
                 logger.warn("Could not correctly decode %s, skipping file",
                         tarinfo.name)
             data_file.close()
+            del data_file
 
             logger.debug("Done parsing file %s/%s", pkgid, fname)
 
@@ -498,10 +499,10 @@ def read_repo(primary_arch, repo_file, options):
             logger.warning("Package %s arch = %s", package.name, package.arch)
     del packages
 
-    logger.info('Starting database updates.')
+    logger.info('Starting database updates for %s.', repo_file)
     for arch in sorted(packages_arches.keys()):
         db_update(arch, repo, packages_arches[arch], options)
-    logger.info('Finished database updates.')
+    logger.info('Finished database updates for %s.', repo_file)
     return 0
 
 # vim: set ts=4 sw=4 et:
-- 
cgit v1.2.3-54-g00ecf


From b1e406d73844d5a30344ca8ac855fe850c52bc2f Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Wed, 16 Nov 2011 12:49:17 -0600
Subject: reporead_inotify: spin up read_repo() in separate thread

This prevents memory usage from ballooning to absolutely huge values,
such as when multiple threads kick off at the same time. The bulk of our
memory allocation obviously comes in these threads and not the main
threads, so being able to isolate them in processes helps a lot.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead_inotify.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py
index 4c865ce1..ffd49b8f 100755
--- a/devel/management/commands/reporead_inotify.py
+++ b/devel/management/commands/reporead_inotify.py
@@ -15,6 +15,7 @@
 """
 
 import logging
+import multiprocessing
 import os.path
 import pyinotify
 import sys
@@ -133,6 +134,7 @@ def queue_for_update(self, mtime):
                 return
             if self.update_thread:
                 self.update_thread.cancel()
+                self.update_thread = None
             self._start_update_countdown()
 
     def update(self):
@@ -142,8 +144,13 @@ def update(self):
             self.updating = True
 
         try:
-            # invoke reporead's primary method
-            read_repo(self.arch, self.path, {})
+            # invoke reporead's primary method. we do this in a separate
+            # process for memory conservation purposes; these processes grow
+            # rather large so it is best to free up the memory ASAP.
+            process = multiprocessing.Process(target=read_repo,
+                    args=[self.arch, self.path, {}])
+            process.start()
+            process.join()
         finally:
             logger.debug('Done updating database %s.', self.path)
             with self.lock:
-- 
cgit v1.2.3-54-g00ecf


From aa20c798ca8af365b2549591700e932a74d068b8 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Wed, 16 Nov 2011 13:02:35 -0600
Subject: reporead_inotify: close connection once we are done with it

This prevents an otherwise idle connection from sitting around and being
totally useless.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead_inotify.py | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'devel')

diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py
index ffd49b8f..acb53a54 100755
--- a/devel/management/commands/reporead_inotify.py
+++ b/devel/management/commands/reporead_inotify.py
@@ -23,6 +23,7 @@
 import time
 
 from django.core.management.base import BaseCommand, CommandError
+from django.db import connection
 
 from main.models import Arch, Repo
 from .reporead import read_repo
@@ -90,6 +91,11 @@ def setup_notifier(self):
             raise CommandError('path template did not uniquely '
                     'determine architecture for each file')
 
+        # this thread is done using the database; all future access is done in
+        # the spawned read_repo() processes, so close the otherwise completely
+        # idle connection.
+        connection.close()
+
         # A proper atomic replacement of the database as done by rsync is type
         # IN_MOVED_TO. repo-add/remove will finish with a IN_CLOSE_WRITE.
         mask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO
-- 
cgit v1.2.3-54-g00ecf


From 9d2fdbe5bc6a0d9ab2907b377056851fc5eb56c3 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Thu, 17 Nov 2011 10:17:55 -0600
Subject: reporead_inotify: nice the spawned subprocesses

This prevents the reporead job from taking over time from more important
processes; this is not a rush task.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead_inotify.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py
index acb53a54..c74762eb 100755
--- a/devel/management/commands/reporead_inotify.py
+++ b/devel/management/commands/reporead_inotify.py
@@ -16,7 +16,7 @@
 
 import logging
 import multiprocessing
-import os.path
+import os
 import pyinotify
 import sys
 import threading
@@ -113,10 +113,11 @@ class Database(object):
     various bits of metadata and state representing the file path, when we last
     updated, how long our delay is before performing the update, whether we are
     updating now, etc.'''
-    def __init__(self, arch, path, delay=60.0):
+    def __init__(self, arch, path, delay=60.0, nice=3):
         self.arch = arch
         self.path = path
         self.delay = delay
+        self.nice = nice
         self.mtime = None
         self.last_import = None
         self.update_thread = None
@@ -153,8 +154,12 @@ def update(self):
             # invoke reporead's primary method. we do this in a separate
             # process for memory conservation purposes; these processes grow
             # rather large so it is best to free up the memory ASAP.
-            process = multiprocessing.Process(target=read_repo,
-                    args=[self.arch, self.path, {}])
+            def run():
+                if self.nice != 0:
+                    os.nice(self.nice)
+                read_repo(self.arch, self.path, {})
+
+            process = multiprocessing.Process(target=run)
             process.start()
             process.join()
         finally:
-- 
cgit v1.2.3-54-g00ecf


From a9819e3d715ce3e5c20c9665db9a6100f06ab562 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Thu, 17 Nov 2011 12:34:12 -0600
Subject: Ensure reporead is protected against simultaneous runs

This adds a bunch of transaction magic and SELECT FOR UPDATE stuff to
reporead to cope with the now-concurrent runs of reporead we get when
invoked from our inotify-based updater. The collision occurs with 'any'
architecture packages as both repo databases contain the new version,
and the updates occur at exactly the same time.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py          | 206 +++++++++++++------------
 main/migrations/0055_unique_package_in_repo.py | 155 +++++++++++++++++++
 main/models.py                                 |   3 +-
 3 files changed, 263 insertions(+), 101 deletions(-)
 create mode 100644 main/migrations/0055_unique_package_in_repo.py

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index ad76db4d..b6bd8457 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -13,10 +13,6 @@
   ./manage.py reporead i686 /tmp/core.db.tar.gz
 """
 
-from django.core.management.base import BaseCommand, CommandError
-from django.contrib.auth.models import User
-from django.db import transaction
-
 from collections import defaultdict
 import io
 import os
@@ -27,6 +23,11 @@
 from datetime import datetime
 from optparse import make_option
 
+from django.core.management.base import BaseCommand, CommandError
+from django.contrib.auth.models import User
+from django.db import connections, router, transaction
+from django.db.utils import IntegrityError
+
 from devel.utils import UserFinder
 from main.models import Arch, Package, PackageDepend, PackageFile, Repo
 from packages.models import Conflict, Provision, Replacement
@@ -189,8 +190,6 @@ def create_multivalued(dbpkg, repopkg, db_attr, repo_attr):
 finder = UserFinder()
 
 def populate_pkg(dbpkg, repopkg, force=False, timestamp=None):
-    db_score = 1
-
     if repopkg.base:
         dbpkg.pkgbase = repopkg.base
     else:
@@ -214,7 +213,7 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None):
         dbpkg.last_update = timestamp
     dbpkg.save()
 
-    db_score += populate_files(dbpkg, repopkg, force=force)
+    populate_files(dbpkg, repopkg, force=force)
 
     dbpkg.packagedepend_set.all().delete()
     for y in repopkg.depends:
@@ -235,28 +234,23 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None):
     create_multivalued(dbpkg, repopkg, 'groups', 'groups')
     create_multivalued(dbpkg, repopkg, 'licenses', 'license')
 
-    related_score = (len(repopkg.depends) + len(repopkg.optdepends)
-            + len(repopkg.conflicts) + len(repopkg.provides)
-            + len(repopkg.replaces) + len(repopkg.groups)
-            + len(repopkg.license))
-    if related_score:
-        db_score += (related_score / 20) + 1
 
-    return db_score
+pkg_same_version = lambda pkg, dbpkg: pkg.ver == dbpkg.pkgver \
+        and pkg.rel == dbpkg.pkgrel and pkg.epoch == dbpkg.epoch
 
 
 def populate_files(dbpkg, repopkg, force=False):
     if not force:
-        if dbpkg.pkgver != repopkg.ver or dbpkg.pkgrel != repopkg.rel \
-                or dbpkg.epoch != repopkg.epoch:
+        if not pkg_same_version(repopkg, dbpkg):
             logger.info("DB version (%s) didn't match repo version "
                     "(%s) for package %s, skipping file list addition",
                     dbpkg.full_version, repopkg.full_version, dbpkg.pkgname)
-            return 0
+            return
         if not dbpkg.files_last_update or not dbpkg.last_update:
             pass
         elif dbpkg.files_last_update > dbpkg.last_update:
-            return 0
+            return
+
     # only delete files if we are reading a DB that contains them
     if repopkg.has_files:
         dbpkg.packagefile_set.all().delete()
@@ -275,30 +269,19 @@ def populate_files(dbpkg, repopkg, force=False):
             pkgfile.save(force_insert=True)
         dbpkg.files_last_update = datetime.utcnow()
         dbpkg.save()
-        return (len(repopkg.files) / 50) + 1
-    return 0
-
 
-class Batcher(object):
-    def __init__(self, threshold, start=0):
-        self.threshold = threshold
-        self.meter = start
 
-    def batch_commit(self, score):
-        """
-        Track updates to the database and perform a commit if the batch
-        becomes sufficiently large. "Large" is defined by waiting for the
-        sum of scores to exceed the arbitrary threshold value; once it is
-        hit a commit is issued.
-        """
-        self.meter += score
-        if self.meter > self.threshold:
-            logger.debug("Committing transaction, batch threshold hit")
-            transaction.commit()
-            self.meter = 0
+def select_pkg_for_update(dbpkg):
+    database = router.db_for_write(Package, instance=dbpkg)
+    connection = connections[database]
+    if 'sqlite' in connection.settings_dict['ENGINE'].lower():
+        return dbpkg
+    new_pkg = Package.objects.raw(
+            'SELECT * FROM packages WHERE id = %s FOR UPDATE',
+            [dbpkg.id])
+    return list(new_pkg)[0]
 
 
-@transaction.commit_on_success
 def db_update(archname, reponame, pkgs, options):
     """
     Parses a list and updates the Arch dev database accordingly.
@@ -310,88 +293,111 @@ def db_update(archname, reponame, pkgs, options):
     logger.info('Updating Arch: %s', archname)
     force = options.get('force', False)
     filesonly = options.get('filesonly', False)
-    repository = Repo.objects.get(name__iexact=reponame)
-    architecture = Arch.objects.get(name__iexact=archname)
-    # no-arg order_by() removes even the default ordering; we don't need it
-    dbpkgs = Package.objects.filter(
-            arch=architecture, repo=repository).order_by()
-    # This makes our inner loop where we find packages by name *way* more
-    # efficient by not having to go to the database for each package to
-    # SELECT them by name.
-    dbdict = dict([(pkg.pkgname, pkg) for pkg in dbpkgs])
-
-    logger.debug("Creating sets")
-    dbset = set(dbdict.keys())
-    syncset = set([pkg.name for pkg in pkgs])
-    logger.info("%d packages in current web DB", len(dbset))
-    logger.info("%d packages in new updating db", len(syncset))
-    in_sync_not_db = syncset - dbset
-    logger.info("%d packages in sync not db", len(in_sync_not_db))
-
-    # Try to catch those random package deletions that make Eric so unhappy.
-    if len(dbset):
-        dbpercent = 100.0 * len(syncset) / len(dbset)
-    else:
-        dbpercent = 0.0
-    logger.info("DB package ratio: %.1f%%", dbpercent)
-
-    # Fewer than 20 packages makes the percentage check unreliable, but it also
-    # means we expect the repo to fluctuate a lot.
-    msg = "Package database has %.1f%% the number of packages in the " \
-            "web database" % dbpercent
-    if len(dbset) == 0 and len(syncset) == 0:
-        pass
-    elif not filesonly and \
-            len(dbset) > 20 and dbpercent < 50.0 and \
-            not repository.testing and not repository.staging:
-        logger.error(msg)
-        raise Exception(msg)
-    elif dbpercent < 75.0:
-        logger.warning(msg)
-
-    batcher = Batcher(100)
+
+    with transaction.commit_manually():
+        repository = Repo.objects.get(name__iexact=reponame)
+        architecture = Arch.objects.get(name__iexact=archname)
+        # no-arg order_by() removes even the default ordering; we don't need it
+        dbpkgs = Package.objects.filter(
+                arch=architecture, repo=repository).order_by()
+        # This makes our inner loop where we find packages by name *way* more
+        # efficient by not having to go to the database for each package to
+        # SELECT them by name.
+        dbdict = dict((dbpkg.pkgname, dbpkg) for dbpkg in dbpkgs)
+
+        logger.debug("Creating sets")
+        dbset = set(dbdict.keys())
+        syncset = set([pkg.name for pkg in pkgs])
+        logger.info("%d packages in current web DB", len(dbset))
+        logger.info("%d packages in new updating db", len(syncset))
+        in_sync_not_db = syncset - dbset
+        logger.info("%d packages in sync not db", len(in_sync_not_db))
+
+        # Try to catch those random package deletions that make Eric so unhappy.
+        if len(dbset):
+            dbpercent = 100.0 * len(syncset) / len(dbset)
+        else:
+            dbpercent = 0.0
+        logger.info("DB package ratio: %.1f%%", dbpercent)
+
+        # Fewer than 20 packages makes the percentage check unreliable, but it also
+        # means we expect the repo to fluctuate a lot.
+        msg = "Package database has %.1f%% the number of packages in the " \
+                "web database" % dbpercent
+        if len(dbset) == 0 and len(syncset) == 0:
+            pass
+        elif not filesonly and \
+                len(dbset) > 20 and dbpercent < 50.0 and \
+                not repository.testing and not repository.staging:
+            logger.error(msg)
+            raise Exception(msg)
+        elif dbpercent < 75.0:
+            logger.warning(msg)
+
+        # If isolation level is repeatable-read, we need to ensure each package
+        # update starts a new transaction and re-queries the database as necessary
+        # to guard against simultaneous updates
+        transaction.commit()
 
     if not filesonly:
         # packages in syncdb and not in database (add to database)
-        for p in [x for x in pkgs if x.name in in_sync_not_db]:
-            logger.info("Adding package %s", p.name)
-            pkg = Package(pkgname=p.name, arch=architecture, repo=repository)
-            score = populate_pkg(pkg, p, timestamp=datetime.utcnow())
-            batcher.batch_commit(score)
+        for pkg in (pkg for pkg in pkgs if pkg.name in in_sync_not_db):
+            logger.info("Adding package %s", pkg.name)
+            dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository)
+            try:
+                with transaction.commit_on_success():
+                    populate_pkg(dbpkg, pkg, timestamp=datetime.utcnow())
+            except IntegrityError:
+                logger.warning("Could not add package %s; "
+                        "not fatal if another thread beat us to it.",
+                        pkg.name, exc_info=True)
 
         # packages in database and not in syncdb (remove from database)
-        in_db_not_sync = dbset - syncset
-        for p in in_db_not_sync:
-            logger.info("Removing package %s", p)
-            dbp = dbdict[p]
-            dbp.delete()
-            batcher.batch_commit(1)
+        for pkgname in (dbset - syncset):
+            logger.info("Removing package %s", pkgname)
+            dbpkg = dbdict[pkgname]
+            with transaction.commit_on_success():
+                # no race condition here as long as simultaneous threads both
+                # issue deletes; second delete will be a no-op
+                dbpkg.delete()
 
     # packages in both database and in syncdb (update in database)
     pkg_in_both = syncset & dbset
-    for p in [x for x in pkgs if x.name in pkg_in_both]:
-        logger.debug("Checking package %s", p.name)
-        dbp = dbdict[p.name]
+    for pkg in (x for x in pkgs if x.name in pkg_in_both):
+        logger.debug("Checking package %s", pkg.name)
+        dbpkg = dbdict[pkg.name]
         timestamp = None
         # for a force, we don't want to update the timestamp.
         # for a non-force, we don't want to do anything at all.
         if filesonly:
             pass
-        elif p.ver == dbp.pkgver and p.rel == dbp.pkgrel \
-                and p.epoch == dbp.epoch:
+        elif pkg_same_version(pkg, dbpkg):
             if not force:
                 continue
         else:
             timestamp = datetime.utcnow()
 
+        # The odd select_for_update song and dance here are to ensure
+        # simultaneous updates don't happen on a package, causing
+        # files/depends/all related items to be double-imported.
         if filesonly:
-            logger.debug("Checking files for package %s", p.name)
-            score = populate_files(dbp, p, force=force)
+            with transaction.commit_on_success():
+                # TODO Django 1.4 select_for_update() will work once released
+                dbpkg = select_pkg_for_update(dbpkg)
+                if pkg_same_version(pkg, dbpkg):
+                    logger.debug("Package %s was already updated", pkg.name)
+                    continue
+                logger.debug("Checking files for package %s", pkg.name)
+                populate_files(dbpkg, pkg, force=force)
         else:
-            logger.info("Updating package %s", p.name)
-            score = populate_pkg(dbp, p, force=force, timestamp=timestamp)
-
-        batcher.batch_commit(score)
+            with transaction.commit_on_success():
+                # TODO Django 1.4 select_for_update() will work once released
+                dbpkg = select_pkg_for_update(dbpkg)
+                if pkg_same_version(pkg, dbpkg):
+                    logger.debug("Package %s was already updated", pkg.name)
+                    continue
+                logger.info("Updating package %s", pkg.name)
+                populate_pkg(dbpkg, pkg, force=force, timestamp=timestamp)
 
     logger.info('Finished updating Arch: %s', archname)
 
diff --git a/main/migrations/0055_unique_package_in_repo.py b/main/migrations/0055_unique_package_in_repo.py
new file mode 100644
index 00000000..63951a08
--- /dev/null
+++ b/main/migrations/0055_unique_package_in_repo.py
@@ -0,0 +1,155 @@
+# encoding: utf-8
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        db.delete_index('packages', ['pkgname'])
+        db.create_unique('packages', ['pkgname', 'repo_id', 'arch_id'])
+
+    def backwards(self, orm):
+        db.delete_unique('packages', ['pkgname', 'repo_id', 'arch_id'])
+        db.create_index('packages', ['pkgname'])
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'main.arch': {
+            'Meta': {'ordering': "['name']", 'object_name': 'Arch', 'db_table': "'arches'"},
+            'agnostic': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+        },
+        'main.donor': {
+            'Meta': {'ordering': "('name',)", 'object_name': 'Donor', 'db_table': "'donors'"},
+            'created': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+            'visible': ('django.db.models.fields.BooleanField', [], {'default': 'True'})
+        },
+        'main.package': {
+            'Meta': {'ordering': "('pkgname',)", 'unique_together': "(('pkgname', 'repo', 'arch'),)", 'object_name': 'Package', 'db_table': "'packages'"},
+            'arch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'to': "orm['main.Arch']"}),
+            'build_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+            'compressed_size': ('main.models.PositiveBigIntegerField', [], {}),
+            'epoch': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}),
+            'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'files_last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'flag_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'installed_size': ('main.models.PositiveBigIntegerField', [], {}),
+            'last_update': ('django.db.models.fields.DateTimeField', [], {}),
+            'packager': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True'}),
+            'packager_str': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'pgp_signature': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'pkgdesc': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}),
+            'pkgname': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'repo': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'to': "orm['main.Repo']"}),
+            'url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'})
+        },
+        'main.packagedepend': {
+            'Meta': {'object_name': 'PackageDepend', 'db_table': "'package_depends'"},
+            'depname': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'depvcmp': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}),
+            'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'optional': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"})
+        },
+        'main.packagefile': {
+            'Meta': {'object_name': 'PackageFile', 'db_table': "'package_files'"},
+            'directory': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'filename': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_directory': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"})
+        },
+        'main.repo': {
+            'Meta': {'ordering': "['name']", 'object_name': 'Repo', 'db_table': "'repos'"},
+            'bugs_category': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'bugs_project': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+            'staging': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'svn_root': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False'})
+        },
+        'main.todolist': {
+            'Meta': {'object_name': 'Todolist', 'db_table': "'todolists'"},
+            'creator': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}),
+            'date_added': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'})
+        },
+        'main.todolistpkg': {
+            'Meta': {'unique_together': "(('list', 'pkg'),)", 'object_name': 'TodolistPkg', 'db_table': "'todolist_pkgs'"},
+            'complete': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'list': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Todolist']"}),
+            'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"})
+        },
+        'main.userprofile': {
+            'Meta': {'object_name': 'UserProfile', 'db_table': "'user_profiles'"},
+            'alias': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'allowed_repos': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['main.Repo']", 'symmetrical': 'False', 'blank': 'True'}),
+            'favorite_distros': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'interests': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'languages': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'location': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'notify': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'occupation': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'other_contact': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'pgp_key': ('main.models.PGPKeyField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}),
+            'picture': ('django.db.models.fields.files.FileField', [], {'default': "'devs/silhouette.png'", 'max_length': '100'}),
+            'public_email': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'roles': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'time_zone': ('django.db.models.fields.CharField', [], {'default': "'UTC'", 'max_length': '100'}),
+            'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'userprofile'", 'unique': 'True', 'to': "orm['auth.User']"}),
+            'website': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'yob': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['main']
diff --git a/main/models.py b/main/models.py
index caf36be0..cad4f9fc 100644
--- a/main/models.py
+++ b/main/models.py
@@ -159,7 +159,7 @@ class Package(models.Model):
             on_delete=models.PROTECT)
     arch = models.ForeignKey(Arch, related_name="packages",
             on_delete=models.PROTECT)
-    pkgname = models.CharField(max_length=255, db_index=True)
+    pkgname = models.CharField(max_length=255)
     pkgbase = models.CharField(max_length=255, db_index=True)
     pkgver = models.CharField(max_length=255)
     pkgrel = models.CharField(max_length=255)
@@ -184,6 +184,7 @@ class Meta:
         db_table = 'packages'
         ordering = ('pkgname',)
         get_latest_by = 'last_update'
+        unique_together = (('pkgname', 'repo', 'arch'),)
 
     def __unicode__(self):
         return self.pkgname
-- 
cgit v1.2.3-54-g00ecf


From 2cb4f97bb235217d6e56deded1444f5e84f08b71 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Thu, 17 Nov 2011 13:36:27 -0600
Subject: reporead: don't trim pkgdesc length

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index b6bd8457..cf101d97 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -73,7 +73,7 @@ def handle(self, arch=None, filename=None, **options):
 
 class Pkg(object):
     """An interim 'container' object for holding Arch package data."""
-    bare = ( 'name', 'base', 'arch', 'desc', 'filename',
+    bare = ( 'name', 'base', 'arch', 'filename',
             'md5sum', 'sha256sum', 'url', 'packager' )
     number = ( 'csize', 'isize' )
     collections = ( 'depends', 'optdepends', 'conflicts',
@@ -101,8 +101,8 @@ def populate(self, values):
                 setattr(self, k, v[0][:254])
             elif k in self.number:
                 setattr(self, k, long(v[0]))
-            elif k == 'pgpsig':
-                # do NOT prune this value at all
+            elif k in ('desc', 'pgpsig'):
+                # do NOT prune these values at all
                 setattr(self, k, v[0])
             elif k == 'version':
                 match = self.version_re.match(v[0])
-- 
cgit v1.2.3-54-g00ecf


From 85657db05d7f65604340699cfcb9967c9e81a0ef Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Mon, 21 Nov 2011 10:08:23 -0600
Subject: Better support for non-latin full names

Add a 'latin_name' field to the user profile so we can better support
those developers with names in non-Latin scripts, and yet still show a
Latin name as necessary on the developer profile page. This field only
shows up if populated.

Also, use consistent sorting everywhere- rather than using username,
always use first_name and last_name fields.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/views.py                                     |   2 +-
 .../0057_auto__add_field_userprofile_latin_name.py | 153 +++++++++++++++++++++
 main/models.py                                     |   2 +
 packages/views/search.py                           |   3 +-
 public/views.py                                    |   6 +-
 templates/devel/clock.html                         |   2 +-
 templates/public/developer_list.html               |   2 +-
 7 files changed, 164 insertions(+), 6 deletions(-)
 create mode 100644 main/migrations/0057_auto__add_field_userprofile_latin_name.py

(limited to 'devel')

diff --git a/devel/views.py b/devel/views.py
index 0002df04..08b19cd7 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -83,7 +83,7 @@ def index(request):
 @never_cache
 def clock(request):
     devs = User.objects.filter(is_active=True).order_by(
-            'username').select_related('userprofile')
+            'first_name', 'last_name').select_related('userprofile')
 
     now = datetime.now()
     utc_now = datetime.utcnow().replace(tzinfo=pytz.utc)
diff --git a/main/migrations/0057_auto__add_field_userprofile_latin_name.py b/main/migrations/0057_auto__add_field_userprofile_latin_name.py
new file mode 100644
index 00000000..ffde1885
--- /dev/null
+++ b/main/migrations/0057_auto__add_field_userprofile_latin_name.py
@@ -0,0 +1,153 @@
+# encoding: utf-8
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        db.add_column('user_profiles', 'latin_name', self.gf('django.db.models.fields.CharField')(max_length=255, null=True, blank=True), keep_default=False)
+
+    def backwards(self, orm):
+        db.delete_column('user_profiles', 'latin_name')
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'main.arch': {
+            'Meta': {'ordering': "['name']", 'object_name': 'Arch', 'db_table': "'arches'"},
+            'agnostic': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+        },
+        'main.donor': {
+            'Meta': {'ordering': "('name',)", 'object_name': 'Donor', 'db_table': "'donors'"},
+            'created': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+            'visible': ('django.db.models.fields.BooleanField', [], {'default': 'True'})
+        },
+        'main.package': {
+            'Meta': {'ordering': "('pkgname',)", 'unique_together': "(('pkgname', 'repo', 'arch'),)", 'object_name': 'Package', 'db_table': "'packages'"},
+            'arch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'to': "orm['main.Arch']"}),
+            'build_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+            'compressed_size': ('main.models.PositiveBigIntegerField', [], {}),
+            'epoch': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}),
+            'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'files_last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'flag_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'installed_size': ('main.models.PositiveBigIntegerField', [], {}),
+            'last_update': ('django.db.models.fields.DateTimeField', [], {}),
+            'packager': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True'}),
+            'packager_str': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'pgp_signature': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'pkgdesc': ('django.db.models.fields.TextField', [], {'null': 'True'}),
+            'pkgname': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'repo': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'to': "orm['main.Repo']"}),
+            'url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'})
+        },
+        'main.packagedepend': {
+            'Meta': {'object_name': 'PackageDepend', 'db_table': "'package_depends'"},
+            'depname': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'depvcmp': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}),
+            'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'optional': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"})
+        },
+        'main.packagefile': {
+            'Meta': {'object_name': 'PackageFile', 'db_table': "'package_files'"},
+            'directory': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'filename': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_directory': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"})
+        },
+        'main.repo': {
+            'Meta': {'ordering': "['name']", 'object_name': 'Repo', 'db_table': "'repos'"},
+            'bugs_category': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'bugs_project': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+            'staging': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'svn_root': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False'})
+        },
+        'main.todolist': {
+            'Meta': {'object_name': 'Todolist', 'db_table': "'todolists'"},
+            'creator': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}),
+            'date_added': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'})
+        },
+        'main.todolistpkg': {
+            'Meta': {'unique_together': "(('list', 'pkg'),)", 'object_name': 'TodolistPkg', 'db_table': "'todolist_pkgs'"},
+            'complete': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'list': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Todolist']"}),
+            'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"})
+        },
+        'main.userprofile': {
+            'Meta': {'object_name': 'UserProfile', 'db_table': "'user_profiles'"},
+            'alias': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'allowed_repos': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['main.Repo']", 'symmetrical': 'False', 'blank': 'True'}),
+            'favorite_distros': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'interests': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'languages': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'latin_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'location': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'notify': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'occupation': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'other_contact': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'pgp_key': ('main.models.PGPKeyField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}),
+            'picture': ('django.db.models.fields.files.FileField', [], {'default': "'devs/silhouette.png'", 'max_length': '100'}),
+            'public_email': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'roles': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'time_zone': ('django.db.models.fields.CharField', [], {'default': "'UTC'", 'max_length': '100'}),
+            'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'userprofile'", 'unique': 'True', 'to': "orm['auth.User']"}),
+            'website': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'yob': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['main']
diff --git a/main/models.py b/main/models.py
index b37468f9..d7780b91 100644
--- a/main/models.py
+++ b/main/models.py
@@ -77,6 +77,8 @@ class UserProfile(models.Model):
         help_text="Ideally 125px by 125px")
     user = models.OneToOneField(User, related_name='userprofile')
     allowed_repos = models.ManyToManyField('Repo', blank=True)
+    latin_name = models.CharField(max_length=255, null=True, blank=True,
+        help_text="Latin-form name; used only for non-Latin full names")
 
     class Meta:
         db_table = 'user_profiles'
diff --git a/packages/views/search.py b/packages/views/search.py
index 57481614..65fcddb3 100644
--- a/packages/views/search.py
+++ b/packages/views/search.py
@@ -60,7 +60,8 @@ def __init__(self, *args, **kwargs):
         self.fields['arch'].choices = make_choice(
                         [arch.name for arch in Arch.objects.all()])
         self.fields['q'].widget.attrs.update({"size": "30"})
-        maints = User.objects.filter(is_active=True).order_by('username')
+        maints = User.objects.filter(is_active=True).order_by(
+                'first_name', 'last_name')
         self.fields['maintainer'].choices = \
                 [('', 'All'), ('orphan', 'Orphan')] + \
                 [(m.username, m.get_full_name()) for m in maints]
diff --git a/public/views.py b/public/views.py
index 14dd6353..c28fd303 100644
--- a/public/views.py
+++ b/public/views.py
@@ -34,13 +34,15 @@ def index(request):
 }
 
 def userlist(request, user_type='devs'):
-    users = User.objects.order_by('username').select_related('userprofile')
+    users = User.objects.order_by(
+            'first_name', 'last_name').select_related('userprofile')
     if user_type == 'devs':
         users = users.filter(is_active=True, groups__name="Developers")
     elif user_type == 'tus':
         users = users.filter(is_active=True, groups__name="Trusted Users")
     elif user_type == 'fellows':
-        users = users.filter(is_active=False, groups__name__in=["Developers", "Trusted Users"])
+        users = users.filter(is_active=False,
+                groups__name__in=["Developers", "Trusted Users"])
     else:
         raise Http404
 
diff --git a/templates/devel/clock.html b/templates/devel/clock.html
index 0f0e20c5..d2eb0a8d 100644
--- a/templates/devel/clock.html
+++ b/templates/devel/clock.html
@@ -45,7 +45,7 @@ <h2>Developer World Clocks</h2>
 <script type="text/javascript">
 $(document).ready(function() {
     $("#clocks-table:has(tbody tr)").tablesorter(
-        {widgets: ['zebra'], sortList: [[1,0]]});
+        {widgets: ['zebra'], sortList: [[0,0]]});
 });
 </script>
 {% endblock %}
diff --git a/templates/public/developer_list.html b/templates/public/developer_list.html
index 0ac444e5..5aa4c6b2 100644
--- a/templates/public/developer_list.html
+++ b/templates/public/developer_list.html
@@ -21,7 +21,7 @@
             <table class="bio bio-{{ dev.username }}" cellspacing="0">
                 <tr>
                     <th>Name:</th>
-                    <td>{{ dev.get_full_name }}</td>
+                    <td>{{ dev.get_full_name }}{% if prof.latin_name %} ({{ prof.latin_name}}){% endif %}</td>
                 </tr><tr>
                     <th>Alias:</th>
                     <td>{{ prof.alias }}</td>
-- 
cgit v1.2.3-54-g00ecf