diff options
Diffstat (limited to 'devel')
| -rw-r--r-- | devel/admin.py | 12 | ||||
| -rw-r--r-- | devel/forms.py | 100 | ||||
| -rw-r--r-- | devel/management/commands/generate_keyring.py | 6 | ||||
| -rw-r--r-- | devel/management/commands/import_signatures.py | 123 | ||||
| -rw-r--r-- | devel/management/commands/pgp_import.py | 242 | ||||
| -rw-r--r-- | devel/management/commands/rematch_developers.py | 63 | ||||
| -rw-r--r-- | devel/management/commands/reporead.py | 112 | ||||
| -rw-r--r--[-rwxr-xr-x] | devel/management/commands/reporead_inotify.py | 19 | ||||
| -rw-r--r-- | devel/migrations/0008_auto__add_field_userprofile_last_modified.py | 110 | ||||
| -rw-r--r-- | devel/migrations/0009_auto__add_developerkey.py | 126 | ||||
| -rw-r--r-- | devel/models.py | 31 | ||||
| -rw-r--r-- | devel/utils.py | 50 | ||||
| -rw-r--r-- | devel/views.py | 270 |
13 files changed, 889 insertions, 375 deletions
diff --git a/devel/admin.py b/devel/admin.py index 5a704c0b..971933b7 100644 --- a/devel/admin.py +++ b/devel/admin.py @@ -2,7 +2,7 @@ from django.contrib import admin from django.contrib.auth.admin import UserAdmin from django.contrib.auth.models import User -from .models import UserProfile, MasterKey, PGPSignature +from .models import UserProfile, MasterKey, DeveloperKey, PGPSignature class UserProfileInline(admin.StackedInline): @@ -17,7 +17,14 @@ class UserProfileAdmin(UserAdmin): class MasterKeyAdmin(admin.ModelAdmin): list_display = ('pgp_key', 'owner', 'created', 'revoker', 'revoked') - search_fields = ('pgp_key', 'owner', 'revoker') + search_fields = ('pgp_key', 'owner__username', 'revoker__username') + date_hierarchy = 'created' + + +class DeveloperKeyAdmin(admin.ModelAdmin): + list_display = ('key', 'parent', 'owner', 'created', 'expires', 'revoked') + search_fields = ('key', 'owner__username') + list_filter = ('owner',) date_hierarchy = 'created' @@ -32,6 +39,7 @@ admin.site.unregister(User) admin.site.register(User, UserProfileAdmin) admin.site.register(MasterKey, MasterKeyAdmin) +admin.site.register(DeveloperKey, DeveloperKeyAdmin) admin.site.register(PGPSignature, PGPSignatureAdmin) # vim: set ts=4 sw=4 et: diff --git a/devel/forms.py b/devel/forms.py new file mode 100644 index 00000000..7f7c281e --- /dev/null +++ b/devel/forms.py @@ -0,0 +1,100 @@ +import random +from string import ascii_letters, digits + +from django import forms +from django.conf import settings +from django.contrib.auth.models import User, Group +from django.contrib.sites.models import Site +from django.core.mail import send_mail +from django.template import loader, Context + +from .models import UserProfile + + +class ProfileForm(forms.Form): + email = forms.EmailField(label='Private email (not shown publicly):', + help_text="Used for out-of-date notifications, etc.") + passwd1 = forms.CharField(label='New Password', required=False, + widget=forms.PasswordInput) + passwd2 = forms.CharField(label='Confirm Password', required=False, + widget=forms.PasswordInput) + + def clean(self): + if self.cleaned_data['passwd1'] != self.cleaned_data['passwd2']: + raise forms.ValidationError('Passwords do not match.') + return self.cleaned_data + + +class UserProfileForm(forms.ModelForm): + def clean_pgp_key(self): + data = self.cleaned_data['pgp_key'] + # strip 0x prefix if provided; store uppercase + if data.startswith('0x'): + data = data[2:] + return data.upper() + + class Meta: + model = UserProfile + exclude = ('allowed_repos', 'user', 'latin_name') + + +class NewUserForm(forms.ModelForm): + username = forms.CharField(max_length=30) + private_email = forms.EmailField() + first_name = forms.CharField(required=False) + last_name = forms.CharField(required=False) + groups = forms.ModelMultipleChoiceField(required=False, + queryset=Group.objects.all()) + + class Meta: + model = UserProfile + exclude = ('picture', 'user') + + def __init__(self, *args, **kwargs): + super(NewUserForm, self).__init__(*args, **kwargs) + # Hack ourself so certain fields appear first. self.fields is a + # SortedDict object where we can manipulate the keyOrder list. + order = self.fields.keyOrder + keys = ('username', 'private_email', 'first_name', 'last_name') + for key in reversed(keys): + order.remove(key) + order.insert(0, key) + + def clean_username(self): + username = self.cleaned_data['username'] + if User.objects.filter(username=username).exists(): + raise forms.ValidationError( + "A user with that username already exists.") + return username + + def save(self, commit=True): + profile = super(NewUserForm, self).save(False) + pwletters = ascii_letters + digits + password = ''.join([random.choice(pwletters) for _ in xrange(8)]) + user = User.objects.create_user(username=self.cleaned_data['username'], + email=self.cleaned_data['private_email'], password=password) + user.first_name = self.cleaned_data['first_name'] + user.last_name = self.cleaned_data['last_name'] + user.save() + # sucks that the MRM.add() method can't take a list directly... we have + # to resort to dirty * magic. + user.groups.add(*self.cleaned_data['groups']) + profile.user = user + if commit: + profile.save() + self.save_m2m() + + template = loader.get_template('devel/new_account.txt') + ctx = Context({ + 'site': Site.objects.get_current(), + 'user': user, + 'password': password, + }) + + send_mail("Your new archweb account", + template.render(ctx), + settings.BRANDING_EMAIL, + [user.email], + fail_silently=False) + +# vim: set ts=4 sw=4 et: diff --git a/devel/management/commands/generate_keyring.py b/devel/management/commands/generate_keyring.py index b9117c84..34bcd2f8 100644 --- a/devel/management/commands/generate_keyring.py +++ b/devel/management/commands/generate_keyring.py @@ -48,13 +48,17 @@ def generate_keyring(keyserver, keyring): logger.info("getting all known key IDs") # Screw you Django, for not letting one natively do value != <empty string> - key_ids = UserProfile.objects.filter(user__is_active=True, + key_ids = UserProfile.objects.filter( pgp_key__isnull=False).extra(where=["pgp_key != ''"]).values_list( "pgp_key", flat=True) logger.info("%d keys fetched from user profiles", len(key_ids)) master_key_ids = MasterKey.objects.values_list("pgp_key", flat=True) logger.info("%d keys fetched from master keys", len(master_key_ids)) + # GPG is stupid and interprets any filename without path portion as being + # in ~/.gnupg/. Fake it out if we just get a bare filename. + if '/' not in keyring: + keyring = './%s' % keyring gpg_cmd = ["gpg", "--no-default-keyring", "--keyring", keyring, "--keyserver", keyserver, "--recv-keys"] logger.info("running command: %r", gpg_cmd) diff --git a/devel/management/commands/import_signatures.py b/devel/management/commands/import_signatures.py deleted file mode 100644 index ce1aba90..00000000 --- a/devel/management/commands/import_signatures.py +++ /dev/null @@ -1,123 +0,0 @@ -# -*- coding: utf-8 -*- -""" -import_signatures command - -Import signatures from a given GPG keyring. - -Usage: ./manage.py generate_keyring <keyring_path> -""" - -from collections import namedtuple -from datetime import datetime -import logging -import subprocess -import sys - -from django.core.management.base import BaseCommand, CommandError -from django.db import transaction - -from devel.models import PGPSignature - -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s -> %(levelname)s: %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', - stream=sys.stderr) -logger = logging.getLogger() - -class Command(BaseCommand): - args = "<keyring_path>" - help = "Import signatures from a given GPG keyring." - - def handle(self, *args, **options): - v = int(options.get('verbosity', None)) - if v == 0: - logger.level = logging.ERROR - elif v == 1: - logger.level = logging.INFO - elif v == 2: - logger.level = logging.DEBUG - - if len(args) < 1: - raise CommandError("keyring_path must be provided") - - import_signatures(args[0]) - - -SignatureData = namedtuple('SignatureData', - ('signer', 'signee', 'created', 'expires', 'valid')) - - -def get_date(epoch_string): - '''Convert a epoch string into a python 'date' object (not datetime).''' - return datetime.utcfromtimestamp(int(epoch_string)).date() - - -def parse_sigdata(data): - nodes = {} - edges = [] - current_pubkey = None - - # parse all of the output from our successful GPG command - logger.info("parsing command output") - for line in data.split('\n'): - parts = line.split(':') - if parts[0] == 'pub': - current_pubkey = parts[4] - nodes[current_pubkey] = None - if parts[0] == 'uid': - uid = parts[9] - # only set uid if this is the first one encountered - if nodes[current_pubkey] is None: - nodes[current_pubkey] = uid - if parts[0] == 'sig': - signer = parts[4] - created = get_date(parts[5]) - expires = None - if parts[6]: - expires = get_date(parts[6]) - valid = parts[1] != '-' - edge = SignatureData(signer, current_pubkey, - created, expires, valid) - edges.append(edge) - - return nodes, edges - - -def import_signatures(keyring): - gpg_cmd = ["gpg", "--no-default-keyring", "--keyring", keyring, - "--list-sigs", "--with-colons", "--fixed-list-mode"] - logger.info("running command: %r", gpg_cmd) - proc = subprocess.Popen(gpg_cmd, stdout=subprocess.PIPE) - outdata, errdata = proc.communicate() - if proc.returncode != 0: - logger.error(errdata) - raise subprocess.CalledProcessError(proc.returncode, gpg_cmd) - - nodes, edges = parse_sigdata(outdata) - - # now prune the data down to what we actually want. - # prune edges not in nodes, remove duplicates, and self-sigs - pruned_edges = set(edge for edge in edges - if edge.signer in nodes and edge.signer != edge.signee) - - logger.info("creating or finding %d signatures", len(pruned_edges)) - created_ct = updated_ct = 0 - with transaction.commit_on_success(): - for edge in pruned_edges: - sig, created = PGPSignature.objects.get_or_create( - signer=edge.signer, signee=edge.signee, - created=edge.created, expires=edge.expires, - defaults={ 'valid': edge.valid }) - if sig.valid != edge.valid: - sig.valid = edge.valid - sig.save() - updated_ct = 1 - if created: - created_ct += 1 - - sig_ct = PGPSignature.objects.all().count() - logger.info("%d total signatures in database", sig_ct) - logger.info("created %d, updated %d signatures", created_ct, updated_ct) - -# vim: set ts=4 sw=4 et: diff --git a/devel/management/commands/pgp_import.py b/devel/management/commands/pgp_import.py new file mode 100644 index 00000000..b1f29d77 --- /dev/null +++ b/devel/management/commands/pgp_import.py @@ -0,0 +1,242 @@ +# -*- coding: utf-8 -*- +""" +pgp_import command + +Import keys and signatures from a given GPG keyring. + +Usage: ./manage.py pgp_import <keyring_path> +""" + +from collections import namedtuple, OrderedDict +from datetime import datetime +import logging +from pytz import utc +import subprocess +import sys + +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction + +from devel.models import DeveloperKey, PGPSignature +from devel.utils import UserFinder + + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s -> %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + stream=sys.stderr) +logger = logging.getLogger() + +class Command(BaseCommand): + args = "<keyring_path>" + help = "Import keys and signatures from a given GPG keyring." + + def handle(self, *args, **options): + v = int(options.get('verbosity', None)) + if v == 0: + logger.level = logging.ERROR + elif v == 1: + logger.level = logging.INFO + elif v == 2: + logger.level = logging.DEBUG + + if len(args) < 1: + raise CommandError("keyring_path must be provided") + + import_keys(args[0]) + import_signatures(args[0]) + + +def get_date(epoch_string): + '''Convert a epoch string into a python 'date' object (not datetime).''' + if not epoch_string: + return None + return datetime.utcfromtimestamp(int(epoch_string)).date() + + +def get_datetime(epoch_string): + '''Convert a epoch string into a python 'datetime' object.''' + if not epoch_string: + return None + return datetime.utcfromtimestamp(int(epoch_string)).replace(tzinfo=utc) + + +def call_gpg(keyring, *args): + # GPG is stupid and interprets any filename without path portion as being + # in ~/.gnupg/. Fake it out if we just get a bare filename. + if '/' not in keyring: + keyring = './%s' % keyring + gpg_cmd = ["gpg2", "--no-default-keyring", "--keyring", keyring, + "--with-colons", "--fixed-list-mode"] + gpg_cmd.extend(args) + logger.info("running command: %s", ' '.join(gpg_cmd)) + proc = subprocess.Popen(gpg_cmd, stdout=subprocess.PIPE) + outdata, errdata = proc.communicate() + if proc.returncode != 0: + logger.error(errdata) + raise subprocess.CalledProcessError(proc.returncode, gpg_cmd) + return outdata + + +class KeyData(object): + def __init__(self, key, created, expires): + self.key = key + self.created = get_datetime(created) + self.expires = get_datetime(expires) + self.parent = None + self.revoked = None + self.db_id = None + + +def parse_keydata(data): + keys = OrderedDict() + current_pubkey = None + + # parse all of the output from our successful GPG command + logger.info("parsing command output") + node = None + for line in data.split('\n'): + parts = line.split(':') + if parts[0] == 'pub': + key = parts[4] + current_pubkey = key + keys[key] = KeyData(key, parts[5], parts[6]) + node = parts[0] + elif parts[0] == 'sub': + key = parts[4] + keys[key] = KeyData(key, parts[5], parts[6]) + keys[key].parent = current_pubkey + node = parts[0] + elif parts[0] == 'uid': + node = parts[0] + elif parts[0] == 'rev' and node in ('pub', 'sub'): + keys[current_pubkey].revoked = get_datetime(parts[5]) + + return keys + + +def find_key_owner(key, keys, finder): + '''Recurse up the chain, looking for an owner.''' + if key is None: + return None + owner = finder.find_by_pgp_key(key.key) + if owner: + return owner + if key.parent: + return find_key_owner(keys[key.parent], keys, finder) + return None + + +def import_keys(keyring): + outdata = call_gpg(keyring, "--list-sigs") + keydata = parse_keydata(outdata) + + logger.info("creating or finding %d keys", len(keydata)) + created_ct = updated_ct = 0 + with transaction.commit_on_success(): + finder = UserFinder() + # we are dependent on parents coming before children; parse_keydata + # uses an OrderedDict to ensure this is the case. + for data in keydata.values(): + parent_id = None + if data.parent: + parent_data = keydata.get(data.parent, None) + if parent_data: + parent_id = parent_data.db_id + other = { + 'expires': data.expires, + 'revoked': data.revoked, + 'parent_id': parent_id, + } + dkey, created = DeveloperKey.objects.get_or_create( + key=data.key, created=data.created, defaults=other) + data.db_id = dkey.id + + # set or update any additional data we might need to + needs_save = False + if created: + created_ct += 1 + else: + for k, v in other.items(): + if getattr(dkey, k) != v: + setattr(dkey, k, v) + needs_save = True + if dkey.owner_id is None: + owner = find_key_owner(data, keydata, finder) + if owner is not None: + dkey.owner = owner + needs_save = True + if needs_save: + dkey.save() + updated_ct += 1 + + key_ct = DeveloperKey.objects.all().count() + logger.info("%d total keys in database", key_ct) + logger.info("created %d, updated %d keys", created_ct, updated_ct) + + +SignatureData = namedtuple('SignatureData', + ('signer', 'signee', 'created', 'expires', 'valid')) + + +def parse_sigdata(data): + nodes = {} + edges = [] + current_pubkey = None + + # parse all of the output from our successful GPG command + logger.info("parsing command output") + for line in data.split('\n'): + parts = line.split(':') + if parts[0] == 'pub': + current_pubkey = parts[4] + nodes[current_pubkey] = None + if parts[0] == 'uid': + uid = parts[9] + # only set uid if this is the first one encountered + if nodes[current_pubkey] is None: + nodes[current_pubkey] = uid + if parts[0] == 'sig': + signer = parts[4] + created = get_date(parts[5]) + expires = None + if parts[6]: + expires = get_date(parts[6]) + valid = parts[1] != '-' + edge = SignatureData(signer, current_pubkey, + created, expires, valid) + edges.append(edge) + + return nodes, edges + + +def import_signatures(keyring): + outdata = call_gpg(keyring, "--list-sigs") + nodes, edges = parse_sigdata(outdata) + + # now prune the data down to what we actually want. + # prune edges not in nodes, remove duplicates, and self-sigs + pruned_edges = {edge for edge in edges + if edge.signer in nodes and edge.signer != edge.signee} + + logger.info("creating or finding %d signatures", len(pruned_edges)) + created_ct = updated_ct = 0 + with transaction.commit_on_success(): + for edge in pruned_edges: + sig, created = PGPSignature.objects.get_or_create( + signer=edge.signer, signee=edge.signee, + created=edge.created, expires=edge.expires, + defaults={ 'valid': edge.valid }) + if sig.valid != edge.valid: + sig.valid = edge.valid + sig.save() + updated_ct = 1 + if created: + created_ct += 1 + + sig_ct = PGPSignature.objects.all().count() + logger.info("%d total signatures in database", sig_ct) + logger.info("created %d, updated %d signatures", created_ct, updated_ct) + +# vim: set ts=4 sw=4 et: diff --git a/devel/management/commands/rematch_developers.py b/devel/management/commands/rematch_developers.py index 8383cc8d..2b379588 100644 --- a/devel/management/commands/rematch_developers.py +++ b/devel/management/commands/rematch_developers.py @@ -46,52 +46,53 @@ class Command(NoArgsCommand): @transaction.commit_on_success def match_packager(finder): - logger.info("getting all unmatched packages") + logger.info("getting all unmatched packager strings") package_count = matched_count = 0 - unknown = set() - - for package in Package.objects.filter(packager__isnull=True): - if package.packager_str in unknown: - continue - logger.debug("package %s, packager string %s", - package.pkgname, package.packager_str) - package_count += 1 - user = finder.find(package.packager_str) + mapping = {} + + unmatched = Package.objects.filter(packager__isnull=True).values_list( + 'packager_str', flat=True).order_by().distinct() + + logger.info("%d packager strings retrieved", len(unmatched)) + for packager in unmatched: + logger.debug("packager string %s", packager) + user = finder.find(packager) if user: - package.packager = user + mapping[packager] = user logger.debug(" found user %s" % user.username) - package.save() matched_count += 1 - else: - unknown.add(package.packager_str) - logger.info("%d packager strings checked, %d newly matched", + for packager_str, user in mapping.items(): + package_count += Package.objects.filter(packager__isnull=True, + packager_str=packager_str).update(packager=user) + + logger.info("%d packages updated, %d packager strings matched", package_count, matched_count) - logger.debug("unknown packagers:\n%s", - "\n".join(unknown)) @transaction.commit_on_success def match_flagrequest(finder): - logger.info("getting all non-user flag requests") + logger.info("getting all flag request email addresses from unknown users") req_count = matched_count = 0 - unknown = set() - - for request in FlagRequest.objects.filter(user__isnull=True): - if request.user_email in unknown: - continue - logger.debug("email %s", request.user_email) - req_count += 1 - user = finder.find_by_email(request.user_email) + mapping = {} + + unmatched = FlagRequest.objects.filter(user__isnull=True).values_list( + 'user_email', flat=True).order_by().distinct() + + logger.info("%d email addresses retrieved", len(unmatched)) + for user_email in unmatched: + logger.debug("email %s", user_email) + user = finder.find_by_email(user_email) if user: - request.user = user + mapping[user_email] = user logger.debug(" found user %s" % user.username) - request.save() matched_count += 1 - else: - unknown.add(request.user_email) - logger.info("%d request emails checked, %d newly matched", + for user_email, user in mapping.items(): + req_count += FlagRequest.objects.filter(user__isnull=True, + user_email=user_email).update(user=user) + + logger.info("%d request emails updated, %d emails matched", req_count, matched_count) # vim: set ts=4 sw=4 et: diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index cf98f004..3e835f7c 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -14,6 +14,7 @@ Example: """ from collections import defaultdict +from copy import copy import io import os import re @@ -27,11 +28,12 @@ from pytz import utc from django.core.management.base import BaseCommand, CommandError from django.db import connections, router, transaction from django.db.utils import IntegrityError +from django.utils.timezone import now from devel.utils import UserFinder -from main.models import Arch, Package, PackageDepend, PackageFile, Repo -from main.utils import utc_now -from packages.models import Conflict, Provision, Replacement +from main.models import Arch, Package, PackageFile, Repo +from packages.models import Depend, Conflict, Provision, Replacement, Update +from packages.utils import parse_version logging.basicConfig( @@ -78,10 +80,9 @@ class RepoPackage(object): bare = ( 'name', 'base', 'arch', 'filename', 'md5sum', 'sha256sum', 'url', 'packager' ) number = ( 'csize', 'isize' ) - collections = ( 'depends', 'optdepends', 'conflicts', - 'provides', 'replaces', 'groups', 'license', 'files' ) - - version_re = re.compile(r'^((\d+):)?(.+)-([^-]+)$') + collections = ( 'depends', 'optdepends', 'makedepends', 'checkdepends', + 'conflicts', 'provides', 'replaces', 'groups', 'license', + 'files' ) def __init__(self, repo): self.repo = repo @@ -111,23 +112,15 @@ class RepoPackage(object): v[0] = 'missing' setattr(self, k, v[0]) elif k == 'version': - match = self.version_re.match(v[0]) - self.ver = match.group(3) - self.rel = match.group(4) - if match.group(2): - self.epoch = int(match.group(2)) + self.ver, self.rel, self.epoch = parse_version(v[0]) elif k == 'builddate': try: builddate = datetime.utcfromtimestamp(int(v[0])) self.builddate = builddate.replace(tzinfo=utc) except ValueError: - try: - self.builddate = datetime.strptime(v[0], - '%a %b %d %H:%M:%S %Y') - except ValueError: - logger.warning( - 'Package %s had unparsable build date %s', - self.name, v[0]) + logger.warning( + 'Package %s had unparsable build date %s', + self.name, v[0]) elif k == 'files': self.files = tuple(v) self.has_files = True @@ -143,19 +136,21 @@ class RepoPackage(object): return u'%s-%s' % (self.ver, self.rel) -DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.*))?$") +DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.+))?$") -def create_depend(package, dep_str, optional=False): - depend = PackageDepend(pkg=package, optional=optional) +def create_depend(package, dep_str, deptype='D'): + depend = Depend(pkg=package, deptype=deptype) # lop off any description first parts = dep_str.split(':', 1) if len(parts) > 1: depend.description = parts[1].strip() match = DEPEND_RE.match(parts[0].strip()) if match: - depend.depname = match.group(1) - if match.group(2): - depend.depvcmp = match.group(2) + depend.name = match.group(1) + if match.group(3): + depend.comparison = match.group(3) + if match.group(4): + depend.version = match.group(4) else: logger.warning('Package %s had unparsable depend string %s', package.pkgname, dep_str) @@ -183,6 +178,7 @@ def create_related(model, package, rel_str, equals_only=False): return None return related + def create_multivalued(dbpkg, repopkg, db_attr, repo_attr): '''Populate the simplest of multivalued attributes. These are those that only deal with a 'name' attribute, such as licenses, groups, etc. The input @@ -236,8 +232,10 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): dbpkg.depends.all().delete() deps = [create_depend(dbpkg, y) for y in repopkg.depends] - deps += [create_depend(dbpkg, y, True) for y in repopkg.optdepends] - PackageDepend.objects.bulk_create(deps) + deps += [create_depend(dbpkg, y, 'O') for y in repopkg.optdepends] + deps += [create_depend(dbpkg, y, 'M') for y in repopkg.makedepends] + deps += [create_depend(dbpkg, y, 'C') for y in repopkg.checkdepends] + Depend.objects.bulk_create(deps) dbpkg.conflicts.all().delete() conflicts = [create_related(Conflict, dbpkg, y) for y in repopkg.conflicts] @@ -275,7 +273,7 @@ def populate_files(dbpkg, repopkg, force=False): return if not dbpkg.files_last_update or not dbpkg.last_update: pass - elif dbpkg.files_last_update > dbpkg.last_update: + elif dbpkg.files_last_update >= dbpkg.last_update: return # only delete files if we are reading a DB that contains them @@ -284,17 +282,24 @@ def populate_files(dbpkg, repopkg, force=False): logger.info("adding %d files for package %s", len(repopkg.files), dbpkg.pkgname) pkg_files = [] - for f in repopkg.files: - dirname, filename = f.rsplit('/', 1) + # sort in normal alpha-order that pacman uses, rather than makepkg's + # default breadth-first, directory-first ordering + files = sorted(repopkg.files) + for f in files: + if '/' in f: + dirname, filename = f.rsplit('/', 1) + dirname += '/' + else: + dirname, filename = '', f if filename == '': filename = None pkgfile = PackageFile(pkg=dbpkg, is_directory=(filename is None), - directory=dirname + '/', + directory=dirname, filename=filename) pkg_files.append(pkgfile) PackageFile.objects.bulk_create(pkg_files) - dbpkg.files_last_update = utc_now() + dbpkg.files_last_update = now() dbpkg.save() @@ -307,7 +312,7 @@ def update_common(archname, reponame, pkgs, sanity_check=True): transaction.set_dirty() repository = Repo.objects.get(name__iexact=reponame) - architecture = Arch.objects.get(name__iexact=archname) + architecture = Arch.objects.get(name=archname) # no-arg order_by() removes even the default ordering; we don't need it dbpkgs = Package.objects.filter( arch=architecture, repo=repository).order_by() @@ -346,35 +351,42 @@ def db_update(archname, reponame, pkgs, force=False): logger.info('Updating %s (%s)', reponame, archname) dbpkgs = update_common(archname, reponame, pkgs, True) repository = Repo.objects.get(name__iexact=reponame) - architecture = Arch.objects.get(name__iexact=archname) + architecture = Arch.objects.get(name=archname) # This makes our inner loop where we find packages by name *way* more # efficient by not having to go to the database for each package to # SELECT them by name. - dbdict = dict((dbpkg.pkgname, dbpkg) for dbpkg in dbpkgs) + dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs} dbset = set(dbdict.keys()) - syncset = set([pkg.name for pkg in pkgs]) + syncset = {pkg.name for pkg in pkgs} in_sync_not_db = syncset - dbset logger.info("%d packages in sync not db", len(in_sync_not_db)) # packages in syncdb and not in database (add to database) for pkg in (pkg for pkg in pkgs if pkg.name in in_sync_not_db): logger.info("Adding package %s", pkg.name) - dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository) + timestamp = now() + dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository, + created=timestamp) try: with transaction.commit_on_success(): - populate_pkg(dbpkg, pkg, timestamp=utc_now()) + populate_pkg(dbpkg, pkg, timestamp=timestamp) + Update.objects.log_update(None, dbpkg) except IntegrityError: - logger.warning("Could not add package %s; " - "not fatal if another thread beat us to it.", - pkg.name, exc_info=True) + if architecture.agnostic: + logger.warning("Could not add package %s; " + "not fatal if another thread beat us to it.", + pkg.name) + else: + logger.exception("Could not add package %s", pkg.name) # packages in database and not in syncdb (remove from database) for pkgname in (dbset - syncset): logger.info("Removing package %s", pkgname) dbpkg = dbdict[pkgname] with transaction.commit_on_success(): + Update.objects.log_update(dbpkg, None) # no race condition here as long as simultaneous threads both # issue deletes; second delete will be a no-op delete_pkg_files(dbpkg) @@ -391,7 +403,7 @@ def db_update(archname, reponame, pkgs, force=False): if not force and pkg_same_version(pkg, dbpkg): continue elif not force: - timestamp = utc_now() + timestamp = now() # The odd select_for_update song and dance here are to ensure # simultaneous updates don't happen on a package, causing @@ -402,7 +414,9 @@ def db_update(archname, reponame, pkgs, force=False): logger.debug("Package %s was already updated", pkg.name) continue logger.info("Updating package %s", pkg.name) + prevpkg = copy(dbpkg) populate_pkg(dbpkg, pkg, force=force, timestamp=timestamp) + Update.objects.log_update(prevpkg, dbpkg) logger.info('Finished updating arch: %s', archname) @@ -413,7 +427,7 @@ def filesonly_update(archname, reponame, pkgs, force=False): """ logger.info('Updating files for %s (%s)', reponame, archname) dbpkgs = update_common(archname, reponame, pkgs, False) - dbdict = dict((dbpkg.pkgname, dbpkg) for dbpkg in dbpkgs) + dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs} dbset = set(dbdict.keys()) for pkg in (pkg for pkg in pkgs if pkg.name in dbset): @@ -425,7 +439,7 @@ def filesonly_update(archname, reponame, pkgs, force=False): with transaction.commit_on_success(): if not dbpkg.files_last_update or not dbpkg.last_update: pass - elif not force and dbpkg.files_last_update > dbpkg.last_update: + elif not force and dbpkg.files_last_update >= dbpkg.last_update: logger.debug("Files for %s are up to date", pkg.name) continue dbpkg = Package.objects.select_for_update().get(id=dbpkg.id) @@ -509,7 +523,7 @@ def locate_arch(arch): if isinstance(arch, Arch): return arch try: - return Arch.objects.get(name__iexact=arch) + return Arch.objects.get(name=arch) except Arch.DoesNotExist: raise CommandError( 'Specified architecture %s is not currently known.' % arch) @@ -541,6 +555,12 @@ def read_repo(primary_arch, repo_file, options): package.name, repo_file, package.arch)) del packages + database = router.db_for_write(Package) + connection = connections[database] + if connection.vendor == 'sqlite': + cursor = connection.cursor() + cursor.execute('PRAGMA synchronous = NORMAL') + logger.info('Starting database updates for %s.', repo_file) for arch in sorted(packages_arches.keys()): if filesonly: @@ -548,6 +568,8 @@ def read_repo(primary_arch, repo_file, options): else: db_update(arch, repo, packages_arches[arch], force) logger.info('Finished database updates for %s.', repo_file) + connection.commit() + connection.close() return 0 # vim: set ts=4 sw=4 et: diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py index c74762eb..8c1e47bf 100755..100644 --- a/devel/management/commands/reporead_inotify.py +++ b/devel/management/commands/reporead_inotify.py @@ -23,7 +23,7 @@ import threading import time from django.core.management.base import BaseCommand, CommandError -from django.db import connection +from django.db import connection, transaction from main.models import Arch, Repo from .reporead import read_repo @@ -53,6 +53,11 @@ class Command(BaseCommand): self.path_template = path_template notifier = self.setup_notifier() + # this thread is done using the database; all future access is done in + # the spawned read_repo() processes, so close the otherwise completely + # idle connection. + connection.close() + logger.info('Entering notifier loop') notifier.loop() @@ -61,15 +66,18 @@ class Command(BaseCommand): if hasattr(thread, 'cancel'): thread.cancel() + @transaction.commit_on_success def setup_notifier(self): '''Set up and configure the inotify machinery and logic. This takes the provided or default path_template and builds a list of directories we need to watch for database updates. It then validates and passes these on to the various pyinotify pieces as necessary and finally builds and returns a notifier object.''' + transaction.commit_manually() arches = Arch.objects.filter(agnostic=False) repos = Repo.objects.all() - arch_path_map = dict((arch, None) for arch in arches) + transaction.set_dirty() + arch_path_map = {arch: None for arch in arches} all_paths = set() total_paths = 0 for arch in arches: @@ -77,7 +85,7 @@ class Command(BaseCommand): for repo in repos) # take a python format string and generate all unique combinations # of directories from it; using set() ensures we filter it down - paths = set(self.path_template % values for values in combos) + paths = {self.path_template % values for values in combos} total_paths += len(paths) all_paths |= paths arch_path_map[arch] = paths @@ -91,11 +99,6 @@ class Command(BaseCommand): raise CommandError('path template did not uniquely ' 'determine architecture for each file') - # this thread is done using the database; all future access is done in - # the spawned read_repo() processes, so close the otherwise completely - # idle connection. - connection.close() - # A proper atomic replacement of the database as done by rsync is type # IN_MOVED_TO. repo-add/remove will finish with a IN_CLOSE_WRITE. mask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO diff --git a/devel/migrations/0008_auto__add_field_userprofile_last_modified.py b/devel/migrations/0008_auto__add_field_userprofile_last_modified.py new file mode 100644 index 00000000..08972e1b --- /dev/null +++ b/devel/migrations/0008_auto__add_field_userprofile_last_modified.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models +from pytz import utc + + +class Migration(SchemaMigration): + def forwards(self, orm): + default = datetime.datetime(2000, 1, 1, 0, 0).replace(tzinfo=utc) + db.add_column('user_profiles', 'last_modified', + self.gf('django.db.models.fields.DateTimeField')(default=default), + keep_default=False) + + def backwards(self, orm): + db.delete_column('user_profiles', 'last_modified') + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'devel.masterkey': { + 'Meta': {'ordering': "('created',)", 'object_name': 'MasterKey'}, + 'created': ('django.db.models.fields.DateField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'owner': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'masterkey_owner'", 'to': "orm['auth.User']"}), + 'pgp_key': ('devel.fields.PGPKeyField', [], {'max_length': '40'}), + 'revoked': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'revoker': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'masterkey_revoker'", 'to': "orm['auth.User']"}) + }, + 'devel.pgpsignature': { + 'Meta': {'object_name': 'PGPSignature'}, + 'created': ('django.db.models.fields.DateField', [], {}), + 'expires': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'signee': ('devel.fields.PGPKeyField', [], {'max_length': '40'}), + 'signer': ('devel.fields.PGPKeyField', [], {'max_length': '40'}), + 'valid': ('django.db.models.fields.BooleanField', [], {'default': 'True'}) + }, + 'devel.userprofile': { + 'Meta': {'object_name': 'UserProfile', 'db_table': "'user_profiles'"}, + 'alias': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'allowed_repos': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['main.Repo']", 'symmetrical': 'False', 'blank': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2', 'blank': 'True'}), + 'favorite_distros': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interests': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'languages': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'last_modified': ('django.db.models.fields.DateTimeField', [], {}), + 'latin_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'location': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'notify': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'occupation': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'other_contact': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'pgp_key': ('devel.fields.PGPKeyField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}), + 'picture': ('django.db.models.fields.files.FileField', [], {'default': "'devs/silhouette.png'", 'max_length': '100'}), + 'public_email': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'roles': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'time_zone': ('django.db.models.fields.CharField', [], {'default': "'UTC'", 'max_length': '100'}), + 'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'userprofile'", 'unique': 'True', 'to': "orm['auth.User']"}), + 'website': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'yob': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) + }, + 'main.repo': { + 'Meta': {'ordering': "['name']", 'object_name': 'Repo', 'db_table': "'repos'"}, + 'bugs_category': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'bugs_project': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'staging': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'svn_root': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + } + } + + complete_apps = ['devel'] diff --git a/devel/migrations/0009_auto__add_developerkey.py b/devel/migrations/0009_auto__add_developerkey.py new file mode 100644 index 00000000..60d3f7b8 --- /dev/null +++ b/devel/migrations/0009_auto__add_developerkey.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.create_table('devel_developerkey', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('owner', self.gf('django.db.models.fields.related.ForeignKey')(related_name='all_keys', null=True, to=orm['auth.User'])), + ('key', self.gf('devel.fields.PGPKeyField')(unique=True, max_length=40)), + ('created', self.gf('django.db.models.fields.DateTimeField')()), + ('expires', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)), + ('revoked', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)), + ('parent', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['devel.DeveloperKey'], null=True, on_delete=models.SET_NULL)), + )) + db.send_create_signal('devel', ['DeveloperKey']) + + def backwards(self, orm): + db.delete_table('devel_developerkey') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'devel.developerkey': { + 'Meta': {'object_name': 'DeveloperKey'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'expires': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'key': ('devel.fields.PGPKeyField', [], {'unique': 'True', 'max_length': '40'}), + 'owner': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'all_keys'", 'null': 'True', 'to': "orm['auth.User']"}), + 'parent': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['devel.DeveloperKey']", 'null': 'True', 'on_delete': 'models.SET_NULL'}), + 'revoked': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}) + }, + 'devel.masterkey': { + 'Meta': {'ordering': "('created',)", 'object_name': 'MasterKey'}, + 'created': ('django.db.models.fields.DateField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'owner': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'masterkey_owner'", 'to': "orm['auth.User']"}), + 'pgp_key': ('devel.fields.PGPKeyField', [], {'max_length': '40'}), + 'revoked': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'revoker': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'masterkey_revoker'", 'to': "orm['auth.User']"}) + }, + 'devel.pgpsignature': { + 'Meta': {'ordering': "('signer', 'signee')", 'object_name': 'PGPSignature'}, + 'created': ('django.db.models.fields.DateField', [], {}), + 'expires': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'signee': ('devel.fields.PGPKeyField', [], {'max_length': '40'}), + 'signer': ('devel.fields.PGPKeyField', [], {'max_length': '40'}), + 'valid': ('django.db.models.fields.BooleanField', [], {'default': 'True'}) + }, + 'devel.userprofile': { + 'Meta': {'object_name': 'UserProfile', 'db_table': "'user_profiles'"}, + 'alias': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'allowed_repos': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['main.Repo']", 'symmetrical': 'False', 'blank': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2', 'blank': 'True'}), + 'favorite_distros': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interests': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'languages': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'last_modified': ('django.db.models.fields.DateTimeField', [], {}), + 'latin_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'location': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'notify': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'occupation': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'other_contact': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'pgp_key': ('devel.fields.PGPKeyField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}), + 'picture': ('django.db.models.fields.files.FileField', [], {'default': "'devs/silhouette.png'", 'max_length': '100'}), + 'public_email': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'roles': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'time_zone': ('django.db.models.fields.CharField', [], {'default': "'UTC'", 'max_length': '100'}), + 'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'userprofile'", 'unique': 'True', 'to': "orm['auth.User']"}), + 'website': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'yob': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) + }, + 'main.repo': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Repo', 'db_table': "'repos'"}, + 'bugs_category': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'bugs_project': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'staging': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'svn_root': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + } + } + + complete_apps = ['devel'] diff --git a/devel/models.py b/devel/models.py index fd5a0347..4354e0f2 100644 --- a/devel/models.py +++ b/devel/models.py @@ -2,11 +2,12 @@ import pytz from django.db import models +from django.db.models.signals import pre_save from django.contrib.auth.models import User from django_countries import CountryField from .fields import PGPKeyField -from main.utils import make_choice +from main.utils import make_choice, set_created_field class UserProfile(models.Model): @@ -44,11 +45,13 @@ class UserProfile(models.Model): allowed_repos = models.ManyToManyField('main.Repo', blank=True) latin_name = models.CharField(max_length=255, null=True, blank=True, help_text="Latin-form name; used only for non-Latin full names") + last_modified = models.DateTimeField(editable=False) class Meta: db_table = 'user_profiles' - verbose_name = 'Additional Profile Data' - verbose_name_plural = 'Additional Profile Data' + get_latest_by = 'last_modified' + verbose_name = 'additional profile data' + verbose_name_plural = 'additional profile data' def get_absolute_url(self): # TODO: this is disgusting. find a way to consolidate this logic with @@ -64,7 +67,6 @@ class UserProfile(models.Model): return '/%s/#%s' % (prefix, self.user.username) - class MasterKey(models.Model): owner = models.ForeignKey(User, related_name='masterkey_owner', help_text="The developer holding this master key") @@ -77,12 +79,27 @@ class MasterKey(models.Model): class Meta: ordering = ('created',) + get_latest_by = 'created' def __unicode__(self): return u'%s, created %s' % ( self.owner.get_full_name(), self.created) +class DeveloperKey(models.Model): + owner = models.ForeignKey(User, related_name='all_keys', null=True, + help_text="The developer this key belongs to") + key = PGPKeyField(max_length=40, verbose_name="PGP key fingerprint", + unique=True) + created = models.DateTimeField() + expires = models.DateTimeField(null=True, blank=True) + revoked = models.DateTimeField(null=True, blank=True) + parent = models.ForeignKey('self', null=True, on_delete=models.SET_NULL) + + def __unicode__(self): + return self.key + + class PGPSignature(models.Model): signer = PGPKeyField(max_length=40, verbose_name="Signer key fingerprint") signee = PGPKeyField(max_length=40, verbose_name="Signee key fingerprint") @@ -91,9 +108,15 @@ class PGPSignature(models.Model): valid = models.BooleanField(default=True) class Meta: + ordering = ('signer', 'signee') + get_latest_by = 'created' verbose_name = 'PGP signature' def __unicode__(self): return u'%s → %s' % (self.signer, self.signee) + +pre_save.connect(set_created_field, sender=UserProfile, + dispatch_uid="devel.models") + # vim: set ts=4 sw=4 et: diff --git a/devel/utils.py b/devel/utils.py index 85b4e42f..7dd64972 100644 --- a/devel/utils.py +++ b/devel/utils.py @@ -1,6 +1,8 @@ import re +from django.conf import settings from django.contrib.auth.models import User +from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned from django.db import connection from django.db.models import Count, Q @@ -44,6 +46,15 @@ SELECT pr.user_id, COUNT(*), COUNT(p.flag_date) return maintainers +def ignore_does_not_exist(func): + def new_func(*args, **kwargs): + try: + return func(*args, **kwargs) + except (ObjectDoesNotExist, MultipleObjectsReturned): + return None + return new_func + + class UserFinder(object): def __init__(self): self.cache = {} @@ -52,18 +63,32 @@ class UserFinder(object): self.pgp_cache = {} @staticmethod + @ignore_does_not_exist def user_email(name, email): if email: return User.objects.get(email=email) return None @staticmethod + @ignore_does_not_exist + def username_email(name, email): + if email and '@' in email: + # split email addr at '@' symbol, ensure domain matches + # or is a subdomain of archlinux.org + username, domain = email.split('@', 1) + if re.match(settings.DOMAIN_RE, domain): + return User.objects.get(username=username) + return None + + @staticmethod + @ignore_does_not_exist def profile_email(name, email): if email: return User.objects.get(userprofile__public_email=email) return None @staticmethod + @ignore_does_not_exist def user_name(name, email): # yes, a bit odd but this is the easiest way since we can't always be # sure how to split the name. Ensure every 'token' appears in at least @@ -102,14 +127,12 @@ class UserFinder(object): email = matches.group(2) user = None - find_methods = (self.user_email, self.profile_email, self.user_name) + find_methods = (self.user_email, self.profile_email, + self.username_email, self.user_name) for matcher in find_methods: - try: - user = matcher(name, email) - if user != None: - break - except (User.DoesNotExist, User.MultipleObjectsReturned): - pass + user = matcher(name, email) + if user is not None: + break self.cache[userstring] = user self.email_cache[email] = user @@ -135,14 +158,11 @@ class UserFinder(object): if email in self.email_cache: return self.email_cache[email] - user = None - try: - user = self.user_email(None, email) - except User.DoesNotExist: - try: - user = self.profile_email(None, email) - except User.DoesNotExist: - pass + user = self.user_email(None, email) + if user is None: + user = self.profile_email(None, email) + if user is None: + user = self.username_email(None, email) self.email_cache[email] = user return user diff --git a/devel/views.py b/devel/views.py index 7be3dc17..4258ea7f 100644 --- a/devel/views.py +++ b/devel/views.py @@ -1,42 +1,40 @@ -from datetime import date, datetime, timedelta +from datetime import timedelta import operator import pytz -import random -from string import ascii_letters, digits import time -from django import forms -from django.conf import settings from django.http import HttpResponseRedirect from django.contrib.auth.decorators import \ login_required, permission_required, user_passes_test -from django.contrib.auth.models import User, Group -from django.contrib.sites.models import Site -from django.core.mail import send_mail +from django.contrib.admin.models import LogEntry, ADDITION +from django.contrib.auth.models import User +from django.contrib.contenttypes.models import ContentType from django.db import transaction -from django.db.models import F +from django.db.models import F, Count, Max from django.http import Http404 -from django.shortcuts import get_object_or_404 -from django.template import loader, Context +from django.shortcuts import get_object_or_404, render from django.template.defaultfilters import filesizeformat from django.views.decorators.cache import never_cache -from django.views.generic.simple import direct_to_template +from django.utils.encoding import force_unicode from django.utils.http import http_date +from django.utils.timezone import now -from .models import UserProfile -from main.models import Package, PackageDepend, PackageFile, TodolistPkg +from .forms import ProfileForm, UserProfileForm, NewUserForm +from .models import DeveloperKey +from main.models import Package, PackageFile from main.models import Arch, Repo -from main.utils import utc_now -from packages.models import PackageRelation +from news.models import News +from packages.models import PackageRelation, Signoff, FlagRequest, Depend from packages.utils import get_signoff_groups +from todolists.models import TodolistPackage from todolists.utils import get_annotated_todolists -from .utils import get_annotated_maintainers, UserFinder +from .utils import get_annotated_maintainers @login_required def index(request): '''the developer dashboard''' - if(request.user.is_authenticated()): + if request.user.is_authenticated(): inner_q = PackageRelation.objects.filter(user=request.user) else: inner_q = PackageRelation.objects.none() @@ -45,17 +43,28 @@ def index(request): flagged = Package.objects.normal().filter( flag_date__isnull=False, pkgbase__in=inner_q).order_by('pkgname') - todopkgs = TodolistPkg.objects.select_related( - 'pkg', 'pkg__arch', 'pkg__repo').filter(complete=False) - todopkgs = todopkgs.filter(pkg__pkgbase__in=inner_q).order_by( - 'list__name', 'pkg__pkgname') + todopkgs = TodolistPackage.objects.select_related( + 'todolist', 'pkg', 'arch', 'repo').exclude( + status=TodolistPackage.COMPLETE).filter(removed__isnull=True) + todopkgs = todopkgs.filter(pkgbase__in=inner_q).order_by( + 'todolist__name', 'pkgname') - todolists = get_annotated_todolists() - todolists = [todolist for todolist in todolists if todolist.incomplete_count > 0] + todolists = get_annotated_todolists(incomplete_only=True) signoffs = sorted(get_signoff_groups(user=request.user), key=operator.attrgetter('pkgbase')) + arches = Arch.objects.all().annotate( + total_ct=Count('packages'), flagged_ct=Count('packages__flag_date')) + repos = Repo.objects.all().annotate( + total_ct=Count('packages'), flagged_ct=Count('packages__flag_date')) + # the join is huge unless we do this separately, so merge the result here + repo_maintainers = dict(Repo.objects.order_by().filter( + userprofile__user__is_active=True).values_list('id').annotate( + Count('userprofile'))) + for repo in repos: + repo.maintainer_ct = repo_maintainers.get(repo.id, 0) + maintainers = get_annotated_maintainers() maintained = PackageRelation.objects.filter( @@ -72,80 +81,93 @@ def index(request): page_dict = { 'todos': todolists, - 'repos': Repo.objects.all(), - 'arches': Arch.objects.all(), + 'arches': arches, + 'repos': repos, 'maintainers': maintainers, 'orphan': orphan, - 'flagged' : flagged, - 'todopkgs' : todopkgs, + 'flagged': flagged, + 'todopkgs': todopkgs, 'signoffs': signoffs } - return direct_to_template(request, 'devel/index.html', page_dict) + return render(request, 'devel/index.html', page_dict) + @login_required def clock(request): devs = User.objects.filter(is_active=True).order_by( 'first_name', 'last_name').select_related('userprofile') - now = utc_now() + latest_news = dict(News.objects.filter( + author__is_active=True).values_list('author').order_by( + ).annotate(last_post=Max('postdate'))) + latest_package = dict(Package.objects.filter( + packager__is_active=True).values_list('packager').order_by( + ).annotate(last_build=Max('build_date'))) + latest_signoff = dict(Signoff.objects.filter( + user__is_active=True).values_list('user').order_by( + ).annotate(last_signoff=Max('created'))) + # The extra() bit ensures we can use our 'user_id IS NOT NULL' index + latest_flagreq = dict(FlagRequest.objects.filter( + user__is_active=True).extra( + where=['user_id IS NOT NULL']).values_list('user_id').order_by( + ).annotate(last_flagrequest=Max('created'))) + latest_log = dict(LogEntry.objects.filter( + user__is_active=True).values_list('user').order_by( + ).annotate(last_log=Max('action_time'))) + + for dev in devs: + dates = [ + latest_news.get(dev.id, None), + latest_package.get(dev.id, None), + latest_signoff.get(dev.id, None), + latest_flagreq.get(dev.id, None), + latest_log.get(dev.id, None), + dev.last_login, + ] + dates = [d for d in dates if d is not None] + if dates: + dev.last_action = max(dates) + else: + dev.last_action = None + + current_time = now() page_dict = { 'developers': devs, - 'utc_now': now, + 'utc_now': current_time, } - response = direct_to_template(request, 'devel/clock.html', page_dict) + response = render(request, 'devel/clock.html', page_dict) if not response.has_header('Expires'): - expire_time = now.replace(second=0, microsecond=0) + expire_time = current_time.replace(second=0, microsecond=0) expire_time += timedelta(minutes=1) expire_time = time.mktime(expire_time.timetuple()) response['Expires'] = http_date(expire_time) return response -class ProfileForm(forms.Form): - email = forms.EmailField(label='Private email (not shown publicly):', - help_text="Used for out-of-date notifications, etc.") - passwd1 = forms.CharField(label='New Password', required=False, - widget=forms.PasswordInput) - passwd2 = forms.CharField(label='Confirm Password', required=False, - widget=forms.PasswordInput) - - def clean(self): - if self.cleaned_data['passwd1'] != self.cleaned_data['passwd2']: - raise forms.ValidationError('Passwords do not match.') - return self.cleaned_data - -class UserProfileForm(forms.ModelForm): - def clean_pgp_key(self): - data = self.cleaned_data['pgp_key'] - # strip 0x prefix if provided; store uppercase - if data.startswith('0x'): - data = data[2:] - return data.upper() - - class Meta: - model = UserProfile - exclude = ('allowed_repos', 'user', 'latin_name') @login_required @never_cache def change_profile(request): if request.POST: form = ProfileForm(request.POST) - profile_form = UserProfileForm(request.POST, request.FILES, instance=request.user.get_profile()) + profile_form = UserProfileForm(request.POST, request.FILES, + instance=request.user.userprofile) if form.is_valid() and profile_form.is_valid(): request.user.email = form.cleaned_data['email'] if form.cleaned_data['passwd1']: request.user.set_password(form.cleaned_data['passwd1']) - request.user.save() - profile_form.save() + with transaction.commit_on_success(): + request.user.save() + profile_form.save() return HttpResponseRedirect('/devel/') else: form = ProfileForm(initial={'email': request.user.email}) - profile_form = UserProfileForm(instance=request.user.get_profile()) - return direct_to_template(request, 'devel/profile.html', + profile_form = UserProfileForm(instance=request.user.userprofile) + return render(request, 'devel/profile.html', {'form': form, 'profile_form': profile_form}) + @login_required def report(request, report_name, username=None): title = 'Developer Report' @@ -163,12 +185,12 @@ def report(request, report_name, username=None): if report_name == 'old': title = 'Packages last built more than one year ago' - cutoff = utc_now() - timedelta(days=365) + cutoff = now() - timedelta(days=365) packages = packages.filter( build_date__lt=cutoff).order_by('build_date') elif report_name == 'long-out-of-date': title = 'Packages marked out-of-date more than 90 days ago' - cutoff = utc_now() - timedelta(days=90) + cutoff = now() - timedelta(days=90) packages = packages.filter( flag_date__lt=cutoff).order_by('flag_date') elif report_name == 'big': @@ -199,7 +221,7 @@ def report(request, report_name, username=None): package.installed_size_pretty = filesizeformat( package.installed_size) ratio = package.compressed_size / float(package.installed_size) - package.ratio = '%.2f' % ratio + package.ratio = '%.3f' % ratio package.compress_type = package.filename.split('.')[-1] elif report_name == 'uncompressed-man': title = 'Packages with uncompressed manpages' @@ -214,7 +236,8 @@ def report(request, report_name, username=None): if username: pkg_ids = set(packages.values_list('id', flat=True)) bad_files = bad_files.filter(pkg__in=pkg_ids) - bad_files = bad_files.values_list('pkg_id', flat=True).distinct() + bad_files = bad_files.values_list( + 'pkg_id', flat=True).order_by().distinct() packages = packages.filter(id__in=set(bad_files)) elif report_name == 'uncompressed-info': title = 'Packages with uncompressed infopages' @@ -225,12 +248,13 @@ def report(request, report_name, username=None): if username: pkg_ids = set(packages.values_list('id', flat=True)) bad_files = bad_files.filter(pkg__in=pkg_ids) - bad_files = bad_files.values_list('pkg_id', flat=True).distinct() + bad_files = bad_files.values_list( + 'pkg_id', flat=True).order_by().distinct() packages = packages.filter(id__in=set(bad_files)) elif report_name == 'unneeded-orphans': title = 'Orphan packages required by no other packages' owned = PackageRelation.objects.all().values('pkgbase') - required = PackageDepend.objects.all().values('depname') + required = Depend.objects.all().values('name') # The two separate calls to exclude is required to do the right thing packages = packages.exclude(pkgbase__in=owned).exclude( pkgname__in=required) @@ -239,98 +263,52 @@ def report(request, report_name, username=None): names = [ 'Signature Date', 'Signed By', 'Packager' ] attrs = [ 'sig_date', 'sig_by', 'packager' ] cutoff = timedelta(hours=24) - finder = UserFinder() filtered = [] - packages = packages.filter(pgp_signature__isnull=False) + packages = packages.select_related( + 'arch', 'repo', 'packager').filter(pgp_signature__isnull=False) + known_keys = DeveloperKey.objects.select_related( + 'owner').filter(owner__isnull=False) + known_keys = {dk.key: dk for dk in known_keys} for package in packages: - sig_date = package.signature.datetime.replace(tzinfo=pytz.utc) + bad = False + sig = package.signature + sig_date = sig.creation_time.replace(tzinfo=pytz.utc) package.sig_date = sig_date.date() - key_id = package.signature.key_id - signer = finder.find_by_pgp_key(key_id) - package.sig_by = signer or key_id - if signer is None or signer.id != package.packager_id: - filtered.append(package) - elif sig_date > package.build_date + cutoff: + dev_key = known_keys.get(sig.key_id, None) + if dev_key: + package.sig_by = dev_key.owner + if dev_key.owner_id != package.packager_id: + bad = True + else: + package.sig_by = sig.key_id + bad = True + + if sig_date > package.build_date + cutoff: + bad = True + + if bad: filtered.append(package) packages = filtered else: raise Http404 + arches = {pkg.arch for pkg in packages} + repos = {pkg.repo for pkg in packages} context = { 'all_maintainers': maints, 'title': title, 'maintainer': user, 'packages': packages, + 'arches': sorted(arches), + 'repos': sorted(repos), 'column_names': names, 'column_attrs': attrs, } - return direct_to_template(request, 'devel/packages.html', context) - - -class NewUserForm(forms.ModelForm): - username = forms.CharField(max_length=30) - private_email = forms.EmailField() - first_name = forms.CharField(required=False) - last_name = forms.CharField(required=False) - groups = forms.ModelMultipleChoiceField(required=False, - queryset=Group.objects.all()) - - class Meta: - model = UserProfile - exclude = ('picture', 'user') - - def __init__(self, *args, **kwargs): - super(NewUserForm, self).__init__(*args, **kwargs) - # Hack ourself so certain fields appear first. self.fields is a - # SortedDict object where we can manipulate the keyOrder list. - order = self.fields.keyOrder - keys = ('username', 'private_email', 'first_name', 'last_name') - for key in reversed(keys): - order.remove(key) - order.insert(0, key) - - def clean_username(self): - username = self.cleaned_data['username'] - if User.objects.filter(username=username).exists(): - raise forms.ValidationError( - "A user with that username already exists.") - return username - - def save(self, commit=True): - profile = super(NewUserForm, self).save(False) - pwletters = ascii_letters + digits - password = ''.join([random.choice(pwletters) for _ in xrange(8)]) - user = User.objects.create_user(username=self.cleaned_data['username'], - email=self.cleaned_data['private_email'], password=password) - user.first_name = self.cleaned_data['first_name'] - user.last_name = self.cleaned_data['last_name'] - user.save() - # sucks that the MRM.add() method can't take a list directly... we have - # to resort to dirty * magic. - user.groups.add(*self.cleaned_data['groups']) - profile.user = user - if commit: - profile.save() - self.save_m2m() - - template = loader.get_template('devel/new_account.txt') - ctx = Context({ - 'site': Site.objects.get_current(), - 'user': user, - 'password': password, - }) - - send_mail("Your new "+settings.BRANDING_APPNAME+" account", - template.render(ctx), - settings.BRANDING_EMAIL, - [user.email], - fail_silently=False) + return render(request, 'devel/packages.html', context) + def log_addition(request, obj): """Cribbed from ModelAdmin.log_addition.""" - from django.contrib.admin.models import LogEntry, ADDITION - from django.contrib.contenttypes.models import ContentType - from django.utils.encoding import force_unicode LogEntry.objects.log_action( user_id = request.user.pk, content_type_id = ContentType.objects.get_for_model(obj).pk, @@ -340,17 +318,16 @@ def log_addition(request, obj): change_message = "Added via Create New User form." ) + @permission_required('auth.add_user') @never_cache def new_user_form(request): if request.POST: form = NewUserForm(request.POST) if form.is_valid(): - @transaction.commit_on_success - def inner_save(): + with transaction.commit_on_success(): form.save() log_addition(request, form.instance.user) - inner_save() return HttpResponseRedirect('/admin/auth/user/%d/' % \ form.instance.user.id) else: @@ -365,7 +342,8 @@ def new_user_form(request): 'title': 'Create User', 'submit_text': 'Create User' } - return direct_to_template(request, 'general_form.html', context) + return render(request, 'general_form.html', context) + @user_passes_test(lambda u: u.is_superuser) def admin_log(request, username=None): @@ -376,6 +354,6 @@ def admin_log(request, username=None): 'title': "Admin Action Log", 'log_user': user, } - return direct_to_template(request, 'devel/admin_log.html', context) + return render(request, 'devel/admin_log.html', context) # vim: set ts=4 sw=4 et: |