diff options
Diffstat (limited to 'packages')
25 files changed, 2693 insertions, 857 deletions
diff --git a/packages/admin.py b/packages/admin.py new file mode 100644 index 00000000..5df0043a --- /dev/null +++ b/packages/admin.py @@ -0,0 +1,62 @@ +from django.contrib import admin + +from .models import (PackageRelation, FlagRequest, + Signoff, SignoffSpecification, Update) + + +class PackageRelationAdmin(admin.ModelAdmin): + list_display = ('pkgbase', 'user', 'type', 'created') + list_filter = ('type', 'user') + search_fields = ('pkgbase', 'user__username') + ordering = ('pkgbase', 'user') + date_hierarchy = 'created' + + +class FlagRequestAdmin(admin.ModelAdmin): + list_display = ('pkgbase', 'full_version', 'repo', 'created', 'who', + 'is_spam', 'is_legitimate', 'message') + list_filter = ('is_spam', 'is_legitimate', 'repo', 'created') + search_fields = ('pkgbase', 'user_email', 'message') + ordering = ('-created',) + + def get_queryset(self, request): + qs = super(FlagRequestAdmin, self).queryset(request) + return qs.select_related('repo', 'user') + + +class SignoffAdmin(admin.ModelAdmin): + list_display = ('pkgbase', 'full_version', 'arch', 'repo', + 'user', 'created', 'revoked') + list_filter = ('arch', 'repo', 'user', 'created') + search_fields = ('pkgbase', 'user__username') + ordering = ('-created',) + + +class SignoffSpecificationAdmin(admin.ModelAdmin): + list_display = ('pkgbase', 'full_version', 'arch', 'repo', + 'user', 'created', 'comments') + list_filter = ('arch', 'repo', 'user', 'created') + search_fields = ('pkgbase', 'user__username') + ordering = ('-created',) + + def get_queryset(self, request): + qs = super(SignoffSpecificationAdmin, self).queryset(request) + return qs.select_related('arch', 'repo', 'user') + + +class UpdateAdmin(admin.ModelAdmin): + list_display = ('pkgname', 'repo', 'arch', 'action_flag', + 'old_version', 'new_version', 'created') + list_filter = ('action_flag', 'repo', 'arch', 'created') + search_fields = ('pkgname',) + ordering = ('-created',) + raw_id_fields = ('package',) + + +admin.site.register(PackageRelation, PackageRelationAdmin) +admin.site.register(FlagRequest, FlagRequestAdmin) +admin.site.register(Signoff, SignoffAdmin) +admin.site.register(SignoffSpecification, SignoffSpecificationAdmin) +admin.site.register(Update, UpdateAdmin) + +# vim: set ts=4 sw=4 et: diff --git a/packages/alpm.py b/packages/alpm.py new file mode 100644 index 00000000..3762ea68 --- /dev/null +++ b/packages/alpm.py @@ -0,0 +1,75 @@ +import ctypes +from ctypes.util import find_library +import operator + + +def load_alpm(name=None): + # Load the alpm library and set up some of the functions we might use + if name is None: + name = find_library('alpm') + if name is None: + # couldn't locate the correct library + return None + try: + alpm = ctypes.cdll.LoadLibrary(name) + except OSError: + return None + try: + alpm.alpm_version.argtypes = () + alpm.alpm_version.restype = ctypes.c_char_p + alpm.alpm_pkg_vercmp.argtypes = (ctypes.c_char_p, ctypes.c_char_p) + alpm.alpm_pkg_vercmp.restype = ctypes.c_int + except AttributeError: + return None + + return alpm + + +ALPM = load_alpm() + +class AlpmAPI(object): + OPERATOR_MAP = { + '=': operator.eq, + '==': operator.eq, + '!=': operator.ne, + '<': operator.lt, + '<=': operator.le, + '>': operator.gt, + '>=': operator.ge, + } + + def __init__(self): + self.alpm = ALPM + self.available = ALPM is not None + + def version(self): + if not self.available: + return None + return ALPM.alpm_version() + + def vercmp(self, ver1, ver2): + if not self.available: + return None + return ALPM.alpm_pkg_vercmp(str(ver1), str(ver2)) + + def compare_versions(self, ver1, oper, ver2): + func = self.OPERATOR_MAP.get(oper, None) + if func is None: + raise Exception("Invalid operator %s specified" % oper) + if not self.available: + return None + res = self.vercmp(ver1, ver2) + return func(res, 0) + + +def main(): + api = AlpmAPI() + print api.version() + print api.vercmp(1, 2) + print api.compare_versions(1, '<', 2) + + +if __name__ == '__main__': + main() + +# vim: set ts=4 sw=4 et: diff --git a/packages/management/__init__.py b/packages/management/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/packages/management/__init__.py diff --git a/packages/management/commands/__init__.py b/packages/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/packages/management/commands/__init__.py diff --git a/packages/management/commands/populate_signoffs.py b/packages/management/commands/populate_signoffs.py new file mode 100644 index 00000000..8a025f4e --- /dev/null +++ b/packages/management/commands/populate_signoffs.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +""" +populate_signoffs command + +Pull the latest commit message from SVN for a given package that is +signoff-eligible and does not have an existing comment attached. + +Usage: ./manage.py populate_signoffs +""" + +from datetime import datetime +import logging +import subprocess +import sys +from xml.etree.ElementTree import XML + +from django.conf import settings +from django.core.management.base import NoArgsCommand + +from ...models import SignoffSpecification +from ...utils import get_signoff_groups +from devel.utils import UserFinder + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s -> %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + stream=sys.stderr) +logger = logging.getLogger() + +class Command(NoArgsCommand): + help = """Pull the latest commit message from SVN for a given package that +is signoff-eligible and does not have an existing comment attached""" + + def handle_noargs(self, **options): + v = int(options.get('verbosity', None)) + if v == 0: + logger.level = logging.ERROR + elif v == 1: + logger.level = logging.INFO + elif v >= 2: + logger.level = logging.DEBUG + + return add_signoff_comments() + +def svn_log(pkgbase, repo): + '''Retrieve the most recent SVN log entry for the given pkgbase and + repository. The configured setting SVN_BASE_URL is used along with the + svn_root for each repository to form the correct URL.''' + path = '%s%s/%s/trunk/' % (settings.SVN_BASE_URL, repo.svn_root, pkgbase) + cmd = ['svn', 'log', '--limit=1', '--xml', path] + log_data = subprocess.check_output(cmd) + # the XML format is very very simple, especially with only one revision + xml = XML(log_data) + revision = int(xml.find('logentry').get('revision')) + date = datetime.strptime(xml.findtext('logentry/date'), + '%Y-%m-%dT%H:%M:%S.%fZ') + return { + 'revision': revision, + 'date': date, + 'author': xml.findtext('logentry/author'), + 'message': xml.findtext('logentry/msg'), + } + +def cached_svn_log(pkgbase, repo): + '''Retrieve the cached version of the SVN log if possible, else delegate to + svn_log() to do the work and cache the result.''' + key = (pkgbase, repo) + if key in cached_svn_log.cache: + return cached_svn_log.cache[key] + log = svn_log(pkgbase, repo) + cached_svn_log.cache[key] = log + return log +cached_svn_log.cache = {} + +def create_specification(package, log, finder): + trimmed_message = log['message'].strip() + required = package.arch.required_signoffs + spec = SignoffSpecification(pkgbase=package.pkgbase, + pkgver=package.pkgver, pkgrel=package.pkgrel, + epoch=package.epoch, arch=package.arch, repo=package.repo, + comments=trimmed_message, required=required) + spec.user = finder.find_by_username(log['author']) + return spec + +def add_signoff_comments(): + logger.info("getting all signoff groups") + groups = get_signoff_groups() + logger.info("%d signoff groups found", len(groups)) + + finder = UserFinder() + + for group in groups: + if not group.default_spec: + continue + + logger.debug("getting SVN log for %s (%s)", group.pkgbase, group.repo) + try: + log = cached_svn_log(group.pkgbase, group.repo) + logger.info("creating spec with SVN message for %s", group.pkgbase) + spec = create_specification(group.packages[0], log, finder) + spec.save() + except: + logger.exception("error getting SVN log for %s", group.pkgbase) + +# vim: set ts=4 sw=4 et: diff --git a/packages/management/commands/signoff_report.py b/packages/management/commands/signoff_report.py new file mode 100644 index 00000000..9724e562 --- /dev/null +++ b/packages/management/commands/signoff_report.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +""" +signoff_report command + +Send an email summarizing the state of outstanding signoffs for the given +repository. + +Usage: ./manage.py signoff_report <email> <repository> +""" + +from django.conf import settings +from django.core.mail import send_mail +from django.core.urlresolvers import reverse +from django.core.management.base import BaseCommand, CommandError +from django.contrib.auth.models import User +from django.contrib.sites.models import Site +from django.db.models import Count +from django.template import loader, Context +from django.utils.timezone import now + +from collections import namedtuple +from datetime import timedelta +import logging +from operator import attrgetter +import sys + +from main.models import Repo +from packages.models import Signoff +from packages.utils import get_signoff_groups + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s -> %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + stream=sys.stderr) +logger = logging.getLogger() + +class Command(BaseCommand): + args = "<email> <repository>" + help = "Send a signoff report for the given repository." + + def handle(self, *args, **options): + v = int(options.get('verbosity', None)) + if v == 0: + logger.level = logging.ERROR + elif v == 1: + logger.level = logging.INFO + elif v >= 2: + logger.level = logging.DEBUG + + if len(args) != 2: + raise CommandError("email and repository must be provided") + + return generate_report(args[0], args[1]) + +def generate_report(email, repo_name): + repo = Repo.objects.get(name__iexact=repo_name) + # Collect all existing signoffs for these packages + signoff_groups = sorted(get_signoff_groups([repo]), + key=attrgetter('target_repo', 'arch', 'pkgbase')) + disabled = [] + bad = [] + complete = [] + incomplete = [] + new = [] + old = [] + + new_hours = 24 + old_days = 14 + current_time = now() + new_cutoff = current_time - timedelta(hours=new_hours) + old_cutoff = current_time - timedelta(days=old_days) + + if len(signoff_groups) == 0: + # no need to send an email at all + return + + for group in signoff_groups: + spec = group.specification + if spec.known_bad: + bad.append(group) + elif not spec.enabled: + disabled.append(group) + elif group.approved(): + complete.append(group) + else: + incomplete.append(group) + + if group.package.last_update > new_cutoff: + new.append(group) + if group.package.last_update < old_cutoff: + old.append(group) + + old.sort(key=attrgetter('last_update')) + + proto = 'https' + domain = Site.objects.get_current().domain + signoffs_url = '%s://%s%s' % (proto, domain, reverse('package-signoffs')) + + # and the fun bit + Leader = namedtuple('Leader', ['user', 'count']) + leaders = Signoff.objects.filter(created__gt=new_cutoff, + revoked__isnull=True).values_list('user').annotate( + signoff_count=Count('pk')).order_by('-signoff_count')[:5] + users = User.objects.in_bulk([l[0] for l in leaders]) + leaders = (Leader(users[l[0]], l[1]) for l in leaders) + + subject = 'Signoff report for [%s]' % repo.name.lower() + t = loader.get_template('packages/signoff_report.txt') + c = Context({ + 'repo': repo, + 'signoffs_url': signoffs_url, + 'disabled': disabled, + 'bad': bad, + 'all': signoff_groups, + 'incomplete': incomplete, + 'complete': complete, + 'new': new, + 'new_hours': new_hours, + 'old': old, + 'old_days': old_days, + 'leaders': leaders, + }) + from_addr = settings.BRANDING_EMAIL + send_mail(subject, t.render(c), from_addr, [email]) + +# vim: set ts=4 sw=4 et: diff --git a/packages/migrations/0001_initial.py b/packages/migrations/0001_initial.py index 76e97340..5ecac84a 100644 --- a/packages/migrations/0001_initial.py +++ b/packages/migrations/0001_initial.py @@ -1,72 +1,205 @@ -# encoding: utf-8 -import datetime -from south.db import db -from south.v2 import SchemaMigration -from django.db import models +# -*- coding: utf-8 -*- +from __future__ import unicode_literals -class Migration(SchemaMigration): - def forwards(self, orm): - # Adding model 'PackageRelation' - db.create_table('packages_packagerelation', ( - ('pkgbase', self.gf('django.db.models.fields.CharField')(max_length=255)), - ('type', self.gf('django.db.models.fields.PositiveIntegerField')(default=1)), - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), - ('user', self.gf('django.db.models.fields.related.ForeignKey')(related_name='package_relations', to=orm['auth.User'])), - )) - db.send_create_signal('packages', ['PackageRelation']) - # Adding unique constraint on 'PackageRelation', fields ['pkgbase', 'user', 'type'] - db.create_unique('packages_packagerelation', ['pkgbase', 'user_id', 'type']) +from django.db import models, migrations +import django.db.models.deletion +from django.conf import settings - def backwards(self, orm): - # Deleting model 'PackageRelation' - db.delete_table('packages_packagerelation') - # Removing unique constraint on 'PackageRelation', fields ['pkgbase', 'user', 'type'] - db.delete_unique('packages_packagerelation', ['pkgbase', 'user_id', 'type']) - models = { - 'auth.group': { - 'Meta': {'object_name': 'Group'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), - 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'blank': 'True'}) - }, - 'auth.permission': { - 'Meta': {'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, - 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), - 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) - }, - 'auth.user': { - 'Meta': {'object_name': 'User'}, - 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), - 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), - 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), - 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'blank': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), - 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), - 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), - 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), - 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), - 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), - 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'blank': 'True'}), - 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) - }, - 'contenttypes.contenttype': { - 'Meta': {'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, - 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) - }, - 'packages.packagerelation': { - 'Meta': {'unique_together': "(('pkgbase', 'user', 'type'),)", 'object_name': 'PackageRelation'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'type': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}), - 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'package_relations'", 'to': "orm['auth.User']"}) - } - } +class Migration(migrations.Migration): - complete_apps = ['packages'] + dependencies = [ + ('main', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Conflict', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(max_length=255, db_index=True)), + ('version', models.CharField(default=b'', max_length=255)), + ('comparison', models.CharField(default=b'', max_length=255)), + ('pkg', models.ForeignKey(related_name=b'conflicts', to='main.Package')), + ], + options={ + 'ordering': ('name',), + 'abstract': False, + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='Depend', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(max_length=255, db_index=True)), + ('version', models.CharField(default=b'', max_length=255)), + ('comparison', models.CharField(default=b'', max_length=255)), + ('description', models.TextField(null=True, blank=True)), + ('deptype', models.CharField(default=b'D', max_length=1, choices=[(b'D', b'Depend'), (b'O', b'Optional Depend'), (b'M', b'Make Depend'), (b'C', b'Check Depend')])), + ('pkg', models.ForeignKey(related_name=b'depends', to='main.Package')), + ], + options={ + 'ordering': ('name',), + 'abstract': False, + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='FlagRequest', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('user_email', models.EmailField(max_length=75, verbose_name=b'email address')), + ('created', models.DateTimeField(editable=False, db_index=True)), + ('ip_address', models.GenericIPAddressField(verbose_name=b'IP address', unpack_ipv4=True)), + ('pkgbase', models.CharField(max_length=255, db_index=True)), + ('pkgver', models.CharField(max_length=255)), + ('pkgrel', models.CharField(max_length=255)), + ('epoch', models.PositiveIntegerField(default=0)), + ('num_packages', models.PositiveIntegerField(default=1, verbose_name=b'number of packages')), + ('message', models.TextField(verbose_name=b'message to developer', blank=True)), + ('is_spam', models.BooleanField(default=False, help_text=b'Is this comment from a real person?')), + ('is_legitimate', models.BooleanField(default=True, help_text=b'Is this actually an out-of-date flag request?')), + ('repo', models.ForeignKey(to='main.Repo')), + ('user', models.ForeignKey(blank=True, to=settings.AUTH_USER_MODEL, null=True)), + ], + options={ + 'get_latest_by': 'created', + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='License', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(max_length=255)), + ('pkg', models.ForeignKey(related_name=b'licenses', to='main.Package')), + ], + options={ + 'ordering': ('name',), + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='PackageGroup', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(max_length=255, db_index=True)), + ('pkg', models.ForeignKey(related_name=b'groups', to='main.Package')), + ], + options={ + 'ordering': ('name',), + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='PackageRelation', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('pkgbase', models.CharField(max_length=255)), + ('type', models.PositiveIntegerField(default=1, choices=[(1, b'Maintainer'), (2, b'Watcher')])), + ('created', models.DateTimeField(editable=False)), + ('user', models.ForeignKey(related_name=b'package_relations', to=settings.AUTH_USER_MODEL)), + ], + options={ + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='Provision', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(max_length=255, db_index=True)), + ('version', models.CharField(default=b'', max_length=255)), + ('pkg', models.ForeignKey(related_name=b'provides', to='main.Package')), + ], + options={ + 'ordering': ('name',), + 'abstract': False, + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='Replacement', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(max_length=255, db_index=True)), + ('version', models.CharField(default=b'', max_length=255)), + ('comparison', models.CharField(default=b'', max_length=255)), + ('pkg', models.ForeignKey(related_name=b'replaces', to='main.Package')), + ], + options={ + 'ordering': ('name',), + 'abstract': False, + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='Signoff', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('pkgbase', models.CharField(max_length=255, db_index=True)), + ('pkgver', models.CharField(max_length=255)), + ('pkgrel', models.CharField(max_length=255)), + ('epoch', models.PositiveIntegerField(default=0)), + ('created', models.DateTimeField(editable=False, db_index=True)), + ('revoked', models.DateTimeField(null=True)), + ('comments', models.TextField(null=True, blank=True)), + ('arch', models.ForeignKey(to='main.Arch')), + ('repo', models.ForeignKey(to='main.Repo')), + ('user', models.ForeignKey(related_name=b'package_signoffs', to=settings.AUTH_USER_MODEL)), + ], + options={ + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='SignoffSpecification', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('pkgbase', models.CharField(max_length=255, db_index=True)), + ('pkgver', models.CharField(max_length=255)), + ('pkgrel', models.CharField(max_length=255)), + ('epoch', models.PositiveIntegerField(default=0)), + ('created', models.DateTimeField(editable=False)), + ('required', models.PositiveIntegerField(default=2, help_text=b'How many signoffs are required for this package?')), + ('enabled', models.BooleanField(default=True, help_text=b'Is this package eligible for signoffs?')), + ('known_bad', models.BooleanField(default=False, help_text=b'Is package is known to be broken in some way?')), + ('comments', models.TextField(null=True, blank=True)), + ('arch', models.ForeignKey(to='main.Arch')), + ('repo', models.ForeignKey(to='main.Repo')), + ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, null=True)), + ], + options={ + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='Update', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('pkgname', models.CharField(max_length=255, db_index=True)), + ('pkgbase', models.CharField(max_length=255)), + ('action_flag', models.PositiveSmallIntegerField(verbose_name=b'action flag', choices=[(1, b'Addition'), (2, b'Change'), (3, b'Deletion')])), + ('created', models.DateTimeField(editable=False, db_index=True)), + ('old_pkgver', models.CharField(max_length=255, null=True)), + ('old_pkgrel', models.CharField(max_length=255, null=True)), + ('old_epoch', models.PositiveIntegerField(null=True)), + ('new_pkgver', models.CharField(max_length=255, null=True)), + ('new_pkgrel', models.CharField(max_length=255, null=True)), + ('new_epoch', models.PositiveIntegerField(null=True)), + ('arch', models.ForeignKey(related_name=b'updates', to='main.Arch')), + ('package', models.ForeignKey(related_name=b'updates', on_delete=django.db.models.deletion.SET_NULL, to='main.Package', null=True)), + ('repo', models.ForeignKey(related_name=b'updates', to='main.Repo')), + ], + options={ + 'get_latest_by': 'created', + }, + bases=(models.Model,), + ), + migrations.AlterUniqueTogether( + name='packagerelation', + unique_together=set([('pkgbase', 'user', 'type')]), + ), + ] diff --git a/packages/migrations/0002_populate_package_relation.py b/packages/migrations/0002_populate_package_relation.py deleted file mode 100644 index 738e068f..00000000 --- a/packages/migrations/0002_populate_package_relation.py +++ /dev/null @@ -1,235 +0,0 @@ -# encoding: utf-8 -import datetime -from south.db import db -from south.v2 import DataMigration -from django.db import models - -class Migration(DataMigration): - - depends_on = ( - ("main", "0003_migrate_maintainer"), - ) - - def forwards(self, orm): - "Write your forwards methods here." - # search by pkgbase first and insert those records - qs = orm['main.Package'].objects.exclude(maintainer=None).exclude( - pkgbase=None).distinct().values('pkgbase', 'maintainer_id') - for row in qs: - pr, created = orm.PackageRelation.objects.get_or_create( - pkgbase=row['pkgbase'], user__id=row['maintainer_id'], - defaults={'user_id': row['maintainer_id']}) - - # next search by pkgname first and insert those records - qs = orm['main.Package'].objects.exclude(maintainer=None).filter( - pkgbase=None).distinct().values('pkgname', 'maintainer_id') - for row in qs: - pr, created = orm.PackageRelation.objects.get_or_create( - pkgbase=row['pkgname'], user__id=row['maintainer_id'], - defaults={'user_id': row['maintainer_id']}) - - def backwards(self, orm): - "Write your backwards methods here." - if not db.dry_run: - orm.PackageRelation.objects.all().delete() - pass - - models = { - 'auth.group': { - 'Meta': {'object_name': 'Group'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), - 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'blank': 'True'}) - }, - 'auth.permission': { - 'Meta': {'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, - 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), - 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) - }, - 'auth.user': { - 'Meta': {'object_name': 'User'}, - 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), - 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), - 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), - 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'blank': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), - 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), - 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), - 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), - 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), - 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), - 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'blank': 'True'}), - 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) - }, - 'contenttypes.contenttype': { - 'Meta': {'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, - 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) - }, - 'main.altforum': { - 'Meta': {'object_name': 'AltForum', 'db_table': "'alt_forums'"}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'language': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'url': ('django.db.models.fields.CharField', [], {'max_length': '255'}) - }, - 'main.arch': { - 'Meta': {'object_name': 'Arch', 'db_table': "'arches'"}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) - }, - 'main.donor': { - 'Meta': {'object_name': 'Donor', 'db_table': "'donors'"}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) - }, - 'main.externalproject': { - 'Meta': {'object_name': 'ExternalProject'}, - 'description': ('django.db.models.fields.CharField', [], {'max_length': '128'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '64'}), - 'url': ('django.db.models.fields.URLField', [], {'max_length': '200'}) - }, - 'main.mirror': { - 'Meta': {'object_name': 'Mirror'}, - 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), - 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), - 'country': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), - 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), - 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), - 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), - 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), - 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Mirror']", 'null': 'True'}) - }, - 'main.mirrorprotocol': { - 'Meta': {'object_name': 'MirrorProtocol'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) - }, - 'main.mirrorrsync': { - 'Meta': {'object_name': 'MirrorRsync'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), - 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['main.Mirror']"}) - }, - 'main.mirrorurl': { - 'Meta': {'object_name': 'MirrorUrl'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['main.Mirror']"}), - 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['main.MirrorProtocol']"}), - 'url': ('django.db.models.fields.CharField', [], {'max_length': '255'}) - }, - 'main.news': { - 'Meta': {'object_name': 'News', 'db_table': "'news'"}, - 'author': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'news_author'", 'to': "orm['auth.User']"}), - 'content': ('django.db.models.fields.TextField', [], {}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'postdate': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}), - 'title': ('django.db.models.fields.CharField', [], {'max_length': '255'}) - }, - 'main.package': { - 'Meta': {'object_name': 'Package', 'db_table': "'packages'"}, - 'arch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'to': "orm['main.Arch']"}), - 'build_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), - 'compressed_size': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True'}), - 'files_last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'installed_size': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True'}), - 'last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), - 'license': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'maintained_packages'", 'null': 'True', 'to': "orm['auth.User']"}), - 'needupdate': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), - 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), - 'pkgdesc': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'pkgname': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), - 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'repo': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'to': "orm['main.Repo']"}), - 'url': ('django.db.models.fields.CharField', [], {'max_length': '255'}) - }, - 'main.packagedepend': { - 'Meta': {'object_name': 'PackageDepend', 'db_table': "'package_depends'"}, - 'depname': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), - 'depvcmp': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}) - }, - 'main.packagefile': { - 'Meta': {'object_name': 'PackageFile', 'db_table': "'package_files'"}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}) - }, - 'main.press': { - 'Meta': {'object_name': 'Press', 'db_table': "'press'"}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'url': ('django.db.models.fields.CharField', [], {'max_length': '255'}) - }, - 'main.repo': { - 'Meta': {'object_name': 'Repo', 'db_table': "'repos'"}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), - 'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}) - }, - 'main.signoff': { - 'Meta': {'object_name': 'Signoff'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'packager': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}), - 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}), - 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}) - }, - 'main.todolist': { - 'Meta': {'object_name': 'Todolist', 'db_table': "'todolists'"}, - 'creator': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}), - 'date_added': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}), - 'description': ('django.db.models.fields.TextField', [], {}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}) - }, - 'main.todolistpkg': { - 'Meta': {'unique_together': "(('list', 'pkg'),)", 'object_name': 'TodolistPkg', 'db_table': "'todolist_pkgs'"}, - 'complete': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'list': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Todolist']"}), - 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}) - }, - 'main.userprofile': { - 'Meta': {'object_name': 'UserProfile', 'db_table': "'user_profiles'"}, - 'alias': ('django.db.models.fields.CharField', [], {'max_length': '50'}), - 'allowed_repos': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['main.Repo']", 'blank': 'True'}), - 'favorite_distros': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'interests': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), - 'languages': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), - 'location': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), - 'notify': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), - 'occupation': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), - 'other_contact': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), - 'picture': ('django.db.models.fields.files.FileField', [], {'default': "'devs/silhouette.png'", 'max_length': '100'}), - 'public_email': ('django.db.models.fields.CharField', [], {'max_length': '50'}), - 'roles': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), - 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'userprofile_user'", 'unique': 'True', 'to': "orm['auth.User']"}), - 'website': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), - 'yob': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) - }, - 'packages.packagerelation': { - 'Meta': {'unique_together': "(('pkgbase', 'user', 'type'),)", 'object_name': 'PackageRelation'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'type': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}), - 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'package_relations'", 'to': "orm['auth.User']"}) - } - } - - complete_apps = ['main', 'packages'] diff --git a/packages/migrations/0003_auto__add_packagegroup.py b/packages/migrations/0003_auto__add_packagegroup.py deleted file mode 100644 index c40b6429..00000000 --- a/packages/migrations/0003_auto__add_packagegroup.py +++ /dev/null @@ -1,109 +0,0 @@ -# encoding: utf-8 -import datetime -from south.db import db -from south.v2 import SchemaMigration -from django.db import models - -class Migration(SchemaMigration): - - def forwards(self, orm): - - # Adding model 'PackageGroup' - db.create_table('packages_packagegroup', ( - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), - ('pkg', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['main.Package'])), - ('name', self.gf('django.db.models.fields.CharField')(max_length=255)), - )) - db.send_create_signal('packages', ['PackageGroup']) - - - def backwards(self, orm): - - # Deleting model 'PackageGroup' - db.delete_table('packages_packagegroup') - - - models = { - 'auth.group': { - 'Meta': {'object_name': 'Group'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), - 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) - }, - 'auth.permission': { - 'Meta': {'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, - 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), - 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) - }, - 'auth.user': { - 'Meta': {'object_name': 'User'}, - 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), - 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), - 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), - 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), - 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), - 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), - 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), - 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), - 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), - 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), - 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) - }, - 'contenttypes.contenttype': { - 'Meta': {'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, - 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) - }, - 'main.arch': { - 'Meta': {'object_name': 'Arch', 'db_table': "'arches'"}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) - }, - 'main.package': { - 'Meta': {'object_name': 'Package', 'db_table': "'packages'"}, - 'arch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'to': "orm['main.Arch']"}), - 'build_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), - 'compressed_size': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True'}), - 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'files_last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), - 'flag_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'installed_size': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True'}), - 'last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), - 'license': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), - 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), - 'pkgdesc': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), - 'pkgname': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), - 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'repo': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'to': "orm['main.Repo']"}), - 'url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}) - }, - 'main.repo': { - 'Meta': {'object_name': 'Repo', 'db_table': "'repos'"}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), - 'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}) - }, - 'packages.packagegroup': { - 'Meta': {'object_name': 'PackageGroup'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}) - }, - 'packages.packagerelation': { - 'Meta': {'unique_together': "(('pkgbase', 'user', 'type'),)", 'object_name': 'PackageRelation'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'type': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}), - 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'package_relations'", 'to': "orm['auth.User']"}) - } - } - - complete_apps = ['packages'] diff --git a/packages/models.py b/packages/models.py index 70ac4fe5..03f03422 100644 --- a/packages/models.py +++ b/packages/models.py @@ -1,6 +1,15 @@ +from collections import namedtuple + from django.db import models +from django.db.models.signals import pre_save +from django.contrib.admin.models import ADDITION, CHANGE, DELETION from django.contrib.auth.models import User +from main.models import Arch, Repo, Package +from main.utils import set_created_field, database_vendor +from packages.alpm import AlpmAPI + + class PackageRelation(models.Model): ''' Represents maintainership (or interest) in a package by a given developer. @@ -17,15 +26,488 @@ class PackageRelation(models.Model): pkgbase = models.CharField(max_length=255) user = models.ForeignKey(User, related_name="package_relations") type = models.PositiveIntegerField(choices=TYPE_CHOICES, default=MAINTAINER) + created = models.DateTimeField(editable=False) + class Meta: unique_together = (('pkgbase', 'user', 'type'),) + def get_associated_packages(self): + return Package.objects.normal().filter(pkgbase=self.pkgbase) + + def repositories(self): + packages = self.get_associated_packages() + return sorted({p.repo for p in packages}) + + def last_update(self): + return Update.objects.filter(pkgbase=self.pkgbase).latest() + + def __unicode__(self): + return u'%s: %s (%s)' % ( + self.pkgbase, self.user, self.get_type_display()) + + +class SignoffSpecificationManager(models.Manager): + def get_from_package(self, pkg): + '''Utility method to pull all relevant name-version fields from a + package and get a matching signoff specification.''' + return self.get( + pkgbase=pkg.pkgbase, pkgver=pkg.pkgver, pkgrel=pkg.pkgrel, + epoch=pkg.epoch, arch=pkg.arch, repo=pkg.repo) + + def get_or_default_from_package(self, pkg): + '''utility method to pull all relevant name-version fields from a + package and get a matching signoff specification, or return the default + base case.''' + try: + return self.get( + pkgbase=pkg.pkgbase, pkgver=pkg.pkgver, pkgrel=pkg.pkgrel, + epoch=pkg.epoch, arch=pkg.arch, repo=pkg.repo) + except SignoffSpecification.DoesNotExist: + return fake_signoff_spec(pkg.arch) + + +class SignoffSpecification(models.Model): + ''' + A specification for the signoff policy for this particular revision of a + package. The default is requiring two signoffs for a given package. These + are created only if necessary; e.g., if one wanted to override the + required=2 attribute, otherwise a sane default object is used. + ''' + pkgbase = models.CharField(max_length=255, db_index=True) + pkgver = models.CharField(max_length=255) + pkgrel = models.CharField(max_length=255) + epoch = models.PositiveIntegerField(default=0) + arch = models.ForeignKey(Arch) + repo = models.ForeignKey(Repo) + user = models.ForeignKey(User, null=True) + created = models.DateTimeField(editable=False) + required = models.PositiveIntegerField(default=2, + help_text="How many signoffs are required for this package?") + enabled = models.BooleanField(default=True, + help_text="Is this package eligible for signoffs?") + known_bad = models.BooleanField(default=False, + help_text="Is package is known to be broken in some way?") + comments = models.TextField(null=True, blank=True) + + objects = SignoffSpecificationManager() + + @property + def full_version(self): + if self.epoch > 0: + return u'%d:%s-%s' % (self.epoch, self.pkgver, self.pkgrel) + return u'%s-%s' % (self.pkgver, self.pkgrel) + + def __unicode__(self): + return u'%s-%s' % (self.pkgbase, self.full_version) + + +# Fake signoff specs for when we don't have persisted ones in the database. +# These have all necessary attributes of the real thing but are lighter weight +# and have no chance of being persisted. +FakeSignoffSpecification = namedtuple('FakeSignoffSpecification', + ('required', 'enabled', 'known_bad', 'comments')) + + +def fake_signoff_spec(arch): + return FakeSignoffSpecification(arch.required_signoffs, True, False, u'') + + +class SignoffManager(models.Manager): + def get_from_package(self, pkg, user, revoked=False): + '''Utility method to pull all relevant name-version fields from a + package and get a matching signoff.''' + not_revoked = not revoked + return self.get( + pkgbase=pkg.pkgbase, pkgver=pkg.pkgver, pkgrel=pkg.pkgrel, + epoch=pkg.epoch, arch=pkg.arch, repo=pkg.repo, + revoked__isnull=not_revoked, user=user) + + def get_or_create_from_package(self, pkg, user): + '''Utility method to pull all relevant name-version fields from a + package and get or create a matching signoff.''' + return self.get_or_create( + pkgbase=pkg.pkgbase, pkgver=pkg.pkgver, pkgrel=pkg.pkgrel, + epoch=pkg.epoch, arch=pkg.arch, repo=pkg.repo, + revoked=None, user=user) + + def for_package(self, pkg): + return self.select_related('user').filter( + pkgbase=pkg.pkgbase, pkgver=pkg.pkgver, pkgrel=pkg.pkgrel, + epoch=pkg.epoch, arch=pkg.arch, repo=pkg.repo) + +class Signoff(models.Model): + ''' + A signoff for a package (by pkgbase) at a given point in time. These are + not keyed directly to a Package object so they don't ever get deleted when + Packages come and go from testing repositories. + ''' + pkgbase = models.CharField(max_length=255, db_index=True) + pkgver = models.CharField(max_length=255) + pkgrel = models.CharField(max_length=255) + epoch = models.PositiveIntegerField(default=0) + arch = models.ForeignKey(Arch) + repo = models.ForeignKey(Repo) + user = models.ForeignKey(User, related_name="package_signoffs") + created = models.DateTimeField(editable=False, db_index=True) + revoked = models.DateTimeField(null=True) + comments = models.TextField(null=True, blank=True) + + objects = SignoffManager() + + @property + def packages(self): + return Package.objects.normal().filter(pkgbase=self.pkgbase, + pkgver=self.pkgver, pkgrel=self.pkgrel, epoch=self.epoch, + arch=self.arch, repo=self.repo) + + @property + def full_version(self): + if self.epoch > 0: + return u'%d:%s-%s' % (self.epoch, self.pkgver, self.pkgrel) + return u'%s-%s' % (self.pkgver, self.pkgrel) + + def __unicode__(self): + revoked = u'' + if self.revoked: + revoked = u' (revoked)' + return u'%s-%s: %s%s' % ( + self.pkgbase, self.full_version, self.user, revoked) + + +class FlagRequest(models.Model): + ''' + A notification the package is out-of-date submitted through the web site. + ''' + user = models.ForeignKey(User, blank=True, null=True) + user_email = models.EmailField('email address') + created = models.DateTimeField(editable=False, db_index=True) + ip_address = models.GenericIPAddressField('IP address', unpack_ipv4=True) + pkgbase = models.CharField(max_length=255, db_index=True) + pkgver = models.CharField(max_length=255) + pkgrel = models.CharField(max_length=255) + epoch = models.PositiveIntegerField(default=0) + repo = models.ForeignKey(Repo) + num_packages = models.PositiveIntegerField('number of packages', default=1) + message = models.TextField('message to developer', blank=True) + is_spam = models.BooleanField(default=False, + help_text="Is this comment from a real person?") + is_legitimate = models.BooleanField(default=True, + help_text="Is this actually an out-of-date flag request?") + + class Meta: + get_latest_by = 'created' + + def who(self): + if self.user: + return self.user.get_full_name() + return self.user_email + + @property + def full_version(self): + # Difference here from other implementations at the moment: we need to + # handle the case of pkgver and pkgrel being null as this table didn't + # originally have version columns. + if self.pkgver == '' and self.pkgrel == '': + return u'' + if self.epoch > 0: + return u'%d:%s-%s' % (self.epoch, self.pkgver, self.pkgrel) + return u'%s-%s' % (self.pkgver, self.pkgrel) + + def get_associated_packages(self): + return Package.objects.normal().filter( + pkgbase=self.pkgbase, + repo__testing=self.repo.testing, + repo__staging=self.repo.staging).order_by( + 'pkgname', 'repo__name', 'arch__name') + + def __unicode__(self): + return u'%s from %s on %s' % (self.pkgbase, self.who(), self.created) + + +class UpdateManager(models.Manager): + def log_update(self, old_pkg, new_pkg): + '''Utility method to help log an update. This will determine the type + based on how many packages are passed in, and will pull the relevant + necesary fields off the given packages. + Note that in some cases, this is a no-op if we know this database type + supports triggers to add these rows instead.''' + if database_vendor(Package, 'write') in ('sqlite', 'postgresql'): + # we log updates using database triggers for these backends + return + update = Update() + if new_pkg: + update.action_flag = ADDITION + update.package = new_pkg + update.arch = new_pkg.arch + update.repo = new_pkg.repo + update.pkgname = new_pkg.pkgname + update.pkgbase = new_pkg.pkgbase + update.new_pkgver = new_pkg.pkgver + update.new_pkgrel = new_pkg.pkgrel + update.new_epoch = new_pkg.epoch + if old_pkg: + if new_pkg: + update.action_flag = CHANGE + # ensure we should even be logging this + if (old_pkg.pkgver == new_pkg.pkgver and + old_pkg.pkgrel == new_pkg.pkgrel and + old_pkg.epoch == new_pkg.epoch): + # all relevant fields were the same; e.g. a force update + return + else: + update.action_flag = DELETION + update.arch = old_pkg.arch + update.repo = old_pkg.repo + update.pkgname = old_pkg.pkgname + update.pkgbase = old_pkg.pkgbase + + update.old_pkgver = old_pkg.pkgver + update.old_pkgrel = old_pkg.pkgrel + update.old_epoch = old_pkg.epoch + + update.save(force_insert=True) + return update + + +class Update(models.Model): + UPDATE_ACTION_CHOICES = ( + (ADDITION, 'Addition'), + (CHANGE, 'Change'), + (DELETION, 'Deletion'), + ) + + package = models.ForeignKey(Package, related_name="updates", + null=True, on_delete=models.SET_NULL) + repo = models.ForeignKey(Repo, related_name="updates") + arch = models.ForeignKey(Arch, related_name="updates") + pkgname = models.CharField(max_length=255, db_index=True) + pkgbase = models.CharField(max_length=255) + action_flag = models.PositiveSmallIntegerField('action flag', + choices=UPDATE_ACTION_CHOICES) + created = models.DateTimeField(editable=False, db_index=True) + + old_pkgver = models.CharField(max_length=255, null=True) + old_pkgrel = models.CharField(max_length=255, null=True) + old_epoch = models.PositiveIntegerField(null=True) + + new_pkgver = models.CharField(max_length=255, null=True) + new_pkgrel = models.CharField(max_length=255, null=True) + new_epoch = models.PositiveIntegerField(null=True) + + objects = UpdateManager() + + class Meta: + get_latest_by = 'created' + + def is_addition(self): + return self.action_flag == ADDITION + + def is_change(self): + return self.action_flag == CHANGE + + def is_deletion(self): + return self.action_flag == DELETION + + @property + def old_version(self): + if self.action_flag == ADDITION: + return None + if self.old_epoch > 0: + return u'%d:%s-%s' % (self.old_epoch, self.old_pkgver, self.old_pkgrel) + return u'%s-%s' % (self.old_pkgver, self.old_pkgrel) + + @property + def new_version(self): + if self.action_flag == DELETION: + return None + if self.new_epoch > 0: + return u'%d:%s-%s' % (self.new_epoch, self.new_pkgver, self.new_pkgrel) + return u'%s-%s' % (self.new_pkgver, self.new_pkgrel) + + def elsewhere(self): + return Package.objects.normal().filter( + pkgname=self.pkgname, arch=self.arch) + + def replacements(self): + pkgs = Package.objects.normal().filter( + replaces__name=self.pkgname) + if not self.arch.agnostic: + # make sure we match architectures if possible + arches = set(Arch.objects.filter(agnostic=True)) + arches.add(self.arch) + pkgs = pkgs.filter(arch__in=arches) + return pkgs + + def __unicode__(self): + return u'%s of %s on %s' % (self.get_action_flag_display(), + self.pkgname, self.created) + + class PackageGroup(models.Model): ''' Represents a group a package is in. There is no actual group entity, only names that link to given packages. ''' - pkg = models.ForeignKey('main.Package') + pkg = models.ForeignKey(Package, related_name='groups') + name = models.CharField(max_length=255, db_index=True) + + def __unicode__(self): + return "%s: %s" % (self.name, self.pkg) + + class Meta: + ordering = ('name',) + + +class License(models.Model): + pkg = models.ForeignKey(Package, related_name='licenses') name = models.CharField(max_length=255) + def __unicode__(self): + return self.name + + class Meta: + ordering = ('name',) + + +class RelatedToBase(models.Model): + '''A base class for conflicts/provides/replaces/etc.''' + name = models.CharField(max_length=255, db_index=True) + version = models.CharField(max_length=255, default='') + + def get_best_satisfier(self): + '''Find a satisfier for this related package that best matches the + given criteria. It will not search provisions, but will find packages + named and matching repo characteristics if possible.''' + pkgs = Package.objects.normal().filter(pkgname=self.name) + # TODO: this may in fact be faster- select only the fields we know will + # actually get used, saving us some bandwidth and hopefully query + # construction time. However, reality hasn't quite proved it out yet. + #pkgs = Package.objects.select_related('repo', 'arch').only( + # 'id', 'pkgname', 'epoch', 'pkgver', 'pkgrel', + # 'repo__id', 'repo__name', 'repo__testing', 'repo__staging', + # 'arch__id', 'arch__name').filter(pkgname=self.name) + if not self.pkg.arch.agnostic: + # make sure we match architectures if possible + arches = self.pkg.applicable_arches() + pkgs = pkgs.filter(arch__in=arches) + # if we have a comparison operation, make sure the packages we grab + # actually satisfy the requirements + if self.comparison and self.version: + alpm = AlpmAPI() + pkgs = [pkg for pkg in pkgs if not alpm.available or + alpm.compare_versions(pkg.full_version, self.comparison, + self.version)] + if len(pkgs) == 0: + # couldn't find a package in the DB + # it should be a virtual depend (or a removed package) + return None + if len(pkgs) == 1: + return pkgs[0] + # more than one package, see if we can't shrink it down + # grab the first though in case we fail + pkg = pkgs[0] + # prevents yet more DB queries, these lists should be short; + # after each grab the best available in case we remove all entries + pkgs = [p for p in pkgs if p.repo.staging == self.pkg.repo.staging] + if len(pkgs) > 0: + pkg = pkgs[0] + + pkgs = [p for p in pkgs if p.repo.testing == self.pkg.repo.testing] + if len(pkgs) > 0: + pkg = pkgs[0] + + return pkg + + def get_providers(self): + '''Return providers of this related package. Does *not* include exact + matches as it checks the Provision names only, use get_best_satisfier() + instead for exact matches.''' + pkgs = Package.objects.normal().filter( + provides__name=self.name).order_by().distinct() + if not self.pkg.arch.agnostic: + # make sure we match architectures if possible + arches = self.pkg.applicable_arches() + pkgs = pkgs.filter(arch__in=arches) + + # If we have a comparison operation, make sure the packages we grab + # actually satisfy the requirements. + alpm = AlpmAPI() + if alpm.available and self.comparison and self.version: + pkgs = pkgs.prefetch_related('provides') + new_pkgs = [] + for package in pkgs: + for provide in package.provides.all(): + if provide.name != self.name: + continue + if alpm.compare_versions(provide.version, + self.comparison, self.version): + new_pkgs.append(package) + pkgs = new_pkgs + + # Sort providers by preference. We sort those in same staging/testing + # combination first, followed by others. We sort by a (staging, + # testing) match tuple that will be (True, True) in the best case. + key_func = lambda x: (x.repo.staging == self.pkg.repo.staging, + x.repo.testing == self.pkg.repo.testing) + return sorted(pkgs, key=key_func, reverse=True) + + def __unicode__(self): + if self.version: + return u'%s%s%s' % (self.name, self.comparison, self.version) + return self.name + + class Meta: + abstract = True + ordering = ('name',) + + +class Depend(RelatedToBase): + DEPTYPE_CHOICES = ( + ('D', 'Depend'), + ('O', 'Optional Depend'), + ('M', 'Make Depend'), + ('C', 'Check Depend'), + ) + + pkg = models.ForeignKey(Package, related_name='depends') + comparison = models.CharField(max_length=255, default='') + description = models.TextField(null=True, blank=True) + deptype = models.CharField(max_length=1, default='D', + choices=DEPTYPE_CHOICES) + + def __unicode__(self): + '''For depends, we may also have a description and a modifier.''' + to_str = super(Depend, self).__unicode__() + if self.description: + return u'%s: %s' % (to_str, self.description) + return to_str + + +class Conflict(RelatedToBase): + pkg = models.ForeignKey(Package, related_name='conflicts') + comparison = models.CharField(max_length=255, default='') + + +class Provision(RelatedToBase): + pkg = models.ForeignKey(Package, related_name='provides') + # comparison must be '=' for provides + + @property + def comparison(self): + if self.version is not None and self.version != '': + return '=' + return None + + +class Replacement(RelatedToBase): + pkg = models.ForeignKey(Package, related_name='replaces') + comparison = models.CharField(max_length=255, default='') + + +# hook up some signals +for sender in (FlagRequest, PackageRelation, + SignoffSpecification, Signoff, Update): + pre_save.connect(set_created_field, sender=sender, + dispatch_uid="packages.models") + # vim: set ts=4 sw=4 et: diff --git a/packages/sql/search_indexes.postgresql_psycopg2.sql b/packages/sql/search_indexes.postgresql_psycopg2.sql new file mode 100644 index 00000000..a7eaf998 --- /dev/null +++ b/packages/sql/search_indexes.postgresql_psycopg2.sql @@ -0,0 +1,3 @@ +CREATE EXTENSION IF NOT EXISTS pg_trgm; +CREATE INDEX packages_pkgname_trgm_gist ON packages USING gist (UPPER(pkgname) gist_trgm_ops); +CREATE INDEX packages_pkgdesc_trgm_gist ON packages USING gist (UPPER(pkgdesc) gist_trgm_ops); diff --git a/packages/sql/update.postgresql_psycopg2.sql b/packages/sql/update.postgresql_psycopg2.sql new file mode 100644 index 00000000..6d678387 --- /dev/null +++ b/packages/sql/update.postgresql_psycopg2.sql @@ -0,0 +1,45 @@ +CREATE OR REPLACE FUNCTION packages_on_insert() RETURNS trigger AS $body$ +BEGIN + INSERT INTO packages_update + (action_flag, created, package_id, arch_id, repo_id, pkgname, pkgbase, new_pkgver, new_pkgrel, new_epoch) + VALUES (1, now(), NEW.id, NEW.arch_id, NEW.repo_id, NEW.pkgname, NEW.pkgbase, NEW.pkgver, NEW.pkgrel, NEW.epoch); + RETURN NULL; +END; +$body$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION packages_on_update() RETURNS trigger AS $body$ +BEGIN + INSERT INTO packages_update + (action_flag, created, package_id, arch_id, repo_id, pkgname, pkgbase, old_pkgver, old_pkgrel, old_epoch, new_pkgver, new_pkgrel, new_epoch) + VALUES (2, now(), NEW.id, NEW.arch_id, NEW.repo_id, NEW.pkgname, NEW.pkgbase, OLD.pkgver, OLD.pkgrel, OLD.epoch, NEW.pkgver, NEW.pkgrel, NEW.epoch); + RETURN NULL; +END; +$body$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION packages_on_delete() RETURNS trigger AS $body$ +BEGIN + INSERT INTO packages_update + (action_flag, created, arch_id, repo_id, pkgname, pkgbase, old_pkgver, old_pkgrel, old_epoch) + VALUES (3, now(), OLD.arch_id, OLD.repo_id, OLD.pkgname, OLD.pkgbase, OLD.pkgver, OLD.pkgrel, OLD.epoch); + RETURN NULL; +END; +$body$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS packages_insert ON packages; +CREATE TRIGGER packages_insert + AFTER INSERT ON packages + FOR EACH ROW + EXECUTE PROCEDURE packages_on_insert(); + +DROP TRIGGER IF EXISTS packages_update ON packages; +CREATE TRIGGER packages_update + AFTER UPDATE ON packages + FOR EACH ROW + WHEN (OLD.pkgver != NEW.pkgver OR OLD.pkgrel != NEW.pkgrel OR OLD.epoch != NEW.epoch) + EXECUTE PROCEDURE packages_on_update(); + +DROP TRIGGER IF EXISTS packages_delete ON packages; +CREATE TRIGGER packages_delete + AFTER DELETE ON packages + FOR EACH ROW + EXECUTE PROCEDURE packages_on_delete(); diff --git a/packages/sql/update.sqlite3.sql b/packages/sql/update.sqlite3.sql new file mode 100644 index 00000000..6f151bdd --- /dev/null +++ b/packages/sql/update.sqlite3.sql @@ -0,0 +1,30 @@ +DROP TRIGGER IF EXISTS packages_insert; +CREATE TRIGGER packages_insert + AFTER INSERT ON packages + FOR EACH ROW + BEGIN + INSERT INTO packages_update + (action_flag, created, package_id, arch_id, repo_id, pkgname, pkgbase, new_pkgver, new_pkgrel, new_epoch) + VALUES (1, strftime('%Y-%m-%d %H:%M:%f', 'now'), NEW.id, NEW.arch_id, NEW.repo_id, NEW.pkgname, NEW.pkgbase, NEW.pkgver, NEW.pkgrel, NEW.epoch); + END; + +DROP TRIGGER IF EXISTS packages_update; +CREATE TRIGGER packages_update + AFTER UPDATE ON packages + FOR EACH ROW + WHEN (OLD.pkgver != NEW.pkgver OR OLD.pkgrel != NEW.pkgrel OR OLD.epoch != NEW.epoch) + BEGIN + INSERT INTO packages_update + (action_flag, created, package_id, arch_id, repo_id, pkgname, pkgbase, old_pkgver, old_pkgrel, old_epoch, new_pkgver, new_pkgrel, new_epoch) + VALUES (2, strftime('%Y-%m-%d %H:%M:%f', 'now'), NEW.id, NEW.arch_id, NEW.repo_id, NEW.pkgname, NEW.pkgbase, OLD.pkgver, OLD.pkgrel, OLD.epoch, NEW.pkgver, NEW.pkgrel, NEW.epoch); + END; + +DROP TRIGGER IF EXISTS packages_delete; +CREATE TRIGGER packages_delete + AFTER DELETE ON packages + FOR EACH ROW + BEGIN + INSERT INTO packages_update + (action_flag, created, arch_id, repo_id, pkgname, pkgbase, old_pkgver, old_pkgrel, old_epoch) + VALUES (3, strftime('%Y-%m-%d %H:%M:%f', 'now'), OLD.arch_id, OLD.repo_id, OLD.pkgname, OLD.pkgbase, OLD.pkgver, OLD.pkgrel, OLD.epoch); + END; diff --git a/packages/templatetags/jinja2.py b/packages/templatetags/jinja2.py new file mode 100644 index 00000000..f0b42a09 --- /dev/null +++ b/packages/templatetags/jinja2.py @@ -0,0 +1,91 @@ +from urllib import urlencode, quote as urlquote, unquote +from django.utils.html import escape +from django_jinja import library +from main.templatetags import pgp + + +@library.filter +def url_unquote(original_url): + try: + url = original_url + if isinstance(url, unicode): + url = url.encode('ascii') + url = unquote(url).decode('utf-8') + return url + except UnicodeError: + return original_url + + +def link_encode(url, query): + # massage the data into all utf-8 encoded strings first, so urlencode + # doesn't barf at the data we pass it + query = {k: unicode(v).encode('utf-8') for k, v in query.items()} + data = urlencode(query) + return "%s?%s" % (url, data) + + +@library.global_function +def pgp_key_link(key_id, link_text=None): + return pgp.pgp_key_link(key_id, link_text) + + +@library.global_function +def scm_link(package, operation): + parts = ("abslibre", operation, package.repo.name.lower(), package.pkgbase) + linkbase = ( + "https://projects.parabola.nu/%s.git/%s/%s/%s") + return linkbase % tuple(urlquote(part.encode('utf-8')) for part in parts) + + +@library.global_function +def wiki_link(package): + url = "https://wiki.parabola.nu/index.php" + data = { + 'title': "Special:Search", + 'search': package.pkgname, + } + return link_encode(url, data) + +@library.global_function +def bugs_list(package): + if package.arch.name == 'mips64el': + project = "mips64el" + else: + project = "issue-tracker" + url = "https://labs.parabola.nu/projects/%s/search" % project + data = { + 'titles_only': '1', + 'issues': '1', + 'q': package.pkgname, + } + return link_encode(url, data) + + +@library.global_function +def bug_report(package): + url = "https://labs.parabola.nu/projects/" + if package.arch.name == 'mips64el': + url = url + "mips64el/issues/new" + else: + url = url + "issue-tracker/issues/new" + data = { + 'issue[subject]': '[%s] PLEASE ENTER SUMMARY' % package.pkgname, + } + return link_encode(url, data) + +@library.global_function +def flag_unfree(package): + url = "https://labs.parabola.nu/projects/" + if package.arch.name == 'mips64el': + url = url + "mips64el/issues/new" + else: + url = url + "issue-tracker/issues/new" + data = { + 'issue[tracker_id]': '4', # "freedom issue" + 'issue[priority_id]': '1', # "freedom issue" + 'issue[watcher_user_ids][]': '62', # "dev-list" + 'issue[subject]': '[%s] Please put your reasons here (register first if you haven\'t)' % package.pkgname, + } + return link_encode(url, data) + +# vim: set ts=4 sw=4 et: diff --git a/packages/templatetags/package_extras.py b/packages/templatetags/package_extras.py index 3cd2b91b..73a39092 100644 --- a/packages/templatetags/package_extras.py +++ b/packages/templatetags/package_extras.py @@ -1,23 +1,37 @@ -import cgi, urllib +from urllib import urlencode +try: + from urlparse import parse_qs +except ImportError: + from cgi import parse_qs + from django import template -from django.utils.html import escape + register = template.Library() + class BuildQueryStringNode(template.Node): def __init__(self, sortfield): self.sortfield = sortfield + super(BuildQueryStringNode, self).__init__() def render(self, context): - qs = dict(cgi.parse_qsl(context['current_query'][1:])) - if qs.has_key('sort') and qs['sort'] == self.sortfield: + qs = parse_qs(context['current_query']) + # This is really dirty. The crazy round trips we do on our query string + # mean we get things like u'\xe2\x98\x83' in our views, when we should + # have simply u'\u2603' or a byte string of the UTF-8 value. Force the + # keys and list of values to be byte strings only. + qs = {k.encode('latin-1'): [v.encode('latin-1') for v in vals] + for k, vals in qs.items()} + if 'sort' in qs and self.sortfield in qs['sort']: if self.sortfield.startswith('-'): - qs['sort'] = self.sortfield[1:] + qs['sort'] = [self.sortfield[1:]] else: - qs['sort'] = '-' + self.sortfield + qs['sort'] = ['-' + self.sortfield] else: - qs['sort'] = self.sortfield - return '?' + urllib.urlencode(qs) + qs['sort'] = [self.sortfield] + return urlencode(qs, True).replace('&', '&') + @register.tag(name='buildsortqs') def do_buildsortqs(parser, token): @@ -25,35 +39,24 @@ def do_buildsortqs(parser, token): tagname, sortfield = token.split_contents() except ValueError: raise template.TemplateSyntaxError( - "%r tag requires a single argument" % tagname) + "%r tag requires a single argument" % token) if not (sortfield[0] == sortfield[-1] and sortfield[0] in ('"', "'")): raise template.TemplateSyntaxError( - "%r tag's argument should be in quotes" % tagname) + "%r tag's argument should be in quotes" % token) return BuildQueryStringNode(sortfield[1:-1]) -@register.tag -def userpkgs(parser, token): - try: - tagname, user = token.split_contents() - except ValueError: - raise template.TemplateSyntaxError( - "%r tag requires a single argument" % tagname) - return UserPkgsNode(user) -class UserPkgsNode(template.Node): - def __init__(self, user): - self.user = template.Variable(user) +@register.simple_tag +def pkg_details_link(pkg, link_title=None, honor_flagged=False): + if not pkg: + return link_title or '' + if link_title is None: + link_title = pkg.pkgname + link_content = link_title + if honor_flagged and pkg.flag_date: + link_content = '<span class="flagged">%s</span>' % link_title + link = '<a href="%s" title="View package details for %s">%s</a>' + return link % (pkg.get_absolute_url(), pkg.pkgname, link_content) - def render(self, context): - try: - real_user = self.user.resolve(context) - # TODO don't hardcode - title = escape('View packages maintained by ' + real_user.get_full_name()) - return '<a href="/packages/search/?maintainer=%s" title="%s">%s</a>' % ( - real_user.username, - title, - real_user.get_full_name(), - ) - except template.VariableDoesNotExist: - return '' - pass + +# vim: set ts=4 sw=4 et: diff --git a/packages/tests.py b/packages/tests.py new file mode 100644 index 00000000..bbe9f00e --- /dev/null +++ b/packages/tests.py @@ -0,0 +1,46 @@ +import unittest + +from .alpm import AlpmAPI + + +alpm = AlpmAPI() + + +class AlpmTestCase(unittest.TestCase): + + @unittest.skipUnless(alpm.available, "ALPM is unavailable") + def test_version(self): + version = alpm.version() + self.assertIsNotNone(version) + version = version.split('.') + # version is a 3-tuple, e.g., '7.0.2' + self.assertEqual(3, len(version)) + + @unittest.skipUnless(alpm.available, "ALPM is unavailable") + def test_vercmp(self): + self.assertEqual(0, alpm.vercmp("1.0", "1.0")) + self.assertEqual(1, alpm.vercmp("1.1", "1.0")) + + @unittest.skipUnless(alpm.available, "ALPM is unavailable") + def test_compare_versions(self): + self.assertTrue(alpm.compare_versions("1.0", "<=", "2.0")) + self.assertTrue(alpm.compare_versions("1.0", "<", "2.0")) + self.assertFalse(alpm.compare_versions("1.0", ">=", "2.0")) + self.assertFalse(alpm.compare_versions("1.0", ">", "2.0")) + self.assertTrue(alpm.compare_versions("1:1.0", ">", "2.0")) + self.assertFalse(alpm.compare_versions("1.0.2", ">=", "2.1.0")) + + self.assertTrue(alpm.compare_versions("1.0", "=", "1.0")) + self.assertTrue(alpm.compare_versions("1.0", "=", "1.0-1")) + self.assertFalse(alpm.compare_versions("1.0", "!=", "1.0")) + + def test_behavior_when_unavailable(self): + mock_alpm = AlpmAPI() + mock_alpm.available = False + + self.assertIsNone(mock_alpm.version()) + self.assertIsNone(mock_alpm.vercmp("1.0", "1.0")) + self.assertIsNone(mock_alpm.compare_versions("1.0", "=", "1.0")) + + +# vim: set ts=4 sw=4 et: diff --git a/packages/urls.py b/packages/urls.py new file mode 100644 index 00000000..dfe19207 --- /dev/null +++ b/packages/urls.py @@ -0,0 +1,42 @@ +from django.conf.urls import include, patterns + +from .views.search import SearchListView + +package_patterns = patterns('packages.views', + (r'^$', 'details'), + (r'^json/$', 'details_json'), + (r'^files/$', 'files'), + (r'^files/json/$', 'files_json'), + (r'^flag/$', 'flag'), + (r'^flag/done/$', 'flag_confirmed', {}, 'package-flag-confirmed'), + (r'^unflag/$', 'unflag'), + (r'^unflag/all/$', 'unflag_all'), + (r'^signoff/$', 'signoff_package'), + (r'^signoff/revoke/$', 'signoff_package', {'revoke': True}), + (r'^signoff/options/$', 'signoff_options'), + (r'^download/$', 'download'), +) + +urlpatterns = patterns('packages.views', + (r'^flaghelp/$', 'flaghelp'), + (r'^signoffs/$', 'signoffs', {}, 'package-signoffs'), + (r'^signoffs/json/$', 'signoffs_json', {}, 'package-signoffs-json'), + (r'^update/$', 'update'), + + (r'^$', SearchListView.as_view(), {}, 'packages-search'), + (r'^search/json/$', 'search_json'), + + (r'^differences/$', 'arch_differences', {}, 'packages-differences'), + (r'^stale_relations/$', 'stale_relations'), + (r'^stale_relations/update/$','stale_relations_update'), + + (r'^(?P<name>[^ /]+)/$', + 'details'), + (r'^(?P<repo>[A-z0-9~\-]+)/(?P<name>[^ /]+)/$', + 'details'), + # canonical package url. subviews defined above + (r'^(?P<repo>[A-z0-9~\-]+)/(?P<arch>[A-z0-9]+)/(?P<name>[^ /]+)/', + include(package_patterns)), +) + +# vim: set ts=4 sw=4 et: diff --git a/packages/urls_groups.py b/packages/urls_groups.py new file mode 100644 index 00000000..49ced145 --- /dev/null +++ b/packages/urls_groups.py @@ -0,0 +1,9 @@ +from django.conf.urls import patterns + +urlpatterns = patterns('packages.views', + (r'^$', 'groups', {}, 'groups-list'), + (r'^(?P<arch>[A-z0-9]+)/$', 'groups'), + (r'^(?P<arch>[A-z0-9]+)/(?P<name>[^ /]+)/$', 'group_details'), +) + +# vim: set ts=4 sw=4 et: diff --git a/packages/utils.py b/packages/utils.py index 55b7acf9..c38aa840 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -1,17 +1,42 @@ +from collections import defaultdict +from itertools import chain +from operator import attrgetter, itemgetter +import re + +from django.core.serializers.json import DjangoJSONEncoder from django.db import connection -from django.db.models import Count, Max +from django.db.models import Count, Max, F +from django.db.models.query import QuerySet +from django.contrib.auth.models import User + +from main.models import Package, PackageFile, Arch, Repo +from main.utils import (database_vendor, + groupby_preserve_order, PackageStandin) +from .models import (PackageGroup, PackageRelation, + License, Depend, Conflict, Provision, Replacement, + SignoffSpecification, Signoff, fake_signoff_spec) + + +VERSION_RE = re.compile(r'^((\d+):)?(.+)-([^-]+)$') + -from operator import itemgetter +def parse_version(version): + match = VERSION_RE.match(version) + if not match: + return None, None, 0 + ver = match.group(3) + rel = match.group(4) + if match.group(2): + epoch = int(match.group(2)) + else: + epoch = 0 + return ver, rel, epoch -from main.models import Package -from main.utils import cache_function -from .models import PackageGroup -@cache_function(300) -def get_group_info(): +def get_group_info(include_arches=None): raw_groups = PackageGroup.objects.values_list( 'name', 'pkg__arch__name').order_by('name').annotate( - cnt=Count('pkg'), last_update=Max('pkg__last_update')) + cnt=Count('pkg'), last_update=Max('pkg__last_update')) # now for post_processing. we need to seperate things out and add # the count in for 'any' to all of the other architectures. group_mapping = {} @@ -38,12 +63,30 @@ def get_group_info(): new_g['arch'] = arch arch_groups[grp['name']] = new_g - # now transform it back into a sorted list + # now transform it back into a sorted list, including only the specified + # architectures if we got a list groups = [] - for val in group_mapping.itervalues(): - groups.extend(val.itervalues()) + for key, val in group_mapping.iteritems(): + if not include_arches or key in include_arches: + groups.extend(val.itervalues()) return sorted(groups, key=itemgetter('name', 'arch')) + +def get_split_packages_info(): + '''Return info on split packages that do not have an actual package name + matching the split pkgbase.''' + pkgnames = Package.objects.values('pkgname') + split_pkgs = Package.objects.exclude(pkgname=F('pkgbase')).exclude( + pkgbase__in=pkgnames).values('pkgbase', 'repo', 'arch').annotate( + last_update=Max('last_update')).order_by().distinct() + all_arches = Arch.objects.in_bulk({s['arch'] for s in split_pkgs}) + all_repos = Repo.objects.in_bulk({s['repo'] for s in split_pkgs}) + for split in split_pkgs: + split['arch'] = all_arches[split['arch']] + split['repo'] = all_repos[split['repo']] + return split_pkgs + + class Difference(object): def __init__(self, pkgname, repo, pkg_a, pkg_b): self.pkgname = pkgname @@ -65,12 +108,17 @@ class Difference(object): css_classes.append(self.pkg_b.arch.name) return ' '.join(css_classes) - def __cmp__(self, other): - if isinstance(other, Difference): - return cmp(self.__dict__, other.__dict__) - return False + def __key(self): + return (self.pkgname, hash(self.repo), + hash(self.pkg_a), hash(self.pkg_b)) + + def __eq__(self, other): + return self.__key() == other.__key() + + def __hash__(self): + return hash(self.__key()) + -@cache_function(300) def get_differences_info(arch_a, arch_b): # This is a monster. Join packages against itself, looking for packages in # our non-'any' architectures only, and not having a corresponding package @@ -89,24 +137,25 @@ SELECT p.id, q.id ) WHERE p.arch_id IN (%s, %s) AND ( + q.arch_id IN (%s, %s) + OR q.id IS NULL + ) + AND ( q.id IS NULL - OR - p.pkgver != q.pkgver - OR - p.pkgrel != q.pkgrel + OR p.pkgver != q.pkgver + OR p.pkgrel != q.pkgrel + OR p.epoch != q.epoch ) """ cursor = connection.cursor() - cursor.execute(sql, [arch_a.id, arch_b.id]) + cursor.execute(sql, [arch_a.id, arch_b.id, arch_a.id, arch_b.id]) results = cursor.fetchall() - to_fetch = [] - for row in results: - # column A will always have a value, column B might be NULL - to_fetch.append(row[0]) + # column A will always have a value, column B might be NULL + to_fetch = {row[0] for row in results} # fetch all of the necessary packages - pkgs = Package.objects.in_bulk(to_fetch) - # now build a list of tuples containing differences - differences = [] + pkgs = Package.objects.normal().in_bulk(to_fetch) + # now build a set containing differences + differences = set() for row in results: pkg_a = pkgs.get(row[0]) pkg_b = pkgs.get(row[1]) @@ -119,11 +168,342 @@ SELECT p.id, q.id name = pkg_a.pkgname if pkg_a else pkg_b.pkgname repo = pkg_a.repo if pkg_a else pkg_b.repo item = Difference(name, repo, pkg_b, pkg_a) - if item not in differences: - differences.append(item) + differences.add(item) # now sort our list by repository, package name - differences.sort(key=lambda a: (a.repo.name, a.pkgname)) + key_func = attrgetter('repo.name', 'pkgname') + differences = sorted(differences, key=key_func) return differences + +def multilib_differences(): + # Query for checking multilib out of date-ness + if database_vendor(Package) == 'sqlite': + pkgname_sql = """ + CASE WHEN ml.pkgname LIKE %s + THEN SUBSTR(ml.pkgname, 7) + WHEN ml.pkgname LIKE %s + THEN SUBSTR(ml.pkgname, 1, LENGTH(ml.pkgname) - 9) + ELSE + ml.pkgname + END + """ + else: + pkgname_sql = """ + CASE WHEN ml.pkgname LIKE %s + THEN SUBSTRING(ml.pkgname, 7) + WHEN ml.pkgname LIKE %s + THEN SUBSTRING(ml.pkgname FROM 1 FOR CHAR_LENGTH(ml.pkgname) - 9) + ELSE + ml.pkgname + END + """ + sql = """ +SELECT ml.id, reg.id + FROM packages ml + JOIN packages reg + ON ( + reg.pkgname = (""" + pkgname_sql + """) + AND reg.pkgver != ml.pkgver + ) + JOIN repos r ON reg.repo_id = r.id + WHERE ml.repo_id = %s + AND r.testing = %s + AND r.staging = %s + AND reg.arch_id = %s + ORDER BY ml.last_update + """ + multilib = Repo.objects.get(name__iexact='multilib') + i686 = Arch.objects.get(name='i686') + params = ['lib32-%', '%-multilib', multilib.id, False, False, i686.id] + + cursor = connection.cursor() + cursor.execute(sql, params) + results = cursor.fetchall() + + # fetch all of the necessary packages + to_fetch = set(chain.from_iterable(results)) + pkgs = Package.objects.normal().in_bulk(to_fetch) + + return [(pkgs[ml], pkgs[reg]) for ml, reg in results] + + +def get_wrong_permissions(): + sql = """ +SELECT DISTINCT id + FROM ( + SELECT pr.id, p.repo_id, pr.user_id + FROM packages p + JOIN packages_packagerelation pr ON p.pkgbase = pr.pkgbase + WHERE pr.type = %s + ) mp + LEFT JOIN ( + SELECT user_id, repo_id FROM user_profiles_allowed_repos ar + INNER JOIN user_profiles up ON ar.userprofile_id = up.id + ) ur + ON mp.user_id = ur.user_id AND mp.repo_id = ur.repo_id + WHERE ur.user_id IS NULL; +""" + cursor = connection.cursor() + cursor.execute(sql, [PackageRelation.MAINTAINER]) + to_fetch = [row[0] for row in cursor.fetchall()] + relations = PackageRelation.objects.select_related( + 'user', 'user__userprofile').filter( + id__in=to_fetch) + return relations + + +def attach_maintainers(packages): + '''Given a queryset or something resembling it of package objects, find all + the maintainers and attach them to the packages to prevent N+1 query + cascading.''' + if isinstance(packages, QuerySet): + pkgbases = packages.values('pkgbase') + else: + packages = list(packages) + pkgbases = {p.pkgbase for p in packages if p is not None} + rels = PackageRelation.objects.filter(type=PackageRelation.MAINTAINER, + pkgbase__in=pkgbases).values_list( + 'pkgbase', 'user_id').order_by().distinct() + + # get all the user objects we will need + user_ids = {rel[1] for rel in rels} + users = User.objects.in_bulk(user_ids) + + # now build a pkgbase -> [maintainers...] map + maintainers = defaultdict(list) + for rel in rels: + maintainers[rel[0]].append(users[rel[1]]) + + annotated = [] + # and finally, attach the maintainer lists on the original packages + for package in packages: + if package is None: + continue + package.maintainers = maintainers[package.pkgbase] + annotated.append(package) + + return annotated + + +def approved_by_signoffs(signoffs, spec): + if signoffs: + good_signoffs = sum(1 for s in signoffs if not s.revoked) + return good_signoffs >= spec.required + return False + + +class PackageSignoffGroup(object): + '''Encompasses all packages in testing with the same pkgbase.''' + def __init__(self, packages): + if len(packages) == 0: + raise Exception + self.packages = packages + self.user = None + self.target_repo = None + self.signoffs = set() + self.default_spec = True + + first = packages[0] + self.pkgbase = first.pkgbase + self.arch = first.arch + self.repo = first.repo + self.version = '' + self.last_update = first.last_update + self.packager = first.packager + self.maintainers = first.maintainers + self.specification = fake_signoff_spec(first.arch) + + version = first.full_version + if all(version == pkg.full_version for pkg in packages): + self.version = version + + @property + def package(self): + '''Try and return a relevant single package object representing this + group. Start by seeing if there is only one package, then look for the + matching package by name, finally falling back to a standin package + object.''' + if len(self.packages) == 1: + return self.packages[0] + + same_pkgs = [p for p in self.packages if p.pkgname == p.pkgbase] + if same_pkgs: + return same_pkgs[0] + + return PackageStandin(self.packages[0]) + + def find_signoffs(self, all_signoffs): + '''Look through a list of Signoff objects for ones matching this + particular group and store them on the object.''' + for s in all_signoffs: + if s.pkgbase != self.pkgbase: + continue + if self.version and not s.full_version == self.version: + continue + if s.arch_id == self.arch.id and s.repo_id == self.repo.id: + self.signoffs.add(s) + + def find_specification(self, specifications): + for spec in specifications: + if spec.pkgbase != self.pkgbase: + continue + if self.version and not spec.full_version == self.version: + continue + if spec.arch_id == self.arch.id and spec.repo_id == self.repo.id: + self.specification = spec + self.default_spec = False + return + + def approved(self): + return approved_by_signoffs(self.signoffs, self.specification) + + @property + def completed(self): + return sum(1 for s in self.signoffs if not s.revoked) + + @property + def required(self): + return self.specification.required + + def user_signed_off(self, user=None): + '''Did a given user signoff on this package? user can be passed as an + argument, or attached to the group object itself so this can be called + from a template.''' + if user is None: + user = self.user + return user in (s.user for s in self.signoffs if not s.revoked) + + def __unicode__(self): + return u'%s-%s (%s): %d' % ( + self.pkgbase, self.version, self.arch, len(self.signoffs)) + + +def signoffs_id_query(model, repos): + sql = """ +SELECT DISTINCT s.id + FROM %s s + JOIN packages p ON ( + s.pkgbase = p.pkgbase + AND s.pkgver = p.pkgver + AND s.pkgrel = p.pkgrel + AND s.epoch = p.epoch + AND s.arch_id = p.arch_id + AND s.repo_id = p.repo_id + ) + WHERE p.repo_id IN (%s) + AND s.repo_id IN (%s) + """ + cursor = connection.cursor() + # query pre-process- fill in table name and placeholders for IN + repo_sql = ','.join(['%s' for _ in repos]) + sql = sql % (model._meta.db_table, repo_sql, repo_sql) + repo_ids = [r.pk for r in repos] + # repo_ids are needed twice, so double the array + cursor.execute(sql, repo_ids * 2) + + results = cursor.fetchall() + return [row[0] for row in results] + + +def get_current_signoffs(repos): + '''Returns a list of signoff objects for the given repos.''' + to_fetch = signoffs_id_query(Signoff, repos) + return Signoff.objects.select_related('user').in_bulk(to_fetch).values() + + +def get_current_specifications(repos): + '''Returns a list of signoff specification objects for the given repos.''' + to_fetch = signoffs_id_query(SignoffSpecification, repos) + return SignoffSpecification.objects.select_related('arch').in_bulk( + to_fetch).values() + + +def get_target_repo_map(repos): + sql = """ +SELECT DISTINCT p1.pkgbase, r.name + FROM packages p1 + JOIN repos r ON p1.repo_id = r.id + JOIN packages p2 ON p1.pkgbase = p2.pkgbase + WHERE r.staging = %s + AND r.testing = %s + AND p2.repo_id IN ( + """ + sql += ','.join(['%s' for _ in repos]) + sql += ")" + + params = [False, False] + params.extend(r.pk for r in repos) + + cursor = connection.cursor() + cursor.execute(sql, params) + return dict(cursor.fetchall()) + + +def get_signoff_groups(repos=None, user=None): + if repos is None: + repos = Repo.objects.filter(testing=True) + repo_ids = [r.pk for r in repos] + + test_pkgs = Package.objects.select_related( + 'arch', 'repo', 'packager').filter(repo__in=repo_ids) + packages = test_pkgs.order_by('pkgname') + packages = attach_maintainers(packages) + + # Filter by user if asked to do so + if user is not None: + packages = [p for p in packages if user == p.packager + or user in p.maintainers] + + # Collect all pkgbase values in testing repos + pkgtorepo = get_target_repo_map(repos) + + # Collect all possible signoffs and specifications for these packages + signoffs = get_current_signoffs(repos) + specs = get_current_specifications(repos) + + same_pkgbase_key = lambda x: (x.repo.name, x.arch.name, x.pkgbase) + grouped = groupby_preserve_order(packages, same_pkgbase_key) + signoff_groups = [] + for group in grouped: + signoff_group = PackageSignoffGroup(group) + signoff_group.target_repo = pkgtorepo.get(signoff_group.pkgbase, + "Unknown") + signoff_group.find_signoffs(signoffs) + signoff_group.find_specification(specs) + signoff_groups.append(signoff_group) + + return signoff_groups + + +class PackageJSONEncoder(DjangoJSONEncoder): + pkg_attributes = ['pkgname', 'pkgbase', 'repo', 'arch', 'pkgver', + 'pkgrel', 'epoch', 'pkgdesc', 'url', 'filename', 'compressed_size', + 'installed_size', 'build_date', 'last_update', 'flag_date', + 'maintainers', 'packager'] + pkg_list_attributes = ['groups', 'licenses', 'conflicts', + 'provides', 'replaces', 'depends'] + + def default(self, obj): + if hasattr(obj, '__iter__'): + # mainly for queryset serialization + return list(obj) + if isinstance(obj, Package): + data = {attr: getattr(obj, attr) for attr in self.pkg_attributes} + for attr in self.pkg_list_attributes: + data[attr] = getattr(obj, attr).all() + return data + if isinstance(obj, PackageFile): + filename = obj.filename or '' + return obj.directory + filename + if isinstance(obj, (Repo, Arch)): + return obj.name.lower() + if isinstance(obj, (PackageGroup, License)): + return obj.name + if isinstance(obj, (Depend, Conflict, Provision, Replacement)): + return unicode(obj) + elif isinstance(obj, User): + return obj.username + return super(PackageJSONEncoder, self).default(obj) + # vim: set ts=4 sw=4 et: diff --git a/packages/views.py b/packages/views.py deleted file mode 100644 index 4cc4cc2f..00000000 --- a/packages/views.py +++ /dev/null @@ -1,379 +0,0 @@ -from django import forms -from django.contrib import messages -from django.core.mail import send_mail -from django.template import loader, Context, RequestContext -from django.http import HttpResponse, Http404 -from django.shortcuts import get_object_or_404, redirect -from django.contrib.auth.models import User -from django.contrib.auth.decorators import permission_required -from django.contrib.admin.widgets import AdminDateWidget -from django.views.decorators.cache import never_cache -from django.views.decorators.vary import vary_on_headers -from django.views.generic import list_detail -from django.views.generic.simple import direct_to_template -from django.db.models import Q - -from datetime import datetime -import string - -from main.models import Package, PackageFile -from main.models import Arch, Repo, Signoff -from main.utils import make_choice -from mirrors.models import MirrorUrl -from .models import PackageRelation -from .utils import get_group_info, get_differences_info - -def opensearch(request): - if request.is_secure(): - domain = "https://%s" % request.META['HTTP_HOST'] - else: - domain = "http://%s" % request.META['HTTP_HOST'] - - return direct_to_template(request, 'packages/opensearch.xml', - {'domain': domain}, - mimetype='application/opensearchdescription+xml') - -@permission_required('main.change_package') -def update(request): - ids = request.POST.getlist('pkgid') - mode = None - if request.POST.has_key('adopt'): - mode = 'adopt' - if request.POST.has_key('disown'): - mode = 'disown' - - if mode: - repos = request.user.userprofile_user.all()[0].allowed_repos.all() - pkgs = Package.objects.filter(id__in=ids, repo__in=repos) - disallowed_pkgs = Package.objects.filter(id__in=ids).exclude( - repo__in=repos) - count = 0 - for pkg in pkgs: - maints = pkg.maintainers - if mode == 'adopt' and request.user not in maints: - prel = PackageRelation(pkgbase=pkg.pkgbase, - user=request.user, - type=PackageRelation.MAINTAINER) - count += 1 - prel.save() - elif mode == 'disown' and request.user in maints: - rels = PackageRelation.objects.filter(pkgbase=pkg.pkgbase, - user=request.user) - count += rels.count() - rels.delete() - - messages.info(request, "%d base packages %sed." % (count, mode)) - if disallowed_pkgs: - messages.warning(request, - "You do not have permission to %s: %s" % ( - mode, ' '.join([p.pkgname for p in disallowed_pkgs]) - )) - else: - messages.error(request, "Are you trying to adopt or disown?") - return redirect('/packages/') - -def details(request, name='', repo='', arch=''): - if all([name, repo, arch]): - pkg = get_object_or_404(Package, - pkgname=name, repo__name__iexact=repo, arch__name=arch) - return direct_to_template(request, 'packages/details.html', {'pkg': pkg, }) - else: - return redirect("/packages/?arch=%s&repo=%s&q=%s" % ( - arch.lower(), repo.title(), name)) - -def groups(request): - grps = get_group_info() - return direct_to_template(request, 'packages/groups.html', {'groups': grps}) - -def group_details(request, arch, name): - arch = get_object_or_404(Arch, name=arch) - arches = [ arch ] - arches.extend(Arch.objects.filter(agnostic=True)) - pkgs = Package.objects.filter(packagegroup__name=name, - arch__in=arches) - pkgs = pkgs.order_by('pkgname') - if len(pkgs) == 0: - raise Http404 - context = { - 'groupname': name, - 'arch': arch, - 'packages': pkgs, - } - return direct_to_template(request, 'packages/group_details.html', context) - -def getmaintainer(request, name, repo, arch): - "Returns the maintainers as plaintext." - - pkg = get_object_or_404(Package, - pkgname=name, repo__name__iexact=repo, arch__name=arch) - names = [m.username for m in pkg.maintainers] - - return HttpResponse(str('\n'.join(names)), mimetype='text/plain') - -class PackageSearchForm(forms.Form): - repo = forms.ChoiceField(required=False) - arch = forms.ChoiceField(required=False) - q = forms.CharField(required=False) - maintainer = forms.ChoiceField(required=False) - last_update = forms.DateField(required=False, widget=AdminDateWidget(), - label='Last Updated After') - flagged = forms.ChoiceField( - choices=[('', 'All')] + make_choice(['Flagged', 'Not Flagged']), - required=False) - limit = forms.ChoiceField( - choices=make_choice([50, 100, 250]) + [('all', 'All')], - required=False, - initial=50) - - def clean_limit(self): - limit = self.cleaned_data['limit'] - if limit == 'all': - limit = None - elif limit: - try: - limit = int(limit) - except: - raise forms.ValidationError("Should be an integer") - else: - limit = 50 - return limit - - - def __init__(self, *args, **kwargs): - super(PackageSearchForm, self).__init__(*args, **kwargs) - self.fields['repo'].choices = [('', 'All')] + make_choice( - [repo.name for repo in Repo.objects.all()]) - self.fields['arch'].choices = [('', 'All')] + make_choice( - [arch.name for arch in Arch.objects.all()]) - self.fields['q'].widget.attrs.update({"size": "30"}) - maints = User.objects.filter(is_active=True).order_by('username') - self.fields['maintainer'].choices = \ - [('', 'All'), ('orphan', 'Orphan')] + \ - [(m.username, m.get_full_name()) for m in maints] - -def search(request, page=None): - current_query = '?' - limit = 50 - packages = Package.objects.select_related('arch', 'repo') - - if request.GET: - current_query += request.GET.urlencode() - form = PackageSearchForm(data=request.GET) - if form.is_valid(): - if form.cleaned_data['repo']: - packages = packages.filter( - repo__name=form.cleaned_data['repo']) - - if form.cleaned_data['arch']: - packages = packages.filter( - arch__name=form.cleaned_data['arch']) - - if form.cleaned_data['maintainer'] == 'orphan': - inner_q = PackageRelation.objects.all().values('pkgbase') - packages = packages.exclude(pkgbase__in=inner_q) - elif form.cleaned_data['maintainer']: - inner_q = PackageRelation.objects.filter(user__username=form.cleaned_data['maintainer']).values('pkgbase') - packages = packages.filter(pkgbase__in=inner_q) - - if form.cleaned_data['flagged'] == 'Flagged': - packages=packages.filter(flag_date__isnull=False) - elif form.cleaned_data['flagged'] == 'Not Flagged': - packages = packages.filter(flag_date__isnull=True) - - if form.cleaned_data['q']: - query = form.cleaned_data['q'] - q = Q(pkgname__icontains=query) | Q(pkgdesc__icontains=query) - packages = packages.filter(q) - if form.cleaned_data['last_update']: - lu = form.cleaned_data['last_update'] - packages = packages.filter(last_update__gte= - datetime(lu.year, lu.month, lu.day, 0, 0)) - limit = form.cleaned_data['limit'] - else: - form = PackageSearchForm() - - page_dict = {'search_form': form, - 'current_query': current_query - } - if packages.count() == 1: - return redirect(packages[0]) - - allowed_sort = ["arch", "repo", "pkgname", "last_update"] - allowed_sort += ["-" + s for s in allowed_sort] - sort = request.GET.get('sort', None) - # TODO: sorting by multiple fields makes using a DB index much harder - if sort in allowed_sort: - packages = packages.order_by( - request.GET['sort'], 'repo', 'arch', 'pkgname') - page_dict['sort'] = sort - else: - packages = packages.order_by('repo', 'arch', '-last_update', 'pkgname') - - return list_detail.object_list(request, packages, - template_name="packages/search.html", - page=page, - paginate_by=limit, - template_object_name="package", - extra_context=page_dict) - -@vary_on_headers('X-Requested-With') -def files(request, name='', repo='', arch=''): - pkg = get_object_or_404(Package, - pkgname=name, repo__name__iexact=repo, arch__name=arch) - fileslist = PackageFile.objects.filter(pkg=pkg).order_by('path') - template = 'packages/files.html' - if request.is_ajax(): - template = 'packages/files-list.html' - return direct_to_template(request, template, - {'pkg':pkg, 'files':fileslist}) - -@permission_required('main.change_package') -def unflag(request, name='', repo='', arch=''): - pkg = get_object_or_404(Package, - pkgname=name, repo__name__iexact=repo, arch__name=arch) - pkg.flag_date = None - pkg.save() - return redirect(pkg) - -@permission_required('main.change_package') -@never_cache -def signoffs(request): - packages = Package.objects.select_related('arch', 'repo', 'signoffs').filter(repo__testing=True).order_by("pkgname") - package_list = [] - - q_pkgname = Package.objects.filter(repo__testing=True).values('pkgname').distinct().query - package_repos = Package.objects.values('pkgname', 'repo__name').exclude(repo__testing=True).filter(pkgname__in=q_pkgname) - pkgtorepo = dict() - for pr in package_repos: - pkgtorepo[pr['pkgname']] = pr['repo__name'] - - for package in packages: - if package.pkgname in pkgtorepo: - repo = pkgtorepo[package.pkgname] - else: - repo = "Unknown" - package_list.append((package, repo)) - return direct_to_template(request, 'packages/signoffs.html', - {'packages': package_list}) - -@permission_required('main.change_package') -@never_cache -def signoff_package(request, arch, pkgname): - pkg = get_object_or_404(Package, - arch__name=arch, - pkgname=pkgname, - repo__testing=True) - - signoff, created = Signoff.objects.get_or_create( - pkg=pkg, - pkgver=pkg.pkgver, - pkgrel=pkg.pkgrel, - packager=request.user) - - if created: - messages.info(request, - "You have successfully signed off for %s on %s." % \ - (pkg.pkgname, pkg.arch)) - else: - messages.warning(request, - "You have already signed off for %s on %s." % \ - (pkg.pkgname, pkg.arch)) - return signoffs(request) - -def flaghelp(request): - return direct_to_template(request, 'packages/flaghelp.html') - -class FlagForm(forms.Form): - email = forms.EmailField(label='* E-mail Address') - usermessage = forms.CharField(label='Message To Dev', - widget=forms.Textarea, required=False) - # The field below is used to filter out bots that blindly fill out all input elements - website = forms.CharField(label='', - widget=forms.TextInput(attrs={'style': 'display:none;'}), - required=False) - -@never_cache -def flag(request, name='', repo='', arch=''): - pkg = get_object_or_404(Package, - pkgname=name, repo__name__iexact=repo, arch__name=arch) - context = {'pkg': pkg} - if pkg.flag_date is not None: - # already flagged. do nothing. - return direct_to_template(request, 'packages/flagged.html', context) - - if request.POST: - form = FlagForm(request.POST) - if form.is_valid() and form.cleaned_data['website'] == '': - # find all packages from (hopefully) the same PKGBUILD - pkgs = Package.objects.filter( - pkgbase=pkg.pkgbase, repo__testing=pkg.repo.testing) - pkgs.update(flag_date=datetime.now()) - - maints = pkg.maintainers - if not maints: - toemail = ['arch-notifications@archlinux.org'] - subject = 'Orphan %s package [%s] marked out-of-date' % \ - (pkg.repo.name, pkg.pkgname) - else: - toemail = [] - subject = '%s package [%s] marked out-of-date' % \ - (pkg.repo.name, pkg.pkgname) - for maint in maints: - if maint.get_profile().notify == True: - toemail.append(maint.email) - - if toemail: - # send notification email to the maintainer - t = loader.get_template('packages/outofdate.txt') - c = Context({ - 'email': form.cleaned_data['email'], - 'message': form.cleaned_data['usermessage'], - 'pkg': pkg, - 'weburl': pkg.get_full_url(), - }) - send_mail(subject, - t.render(c), - 'Arch Website Notification <nobody@archlinux.org>', - toemail, - fail_silently=True) - - context['confirmed'] = True - else: - form = FlagForm() - - context['form'] = form - - return direct_to_template(request, 'packages/flag.html', context) - -def download(request, name='', repo='', arch=''): - pkg = get_object_or_404(Package, - pkgname=name, repo__name__iexact=repo, arch__name=arch) - mirrorurl = MirrorUrl.objects.filter(mirror__country='Any', - mirror__public=True, mirror__active=True, - protocol__protocol__iexact='HTTP')[0] - arch = pkg.arch.name - if pkg.arch.agnostic: - # grab the first non-any arch to fake the download path - arch = Arch.objects.exclude(agnostic=True)[0].name - details = { - 'host': mirrorurl.url, - 'arch': arch, - 'repo': pkg.repo.name.lower(), - 'file': pkg.filename, - } - url = string.Template('${host}${repo}/os/${arch}/${file}').substitute(details) - return redirect(url) - -def arch_differences(request): - # TODO: we have some hardcoded magic here with respect to the arches. - arch_a = Arch.objects.get(name='i686') - arch_b = Arch.objects.get(name='x86_64') - differences = get_differences_info(arch_a, arch_b) - context = { - 'arch_a': arch_a, - 'arch_b': arch_b, - 'differences': differences, - } - return direct_to_template(request, 'packages/differences.html', context) - -# vim: set ts=4 sw=4 et: diff --git a/packages/views/__init__.py b/packages/views/__init__.py new file mode 100644 index 00000000..84ca37f2 --- /dev/null +++ b/packages/views/__init__.py @@ -0,0 +1,153 @@ +import hashlib +import json + +from django.contrib import messages +from django.contrib.auth.decorators import permission_required +from django.contrib.auth.models import User +from django.core.cache import cache +from django.db.models import Q +from django.http import HttpResponse +from django.shortcuts import redirect, render +from django.views.decorators.cache import cache_control +from django.views.decorators.http import require_safe, require_POST + +from main.models import Package, Arch +from ..models import PackageRelation +from ..utils import (get_differences_info, + multilib_differences, get_wrong_permissions) + +# make other views available from this same package +from .display import (details, groups, group_details, files, details_json, + files_json, download) +from .flag import flaghelp, flag, flag_confirmed, unflag, unflag_all +from .search import search_json +from .signoff import signoffs, signoff_package, signoff_options, signoffs_json + + +@require_safe +@cache_control(public=True, max_age=86400) +def opensearch(request): + domain = "%s://%s" % (request.scheme, request.META.get('HTTP_HOST')) + + return render(request, 'packages/opensearch.xml', + {'domain': domain}, + content_type='application/opensearchdescription+xml') + + +@require_safe +@cache_control(public=True, max_age=613) +def opensearch_suggest(request): + search_term = request.GET.get('q', '') + if search_term == '': + return HttpResponse('', content_type='application/x-suggestions+json') + + cache_key = 'opensearch:packages:' + \ + hashlib.md5(search_term.encode('utf-8')).hexdigest() + to_json = cache.get(cache_key, None) + if to_json is None: + q = Q(pkgname__startswith=search_term) + lookup = search_term.lower() + if search_term != lookup: + # package names are lowercase by convention, so include that in + # search if original wasn't lowercase already + q |= Q(pkgname__startswith=lookup) + names = Package.objects.filter(q).values_list( + 'pkgname', flat=True).order_by('pkgname').distinct()[:10] + results = [search_term, list(names)] + to_json = json.dumps(results, ensure_ascii=False) + cache.set(cache_key, to_json, 613) + return HttpResponse(to_json, content_type='application/x-suggestions+json') + + +@permission_required('main.change_package') +@require_POST +def update(request): + ids = request.POST.getlist('pkgid') + count = 0 + + if request.POST.has_key('adopt'): + repos = request.user.userprofile.allowed_repos.all() + pkgs = Package.objects.filter(id__in=ids, repo__in=repos) + disallowed_pkgs = Package.objects.filter(id__in=ids).exclude( + repo__in=repos) + + if disallowed_pkgs: + messages.warning(request, + "You do not have permission to adopt: %s." % ( + ' '.join([p.pkgname for p in disallowed_pkgs]) + )) + + for pkg in pkgs: + if request.user not in pkg.maintainers: + prel = PackageRelation(pkgbase=pkg.pkgbase, + user=request.user, + type=PackageRelation.MAINTAINER) + count += 1 + prel.save() + + messages.info(request, "%d base packages adopted." % count) + + elif request.POST.has_key('disown'): + # allow disowning regardless of allowed repos, helps things like + # [community] -> [extra] moves + for pkg in Package.objects.filter(id__in=ids): + if request.user in pkg.maintainers: + rels = PackageRelation.objects.filter(pkgbase=pkg.pkgbase, + user=request.user, + type=PackageRelation.MAINTAINER) + count += rels.count() + rels.delete() + + messages.info(request, "%d base packages disowned." % count) + + else: + messages.error(request, "Are you trying to adopt or disown?") + return redirect('/packages/') + + +def arch_differences(request): + # TODO: we have some hardcoded magic here with respect to the arches. + arch_a = Arch.objects.get(name=request.GET.get('arch_a', 'i686')) + arch_b = Arch.objects.get(name=request.GET.get('arch_b', 'x86_64')) + arch_c = Arch.objects.get(name=request.GET.get('arch_c', 'armv7h')) + differences = get_differences_info(arch_a, arch_b) + multilib_diffs = multilib_differences() + context = { + 'arch_a': arch_a, + 'arch_b': arch_b, + 'arch_c': arch_c, + 'differences': differences, + 'arches': Arch.objects.filter(agnostic=False), + 'multilib_differences': multilib_diffs + } + return render(request, 'packages/differences.html', context) + +@permission_required('main.change_package') +def stale_relations(request): + relations = PackageRelation.objects.select_related('user') + pkgbases = Package.objects.all().values('pkgbase') + + inactive_user = relations.filter(user__is_active=False) + missing_pkgbase = relations.exclude( + pkgbase__in=pkgbases).order_by('pkgbase') + wrong_permissions = get_wrong_permissions() + + context = { + 'inactive_user': inactive_user, + 'missing_pkgbase': missing_pkgbase, + 'wrong_permissions': wrong_permissions, + } + return render(request, 'packages/stale_relations.html', context) + +@permission_required('packages.delete_packagerelation') +@require_POST +def stale_relations_update(request): + ids = set(request.POST.getlist('relation_id')) + + if ids: + PackageRelation.objects.filter(id__in=ids).delete() + + messages.info(request, "%d package relations deleted." % len(ids)) + return redirect('/packages/stale_relations/') + +# vim: set ts=4 sw=4 et: diff --git a/packages/views/display.py b/packages/views/display.py new file mode 100644 index 00000000..021c7ed8 --- /dev/null +++ b/packages/views/display.py @@ -0,0 +1,235 @@ +import datetime +import json +from urllib import urlencode + +from django.http import HttpResponse, Http404 +from django.shortcuts import get_object_or_404, redirect, render +from django.utils.timezone import now + +from main.models import Package, PackageFile, Arch, Repo +from main.utils import empty_response +from mirrors.utils import get_mirror_url_for_download +from ..models import Update +from ..utils import get_group_info, PackageJSONEncoder + + +def arch_plus_agnostic(arch): + arches = [ arch ] + arches.extend(Arch.objects.filter(agnostic=True).order_by()) + return arches + + +def split_package_details(request, name, repo, arch): + '''Check if we have a split package (e.g. pkgbase) value matching this + name. If so, we can show a listing page for the entire set of packages.''' + arches = arch_plus_agnostic(arch) + pkgs = Package.objects.normal().filter(pkgbase=name, + repo__testing=repo.testing, repo__staging=repo.staging, + arch__in=arches).order_by('pkgname') + if len(pkgs) == 0: + return None + # we have packages, but ensure at least one is in the given repo + if not any(True for pkg in pkgs if pkg.repo == repo): + return None + context = { + 'list_title': 'Split Package Details', + 'name': name, + 'arch': arch, + 'packages': pkgs, + } + return render(request, 'packages/packages_list.html', context) + + +CUTOFF = datetime.timedelta(days=60) + + +def recently_removed_package(request, name, repo, arch, cutoff=CUTOFF): + '''Check our packages update table to see if this package has existed in + this repo before. If so, we can show a 410 Gone page and point the + requester in the right direction.''' + arches = arch_plus_agnostic(arch) + match = Update.objects.select_related('arch', 'repo').filter( + pkgname=name, repo=repo, arch__in=arches) + if cutoff is not None: + when = now() - cutoff + match = match.filter(created__gte=when) + try: + update = match.latest() + elsewhere = update.elsewhere() + if len(elsewhere) == 0: + elsewhere = update.replacements() + if len(elsewhere) == 1: + return redirect(elsewhere[0]) + context = { + 'update': update, + 'elsewhere': elsewhere, + 'name': name, + 'version': update.old_version, + 'arch': arch, + 'repo': repo, + } + return render(request, 'packages/removed.html', context, status=410) + except Update.DoesNotExist: + return None + + +def replaced_package(request, name, repo, arch): + '''Check our package replacements to see if this is a package we used to + have but no longer do.''' + match = Package.objects.filter(replaces__name=name, repo=repo, arch=arch) + if len(match) == 1: + return redirect(match[0], permanent=True) + elif len(match) > 1: + context = { + 'elsewhere': match, + 'name': name, + 'version': '', + 'arch': arch, + 'repo': repo, + } + return render(request, 'packages/removed.html', context, status=410) + return None + + +def redirect_agnostic(request, name, repo, arch): + '''For arch='any' packages, we can issue a redirect to them if we have a + single non-ambiguous option by changing the arch to match any arch-agnostic + package.''' + if not arch.agnostic: + # limit to 2 results, we only need to know whether there is anything + # except only one matching result + pkgs = Package.objects.select_related( + 'arch', 'repo', 'packager').filter(pkgname=name, + repo=repo, arch__agnostic=True)[:2] + if len(pkgs) == 1: + return redirect(pkgs[0], permanent=True) + return None + + +def redirect_to_search(request, name, repo, arch): + if request.GET.get('q'): + name = request.GET.get('q') + pkg_data = [ + ('arch', arch.lower()), + ('repo', repo.lower()), + ('q', name), + ] + # only include non-blank values in the query we generate + pkg_data = [(x, y.encode('utf-8')) for x, y in pkg_data if y] + return redirect("/packages/?%s" % urlencode(pkg_data)) + + +def details(request, name='', repo='', arch=''): + if all([name, repo, arch]): + arch_obj = get_object_or_404(Arch, name=arch) + repo_obj = get_object_or_404(Repo, name__iexact=repo) + try: + pkg = Package.objects.select_related( + 'arch', 'repo', 'packager').get(pkgname=name, + repo=repo_obj, arch=arch_obj) + if request.method == 'HEAD': + return empty_response() + return render(request, 'packages/details.html', {'pkg': pkg}) + except Package.DoesNotExist: + # attempt a variety of fallback options before 404ing + options = (redirect_agnostic, split_package_details, + recently_removed_package, replaced_package) + for method in options: + ret = method(request, name, repo_obj, arch_obj) + if ret: + return ret + # we've tried everything at this point, nothing to see + raise Http404 + else: + return redirect_to_search(request, name, repo, arch) + + +def groups(request, arch=None): + arches = [] + if arch: + get_object_or_404(Arch, name=arch, agnostic=False) + arches.append(arch) + grps = get_group_info(arches) + context = { + 'groups': grps, + 'arch': arch, + } + return render(request, 'packages/groups.html', context) + + +def group_details(request, arch, name): + arch = get_object_or_404(Arch, name=arch) + arches = arch_plus_agnostic(arch) + pkgs = Package.objects.normal().filter( + groups__name=name, arch__in=arches).order_by('pkgname') + if len(pkgs) == 0: + raise Http404 + context = { + 'list_title': 'Group Details', + 'name': name, + 'arch': arch, + 'packages': pkgs, + } + return render(request, 'packages/packages_list.html', context) + + +def files(request, name, repo, arch): + pkg = get_object_or_404(Package.objects.normal(), + pkgname=name, repo__name__iexact=repo, arch__name=arch) + # files are inserted in sorted order, so preserve that + fileslist = PackageFile.objects.filter(pkg=pkg).order_by('id') + dir_count = sum(1 for f in fileslist if f.is_directory) + files_count = len(fileslist) - dir_count + context = { + 'pkg': pkg, + 'files': fileslist, + 'files_count': files_count, + 'dir_count': dir_count, + } + template = 'packages/files.html' + return render(request, template, context) + + +def details_json(request, name, repo, arch): + pkg = get_object_or_404(Package.objects.normal(), + pkgname=name, repo__name__iexact=repo, arch__name=arch) + to_json = json.dumps(pkg, ensure_ascii=False, cls=PackageJSONEncoder) + return HttpResponse(to_json, content_type='application/json') + + +def files_json(request, name, repo, arch): + pkg = get_object_or_404(Package.objects.normal(), + pkgname=name, repo__name__iexact=repo, arch__name=arch) + # files are inserted in sorted order, so preserve that + fileslist = PackageFile.objects.filter(pkg=pkg).order_by('id') + dir_count = sum(1 for f in fileslist if f.is_directory) + files_count = len(fileslist) - dir_count + data = { + 'pkgname': pkg.pkgname, + 'repo': pkg.repo.name.lower(), + 'arch': pkg.arch.name.lower(), + 'pkg_last_update': pkg.last_update, + 'files_last_update': pkg.files_last_update, + 'files_count': files_count, + 'dir_count': dir_count, + 'files': fileslist, + } + to_json = json.dumps(data, ensure_ascii=False, cls=PackageJSONEncoder) + return HttpResponse(to_json, content_type='application/json') + + +def download(request, name, repo, arch): + pkg = get_object_or_404(Package.objects.normal(), + pkgname=name, repo__name__iexact=repo, arch__name=arch) + url = get_mirror_url_for_download() + if not url: + raise Http404 + arch = pkg.arch.name + if pkg.arch.agnostic: + # grab the first non-any arch to fake the download path + arch = Arch.objects.exclude(agnostic=True)[0].name + url = '{host}{repo}/os/{arch}/{filename}'.format(host=url.url, + repo=pkg.repo.name.lower(), arch=arch, filename=pkg.filename) + return redirect(url) + +# vim: set ts=4 sw=4 et: diff --git a/packages/views/flag.py b/packages/views/flag.py new file mode 100644 index 00000000..c6936ac4 --- /dev/null +++ b/packages/views/flag.py @@ -0,0 +1,178 @@ +import re + +from django import forms +from django.conf import settings +from django.contrib.auth.decorators import permission_required +from django.core.mail import EmailMessage +from django.db import transaction +from django.db.models import Q +from django.shortcuts import get_object_or_404, redirect, render +from django.template import loader, Context +from django.utils.timezone import now +from django.views.decorators.cache import cache_page, never_cache + +from ..models import FlagRequest +from main.models import Package + + +class FlagForm(forms.Form): + email = forms.EmailField(label='E-mail Address') + message = forms.CharField(label='Message To Developer', + widget=forms.Textarea) + # The field below is used to filter out bots that blindly fill out all + # input elements + website = forms.CharField(label='', + widget=forms.TextInput(attrs={'style': 'display:none;'}), + required=False) + + def __init__(self, *args, **kwargs): + # we remove the 'email' field if this form is being shown to a + # logged-in user, e.g., a developer. + auth = kwargs.pop('authenticated', False) + super(FlagForm, self).__init__(*args, **kwargs) + if auth: + del self.fields['email'] + + def clean_message(self): + data = self.cleaned_data['message'] + # make sure the message isn't garbage (only punctuation or whitespace) + # and ensure a certain minimum length + if re.match(r'^[^0-9A-Za-z]+$', data) or len(data) < 3: + raise forms.ValidationError( + "Enter a valid and useful out-of-date message.") + return data + + +@cache_page(3600) +def flaghelp(request): + return render(request, 'packages/flaghelp.html') + + +@never_cache +def flag(request, name, repo, arch): + pkg = get_object_or_404(Package.objects.normal(), + pkgname=name, repo__name__iexact=repo, arch__name=arch) + if pkg.flag_date is not None: + # already flagged. do nothing. + return render(request, 'packages/flagged.html', {'pkg': pkg}) + # find all packages from (hopefully) the same PKGBUILD + pkgs = Package.objects.normal().filter( + pkgbase=pkg.pkgbase, flag_date__isnull=True, + repo__testing=pkg.repo.testing, + repo__staging=pkg.repo.staging).filter( + Q(arch__name='mips64el') | Q(repo__name='Libre') | Q(repo__name='Pcr')).order_by( + 'pkgname', 'repo__name', 'arch__name') + + authenticated = request.user.is_authenticated() + + if request.POST: + form = FlagForm(request.POST, authenticated=authenticated) + if form.is_valid() and form.cleaned_data['website'] == '': + # save the package list for later use + flagged_pkgs = list(pkgs) + + # find a common version if there is one available to store + versions = set((pkg.pkgver, pkg.pkgrel, pkg.epoch) + for pkg in flagged_pkgs) + if len(versions) == 1: + version = versions.pop() + else: + version = ('', '', 0) + + message = form.cleaned_data['message'] + ip_addr = request.META.get('REMOTE_ADDR') + if authenticated: + email = request.user.email + else: + email = form.cleaned_data['email'] + + @transaction.atomic + def perform_updates(): + current_time = now() + pkgs.update(flag_date=current_time) + # store our flag request + flag_request = FlagRequest(created=current_time, + user_email=email, message=message, + ip_address=ip_addr, pkgbase=pkg.pkgbase, + repo=pkg.repo, pkgver=version[0], pkgrel=version[1], + epoch=version[2], num_packages=len(flagged_pkgs)) + if authenticated: + flag_request.user = request.user + flag_request.save() + + perform_updates() + + maints = pkg.maintainers + if not maints: + toemail = settings.NOTIFICATIONS + subject = 'Orphan %s package [%s] marked out-of-date' % \ + (pkg.repo.name, pkg.pkgname) + else: + toemail = [] + subject = '%s package [%s] marked out-of-date' % \ + (pkg.repo.name, pkg.pkgname) + for maint in maints: + if maint.userprofile.notify is True: + toemail.append(maint.email) + + if toemail: + # send notification email to the maintainers + tmpl = loader.get_template('packages/outofdate.txt') + ctx = Context({ + 'email': email, + 'message': message, + 'pkg': pkg, + 'packages': flagged_pkgs, + }) + msg = EmailMessage(subject, + tmpl.render(ctx), + settings.BRANDING_EMAIL, + toemail, + headers={"Reply-To": email } + ) + msg.send(fail_silently=True) + + return redirect('package-flag-confirmed', name=name, repo=repo, + arch=arch) + else: + form = FlagForm(authenticated=authenticated) + + context = { + 'package': pkg, + 'packages': pkgs, + 'form': form + } + return render(request, 'packages/flag.html', context) + +def flag_confirmed(request, name, repo, arch): + pkg = get_object_or_404(Package, + pkgname=name, repo__name__iexact=repo, arch__name=arch) + pkgs = Package.objects.normal().filter( + pkgbase=pkg.pkgbase, flag_date=pkg.flag_date, + repo__testing=pkg.repo.testing, + repo__staging=pkg.repo.staging).order_by( + 'pkgname', 'repo__name', 'arch__name') + + context = {'package': pkg, 'packages': pkgs} + + return render(request, 'packages/flag_confirmed.html', context) + +@permission_required('main.change_package') +def unflag(request, name, repo, arch): + pkg = get_object_or_404(Package.objects.normal(), + pkgname=name, repo__name__iexact=repo, arch__name=arch) + pkg.flag_date = None + pkg.save() + return redirect(pkg) + +@permission_required('main.change_package') +def unflag_all(request, name, repo, arch): + pkg = get_object_or_404(Package.objects.normal(), + pkgname=name, repo__name__iexact=repo, arch__name=arch) + # find all packages from (hopefully) the same PKGBUILD + pkgs = Package.objects.filter(pkgbase=pkg.pkgbase, + repo__testing=pkg.repo.testing, repo__staging=pkg.repo.staging) + pkgs.update(flag_date=None) + return redirect(pkg) + +# vim: set ts=4 sw=4 et: diff --git a/packages/views/search.py b/packages/views/search.py new file mode 100644 index 00000000..6e892251 --- /dev/null +++ b/packages/views/search.py @@ -0,0 +1,172 @@ +import json + +from django import forms +from django.contrib.auth.models import User +from django.db.models import Q +from django.http import HttpResponse +from django.views.generic import ListView + +from devel.models import UserProfile +from main.models import Package, Arch, Repo +from main.utils import empty_response, make_choice +from ..models import PackageRelation +from ..utils import attach_maintainers, PackageJSONEncoder + + +class PackageSearchForm(forms.Form): + repo = forms.MultipleChoiceField(required=False) + arch = forms.MultipleChoiceField(required=False) + name = forms.CharField(required=False) + desc = forms.CharField(required=False) + q = forms.CharField(required=False) + sort = forms.CharField(required=False, widget=forms.HiddenInput()) + maintainer = forms.ChoiceField(required=False) + packager = forms.ChoiceField(required=False) + flagged = forms.ChoiceField( + choices=[('', 'All')] + make_choice(['Flagged', 'Not Flagged']), + required=False) + + def __init__(self, *args, **kwargs): + show_staging = kwargs.pop('show_staging', False) + super(PackageSearchForm, self).__init__(*args, **kwargs) + repos = Repo.objects.all() + if not show_staging: + repos = repos.filter(staging=False) + self.fields['repo'].choices = make_choice( + [repo.name for repo in repos]) + self.fields['arch'].choices = make_choice( + [arch.name for arch in Arch.objects.all()]) + self.fields['q'].widget.attrs.update({"size": "30"}) + + profile_ids = UserProfile.allowed_repos.through.objects.values('userprofile_id') + people = User.objects.filter( + is_active=True, userprofile__id__in=profile_ids).order_by( + 'first_name', 'last_name') + maintainers = [('', 'All'), ('orphan', 'Orphan')] + \ + [(p.username, p.get_full_name()) for p in people] + packagers = [('', 'All'), ('unknown', 'Unknown')] + \ + [(p.username, p.get_full_name()) for p in people] + + self.fields['maintainer'].choices = maintainers + self.fields['packager'].choices = packagers + + def exact_matches(self): + # only do exact match search if 'q' is sole parameter + if self.changed_data != ['q']: + return [] + return Package.objects.normal().filter(pkgname=self.cleaned_data['q']) + + +def parse_form(form, packages): + if form.cleaned_data['repo']: + packages = packages.filter( + repo__name__in=form.cleaned_data['repo']) + + if form.cleaned_data['arch']: + packages = packages.filter( + arch__name__in=form.cleaned_data['arch']) + + if form.cleaned_data['maintainer'] == 'orphan': + inner_q = PackageRelation.objects.all().values('pkgbase') + packages = packages.exclude(pkgbase__in=inner_q) + elif form.cleaned_data['maintainer']: + inner_q = PackageRelation.objects.filter( + user__username=form.cleaned_data['maintainer']).values('pkgbase') + packages = packages.filter(pkgbase__in=inner_q) + + if form.cleaned_data['packager'] == 'unknown': + packages = packages.filter(packager__isnull=True) + elif form.cleaned_data['packager']: + packages = packages.filter( + packager__username=form.cleaned_data['packager']) + + if form.cleaned_data['flagged'] == 'Flagged': + packages = packages.filter(flag_date__isnull=False) + elif form.cleaned_data['flagged'] == 'Not Flagged': + packages = packages.filter(flag_date__isnull=True) + + if form.cleaned_data['name']: + name = form.cleaned_data['name'] + packages = packages.filter(pkgname=name) + + if form.cleaned_data['desc']: + desc = form.cleaned_data['desc'] + packages = packages.filter(pkgdesc__icontains=desc) + + if form.cleaned_data['q']: + query = form.cleaned_data['q'] + q = Q(pkgname__icontains=query) | Q(pkgdesc__icontains=query) + packages = packages.filter(q) + + return packages + + +class SearchListView(ListView): + template_name = "packages/search.html" + paginate_by = 100 + + sort_fields = ("arch", "repo", "pkgname", "pkgbase", "compressed_size", + "installed_size", "build_date", "last_update", "flag_date") + allowed_sort = list(sort_fields) + ["-" + s for s in sort_fields] + + def get(self, request, *args, **kwargs): + if request.method == 'HEAD': + return empty_response() + self.form = PackageSearchForm(data=request.GET, + show_staging=self.request.user.is_authenticated()) + return super(SearchListView, self).get(request, *args, **kwargs) + + def get_queryset(self): + packages = Package.objects.normal() + if not self.request.user.is_authenticated(): + packages = packages.filter(repo__staging=False) + if self.form.is_valid(): + packages = parse_form(self.form, packages) + sort = self.form.cleaned_data['sort'] + if sort in self.allowed_sort: + packages = packages.order_by(sort) + else: + packages = packages.order_by('pkgname') + return packages + + # Form had errors so don't return any results + return Package.objects.none() + + def get_context_data(self, **kwargs): + context = super(SearchListView, self).get_context_data(**kwargs) + query_params = self.request.GET.copy() + query_params.pop('page', None) + context['current_query'] = query_params.urlencode() + context['search_form'] = self.form + return context + + +def search_json(request): + limit = 250 + + container = { + 'version': 2, + 'limit': limit, + 'valid': False, + 'results': [], + } + + if request.GET: + form = PackageSearchForm(data=request.GET, + show_staging=request.user.is_authenticated()) + if form.is_valid(): + packages = Package.objects.select_related('arch', 'repo', + 'packager') + if not request.user.is_authenticated(): + packages = packages.filter(repo__staging=False) + packages = parse_form(form, packages)[:limit] + packages = packages.prefetch_related('groups', 'licenses', + 'conflicts', 'provides', 'replaces', 'depends') + attach_maintainers(packages) + container['results'] = packages + container['valid'] = True + + to_json = json.dumps(container, ensure_ascii=False, cls=PackageJSONEncoder) + return HttpResponse(to_json, content_type='application/json') + +# vim: set ts=4 sw=4 et: diff --git a/packages/views/signoff.py b/packages/views/signoff.py new file mode 100644 index 00000000..fcc6de45 --- /dev/null +++ b/packages/views/signoff.py @@ -0,0 +1,187 @@ +import json +from operator import attrgetter + +from django import forms +from django.contrib.auth.decorators import permission_required +from django.contrib.auth.models import User +from django.core.serializers.json import DjangoJSONEncoder +from django.db import transaction +from django.http import HttpResponse, Http404 +from django.shortcuts import get_list_or_404, redirect, render +from django.utils.timezone import now +from django.views.decorators.cache import never_cache + +from main.models import Package, Arch, Repo +from ..models import SignoffSpecification, Signoff +from ..utils import (get_signoff_groups, approved_by_signoffs, + PackageSignoffGroup) + +@permission_required('main.change_package') +def signoffs(request): + signoff_groups = sorted(get_signoff_groups(), key=attrgetter('pkgbase')) + for group in signoff_groups: + group.user = request.user + + context = { + 'signoff_groups': signoff_groups, + 'arches': Arch.objects.all(), + 'repo_names': sorted({g.target_repo for g in signoff_groups}), + } + return render(request, 'packages/signoffs.html', context) + +@permission_required('main.change_package') +@never_cache +def signoff_package(request, name, repo, arch, revoke=False): + packages = get_list_or_404(Package, pkgbase=name, + arch__name=arch, repo__name__iexact=repo, repo__testing=True) + package = packages[0] + + spec = SignoffSpecification.objects.get_or_default_from_package(package) + + if revoke: + try: + signoff = Signoff.objects.get_from_package( + package, request.user, False) + except Signoff.DoesNotExist: + raise Http404 + signoff.revoked = now() + signoff.save(update_fields=('revoked',)) + created = False + else: + # ensure we should even be accepting signoffs + if spec.known_bad or not spec.enabled: + return render(request, '403.html', status=403) + signoff, created = Signoff.objects.get_or_create_from_package( + package, request.user) + + all_signoffs = Signoff.objects.for_package(package) + + if request.is_ajax(): + data = { + 'created': created, + 'revoked': bool(signoff.revoked), + 'approved': approved_by_signoffs(all_signoffs, spec), + 'required': spec.required, + 'enabled': spec.enabled, + 'known_bad': spec.known_bad, + 'user': str(request.user), + } + return HttpResponse(json.dumps(data, ensure_ascii=False), + content_type='application/json') + + return redirect('package-signoffs') + +class SignoffOptionsForm(forms.ModelForm): + apply_all = forms.BooleanField(required=False, + help_text="Apply these options to all architectures?") + + class Meta: + model = SignoffSpecification + fields = ('required', 'enabled', 'known_bad', 'comments') + +def _signoff_options_all(request, name, repo): + seen_ids = set() + with transaction.atomic(): + # find or create a specification for all architectures, then + # graft the form data onto them + packages = Package.objects.filter(pkgbase=name, + repo__name__iexact=repo, repo__testing=True) + for package in packages: + try: + spec = SignoffSpecification.objects.get_from_package(package) + if spec.pk in seen_ids: + continue + except SignoffSpecification.DoesNotExist: + spec = SignoffSpecification(pkgbase=package.pkgbase, + pkgver=package.pkgver, pkgrel=package.pkgrel, + epoch=package.epoch, arch=package.arch, + repo=package.repo) + + if spec.user is None: + spec.user = request.user + + form = SignoffOptionsForm(request.POST, instance=spec) + if form.is_valid(): + form.save() + seen_ids.add(form.instance.pk) + +@permission_required('main.change_package') +@never_cache +def signoff_options(request, name, repo, arch): + packages = get_list_or_404(Package, pkgbase=name, + arch__name=arch, repo__name__iexact=repo, repo__testing=True) + package = packages[0] + + if request.user != package.packager and \ + request.user not in package.maintainers: + return render(request, '403.html', status=403) + + try: + spec = SignoffSpecification.objects.get_from_package(package) + except SignoffSpecification.DoesNotExist: + # create a fake one, but don't save it just yet + spec = SignoffSpecification(pkgbase=package.pkgbase, + pkgver=package.pkgver, pkgrel=package.pkgrel, + epoch=package.epoch, arch=package.arch, repo=package.repo) + + if spec.user is None: + spec.user = request.user + + if request.POST: + form = SignoffOptionsForm(request.POST, instance=spec) + if form.is_valid(): + if form.cleaned_data['apply_all']: + _signoff_options_all(request, name, repo) + else: + form.save() + return redirect('package-signoffs') + else: + form = SignoffOptionsForm(instance=spec) + + context = { + 'packages': packages, + 'package': package, + 'form': form, + } + return render(request, 'packages/signoff_options.html', context) + +class SignoffJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle all serialization of all classes + related to signoffs.''' + signoff_group_attrs = ['arch', 'last_update', 'maintainers', 'packager', + 'pkgbase', 'repo', 'signoffs', 'target_repo', 'version'] + signoff_spec_attrs = ['required', 'enabled', 'known_bad', 'comments'] + signoff_attrs = ['user', 'created', 'revoked'] + + def default(self, obj): + if isinstance(obj, PackageSignoffGroup): + data = {attr: getattr(obj, attr) + for attr in self.signoff_group_attrs} + data['pkgnames'] = [p.pkgname for p in obj.packages] + data['package_count'] = len(obj.packages) + data['approved'] = obj.approved() + data.update((attr, getattr(obj.specification, attr)) + for attr in self.signoff_spec_attrs) + return data + elif isinstance(obj, Signoff): + return {attr: getattr(obj, attr) for attr in self.signoff_attrs} + elif isinstance(obj, Arch) or isinstance(obj, Repo): + return unicode(obj) + elif isinstance(obj, User): + return obj.username + elif isinstance(obj, set): + return list(obj) + return super(SignoffJSONEncoder, self).default(obj) + +@permission_required('main.change_package') +def signoffs_json(request): + signoff_groups = sorted(get_signoff_groups(), key=attrgetter('pkgbase')) + data = { + 'version': 2, + 'signoff_groups': signoff_groups, + } + to_json = json.dumps(data, ensure_ascii=False, cls=SignoffJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + +# vim: set ts=4 sw=4 et: |