summaryrefslogtreecommitdiff
path: root/devel
diff options
context:
space:
mode:
authorLuke Shumaker <LukeShu@sbcglobal.net>2014-05-25 19:34:21 -0400
committerLuke Shumaker <LukeShu@sbcglobal.net>2014-05-25 19:34:21 -0400
commit1a1fd4a3d6aa7f6b2338f7751f5f97a30a059a04 (patch)
treef608874663d24f19ecf175ec556f2a330d47a726 /devel
parent747a562ddca10e29dd003e8b3c59f8f384d1bc8d (diff)
parentcd7222047b41ac3efa389a22fdd22148d03c1d61 (diff)
Merge tag 'release_2014-03-22' into archweb-generic
Fix long filenames; local storage for filter prefs Conflicts: templates/public/download.html
Diffstat (limited to 'devel')
-rw-r--r--devel/models.py2
-rw-r--r--devel/reports.py198
-rw-r--r--devel/views.py141
3 files changed, 217 insertions, 124 deletions
diff --git a/devel/models.py b/devel/models.py
index 44bbc66e..5c4d4fe7 100644
--- a/devel/models.py
+++ b/devel/models.py
@@ -4,7 +4,7 @@ import pytz
from django.db import models
from django.db.models.signals import pre_save
from django.contrib.auth.models import User
-from django_countries import CountryField
+from django_countries.fields import CountryField
from .fields import PGPKeyField
from main.utils import make_choice, set_created_field
diff --git a/devel/reports.py b/devel/reports.py
new file mode 100644
index 00000000..ca1a3321
--- /dev/null
+++ b/devel/reports.py
@@ -0,0 +1,198 @@
+from datetime import timedelta
+import pytz
+
+from django.db.models import F
+from django.template.defaultfilters import filesizeformat
+from django.utils.timezone import now
+
+from .models import DeveloperKey, UserProfile
+from main.models import PackageFile
+from packages.models import PackageRelation, Depend
+
+class DeveloperReport(object):
+ def __init__(self, slug, name, desc, packages_func,
+ names=None, attrs=None, personal=True):
+ self.slug = slug
+ self.name = name
+ self.description = desc
+ self.packages = packages_func
+ self.names = names
+ self.attrs = attrs
+ self.personal = personal
+
+
+def old(packages, username):
+ cutoff = now() - timedelta(days=365 * 2)
+ return packages.filter(
+ build_date__lt=cutoff).order_by('build_date')
+
+
+def outofdate(packages, username):
+ cutoff = now() - timedelta(days=30)
+ return packages.filter(
+ flag_date__lt=cutoff).order_by('flag_date')
+
+
+def big(packages, username):
+ cutoff = 50 * 1024 * 1024
+ packages = packages.filter(
+ compressed_size__gte=cutoff).order_by('-compressed_size')
+ # Format the compressed and installed sizes with MB/GB/etc suffixes
+ for package in packages:
+ package.compressed_size_pretty = filesizeformat(
+ package.compressed_size)
+ package.installed_size_pretty = filesizeformat(
+ package.installed_size)
+ return packages
+
+
+def badcompression(packages, username):
+ cutoff = 0.90 * F('installed_size')
+ packages = packages.filter(compressed_size__gt=0, installed_size__gt=0,
+ compressed_size__gte=cutoff).order_by('-compressed_size')
+
+ # Format the compressed and installed sizes with MB/GB/etc suffixes
+ for package in packages:
+ package.compressed_size_pretty = filesizeformat(
+ package.compressed_size)
+ package.installed_size_pretty = filesizeformat(
+ package.installed_size)
+ ratio = package.compressed_size / float(package.installed_size)
+ package.ratio = '%.3f' % ratio
+ package.compress_type = package.filename.split('.')[-1]
+
+ return packages
+
+
+def uncompressed_man(packages, username):
+ # checking for all '.0'...'.9' + '.n' extensions
+ bad_files = PackageFile.objects.filter(is_directory=False,
+ directory__contains='/man/',
+ filename__regex=r'\.[0-9n]').exclude(
+ filename__endswith='.gz').exclude(
+ filename__endswith='.xz').exclude(
+ filename__endswith='.bz2').exclude(
+ filename__endswith='.html')
+ if username:
+ pkg_ids = set(packages.values_list('id', flat=True))
+ bad_files = bad_files.filter(pkg__in=pkg_ids)
+ bad_files = bad_files.values_list(
+ 'pkg_id', flat=True).order_by().distinct()
+ return packages.filter(id__in=set(bad_files))
+
+
+def uncompressed_info(packages, username):
+ # we don't worry about looking for '*.info-1', etc., given that an
+ # uncompressed root page probably exists in the package anyway
+ bad_files = PackageFile.objects.filter(is_directory=False,
+ directory__endswith='/info/', filename__endswith='.info')
+ if username:
+ pkg_ids = set(packages.values_list('id', flat=True))
+ bad_files = bad_files.filter(pkg__in=pkg_ids)
+ bad_files = bad_files.values_list(
+ 'pkg_id', flat=True).order_by().distinct()
+ return packages.filter(id__in=set(bad_files))
+
+
+def unneeded_orphans(packages, username):
+ owned = PackageRelation.objects.all().values('pkgbase')
+ required = Depend.objects.all().values('name')
+ # The two separate calls to exclude is required to do the right thing
+ return packages.exclude(pkgbase__in=owned).exclude(
+ pkgname__in=required)
+
+
+def mismatched_signature(packages, username):
+ filtered = []
+ packages = packages.select_related(
+ 'arch', 'repo', 'packager').filter(signature_bytes__isnull=False)
+ known_keys = DeveloperKey.objects.select_related(
+ 'owner').filter(owner__isnull=False)
+ known_keys = {dk.key: dk for dk in known_keys}
+ for package in packages:
+ bad = False
+ sig = package.signature
+ dev_key = known_keys.get(sig.key_id, None)
+ if dev_key:
+ package.sig_by = dev_key.owner
+ if dev_key.owner_id != package.packager_id:
+ bad = True
+ else:
+ package.sig_by = sig.key_id
+ bad = True
+
+ if bad:
+ filtered.append(package)
+ return filtered
+
+
+def signature_time(packages, username):
+ cutoff = timedelta(hours=24)
+ filtered = []
+ packages = packages.select_related(
+ 'arch', 'repo', 'packager').filter(signature_bytes__isnull=False)
+ for package in packages:
+ sig = package.signature
+ sig_date = sig.creation_time.replace(tzinfo=pytz.utc)
+ package.sig_date = sig_date.date()
+ if sig_date > package.build_date + cutoff:
+ filtered.append(package)
+
+ return filtered
+
+
+REPORT_OLD = DeveloperReport('old', 'Old',
+ 'Packages last built more than two years ago', old)
+
+REPORT_OUTOFDATE = DeveloperReport('long-out-of-date', 'Long Out-of-date',
+ 'Packages marked out-of-date more than 30 days ago', outofdate)
+
+REPORT_BIG = DeveloperReport('big', 'Big',
+ 'Packages with compressed size > 50 MiB', big,
+ ['Compressed Size', 'Installed Size'],
+ ['compressed_size_pretty', 'installed_size_pretty'])
+
+REPORT_BADCOMPRESS = DeveloperReport('badcompression', 'Bad Compression',
+ 'Packages with a compression ratio of less than 10%', badcompression,
+ ['Compressed Size', 'Installed Size', 'Ratio', 'Type'],
+ ['compressed_size_pretty', 'installed_size_pretty','ratio', 'compress_type'])
+
+
+REPORT_MAN = DeveloperReport('uncompressed-man', 'Uncompressed Manpages',
+ 'Packages with uncompressed manpages', uncompressed_man)
+
+REPORT_INFO = DeveloperReport('uncompressed-info', 'Uncompressed Info Pages',
+ 'Packages with uncompressed info pages', uncompressed_info)
+
+REPORT_ORPHANS = DeveloperReport('unneeded-orphans', 'Unneeded Orphans',
+ 'Packages that have no maintainer and are not required by any '
+ + 'other package in any repository', unneeded_orphans,
+ personal=False)
+
+REPORT_SIGNATURE = DeveloperReport('mismatched-signature',
+ 'Mismatched Signatures',
+ 'Packages where the signing key is unknown or signer != packager',
+ mismatched_signature,
+ ['Signed By', 'Packager'],
+ ['sig_by', 'packager'])
+
+REPORT_SIG_TIME = DeveloperReport('signature-time', 'Signature Time',
+ 'Packages where the signature timestamp is more than 24 hours '
+ + 'after the build timestamp',
+ signature_time,
+ ['Signature Date', 'Packager'],
+ ['sig_date', 'packager'])
+
+
+def available_reports():
+ return (
+ REPORT_OLD,
+ REPORT_OUTOFDATE,
+ REPORT_BIG,
+ REPORT_BADCOMPRESS,
+ REPORT_MAN,
+ REPORT_INFO,
+ REPORT_ORPHANS,
+ REPORT_SIGNATURE,
+ REPORT_SIG_TIME,
+ )
diff --git a/devel/views.py b/devel/views.py
index 29954b51..972d0abb 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -1,6 +1,5 @@
from datetime import timedelta
import operator
-import pytz
import time
from django.http import HttpResponseRedirect
@@ -10,21 +9,21 @@ from django.contrib.admin.models import LogEntry, ADDITION
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.db import transaction
-from django.db.models import F, Count, Max
+from django.db.models import Count, Max
from django.http import Http404
from django.shortcuts import get_object_or_404, render
-from django.template.defaultfilters import filesizeformat
from django.views.decorators.cache import never_cache
from django.utils.encoding import force_unicode
from django.utils.http import http_date
from django.utils.timezone import now
from .forms import ProfileForm, UserProfileForm, NewUserForm
-from .models import DeveloperKey, UserProfile
-from main.models import Package, PackageFile
+from .models import UserProfile
+from .reports import available_reports
+from main.models import Package
from main.models import Arch, Repo
from news.models import News
-from packages.models import PackageRelation, Signoff, FlagRequest, Depend
+from packages.models import PackageRelation, Signoff, FlagRequest
from packages.utils import get_signoff_groups
from todolists.models import TodolistPackage
from todolists.utils import get_annotated_todolists
@@ -58,7 +57,8 @@ def index(request):
'todos': todolists,
'flagged': flagged,
'todopkgs': todopkgs,
- 'signoffs': signoffs
+ 'signoffs': signoffs,
+ 'reports': available_reports(),
}
return render(request, 'devel/index.html', page_dict)
@@ -180,10 +180,13 @@ def change_profile(request):
@login_required
def report(request, report_name, username=None):
- title = 'Developer Report'
- packages = Package.objects.normal()
- names = attrs = user = None
+ available = {report.slug: report for report in available_reports()}
+ report = available.get(report_name, None)
+ if report is None:
+ raise Http404
+ packages = Package.objects.normal()
+ user = None
if username:
user = get_object_or_404(User, username=username, is_active=True)
maintained = PackageRelation.objects.filter(user=user,
@@ -193,126 +196,18 @@ def report(request, report_name, username=None):
maints = User.objects.filter(id__in=PackageRelation.objects.filter(
type=PackageRelation.MAINTAINER).values('user'))
- if report_name == 'old':
- title = 'Packages last built more than one year ago'
- cutoff = now() - timedelta(days=365)
- packages = packages.filter(
- build_date__lt=cutoff).order_by('build_date')
- elif report_name == 'long-out-of-date':
- title = 'Packages marked out-of-date more than 90 days ago'
- cutoff = now() - timedelta(days=90)
- packages = packages.filter(
- flag_date__lt=cutoff).order_by('flag_date')
- elif report_name == 'big':
- title = 'Packages with compressed size > 50 MiB'
- cutoff = 50 * 1024 * 1024
- packages = packages.filter(
- compressed_size__gte=cutoff).order_by('-compressed_size')
- names = [ 'Compressed Size', 'Installed Size' ]
- attrs = [ 'compressed_size_pretty', 'installed_size_pretty' ]
- # Format the compressed and installed sizes with MB/GB/etc suffixes
- for package in packages:
- package.compressed_size_pretty = filesizeformat(
- package.compressed_size)
- package.installed_size_pretty = filesizeformat(
- package.installed_size)
- elif report_name == 'badcompression':
- title = 'Packages that have little need for compression'
- cutoff = 0.90 * F('installed_size')
- packages = packages.filter(compressed_size__gt=0, installed_size__gt=0,
- compressed_size__gte=cutoff).order_by('-compressed_size')
- names = [ 'Compressed Size', 'Installed Size', 'Ratio', 'Type' ]
- attrs = [ 'compressed_size_pretty', 'installed_size_pretty',
- 'ratio', 'compress_type' ]
- # Format the compressed and installed sizes with MB/GB/etc suffixes
- for package in packages:
- package.compressed_size_pretty = filesizeformat(
- package.compressed_size)
- package.installed_size_pretty = filesizeformat(
- package.installed_size)
- ratio = package.compressed_size / float(package.installed_size)
- package.ratio = '%.3f' % ratio
- package.compress_type = package.filename.split('.')[-1]
- elif report_name == 'uncompressed-man':
- title = 'Packages with uncompressed manpages'
- # checking for all '.0'...'.9' + '.n' extensions
- bad_files = PackageFile.objects.filter(is_directory=False,
- directory__contains='/man/',
- filename__regex=r'\.[0-9n]').exclude(
- filename__endswith='.gz').exclude(
- filename__endswith='.xz').exclude(
- filename__endswith='.bz2').exclude(
- filename__endswith='.html')
- if username:
- pkg_ids = set(packages.values_list('id', flat=True))
- bad_files = bad_files.filter(pkg__in=pkg_ids)
- bad_files = bad_files.values_list(
- 'pkg_id', flat=True).order_by().distinct()
- packages = packages.filter(id__in=set(bad_files))
- elif report_name == 'uncompressed-info':
- title = 'Packages with uncompressed infopages'
- # we don't worry about looking for '*.info-1', etc., given that an
- # uncompressed root page probably exists in the package anyway
- bad_files = PackageFile.objects.filter(is_directory=False,
- directory__endswith='/info/', filename__endswith='.info')
- if username:
- pkg_ids = set(packages.values_list('id', flat=True))
- bad_files = bad_files.filter(pkg__in=pkg_ids)
- bad_files = bad_files.values_list(
- 'pkg_id', flat=True).order_by().distinct()
- packages = packages.filter(id__in=set(bad_files))
- elif report_name == 'unneeded-orphans':
- title = 'Orphan packages required by no other packages'
- owned = PackageRelation.objects.all().values('pkgbase')
- required = Depend.objects.all().values('name')
- # The two separate calls to exclude is required to do the right thing
- packages = packages.exclude(pkgbase__in=owned).exclude(
- pkgname__in=required)
- elif report_name == 'mismatched-signature':
- title = 'Packages with mismatched signatures'
- names = [ 'Signature Date', 'Signed By', 'Packager' ]
- attrs = [ 'sig_date', 'sig_by', 'packager' ]
- cutoff = timedelta(hours=24)
- filtered = []
- packages = packages.select_related(
- 'arch', 'repo', 'packager').filter(signature_bytes__isnull=False)
- known_keys = DeveloperKey.objects.select_related(
- 'owner').filter(owner__isnull=False)
- known_keys = {dk.key: dk for dk in known_keys}
- for package in packages:
- bad = False
- sig = package.signature
- sig_date = sig.creation_time.replace(tzinfo=pytz.utc)
- package.sig_date = sig_date.date()
- dev_key = known_keys.get(sig.key_id, None)
- if dev_key:
- package.sig_by = dev_key.owner
- if dev_key.owner_id != package.packager_id:
- bad = True
- else:
- package.sig_by = sig.key_id
- bad = True
-
- if sig_date > package.build_date + cutoff:
- bad = True
-
- if bad:
- filtered.append(package)
- packages = filtered
- else:
- raise Http404
-
arches = {pkg.arch for pkg in packages}
repos = {pkg.repo for pkg in packages}
context = {
'all_maintainers': maints,
- 'title': title,
+ 'title': report.description,
+ 'report': report,
'maintainer': user,
- 'packages': packages,
+ 'packages': report.packages(packages, username),
'arches': sorted(arches),
'repos': sorted(repos),
- 'column_names': names,
- 'column_attrs': attrs,
+ 'column_names': report.names,
+ 'column_attrs': report.attrs,
}
return render(request, 'devel/packages.html', context)