From 7d08d59280dc4ddbe43f277e177bce683dd51188 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 7 Apr 2011 16:14:33 -0500 Subject: Show a few more fields in package admin Signed-off-by: Dan McGee --- main/admin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'main') diff --git a/main/admin.py b/main/admin.py index 45bc5ab2..f93838d3 100644 --- a/main/admin.py +++ b/main/admin.py @@ -19,7 +19,8 @@ class RepoAdmin(admin.ModelAdmin): search_fields = ('name',) class PackageAdmin(admin.ModelAdmin): - list_display = ('pkgname', 'repo', 'arch', 'last_update') + list_display = ('pkgname', 'full_version', 'repo', 'arch', 'packager', + 'last_update', 'build_date') list_filter = ('repo', 'arch') search_fields = ('pkgname',) -- cgit v1.2.3-54-g00ecf From 01db07bad844e17e084f650b6732647f77a91c5c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 7 Apr 2011 15:39:53 -0500 Subject: Use UTC datetime objects everywhere Rather than the twisted mix of local times and UTC times we currently have. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 6 +++--- feeds.py | 22 ++++------------------ main/models.py | 13 +++++++++++-- news/models.py | 2 +- packages/views.py | 2 +- 5 files changed, 20 insertions(+), 25 deletions(-) (limited to 'main') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index e26bb800..a8875c7e 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -315,7 +315,7 @@ def populate_files(dbpkg, repopkg, force=False): directory=dirname + '/', filename=filename) pkgfile.save(force_insert=True) - dbpkg.files_last_update = datetime.now() + dbpkg.files_last_update = datetime.utcnow() dbpkg.save() @transaction.commit_on_success @@ -374,7 +374,7 @@ def db_update(archname, reponame, pkgs, options): for p in [x for x in pkgs if x.name in in_sync_not_db]: logger.info("Adding package %s", p.name) pkg = Package(pkgname = p.name, arch = architecture, repo = repository) - populate_pkg(pkg, p, timestamp=datetime.now()) + populate_pkg(pkg, p, timestamp=datetime.utcnow()) # packages in database and not in syncdb (remove from database) in_db_not_sync = dbset - syncset @@ -398,7 +398,7 @@ def db_update(archname, reponame, pkgs, options): if not force: continue else: - timestamp = datetime.now() + timestamp = datetime.utcnow() if filesonly: logger.debug("Checking files for package %s in database", p.name) populate_files(dbp, p, force=force) diff --git a/feeds.py b/feeds.py index 7a2f2e40..0be12531 100644 --- a/feeds.py +++ b/feeds.py @@ -1,5 +1,4 @@ -import datetime -from decimal import Decimal, ROUND_HALF_DOWN +import pytz from django.contrib.sites.models import Site from django.contrib.syndication.views import Feed @@ -10,7 +9,7 @@ from django.views.decorators.http import condition from main.models import Arch, Repo, Package -from main.utils import CACHE_TIMEOUT, INVALIDATE_TIMEOUT +from main.utils import CACHE_TIMEOUT from main.utils import CACHE_PACKAGE_KEY, CACHE_NEWS_KEY from news.models import News @@ -32,17 +31,6 @@ def write_items(self, handler): super(GuidNotPermalinkFeed, self).write_items(handler) -def utc_offset(): - '''Calculate the UTC offset from local time. Useful for converting values - stored in local time to things like cache last modifed headers.''' - timediff = datetime.datetime.utcnow() - datetime.datetime.now() - secs = timediff.days * 86400 + timediff.seconds - # round to nearest minute - mins = Decimal(secs) / Decimal(60) - mins = mins.quantize(Decimal('0'), rounding=ROUND_HALF_DOWN) - return datetime.timedelta(minutes=int(mins)) - - def retrieve_package_latest(): # we could break this down based on the request url, but it would probably # cost us more in query time to do so. @@ -52,7 +40,6 @@ def retrieve_package_latest(): try: latest = Package.objects.values('last_update').latest( 'last_update')['last_update'] - latest = latest + utc_offset() # Using add means "don't overwrite anything in there". What could be in # there is an explicit None value that our refresh signal set, which # means we want to avoid race condition possibilities for a bit. @@ -132,7 +119,7 @@ def item_guid(self, item): date.strftime('%Y%m%d%H%M')) def item_pubdate(self, item): - return item.last_update + return item.last_update.replace(tzinfo=pytz.utc) def item_categories(self, item): return (item.repo.name, item.arch.name) @@ -145,7 +132,6 @@ def retrieve_news_latest(): try: latest = News.objects.values('last_modified').latest( 'last_modified')['last_modified'] - latest = latest + utc_offset() # same thoughts apply as in retrieve_package_latest cache.add(CACHE_NEWS_KEY, latest, CACHE_TIMEOUT) return latest @@ -184,7 +170,7 @@ def item_guid(self, item): return item.guid def item_pubdate(self, item): - return item.postdate + return item.postdate.replace(tzinfo=pytz.utc) def item_author_name(self, item): return item.author.get_full_name() diff --git a/main/models.py b/main/models.py index 4370fa24..8d34731f 100644 --- a/main/models.py +++ b/main/models.py @@ -5,6 +5,7 @@ from main.utils import cache_function, make_choice from packages.models import PackageRelation +from datetime import datetime from itertools import groupby import pytz @@ -351,7 +352,7 @@ class Todolist(models.Model): creator = models.ForeignKey(User) name = models.CharField(max_length=255) description = models.TextField() - date_added = models.DateTimeField(auto_now_add=True, db_index=True) + date_added = models.DateTimeField(db_index=True) objects = TodolistManager() def __unicode__(self): @@ -383,10 +384,18 @@ class Meta: db_table = 'todolist_pkgs' unique_together = (('list','pkg'),) +def set_todolist_fields(sender, **kwargs): + todolist = kwargs['instance'] + if not todolist.date_added: + todolist.date_added = datetime.utcnow() + # connect signals needed to keep cache in line with reality from main.utils import refresh_package_latest -from django.db.models.signals import post_save +from django.db.models.signals import pre_save, post_save + post_save.connect(refresh_package_latest, sender=Package, dispatch_uid="main.models") +pre_save.connect(set_todolist_fields, sender=Todolist, + dispatch_uid="main.models") # vim: set ts=4 sw=4 et: diff --git a/news/models.py b/news/models.py index c4fb136a..17d51de9 100644 --- a/news/models.py +++ b/news/models.py @@ -27,7 +27,7 @@ class Meta: def set_news_fields(sender, **kwargs): news = kwargs['instance'] - now = datetime.now() + now = datetime.utcnow() news.last_modified = now if not news.postdate: news.postdate = now diff --git a/packages/views.py b/packages/views.py index 2f614e34..1587563b 100644 --- a/packages/views.py +++ b/packages/views.py @@ -371,7 +371,7 @@ def flag(request, name, repo, arch): # find all packages from (hopefully) the same PKGBUILD pkgs = Package.objects.filter( pkgbase=pkg.pkgbase, repo__testing=pkg.repo.testing) - pkgs.update(flag_date=datetime.now()) + pkgs.update(flag_date=datetime.utcnow()) maints = pkg.maintainers if not maints: -- cgit v1.2.3-54-g00ecf From d7665959652171b93db5e084c6738e8e1773e7f0 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 7 Apr 2011 16:14:48 -0500 Subject: Add some migrations to convert database to UTC time This follows the earlier commit where we make sure any value going to or being pulled from the database is UTC. Signed-off-by: Dan McGee --- main/migrations/0047_utc_datetimes.py | 180 ++++++++++++++++++++++++++++++++++ news/migrations/0009_utc_datetimes.py | 85 ++++++++++++++++ 2 files changed, 265 insertions(+) create mode 100644 main/migrations/0047_utc_datetimes.py create mode 100644 news/migrations/0009_utc_datetimes.py (limited to 'main') diff --git a/main/migrations/0047_utc_datetimes.py b/main/migrations/0047_utc_datetimes.py new file mode 100644 index 00000000..83153b78 --- /dev/null +++ b/main/migrations/0047_utc_datetimes.py @@ -0,0 +1,180 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import DataMigration +from django.db import models +from django.utils.tzinfo import LocalTimezone + +def new_date(old_date, reverse=False): + if old_date is None: + return None + tz = LocalTimezone(old_date) + offset = tz.utcoffset(old_date) + if reverse: + offset = -offset + return old_date - offset + +class Migration(DataMigration): + + def forwards(self, orm): + all_pkgs = orm.Package.objects.all() + for package in all_pkgs: + # prevents full object updates + orm.Package.objects.filter(pk=package.pk).update( + last_update=new_date(package.last_update), + files_last_update=new_date(package.files_last_update), + flag_date=new_date(package.flag_date)) + # We could do todolists, but they just don't matter that much. + + def backwards(self, orm): + all_pkgs = orm.Package.objects.all() + for package in all_pkgs: + # prevents full object updates + orm.Package.objects.filter(pk=package.pk).update( + last_update=new_date(package.last_update, True), + files_last_update=new_date(package.files_last_update, True), + flag_date=new_date(package.flag_date, True)) + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'main.arch': { + 'Meta': {'ordering': "['name']", 'object_name': 'Arch', 'db_table': "'arches'"}, + 'agnostic': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'main.donor': { + 'Meta': {'ordering': "['name']", 'object_name': 'Donor', 'db_table': "'donors'"}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'visible': ('django.db.models.fields.BooleanField', [], {'default': 'True'}) + }, + 'main.package': { + 'Meta': {'ordering': "('pkgname',)", 'object_name': 'Package', 'db_table': "'packages'"}, + 'arch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'to': "orm['main.Arch']"}), + 'build_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'compressed_size': ('django.db.models.fields.BigIntegerField', [], {'null': 'True'}), + 'epoch': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'files_last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'flag_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'installed_size': ('django.db.models.fields.BigIntegerField', [], {'null': 'True'}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'packager': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True'}), + 'packager_str': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkgdesc': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), + 'pkgname': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'repo': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'to': "orm['main.Repo']"}), + 'url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}) + }, + 'main.packagedepend': { + 'Meta': {'object_name': 'PackageDepend', 'db_table': "'package_depends'"}, + 'depname': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'depvcmp': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}), + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'optional': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}) + }, + 'main.packagefile': { + 'Meta': {'object_name': 'PackageFile', 'db_table': "'package_files'"}, + 'directory': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_directory': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}) + }, + 'main.repo': { + 'Meta': {'ordering': "['name']", 'object_name': 'Repo', 'db_table': "'repos'"}, + 'bugs_project': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'staging': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'svn_root': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + 'main.signoff': { + 'Meta': {'object_name': 'Signoff'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'packager': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}), + 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + 'main.todolist': { + 'Meta': {'object_name': 'Todolist', 'db_table': "'todolists'"}, + 'creator': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}), + 'date_added': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + 'main.todolistpkg': { + 'Meta': {'unique_together': "(('list', 'pkg'),)", 'object_name': 'TodolistPkg', 'db_table': "'todolist_pkgs'"}, + 'complete': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'list': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Todolist']"}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}) + }, + 'main.userprofile': { + 'Meta': {'object_name': 'UserProfile', 'db_table': "'user_profiles'"}, + 'alias': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'allowed_repos': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['main.Repo']", 'symmetrical': 'False', 'blank': 'True'}), + 'favorite_distros': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interests': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'languages': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'location': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'notify': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'occupation': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'other_contact': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'picture': ('django.db.models.fields.files.FileField', [], {'default': "'devs/silhouette.png'", 'max_length': '100'}), + 'public_email': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'roles': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'time_zone': ('django.db.models.fields.CharField', [], {'default': "'UTC'", 'max_length': '100'}), + 'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'userprofile'", 'unique': 'True', 'to': "orm['auth.User']"}), + 'website': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'yob': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) + } + } + + complete_apps = ['main'] diff --git a/news/migrations/0009_utc_datetimes.py b/news/migrations/0009_utc_datetimes.py new file mode 100644 index 00000000..6cddf783 --- /dev/null +++ b/news/migrations/0009_utc_datetimes.py @@ -0,0 +1,85 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import DataMigration +from django.db import models +from django.utils.tzinfo import LocalTimezone + +def new_date(old_date, reverse=False): + if old_date is None: + return None + tz = LocalTimezone(old_date) + offset = tz.utcoffset(old_date) + if reverse: + offset = -offset + return old_date - offset + +class Migration(DataMigration): + + def forwards(self, orm): + all_news = orm.News.objects.all().defer('content') + for news in all_news: + # prevents full object updates + orm.News.objects.filter(pk=news.pk).update( + postdate=new_date(news.postdate), + last_modified=new_date(news.last_modified)) + + def backwards(self, orm): + all_news = orm.News.objects.all().defer('content') + for news in all_news: + # prevents full object updates + orm.News.objects.filter(pk=news.pk).update( + postdate=new_date(news.postdate, True), + last_modified=new_date(news.last_modified, True)) + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'news.news': { + 'Meta': {'ordering': "['-postdate']", 'object_name': 'News', 'db_table': "'news'"}, + 'author': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'news_author'", 'to': "orm['auth.User']"}), + 'content': ('django.db.models.fields.TextField', [], {}), + 'guid': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_modified': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'postdate': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + } + } + + complete_apps = ['news'] -- cgit v1.2.3-54-g00ecf From 77842a6c76095277b024505708bf528d455b9c89 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 7 Apr 2011 16:52:52 -0500 Subject: Consolidate caching black magic Get the stuff used to retrieve and refresh the latest date values all in the same place, and make it a bit more beautiful by refactoring it all into a common set of methods. Signed-off-by: Dan McGee --- feeds.py | 44 +++++--------------------------------------- main/models.py | 7 +++---- main/utils.py | 34 +++++++++++++++++++++++++--------- news/models.py | 4 ++-- 4 files changed, 35 insertions(+), 54 deletions(-) (limited to 'main') diff --git a/feeds.py b/feeds.py index 0be12531..269d0a38 100644 --- a/feeds.py +++ b/feeds.py @@ -2,15 +2,13 @@ from django.contrib.sites.models import Site from django.contrib.syndication.views import Feed -from django.core.cache import cache from django.db.models import Q from django.utils.feedgenerator import Rss201rev2Feed from django.utils.hashcompat import md5_constructor from django.views.decorators.http import condition +from main.utils import retrieve_latest from main.models import Arch, Repo, Package -from main.utils import CACHE_TIMEOUT -from main.utils import CACHE_PACKAGE_KEY, CACHE_NEWS_KEY from news.models import News def check_for_unique_id(f): @@ -31,32 +29,14 @@ def write_items(self, handler): super(GuidNotPermalinkFeed, self).write_items(handler) -def retrieve_package_latest(): - # we could break this down based on the request url, but it would probably - # cost us more in query time to do so. - latest = cache.get(CACHE_PACKAGE_KEY) - if latest: - return latest - try: - latest = Package.objects.values('last_update').latest( - 'last_update')['last_update'] - # Using add means "don't overwrite anything in there". What could be in - # there is an explicit None value that our refresh signal set, which - # means we want to avoid race condition possibilities for a bit. - cache.add(CACHE_PACKAGE_KEY, latest, CACHE_TIMEOUT) - return latest - except Package.DoesNotExist: - pass - return None - def package_etag(request, *args, **kwargs): - latest = retrieve_package_latest() + latest = retrieve_latest(Package) if latest: return md5_constructor(str(kwargs) + str(latest)).hexdigest() return None def package_last_modified(request, *args, **kwargs): - return retrieve_package_latest() + return retrieve_latest(Package) class PackageFeed(Feed): feed_type = GuidNotPermalinkFeed @@ -125,28 +105,14 @@ def item_categories(self, item): return (item.repo.name, item.arch.name) -def retrieve_news_latest(): - latest = cache.get(CACHE_NEWS_KEY) - if latest: - return latest - try: - latest = News.objects.values('last_modified').latest( - 'last_modified')['last_modified'] - # same thoughts apply as in retrieve_package_latest - cache.add(CACHE_NEWS_KEY, latest, CACHE_TIMEOUT) - return latest - except News.DoesNotExist: - pass - return None - def news_etag(request, *args, **kwargs): - latest = retrieve_news_latest() + latest = retrieve_latest(News) if latest: return md5_constructor(str(latest)).hexdigest() return None def news_last_modified(request, *args, **kwargs): - return retrieve_news_latest() + return retrieve_latest(News) class NewsFeed(Feed): feed_type = GuidNotPermalinkFeed diff --git a/main/models.py b/main/models.py index 8d34731f..38120134 100644 --- a/main/models.py +++ b/main/models.py @@ -128,8 +128,7 @@ class Package(models.Model): class Meta: db_table = 'packages' ordering = ('pkgname',) - #get_latest_by = 'last_update' - #ordering = ('-last_update',) + get_latest_by = 'last_update' def __unicode__(self): return self.pkgname @@ -390,10 +389,10 @@ def set_todolist_fields(sender, **kwargs): todolist.date_added = datetime.utcnow() # connect signals needed to keep cache in line with reality -from main.utils import refresh_package_latest +from main.utils import refresh_latest from django.db.models.signals import pre_save, post_save -post_save.connect(refresh_package_latest, sender=Package, +post_save.connect(refresh_latest, sender=Package, dispatch_uid="main.models") pre_save.connect(set_todolist_fields, sender=Todolist, dispatch_uid="main.models") diff --git a/main/utils.py b/main/utils.py index d7681cb6..12d12503 100644 --- a/main/utils.py +++ b/main/utils.py @@ -6,10 +6,8 @@ from django.utils.hashcompat import md5_constructor CACHE_TIMEOUT = 1800 -INVALIDATE_TIMEOUT = 15 - -CACHE_PACKAGE_KEY = 'cache_package_latest' -CACHE_NEWS_KEY = 'cache_news_latest' +INVALIDATE_TIMEOUT = 10 +CACHE_LATEST_PREFIX = 'cache_latest_' def cache_function_key(func, args, kwargs): raw = [func.__name__, func.__module__, args, kwargs] @@ -53,16 +51,34 @@ def clear_cache_function(func, args, kwargs): # and hoops otherwise. The only thing currently using these keys is the feed # caching stuff. -def refresh_package_latest(**kwargs): +def refresh_latest(**kwargs): + '''A post_save signal handler to clear out the cached latest value for a + given model.''' + cache_key = CACHE_LATEST_PREFIX + kwargs['sender'].__name__ # We could delete the value, but that could open a race condition # where the new data wouldn't have been committed yet by the calling # thread. Instead, explicitly set it to None for a short amount of time. # Hopefully by the time it expires we will have committed, and the cache # will be valid again. See "Scaling Django" by Mike Malone, slide 30. - cache.set(CACHE_PACKAGE_KEY, None, INVALIDATE_TIMEOUT) + cache.set(cache_key, None, INVALIDATE_TIMEOUT) -def refresh_news_latest(**kwargs): - # same thoughts apply as in refresh_package_latest - cache.set(CACHE_NEWS_KEY, None, INVALIDATE_TIMEOUT) +def retrieve_latest(sender): + # we could break this down based on the request url, but it would probably + # cost us more in query time to do so. + cache_key = CACHE_LATEST_PREFIX + sender.__name__ + latest = cache.get(cache_key) + if latest: + return latest + try: + latest_by = sender._meta.get_latest_by + latest = sender.objects.values(latest_by).latest()[latest_by] + # Using add means "don't overwrite anything in there". What could be in + # there is an explicit None value that our refresh signal set, which + # means we want to avoid race condition possibilities for a bit. + cache.add(cache_key, latest, CACHE_TIMEOUT) + return latest + except sender.DoesNotExist: + pass + return None # vim: set ts=4 sw=4 et: diff --git a/news/models.py b/news/models.py index 17d51de9..5e467515 100644 --- a/news/models.py +++ b/news/models.py @@ -36,10 +36,10 @@ def set_news_fields(sender, **kwargs): now.strftime('%Y-%m-%d'), news.get_absolute_url()) # connect signals needed to keep cache in line with reality -from main.utils import refresh_news_latest +from main.utils import refresh_latest from django.db.models.signals import pre_save, post_save -post_save.connect(refresh_news_latest, sender=News, +post_save.connect(refresh_latest, sender=News, dispatch_uid="news.models") pre_save.connect(set_news_fields, sender=News, dispatch_uid="news.models") -- cgit v1.2.3-54-g00ecf From 6fe08cd68901e698f4a0741e177354a45c753b46 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 29 Mar 2011 17:39:59 -0500 Subject: Utilize Django 1.3 'on_delete' feature on several foreign keys The most important one here is PROTECT to keep people from making bone-headed plays and deleting an Arch or Repo and every package along with it. We can use this in a few other places, as well as some carefully placed SET_NULL indicators. Note that nothing here pushes deletion responsibilities down to the database, although that will probably happen in a future commit. Signed-off-by: Dan McGee --- main/models.py | 11 +++++++---- mirrors/models.py | 4 ++-- news/models.py | 3 ++- 3 files changed, 11 insertions(+), 7 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index 38120134..772d85bb 100644 --- a/main/models.py +++ b/main/models.py @@ -104,8 +104,10 @@ class Meta: verbose_name_plural = 'repos' class Package(models.Model): - repo = models.ForeignKey(Repo, related_name="packages") - arch = models.ForeignKey(Arch, related_name="packages") + repo = models.ForeignKey(Repo, related_name="packages", + on_delete=models.PROTECT) + arch = models.ForeignKey(Arch, related_name="packages", + on_delete=models.PROTECT) pkgname = models.CharField(max_length=255, db_index=True) pkgbase = models.CharField(max_length=255, db_index=True) pkgver = models.CharField(max_length=255) @@ -121,7 +123,8 @@ class Package(models.Model): last_update = models.DateTimeField(null=True, blank=True) files_last_update = models.DateTimeField(null=True, blank=True) packager_str = models.CharField(max_length=255) - packager = models.ForeignKey(User, null=True) + packager = models.ForeignKey(User, null=True, + on_delete=models.SET_NULL) flag_date = models.DateTimeField(null=True) objects = PackageManager() @@ -348,7 +351,7 @@ class Meta: db_table = 'package_depends' class Todolist(models.Model): - creator = models.ForeignKey(User) + creator = models.ForeignKey(User, on_delete=models.PROTECT) name = models.CharField(max_length=255) description = models.TextField() date_added = models.DateTimeField(db_index=True) diff --git a/mirrors/models.py b/mirrors/models.py index dce17b5e..80808e2e 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -14,7 +14,7 @@ class Mirror(models.Model): name = models.CharField(max_length=255, unique=True) tier = models.SmallIntegerField(default=2, choices=TIER_CHOICES) - upstream = models.ForeignKey('self', null=True) + upstream = models.ForeignKey('self', null=True, on_delete=models.SET_NULL) country = models.CharField(max_length=255, db_index=True) admin_email = models.EmailField(max_length=255, blank=True) public = models.BooleanField(default=True) @@ -56,7 +56,7 @@ class Meta: class MirrorUrl(models.Model): url = models.CharField(max_length=255, unique=True) protocol = models.ForeignKey(MirrorProtocol, related_name="urls", - editable=False) + editable=False, on_delete=models.PROTECT) mirror = models.ForeignKey(Mirror, related_name="urls") has_ipv4 = models.BooleanField("IPv4 capable", default=True, editable=False) diff --git a/news/models.py b/news/models.py index 5e467515..33d958e0 100644 --- a/news/models.py +++ b/news/models.py @@ -6,7 +6,8 @@ class News(models.Model): slug = models.SlugField(max_length=255, unique=True) - author = models.ForeignKey(User, related_name='news_author') + author = models.ForeignKey(User, related_name='news_author', + on_delete=models.PROTECT) postdate = models.DateTimeField("post date", db_index=True) last_modified = models.DateTimeField(editable=False, db_index=True) title = models.CharField(max_length=255) -- cgit v1.2.3-54-g00ecf From d8022fd5720a8367a03bbff58668ed701a0bebcf Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 18 Apr 2011 23:00:30 -0500 Subject: Add a "Report a Bug" link We need Flyspray category data to make this more useful, and we can prefill the Subject and Category fields (along with putting it on the right project). Implements FS#23751. Signed-off-by: Dan McGee --- main/admin.py | 3 +- .../0048_auto__add_field_repo_bugs_category.py | 158 +++++++++++++++++++++ main/models.py | 11 ++ templates/packages/details.html | 1 + 4 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 main/migrations/0048_auto__add_field_repo_bugs_category.py (limited to 'main') diff --git a/main/admin.py b/main/admin.py index f93838d3..e86e5cab 100644 --- a/main/admin.py +++ b/main/admin.py @@ -14,7 +14,8 @@ class ArchAdmin(admin.ModelAdmin): search_fields = ('name',) class RepoAdmin(admin.ModelAdmin): - list_display = ('name', 'testing', 'staging', 'bugs_project', 'svn_root') + list_display = ('name', 'testing', 'staging', 'bugs_project', + 'bugs_category', 'svn_root') list_filter = ('testing', 'staging') search_fields = ('name',) diff --git a/main/migrations/0048_auto__add_field_repo_bugs_category.py b/main/migrations/0048_auto__add_field_repo_bugs_category.py new file mode 100644 index 00000000..30575126 --- /dev/null +++ b/main/migrations/0048_auto__add_field_repo_bugs_category.py @@ -0,0 +1,158 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.add_column('repos', 'bugs_category', self.gf('django.db.models.fields.SmallIntegerField')(default=0), keep_default=False) + + def backwards(self, orm): + db.delete_column('repos', 'bugs_category') + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'main.arch': { + 'Meta': {'ordering': "['name']", 'object_name': 'Arch', 'db_table': "'arches'"}, + 'agnostic': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'main.donor': { + 'Meta': {'ordering': "['name']", 'object_name': 'Donor', 'db_table': "'donors'"}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'visible': ('django.db.models.fields.BooleanField', [], {'default': 'True'}) + }, + 'main.package': { + 'Meta': {'ordering': "('pkgname',)", 'object_name': 'Package', 'db_table': "'packages'"}, + 'arch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'to': "orm['main.Arch']"}), + 'build_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'compressed_size': ('django.db.models.fields.BigIntegerField', [], {'null': 'True'}), + 'epoch': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'files_last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'flag_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'installed_size': ('django.db.models.fields.BigIntegerField', [], {'null': 'True'}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'packager': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True'}), + 'packager_str': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkgdesc': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), + 'pkgname': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'repo': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'to': "orm['main.Repo']"}), + 'url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}) + }, + 'main.packagedepend': { + 'Meta': {'object_name': 'PackageDepend', 'db_table': "'package_depends'"}, + 'depname': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'depvcmp': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}), + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'optional': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}) + }, + 'main.packagefile': { + 'Meta': {'object_name': 'PackageFile', 'db_table': "'package_files'"}, + 'directory': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_directory': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}) + }, + 'main.repo': { + 'Meta': {'ordering': "['name']", 'object_name': 'Repo', 'db_table': "'repos'"}, + 'bugs_category': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}), + 'bugs_project': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'staging': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'svn_root': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + 'main.signoff': { + 'Meta': {'object_name': 'Signoff'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'packager': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}), + 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + 'main.todolist': { + 'Meta': {'object_name': 'Todolist', 'db_table': "'todolists'"}, + 'creator': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}), + 'date_added': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + 'main.todolistpkg': { + 'Meta': {'unique_together': "(('list', 'pkg'),)", 'object_name': 'TodolistPkg', 'db_table': "'todolist_pkgs'"}, + 'complete': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'list': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Todolist']"}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}) + }, + 'main.userprofile': { + 'Meta': {'object_name': 'UserProfile', 'db_table': "'user_profiles'"}, + 'alias': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'allowed_repos': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['main.Repo']", 'symmetrical': 'False', 'blank': 'True'}), + 'favorite_distros': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interests': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'languages': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'location': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'notify': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'occupation': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'other_contact': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'picture': ('django.db.models.fields.files.FileField', [], {'default': "'devs/silhouette.png'", 'max_length': '100'}), + 'public_email': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'roles': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'time_zone': ('django.db.models.fields.CharField', [], {'default': "'UTC'", 'max_length': '100'}), + 'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'userprofile'", 'unique': 'True', 'to': "orm['auth.User']"}), + 'website': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'yob': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) + } + } + + complete_apps = ['main'] diff --git a/main/models.py b/main/models.py index 772d85bb..d84a6af3 100644 --- a/main/models.py +++ b/main/models.py @@ -8,6 +8,7 @@ from datetime import datetime from itertools import groupby import pytz +from urllib import urlencode class UserProfile(models.Model): notify = models.BooleanField( @@ -89,6 +90,8 @@ class Repo(models.Model): help_text="Is this repo meant for package staging?") bugs_project = models.SmallIntegerField(default=1, help_text="Flyspray project ID for this repository.") + bugs_category = models.SmallIntegerField(default=0, + help_text="Flyspray category ID for this repository.") svn_root = models.CharField(max_length=64, help_text="SVN root (e.g. path) for this repository.") @@ -294,6 +297,14 @@ def get_bugs_link(self): return "https://bugs.archlinux.org/?project=%d&string=%s" % \ (self.repo.bugs_project, self.pkgname) + def get_bug_report_link(self): + data = { + 'project': self.repo.bugs_project, + 'product_category': self.repo.bugs_category, + 'item_summary': '[%s]' % self.pkgname, + } + return "https://bugs.archlinux.org/newtask?%s" % urlencode(data) + def is_same_version(self, other): 'is this package similar, name and version-wise, to another' return self.pkgname == other.pkgname \ diff --git a/templates/packages/details.html b/templates/packages/details.html index 6c8aee3a..4c669892 100644 --- a/templates/packages/details.html +++ b/templates/packages/details.html @@ -18,6 +18,7 @@

Package Actions

  • SVN Entries ({{pkg.repo|lower}}-{{pkg.arch}})
  • SVN Entries (trunk)
  • Bug Reports
    • +
  • Report a Bug
    • {% if pkg.flag_date %}
  • Flagged out-of-date on {{ pkg.flag_date|date }}
    • {% with pkg.in_testing as tp %}{% if tp %} -- cgit v1.2.3-54-g00ecf From f1f01ecf0216441dd66f3bc6afc14fe104de291f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 18 Apr 2011 23:26:10 -0500 Subject: Reimplement links code as template tags These were starting to get a bit too much inside the model itself, and they don't really belong there as they are view layer concerns anyway. Signed-off-by: Dan McGee --- main/models.py | 24 --------------------- packages/templatetags/package_extras.py | 37 +++++++++++++++++++++++++++++++-- templates/packages/details.html | 8 +++---- 3 files changed, 39 insertions(+), 30 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index d84a6af3..e8f189e7 100644 --- a/main/models.py +++ b/main/models.py @@ -8,7 +8,6 @@ from datetime import datetime from itertools import groupby import pytz -from urllib import urlencode class UserProfile(models.Model): notify = models.BooleanField( @@ -282,29 +281,6 @@ def split_packages(self): return Package.objects.filter(arch__in=self.applicable_arches(), repo__testing=self.repo.testing, pkgbase=self.pkgbase).exclude(id=self.id) - def get_svn_link(self, svnpath): - linkbase = "http://projects.archlinux.org/svntogit/%s.git/tree/%s/%s/" - return linkbase % (self.repo.svn_root, self.pkgbase, svnpath) - - def get_arch_svn_link(self): - repo = self.repo.name.lower() - return self.get_svn_link("repos/%s-%s" % (repo, self.arch.name)) - - def get_trunk_svn_link(self): - return self.get_svn_link("trunk") - - def get_bugs_link(self): - return "https://bugs.archlinux.org/?project=%d&string=%s" % \ - (self.repo.bugs_project, self.pkgname) - - def get_bug_report_link(self): - data = { - 'project': self.repo.bugs_project, - 'product_category': self.repo.bugs_category, - 'item_summary': '[%s]' % self.pkgname, - } - return "https://bugs.archlinux.org/newtask?%s" % urlencode(data) - def is_same_version(self, other): 'is this package similar, name and version-wise, to another' return self.pkgname == other.pkgname \ diff --git a/packages/templatetags/package_extras.py b/packages/templatetags/package_extras.py index dd5b9347..e089b723 100644 --- a/packages/templatetags/package_extras.py +++ b/packages/templatetags/package_extras.py @@ -1,4 +1,4 @@ -import urllib +from urllib import urlencode, quote as urlquote try: from urlparse import parse_qs except ImportError: @@ -22,7 +22,7 @@ def render(self, context): qs['sort'] = ['-' + self.sortfield] else: qs['sort'] = [self.sortfield] - return urllib.urlencode(qs, True) + return urlencode(qs, True) @register.tag(name='buildsortqs') def do_buildsortqs(parser, token): @@ -48,4 +48,37 @@ def userpkgs(user): ) return '' + +def svn_link(package, svnpath): + '''Helper function for the two real SVN link methods.''' + parts = (package.repo.svn_root, package.pkgbase, svnpath) + linkbase = "http://projects.archlinux.org/svntogit/%s.git/tree/%s/%s/" + return linkbase % tuple(urlquote(part) for part in parts) + +@register.simple_tag +def svn_arch(package): + repo = package.repo.name.lower() + return svn_link(package, "repos/%s-%s" % (repo, package.arch.name)) + +@register.simple_tag +def svn_trunk(package): + return svn_link(package, "trunk") + +@register.simple_tag +def bugs_list(package): + data = { + 'project': package.repo.bugs_project, + 'string': package.pkgname, + } + return "https://bugs.archlinux.org/?%s" % urlencode(data) + +@register.simple_tag +def bug_report(package): + data = { + 'project': package.repo.bugs_project, + 'product_category': package.repo.bugs_category, + 'item_summary': '[%s]' % package.pkgname, + } + return "https://bugs.archlinux.org/newtask?%s" % urlencode(data) + # vim: set ts=4 sw=4 et: diff --git a/templates/packages/details.html b/templates/packages/details.html index 4c669892..1926abc2 100644 --- a/templates/packages/details.html +++ b/templates/packages/details.html @@ -15,10 +15,10 @@

    Package Details: {{ pkg.pkgname }} {{ pkg.full_version }}

    Package Actions

      -
    • SVN Entries ({{pkg.repo|lower}}-{{pkg.arch}})
      • -
    • SVN Entries (trunk)
      • -
    • Bug Reports
      • -
    • Report a Bug
      • +
    • SVN Entries ({{pkg.repo|lower}}-{{pkg.arch}})
      • +
    • SVN Entries (trunk)
      • +
    • Bug Reports
      • +
    • Report a Bug
      • {% if pkg.flag_date %}
    • Flagged out-of-date on {{ pkg.flag_date|date }}
      • {% with pkg.in_testing as tp %}{% if tp %} -- cgit v1.2.3-54-g00ecf From e6717510a0a7976fca1ccd3e5aaf1a16123a1ad4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 23 Apr 2011 14:29:29 -0500 Subject: Update repos fixture with new fields Signed-off-by: Dan McGee --- main/fixtures/repos.json | 58 ++++++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 22 deletions(-) (limited to 'main') diff --git a/main/fixtures/repos.json b/main/fixtures/repos.json index fae96f85..5fd918bb 100644 --- a/main/fixtures/repos.json +++ b/main/fixtures/repos.json @@ -3,70 +3,84 @@ "pk": 5, "model": "main.repo", "fields": { - "svn_root": "community", - "testing": false, + "bugs_category": 33, + "staging": false, "name": "Community", - "bugs_project": 5 + "bugs_project": 5, + "svn_root": "community", + "testing": false } }, { "pk": 6, "model": "main.repo", "fields": { - "svn_root": "community", - "testing": true, + "bugs_category": 41, + "staging": false, "name": "Community-Testing", - "bugs_project": 5 + "bugs_project": 5, + "svn_root": "community", + "testing": true } }, { "pk": 1, "model": "main.repo", "fields": { - "svn_root": "packages", - "testing": false, + "bugs_category": 31, + "staging": false, "name": "Core", - "bugs_project": 1 + "bugs_project": 1, + "svn_root": "packages", + "testing": false } }, { "pk": 2, "model": "main.repo", "fields": { - "svn_root": "packages", - "testing": false, + "bugs_category": 2, + "staging": false, "name": "Extra", - "bugs_project": 1 + "bugs_project": 1, + "svn_root": "packages", + "testing": false } }, { "pk": 7, "model": "main.repo", "fields": { - "svn_root": "community", - "testing": false, + "bugs_category": 46, + "staging": false, "name": "Multilib", - "bugs_project": 5 + "bugs_project": 5, + "svn_root": "community", + "testing": false } }, { "pk": 8, "model": "main.repo", "fields": { - "svn_root": "community", - "testing": true, + "bugs_category": 46, + "staging": false, "name": "Multilib-Testing", - "bugs_project": 5 + "bugs_project": 5, + "svn_root": "community", + "testing": true } }, { "pk": 3, "model": "main.repo", "fields": { - "svn_root": "packages", - "testing": true, + "bugs_category": 10, + "staging": false, "name": "Testing", - "bugs_project": 1 + "bugs_project": 1, + "svn_root": "packages", + "testing": true } } -] \ No newline at end of file +] -- cgit v1.2.3-54-g00ecf From 381e0a787205af530ae11bac1b1a17e567eecc84 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 25 Apr 2011 18:09:39 -0500 Subject: Developer reports This commit adds four initial developer reports that are hopefully useful to developers and packages in checking up on the state of things. They include: * big : the 100 biggest packages in the repos * old : packages built > 2 years ago * uncompressed-man : self-explanatory * uncompressed-info : self-explanatory There should obviously be some sort of index page to access all of these, so that will be coming soon. Signed-off-by: Dan McGee --- devel/urls.py | 9 ++++--- devel/views.py | 54 +++++++++++++++++++++++++++++++++++---- main/templatetags/attributes.py | 21 ++++++++++++++++ templates/devel/packages.html | 56 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 9 deletions(-) create mode 100644 main/templatetags/attributes.py create mode 100644 templates/devel/packages.html (limited to 'main') diff --git a/devel/urls.py b/devel/urls.py index 41be2b31..9bf50f45 100644 --- a/devel/urls.py +++ b/devel/urls.py @@ -1,12 +1,13 @@ from django.conf.urls.defaults import patterns urlpatterns = patterns('devel.views', - (r'^$', 'index'), + (r'^admin_log/$','admin_log'), + (r'^admin_log/(?P.*)/$','admin_log'), (r'^clock/$', 'clock'), - (r'^profile/$', 'change_profile'), + (r'^$', 'index'), (r'^newuser/$', 'new_user_form'), - (r'^admin_log/(?P.*)/$','admin_log'), - (r'^admin_log/$','admin_log'), + (r'^profile/$', 'change_profile'), + (r'^reports/(?P.*)/$', 'report'), ) # vim: set ts=4 sw=4 et: diff --git a/devel/views.py b/devel/views.py index b61e605f..01d54e6f 100644 --- a/devel/views.py +++ b/devel/views.py @@ -6,19 +6,22 @@ from django.contrib.sites.models import Site from django.core.mail import send_mail from django.db import transaction +from django.db.models import Q +from django.http import Http404 from django.shortcuts import get_object_or_404 from django.template import loader, Context from django.views.decorators.cache import never_cache from django.views.generic.simple import direct_to_template -from main.models import Package, TodolistPkg +from main.models import Package, PackageFile, TodolistPkg from main.models import Arch, Repo from main.models import UserProfile from packages.models import PackageRelation from todolists.utils import get_annotated_todolists from .utils import get_annotated_maintainers -import datetime +from datetime import datetime, timedelta +import operator import pytz import random from string import ascii_letters, digits @@ -26,7 +29,7 @@ @login_required @never_cache def index(request): - '''the Developer dashboard''' + '''the developer dashboard''' inner_q = PackageRelation.objects.filter(user=request.user).values('pkgbase') flagged = Package.objects.select_related('arch', 'repo').filter( flag_date__isnull=False, pkgbase__in=inner_q).order_by('pkgname') @@ -70,8 +73,8 @@ def clock(request): 'username').select_related('userprofile') # now annotate each dev object with their current time - now = datetime.datetime.now() - utc_now = datetime.datetime.utcnow().replace(tzinfo=pytz.utc) + now = datetime.now() + utc_now = datetime.utcnow().replace(tzinfo=pytz.utc) for dev in devs: # Work around https://bugs.launchpad.net/pytz/+bug/718673 timezone = str(dev.userprofile.time_zone) @@ -123,6 +126,47 @@ def change_profile(request): return direct_to_template(request, 'devel/profile.html', {'form': form, 'profile_form': profile_form}) +@login_required +def report(request, report): + title = "Developer Report" + packages = Package.objects.select_related('arch', 'repo') + names = attrs = None + if report == "old": + title = "Packages last built more than two years ago" + cutoff = datetime.now() - timedelta(days=730) + packages = packages.filter(build_date__lt=cutoff).order_by('build_date') + elif report == "big": + title = "100 largest compressed packages" + packages = packages.order_by('-compressed_size')[:100] + names = [ 'Compressed Size', 'Installed Size' ] + attrs = [ 'compressed_size', 'installed_size' ] + elif report == "uncompressed-man": + title = "Packages with uncompressed manpages" + # magic going on here! Checking for all '.1'...'.9' extensions + invalid_endings = [Q(filename__endswith='.%d' % n) for n in range(1,10)] + invalid_endings.append(Q(filename__endswith='.n')) + bad_files = PackageFile.objects.filter(Q(directory__contains='man') & ( + reduce(operator.or_, invalid_endings)) + ).values_list('pkg_id', flat=True).distinct() + packages = packages.filter(id__in=set(bad_files)) + elif report == "uncompressed-info": + title = "Packages with uncompressed infopages" + bad_files = PackageFile.objects.filter(directory__contains='/info', + filename__endswith='.info').values_list( + 'pkg_id', flat=True).distinct() + packages = packages.filter(id__in=set(bad_files)) + else: + raise Http404 + + context = { + 'title': title, + 'packages': packages, + 'column_names': names, + 'column_attrs': attrs, + } + return direct_to_template(request, 'devel/packages.html', context) + + class NewUserForm(forms.ModelForm): username = forms.CharField(max_length=30) private_email = forms.EmailField() diff --git a/main/templatetags/attributes.py b/main/templatetags/attributes.py new file mode 100644 index 00000000..bd4ccf3d --- /dev/null +++ b/main/templatetags/attributes.py @@ -0,0 +1,21 @@ +import re +from django import template +from django.conf import settings + +numeric_test = re.compile("^\d+$") +register = template.Library() + +def attribute(value, arg): + """Gets an attribute of an object dynamically from a string name""" + if hasattr(value, str(arg)): + return getattr(value, arg) + elif hasattr(value, 'has_key') and value.has_key(arg): + return value[arg] + elif numeric_test.match(str(arg)) and len(value) > int(arg): + return value[int(arg)] + else: + return settings.TEMPLATE_STRING_IF_INVALID + +register.filter('attribute', attribute) + +# vim: set ts=4 sw=4 et: diff --git a/templates/devel/packages.html b/templates/devel/packages.html new file mode 100644 index 00000000..3b511c98 --- /dev/null +++ b/templates/devel/packages.html @@ -0,0 +1,56 @@ +{% extends "base.html" %} +{% load attributes %} + +{% block title %}Arch Linux - {{ title }}{% endblock %} + +{% block content %} +
      +

      {{ title }}

      + + + + + + + + + + + + {% for name in column_names %} + + {% endfor %} + + + + {% for pkg in packages %} + + + + + {% if pkg.flag_date %} + + {% else %} + + {% endif %} + + + + + {% for attr in column_attrs %} + + {% endfor %} + + {% endfor %} + +
      ArchRepoNameVersionDescriptionLast UpdatedBuild DateFlag Date{{ name }}
      {{ pkg.arch.name }}{{ pkg.repo.name|capfirst }}{{ pkg.pkgname }}{{ pkg.full_version }}{{ pkg.full_version }}{{ pkg.pkgdesc }}{{ pkg.last_update|date }}{{ pkg.build_date|date }}{{ pkg.flag_date|date }}{{ pkg|attribute:attr }}
      +
      +{% load cdn %}{% jquery %} + + +{% endblock %} -- cgit v1.2.3-54-g00ecf From a08cac6dd07ecc3c535c1e1af158c203a2267c09 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 3 May 2011 13:39:51 -0500 Subject: Add two new groups for permissions Signed-off-by: Dan McGee --- main/fixtures/groups.json | 218 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 213 insertions(+), 5 deletions(-) (limited to 'main') diff --git a/main/fixtures/groups.json b/main/fixtures/groups.json index 32416a7a..8a6b2287 100644 --- a/main/fixtures/groups.json +++ b/main/fixtures/groups.json @@ -84,11 +84,6 @@ "mirrors", "mirrorprotocol" ], - [ - "delete_mirrorprotocol", - "mirrors", - "mirrorprotocol" - ], [ "add_mirrorrsync", "mirrors", @@ -122,6 +117,219 @@ ] } }, + { + "pk": 6, + "model": "auth.group", + "fields": { + "name": "Package Relation Maintainers", + "permissions": [ + [ + "add_packagerelation", + "packages", + "packagerelation" + ], + [ + "change_packagerelation", + "packages", + "packagerelation" + ], + [ + "delete_packagerelation", + "packages", + "packagerelation" + ] + ] + } + }, + { + "pk": 5, + "model": "auth.group", + "fields": { + "name": "Release Engineering", + "permissions": [ + [ + "add_architecture", + "releng", + "architecture" + ], + [ + "change_architecture", + "releng", + "architecture" + ], + [ + "delete_architecture", + "releng", + "architecture" + ], + [ + "add_bootloader", + "releng", + "bootloader" + ], + [ + "change_bootloader", + "releng", + "bootloader" + ], + [ + "delete_bootloader", + "releng", + "bootloader" + ], + [ + "add_boottype", + "releng", + "boottype" + ], + [ + "change_boottype", + "releng", + "boottype" + ], + [ + "delete_boottype", + "releng", + "boottype" + ], + [ + "add_clockchoice", + "releng", + "clockchoice" + ], + [ + "change_clockchoice", + "releng", + "clockchoice" + ], + [ + "delete_clockchoice", + "releng", + "clockchoice" + ], + [ + "add_filesystem", + "releng", + "filesystem" + ], + [ + "change_filesystem", + "releng", + "filesystem" + ], + [ + "delete_filesystem", + "releng", + "filesystem" + ], + [ + "add_hardwaretype", + "releng", + "hardwaretype" + ], + [ + "change_hardwaretype", + "releng", + "hardwaretype" + ], + [ + "delete_hardwaretype", + "releng", + "hardwaretype" + ], + [ + "add_installtype", + "releng", + "installtype" + ], + [ + "change_installtype", + "releng", + "installtype" + ], + [ + "delete_installtype", + "releng", + "installtype" + ], + [ + "add_iso", + "releng", + "iso" + ], + [ + "change_iso", + "releng", + "iso" + ], + [ + "delete_iso", + "releng", + "iso" + ], + [ + "add_isotype", + "releng", + "isotype" + ], + [ + "change_isotype", + "releng", + "isotype" + ], + [ + "delete_isotype", + "releng", + "isotype" + ], + [ + "add_module", + "releng", + "module" + ], + [ + "change_module", + "releng", + "module" + ], + [ + "delete_module", + "releng", + "module" + ], + [ + "add_source", + "releng", + "source" + ], + [ + "change_source", + "releng", + "source" + ], + [ + "delete_source", + "releng", + "source" + ], + [ + "add_test", + "releng", + "test" + ], + [ + "change_test", + "releng", + "test" + ], + [ + "delete_test", + "releng", + "test" + ] + ] + } + }, { "pk": 2, "model": "auth.group", -- cgit v1.2.3-54-g00ecf From 72bf6a02f3ee80cba40bb8556e161fd89ad7d886 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 9 May 2011 16:49:20 -0500 Subject: Ensure required by works for arch-agnostic packages We weren't showing required by entries for arch-specific packages that depended on arch-agnostic ones. Make the check a bit less specific for arch-agnostic packages similar to what we do for dependencies. Fixes FS#24184. Signed-off-by: Dan McGee --- main/models.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index e8f189e7..59dc154b 100644 --- a/main/models.py +++ b/main/models.py @@ -188,9 +188,12 @@ def get_requiredby(self): """ requiredby = PackageDepend.objects.select_related('pkg', 'pkg__arch', 'pkg__repo').filter( - pkg__arch__in=self.applicable_arches(), depname=self.pkgname).order_by( - 'pkg__pkgname', 'pkg__id') + 'pkg__pkgname', 'pkg__arch__name', 'pkg__repo__name') + if not self.arch.agnostic: + # make sure we match architectures if possible + requiredby = requiredby.filter( + pkg__arch__in=self.applicable_arches()) # sort out duplicate packages; this happens if something has a double # versioned dep such as a kernel module requiredby = [list(vals)[0] for k, vals in -- cgit v1.2.3-54-g00ecf