diff options
Diffstat (limited to 'mirrors')
| -rw-r--r-- | mirrors/admin.py | 2 | ||||
| -rw-r--r-- | mirrors/migrations/0009_auto__chg_field_mirrorurl_country.py | 6 | ||||
| -rw-r--r-- | mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py | 2 | ||||
| -rw-r--r-- | mirrors/migrations/0011_adjust_protocol_defaults.py | 2 | ||||
| -rw-r--r-- | mirrors/migrations/0012_auto__add_on_delete_attribute.py | 2 | ||||
| -rw-r--r-- | mirrors/migrations/0013_rename_country_fields.py | 68 | ||||
| -rw-r--r-- | mirrors/migrations/0014_add_country_code_fields.py | 74 | ||||
| -rw-r--r-- | mirrors/migrations/0015_assign_country_codes.py | 93 | ||||
| -rw-r--r-- | mirrors/migrations/0016_auto__del_field_mirror_country_old__del_field_mirrorurl_country_old.py | 76 | ||||
| -rw-r--r-- | mirrors/models.py | 15 | ||||
| -rw-r--r-- | mirrors/utils.py | 18 | ||||
| -rw-r--r-- | mirrors/views.py | 82 |
12 files changed, 389 insertions, 51 deletions
diff --git a/mirrors/admin.py b/mirrors/admin.py index a2b59b41..b7b9894c 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -64,7 +64,7 @@ class MirrorAdmin(admin.ModelAdmin): form = MirrorAdminForm list_display = ('name', 'tier', 'country', 'active', 'public', 'isos', 'admin_email') - list_filter = ('tier', 'active', 'public', 'country') + list_filter = ('tier', 'active', 'public', 'country') search_fields = ('name',) inlines = [ MirrorUrlInlineAdmin, diff --git a/mirrors/migrations/0009_auto__chg_field_mirrorurl_country.py b/mirrors/migrations/0009_auto__chg_field_mirrorurl_country.py index 941ecfbb..bca6b6fb 100644 --- a/mirrors/migrations/0009_auto__chg_field_mirrorurl_country.py +++ b/mirrors/migrations/0009_auto__chg_field_mirrorurl_country.py @@ -7,11 +7,11 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): - db.alter_column('mirrors_mirrorurl', 'country', self.gf('mirrors.models.NullCharField')(max_length=255, null=True)) + pass def backwards(self, orm): - db.alter_column('mirrors_mirrorurl', 'country', self.gf('django.db.models.fields.CharField')(max_length=255, null=True)) + pass models = { 'mirrors.mirror': { @@ -53,7 +53,7 @@ class Migration(SchemaMigration): }, 'mirrors.mirrorurl': { 'Meta': {'object_name': 'MirrorUrl'}, - 'country': ('mirrors.models.NullCharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'country': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}), 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), diff --git a/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py b/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py index 6868cb25..d30c78c7 100644 --- a/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py +++ b/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py @@ -53,7 +53,7 @@ class Migration(SchemaMigration): }, 'mirrors.mirrorurl': { 'Meta': {'object_name': 'MirrorUrl'}, - 'country': ('mirrors.models.NullCharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'country': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}), 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), diff --git a/mirrors/migrations/0011_adjust_protocol_defaults.py b/mirrors/migrations/0011_adjust_protocol_defaults.py index 0118de67..a5ffafb4 100644 --- a/mirrors/migrations/0011_adjust_protocol_defaults.py +++ b/mirrors/migrations/0011_adjust_protocol_defaults.py @@ -54,7 +54,7 @@ class Migration(DataMigration): }, 'mirrors.mirrorurl': { 'Meta': {'object_name': 'MirrorUrl'}, - 'country': ('mirrors.models.NullCharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'country': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}), 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), diff --git a/mirrors/migrations/0012_auto__add_on_delete_attribute.py b/mirrors/migrations/0012_auto__add_on_delete_attribute.py index 3990d466..f81fe0ae 100644 --- a/mirrors/migrations/0012_auto__add_on_delete_attribute.py +++ b/mirrors/migrations/0012_auto__add_on_delete_attribute.py @@ -55,7 +55,7 @@ class Migration(SchemaMigration): }, 'mirrors.mirrorurl': { 'Meta': {'object_name': 'MirrorUrl'}, - 'country': ('mirrors.models.NullCharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'country': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}), 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), diff --git a/mirrors/migrations/0013_rename_country_fields.py b/mirrors/migrations/0013_rename_country_fields.py new file mode 100644 index 00000000..8a9bc34a --- /dev/null +++ b/mirrors/migrations/0013_rename_country_fields.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.rename_column('mirrors_mirror', 'country', 'country_old') + db.rename_column('mirrors_mirrorurl', 'country', 'country_old') + + def backwards(self, orm): + db.rename_column('mirrors_mirror', 'country_old', 'country') + db.rename_column('mirrors_mirrorurl', 'country_old', 'country') + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('country_old', 'name')", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'country_old': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country_old': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/migrations/0014_add_country_code_fields.py b/mirrors/migrations/0014_add_country_code_fields.py new file mode 100644 index 00000000..010194d7 --- /dev/null +++ b/mirrors/migrations/0014_add_country_code_fields.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.add_column('mirrors_mirror', 'country', + self.gf('django_countries.fields.CountryField')(default='', max_length=2, blank=True), + keep_default=False) + db.add_column('mirrors_mirrorurl', 'country', + self.gf('django_countries.fields.CountryField')(default='', max_length=2, blank=True), + keep_default=False) + + def backwards(self, orm): + db.delete_column('mirrors_mirror', 'country') + db.delete_column('mirrors_mirrorurl', 'country') + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2', 'blank': 'True'}), + 'country_old': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2', 'blank': 'True'}), + 'country_old': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/migrations/0015_assign_country_codes.py b/mirrors/migrations/0015_assign_country_codes.py new file mode 100644 index 00000000..c1b0f969 --- /dev/null +++ b/mirrors/migrations/0015_assign_country_codes.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import DataMigration +from django.db import models + +from django_countries.countries import OFFICIAL_COUNTRIES + +class Migration(DataMigration): + + def forwards(self, orm): + reverse_map = dict((v, k) for k, v in OFFICIAL_COUNTRIES.items()) + # add a few special cases to the list that we know might exist + reverse_map['GREAT BRITAIN'] = 'GB' + reverse_map['KOREA'] = 'KR' + reverse_map['MACEDONIA'] = 'MK' + reverse_map['RUSSIA'] = 'RU' + reverse_map['SOUTH KOREA'] = 'KR' + reverse_map['TAIWAN'] = 'TW' + reverse_map['VIETNAM'] = 'VN' + + for country_name in orm.Mirror.objects.values_list( + 'country_old', flat=True).order_by().distinct(): + code = reverse_map.get(country_name.upper(), '') + orm.Mirror.objects.filter( + country_old=country_name).update(country=code) + + for country_name in orm.MirrorUrl.objects.filter( + country_old__isnull=False).values_list( + 'country_old', flat=True).order_by().distinct(): + code = reverse_map.get(country_name.upper(), '') + orm.MirrorUrl.objects.filter( + country_old=country_name).update(country=code) + + def backwards(self, orm): + orm.MirrorUrl.objects.all().update(country='') + orm.Mirror.objects.all().update(country='') + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2', 'blank': 'True'}), + 'country_old': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2', 'blank': 'True'}), + 'country_old': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] + symmetrical = True diff --git a/mirrors/migrations/0016_auto__del_field_mirror_country_old__del_field_mirrorurl_country_old.py b/mirrors/migrations/0016_auto__del_field_mirror_country_old__del_field_mirrorurl_country_old.py new file mode 100644 index 00000000..b296d7ae --- /dev/null +++ b/mirrors/migrations/0016_auto__del_field_mirror_country_old__del_field_mirrorurl_country_old.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.delete_column('mirrors_mirror', 'country_old') + db.delete_column('mirrors_mirrorurl', 'country_old') + db.create_index('mirrors_mirror', ['country']) + db.create_index('mirrors_mirrorurl', ['country']) + + def backwards(self, orm): + db.delete_index('mirrors_mirrorurl', ['country']) + db.delete_index('mirrors_mirror', ['country']) + db.add_column('mirrors_mirror', 'country_old', + self.gf('django.db.models.fields.CharField')(default='Any', max_length=255, db_index=True), + keep_default=False) + db.add_column('mirrors_mirrorurl', 'country_old', + self.gf('django.db.models.fields.CharField')(blank=True, max_length=255, null=True, db_index=True), + keep_default=False) + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index 86905eea..19437610 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -3,15 +3,8 @@ from urlparse import urlparse from django.db import models from django.core.exceptions import ValidationError +from django_countries import CountryField -class NullCharField(models.CharField): - description = "String (up to %(max_length)s), NULL if value is empty" - _south_introspects = True - - def get_prep_value(self, value): - if value == '': - return None - return self.to_python(value) TIER_CHOICES = ( (0, 'Tier 0'), @@ -20,11 +13,12 @@ TIER_CHOICES = ( (-1, 'Untiered'), ) + class Mirror(models.Model): name = models.CharField(max_length=255, unique=True) tier = models.SmallIntegerField(default=2, choices=TIER_CHOICES) upstream = models.ForeignKey('self', null=True, on_delete=models.SET_NULL) - country = models.CharField(max_length=255, db_index=True) + country = CountryField(blank=True, db_index=True) admin_email = models.EmailField(max_length=255, blank=True) public = models.BooleanField(default=True) active = models.BooleanField(default=True) @@ -68,8 +62,7 @@ class MirrorUrl(models.Model): protocol = models.ForeignKey(MirrorProtocol, related_name="urls", editable=False, on_delete=models.PROTECT) mirror = models.ForeignKey(Mirror, related_name="urls") - country = NullCharField(max_length=255, null=True, blank=True, - db_index=True) + country = CountryField(blank=True, db_index=True) has_ipv4 = models.BooleanField("IPv4 capable", default=True, editable=False) has_ipv6 = models.BooleanField("IPv6 capable", default=False, diff --git a/mirrors/utils.py b/mirrors/utils.py index ddecb095..32fa3587 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,6 +1,7 @@ from datetime import timedelta from django.db.models import Avg, Count, Max, Min, StdDev +from django_countries.fields import Country from main.utils import cache_function, utc_now from .models import MirrorLog, MirrorProtocol, MirrorUrl @@ -42,8 +43,7 @@ def get_mirror_statuses(cutoff=default_cutoff): last_sync=Max('logs__last_sync'), last_check=Max('logs__check_time'), duration_avg=Avg('logs__duration'), - duration_stddev=StdDev('logs__duration') - ).order_by('-last_sync', '-duration_avg') + duration_stddev=StdDev('logs__duration')) # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the @@ -95,7 +95,8 @@ def get_mirror_errors(cutoff=default_cutoff): ).order_by('-last_occurred', '-error_count') errors = list(errors) for err in errors: - err['country'] = err['url__country'] or err['url__mirror__country'] + ctry_code = err['url__country'] or err['url__mirror__country'] + err['country'] = Country(ctry_code) return errors @@ -110,20 +111,19 @@ def get_mirror_url_for_download(cutoff=default_cutoff): Max('check_time'), Max('last_sync')) if status_data['check_time__max'] is not None: min_check_time = status_data['check_time__max'] - timedelta(minutes=5) - min_sync_time = status_data['last_sync__max'] - timedelta(minutes=30) + min_sync_time = status_data['last_sync__max'] - timedelta(minutes=20) best_logs = MirrorLog.objects.filter(is_success=True, check_time__gte=min_check_time, last_sync__gte=min_sync_time, url__mirror__public=True, url__mirror__active=True, - url__protocol__protocol__iexact='HTTP').order_by( + url__protocol__default=True).order_by( 'duration')[:1] if best_logs: return MirrorUrl.objects.get(id=best_logs[0].url_id) mirror_urls = MirrorUrl.objects.filter( - mirror__public=True, mirror__active=True, - protocol__protocol__iexact='HTTP') - # look first for an 'Any' URL, then fall back to any HTTP URL - filtered_urls = mirror_urls.filter(mirror__country='Any')[:1] + mirror__public=True, mirror__active=True, protocol__default=True) + # look first for a country-agnostic URL, then fall back to any HTTP URL + filtered_urls = mirror_urls.filter(mirror__country='')[:1] if not filtered_urls: filtered_urls = mirror_urls[:1] if not filtered_urls: diff --git a/mirrors/views.py b/mirrors/views.py index e93097a3..c52656f7 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -1,4 +1,8 @@ +from datetime import timedelta +from operator import attrgetter, itemgetter + from django import forms +from django.forms.widgets import CheckboxSelectMultiple from django.core.serializers.json import DjangoJSONEncoder from django.db.models import Q from django.http import Http404, HttpResponse @@ -6,33 +10,55 @@ from django.shortcuts import get_object_or_404 from django.views.decorators.csrf import csrf_exempt from django.views.generic.simple import direct_to_template from django.utils import simplejson +from django_countries.countries import COUNTRIES -from main.utils import make_choice from .models import Mirror, MirrorUrl, MirrorProtocol from .utils import get_mirror_statuses, get_mirror_errors -import datetime +COUNTRY_LOOKUP = dict(COUNTRIES) + class MirrorlistForm(forms.Form): country = forms.MultipleChoiceField(required=False) - protocol = forms.MultipleChoiceField(required=False) + protocol = forms.MultipleChoiceField(required=False, + widget=CheckboxSelectMultiple) ip_version = forms.MultipleChoiceField(required=False, - label="IP version", choices=(('4','IPv4'), ('6','IPv6'))) + label="IP version", choices=(('4','IPv4'), ('6','IPv6')), + widget=CheckboxSelectMultiple) use_mirror_status = forms.BooleanField(required=False) def __init__(self, *args, **kwargs): super(MirrorlistForm, self).__init__(*args, **kwargs) - countries = Mirror.objects.filter(active=True).values_list( - 'country', flat=True).distinct().order_by('country') - self.fields['country'].choices = [('all','All')] + make_choice( - countries) - self.fields['country'].initial = ['all'] - protos = make_choice( - MirrorProtocol.objects.filter(is_download=True)) + fields = self.fields + fields['country'].choices = [('all','All')] + self.get_countries() + fields['country'].initial = ['all'] + protos = [(p.protocol, p.protocol) for p in + MirrorProtocol.objects.filter(is_download=True)] initial = MirrorProtocol.objects.filter(is_download=True, default=True) - self.fields['protocol'].choices = protos - self.fields['protocol'].initial = [p.protocol for p in initial] - self.fields['ip_version'].initial = ['4'] + fields['protocol'].choices = protos + fields['protocol'].initial = [p.protocol for p in initial] + fields['ip_version'].initial = ['4'] + + def get_countries(self): + country_codes = set() + country_codes.update(Mirror.objects.filter(active=True).exclude( + country='').values_list( + 'country', flat=True).order_by().distinct()) + country_codes.update(MirrorUrl.objects.filter( + mirror__active=True).exclude(country='').values_list( + 'country', flat=True).order_by().distinct()) + countries = [(code, COUNTRY_LOOKUP[code]) for code in country_codes] + return sorted(countries, key=itemgetter(1)) + + def as_div(self): + "Returns this form rendered as HTML <divs>s." + return self._html_output( + normal_row = u'<div%(html_class_attr)s>%(label)s %(field)s%(help_text)s</div>', + error_row = u'%s', + row_ender = '</div>', + help_text_html = u' <span class="helptext">%s</span>', + errors_on_separate_row = True) + @csrf_exempt def generate_mirrorlist(request): @@ -49,9 +75,10 @@ def generate_mirrorlist(request): else: form = MirrorlistForm() - return direct_to_template(request, 'mirrors/index.html', + return direct_to_template(request, 'mirrors/mirrorlist_generate.html', {'mirrorlist_form': form}) + def find_mirrors(request, countries=None, protocols=None, use_status=False, ipv4_supported=True, ipv6_supported=True): if not protocols: @@ -74,7 +101,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, if not use_status: urls = qset.order_by('mirror__name', 'url') - urls = sorted(urls, key=lambda x: x.real_country) + urls = sorted(urls, key=attrgetter('real_country')) template = 'mirrors/mirrorlist.txt' else: status_info = get_mirror_statuses() @@ -96,6 +123,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, }, mimetype='text/plain') + def mirrors(request): mirror_list = Mirror.objects.select_related().order_by('tier', 'country') if not request.user.is_authenticated(): @@ -103,6 +131,7 @@ def mirrors(request): return direct_to_template(request, 'mirrors/mirrors.html', {'mirror_list': mirror_list}) + def mirror_details(request, name): mirror = get_object_or_404(Mirror, name=name) if not request.user.is_authenticated() and \ @@ -116,13 +145,14 @@ def mirror_details(request, name): # get each item from checked_urls and supplement with anything in all_urls # if it wasn't there all_urls = set(checked_urls).union(all_urls) - all_urls = sorted(all_urls, key=lambda x: x.url) + all_urls = sorted(all_urls, key=attrgetter('url')) return direct_to_template(request, 'mirrors/mirror_details.html', {'mirror': mirror, 'urls': all_urls}) + def status(request): - bad_timedelta = datetime.timedelta(days=3) + bad_timedelta = timedelta(days=3) status_info = get_mirror_statuses() urls = status_info['urls'] @@ -137,12 +167,13 @@ def status(request): context = status_info.copy() context.update({ - 'good_urls': good_urls, - 'bad_urls': bad_urls, + 'good_urls': sorted(good_urls, key=attrgetter('score')), + 'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max), 'error_logs': get_mirror_errors(), }) return direct_to_template(request, 'mirrors/status.html', context) + class MirrorStatusJSONEncoder(DjangoJSONEncoder): '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl serialization. The base class takes care of datetime.datetime types.''' @@ -150,7 +181,7 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): 'delay', 'duration_avg', 'duration_stddev', 'score'] def default(self, obj): - if isinstance(obj, datetime.timedelta): + if isinstance(obj, timedelta): # always returned as integer seconds return obj.days * 24 * 3600 + obj.seconds if hasattr(obj, '__iter__'): @@ -159,17 +190,20 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): if isinstance(obj, MirrorUrl): data = dict((attr, getattr(obj, attr)) for attr in self.url_attributes) - # separate because it isn't on the URL directly - data['country'] = obj.real_country + # get any override on the country attribute first + country = obj.real_country + data['country'] = unicode(country.name) + data['country_code'] = country.code return data if isinstance(obj, MirrorProtocol): return unicode(obj) return super(MirrorStatusJSONEncoder, self).default(obj) + def status_json(request): status_info = get_mirror_statuses() data = status_info.copy() - data['version'] = 2 + data['version'] = 3 to_json = simplejson.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) response = HttpResponse(to_json, mimetype='application/json') |