summaryrefslogtreecommitdiff
path: root/mirrors
diff options
context:
space:
mode:
Diffstat (limited to 'mirrors')
-rw-r--r--mirrors/admin.py1
-rw-r--r--mirrors/fixtures/mirrorprotocols.json9
-rw-r--r--mirrors/management/commands/mirrorcheck.py2
-rw-r--r--mirrors/management/commands/mirrorresolv.py2
-rw-r--r--mirrors/migrations/0026_auto__add_field_mirrorurl_active.py83
-rw-r--r--mirrors/models.py3
-rw-r--r--mirrors/urls_mirrorlist.py4
-rw-r--r--mirrors/utils.py10
-rw-r--r--mirrors/views.py44
9 files changed, 114 insertions, 44 deletions
diff --git a/mirrors/admin.py b/mirrors/admin.py
index 9c88207d..d0f2f475 100644
--- a/mirrors/admin.py
+++ b/mirrors/admin.py
@@ -10,6 +10,7 @@ from .models import (Mirror, MirrorProtocol, MirrorUrl, MirrorRsync,
class MirrorUrlForm(forms.ModelForm):
class Meta:
model = MirrorUrl
+
def clean_url(self):
# is this a valid-looking URL?
url_parts = urlparse(self.cleaned_data["url"])
diff --git a/mirrors/fixtures/mirrorprotocols.json b/mirrors/fixtures/mirrorprotocols.json
index 8822ef8e..1a07510b 100644
--- a/mirrors/fixtures/mirrorprotocols.json
+++ b/mirrors/fixtures/mirrorprotocols.json
@@ -9,15 +9,6 @@
}
},
{
- "pk": 2,
- "model": "mirrors.mirrorprotocol",
- "fields": {
- "is_download": true,
- "default": false,
- "protocol": "ftp"
- }
- },
- {
"pk": 3,
"model": "mirrors.mirrorprotocol",
"fields": {
diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py
index e7dd7b49..6faf294a 100644
--- a/mirrors/management/commands/mirrorcheck.py
+++ b/mirrors/management/commands/mirrorcheck.py
@@ -63,7 +63,7 @@ class Command(NoArgsCommand):
timeout = options.get('timeout')
urls = MirrorUrl.objects.select_related('protocol').filter(
- mirror__active=True, mirror__public=True)
+ active=True, mirror__active=True, mirror__public=True)
location = options.get('location', None)
if location:
diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py
index a6c2523e..85a3c654 100644
--- a/mirrors/management/commands/mirrorresolv.py
+++ b/mirrors/management/commands/mirrorresolv.py
@@ -39,7 +39,7 @@ class Command(NoArgsCommand):
def resolve_mirrors():
logger.debug("requesting list of mirror URLs")
- for mirrorurl in MirrorUrl.objects.filter(mirror__active=True):
+ for mirrorurl in MirrorUrl.objects.filter(active=True, mirror__active=True):
try:
# save old values, we can skip no-op updates this way
oldvals = (mirrorurl.has_ipv4, mirrorurl.has_ipv6)
diff --git a/mirrors/migrations/0026_auto__add_field_mirrorurl_active.py b/mirrors/migrations/0026_auto__add_field_mirrorurl_active.py
new file mode 100644
index 00000000..f989435f
--- /dev/null
+++ b/mirrors/migrations/0026_auto__add_field_mirrorurl_active.py
@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+ db.add_column(u'mirrors_mirrorurl', 'active',
+ self.gf('django.db.models.fields.BooleanField')(default=True),
+ keep_default=True)
+
+ def backwards(self, orm):
+ db.delete_column(u'mirrors_mirrorurl', 'active')
+
+
+ models = {
+ u'mirrors.checklocation': {
+ 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'},
+ 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}),
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'})
+ },
+ u'mirrors.mirror': {
+ 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'},
+ 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+ 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+ 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}),
+ 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'})
+ },
+ u'mirrors.mirrorlog': {
+ 'Meta': {'object_name': 'MirrorLog'},
+ 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
+ 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+ 'location': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'null': 'True', 'to': u"orm['mirrors.CheckLocation']"}),
+ 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"})
+ },
+ u'mirrors.mirrorprotocol': {
+ 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'},
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'})
+ },
+ u'mirrors.mirrorrsync': {
+ 'Meta': {'object_name': 'MirrorRsync'},
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'ip': ('mirrors.fields.IPNetworkField', [], {'max_length': '44'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"})
+ },
+ u'mirrors.mirrorurl': {
+ 'Meta': {'object_name': 'MirrorUrl'},
+ 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}),
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}),
+ 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}),
+ 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+ }
+ }
+
+ complete_apps = ['mirrors']
diff --git a/mirrors/models.py b/mirrors/models.py
index d8ac7952..975ead39 100644
--- a/mirrors/models.py
+++ b/mirrors/models.py
@@ -47,7 +47,7 @@ class Mirror(models.Model):
class MirrorProtocol(models.Model):
protocol = models.CharField(max_length=10, unique=True)
is_download = models.BooleanField(default=True,
- help_text="Is protocol useful for end-users, e.g. FTP/HTTP")
+ help_text="Is protocol useful for end-users, e.g. HTTP")
default = models.BooleanField(default=True,
help_text="Included by default when building mirror list?")
created = models.DateTimeField(editable=False)
@@ -70,6 +70,7 @@ class MirrorUrl(models.Model):
has_ipv6 = models.BooleanField("IPv6 capable", default=False,
editable=False)
created = models.DateTimeField(editable=False)
+ active = models.BooleanField(default=True)
def address_families(self):
hostname = urlparse(self.url).hostname
diff --git a/mirrors/urls_mirrorlist.py b/mirrors/urls_mirrorlist.py
index 1444eca9..bba54ec9 100644
--- a/mirrors/urls_mirrorlist.py
+++ b/mirrors/urls_mirrorlist.py
@@ -1,9 +1,11 @@
from django.conf.urls import patterns
+
urlpatterns = patterns('mirrors.views',
(r'^$', 'generate_mirrorlist', {}, 'mirrorlist'),
(r'^all/$', 'find_mirrors', {'countries': ['all']}),
- (r'^all/(?P<protocol>[A-z]+)/$', 'find_mirrors_simple')
+ (r'^all/(?P<protocol>[A-z]+)/$', 'find_mirrors_simple',
+ {}, 'mirrorlist_simple')
)
# vim: set ts=4 sw=4 et:
diff --git a/mirrors/utils.py b/mirrors/utils.py
index ba45da5f..e98b5c9f 100644
--- a/mirrors/utils.py
+++ b/mirrors/utils.py
@@ -116,7 +116,10 @@ def annotate_url(url, url_data):
def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None):
cutoff_time = now() - cutoff
- valid_urls = MirrorUrl.objects.filter(
+ # TODO: this prevents grabbing data points from any mirror that was active,
+ # receiving checks, and then marked private. we can probably be smarter and
+ # filter the data later?
+ valid_urls = MirrorUrl.objects.filter(active=True,
mirror__active=True, mirror__public=True,
logs__check_time__gte=cutoff_time).distinct()
@@ -159,7 +162,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None):
def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None):
cutoff_time = now() - cutoff
errors = MirrorLog.objects.filter(
- is_success=False, check_time__gte=cutoff_time,
+ is_success=False, check_time__gte=cutoff_time, url__active=True,
url__mirror__active=True, url__mirror__public=True).values(
'url__url', 'url__country', 'url__protocol__protocol',
'url__mirror__tier', 'error').annotate(
@@ -189,13 +192,14 @@ def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF):
min_sync_time = status_data['last_sync__max'] - timedelta(minutes=20)
best_logs = MirrorLog.objects.filter(is_success=True,
check_time__gte=min_check_time, last_sync__gte=min_sync_time,
+ url__active=True,
url__mirror__public=True, url__mirror__active=True,
url__protocol__default=True).order_by(
'duration')[:1]
if best_logs:
return MirrorUrl.objects.get(id=best_logs[0].url_id)
- mirror_urls = MirrorUrl.objects.filter(
+ mirror_urls = MirrorUrl.objects.filter(active=True,
mirror__public=True, mirror__active=True, protocol__default=True)
# look first for a country-agnostic URL, then fall back to any HTTP URL
filtered_urls = mirror_urls.filter(country='')[:1]
diff --git a/mirrors/views.py b/mirrors/views.py
index 73d40297..6f4ad838 100644
--- a/mirrors/views.py
+++ b/mirrors/views.py
@@ -8,7 +8,7 @@ from django.forms.widgets import CheckboxSelectMultiple
from django.core.serializers.json import DjangoJSONEncoder
from django.db.models import Q
from django.http import Http404, HttpResponse
-from django.shortcuts import get_object_or_404, render
+from django.shortcuts import get_object_or_404, redirect, render
from django.utils.timezone import now
from django.views.decorators.csrf import csrf_exempt
from django_countries.countries import COUNTRIES
@@ -43,7 +43,7 @@ class MirrorlistForm(forms.Form):
def get_countries(self):
country_codes = set()
- country_codes.update(MirrorUrl.objects.filter(
+ country_codes.update(MirrorUrl.objects.filter(active=True,
mirror__active=True).exclude(country='').values_list(
'country', flat=True).order_by().distinct())
countries = [(code, self.countries[code]) for code in country_codes]
@@ -78,18 +78,6 @@ def generate_mirrorlist(request):
{'mirrorlist_form': form})
-def default_protocol_filter(original_urls):
- key_func = attrgetter('country')
- sorted_urls = sorted(original_urls, key=key_func)
- urls = []
- for _, group in groupby(sorted_urls, key=key_func):
- cntry_urls = list(group)
- if any(url.protocol.default for url in cntry_urls):
- cntry_urls = [url for url in cntry_urls if url.protocol.default]
- urls.extend(cntry_urls)
- return urls
-
-
def status_filter(original_urls):
status_info = get_mirror_statuses()
scores = {u.id: u.score for u in status_info['urls']}
@@ -105,7 +93,7 @@ def status_filter(original_urls):
def find_mirrors(request, countries=None, protocols=None, use_status=False,
- ipv4_supported=True, ipv6_supported=True, smart_protocol=False):
+ ipv4_supported=True, ipv6_supported=True):
if not protocols:
protocols = MirrorProtocol.objects.filter(is_download=True)
elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol:
@@ -114,7 +102,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False,
else:
protocols = MirrorProtocol.objects.filter(protocol__in=protocols)
qset = MirrorUrl.objects.select_related().filter(
- protocol__in=protocols,
+ protocol__in=protocols, active=True,
mirror__public=True, mirror__active=True)
if countries and 'all' not in countries:
qset = qset.filter(country__in=countries)
@@ -126,17 +114,12 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False,
ip_version |= Q(has_ipv6=True)
qset = qset.filter(ip_version)
- if smart_protocol:
- urls = default_protocol_filter(qset)
- else:
- urls = qset
-
if not use_status:
sort_key = attrgetter('country.name', 'mirror.name', 'url')
- urls = sorted(urls, key=sort_key)
+ urls = sorted(qset, key=sort_key)
template = 'mirrors/mirrorlist.txt'
else:
- urls = status_filter(urls)
+ urls = status_filter(qset)
template = 'mirrors/mirrorlist_status.txt'
context = {
@@ -147,9 +130,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False,
def find_mirrors_simple(request, protocol):
if protocol == 'smart':
- # generate a 'smart' mirrorlist, one that only includes FTP mirrors if
- # no HTTP mirror is available in that country.
- return find_mirrors(request, smart_protocol=True)
+ return redirect('mirrorlist_simple', 'http', permanent=True)
proto = get_object_or_404(MirrorProtocol, protocol=protocol)
return find_mirrors(request, protocols=[proto])
@@ -175,6 +156,7 @@ def mirror_details(request, name):
if not request.user.is_authenticated() and \
(not mirror.public or not mirror.active):
raise Http404
+ error_cutoff = timedelta(days=7)
status_info = get_mirror_statuses(mirror_id=mirror.id)
checked_urls = {url for url in status_info['urls'] \
@@ -188,9 +170,15 @@ def mirror_details(request, name):
setattr(url, attr, None)
all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url'))
- return render(request, 'mirrors/mirror_details.html',
- {'mirror': mirror, 'urls': all_urls})
+ error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff)
+ context = {
+ 'mirror': mirror,
+ 'urls': all_urls,
+ 'cutoff': error_cutoff,
+ 'error_logs': error_logs,
+ }
+ return render(request, 'mirrors/mirror_details.html', context)
def mirror_details_json(request, name):
mirror = get_object_or_404(Mirror, name=name)