From 6ec4a3589e327ded693ab0c741828fc5ec66b840 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Sun, 21 Feb 2016 19:44:38 +0100 Subject: Send notifications when changing ownership Add a new option that makes it possible to subscribe to package ownership changes (adoption/disownment). Fixes FS#15412. Signed-off-by: Lukas Fleischer --- scripts/notify.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) (limited to 'scripts') diff --git a/scripts/notify.py b/scripts/notify.py index 25102a2..5e5f377 100755 --- a/scripts/notify.py +++ b/scripts/notify.py @@ -115,6 +115,16 @@ def get_update_recipients(cur, pkgbase_id, uid): return [row[0] for row in cur.fetchall()] +def get_ownership_recipients(cur, pkgbase_id, uid): + cur.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'Users.OwnershipNotify = 1 AND ' + + 'PackageNotifications.UserID != %s AND ' + + 'PackageNotifications.PackageBaseID = %s', [uid, pkgbase_id]) + return [row[0] for row in cur.fetchall()] + + def get_request_recipients(cur, reqid): cur.execute('SELECT DISTINCT Users.Email FROM PackageRequests ' + 'INNER JOIN PackageBases ' + @@ -243,6 +253,38 @@ def flag(cur, uid, pkgbase_id): send_notification(to, subject, body, refs) +def adopt(cur, pkgbase_id, uid): + user = username_from_id(cur, uid) + pkgbase = pkgbase_from_id(cur, pkgbase_id) + to = get_ownership_recipients(cur, pkgbase_id, uid) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Ownership Notification for %s' % (pkgbase) + body = 'The package %s [1] was adopted by %s [2].' % (pkgbase, user) + refs = '[1] ' + pkgbase_uri + '\n' + refs += '[2] ' + user_uri + + send_notification(to, subject, body, refs) + + +def disown(cur, pkgbase_id, uid): + user = username_from_id(cur, uid) + pkgbase = pkgbase_from_id(cur, pkgbase_id) + to = get_ownership_recipients(cur, pkgbase_id, uid) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Ownership Notification for %s' % (pkgbase) + body = 'The package %s [1] was disowned by %s [2].' % (pkgbase, user) + refs = '[1] ' + pkgbase_uri + '\n' + refs += '[2] ' + user_uri + + send_notification(to, subject, body, refs) + + def comaintainer_add(cur, pkgbase_id, uid): pkgbase = pkgbase_from_id(cur, pkgbase_id) to = [get_user_email(cur, uid)] @@ -364,6 +406,8 @@ if __name__ == '__main__': 'comment': comment, 'update': update, 'flag': flag, + 'adopt': adopt, + 'disown': disown, 'comaintainer-add': comaintainer_add, 'comaintainer-remove': comaintainer_remove, 'delete': delete, -- cgit v1.2.3-54-g00ecf From b2e97cdd1ee804468b2dd601eafda9574c05a3a7 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Tue, 17 May 2016 19:03:39 +0200 Subject: Add repository information to official providers When updating the list of packages provided by the official repositories, also save the repository names. --- schema/aur-schema.sql | 1 + scripts/aurblup.py | 8 ++++++-- upgrading/4.3.0.txt | 6 ++++++ 3 files changed, 13 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/schema/aur-schema.sql b/schema/aur-schema.sql index 0804ac3..ea19d46 100644 --- a/schema/aur-schema.sql +++ b/schema/aur-schema.sql @@ -310,6 +310,7 @@ CREATE TABLE PackageBlacklist ( CREATE TABLE OfficialProviders ( ID INTEGER UNSIGNED NOT NULL AUTO_INCREMENT, Name VARCHAR(64) NOT NULL, + Repo VARCHAR(64) NOT NULL, Provides VARCHAR(64) NOT NULL, PRIMARY KEY (ID) ) ENGINE = InnoDB; diff --git a/scripts/aurblup.py b/scripts/aurblup.py index 325ef3d..9e11e43 100755 --- a/scripts/aurblup.py +++ b/scripts/aurblup.py @@ -20,6 +20,7 @@ servers = config.get('aurblup', 'servers').split(' ') blacklist = set() providers = set() +repomap = dict() h = pyalpm.Handle("/", db_path) for sync_db in sync_dbs: @@ -33,9 +34,11 @@ for sync_db in sync_dbs: blacklist.add(pkg.name) [blacklist.add(x) for x in pkg.replaces] providers.add((pkg.name, pkg.name)) + repomap[(pkg.name, pkg.name)] = repo.name for provision in pkg.provides: provisionname = re.sub(r'(<|=|>).*', '', provision) providers.add((pkg.name, provisionname)) + repomap[(pkg.name, provisionname)] = repo.name db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, passwd=aur_db_pass, db=aur_db_name, @@ -54,8 +57,9 @@ cur.execute("SELECT Name, Provides FROM OfficialProviders") oldproviders = set(cur.fetchall()) for pkg, provides in providers.difference(oldproviders): - cur.execute("INSERT INTO OfficialProviders (Name, Provides) " - "VALUES (%s, %s)", [pkg, provides]) + repo = repomap[(pkg, provides)] + cur.execute("INSERT INTO OfficialProviders (Name, Repo, Provides) " + "VALUES (%s, %s, %s)", [pkg, repo, provides]) for pkg, provides in oldproviders.difference(providers): cur.execute("DELETE FROM OfficialProviders " "WHERE Name = %s AND Provides = %s", [pkg, provides]) diff --git a/upgrading/4.3.0.txt b/upgrading/4.3.0.txt index 34b19a2..f7f3e08 100644 --- a/upgrading/4.3.0.txt +++ b/upgrading/4.3.0.txt @@ -9,3 +9,9 @@ ALTER TABLE Users ADD COLUMN OwnershipNotify TINYINT(1) NOT NULL DEFAULT 1; ---- ALTER TABLE Users MODIFY LastLoginIPAddress VARCHAR(45) NULL DEFAULT NULL; ---- + +3. Add a new column to store repository information of official providers: + +---- +ALTER TABLE OfficialProviders ADD COLUMN Repo VARCHAR(64) NOT NULL; +---- -- cgit v1.2.3-54-g00ecf From d273ee5eb20ecb6e97d5d6cd8a1f493e3652b584 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Tue, 17 May 2016 19:07:41 +0200 Subject: Use the official provider list to detect duplicates Instead of automatically adding packages from the official binary repositories to the package blacklist, use the official provider list to prevent users from uploading duplicates. This does not only result in reduced disk usage but also has a nice visible side effect. The error messages printed by the update hook now look like error: package already provided by [community]: powerline-fonts instead of error: package is blacklisted: powerline-fonts which was confusing to most end users. Signed-off-by: Lukas Fleischer --- git-interface/git-update.py | 6 ++++++ scripts/aurblup.py | 8 -------- 2 files changed, 6 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/git-interface/git-update.py b/git-interface/git-update.py index e54e0e6..2c24e72 100755 --- a/git-interface/git-update.py +++ b/git-interface/git-update.py @@ -331,12 +331,18 @@ pkgbase_id = cur.fetchone()[0] if cur.rowcount == 1 else 0 cur.execute("SELECT Name FROM PackageBlacklist") blacklist = [row[0] for row in cur.fetchall()] +cur.execute("SELECT Name, Repo FROM OfficialProviders") +providers = dict(cur.fetchall()) + for pkgname in srcinfo.utils.get_package_names(metadata): pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) pkgname = pkginfo['pkgname'] if pkgname in blacklist and not privileged: die('package is blacklisted: {:s}'.format(pkgname)) + if pkgname in providers and not privileged: + repo = providers[pkgname] + die('package already provided by [{:s}]: {:s}'.format(repo, pkgname)) cur.execute("SELECT COUNT(*) FROM Packages WHERE Name = %s AND " + "PackageBaseID <> %s", [pkgname, pkgbase_id]) diff --git a/scripts/aurblup.py b/scripts/aurblup.py index 9e11e43..6733b45 100755 --- a/scripts/aurblup.py +++ b/scripts/aurblup.py @@ -45,14 +45,6 @@ db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, unix_socket=aur_db_socket, buffered=True) cur = db.cursor() -cur.execute("SELECT Name FROM PackageBlacklist") -oldblacklist = set([row[0] for row in cur.fetchall()]) - -for pkg in blacklist.difference(oldblacklist): - cur.execute("INSERT INTO PackageBlacklist (Name) VALUES (%s)", [pkg]) -for pkg in oldblacklist.difference(blacklist): - cur.execute("DELETE FROM PackageBlacklist WHERE Name = %s", [pkg]) - cur.execute("SELECT Name, Provides FROM OfficialProviders") oldproviders = set(cur.fetchall()) -- cgit v1.2.3-54-g00ecf From 60cdad28eebf45ac2822155cea27277e1f7de455 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Thu, 30 Jun 2016 15:57:53 +0200 Subject: Distinguish auto-accepted requests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before commit 9746a65 (Port notification routines to Python, 2015-06-27), notification emails for automatically closed requests explicitly stated that the action was taken "automatically by the Arch User Repository package request system". When porting the notification routines to Python, this feature was overlooked and emails sent by the new script always reported that the requester triggered the acceptance or rejection of a request. This patch reimplements the old behavior such that notifications no longer look as if the requester had accepted the request himself. Reported-by: Johannes Löthberg Signed-off-by: Lukas Fleischer --- scripts/notify.py | 19 +++++++++++++------ web/lib/pkgreqfuncs.inc.php | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) (limited to 'scripts') diff --git a/scripts/notify.py b/scripts/notify.py index 5e5f377..6ea25d1 100755 --- a/scripts/notify.py +++ b/scripts/notify.py @@ -41,7 +41,10 @@ def send_notification(to, subject, body, refs, headers={}): wrapped = '' for line in body.splitlines(): wrapped += textwrap.fill(line, break_long_words=False) + '\n' - body = wrapped + '\n' + refs + if refs: + body = wrapped + '\n' + refs + else: + body = wrapped for recipient in to: msg = email.mime.text.MIMEText(body, 'plain', 'utf-8') @@ -377,20 +380,24 @@ def request_open(cur, uid, reqid, reqtype, pkgbase_id, merge_into=None): def request_close(cur, uid, reqid, reason): - user = username_from_id(cur, uid) to = [aur_request_ml] cc = get_request_recipients(cur, reqid) text = get_request_closure_comment(cur, reqid) - user_uri = aur_location + '/account/' + user + '/' - subject = '[PRQ#%d] Request %s' % (int(reqid), reason.title()) - body = 'Request #%d has been %s by %s [1]' % (int(reqid), reason, user) + if int(uid): + user = username_from_id(cur, uid) + user_uri = aur_location + '/account/' + user + '/' + body = 'Request #%d has been %s by %s [1]' % (int(reqid), reason, user) + refs = '[1] ' + user_uri + else: + body = 'Request #%d has been %s automatically by the Arch User ' \ + 'Repository package request system' % (int(reqid), reason) + refs = None if text.strip() == '': body += '.' else: body += ':\n\n' + text - refs = '[1] ' + user_uri thread_id = '' headers = headers_reply(thread_id) headers.update(headers_cc(cc)) diff --git a/web/lib/pkgreqfuncs.inc.php b/web/lib/pkgreqfuncs.inc.php index cf56663..8ceac8d 100644 --- a/web/lib/pkgreqfuncs.inc.php +++ b/web/lib/pkgreqfuncs.inc.php @@ -221,7 +221,7 @@ function pkgreq_close($id, $reason, $comments, $auto_close=false) { $dbh = DB::connect(); $id = intval($id); - $uid = uid_from_sid($_COOKIE["AURSID"]); + $uid = $auto_close ? 0 : uid_from_sid($_COOKIE["AURSID"]); if (!$auto_close && !has_credential(CRED_PKGREQ_CLOSE)) { return array(false, __("Only TUs and developers can close requests.")); -- cgit v1.2.3-54-g00ecf From f3fb614f196a1feb2738c56364f6e877d474ff2d Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Sun, 18 Sep 2016 15:55:49 +0200 Subject: Send reminders before the TU voting period ends Add a new script that checks for TU votes ending within the next 48 hours and sends reminder emails to all Trusted Users who did not cast their votes yet. The script is designed to be executed as a cron job, such that the check is performed periodically. Since the script does not remember users it already sent emails to, the interval should not be chosen too small to avoid spamming. Signed-off-by: Lukas Fleischer --- scripts/notify.py | 23 +++++++++++++++++++++++ scripts/tuvotereminder.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100755 scripts/tuvotereminder.py (limited to 'scripts') diff --git a/scripts/notify.py b/scripts/notify.py index 6ea25d1..a640e45 100755 --- a/scripts/notify.py +++ b/scripts/notify.py @@ -139,6 +139,15 @@ def get_request_recipients(cur, reqid): return [row[0] for row in cur.fetchall()] +def get_tu_vote_reminder_recipients(cur, vote_id): + cur.execute('SELECT Users.Email FROM Users WHERE AccountTypeID = 2 ' + + 'EXCEPT SELECT Users.Email FROM Users ' + + 'INNER JOIN TU_Votes ' + + 'ON TU_Votes.UserID = Users.ID ' + + 'WHERE TU_Votes.VoteID = %s', [vote_id]) + return [row[0] for row in cur.fetchall()] + + def get_comment(cur, comment_id): cur.execute('SELECT Comments FROM PackageComments WHERE ID = %s', [comment_id]) @@ -405,6 +414,19 @@ def request_close(cur, uid, reqid, reason): send_notification(to, subject, body, refs, headers) +def tu_vote_reminder(cur, vote_id): + to = get_tu_vote_reminder_recipients(cur, vote_id) + + vote_uri = aur_location + '/tu/?id=' + vote_id + + subject = 'TU Vote Reminder: Proposal %d' % (int(vote_id)) + body = 'Please remember to cast your vote on proposal %d [1]. ' \ + 'The voting period ends in less than 48 hours.' % (int(vote_id)) + refs = '[1] ' + vote_uri + + send_notification(to, subject, body, refs) + + if __name__ == '__main__': action = sys.argv[1] action_map = { @@ -420,6 +442,7 @@ if __name__ == '__main__': 'delete': delete, 'request-open': request_open, 'request-close': request_close, + 'tu-vote-reminder': tu_vote_reminder, } db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, diff --git a/scripts/tuvotereminder.py b/scripts/tuvotereminder.py new file mode 100755 index 0000000..0992623 --- /dev/null +++ b/scripts/tuvotereminder.py @@ -0,0 +1,32 @@ +#!/usr/bin/python3 + +import configparser +import mysql.connector +import os +import subprocess +import time + +config = configparser.RawConfigParser() +config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") + +aur_db_host = config.get('database', 'host') +aur_db_name = config.get('database', 'name') +aur_db_user = config.get('database', 'user') +aur_db_pass = config.get('database', 'password') +aur_db_socket = config.get('database', 'socket') +notify_cmd = config.get('notifications', 'notify-cmd') + +db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket, buffered=True) +cur = db.cursor() + +now = int(time.time()) +filter_from = now + 500 +filter_to = now + 172800 + +cur.execute("SELECT ID FROM TU_VoteInfo WHERE End >= %s AND End <= %s", + [filter_from, filter_to]) + +for vote_id in [row[0] for row in cur.fetchall()]: + subprocess.Popen((notify_cmd, 'tu-vote-reminder', str(vote_id))) -- cgit v1.2.3-54-g00ecf From 603b5b5db916da7d2a44e9e89587d62859840287 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Tue, 20 Sep 2016 08:42:59 +0200 Subject: Add a main() method to all Python scripts Move the main program logic of all scripts to main() methods such that they can be used as modules and easily be invoked by setuptools wrapper scripts. Signed-off-by: Lukas Fleischer --- scripts/aurblup.py | 86 +++++++++++++++++++++++++---------------------- scripts/mkpkglists.py | 52 ++++++++++++++++------------ scripts/notify.py | 6 +++- scripts/pkgmaint.py | 23 ++++++++----- scripts/popupdate.py | 31 ++++++++++------- scripts/tuvotereminder.py | 28 +++++++++------ 6 files changed, 132 insertions(+), 94 deletions(-) (limited to 'scripts') diff --git a/scripts/aurblup.py b/scripts/aurblup.py index 6733b45..07119b5 100755 --- a/scripts/aurblup.py +++ b/scripts/aurblup.py @@ -18,43 +18,49 @@ db_path = config.get('aurblup', 'db-path') sync_dbs = config.get('aurblup', 'sync-dbs').split(' ') servers = config.get('aurblup', 'servers').split(' ') -blacklist = set() -providers = set() -repomap = dict() - -h = pyalpm.Handle("/", db_path) -for sync_db in sync_dbs: - repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL) - repo.servers = [server.replace("%s", sync_db) for server in servers] - t = h.init_transaction() - repo.update(False) - t.release() - - for pkg in repo.pkgcache: - blacklist.add(pkg.name) - [blacklist.add(x) for x in pkg.replaces] - providers.add((pkg.name, pkg.name)) - repomap[(pkg.name, pkg.name)] = repo.name - for provision in pkg.provides: - provisionname = re.sub(r'(<|=|>).*', '', provision) - providers.add((pkg.name, provisionname)) - repomap[(pkg.name, provisionname)] = repo.name - -db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) -cur = db.cursor() - -cur.execute("SELECT Name, Provides FROM OfficialProviders") -oldproviders = set(cur.fetchall()) - -for pkg, provides in providers.difference(oldproviders): - repo = repomap[(pkg, provides)] - cur.execute("INSERT INTO OfficialProviders (Name, Repo, Provides) " - "VALUES (%s, %s, %s)", [pkg, repo, provides]) -for pkg, provides in oldproviders.difference(providers): - cur.execute("DELETE FROM OfficialProviders " - "WHERE Name = %s AND Provides = %s", [pkg, provides]) - -db.commit() -db.close() + +def main(): + blacklist = set() + providers = set() + repomap = dict() + + h = pyalpm.Handle("/", db_path) + for sync_db in sync_dbs: + repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL) + repo.servers = [server.replace("%s", sync_db) for server in servers] + t = h.init_transaction() + repo.update(False) + t.release() + + for pkg in repo.pkgcache: + blacklist.add(pkg.name) + [blacklist.add(x) for x in pkg.replaces] + providers.add((pkg.name, pkg.name)) + repomap[(pkg.name, pkg.name)] = repo.name + for provision in pkg.provides: + provisionname = re.sub(r'(<|=|>).*', '', provision) + providers.add((pkg.name, provisionname)) + repomap[(pkg.name, provisionname)] = repo.name + + db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket, buffered=True) + cur = db.cursor() + + cur.execute("SELECT Name, Provides FROM OfficialProviders") + oldproviders = set(cur.fetchall()) + + for pkg, provides in providers.difference(oldproviders): + repo = repomap[(pkg, provides)] + cur.execute("INSERT INTO OfficialProviders (Name, Repo, Provides) " + "VALUES (%s, %s, %s)", [pkg, repo, provides]) + for pkg, provides in oldproviders.difference(providers): + cur.execute("DELETE FROM OfficialProviders " + "WHERE Name = %s AND Provides = %s", [pkg, provides]) + + db.commit() + db.close() + + +if __name__ == '__main__': + main() diff --git a/scripts/mkpkglists.py b/scripts/mkpkglists.py index a6f8a19..8cab0a9 100755 --- a/scripts/mkpkglists.py +++ b/scripts/mkpkglists.py @@ -17,25 +17,33 @@ aur_db_user = config.get('database', 'user') aur_db_pass = config.get('database', 'password') aur_db_socket = config.get('database', 'socket') -db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) -cur = db.cursor() - -datestr = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") -pkglist_header = "# AUR package list, generated on " + datestr -pkgbaselist_header = "# AUR package base list, generated on " + datestr - -with gzip.open(docroot + "packages.gz", "w") as f: - f.write(bytes(pkglist_header + "\n", "UTF-8")) - cur.execute("SELECT Packages.Name FROM Packages INNER JOIN PackageBases " + - "ON PackageBases.ID = Packages.PackageBaseID " + - "WHERE PackageBases.PackagerUID IS NOT NULL") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - -with gzip.open(docroot + "pkgbase.gz", "w") as f: - f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) - cur.execute("SELECT Name FROM PackageBases WHERE PackagerUID IS NOT NULL") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - -db.close() + +def main(): + db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket, buffered=True) + cur = db.cursor() + + datestr = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") + pkglist_header = "# AUR package list, generated on " + datestr + pkgbaselist_header = "# AUR package base list, generated on " + datestr + + with gzip.open(docroot + "packages.gz", "w") as f: + f.write(bytes(pkglist_header + "\n", "UTF-8")) + cur.execute("SELECT Packages.Name FROM Packages " + + "INNER JOIN PackageBases " + + "ON PackageBases.ID = Packages.PackageBaseID " + + "WHERE PackageBases.PackagerUID IS NOT NULL") + f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + + with gzip.open(docroot + "pkgbase.gz", "w") as f: + f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) + cur.execute("SELECT Name FROM PackageBases " + + "WHERE PackagerUID IS NOT NULL") + f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + + db.close() + + +if __name__ == '__main__': + main() diff --git a/scripts/notify.py b/scripts/notify.py index a640e45..e8210a8 100755 --- a/scripts/notify.py +++ b/scripts/notify.py @@ -427,7 +427,7 @@ def tu_vote_reminder(cur, vote_id): send_notification(to, subject, body, refs) -if __name__ == '__main__': +def main(): action = sys.argv[1] action_map = { 'send-resetkey': send_resetkey, @@ -454,3 +454,7 @@ if __name__ == '__main__': db.commit() db.close() + + +if __name__ == '__main__': + main() diff --git a/scripts/pkgmaint.py b/scripts/pkgmaint.py index 0eb9422..346b046 100755 --- a/scripts/pkgmaint.py +++ b/scripts/pkgmaint.py @@ -13,13 +13,20 @@ aur_db_user = config.get('database', 'user') aur_db_pass = config.get('database', 'password') aur_db_socket = config.get('database', 'socket') -db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) -cur = db.cursor() -cur.execute("DELETE FROM PackageBases WHERE " + - "UNIX_TIMESTAMP() - SubmittedTS > 86400 AND PackagerUID IS NULL") +def main(): + db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket, buffered=True) + cur = db.cursor() -db.commit() -db.close() + cur.execute("DELETE FROM PackageBases WHERE " + + "UNIX_TIMESTAMP() - SubmittedTS > 86400 " + + "AND PackagerUID IS NULL") + + db.commit() + db.close() + + +if __name__ == '__main__': + main() diff --git a/scripts/popupdate.py b/scripts/popupdate.py index f3ba513..26d8379 100755 --- a/scripts/popupdate.py +++ b/scripts/popupdate.py @@ -13,18 +13,25 @@ aur_db_user = config.get('database', 'user') aur_db_pass = config.get('database', 'password') aur_db_socket = config.get('database', 'socket') -db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) -cur = db.cursor() -cur.execute("UPDATE PackageBases SET NumVotes = (SELECT COUNT(*) FROM " + - "PackageVotes WHERE PackageVotes.PackageBaseID = PackageBases.ID)") +def main(): + db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket, buffered=True) + cur = db.cursor() -cur.execute("UPDATE PackageBases SET Popularity = (" + - "SELECT COALESCE(SUM(POWER(0.98, (UNIX_TIMESTAMP() - VoteTS) / 86400)), 0.0) " + - "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " + - "PackageBases.ID AND NOT VoteTS IS NULL)") + cur.execute("UPDATE PackageBases SET NumVotes = (" + + "SELECT COUNT(*) FROM PackageVotes " + + "WHERE PackageVotes.PackageBaseID = PackageBases.ID)") -db.commit() -db.close() + cur.execute("UPDATE PackageBases SET Popularity = (" + + "SELECT COALESCE(SUM(POWER(0.98, (UNIX_TIMESTAMP() - VoteTS) / 86400)), 0.0) " + + "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " + + "PackageBases.ID AND NOT VoteTS IS NULL)") + + db.commit() + db.close() + + +if __name__ == '__main__': + main() diff --git a/scripts/tuvotereminder.py b/scripts/tuvotereminder.py index 0992623..dc16397 100755 --- a/scripts/tuvotereminder.py +++ b/scripts/tuvotereminder.py @@ -16,17 +16,23 @@ aur_db_pass = config.get('database', 'password') aur_db_socket = config.get('database', 'socket') notify_cmd = config.get('notifications', 'notify-cmd') -db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) -cur = db.cursor() -now = int(time.time()) -filter_from = now + 500 -filter_to = now + 172800 +def main(): + db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket, buffered=True) + cur = db.cursor() -cur.execute("SELECT ID FROM TU_VoteInfo WHERE End >= %s AND End <= %s", - [filter_from, filter_to]) + now = int(time.time()) + filter_from = now + 500 + filter_to = now + 172800 -for vote_id in [row[0] for row in cur.fetchall()]: - subprocess.Popen((notify_cmd, 'tu-vote-reminder', str(vote_id))) + cur.execute("SELECT ID FROM TU_VoteInfo WHERE End >= %s AND End <= %s", + [filter_from, filter_to]) + + for vote_id in [row[0] for row in cur.fetchall()]: + subprocess.Popen((notify_cmd, 'tu-vote-reminder', str(vote_id))) + + +if __name__ == '__main__': + main() -- cgit v1.2.3-54-g00ecf From dc3fd60715a5b17b9542ec888c6eaeb14c284e2b Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Tue, 20 Sep 2016 20:18:24 +0200 Subject: Use setuptools to install Python modules Instead of using relative imports, add support for installing the config and db Python modules to a proper location using setuptools. Change all git-interface scripts to access those modules from the search path. Signed-off-by: Lukas Fleischer --- aurweb/__init__.py | 0 aurweb/config.py | 31 +++++++++++++++++++++++++++ aurweb/db.py | 51 +++++++++++++++++++++++++++++++++++++++++++++ git-interface/__init__.py | 0 git-interface/config.py | 31 --------------------------- git-interface/db.py | 51 --------------------------------------------- git-interface/git-auth.py | 14 ++++++------- git-interface/git-serve.py | 40 +++++++++++++++++------------------ git-interface/git-update.py | 14 ++++++------- git-interface/test/setup.sh | 4 ++++ scripts/__init__.py | 0 setup.py | 20 ++++++++++++++++++ 12 files changed, 140 insertions(+), 116 deletions(-) create mode 100644 aurweb/__init__.py create mode 100644 aurweb/config.py create mode 100644 aurweb/db.py create mode 100644 git-interface/__init__.py delete mode 100644 git-interface/config.py delete mode 100644 git-interface/db.py create mode 100644 scripts/__init__.py create mode 100644 setup.py (limited to 'scripts') diff --git a/aurweb/__init__.py b/aurweb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aurweb/config.py b/aurweb/config.py new file mode 100644 index 0000000..aac188b --- /dev/null +++ b/aurweb/config.py @@ -0,0 +1,31 @@ +import configparser +import os + +_parser = None + + +def _get_parser(): + global _parser + + if not _parser: + _parser = configparser.RawConfigParser() + if 'AUR_CONFIG' in os.environ: + path = os.environ.get('AUR_CONFIG') + else: + relpath = "/../conf/config" + path = os.path.dirname(os.path.realpath(__file__)) + relpath + _parser.read(path) + + return _parser + + +def get(section, option): + return _get_parser().get(section, option) + + +def getboolean(section, option): + return _get_parser().getboolean(section, option) + + +def getint(section, option): + return _get_parser().getint(section, option) diff --git a/aurweb/db.py b/aurweb/db.py new file mode 100644 index 0000000..0b58197 --- /dev/null +++ b/aurweb/db.py @@ -0,0 +1,51 @@ +import mysql.connector +import sqlite3 + +import aurweb.config + + +class Connection: + _conn = None + _paramstyle = None + + def __init__(self): + aur_db_backend = aurweb.config.get('database', 'backend') + + if aur_db_backend == 'mysql': + aur_db_host = aurweb.config.get('database', 'host') + aur_db_name = aurweb.config.get('database', 'name') + aur_db_user = aurweb.config.get('database', 'user') + aur_db_pass = aurweb.config.get('database', 'password') + aur_db_socket = aurweb.config.get('database', 'socket') + self._conn = mysql.connector.connect(host=aur_db_host, + user=aur_db_user, + passwd=aur_db_pass, + db=aur_db_name, + unix_socket=aur_db_socket, + buffered=True) + self._paramstyle = mysql.connector.paramstyle + elif aur_db_backend == 'sqlite': + aur_db_name = aurweb.config.get('database', 'name') + self._conn = sqlite3.connect(aur_db_name) + self._paramstyle = sqlite3.paramstyle + else: + raise ValueError('unsupported database backend') + + def execute(self, query, params=()): + if self._paramstyle in ('format', 'pyformat'): + query = query.replace('%', '%%').replace('?', '%s') + elif self._paramstyle == 'qmark': + pass + else: + raise ValueError('unsupported paramstyle') + + cur = self._conn.cursor() + cur.execute(query, params) + + return cur + + def commit(self): + self._conn.commit() + + def close(self): + self._conn.close() diff --git a/git-interface/__init__.py b/git-interface/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/git-interface/config.py b/git-interface/config.py deleted file mode 100644 index aac188b..0000000 --- a/git-interface/config.py +++ /dev/null @@ -1,31 +0,0 @@ -import configparser -import os - -_parser = None - - -def _get_parser(): - global _parser - - if not _parser: - _parser = configparser.RawConfigParser() - if 'AUR_CONFIG' in os.environ: - path = os.environ.get('AUR_CONFIG') - else: - relpath = "/../conf/config" - path = os.path.dirname(os.path.realpath(__file__)) + relpath - _parser.read(path) - - return _parser - - -def get(section, option): - return _get_parser().get(section, option) - - -def getboolean(section, option): - return _get_parser().getboolean(section, option) - - -def getint(section, option): - return _get_parser().getint(section, option) diff --git a/git-interface/db.py b/git-interface/db.py deleted file mode 100644 index 75d2283..0000000 --- a/git-interface/db.py +++ /dev/null @@ -1,51 +0,0 @@ -import mysql.connector -import sqlite3 - -import config - - -class Connection: - _conn = None - _paramstyle = None - - def __init__(self): - aur_db_backend = config.get('database', 'backend') - - if aur_db_backend == 'mysql': - aur_db_host = config.get('database', 'host') - aur_db_name = config.get('database', 'name') - aur_db_user = config.get('database', 'user') - aur_db_pass = config.get('database', 'password') - aur_db_socket = config.get('database', 'socket') - self._conn = mysql.connector.connect(host=aur_db_host, - user=aur_db_user, - passwd=aur_db_pass, - db=aur_db_name, - unix_socket=aur_db_socket, - buffered=True) - self._paramstyle = mysql.connector.paramstyle - elif aur_db_backend == 'sqlite': - aur_db_name = config.get('database', 'name') - self._conn = sqlite3.connect(aur_db_name) - self._paramstyle = sqlite3.paramstyle - else: - raise ValueError('unsupported database backend') - - def execute(self, query, params=()): - if self._paramstyle in ('format', 'pyformat'): - query = query.replace('%', '%%').replace('?', '%s') - elif self._paramstyle == 'qmark': - pass - else: - raise ValueError('unsupported paramstyle') - - cur = self._conn.cursor() - cur.execute(query, params) - - return cur - - def commit(self): - self._conn.commit() - - def close(self): - self._conn.close() diff --git a/git-interface/git-auth.py b/git-interface/git-auth.py index d3b0188..022b0ff 100755 --- a/git-interface/git-auth.py +++ b/git-interface/git-auth.py @@ -4,8 +4,8 @@ import shlex import re import sys -import config -import db +import aurweb.config +import aurweb.db def format_command(env_vars, command, ssh_opts, ssh_key): @@ -24,17 +24,17 @@ def format_command(env_vars, command, ssh_opts, ssh_key): def main(): - valid_keytypes = config.get('auth', 'valid-keytypes').split() - username_regex = config.get('auth', 'username-regex') - git_serve_cmd = config.get('auth', 'git-serve-cmd') - ssh_opts = config.get('auth', 'ssh-options') + valid_keytypes = aurweb.config.get('auth', 'valid-keytypes').split() + username_regex = aurweb.config.get('auth', 'username-regex') + git_serve_cmd = aurweb.config.get('auth', 'git-serve-cmd') + ssh_opts = aurweb.config.get('auth', 'ssh-options') keytype = sys.argv[1] keytext = sys.argv[2] if keytype not in valid_keytypes: exit(1) - conn = db.Connection() + conn = aurweb.db.Connection() cur = conn.execute("SELECT Users.Username, Users.AccountTypeID FROM Users " "INNER JOIN SSHPubKeys ON SSHPubKeys.UserID = Users.ID " diff --git a/git-interface/git-serve.py b/git-interface/git-serve.py index 8bcecd2..5f3b26d 100755 --- a/git-interface/git-serve.py +++ b/git-interface/git-serve.py @@ -7,23 +7,23 @@ import subprocess import sys import time -import config -import db +import aurweb.config +import aurweb.db -notify_cmd = config.get('notifications', 'notify-cmd') +notify_cmd = aurweb.config.get('notifications', 'notify-cmd') -repo_path = config.get('serve', 'repo-path') -repo_regex = config.get('serve', 'repo-regex') -git_shell_cmd = config.get('serve', 'git-shell-cmd') -git_update_cmd = config.get('serve', 'git-update-cmd') -ssh_cmdline = config.get('serve', 'ssh-cmdline') +repo_path = aurweb.config.get('serve', 'repo-path') +repo_regex = aurweb.config.get('serve', 'repo-regex') +git_shell_cmd = aurweb.config.get('serve', 'git-shell-cmd') +git_update_cmd = aurweb.config.get('serve', 'git-update-cmd') +ssh_cmdline = aurweb.config.get('serve', 'ssh-cmdline') -enable_maintenance = config.getboolean('options', 'enable-maintenance') -maintenance_exc = config.get('options', 'maintenance-exceptions').split() +enable_maintenance = aurweb.config.getboolean('options', 'enable-maintenance') +maintenance_exc = aurweb.config.get('options', 'maintenance-exceptions').split() def pkgbase_from_name(pkgbase): - conn = db.Connection() + conn = aurweb.db.Connection() cur = conn.execute("SELECT ID FROM PackageBases WHERE Name = ?", [pkgbase]) row = cur.fetchone() @@ -35,7 +35,7 @@ def pkgbase_exists(pkgbase): def list_repos(user): - conn = db.Connection() + conn = aurweb.db.Connection() cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) userid = cur.fetchone()[0] @@ -55,7 +55,7 @@ def create_pkgbase(pkgbase, user): if pkgbase_exists(pkgbase): die('{:s}: package base already exists: {:s}'.format(action, pkgbase)) - conn = db.Connection() + conn = aurweb.db.Connection() cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) userid = cur.fetchone()[0] @@ -81,7 +81,7 @@ def pkgbase_adopt(pkgbase, user, privileged): if not pkgbase_id: die('{:s}: package base not found: {:s}'.format(action, pkgbase)) - conn = db.Connection() + conn = aurweb.db.Connection() cur = conn.execute("SELECT ID FROM PackageBases WHERE ID = ? AND " + "MaintainerUID IS NULL", [pkgbase_id]) @@ -111,7 +111,7 @@ def pkgbase_adopt(pkgbase, user, privileged): def pkgbase_get_comaintainers(pkgbase): - conn = db.Connection() + conn = aurweb.db.Connection() cur = conn.execute("SELECT UserName FROM PackageComaintainers " + "INNER JOIN Users " + @@ -132,7 +132,7 @@ def pkgbase_set_comaintainers(pkgbase, userlist, user, privileged): if not privileged and not pkgbase_has_full_access(pkgbase, user): die('{:s}: permission denied: {:s}'.format(action, user)) - conn = db.Connection() + conn = aurweb.db.Connection() userlist_old = set(pkgbase_get_comaintainers(pkgbase)) @@ -198,7 +198,7 @@ def pkgbase_disown(pkgbase, user, privileged): comaintainers = [] new_maintainer_userid = None - conn = db.Connection() + conn = aurweb.db.Connection() # Make the first co-maintainer the new maintainer, unless the action was # enforced by a Trusted User. @@ -232,7 +232,7 @@ def pkgbase_set_keywords(pkgbase, keywords): if not pkgbase_id: die('{:s}: package base not found: {:s}'.format(action, pkgbase)) - conn = db.Connection() + conn = aurweb.db.Connection() conn.execute("DELETE FROM PackageKeywords WHERE PackageBaseID = ?", [pkgbase_id]) @@ -245,7 +245,7 @@ def pkgbase_set_keywords(pkgbase, keywords): def pkgbase_has_write_access(pkgbase, user): - conn = db.Connection() + conn = aurweb.db.Connection() cur = conn.execute("SELECT COUNT(*) FROM PackageBases " + "LEFT JOIN PackageComaintainers " + @@ -259,7 +259,7 @@ def pkgbase_has_write_access(pkgbase, user): def pkgbase_has_full_access(pkgbase, user): - conn = db.Connection() + conn = aurweb.db.Connection() cur = conn.execute("SELECT COUNT(*) FROM PackageBases " + "INNER JOIN Users " + diff --git a/git-interface/git-update.py b/git-interface/git-update.py index 36c38ae..7337341 100755 --- a/git-interface/git-update.py +++ b/git-interface/git-update.py @@ -10,15 +10,15 @@ import time import srcinfo.parse import srcinfo.utils -import config -import db +import aurweb.config +import aurweb.db -notify_cmd = config.get('notifications', 'notify-cmd') +notify_cmd = aurweb.config.get('notifications', 'notify-cmd') -repo_path = config.get('serve', 'repo-path') -repo_regex = config.get('serve', 'repo-regex') +repo_path = aurweb.config.get('serve', 'repo-path') +repo_regex = aurweb.config.get('serve', 'repo-regex') -max_blob_size = config.getint('update', 'max-blob-size') +max_blob_size = aurweb.config.getint('update', 'max-blob-size') def size_humanize(num): @@ -256,7 +256,7 @@ def main(): if refname != "refs/heads/master": die("pushing to a branch other than master is restricted") - conn = db.Connection() + conn = aurweb.db.Connection() # Detect and deny non-fast-forwards. if sha1_old != "0" * 40 and not privileged: diff --git a/git-interface/test/setup.sh b/git-interface/test/setup.sh index f9c1616..d269af6 100644 --- a/git-interface/test/setup.sh +++ b/git-interface/test/setup.sh @@ -2,6 +2,10 @@ TEST_DIRECTORY="$(pwd)" . ./sharness.sh +# Configure python search path. +PYTHONPATH="$TEST_DIRECTORY/../../" +export PYTHONPATH + # Configure paths to the Git interface scripts. GIT_AUTH="$TEST_DIRECTORY/../git-auth.py" GIT_SERVE="$TEST_DIRECTORY/../git-serve.py" diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..48eb176 --- /dev/null +++ b/setup.py @@ -0,0 +1,20 @@ +import re +from setuptools import setup, find_packages +import sys + +version = None +with open('web/lib/version.inc.php', 'r') as f: + for line in f.readlines(): + match = re.match(r'^define\("AURWEB_VERSION", "v([0-9.]+)"\);$', line) + if match: + version = match.group(1) + +if not version: + sys.stderr.write('error: Failed to parse version file!') + sys.exit(1) + +setup( + name="aurweb", + version=version, + packages=find_packages(), +) -- cgit v1.2.3-54-g00ecf From 8c99184f6d0922a7a4076d0c050a924e07a42b3d Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Tue, 20 Sep 2016 20:48:34 +0200 Subject: Use config and db in scripts Instead of using configparser and mysql.connector directly, change all Python scripts to use the config and db Python modules which are now accessible from a common location. Signed-off-by: Lukas Fleischer --- scripts/aurblup.py | 37 +++---- scripts/mkpkglists.py | 32 ++---- scripts/notify.py | 273 +++++++++++++++++++++++----------------------- scripts/pkgmaint.py | 28 ++--- scripts/popupdate.py | 40 +++---- scripts/tuvotereminder.py | 24 ++-- 6 files changed, 184 insertions(+), 250 deletions(-) (limited to 'scripts') diff --git a/scripts/aurblup.py b/scripts/aurblup.py index 07119b5..0009715 100755 --- a/scripts/aurblup.py +++ b/scripts/aurblup.py @@ -1,22 +1,14 @@ #!/usr/bin/python3 -import configparser -import mysql.connector -import os import pyalpm import re -config = configparser.RawConfigParser() -config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") +import aurweb.config +import aurweb.db -aur_db_host = config.get('database', 'host') -aur_db_name = config.get('database', 'name') -aur_db_user = config.get('database', 'user') -aur_db_pass = config.get('database', 'password') -aur_db_socket = config.get('database', 'socket') -db_path = config.get('aurblup', 'db-path') -sync_dbs = config.get('aurblup', 'sync-dbs').split(' ') -servers = config.get('aurblup', 'servers').split(' ') +db_path = aurweb.config.get('aurblup', 'db-path') +sync_dbs = aurweb.config.get('aurblup', 'sync-dbs').split(' ') +servers = aurweb.config.get('aurblup', 'servers').split(' ') def main(): @@ -42,24 +34,21 @@ def main(): providers.add((pkg.name, provisionname)) repomap[(pkg.name, provisionname)] = repo.name - db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) - cur = db.cursor() + conn = aurweb.db.Connection() - cur.execute("SELECT Name, Provides FROM OfficialProviders") + cur = conn.execute("SELECT Name, Provides FROM OfficialProviders") oldproviders = set(cur.fetchall()) for pkg, provides in providers.difference(oldproviders): repo = repomap[(pkg, provides)] - cur.execute("INSERT INTO OfficialProviders (Name, Repo, Provides) " - "VALUES (%s, %s, %s)", [pkg, repo, provides]) + conn.execute("INSERT INTO OfficialProviders (Name, Repo, Provides) " + "VALUES (?, ?, ?)", [pkg, repo, provides]) for pkg, provides in oldproviders.difference(providers): - cur.execute("DELETE FROM OfficialProviders " - "WHERE Name = %s AND Provides = %s", [pkg, provides]) + conn.execute("DELETE FROM OfficialProviders " + "WHERE Name = ? AND Provides = ?", [pkg, provides]) - db.commit() - db.close() + conn.commit() + conn.close() if __name__ == '__main__': diff --git a/scripts/mkpkglists.py b/scripts/mkpkglists.py index 8cab0a9..70cbd13 100755 --- a/scripts/mkpkglists.py +++ b/scripts/mkpkglists.py @@ -1,28 +1,16 @@ #!/usr/bin/python3 -import configparser import datetime import gzip -import mysql.connector import os -docroot = os.path.dirname(os.path.realpath(__file__)) + "/../web/html/" - -config = configparser.RawConfigParser() -config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") +import aurweb.db -aur_db_host = config.get('database', 'host') -aur_db_name = config.get('database', 'name') -aur_db_user = config.get('database', 'user') -aur_db_pass = config.get('database', 'password') -aur_db_socket = config.get('database', 'socket') +docroot = os.path.dirname(os.path.realpath(__file__)) + "/../web/html/" def main(): - db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) - cur = db.cursor() + conn = aurweb.db.Connection() datestr = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") pkglist_header = "# AUR package list, generated on " + datestr @@ -30,19 +18,19 @@ def main(): with gzip.open(docroot + "packages.gz", "w") as f: f.write(bytes(pkglist_header + "\n", "UTF-8")) - cur.execute("SELECT Packages.Name FROM Packages " + - "INNER JOIN PackageBases " + - "ON PackageBases.ID = Packages.PackageBaseID " + - "WHERE PackageBases.PackagerUID IS NOT NULL") + cur = conn.execute("SELECT Packages.Name FROM Packages " + + "INNER JOIN PackageBases " + + "ON PackageBases.ID = Packages.PackageBaseID " + + "WHERE PackageBases.PackagerUID IS NOT NULL") f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) with gzip.open(docroot + "pkgbase.gz", "w") as f: f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) - cur.execute("SELECT Name FROM PackageBases " + - "WHERE PackagerUID IS NOT NULL") + cur = conn.execute("SELECT Name FROM PackageBases " + + "WHERE PackagerUID IS NOT NULL") f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - db.close() + conn.close() if __name__ == '__main__': diff --git a/scripts/notify.py b/scripts/notify.py index e8210a8..ddd6e49 100755 --- a/scripts/notify.py +++ b/scripts/notify.py @@ -1,28 +1,19 @@ #!/usr/bin/python3 -import configparser import email.mime.text -import mysql.connector -import os import subprocess import sys import textwrap -config = configparser.RawConfigParser() -config.read(os.path.dirname(os.path.realpath(__file__)) + '/../conf/config') +import aurweb.config +import aurweb.db -aur_db_host = config.get('database', 'host') -aur_db_name = config.get('database', 'name') -aur_db_user = config.get('database', 'user') -aur_db_pass = config.get('database', 'password') -aur_db_socket = config.get('database', 'socket') +aur_location = aurweb.config.get('options', 'aur_location') +aur_request_ml = aurweb.config.get('options', 'aur_request_ml') -aur_location = config.get('options', 'aur_location') -aur_request_ml = config.get('options', 'aur_request_ml') - -sendmail = config.get('notifications', 'sendmail') -sender = config.get('notifications', 'sender') -reply_to = config.get('notifications', 'reply-to') +sendmail = aurweb.config.get('notifications', 'sendmail') +sender = aurweb.config.get('notifications', 'sender') +reply_to = aurweb.config.get('notifications', 'reply-to') def headers_cc(cclist): @@ -60,120 +51,127 @@ def send_notification(to, subject, body, refs, headers={}): p.communicate(msg.as_bytes()) -def username_from_id(cur, uid): - cur.execute('SELECT UserName FROM Users WHERE ID = %s', [uid]) +def username_from_id(conn, uid): + cur = conn.execute('SELECT UserName FROM Users WHERE ID = ?', [uid]) return cur.fetchone()[0] -def pkgbase_from_id(cur, pkgbase_id): - cur.execute('SELECT Name FROM PackageBases WHERE ID = %s', [pkgbase_id]) +def pkgbase_from_id(conn, pkgbase_id): + cur = conn.execute('SELECT Name FROM PackageBases WHERE ID = ?', + [pkgbase_id]) return cur.fetchone()[0] -def pkgbase_from_pkgreq(cur, reqid): - cur.execute('SELECT PackageBaseID FROM PackageRequests WHERE ID = %s', - [reqid]) +def pkgbase_from_pkgreq(conn, reqid): + cur = conn.execute('SELECT PackageBaseID FROM PackageRequests ' + + 'WHERE ID = ?', [reqid]) return cur.fetchone()[0] -def get_user_email(cur, uid): - cur.execute('SELECT Email FROM Users WHERE ID = %s', [uid]) +def get_user_email(conn, uid): + cur = conn.execute('SELECT Email FROM Users WHERE ID = ?', [uid]) return cur.fetchone()[0] -def get_maintainer_email(cur, pkgbase_id): - cur.execute('SELECT Users.Email FROM Users ' + - 'INNER JOIN PackageBases ' + - 'ON PackageBases.MaintainerUID = Users.ID WHERE ' + - 'PackageBases.ID = %s', [pkgbase_id]) +def get_maintainer_email(conn, pkgbase_id): + cur = conn.execute('SELECT Users.Email FROM Users ' + + 'INNER JOIN PackageBases ' + + 'ON PackageBases.MaintainerUID = Users.ID WHERE ' + + 'PackageBases.ID = ?', [pkgbase_id]) return cur.fetchone()[0] -def get_recipients(cur, pkgbase_id, uid): - cur.execute('SELECT DISTINCT Users.Email FROM Users ' + - 'INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'PackageNotifications.UserID != %s AND ' + - 'PackageNotifications.PackageBaseID = %s', [uid, pkgbase_id]) +def get_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) return [row[0] for row in cur.fetchall()] -def get_comment_recipients(cur, pkgbase_id, uid): - cur.execute('SELECT DISTINCT Users.Email FROM Users ' + - 'INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'Users.CommentNotify = 1 AND ' + - 'PackageNotifications.UserID != %s AND ' + - 'PackageNotifications.PackageBaseID = %s', [uid, pkgbase_id]) +def get_comment_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'Users.CommentNotify = 1 AND ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) return [row[0] for row in cur.fetchall()] -def get_update_recipients(cur, pkgbase_id, uid): - cur.execute('SELECT DISTINCT Users.Email FROM Users ' + - 'INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'Users.UpdateNotify = 1 AND ' + - 'PackageNotifications.UserID != %s AND ' + - 'PackageNotifications.PackageBaseID = %s', [uid, pkgbase_id]) +def get_update_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'Users.UpdateNotify = 1 AND ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) return [row[0] for row in cur.fetchall()] -def get_ownership_recipients(cur, pkgbase_id, uid): - cur.execute('SELECT DISTINCT Users.Email FROM Users ' + - 'INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'Users.OwnershipNotify = 1 AND ' + - 'PackageNotifications.UserID != %s AND ' + - 'PackageNotifications.PackageBaseID = %s', [uid, pkgbase_id]) +def get_ownership_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'Users.OwnershipNotify = 1 AND ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) return [row[0] for row in cur.fetchall()] -def get_request_recipients(cur, reqid): - cur.execute('SELECT DISTINCT Users.Email FROM PackageRequests ' + - 'INNER JOIN PackageBases ' + - 'ON PackageBases.ID = PackageRequests.PackageBaseID ' + - 'INNER JOIN Users ' + - 'ON Users.ID = PackageRequests.UsersID ' + - 'OR Users.ID = PackageBases.MaintainerUID ' + - 'WHERE PackageRequests.ID = %s', [reqid]) +def get_request_recipients(conn, reqid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM PackageRequests ' + + 'INNER JOIN PackageBases ' + + 'ON PackageBases.ID = PackageRequests.PackageBaseID ' + + 'INNER JOIN Users ' + + 'ON Users.ID = PackageRequests.UsersID ' + + 'OR Users.ID = PackageBases.MaintainerUID ' + + 'WHERE PackageRequests.ID = ?', [reqid]) return [row[0] for row in cur.fetchall()] -def get_tu_vote_reminder_recipients(cur, vote_id): - cur.execute('SELECT Users.Email FROM Users WHERE AccountTypeID = 2 ' + - 'EXCEPT SELECT Users.Email FROM Users ' + - 'INNER JOIN TU_Votes ' + - 'ON TU_Votes.UserID = Users.ID ' + - 'WHERE TU_Votes.VoteID = %s', [vote_id]) +def get_tu_vote_reminder_recipients(conn, vote_id): + cur = conn.execute('SELECT Users.Email FROM Users ' + + 'WHERE AccountTypeID = 2 ' + + 'EXCEPT SELECT Users.Email FROM Users ' + + 'INNER JOIN TU_Votes ' + + 'ON TU_Votes.UserID = Users.ID ' + + 'WHERE TU_Votes.VoteID = ?', [vote_id]) return [row[0] for row in cur.fetchall()] -def get_comment(cur, comment_id): - cur.execute('SELECT Comments FROM PackageComments WHERE ID = %s', - [comment_id]) +def get_comment(conn, comment_id): + cur = conn.execute('SELECT Comments FROM PackageComments WHERE ID = ?', + [comment_id]) return cur.fetchone()[0] -def get_flagger_comment(cur, pkgbase_id): - cur.execute('SELECT FlaggerComment FROM PackageBases WHERE ID = %s', - [pkgbase_id]) +def get_flagger_comment(conn, pkgbase_id): + cur = conn.execute('SELECT FlaggerComment FROM PackageBases WHERE ID = ?', + [pkgbase_id]) return cur.fetchone()[0] -def get_request_comment(cur, reqid): - cur.execute('SELECT Comments FROM PackageRequests WHERE ID = %s', [reqid]) +def get_request_comment(conn, reqid): + cur = conn.execute('SELECT Comments FROM PackageRequests WHERE ID = ?', + [reqid]) return cur.fetchone()[0] -def get_request_closure_comment(cur, reqid): - cur.execute('SELECT ClosureComment FROM PackageRequests WHERE ID = %s', - [reqid]) +def get_request_closure_comment(conn, reqid): + cur = conn.execute('SELECT ClosureComment FROM PackageRequests ' + + 'WHERE ID = ?', [reqid]) return cur.fetchone()[0] -def send_resetkey(cur, uid): - cur.execute('SELECT UserName, Email, ResetKey FROM Users WHERE ID = %s', - [uid]) +def send_resetkey(conn, uid): + cur = conn.execute('SELECT UserName, Email, ResetKey FROM Users ' + + 'WHERE ID = ?', [uid]) username, to, resetkey = cur.fetchone() subject = 'AUR Password Reset' @@ -186,9 +184,9 @@ def send_resetkey(cur, uid): send_notification([to], subject, body, refs) -def welcome(cur, uid): - cur.execute('SELECT UserName, Email, ResetKey FROM Users WHERE ID = %s', - [uid]) +def welcome(conn, uid): + cur = conn.execute('SELECT UserName, Email, ResetKey FROM Users ' + + 'WHERE ID = ?', [uid]) username, to, resetkey = cur.fetchone() subject = 'Welcome to the Arch User Repository' @@ -201,11 +199,11 @@ def welcome(cur, uid): send_notification([to], subject, body, refs) -def comment(cur, uid, pkgbase_id, comment_id): - user = username_from_id(cur, uid) - pkgbase = pkgbase_from_id(cur, pkgbase_id) - to = get_comment_recipients(cur, pkgbase_id, uid) - text = get_comment(cur, comment_id) +def comment(conn, uid, pkgbase_id, comment_id): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_comment_recipients(conn, pkgbase_id, uid) + text = get_comment(conn, comment_id) user_uri = aur_location + '/account/' + user + '/' pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' @@ -224,10 +222,10 @@ def comment(cur, uid, pkgbase_id, comment_id): send_notification(to, subject, body, refs, headers) -def update(cur, uid, pkgbase_id): - user = username_from_id(cur, uid) - pkgbase = pkgbase_from_id(cur, pkgbase_id) - to = get_update_recipients(cur, pkgbase_id, uid) +def update(conn, uid, pkgbase_id): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_update_recipients(conn, pkgbase_id, uid) user_uri = aur_location + '/account/' + user + '/' pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' @@ -246,11 +244,11 @@ def update(cur, uid, pkgbase_id): send_notification(to, subject, body, refs, headers) -def flag(cur, uid, pkgbase_id): - user = username_from_id(cur, uid) - pkgbase = pkgbase_from_id(cur, pkgbase_id) - to = [get_maintainer_email(cur, pkgbase_id)] - text = get_flagger_comment(cur, pkgbase_id) +def flag(conn, uid, pkgbase_id): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = [get_maintainer_email(conn, pkgbase_id)] + text = get_flagger_comment(conn, pkgbase_id) user_uri = aur_location + '/account/' + user + '/' pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' @@ -265,10 +263,10 @@ def flag(cur, uid, pkgbase_id): send_notification(to, subject, body, refs) -def adopt(cur, pkgbase_id, uid): - user = username_from_id(cur, uid) - pkgbase = pkgbase_from_id(cur, pkgbase_id) - to = get_ownership_recipients(cur, pkgbase_id, uid) +def adopt(conn, pkgbase_id, uid): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_ownership_recipients(conn, pkgbase_id, uid) user_uri = aur_location + '/account/' + user + '/' pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' @@ -281,10 +279,10 @@ def adopt(cur, pkgbase_id, uid): send_notification(to, subject, body, refs) -def disown(cur, pkgbase_id, uid): - user = username_from_id(cur, uid) - pkgbase = pkgbase_from_id(cur, pkgbase_id) - to = get_ownership_recipients(cur, pkgbase_id, uid) +def disown(conn, pkgbase_id, uid): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_ownership_recipients(conn, pkgbase_id, uid) user_uri = aur_location + '/account/' + user + '/' pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' @@ -297,9 +295,9 @@ def disown(cur, pkgbase_id, uid): send_notification(to, subject, body, refs) -def comaintainer_add(cur, pkgbase_id, uid): - pkgbase = pkgbase_from_id(cur, pkgbase_id) - to = [get_user_email(cur, uid)] +def comaintainer_add(conn, pkgbase_id, uid): + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = [get_user_email(conn, uid)] pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' @@ -310,9 +308,9 @@ def comaintainer_add(cur, pkgbase_id, uid): send_notification(to, subject, body, refs) -def comaintainer_remove(cur, pkgbase_id, uid): - pkgbase = pkgbase_from_id(cur, pkgbase_id) - to = [get_user_email(cur, uid)] +def comaintainer_remove(conn, pkgbase_id, uid): + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = [get_user_email(conn, uid)] pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' @@ -324,12 +322,12 @@ def comaintainer_remove(cur, pkgbase_id, uid): send_notification(to, subject, body, refs) -def delete(cur, uid, old_pkgbase_id, new_pkgbase_id=None): - user = username_from_id(cur, uid) - old_pkgbase = pkgbase_from_id(cur, old_pkgbase_id) +def delete(conn, uid, old_pkgbase_id, new_pkgbase_id=None): + user = username_from_id(conn, uid) + old_pkgbase = pkgbase_from_id(conn, old_pkgbase_id) if new_pkgbase_id: - new_pkgbase = pkgbase_from_id(cur, new_pkgbase_id) - to = get_recipients(cur, old_pkgbase_id, uid) + new_pkgbase = pkgbase_from_id(conn, new_pkgbase_id) + to = get_recipients(conn, old_pkgbase_id, uid) user_uri = aur_location + '/account/' + user + '/' pkgbase_uri = aur_location + '/pkgbase/' + old_pkgbase + '/' @@ -354,12 +352,12 @@ def delete(cur, uid, old_pkgbase_id, new_pkgbase_id=None): send_notification(to, subject, body, refs) -def request_open(cur, uid, reqid, reqtype, pkgbase_id, merge_into=None): - user = username_from_id(cur, uid) - pkgbase = pkgbase_from_id(cur, pkgbase_id) +def request_open(conn, uid, reqid, reqtype, pkgbase_id, merge_into=None): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) to = [aur_request_ml] - cc = get_request_recipients(cur, reqid) - text = get_request_comment(cur, reqid) + cc = get_request_recipients(conn, reqid) + text = get_request_comment(conn, reqid) user_uri = aur_location + '/account/' + user + '/' pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' @@ -388,14 +386,14 @@ def request_open(cur, uid, reqid, reqtype, pkgbase_id, merge_into=None): send_notification(to, subject, body, refs, headers) -def request_close(cur, uid, reqid, reason): +def request_close(conn, uid, reqid, reason): to = [aur_request_ml] - cc = get_request_recipients(cur, reqid) - text = get_request_closure_comment(cur, reqid) + cc = get_request_recipients(conn, reqid) + text = get_request_closure_comment(conn, reqid) subject = '[PRQ#%d] Request %s' % (int(reqid), reason.title()) if int(uid): - user = username_from_id(cur, uid) + user = username_from_id(conn, uid) user_uri = aur_location + '/account/' + user + '/' body = 'Request #%d has been %s by %s [1]' % (int(reqid), reason, user) refs = '[1] ' + user_uri @@ -414,8 +412,8 @@ def request_close(cur, uid, reqid, reason): send_notification(to, subject, body, refs, headers) -def tu_vote_reminder(cur, vote_id): - to = get_tu_vote_reminder_recipients(cur, vote_id) +def tu_vote_reminder(conn, vote_id): + to = get_tu_vote_reminder_recipients(conn, vote_id) vote_uri = aur_location + '/tu/?id=' + vote_id @@ -445,15 +443,12 @@ def main(): 'tu-vote-reminder': tu_vote_reminder, } - db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) - cur = db.cursor() + conn = aurweb.db.Connection() - action_map[action](cur, *sys.argv[2:]) + action_map[action](conn, *sys.argv[2:]) - db.commit() - db.close() + conn.commit() + conn.close() if __name__ == '__main__': diff --git a/scripts/pkgmaint.py b/scripts/pkgmaint.py index 346b046..d0f8fe7 100755 --- a/scripts/pkgmaint.py +++ b/scripts/pkgmaint.py @@ -1,31 +1,17 @@ #!/usr/bin/python3 -import configparser -import mysql.connector -import os - -config = configparser.RawConfigParser() -config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") - -aur_db_host = config.get('database', 'host') -aur_db_name = config.get('database', 'name') -aur_db_user = config.get('database', 'user') -aur_db_pass = config.get('database', 'password') -aur_db_socket = config.get('database', 'socket') +import aurweb.db def main(): - db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) - cur = db.cursor() + conn = aurweb.db.Connection() - cur.execute("DELETE FROM PackageBases WHERE " + - "UNIX_TIMESTAMP() - SubmittedTS > 86400 " + - "AND PackagerUID IS NULL") + conn.execute("DELETE FROM PackageBases WHERE " + + "UNIX_TIMESTAMP() - SubmittedTS > 86400 " + + "AND PackagerUID IS NULL") - db.commit() - db.close() + conn.commit() + conn.close() if __name__ == '__main__': diff --git a/scripts/popupdate.py b/scripts/popupdate.py index 26d8379..f5e09d9 100755 --- a/scripts/popupdate.py +++ b/scripts/popupdate.py @@ -1,36 +1,22 @@ #!/usr/bin/python3 -import configparser -import mysql.connector -import os +import aurweb.db -config = configparser.RawConfigParser() -config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") -aur_db_host = config.get('database', 'host') -aur_db_name = config.get('database', 'name') -aur_db_user = config.get('database', 'user') -aur_db_pass = config.get('database', 'password') -aur_db_socket = config.get('database', 'socket') +def main(): + conn = aurweb.db.Connection() + conn.execute("UPDATE PackageBases SET NumVotes = (" + + "SELECT COUNT(*) FROM PackageVotes " + + "WHERE PackageVotes.PackageBaseID = PackageBases.ID)") -def main(): - db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) - cur = db.cursor() - - cur.execute("UPDATE PackageBases SET NumVotes = (" + - "SELECT COUNT(*) FROM PackageVotes " + - "WHERE PackageVotes.PackageBaseID = PackageBases.ID)") - - cur.execute("UPDATE PackageBases SET Popularity = (" + - "SELECT COALESCE(SUM(POWER(0.98, (UNIX_TIMESTAMP() - VoteTS) / 86400)), 0.0) " + - "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " + - "PackageBases.ID AND NOT VoteTS IS NULL)") - - db.commit() - db.close() + conn.execute("UPDATE PackageBases SET Popularity = (" + + "SELECT COALESCE(SUM(POWER(0.98, (UNIX_TIMESTAMP() - VoteTS) / 86400)), 0.0) " + + "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " + + "PackageBases.ID AND NOT VoteTS IS NULL)") + + conn.commit() + conn.close() if __name__ == '__main__': diff --git a/scripts/tuvotereminder.py b/scripts/tuvotereminder.py index dc16397..a053608 100755 --- a/scripts/tuvotereminder.py +++ b/scripts/tuvotereminder.py @@ -1,34 +1,24 @@ #!/usr/bin/python3 -import configparser -import mysql.connector -import os import subprocess import time -config = configparser.RawConfigParser() -config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") +import aurweb.config +import aurweb.db -aur_db_host = config.get('database', 'host') -aur_db_name = config.get('database', 'name') -aur_db_user = config.get('database', 'user') -aur_db_pass = config.get('database', 'password') -aur_db_socket = config.get('database', 'socket') -notify_cmd = config.get('notifications', 'notify-cmd') +notify_cmd = aurweb.config.get('notifications', 'notify-cmd') def main(): - db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) - cur = db.cursor() + conn = aurweb.db.Connection() now = int(time.time()) filter_from = now + 500 filter_to = now + 172800 - cur.execute("SELECT ID FROM TU_VoteInfo WHERE End >= %s AND End <= %s", - [filter_from, filter_to]) + cur = conn.execute("SELECT ID FROM TU_VoteInfo " + + "WHERE End >= ? AND End <= ?", + [filter_from, filter_to]) for vote_id in [row[0] for row in cur.fetchall()]: subprocess.Popen((notify_cmd, 'tu-vote-reminder', str(vote_id))) -- cgit v1.2.3-54-g00ecf From ec5779c824f54f9bdc1303fb22e61b4d558ece39 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Wed, 21 Sep 2016 08:11:59 +0200 Subject: mkpkglists: Make output files configurable Instead of writing the output to hardcoded files (relative to the document root), make the output paths of mkpkglists configurable. Signed-off-by: Lukas Fleischer --- conf/config.proto | 4 ++++ scripts/mkpkglists.py | 9 +++++---- 2 files changed, 9 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/conf/config.proto b/conf/config.proto index c56141c..e545977 100644 --- a/conf/config.proto +++ b/conf/config.proto @@ -63,3 +63,7 @@ max-blob-size = 256000 db-path = /srv/http/aurweb/aurblup/ sync-dbs = core extra community multilib testing community-testing servers = ftp://mirrors.kernel.org/archlinux/%s/os/x86_64 + +[mkpkglists] +packagesfile = /srv/http/aurweb/web/html/packages.gz +pkgbasefile = /srv/http/aurweb/web/html/pkgbase.gz diff --git a/scripts/mkpkglists.py b/scripts/mkpkglists.py index 70cbd13..8a0f2e9 100755 --- a/scripts/mkpkglists.py +++ b/scripts/mkpkglists.py @@ -2,11 +2,12 @@ import datetime import gzip -import os +import aurweb.config import aurweb.db -docroot = os.path.dirname(os.path.realpath(__file__)) + "/../web/html/" +packagesfile = aurweb.config.get('mkpkglists', 'packagesfile') +pkgbasefile = aurweb.config.get('mkpkglists', 'pkgbasefile') def main(): @@ -16,7 +17,7 @@ def main(): pkglist_header = "# AUR package list, generated on " + datestr pkgbaselist_header = "# AUR package base list, generated on " + datestr - with gzip.open(docroot + "packages.gz", "w") as f: + with gzip.open(packagesfile, "w") as f: f.write(bytes(pkglist_header + "\n", "UTF-8")) cur = conn.execute("SELECT Packages.Name FROM Packages " + "INNER JOIN PackageBases " + @@ -24,7 +25,7 @@ def main(): "WHERE PackageBases.PackagerUID IS NOT NULL") f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - with gzip.open(docroot + "pkgbase.gz", "w") as f: + with gzip.open(pkgbasefile, "w") as f: f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) cur = conn.execute("SELECT Name FROM PackageBases " + "WHERE PackagerUID IS NOT NULL") -- cgit v1.2.3-54-g00ecf From 91f649f5edaae42f616ad9fc2facb90f1d71f9b9 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Wed, 21 Sep 2016 09:05:56 +0200 Subject: scripts: Do not use UNIX_TIMESTAMP Avoid using UNIX_TIMESTAMP which is not part of the SQL standard. See f2a6bd2 (git-interface: Do not use UNIX_TIMESTAMP, 2016-08-05) for related changes. Signed-off-by: Lukas Fleischer --- scripts/pkgmaint.py | 6 ++++-- scripts/popupdate.py | 7 +++++-- 2 files changed, 9 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/pkgmaint.py b/scripts/pkgmaint.py index d0f8fe7..3ad9ed8 100755 --- a/scripts/pkgmaint.py +++ b/scripts/pkgmaint.py @@ -1,14 +1,16 @@ #!/usr/bin/python3 +import time + import aurweb.db def main(): conn = aurweb.db.Connection() + limit_to = int(time.time()) - 86400 conn.execute("DELETE FROM PackageBases WHERE " + - "UNIX_TIMESTAMP() - SubmittedTS > 86400 " + - "AND PackagerUID IS NULL") + "SubmittedTS < ? AND PackagerUID IS NULL", [limit_to]) conn.commit() conn.close() diff --git a/scripts/popupdate.py b/scripts/popupdate.py index f5e09d9..58cd018 100755 --- a/scripts/popupdate.py +++ b/scripts/popupdate.py @@ -1,5 +1,7 @@ #!/usr/bin/python3 +import time + import aurweb.db @@ -10,10 +12,11 @@ def main(): "SELECT COUNT(*) FROM PackageVotes " + "WHERE PackageVotes.PackageBaseID = PackageBases.ID)") + now = int(time.time()) conn.execute("UPDATE PackageBases SET Popularity = (" + - "SELECT COALESCE(SUM(POWER(0.98, (UNIX_TIMESTAMP() - VoteTS) / 86400)), 0.0) " + + "SELECT COALESCE(SUM(POWER(0.98, (? - VoteTS) / 86400)), 0.0) " + "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " + - "PackageBases.ID AND NOT VoteTS IS NULL)") + "PackageBases.ID AND NOT VoteTS IS NULL)", [now]) conn.commit() conn.close() -- cgit v1.2.3-54-g00ecf From cc66259d339ec805265503662bf5da61f63512cc Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Wed, 21 Sep 2016 21:50:17 +0200 Subject: aurblup: Drop support for multiple servers Support for multiple servers has never been used by the official aurweb setup and the current implementation makes it impossible to use server URIs that contain spaces. For simplicity, change the implementation such that only a single server is supported. Signed-off-by: Lukas Fleischer --- conf/config.proto | 2 +- scripts/aurblup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/conf/config.proto b/conf/config.proto index e545977..21441a9 100644 --- a/conf/config.proto +++ b/conf/config.proto @@ -62,7 +62,7 @@ max-blob-size = 256000 [aurblup] db-path = /srv/http/aurweb/aurblup/ sync-dbs = core extra community multilib testing community-testing -servers = ftp://mirrors.kernel.org/archlinux/%s/os/x86_64 +server = ftp://mirrors.kernel.org/archlinux/%s/os/x86_64 [mkpkglists] packagesfile = /srv/http/aurweb/web/html/packages.gz diff --git a/scripts/aurblup.py b/scripts/aurblup.py index 0009715..1b6de2f 100755 --- a/scripts/aurblup.py +++ b/scripts/aurblup.py @@ -8,7 +8,7 @@ import aurweb.db db_path = aurweb.config.get('aurblup', 'db-path') sync_dbs = aurweb.config.get('aurblup', 'sync-dbs').split(' ') -servers = aurweb.config.get('aurblup', 'servers').split(' ') +server = aurweb.config.get('aurblup', 'server') def main(): @@ -19,7 +19,7 @@ def main(): h = pyalpm.Handle("/", db_path) for sync_db in sync_dbs: repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL) - repo.servers = [server.replace("%s", sync_db) for server in servers] + repo.servers = [server.replace("%s", sync_db)] t = h.init_transaction() repo.update(False) t.release() -- cgit v1.2.3-54-g00ecf From e1709e98ce623aee8062e98a859bcb43553e2faf Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Thu, 29 Sep 2016 21:03:54 +0200 Subject: tuvotereminder: Wait for notify processes Signed-off-by: Lukas Fleischer --- scripts/tuvotereminder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/tuvotereminder.py b/scripts/tuvotereminder.py index a053608..97b1d12 100755 --- a/scripts/tuvotereminder.py +++ b/scripts/tuvotereminder.py @@ -21,7 +21,7 @@ def main(): [filter_from, filter_to]) for vote_id in [row[0] for row in cur.fetchall()]: - subprocess.Popen((notify_cmd, 'tu-vote-reminder', str(vote_id))) + subprocess.Popen((notify_cmd, 'tu-vote-reminder', str(vote_id))).wait() if __name__ == '__main__': -- cgit v1.2.3-54-g00ecf From 37188603b52a3dac23df229ada82c7da0c3d9c00 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Mon, 17 Oct 2016 15:20:29 +0200 Subject: Make maintenance scripts installable Add wrappers for the maintenance scripts to the setuptools configuration. Signed-off-by: Lukas Fleischer --- aurweb/scripts/__init__.py | 0 aurweb/scripts/aurblup.py | 55 +++++ aurweb/scripts/mkpkglists.py | 38 ++++ aurweb/scripts/notify.py | 455 +++++++++++++++++++++++++++++++++++++++ aurweb/scripts/pkgmaint.py | 20 ++ aurweb/scripts/popupdate.py | 26 +++ aurweb/scripts/tuvotereminder.py | 28 +++ scripts/__init__.py | 0 scripts/aurblup.py | 55 ----- scripts/mkpkglists.py | 38 ---- scripts/notify.py | 455 --------------------------------------- scripts/pkgmaint.py | 20 -- scripts/popupdate.py | 26 --- scripts/tuvotereminder.py | 28 --- setup.py | 6 + upgrading/4.4.1.txt | 3 + 16 files changed, 631 insertions(+), 622 deletions(-) create mode 100644 aurweb/scripts/__init__.py create mode 100755 aurweb/scripts/aurblup.py create mode 100755 aurweb/scripts/mkpkglists.py create mode 100755 aurweb/scripts/notify.py create mode 100755 aurweb/scripts/pkgmaint.py create mode 100755 aurweb/scripts/popupdate.py create mode 100755 aurweb/scripts/tuvotereminder.py delete mode 100644 scripts/__init__.py delete mode 100755 scripts/aurblup.py delete mode 100755 scripts/mkpkglists.py delete mode 100755 scripts/notify.py delete mode 100755 scripts/pkgmaint.py delete mode 100755 scripts/popupdate.py delete mode 100755 scripts/tuvotereminder.py (limited to 'scripts') diff --git a/aurweb/scripts/__init__.py b/aurweb/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aurweb/scripts/aurblup.py b/aurweb/scripts/aurblup.py new file mode 100755 index 0000000..1b6de2f --- /dev/null +++ b/aurweb/scripts/aurblup.py @@ -0,0 +1,55 @@ +#!/usr/bin/python3 + +import pyalpm +import re + +import aurweb.config +import aurweb.db + +db_path = aurweb.config.get('aurblup', 'db-path') +sync_dbs = aurweb.config.get('aurblup', 'sync-dbs').split(' ') +server = aurweb.config.get('aurblup', 'server') + + +def main(): + blacklist = set() + providers = set() + repomap = dict() + + h = pyalpm.Handle("/", db_path) + for sync_db in sync_dbs: + repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL) + repo.servers = [server.replace("%s", sync_db)] + t = h.init_transaction() + repo.update(False) + t.release() + + for pkg in repo.pkgcache: + blacklist.add(pkg.name) + [blacklist.add(x) for x in pkg.replaces] + providers.add((pkg.name, pkg.name)) + repomap[(pkg.name, pkg.name)] = repo.name + for provision in pkg.provides: + provisionname = re.sub(r'(<|=|>).*', '', provision) + providers.add((pkg.name, provisionname)) + repomap[(pkg.name, provisionname)] = repo.name + + conn = aurweb.db.Connection() + + cur = conn.execute("SELECT Name, Provides FROM OfficialProviders") + oldproviders = set(cur.fetchall()) + + for pkg, provides in providers.difference(oldproviders): + repo = repomap[(pkg, provides)] + conn.execute("INSERT INTO OfficialProviders (Name, Repo, Provides) " + "VALUES (?, ?, ?)", [pkg, repo, provides]) + for pkg, provides in oldproviders.difference(providers): + conn.execute("DELETE FROM OfficialProviders " + "WHERE Name = ? AND Provides = ?", [pkg, provides]) + + conn.commit() + conn.close() + + +if __name__ == '__main__': + main() diff --git a/aurweb/scripts/mkpkglists.py b/aurweb/scripts/mkpkglists.py new file mode 100755 index 0000000..8a0f2e9 --- /dev/null +++ b/aurweb/scripts/mkpkglists.py @@ -0,0 +1,38 @@ +#!/usr/bin/python3 + +import datetime +import gzip + +import aurweb.config +import aurweb.db + +packagesfile = aurweb.config.get('mkpkglists', 'packagesfile') +pkgbasefile = aurweb.config.get('mkpkglists', 'pkgbasefile') + + +def main(): + conn = aurweb.db.Connection() + + datestr = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") + pkglist_header = "# AUR package list, generated on " + datestr + pkgbaselist_header = "# AUR package base list, generated on " + datestr + + with gzip.open(packagesfile, "w") as f: + f.write(bytes(pkglist_header + "\n", "UTF-8")) + cur = conn.execute("SELECT Packages.Name FROM Packages " + + "INNER JOIN PackageBases " + + "ON PackageBases.ID = Packages.PackageBaseID " + + "WHERE PackageBases.PackagerUID IS NOT NULL") + f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + + with gzip.open(pkgbasefile, "w") as f: + f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) + cur = conn.execute("SELECT Name FROM PackageBases " + + "WHERE PackagerUID IS NOT NULL") + f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + + conn.close() + + +if __name__ == '__main__': + main() diff --git a/aurweb/scripts/notify.py b/aurweb/scripts/notify.py new file mode 100755 index 0000000..ddd6e49 --- /dev/null +++ b/aurweb/scripts/notify.py @@ -0,0 +1,455 @@ +#!/usr/bin/python3 + +import email.mime.text +import subprocess +import sys +import textwrap + +import aurweb.config +import aurweb.db + +aur_location = aurweb.config.get('options', 'aur_location') +aur_request_ml = aurweb.config.get('options', 'aur_request_ml') + +sendmail = aurweb.config.get('notifications', 'sendmail') +sender = aurweb.config.get('notifications', 'sender') +reply_to = aurweb.config.get('notifications', 'reply-to') + + +def headers_cc(cclist): + return {'Cc': str.join(', ', cclist)} + + +def headers_msgid(thread_id): + return {'Message-ID': thread_id} + + +def headers_reply(thread_id): + return {'In-Reply-To': thread_id, 'References': thread_id} + + +def send_notification(to, subject, body, refs, headers={}): + wrapped = '' + for line in body.splitlines(): + wrapped += textwrap.fill(line, break_long_words=False) + '\n' + if refs: + body = wrapped + '\n' + refs + else: + body = wrapped + + for recipient in to: + msg = email.mime.text.MIMEText(body, 'plain', 'utf-8') + msg['Subject'] = subject + msg['From'] = sender + msg['Reply-to'] = reply_to + msg['To'] = recipient + + for key, value in headers.items(): + msg[key] = value + + p = subprocess.Popen([sendmail, '-t', '-oi'], stdin=subprocess.PIPE) + p.communicate(msg.as_bytes()) + + +def username_from_id(conn, uid): + cur = conn.execute('SELECT UserName FROM Users WHERE ID = ?', [uid]) + return cur.fetchone()[0] + + +def pkgbase_from_id(conn, pkgbase_id): + cur = conn.execute('SELECT Name FROM PackageBases WHERE ID = ?', + [pkgbase_id]) + return cur.fetchone()[0] + + +def pkgbase_from_pkgreq(conn, reqid): + cur = conn.execute('SELECT PackageBaseID FROM PackageRequests ' + + 'WHERE ID = ?', [reqid]) + return cur.fetchone()[0] + + +def get_user_email(conn, uid): + cur = conn.execute('SELECT Email FROM Users WHERE ID = ?', [uid]) + return cur.fetchone()[0] + + +def get_maintainer_email(conn, pkgbase_id): + cur = conn.execute('SELECT Users.Email FROM Users ' + + 'INNER JOIN PackageBases ' + + 'ON PackageBases.MaintainerUID = Users.ID WHERE ' + + 'PackageBases.ID = ?', [pkgbase_id]) + return cur.fetchone()[0] + + +def get_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) + return [row[0] for row in cur.fetchall()] + + +def get_comment_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'Users.CommentNotify = 1 AND ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) + return [row[0] for row in cur.fetchall()] + + +def get_update_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'Users.UpdateNotify = 1 AND ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) + return [row[0] for row in cur.fetchall()] + + +def get_ownership_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'Users.OwnershipNotify = 1 AND ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) + return [row[0] for row in cur.fetchall()] + + +def get_request_recipients(conn, reqid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM PackageRequests ' + + 'INNER JOIN PackageBases ' + + 'ON PackageBases.ID = PackageRequests.PackageBaseID ' + + 'INNER JOIN Users ' + + 'ON Users.ID = PackageRequests.UsersID ' + + 'OR Users.ID = PackageBases.MaintainerUID ' + + 'WHERE PackageRequests.ID = ?', [reqid]) + return [row[0] for row in cur.fetchall()] + + +def get_tu_vote_reminder_recipients(conn, vote_id): + cur = conn.execute('SELECT Users.Email FROM Users ' + + 'WHERE AccountTypeID = 2 ' + + 'EXCEPT SELECT Users.Email FROM Users ' + + 'INNER JOIN TU_Votes ' + + 'ON TU_Votes.UserID = Users.ID ' + + 'WHERE TU_Votes.VoteID = ?', [vote_id]) + return [row[0] for row in cur.fetchall()] + + +def get_comment(conn, comment_id): + cur = conn.execute('SELECT Comments FROM PackageComments WHERE ID = ?', + [comment_id]) + return cur.fetchone()[0] + + +def get_flagger_comment(conn, pkgbase_id): + cur = conn.execute('SELECT FlaggerComment FROM PackageBases WHERE ID = ?', + [pkgbase_id]) + return cur.fetchone()[0] + + +def get_request_comment(conn, reqid): + cur = conn.execute('SELECT Comments FROM PackageRequests WHERE ID = ?', + [reqid]) + return cur.fetchone()[0] + + +def get_request_closure_comment(conn, reqid): + cur = conn.execute('SELECT ClosureComment FROM PackageRequests ' + + 'WHERE ID = ?', [reqid]) + return cur.fetchone()[0] + + +def send_resetkey(conn, uid): + cur = conn.execute('SELECT UserName, Email, ResetKey FROM Users ' + + 'WHERE ID = ?', [uid]) + username, to, resetkey = cur.fetchone() + + subject = 'AUR Password Reset' + body = 'A password reset request was submitted for the account %s ' \ + 'associated with your email address. If you wish to reset your ' \ + 'password follow the link [1] below, otherwise ignore this ' \ + 'message and nothing will happen.' % (username) + refs = '[1] ' + aur_location + '/passreset/?resetkey=' + resetkey + + send_notification([to], subject, body, refs) + + +def welcome(conn, uid): + cur = conn.execute('SELECT UserName, Email, ResetKey FROM Users ' + + 'WHERE ID = ?', [uid]) + username, to, resetkey = cur.fetchone() + + subject = 'Welcome to the Arch User Repository' + body = 'Welcome to the Arch User Repository! In order to set an initial ' \ + 'password for your new account, please click the link [1] below. ' \ + 'If the link does not work, try copying and pasting it into your ' \ + 'browser.' + refs = '[1] ' + aur_location + '/passreset/?resetkey=' + resetkey + + send_notification([to], subject, body, refs) + + +def comment(conn, uid, pkgbase_id, comment_id): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_comment_recipients(conn, pkgbase_id, uid) + text = get_comment(conn, comment_id) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Comment for %s' % (pkgbase) + body = '%s [1] added the following comment to %s [2]:' % (user, pkgbase) + body += '\n\n' + text + '\n\n' + body += 'If you no longer wish to receive notifications about this ' \ + 'package, please go to the package page [2] and select "%s".' % \ + ('Disable notifications') + refs = '[1] ' + user_uri + '\n' + refs += '[2] ' + pkgbase_uri + thread_id = '' + headers = headers_reply(thread_id) + + send_notification(to, subject, body, refs, headers) + + +def update(conn, uid, pkgbase_id): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_update_recipients(conn, pkgbase_id, uid) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Package Update: %s' % (pkgbase) + body = '%s [1] pushed a new commit to %s [2].' % (user, pkgbase) + body += '\n\n' + body += 'If you no longer wish to receive notifications about this ' \ + 'package, please go to the package page [2] and select "%s".' % \ + ('Disable notifications') + refs = '[1] ' + user_uri + '\n' + refs += '[2] ' + pkgbase_uri + thread_id = '' + headers = headers_reply(thread_id) + + send_notification(to, subject, body, refs, headers) + + +def flag(conn, uid, pkgbase_id): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = [get_maintainer_email(conn, pkgbase_id)] + text = get_flagger_comment(conn, pkgbase_id) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Out-of-date Notification for %s' % (pkgbase) + body = 'Your package %s [1] has been flagged out-of-date by %s [2]:' % \ + (pkgbase, user) + body += '\n\n' + text + refs = '[1] ' + pkgbase_uri + '\n' + refs += '[2] ' + user_uri + + send_notification(to, subject, body, refs) + + +def adopt(conn, pkgbase_id, uid): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_ownership_recipients(conn, pkgbase_id, uid) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Ownership Notification for %s' % (pkgbase) + body = 'The package %s [1] was adopted by %s [2].' % (pkgbase, user) + refs = '[1] ' + pkgbase_uri + '\n' + refs += '[2] ' + user_uri + + send_notification(to, subject, body, refs) + + +def disown(conn, pkgbase_id, uid): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_ownership_recipients(conn, pkgbase_id, uid) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Ownership Notification for %s' % (pkgbase) + body = 'The package %s [1] was disowned by %s [2].' % (pkgbase, user) + refs = '[1] ' + pkgbase_uri + '\n' + refs += '[2] ' + user_uri + + send_notification(to, subject, body, refs) + + +def comaintainer_add(conn, pkgbase_id, uid): + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = [get_user_email(conn, uid)] + + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Co-Maintainer Notification for %s' % (pkgbase) + body = 'You were added to the co-maintainer list of %s [1].' % (pkgbase) + refs = '[1] ' + pkgbase_uri + '\n' + + send_notification(to, subject, body, refs) + + +def comaintainer_remove(conn, pkgbase_id, uid): + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = [get_user_email(conn, uid)] + + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Co-Maintainer Notification for %s' % (pkgbase) + body = ('You were removed from the co-maintainer list of %s [1].' % + (pkgbase)) + refs = '[1] ' + pkgbase_uri + '\n' + + send_notification(to, subject, body, refs) + + +def delete(conn, uid, old_pkgbase_id, new_pkgbase_id=None): + user = username_from_id(conn, uid) + old_pkgbase = pkgbase_from_id(conn, old_pkgbase_id) + if new_pkgbase_id: + new_pkgbase = pkgbase_from_id(conn, new_pkgbase_id) + to = get_recipients(conn, old_pkgbase_id, uid) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + old_pkgbase + '/' + + subject = 'AUR Package deleted: %s' % (old_pkgbase) + if new_pkgbase_id: + new_pkgbase_uri = aur_location + '/pkgbase/' + new_pkgbase + '/' + body = '%s [1] merged %s [2] into %s [3].\n\n' \ + 'If you no longer wish receive notifications about the new ' \ + 'package, please go to [3] and click "%s".' %\ + (user, old_pkgbase, new_pkgbase, 'Disable notifications') + refs = '[1] ' + user_uri + '\n' + refs += '[2] ' + pkgbase_uri + '\n' + refs += '[3] ' + new_pkgbase_uri + else: + body = '%s [1] deleted %s [2].\n\n' \ + 'You will no longer receive notifications about this ' \ + 'package.' % (user, old_pkgbase) + refs = '[1] ' + user_uri + '\n' + refs += '[2] ' + pkgbase_uri + + send_notification(to, subject, body, refs) + + +def request_open(conn, uid, reqid, reqtype, pkgbase_id, merge_into=None): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = [aur_request_ml] + cc = get_request_recipients(conn, reqid) + text = get_request_comment(conn, reqid) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = '[PRQ#%d] %s Request for %s' % \ + (int(reqid), reqtype.title(), pkgbase) + if merge_into: + merge_into_uri = aur_location + '/pkgbase/' + merge_into + '/' + body = '%s [1] filed a request to merge %s [2] into %s [3]:' % \ + (user, pkgbase, merge_into) + body += '\n\n' + text + refs = '[1] ' + user_uri + '\n' + refs += '[2] ' + pkgbase_uri + '\n' + refs += '[3] ' + merge_into_uri + else: + body = '%s [1] filed a %s request for %s [2]:' % \ + (user, reqtype, pkgbase) + body += '\n\n' + text + refs = '[1] ' + user_uri + '\n' + refs += '[2] ' + pkgbase_uri + '\n' + thread_id = '' + # Use a deterministic Message-ID for the first email referencing a request. + headers = headers_msgid(thread_id) + headers.update(headers_cc(cc)) + + send_notification(to, subject, body, refs, headers) + + +def request_close(conn, uid, reqid, reason): + to = [aur_request_ml] + cc = get_request_recipients(conn, reqid) + text = get_request_closure_comment(conn, reqid) + + subject = '[PRQ#%d] Request %s' % (int(reqid), reason.title()) + if int(uid): + user = username_from_id(conn, uid) + user_uri = aur_location + '/account/' + user + '/' + body = 'Request #%d has been %s by %s [1]' % (int(reqid), reason, user) + refs = '[1] ' + user_uri + else: + body = 'Request #%d has been %s automatically by the Arch User ' \ + 'Repository package request system' % (int(reqid), reason) + refs = None + if text.strip() == '': + body += '.' + else: + body += ':\n\n' + text + thread_id = '' + headers = headers_reply(thread_id) + headers.update(headers_cc(cc)) + + send_notification(to, subject, body, refs, headers) + + +def tu_vote_reminder(conn, vote_id): + to = get_tu_vote_reminder_recipients(conn, vote_id) + + vote_uri = aur_location + '/tu/?id=' + vote_id + + subject = 'TU Vote Reminder: Proposal %d' % (int(vote_id)) + body = 'Please remember to cast your vote on proposal %d [1]. ' \ + 'The voting period ends in less than 48 hours.' % (int(vote_id)) + refs = '[1] ' + vote_uri + + send_notification(to, subject, body, refs) + + +def main(): + action = sys.argv[1] + action_map = { + 'send-resetkey': send_resetkey, + 'welcome': welcome, + 'comment': comment, + 'update': update, + 'flag': flag, + 'adopt': adopt, + 'disown': disown, + 'comaintainer-add': comaintainer_add, + 'comaintainer-remove': comaintainer_remove, + 'delete': delete, + 'request-open': request_open, + 'request-close': request_close, + 'tu-vote-reminder': tu_vote_reminder, + } + + conn = aurweb.db.Connection() + + action_map[action](conn, *sys.argv[2:]) + + conn.commit() + conn.close() + + +if __name__ == '__main__': + main() diff --git a/aurweb/scripts/pkgmaint.py b/aurweb/scripts/pkgmaint.py new file mode 100755 index 0000000..3ad9ed8 --- /dev/null +++ b/aurweb/scripts/pkgmaint.py @@ -0,0 +1,20 @@ +#!/usr/bin/python3 + +import time + +import aurweb.db + + +def main(): + conn = aurweb.db.Connection() + + limit_to = int(time.time()) - 86400 + conn.execute("DELETE FROM PackageBases WHERE " + + "SubmittedTS < ? AND PackagerUID IS NULL", [limit_to]) + + conn.commit() + conn.close() + + +if __name__ == '__main__': + main() diff --git a/aurweb/scripts/popupdate.py b/aurweb/scripts/popupdate.py new file mode 100755 index 0000000..58cd018 --- /dev/null +++ b/aurweb/scripts/popupdate.py @@ -0,0 +1,26 @@ +#!/usr/bin/python3 + +import time + +import aurweb.db + + +def main(): + conn = aurweb.db.Connection() + + conn.execute("UPDATE PackageBases SET NumVotes = (" + + "SELECT COUNT(*) FROM PackageVotes " + + "WHERE PackageVotes.PackageBaseID = PackageBases.ID)") + + now = int(time.time()) + conn.execute("UPDATE PackageBases SET Popularity = (" + + "SELECT COALESCE(SUM(POWER(0.98, (? - VoteTS) / 86400)), 0.0) " + + "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " + + "PackageBases.ID AND NOT VoteTS IS NULL)", [now]) + + conn.commit() + conn.close() + + +if __name__ == '__main__': + main() diff --git a/aurweb/scripts/tuvotereminder.py b/aurweb/scripts/tuvotereminder.py new file mode 100755 index 0000000..97b1d12 --- /dev/null +++ b/aurweb/scripts/tuvotereminder.py @@ -0,0 +1,28 @@ +#!/usr/bin/python3 + +import subprocess +import time + +import aurweb.config +import aurweb.db + +notify_cmd = aurweb.config.get('notifications', 'notify-cmd') + + +def main(): + conn = aurweb.db.Connection() + + now = int(time.time()) + filter_from = now + 500 + filter_to = now + 172800 + + cur = conn.execute("SELECT ID FROM TU_VoteInfo " + + "WHERE End >= ? AND End <= ?", + [filter_from, filter_to]) + + for vote_id in [row[0] for row in cur.fetchall()]: + subprocess.Popen((notify_cmd, 'tu-vote-reminder', str(vote_id))).wait() + + +if __name__ == '__main__': + main() diff --git a/scripts/__init__.py b/scripts/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/scripts/aurblup.py b/scripts/aurblup.py deleted file mode 100755 index 1b6de2f..0000000 --- a/scripts/aurblup.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/python3 - -import pyalpm -import re - -import aurweb.config -import aurweb.db - -db_path = aurweb.config.get('aurblup', 'db-path') -sync_dbs = aurweb.config.get('aurblup', 'sync-dbs').split(' ') -server = aurweb.config.get('aurblup', 'server') - - -def main(): - blacklist = set() - providers = set() - repomap = dict() - - h = pyalpm.Handle("/", db_path) - for sync_db in sync_dbs: - repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL) - repo.servers = [server.replace("%s", sync_db)] - t = h.init_transaction() - repo.update(False) - t.release() - - for pkg in repo.pkgcache: - blacklist.add(pkg.name) - [blacklist.add(x) for x in pkg.replaces] - providers.add((pkg.name, pkg.name)) - repomap[(pkg.name, pkg.name)] = repo.name - for provision in pkg.provides: - provisionname = re.sub(r'(<|=|>).*', '', provision) - providers.add((pkg.name, provisionname)) - repomap[(pkg.name, provisionname)] = repo.name - - conn = aurweb.db.Connection() - - cur = conn.execute("SELECT Name, Provides FROM OfficialProviders") - oldproviders = set(cur.fetchall()) - - for pkg, provides in providers.difference(oldproviders): - repo = repomap[(pkg, provides)] - conn.execute("INSERT INTO OfficialProviders (Name, Repo, Provides) " - "VALUES (?, ?, ?)", [pkg, repo, provides]) - for pkg, provides in oldproviders.difference(providers): - conn.execute("DELETE FROM OfficialProviders " - "WHERE Name = ? AND Provides = ?", [pkg, provides]) - - conn.commit() - conn.close() - - -if __name__ == '__main__': - main() diff --git a/scripts/mkpkglists.py b/scripts/mkpkglists.py deleted file mode 100755 index 8a0f2e9..0000000 --- a/scripts/mkpkglists.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/python3 - -import datetime -import gzip - -import aurweb.config -import aurweb.db - -packagesfile = aurweb.config.get('mkpkglists', 'packagesfile') -pkgbasefile = aurweb.config.get('mkpkglists', 'pkgbasefile') - - -def main(): - conn = aurweb.db.Connection() - - datestr = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") - pkglist_header = "# AUR package list, generated on " + datestr - pkgbaselist_header = "# AUR package base list, generated on " + datestr - - with gzip.open(packagesfile, "w") as f: - f.write(bytes(pkglist_header + "\n", "UTF-8")) - cur = conn.execute("SELECT Packages.Name FROM Packages " + - "INNER JOIN PackageBases " + - "ON PackageBases.ID = Packages.PackageBaseID " + - "WHERE PackageBases.PackagerUID IS NOT NULL") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - - with gzip.open(pkgbasefile, "w") as f: - f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) - cur = conn.execute("SELECT Name FROM PackageBases " + - "WHERE PackagerUID IS NOT NULL") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - - conn.close() - - -if __name__ == '__main__': - main() diff --git a/scripts/notify.py b/scripts/notify.py deleted file mode 100755 index ddd6e49..0000000 --- a/scripts/notify.py +++ /dev/null @@ -1,455 +0,0 @@ -#!/usr/bin/python3 - -import email.mime.text -import subprocess -import sys -import textwrap - -import aurweb.config -import aurweb.db - -aur_location = aurweb.config.get('options', 'aur_location') -aur_request_ml = aurweb.config.get('options', 'aur_request_ml') - -sendmail = aurweb.config.get('notifications', 'sendmail') -sender = aurweb.config.get('notifications', 'sender') -reply_to = aurweb.config.get('notifications', 'reply-to') - - -def headers_cc(cclist): - return {'Cc': str.join(', ', cclist)} - - -def headers_msgid(thread_id): - return {'Message-ID': thread_id} - - -def headers_reply(thread_id): - return {'In-Reply-To': thread_id, 'References': thread_id} - - -def send_notification(to, subject, body, refs, headers={}): - wrapped = '' - for line in body.splitlines(): - wrapped += textwrap.fill(line, break_long_words=False) + '\n' - if refs: - body = wrapped + '\n' + refs - else: - body = wrapped - - for recipient in to: - msg = email.mime.text.MIMEText(body, 'plain', 'utf-8') - msg['Subject'] = subject - msg['From'] = sender - msg['Reply-to'] = reply_to - msg['To'] = recipient - - for key, value in headers.items(): - msg[key] = value - - p = subprocess.Popen([sendmail, '-t', '-oi'], stdin=subprocess.PIPE) - p.communicate(msg.as_bytes()) - - -def username_from_id(conn, uid): - cur = conn.execute('SELECT UserName FROM Users WHERE ID = ?', [uid]) - return cur.fetchone()[0] - - -def pkgbase_from_id(conn, pkgbase_id): - cur = conn.execute('SELECT Name FROM PackageBases WHERE ID = ?', - [pkgbase_id]) - return cur.fetchone()[0] - - -def pkgbase_from_pkgreq(conn, reqid): - cur = conn.execute('SELECT PackageBaseID FROM PackageRequests ' + - 'WHERE ID = ?', [reqid]) - return cur.fetchone()[0] - - -def get_user_email(conn, uid): - cur = conn.execute('SELECT Email FROM Users WHERE ID = ?', [uid]) - return cur.fetchone()[0] - - -def get_maintainer_email(conn, pkgbase_id): - cur = conn.execute('SELECT Users.Email FROM Users ' + - 'INNER JOIN PackageBases ' + - 'ON PackageBases.MaintainerUID = Users.ID WHERE ' + - 'PackageBases.ID = ?', [pkgbase_id]) - return cur.fetchone()[0] - - -def get_recipients(conn, pkgbase_id, uid): - cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + - 'INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'PackageNotifications.UserID != ? AND ' + - 'PackageNotifications.PackageBaseID = ?', - [uid, pkgbase_id]) - return [row[0] for row in cur.fetchall()] - - -def get_comment_recipients(conn, pkgbase_id, uid): - cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + - 'INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'Users.CommentNotify = 1 AND ' + - 'PackageNotifications.UserID != ? AND ' + - 'PackageNotifications.PackageBaseID = ?', - [uid, pkgbase_id]) - return [row[0] for row in cur.fetchall()] - - -def get_update_recipients(conn, pkgbase_id, uid): - cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + - 'INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'Users.UpdateNotify = 1 AND ' + - 'PackageNotifications.UserID != ? AND ' + - 'PackageNotifications.PackageBaseID = ?', - [uid, pkgbase_id]) - return [row[0] for row in cur.fetchall()] - - -def get_ownership_recipients(conn, pkgbase_id, uid): - cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + - 'INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'Users.OwnershipNotify = 1 AND ' + - 'PackageNotifications.UserID != ? AND ' + - 'PackageNotifications.PackageBaseID = ?', - [uid, pkgbase_id]) - return [row[0] for row in cur.fetchall()] - - -def get_request_recipients(conn, reqid): - cur = conn.execute('SELECT DISTINCT Users.Email FROM PackageRequests ' + - 'INNER JOIN PackageBases ' + - 'ON PackageBases.ID = PackageRequests.PackageBaseID ' + - 'INNER JOIN Users ' + - 'ON Users.ID = PackageRequests.UsersID ' + - 'OR Users.ID = PackageBases.MaintainerUID ' + - 'WHERE PackageRequests.ID = ?', [reqid]) - return [row[0] for row in cur.fetchall()] - - -def get_tu_vote_reminder_recipients(conn, vote_id): - cur = conn.execute('SELECT Users.Email FROM Users ' + - 'WHERE AccountTypeID = 2 ' + - 'EXCEPT SELECT Users.Email FROM Users ' + - 'INNER JOIN TU_Votes ' + - 'ON TU_Votes.UserID = Users.ID ' + - 'WHERE TU_Votes.VoteID = ?', [vote_id]) - return [row[0] for row in cur.fetchall()] - - -def get_comment(conn, comment_id): - cur = conn.execute('SELECT Comments FROM PackageComments WHERE ID = ?', - [comment_id]) - return cur.fetchone()[0] - - -def get_flagger_comment(conn, pkgbase_id): - cur = conn.execute('SELECT FlaggerComment FROM PackageBases WHERE ID = ?', - [pkgbase_id]) - return cur.fetchone()[0] - - -def get_request_comment(conn, reqid): - cur = conn.execute('SELECT Comments FROM PackageRequests WHERE ID = ?', - [reqid]) - return cur.fetchone()[0] - - -def get_request_closure_comment(conn, reqid): - cur = conn.execute('SELECT ClosureComment FROM PackageRequests ' + - 'WHERE ID = ?', [reqid]) - return cur.fetchone()[0] - - -def send_resetkey(conn, uid): - cur = conn.execute('SELECT UserName, Email, ResetKey FROM Users ' + - 'WHERE ID = ?', [uid]) - username, to, resetkey = cur.fetchone() - - subject = 'AUR Password Reset' - body = 'A password reset request was submitted for the account %s ' \ - 'associated with your email address. If you wish to reset your ' \ - 'password follow the link [1] below, otherwise ignore this ' \ - 'message and nothing will happen.' % (username) - refs = '[1] ' + aur_location + '/passreset/?resetkey=' + resetkey - - send_notification([to], subject, body, refs) - - -def welcome(conn, uid): - cur = conn.execute('SELECT UserName, Email, ResetKey FROM Users ' + - 'WHERE ID = ?', [uid]) - username, to, resetkey = cur.fetchone() - - subject = 'Welcome to the Arch User Repository' - body = 'Welcome to the Arch User Repository! In order to set an initial ' \ - 'password for your new account, please click the link [1] below. ' \ - 'If the link does not work, try copying and pasting it into your ' \ - 'browser.' - refs = '[1] ' + aur_location + '/passreset/?resetkey=' + resetkey - - send_notification([to], subject, body, refs) - - -def comment(conn, uid, pkgbase_id, comment_id): - user = username_from_id(conn, uid) - pkgbase = pkgbase_from_id(conn, pkgbase_id) - to = get_comment_recipients(conn, pkgbase_id, uid) - text = get_comment(conn, comment_id) - - user_uri = aur_location + '/account/' + user + '/' - pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' - - subject = 'AUR Comment for %s' % (pkgbase) - body = '%s [1] added the following comment to %s [2]:' % (user, pkgbase) - body += '\n\n' + text + '\n\n' - body += 'If you no longer wish to receive notifications about this ' \ - 'package, please go to the package page [2] and select "%s".' % \ - ('Disable notifications') - refs = '[1] ' + user_uri + '\n' - refs += '[2] ' + pkgbase_uri - thread_id = '' - headers = headers_reply(thread_id) - - send_notification(to, subject, body, refs, headers) - - -def update(conn, uid, pkgbase_id): - user = username_from_id(conn, uid) - pkgbase = pkgbase_from_id(conn, pkgbase_id) - to = get_update_recipients(conn, pkgbase_id, uid) - - user_uri = aur_location + '/account/' + user + '/' - pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' - - subject = 'AUR Package Update: %s' % (pkgbase) - body = '%s [1] pushed a new commit to %s [2].' % (user, pkgbase) - body += '\n\n' - body += 'If you no longer wish to receive notifications about this ' \ - 'package, please go to the package page [2] and select "%s".' % \ - ('Disable notifications') - refs = '[1] ' + user_uri + '\n' - refs += '[2] ' + pkgbase_uri - thread_id = '' - headers = headers_reply(thread_id) - - send_notification(to, subject, body, refs, headers) - - -def flag(conn, uid, pkgbase_id): - user = username_from_id(conn, uid) - pkgbase = pkgbase_from_id(conn, pkgbase_id) - to = [get_maintainer_email(conn, pkgbase_id)] - text = get_flagger_comment(conn, pkgbase_id) - - user_uri = aur_location + '/account/' + user + '/' - pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' - - subject = 'AUR Out-of-date Notification for %s' % (pkgbase) - body = 'Your package %s [1] has been flagged out-of-date by %s [2]:' % \ - (pkgbase, user) - body += '\n\n' + text - refs = '[1] ' + pkgbase_uri + '\n' - refs += '[2] ' + user_uri - - send_notification(to, subject, body, refs) - - -def adopt(conn, pkgbase_id, uid): - user = username_from_id(conn, uid) - pkgbase = pkgbase_from_id(conn, pkgbase_id) - to = get_ownership_recipients(conn, pkgbase_id, uid) - - user_uri = aur_location + '/account/' + user + '/' - pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' - - subject = 'AUR Ownership Notification for %s' % (pkgbase) - body = 'The package %s [1] was adopted by %s [2].' % (pkgbase, user) - refs = '[1] ' + pkgbase_uri + '\n' - refs += '[2] ' + user_uri - - send_notification(to, subject, body, refs) - - -def disown(conn, pkgbase_id, uid): - user = username_from_id(conn, uid) - pkgbase = pkgbase_from_id(conn, pkgbase_id) - to = get_ownership_recipients(conn, pkgbase_id, uid) - - user_uri = aur_location + '/account/' + user + '/' - pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' - - subject = 'AUR Ownership Notification for %s' % (pkgbase) - body = 'The package %s [1] was disowned by %s [2].' % (pkgbase, user) - refs = '[1] ' + pkgbase_uri + '\n' - refs += '[2] ' + user_uri - - send_notification(to, subject, body, refs) - - -def comaintainer_add(conn, pkgbase_id, uid): - pkgbase = pkgbase_from_id(conn, pkgbase_id) - to = [get_user_email(conn, uid)] - - pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' - - subject = 'AUR Co-Maintainer Notification for %s' % (pkgbase) - body = 'You were added to the co-maintainer list of %s [1].' % (pkgbase) - refs = '[1] ' + pkgbase_uri + '\n' - - send_notification(to, subject, body, refs) - - -def comaintainer_remove(conn, pkgbase_id, uid): - pkgbase = pkgbase_from_id(conn, pkgbase_id) - to = [get_user_email(conn, uid)] - - pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' - - subject = 'AUR Co-Maintainer Notification for %s' % (pkgbase) - body = ('You were removed from the co-maintainer list of %s [1].' % - (pkgbase)) - refs = '[1] ' + pkgbase_uri + '\n' - - send_notification(to, subject, body, refs) - - -def delete(conn, uid, old_pkgbase_id, new_pkgbase_id=None): - user = username_from_id(conn, uid) - old_pkgbase = pkgbase_from_id(conn, old_pkgbase_id) - if new_pkgbase_id: - new_pkgbase = pkgbase_from_id(conn, new_pkgbase_id) - to = get_recipients(conn, old_pkgbase_id, uid) - - user_uri = aur_location + '/account/' + user + '/' - pkgbase_uri = aur_location + '/pkgbase/' + old_pkgbase + '/' - - subject = 'AUR Package deleted: %s' % (old_pkgbase) - if new_pkgbase_id: - new_pkgbase_uri = aur_location + '/pkgbase/' + new_pkgbase + '/' - body = '%s [1] merged %s [2] into %s [3].\n\n' \ - 'If you no longer wish receive notifications about the new ' \ - 'package, please go to [3] and click "%s".' %\ - (user, old_pkgbase, new_pkgbase, 'Disable notifications') - refs = '[1] ' + user_uri + '\n' - refs += '[2] ' + pkgbase_uri + '\n' - refs += '[3] ' + new_pkgbase_uri - else: - body = '%s [1] deleted %s [2].\n\n' \ - 'You will no longer receive notifications about this ' \ - 'package.' % (user, old_pkgbase) - refs = '[1] ' + user_uri + '\n' - refs += '[2] ' + pkgbase_uri - - send_notification(to, subject, body, refs) - - -def request_open(conn, uid, reqid, reqtype, pkgbase_id, merge_into=None): - user = username_from_id(conn, uid) - pkgbase = pkgbase_from_id(conn, pkgbase_id) - to = [aur_request_ml] - cc = get_request_recipients(conn, reqid) - text = get_request_comment(conn, reqid) - - user_uri = aur_location + '/account/' + user + '/' - pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' - - subject = '[PRQ#%d] %s Request for %s' % \ - (int(reqid), reqtype.title(), pkgbase) - if merge_into: - merge_into_uri = aur_location + '/pkgbase/' + merge_into + '/' - body = '%s [1] filed a request to merge %s [2] into %s [3]:' % \ - (user, pkgbase, merge_into) - body += '\n\n' + text - refs = '[1] ' + user_uri + '\n' - refs += '[2] ' + pkgbase_uri + '\n' - refs += '[3] ' + merge_into_uri - else: - body = '%s [1] filed a %s request for %s [2]:' % \ - (user, reqtype, pkgbase) - body += '\n\n' + text - refs = '[1] ' + user_uri + '\n' - refs += '[2] ' + pkgbase_uri + '\n' - thread_id = '' - # Use a deterministic Message-ID for the first email referencing a request. - headers = headers_msgid(thread_id) - headers.update(headers_cc(cc)) - - send_notification(to, subject, body, refs, headers) - - -def request_close(conn, uid, reqid, reason): - to = [aur_request_ml] - cc = get_request_recipients(conn, reqid) - text = get_request_closure_comment(conn, reqid) - - subject = '[PRQ#%d] Request %s' % (int(reqid), reason.title()) - if int(uid): - user = username_from_id(conn, uid) - user_uri = aur_location + '/account/' + user + '/' - body = 'Request #%d has been %s by %s [1]' % (int(reqid), reason, user) - refs = '[1] ' + user_uri - else: - body = 'Request #%d has been %s automatically by the Arch User ' \ - 'Repository package request system' % (int(reqid), reason) - refs = None - if text.strip() == '': - body += '.' - else: - body += ':\n\n' + text - thread_id = '' - headers = headers_reply(thread_id) - headers.update(headers_cc(cc)) - - send_notification(to, subject, body, refs, headers) - - -def tu_vote_reminder(conn, vote_id): - to = get_tu_vote_reminder_recipients(conn, vote_id) - - vote_uri = aur_location + '/tu/?id=' + vote_id - - subject = 'TU Vote Reminder: Proposal %d' % (int(vote_id)) - body = 'Please remember to cast your vote on proposal %d [1]. ' \ - 'The voting period ends in less than 48 hours.' % (int(vote_id)) - refs = '[1] ' + vote_uri - - send_notification(to, subject, body, refs) - - -def main(): - action = sys.argv[1] - action_map = { - 'send-resetkey': send_resetkey, - 'welcome': welcome, - 'comment': comment, - 'update': update, - 'flag': flag, - 'adopt': adopt, - 'disown': disown, - 'comaintainer-add': comaintainer_add, - 'comaintainer-remove': comaintainer_remove, - 'delete': delete, - 'request-open': request_open, - 'request-close': request_close, - 'tu-vote-reminder': tu_vote_reminder, - } - - conn = aurweb.db.Connection() - - action_map[action](conn, *sys.argv[2:]) - - conn.commit() - conn.close() - - -if __name__ == '__main__': - main() diff --git a/scripts/pkgmaint.py b/scripts/pkgmaint.py deleted file mode 100755 index 3ad9ed8..0000000 --- a/scripts/pkgmaint.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/python3 - -import time - -import aurweb.db - - -def main(): - conn = aurweb.db.Connection() - - limit_to = int(time.time()) - 86400 - conn.execute("DELETE FROM PackageBases WHERE " + - "SubmittedTS < ? AND PackagerUID IS NULL", [limit_to]) - - conn.commit() - conn.close() - - -if __name__ == '__main__': - main() diff --git a/scripts/popupdate.py b/scripts/popupdate.py deleted file mode 100755 index 58cd018..0000000 --- a/scripts/popupdate.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/python3 - -import time - -import aurweb.db - - -def main(): - conn = aurweb.db.Connection() - - conn.execute("UPDATE PackageBases SET NumVotes = (" + - "SELECT COUNT(*) FROM PackageVotes " + - "WHERE PackageVotes.PackageBaseID = PackageBases.ID)") - - now = int(time.time()) - conn.execute("UPDATE PackageBases SET Popularity = (" + - "SELECT COALESCE(SUM(POWER(0.98, (? - VoteTS) / 86400)), 0.0) " + - "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " + - "PackageBases.ID AND NOT VoteTS IS NULL)", [now]) - - conn.commit() - conn.close() - - -if __name__ == '__main__': - main() diff --git a/scripts/tuvotereminder.py b/scripts/tuvotereminder.py deleted file mode 100755 index 97b1d12..0000000 --- a/scripts/tuvotereminder.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/python3 - -import subprocess -import time - -import aurweb.config -import aurweb.db - -notify_cmd = aurweb.config.get('notifications', 'notify-cmd') - - -def main(): - conn = aurweb.db.Connection() - - now = int(time.time()) - filter_from = now + 500 - filter_to = now + 172800 - - cur = conn.execute("SELECT ID FROM TU_VoteInfo " + - "WHERE End >= ? AND End <= ?", - [filter_from, filter_to]) - - for vote_id in [row[0] for row in cur.fetchall()]: - subprocess.Popen((notify_cmd, 'tu-vote-reminder', str(vote_id))).wait() - - -if __name__ == '__main__': - main() diff --git a/setup.py b/setup.py index b64e71c..99dbfed 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,12 @@ setup( 'aurweb-git-auth = aurweb.git.auth:main', 'aurweb-git-serve = aurweb.git.serve:main', 'aurweb-git-update = aurweb.git.update:main', + 'aurweb-aurblup = aurweb.scripts.aurblup:main', + 'aurweb-mkpkglists = aurweb.scripts.mkpkglists:main', + 'aurweb-notify = aurweb.scripts.notify:main', + 'aurweb-pkgmaint = aurweb.scripts.pkgmaint:main', + 'aurweb-popupdate = aurweb.scripts.popupdate:main', + 'aurweb-tuvotereminder = aurweb.scripts.tuvotereminder:main', ], }, ) diff --git a/upgrading/4.4.1.txt b/upgrading/4.4.1.txt index b06696e..726f9e2 100644 --- a/upgrading/4.4.1.txt +++ b/upgrading/4.4.1.txt @@ -1,3 +1,6 @@ 1. The default configuration file search path now points to /etc/aurweb/config. Make sure you copy your aurweb configuration to the new location before upgrading. + +2. The maintenance scripts have been prefixed by "aurweb-" and can now be + installed using `python3 setup.py install`. -- cgit v1.2.3-54-g00ecf