From dc3fd60715a5b17b9542ec888c6eaeb14c284e2b Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Tue, 20 Sep 2016 20:18:24 +0200 Subject: Use setuptools to install Python modules Instead of using relative imports, add support for installing the config and db Python modules to a proper location using setuptools. Change all git-interface scripts to access those modules from the search path. Signed-off-by: Lukas Fleischer --- aurweb/__init__.py | 0 aurweb/config.py | 31 +++++++++++++++++++++++++++++++ aurweb/db.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 aurweb/__init__.py create mode 100644 aurweb/config.py create mode 100644 aurweb/db.py (limited to 'aurweb') diff --git a/aurweb/__init__.py b/aurweb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aurweb/config.py b/aurweb/config.py new file mode 100644 index 0000000..aac188b --- /dev/null +++ b/aurweb/config.py @@ -0,0 +1,31 @@ +import configparser +import os + +_parser = None + + +def _get_parser(): + global _parser + + if not _parser: + _parser = configparser.RawConfigParser() + if 'AUR_CONFIG' in os.environ: + path = os.environ.get('AUR_CONFIG') + else: + relpath = "/../conf/config" + path = os.path.dirname(os.path.realpath(__file__)) + relpath + _parser.read(path) + + return _parser + + +def get(section, option): + return _get_parser().get(section, option) + + +def getboolean(section, option): + return _get_parser().getboolean(section, option) + + +def getint(section, option): + return _get_parser().getint(section, option) diff --git a/aurweb/db.py b/aurweb/db.py new file mode 100644 index 0000000..0b58197 --- /dev/null +++ b/aurweb/db.py @@ -0,0 +1,51 @@ +import mysql.connector +import sqlite3 + +import aurweb.config + + +class Connection: + _conn = None + _paramstyle = None + + def __init__(self): + aur_db_backend = aurweb.config.get('database', 'backend') + + if aur_db_backend == 'mysql': + aur_db_host = aurweb.config.get('database', 'host') + aur_db_name = aurweb.config.get('database', 'name') + aur_db_user = aurweb.config.get('database', 'user') + aur_db_pass = aurweb.config.get('database', 'password') + aur_db_socket = aurweb.config.get('database', 'socket') + self._conn = mysql.connector.connect(host=aur_db_host, + user=aur_db_user, + passwd=aur_db_pass, + db=aur_db_name, + unix_socket=aur_db_socket, + buffered=True) + self._paramstyle = mysql.connector.paramstyle + elif aur_db_backend == 'sqlite': + aur_db_name = aurweb.config.get('database', 'name') + self._conn = sqlite3.connect(aur_db_name) + self._paramstyle = sqlite3.paramstyle + else: + raise ValueError('unsupported database backend') + + def execute(self, query, params=()): + if self._paramstyle in ('format', 'pyformat'): + query = query.replace('%', '%%').replace('?', '%s') + elif self._paramstyle == 'qmark': + pass + else: + raise ValueError('unsupported paramstyle') + + cur = self._conn.cursor() + cur.execute(query, params) + + return cur + + def commit(self): + self._conn.commit() + + def close(self): + self._conn.close() -- cgit v1.2.3-54-g00ecf From d4fe77ac572ef0e60c9ffa5f987c9cda448cf9f2 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Sat, 8 Oct 2016 14:19:11 +0200 Subject: Reorganize Git interface scripts Move the Git interface scripts from git-interface/ to aurweb/git/. Use setuptools to automatically create wrappers which can be installed using `python3 setup.py install`. Update the configuration files, the test suite as well as the INSTALL and README files to reflect these changes. Signed-off-by: Lukas Fleischer --- INSTALL | 28 ++- README | 5 +- aurweb/git/auth.py | 62 +++++++ aurweb/git/serve.py | 409 +++++++++++++++++++++++++++++++++++++++++ aurweb/git/update.py | 419 +++++++++++++++++++++++++++++++++++++++++++ conf/config.proto | 4 +- git-interface/Makefile | 18 -- git-interface/__init__.py | 0 git-interface/config.mk | 1 - git-interface/git-auth.py | 62 ------- git-interface/git-auth.sh.in | 3 - git-interface/git-serve.py | 409 ----------------------------------------- git-interface/git-update.py | 419 ------------------------------------------- setup.py | 7 + test/setup.sh | 8 +- 15 files changed, 916 insertions(+), 938 deletions(-) create mode 100755 aurweb/git/auth.py create mode 100755 aurweb/git/serve.py create mode 100755 aurweb/git/update.py delete mode 100644 git-interface/Makefile delete mode 100644 git-interface/__init__.py delete mode 100644 git-interface/config.mk delete mode 100755 git-interface/git-auth.py delete mode 100644 git-interface/git-auth.sh.in delete mode 100755 git-interface/git-serve.py delete mode 100755 git-interface/git-update.py (limited to 'aurweb') diff --git a/INSTALL b/INSTALL index dab48cc..395915a 100644 --- a/INSTALL +++ b/INSTALL @@ -37,11 +37,16 @@ Setup on Arch Linux $ mysql -uaur -p AUR 1 else '0', + } + key = keytype + ' ' + keytext + + print(format_command(env_vars, git_serve_cmd, ssh_opts, key)) + + +if __name__ == '__main__': + main() diff --git a/aurweb/git/serve.py b/aurweb/git/serve.py new file mode 100755 index 0000000..ebfef94 --- /dev/null +++ b/aurweb/git/serve.py @@ -0,0 +1,409 @@ +#!/usr/bin/python3 + +import os +import re +import shlex +import subprocess +import sys +import time + +import aurweb.config +import aurweb.db + +notify_cmd = aurweb.config.get('notifications', 'notify-cmd') + +repo_path = aurweb.config.get('serve', 'repo-path') +repo_regex = aurweb.config.get('serve', 'repo-regex') +git_shell_cmd = aurweb.config.get('serve', 'git-shell-cmd') +git_update_cmd = aurweb.config.get('serve', 'git-update-cmd') +ssh_cmdline = aurweb.config.get('serve', 'ssh-cmdline') + +enable_maintenance = aurweb.config.getboolean('options', 'enable-maintenance') +maintenance_exc = aurweb.config.get('options', 'maintenance-exceptions').split() + + +def pkgbase_from_name(pkgbase): + conn = aurweb.db.Connection() + cur = conn.execute("SELECT ID FROM PackageBases WHERE Name = ?", [pkgbase]) + + row = cur.fetchone() + return row[0] if row else None + + +def pkgbase_exists(pkgbase): + return pkgbase_from_name(pkgbase) is not None + + +def list_repos(user): + conn = aurweb.db.Connection() + + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) + userid = cur.fetchone()[0] + if userid == 0: + die('{:s}: unknown user: {:s}'.format(action, user)) + + cur = conn.execute("SELECT Name, PackagerUID FROM PackageBases " + + "WHERE MaintainerUID = ?", [userid]) + for row in cur: + print((' ' if row[1] else '*') + row[0]) + conn.close() + + +def create_pkgbase(pkgbase, user): + if not re.match(repo_regex, pkgbase): + die('{:s}: invalid repository name: {:s}'.format(action, pkgbase)) + if pkgbase_exists(pkgbase): + die('{:s}: package base already exists: {:s}'.format(action, pkgbase)) + + conn = aurweb.db.Connection() + + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) + userid = cur.fetchone()[0] + if userid == 0: + die('{:s}: unknown user: {:s}'.format(action, user)) + + now = int(time.time()) + cur = conn.execute("INSERT INTO PackageBases (Name, SubmittedTS, " + + "ModifiedTS, SubmitterUID, MaintainerUID) VALUES " + + "(?, ?, ?, ?, ?)", [pkgbase, now, now, userid, userid]) + pkgbase_id = cur.lastrowid + + cur = conn.execute("INSERT INTO PackageNotifications " + + "(PackageBaseID, UserID) VALUES (?, ?)", + [pkgbase_id, userid]) + + conn.commit() + conn.close() + + +def pkgbase_adopt(pkgbase, user, privileged): + pkgbase_id = pkgbase_from_name(pkgbase) + if not pkgbase_id: + die('{:s}: package base not found: {:s}'.format(action, pkgbase)) + + conn = aurweb.db.Connection() + + cur = conn.execute("SELECT ID FROM PackageBases WHERE ID = ? AND " + + "MaintainerUID IS NULL", [pkgbase_id]) + if not privileged and not cur.fetchone(): + die('{:s}: permission denied: {:s}'.format(action, user)) + + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) + userid = cur.fetchone()[0] + if userid == 0: + die('{:s}: unknown user: {:s}'.format(action, user)) + + cur = conn.execute("UPDATE PackageBases SET MaintainerUID = ? " + + "WHERE ID = ?", [userid, pkgbase_id]) + + cur = conn.execute("SELECT COUNT(*) FROM PackageNotifications WHERE " + + "PackageBaseID = ? AND UserID = ?", + [pkgbase_id, userid]) + if cur.fetchone()[0] == 0: + cur = conn.execute("INSERT INTO PackageNotifications " + + "(PackageBaseID, UserID) VALUES (?, ?)", + [pkgbase_id, userid]) + conn.commit() + + subprocess.Popen((notify_cmd, 'adopt', str(pkgbase_id), str(userid))) + + conn.close() + + +def pkgbase_get_comaintainers(pkgbase): + conn = aurweb.db.Connection() + + cur = conn.execute("SELECT UserName FROM PackageComaintainers " + + "INNER JOIN Users " + + "ON Users.ID = PackageComaintainers.UsersID " + + "INNER JOIN PackageBases " + + "ON PackageBases.ID = PackageComaintainers.PackageBaseID " + + "WHERE PackageBases.Name = ? " + + "ORDER BY Priority ASC", [pkgbase]) + + return [row[0] for row in cur.fetchall()] + + +def pkgbase_set_comaintainers(pkgbase, userlist, user, privileged): + pkgbase_id = pkgbase_from_name(pkgbase) + if not pkgbase_id: + die('{:s}: package base not found: {:s}'.format(action, pkgbase)) + + if not privileged and not pkgbase_has_full_access(pkgbase, user): + die('{:s}: permission denied: {:s}'.format(action, user)) + + conn = aurweb.db.Connection() + + userlist_old = set(pkgbase_get_comaintainers(pkgbase)) + + uids_old = set() + for olduser in userlist_old: + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", + [olduser]) + userid = cur.fetchone()[0] + if userid == 0: + die('{:s}: unknown user: {:s}'.format(action, user)) + uids_old.add(userid) + + uids_new = set() + for newuser in userlist: + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", + [newuser]) + userid = cur.fetchone()[0] + if userid == 0: + die('{:s}: unknown user: {:s}'.format(action, user)) + uids_new.add(userid) + + uids_add = uids_new - uids_old + uids_rem = uids_old - uids_new + + i = 1 + for userid in uids_new: + if userid in uids_add: + cur = conn.execute("INSERT INTO PackageComaintainers " + + "(PackageBaseID, UsersID, Priority) " + + "VALUES (?, ?, ?)", [pkgbase_id, userid, i]) + subprocess.Popen((notify_cmd, 'comaintainer-add', str(pkgbase_id), + str(userid))) + else: + cur = conn.execute("UPDATE PackageComaintainers " + + "SET Priority = ? " + + "WHERE PackageBaseID = ? AND UsersID = ?", + [i, pkgbase_id, userid]) + i += 1 + + for userid in uids_rem: + cur = conn.execute("DELETE FROM PackageComaintainers " + + "WHERE PackageBaseID = ? AND UsersID = ?", + [pkgbase_id, userid]) + subprocess.Popen((notify_cmd, 'comaintainer-remove', + str(pkgbase_id), str(userid))) + + conn.commit() + conn.close() + + +def pkgbase_disown(pkgbase, user, privileged): + pkgbase_id = pkgbase_from_name(pkgbase) + if not pkgbase_id: + die('{:s}: package base not found: {:s}'.format(action, pkgbase)) + + initialized_by_owner = pkgbase_has_full_access(pkgbase, user) + if not privileged and not initialized_by_owner: + die('{:s}: permission denied: {:s}'.format(action, user)) + + # TODO: Support disowning package bases via package request. + # TODO: Scan through pending orphan requests and close them. + + comaintainers = [] + new_maintainer_userid = None + + conn = aurweb.db.Connection() + + # Make the first co-maintainer the new maintainer, unless the action was + # enforced by a Trusted User. + if initialized_by_owner: + comaintainers = pkgbase_get_comaintainers(pkgbase) + if len(comaintainers) > 0: + new_maintainer = comaintainers[0] + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", + [new_maintainer]) + new_maintainer_userid = cur.fetchone()[0] + comaintainers.remove(new_maintainer) + + pkgbase_set_comaintainers(pkgbase, comaintainers, user, privileged) + cur = conn.execute("UPDATE PackageBases SET MaintainerUID = ? " + + "WHERE ID = ?", [new_maintainer_userid, pkgbase_id]) + + conn.commit() + + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) + userid = cur.fetchone()[0] + if userid == 0: + die('{:s}: unknown user: {:s}'.format(action, user)) + + subprocess.Popen((notify_cmd, 'disown', str(pkgbase_id), str(userid))) + + conn.close() + + +def pkgbase_set_keywords(pkgbase, keywords): + pkgbase_id = pkgbase_from_name(pkgbase) + if not pkgbase_id: + die('{:s}: package base not found: {:s}'.format(action, pkgbase)) + + conn = aurweb.db.Connection() + + conn.execute("DELETE FROM PackageKeywords WHERE PackageBaseID = ?", + [pkgbase_id]) + for keyword in keywords: + conn.execute("INSERT INTO PackageKeywords (PackageBaseID, Keyword) " + + "VALUES (?, ?)", [pkgbase_id, keyword]) + + conn.commit() + conn.close() + + +def pkgbase_has_write_access(pkgbase, user): + conn = aurweb.db.Connection() + + cur = conn.execute("SELECT COUNT(*) FROM PackageBases " + + "LEFT JOIN PackageComaintainers " + + "ON PackageComaintainers.PackageBaseID = PackageBases.ID " + + "INNER JOIN Users " + + "ON Users.ID = PackageBases.MaintainerUID " + + "OR PackageBases.MaintainerUID IS NULL " + + "OR Users.ID = PackageComaintainers.UsersID " + + "WHERE Name = ? AND Username = ?", [pkgbase, user]) + return cur.fetchone()[0] > 0 + + +def pkgbase_has_full_access(pkgbase, user): + conn = aurweb.db.Connection() + + cur = conn.execute("SELECT COUNT(*) FROM PackageBases " + + "INNER JOIN Users " + + "ON Users.ID = PackageBases.MaintainerUID " + + "WHERE Name = ? AND Username = ?", [pkgbase, user]) + return cur.fetchone()[0] > 0 + + +def die(msg): + sys.stderr.write("{:s}\n".format(msg)) + exit(1) + + +def die_with_help(msg): + die(msg + "\nTry `{:s} help` for a list of commands.".format(ssh_cmdline)) + + +def warn(msg): + sys.stderr.write("warning: {:s}\n".format(msg)) + + +def usage(cmds): + sys.stderr.write("Commands:\n") + colwidth = max([len(cmd) for cmd in cmds.keys()]) + 4 + for key in sorted(cmds): + sys.stderr.write(" " + key.ljust(colwidth) + cmds[key] + "\n") + exit(0) + + +def main(): + user = os.environ.get('AUR_USER') + privileged = (os.environ.get('AUR_PRIVILEGED', '0') == '1') + ssh_cmd = os.environ.get('SSH_ORIGINAL_COMMAND') + ssh_client = os.environ.get('SSH_CLIENT') + + if not ssh_cmd: + die_with_help("Interactive shell is disabled.") + cmdargv = shlex.split(ssh_cmd) + action = cmdargv[0] + remote_addr = ssh_client.split(' ')[0] if ssh_client else None + + if enable_maintenance: + if remote_addr not in maintenance_exc: + die("The AUR is down due to maintenance. We will be back soon.") + + if action == 'git' and cmdargv[1] in ('upload-pack', 'receive-pack'): + action = action + '-' + cmdargv[1] + del cmdargv[1] + + if action == 'git-upload-pack' or action == 'git-receive-pack': + if len(cmdargv) < 2: + die_with_help("{:s}: missing path".format(action)) + + path = cmdargv[1].rstrip('/') + if not path.startswith('/'): + path = '/' + path + if not path.endswith('.git'): + path = path + '.git' + pkgbase = path[1:-4] + if not re.match(repo_regex, pkgbase): + die('{:s}: invalid repository name: {:s}'.format(action, pkgbase)) + + if action == 'git-receive-pack' and pkgbase_exists(pkgbase): + if not privileged and not pkgbase_has_write_access(pkgbase, user): + die('{:s}: permission denied: {:s}'.format(action, user)) + + os.environ["AUR_USER"] = user + os.environ["AUR_PKGBASE"] = pkgbase + os.environ["GIT_NAMESPACE"] = pkgbase + cmd = action + " '" + repo_path + "'" + os.execl(git_shell_cmd, git_shell_cmd, '-c', cmd) + elif action == 'set-keywords': + if len(cmdargv) < 2: + die_with_help("{:s}: missing repository name".format(action)) + pkgbase_set_keywords(cmdargv[1], cmdargv[2:]) + elif action == 'list-repos': + if len(cmdargv) > 1: + die_with_help("{:s}: too many arguments".format(action)) + list_repos(user) + elif action == 'setup-repo': + if len(cmdargv) < 2: + die_with_help("{:s}: missing repository name".format(action)) + if len(cmdargv) > 2: + die_with_help("{:s}: too many arguments".format(action)) + warn('{:s} is deprecated. ' + 'Use `git push` to create new repositories.'.format(action)) + create_pkgbase(cmdargv[1], user) + elif action == 'restore': + if len(cmdargv) < 2: + die_with_help("{:s}: missing repository name".format(action)) + if len(cmdargv) > 2: + die_with_help("{:s}: too many arguments".format(action)) + + pkgbase = cmdargv[1] + if not re.match(repo_regex, pkgbase): + die('{:s}: invalid repository name: {:s}'.format(action, pkgbase)) + + if pkgbase_exists(pkgbase): + die('{:s}: package base exists: {:s}'.format(action, pkgbase)) + create_pkgbase(pkgbase, user) + + os.environ["AUR_USER"] = user + os.environ["AUR_PKGBASE"] = pkgbase + os.execl(git_update_cmd, git_update_cmd, 'restore') + elif action == 'adopt': + if len(cmdargv) < 2: + die_with_help("{:s}: missing repository name".format(action)) + if len(cmdargv) > 2: + die_with_help("{:s}: too many arguments".format(action)) + + pkgbase = cmdargv[1] + pkgbase_adopt(pkgbase, user, privileged) + elif action == 'disown': + if len(cmdargv) < 2: + die_with_help("{:s}: missing repository name".format(action)) + if len(cmdargv) > 2: + die_with_help("{:s}: too many arguments".format(action)) + + pkgbase = cmdargv[1] + pkgbase_disown(pkgbase, user, privileged) + elif action == 'set-comaintainers': + if len(cmdargv) < 2: + die_with_help("{:s}: missing repository name".format(action)) + + pkgbase = cmdargv[1] + userlist = cmdargv[2:] + pkgbase_set_comaintainers(pkgbase, userlist, user, privileged) + elif action == 'help': + cmds = { + "adopt ": "Adopt a package base.", + "disown ": "Disown a package base.", + "help": "Show this help message and exit.", + "list-repos": "List all your repositories.", + "restore ": "Restore a deleted package base.", + "set-comaintainers [...]": "Set package base co-maintainers.", + "set-keywords [...]": "Change package base keywords.", + "setup-repo ": "Create a repository (deprecated).", + "git-receive-pack": "Internal command used with Git.", + "git-upload-pack": "Internal command used with Git.", + } + usage(cmds) + else: + die_with_help("invalid command: {:s}".format(action)) + + +if __name__ == '__main__': + main() diff --git a/aurweb/git/update.py b/aurweb/git/update.py new file mode 100755 index 0000000..7337341 --- /dev/null +++ b/aurweb/git/update.py @@ -0,0 +1,419 @@ +#!/usr/bin/python3 + +import os +import pygit2 +import re +import subprocess +import sys +import time + +import srcinfo.parse +import srcinfo.utils + +import aurweb.config +import aurweb.db + +notify_cmd = aurweb.config.get('notifications', 'notify-cmd') + +repo_path = aurweb.config.get('serve', 'repo-path') +repo_regex = aurweb.config.get('serve', 'repo-regex') + +max_blob_size = aurweb.config.getint('update', 'max-blob-size') + + +def size_humanize(num): + for unit in ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB']: + if abs(num) < 2048.0: + if isinstance(num, int): + return "{}{}".format(num, unit) + else: + return "{:.2f}{}".format(num, unit) + num /= 1024.0 + return "{:.2f}{}".format(num, 'YiB') + + +def extract_arch_fields(pkginfo, field): + values = [] + + if field in pkginfo: + for val in pkginfo[field]: + values.append({"value": val, "arch": None}) + + for arch in ['i686', 'x86_64']: + if field + '_' + arch in pkginfo: + for val in pkginfo[field + '_' + arch]: + values.append({"value": val, "arch": arch}) + + return values + + +def parse_dep(depstring): + dep, _, desc = depstring.partition(': ') + depname = re.sub(r'(<|=|>).*', '', dep) + depcond = dep[len(depname):] + + if (desc): + return (depname + ': ' + desc, depcond) + else: + return (depname, depcond) + + +def create_pkgbase(conn, pkgbase, user): + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) + userid = cur.fetchone()[0] + + now = int(time.time()) + cur = conn.execute("INSERT INTO PackageBases (Name, SubmittedTS, " + + "ModifiedTS, SubmitterUID, MaintainerUID) VALUES " + + "(?, ?, ?, ?, ?)", [pkgbase, now, now, userid, userid]) + pkgbase_id = cur.lastrowid + + cur = conn.execute("INSERT INTO PackageNotifications " + + "(PackageBaseID, UserID) VALUES (?, ?)", + [pkgbase_id, userid]) + + conn.commit() + + return pkgbase_id + + +def save_metadata(metadata, conn, user): + # Obtain package base ID and previous maintainer. + pkgbase = metadata['pkgbase'] + cur = conn.execute("SELECT ID, MaintainerUID FROM PackageBases " + "WHERE Name = ?", [pkgbase]) + (pkgbase_id, maintainer_uid) = cur.fetchone() + was_orphan = not maintainer_uid + + # Obtain the user ID of the new maintainer. + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) + user_id = int(cur.fetchone()[0]) + + # Update package base details and delete current packages. + now = int(time.time()) + conn.execute("UPDATE PackageBases SET ModifiedTS = ?, " + + "PackagerUID = ?, OutOfDateTS = NULL WHERE ID = ?", + [now, user_id, pkgbase_id]) + conn.execute("UPDATE PackageBases SET MaintainerUID = ? " + + "WHERE ID = ? AND MaintainerUID IS NULL", + [user_id, pkgbase_id]) + for table in ('Sources', 'Depends', 'Relations', 'Licenses', 'Groups'): + conn.execute("DELETE FROM Package" + table + " WHERE EXISTS (" + + "SELECT * FROM Packages " + + "WHERE Packages.PackageBaseID = ? AND " + + "Package" + table + ".PackageID = Packages.ID)", + [pkgbase_id]) + conn.execute("DELETE FROM Packages WHERE PackageBaseID = ?", [pkgbase_id]) + + for pkgname in srcinfo.utils.get_package_names(metadata): + pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) + + if 'epoch' in pkginfo and int(pkginfo['epoch']) > 0: + ver = '{:d}:{:s}-{:s}'.format(int(pkginfo['epoch']), + pkginfo['pkgver'], + pkginfo['pkgrel']) + else: + ver = '{:s}-{:s}'.format(pkginfo['pkgver'], pkginfo['pkgrel']) + + for field in ('pkgdesc', 'url'): + if field not in pkginfo: + pkginfo[field] = None + + # Create a new package. + cur = conn.execute("INSERT INTO Packages (PackageBaseID, Name, " + + "Version, Description, URL) " + + "VALUES (?, ?, ?, ?, ?)", + [pkgbase_id, pkginfo['pkgname'], ver, + pkginfo['pkgdesc'], pkginfo['url']]) + conn.commit() + pkgid = cur.lastrowid + + # Add package sources. + for source_info in extract_arch_fields(pkginfo, 'source'): + conn.execute("INSERT INTO PackageSources (PackageID, Source, " + + "SourceArch) VALUES (?, ?, ?)", + [pkgid, source_info['value'], source_info['arch']]) + + # Add package dependencies. + for deptype in ('depends', 'makedepends', + 'checkdepends', 'optdepends'): + cur = conn.execute("SELECT ID FROM DependencyTypes WHERE Name = ?", + [deptype]) + deptypeid = cur.fetchone()[0] + for dep_info in extract_arch_fields(pkginfo, deptype): + depname, depcond = parse_dep(dep_info['value']) + deparch = dep_info['arch'] + conn.execute("INSERT INTO PackageDepends (PackageID, " + + "DepTypeID, DepName, DepCondition, DepArch) " + + "VALUES (?, ?, ?, ?, ?)", + [pkgid, deptypeid, depname, depcond, deparch]) + + # Add package relations (conflicts, provides, replaces). + for reltype in ('conflicts', 'provides', 'replaces'): + cur = conn.execute("SELECT ID FROM RelationTypes WHERE Name = ?", + [reltype]) + reltypeid = cur.fetchone()[0] + for rel_info in extract_arch_fields(pkginfo, reltype): + relname, relcond = parse_dep(rel_info['value']) + relarch = rel_info['arch'] + conn.execute("INSERT INTO PackageRelations (PackageID, " + + "RelTypeID, RelName, RelCondition, RelArch) " + + "VALUES (?, ?, ?, ?, ?)", + [pkgid, reltypeid, relname, relcond, relarch]) + + # Add package licenses. + if 'license' in pkginfo: + for license in pkginfo['license']: + cur = conn.execute("SELECT ID FROM Licenses WHERE Name = ?", + [license]) + row = cur.fetchone() + if row: + licenseid = row[0] + else: + cur = conn.execute("INSERT INTO Licenses (Name) " + + "VALUES (?)", [license]) + conn.commit() + licenseid = cur.lastrowid + conn.execute("INSERT INTO PackageLicenses (PackageID, " + + "LicenseID) VALUES (?, ?)", + [pkgid, licenseid]) + + # Add package groups. + if 'groups' in pkginfo: + for group in pkginfo['groups']: + cur = conn.execute("SELECT ID FROM Groups WHERE Name = ?", + [group]) + row = cur.fetchone() + if row: + groupid = row[0] + else: + cur = conn.execute("INSERT INTO Groups (Name) VALUES (?)", + [group]) + conn.commit() + groupid = cur.lastrowid + conn.execute("INSERT INTO PackageGroups (PackageID, " + "GroupID) VALUES (?, ?)", [pkgid, groupid]) + + # Add user to notification list on adoption. + if was_orphan: + cur = conn.execute("SELECT COUNT(*) FROM PackageNotifications WHERE " + + "PackageBaseID = ? AND UserID = ?", + [pkgbase_id, user_id]) + if cur.fetchone()[0] == 0: + conn.execute("INSERT INTO PackageNotifications " + + "(PackageBaseID, UserID) VALUES (?, ?)", + [pkgbase_id, user_id]) + + conn.commit() + + +def update_notify(conn, user, pkgbase_id): + # Obtain the user ID of the new maintainer. + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) + user_id = int(cur.fetchone()[0]) + + # Execute the notification script. + subprocess.Popen((notify_cmd, 'update', str(user_id), str(pkgbase_id))) + + +def die(msg): + sys.stderr.write("error: {:s}\n".format(msg)) + exit(1) + + +def warn(msg): + sys.stderr.write("warning: {:s}\n".format(msg)) + + +def die_commit(msg, commit): + sys.stderr.write("error: The following error " + + "occurred when parsing commit\n") + sys.stderr.write("error: {:s}:\n".format(commit)) + sys.stderr.write("error: {:s}\n".format(msg)) + exit(1) + + +def main(): + repo = pygit2.Repository(repo_path) + + user = os.environ.get("AUR_USER") + pkgbase = os.environ.get("AUR_PKGBASE") + privileged = (os.environ.get("AUR_PRIVILEGED", '0') == '1') + warn_or_die = warn if privileged else die + + if len(sys.argv) == 2 and sys.argv[1] == "restore": + if 'refs/heads/' + pkgbase not in repo.listall_references(): + die('{:s}: repository not found: {:s}'.format(sys.argv[1], + pkgbase)) + refname = "refs/heads/master" + branchref = 'refs/heads/' + pkgbase + sha1_old = sha1_new = repo.lookup_reference(branchref).target + elif len(sys.argv) == 4: + refname, sha1_old, sha1_new = sys.argv[1:4] + else: + die("invalid arguments") + + if refname != "refs/heads/master": + die("pushing to a branch other than master is restricted") + + conn = aurweb.db.Connection() + + # Detect and deny non-fast-forwards. + if sha1_old != "0" * 40 and not privileged: + walker = repo.walk(sha1_old, pygit2.GIT_SORT_TOPOLOGICAL) + walker.hide(sha1_new) + if next(walker, None) is not None: + die("denying non-fast-forward (you should pull first)") + + # Prepare the walker that validates new commits. + walker = repo.walk(sha1_new, pygit2.GIT_SORT_TOPOLOGICAL) + if sha1_old != "0" * 40: + walker.hide(sha1_old) + + # Validate all new commits. + for commit in walker: + for fname in ('.SRCINFO', 'PKGBUILD'): + if fname not in commit.tree: + die_commit("missing {:s}".format(fname), str(commit.id)) + + for treeobj in commit.tree: + blob = repo[treeobj.id] + + if isinstance(blob, pygit2.Tree): + die_commit("the repository must not contain subdirectories", + str(commit.id)) + + if not isinstance(blob, pygit2.Blob): + die_commit("not a blob object: {:s}".format(treeobj), + str(commit.id)) + + if blob.size > max_blob_size: + die_commit("maximum blob size ({:s}) exceeded".format( + size_humanize(max_blob_size)), str(commit.id)) + + metadata_raw = repo[commit.tree['.SRCINFO'].id].data.decode() + (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw) + if errors: + sys.stderr.write("error: The following errors occurred " + "when parsing .SRCINFO in commit\n") + sys.stderr.write("error: {:s}:\n".format(str(commit.id))) + for error in errors: + for err in error['error']: + sys.stderr.write("error: line {:d}: {:s}\n".format( + error['line'], err)) + exit(1) + + metadata_pkgbase = metadata['pkgbase'] + if not re.match(repo_regex, metadata_pkgbase): + die_commit('invalid pkgbase: {:s}'.format(metadata_pkgbase), + str(commit.id)) + + for pkgname in set(metadata['packages'].keys()): + pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) + + for field in ('pkgver', 'pkgrel', 'pkgname'): + if field not in pkginfo: + die_commit('missing mandatory field: {:s}'.format(field), + str(commit.id)) + + if 'epoch' in pkginfo and not pkginfo['epoch'].isdigit(): + die_commit('invalid epoch: {:s}'.format(pkginfo['epoch']), + str(commit.id)) + + if not re.match(r'[a-z0-9][a-z0-9\.+_-]*$', pkginfo['pkgname']): + die_commit('invalid package name: {:s}'.format( + pkginfo['pkgname']), str(commit.id)) + + for field in ('pkgname', 'pkgdesc', 'url'): + if field in pkginfo and len(pkginfo[field]) > 255: + die_commit('{:s} field too long: {:s}'.format(field, + pkginfo[field]), str(commit.id)) + + for field in ('install', 'changelog'): + if field in pkginfo and not pkginfo[field] in commit.tree: + die_commit('missing {:s} file: {:s}'.format(field, + pkginfo[field]), str(commit.id)) + + for field in extract_arch_fields(pkginfo, 'source'): + fname = field['value'] + if "://" in fname or "lp:" in fname: + continue + if fname not in commit.tree: + die_commit('missing source file: {:s}'.format(fname), + str(commit.id)) + + # Display a warning if .SRCINFO is unchanged. + if sha1_old not in ("0000000000000000000000000000000000000000", sha1_new): + srcinfo_id_old = repo[sha1_old].tree['.SRCINFO'].id + srcinfo_id_new = repo[sha1_new].tree['.SRCINFO'].id + if srcinfo_id_old == srcinfo_id_new: + warn(".SRCINFO unchanged. " + "The package database will not be updated!") + + # Read .SRCINFO from the HEAD commit. + metadata_raw = repo[repo[sha1_new].tree['.SRCINFO'].id].data.decode() + (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw) + + # Ensure that the package base name matches the repository name. + metadata_pkgbase = metadata['pkgbase'] + if metadata_pkgbase != pkgbase: + die('invalid pkgbase: {:s}, expected {:s}'.format(metadata_pkgbase, + pkgbase)) + + # Ensure that packages are neither blacklisted nor overwritten. + pkgbase = metadata['pkgbase'] + cur = conn.execute("SELECT ID FROM PackageBases WHERE Name = ?", [pkgbase]) + row = cur.fetchone() + pkgbase_id = row[0] if row else 0 + + cur = conn.execute("SELECT Name FROM PackageBlacklist") + blacklist = [row[0] for row in cur.fetchall()] + + cur = conn.execute("SELECT Name, Repo FROM OfficialProviders") + providers = dict(cur.fetchall()) + + for pkgname in srcinfo.utils.get_package_names(metadata): + pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) + pkgname = pkginfo['pkgname'] + + if pkgname in blacklist: + warn_or_die('package is blacklisted: {:s}'.format(pkgname)) + if pkgname in providers: + warn_or_die('package already provided by [{:s}]: {:s}'.format( + providers[pkgname], pkgname)) + + cur = conn.execute("SELECT COUNT(*) FROM Packages WHERE Name = ? " + + "AND PackageBaseID <> ?", [pkgname, pkgbase_id]) + if cur.fetchone()[0] > 0: + die('cannot overwrite package: {:s}'.format(pkgname)) + + # Create a new package base if it does not exist yet. + if pkgbase_id == 0: + pkgbase_id = create_pkgbase(conn, pkgbase, user) + + # Store package base details in the database. + save_metadata(metadata, conn, user) + + # Create (or update) a branch with the name of the package base for better + # accessibility. + branchref = 'refs/heads/' + pkgbase + repo.create_reference(branchref, sha1_new, True) + + # Work around a Git bug: The HEAD ref is not updated when using + # gitnamespaces. This can be removed once the bug fix is included in Git + # mainline. See + # http://git.661346.n2.nabble.com/PATCH-receive-pack-Create-a-HEAD-ref-for-ref-namespace-td7632149.html + # for details. + headref = 'refs/namespaces/' + pkgbase + '/HEAD' + repo.create_reference(headref, sha1_new, True) + + # Send package update notifications. + update_notify(conn, user, pkgbase_id) + + # Close the database. + cur.close() + conn.close() + + +if __name__ == '__main__': + main() diff --git a/conf/config.proto b/conf/config.proto index 21441a9..96fad80 100644 --- a/conf/config.proto +++ b/conf/config.proto @@ -46,14 +46,14 @@ RSA = SHA256:Ju+yWiMb/2O+gKQ9RJCDqvRg7l+Q95KFAeqM5sr6l2s [auth] valid-keytypes = ssh-rsa ssh-dss ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 ssh-ed25519 username-regex = [a-zA-Z0-9]+[.\-_]?[a-zA-Z0-9]+$ -git-serve-cmd = /srv/http/aurweb/git-interface/git-serve.py +git-serve-cmd = /usr/local/bin/aurweb-git-serve ssh-options = restrict [serve] repo-path = /srv/http/aurweb/aur.git/ repo-regex = [a-z0-9][a-z0-9.+_-]*$ git-shell-cmd = /usr/bin/git-shell -git-update-cmd = /srv/http/aurweb/git-interface/git-update.py +git-update-cmd = /usr/local/bin/aurweb-git-update ssh-cmdline = ssh aur@aur.archlinux.org [update] diff --git a/git-interface/Makefile b/git-interface/Makefile deleted file mode 100644 index 8865790..0000000 --- a/git-interface/Makefile +++ /dev/null @@ -1,18 +0,0 @@ -GIT_INTERFACE_DIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) - -include config.mk - -git-auth.sh: - sed 's#%GIT_INTERFACE_DIR%#$(GIT_INTERFACE_DIR)#' git-auth.sh - chmod +x git-auth.sh - -install: git-auth.sh - install -Dm0755 git-auth.sh "$(DESTDIR)$(PREFIX)/bin/aur-git-auth" - -uninstall: - rm -f "$(DESTDIR)$(PREFIX)/bin/aur-git-auth" - -clean: - rm -f git-auth.sh - -.PHONY: install uninstall clean diff --git a/git-interface/__init__.py b/git-interface/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/git-interface/config.mk b/git-interface/config.mk deleted file mode 100644 index 4d794a1..0000000 --- a/git-interface/config.mk +++ /dev/null @@ -1 +0,0 @@ -PREFIX = /usr/local diff --git a/git-interface/git-auth.py b/git-interface/git-auth.py deleted file mode 100755 index 022b0ff..0000000 --- a/git-interface/git-auth.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/python3 - -import shlex -import re -import sys - -import aurweb.config -import aurweb.db - - -def format_command(env_vars, command, ssh_opts, ssh_key): - environment = '' - for key, var in env_vars.items(): - environment += '{}={} '.format(key, shlex.quote(var)) - - command = shlex.quote(command) - command = '{}{}'.format(environment, command) - - # The command is being substituted into an authorized_keys line below, - # so we need to escape the double quotes. - command = command.replace('"', '\\"') - msg = 'command="{}",{} {}'.format(command, ssh_opts, ssh_key) - return msg - - -def main(): - valid_keytypes = aurweb.config.get('auth', 'valid-keytypes').split() - username_regex = aurweb.config.get('auth', 'username-regex') - git_serve_cmd = aurweb.config.get('auth', 'git-serve-cmd') - ssh_opts = aurweb.config.get('auth', 'ssh-options') - - keytype = sys.argv[1] - keytext = sys.argv[2] - if keytype not in valid_keytypes: - exit(1) - - conn = aurweb.db.Connection() - - cur = conn.execute("SELECT Users.Username, Users.AccountTypeID FROM Users " - "INNER JOIN SSHPubKeys ON SSHPubKeys.UserID = Users.ID " - "WHERE SSHPubKeys.PubKey = ? AND Users.Suspended = 0", - (keytype + " " + keytext,)) - - row = cur.fetchone() - if not row or cur.fetchone(): - exit(1) - - user, account_type = row - if not re.match(username_regex, user): - exit(1) - - env_vars = { - 'AUR_USER': user, - 'AUR_PRIVILEGED': '1' if account_type > 1 else '0', - } - key = keytype + ' ' + keytext - - print(format_command(env_vars, git_serve_cmd, ssh_opts, key)) - - -if __name__ == '__main__': - main() diff --git a/git-interface/git-auth.sh.in b/git-interface/git-auth.sh.in deleted file mode 100644 index 223816a..0000000 --- a/git-interface/git-auth.sh.in +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -%GIT_INTERFACE_DIR%/git-auth.py "$1" "$2" diff --git a/git-interface/git-serve.py b/git-interface/git-serve.py deleted file mode 100755 index ebfef94..0000000 --- a/git-interface/git-serve.py +++ /dev/null @@ -1,409 +0,0 @@ -#!/usr/bin/python3 - -import os -import re -import shlex -import subprocess -import sys -import time - -import aurweb.config -import aurweb.db - -notify_cmd = aurweb.config.get('notifications', 'notify-cmd') - -repo_path = aurweb.config.get('serve', 'repo-path') -repo_regex = aurweb.config.get('serve', 'repo-regex') -git_shell_cmd = aurweb.config.get('serve', 'git-shell-cmd') -git_update_cmd = aurweb.config.get('serve', 'git-update-cmd') -ssh_cmdline = aurweb.config.get('serve', 'ssh-cmdline') - -enable_maintenance = aurweb.config.getboolean('options', 'enable-maintenance') -maintenance_exc = aurweb.config.get('options', 'maintenance-exceptions').split() - - -def pkgbase_from_name(pkgbase): - conn = aurweb.db.Connection() - cur = conn.execute("SELECT ID FROM PackageBases WHERE Name = ?", [pkgbase]) - - row = cur.fetchone() - return row[0] if row else None - - -def pkgbase_exists(pkgbase): - return pkgbase_from_name(pkgbase) is not None - - -def list_repos(user): - conn = aurweb.db.Connection() - - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - userid = cur.fetchone()[0] - if userid == 0: - die('{:s}: unknown user: {:s}'.format(action, user)) - - cur = conn.execute("SELECT Name, PackagerUID FROM PackageBases " + - "WHERE MaintainerUID = ?", [userid]) - for row in cur: - print((' ' if row[1] else '*') + row[0]) - conn.close() - - -def create_pkgbase(pkgbase, user): - if not re.match(repo_regex, pkgbase): - die('{:s}: invalid repository name: {:s}'.format(action, pkgbase)) - if pkgbase_exists(pkgbase): - die('{:s}: package base already exists: {:s}'.format(action, pkgbase)) - - conn = aurweb.db.Connection() - - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - userid = cur.fetchone()[0] - if userid == 0: - die('{:s}: unknown user: {:s}'.format(action, user)) - - now = int(time.time()) - cur = conn.execute("INSERT INTO PackageBases (Name, SubmittedTS, " + - "ModifiedTS, SubmitterUID, MaintainerUID) VALUES " + - "(?, ?, ?, ?, ?)", [pkgbase, now, now, userid, userid]) - pkgbase_id = cur.lastrowid - - cur = conn.execute("INSERT INTO PackageNotifications " + - "(PackageBaseID, UserID) VALUES (?, ?)", - [pkgbase_id, userid]) - - conn.commit() - conn.close() - - -def pkgbase_adopt(pkgbase, user, privileged): - pkgbase_id = pkgbase_from_name(pkgbase) - if not pkgbase_id: - die('{:s}: package base not found: {:s}'.format(action, pkgbase)) - - conn = aurweb.db.Connection() - - cur = conn.execute("SELECT ID FROM PackageBases WHERE ID = ? AND " + - "MaintainerUID IS NULL", [pkgbase_id]) - if not privileged and not cur.fetchone(): - die('{:s}: permission denied: {:s}'.format(action, user)) - - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - userid = cur.fetchone()[0] - if userid == 0: - die('{:s}: unknown user: {:s}'.format(action, user)) - - cur = conn.execute("UPDATE PackageBases SET MaintainerUID = ? " + - "WHERE ID = ?", [userid, pkgbase_id]) - - cur = conn.execute("SELECT COUNT(*) FROM PackageNotifications WHERE " + - "PackageBaseID = ? AND UserID = ?", - [pkgbase_id, userid]) - if cur.fetchone()[0] == 0: - cur = conn.execute("INSERT INTO PackageNotifications " + - "(PackageBaseID, UserID) VALUES (?, ?)", - [pkgbase_id, userid]) - conn.commit() - - subprocess.Popen((notify_cmd, 'adopt', str(pkgbase_id), str(userid))) - - conn.close() - - -def pkgbase_get_comaintainers(pkgbase): - conn = aurweb.db.Connection() - - cur = conn.execute("SELECT UserName FROM PackageComaintainers " + - "INNER JOIN Users " + - "ON Users.ID = PackageComaintainers.UsersID " + - "INNER JOIN PackageBases " + - "ON PackageBases.ID = PackageComaintainers.PackageBaseID " + - "WHERE PackageBases.Name = ? " + - "ORDER BY Priority ASC", [pkgbase]) - - return [row[0] for row in cur.fetchall()] - - -def pkgbase_set_comaintainers(pkgbase, userlist, user, privileged): - pkgbase_id = pkgbase_from_name(pkgbase) - if not pkgbase_id: - die('{:s}: package base not found: {:s}'.format(action, pkgbase)) - - if not privileged and not pkgbase_has_full_access(pkgbase, user): - die('{:s}: permission denied: {:s}'.format(action, user)) - - conn = aurweb.db.Connection() - - userlist_old = set(pkgbase_get_comaintainers(pkgbase)) - - uids_old = set() - for olduser in userlist_old: - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", - [olduser]) - userid = cur.fetchone()[0] - if userid == 0: - die('{:s}: unknown user: {:s}'.format(action, user)) - uids_old.add(userid) - - uids_new = set() - for newuser in userlist: - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", - [newuser]) - userid = cur.fetchone()[0] - if userid == 0: - die('{:s}: unknown user: {:s}'.format(action, user)) - uids_new.add(userid) - - uids_add = uids_new - uids_old - uids_rem = uids_old - uids_new - - i = 1 - for userid in uids_new: - if userid in uids_add: - cur = conn.execute("INSERT INTO PackageComaintainers " + - "(PackageBaseID, UsersID, Priority) " + - "VALUES (?, ?, ?)", [pkgbase_id, userid, i]) - subprocess.Popen((notify_cmd, 'comaintainer-add', str(pkgbase_id), - str(userid))) - else: - cur = conn.execute("UPDATE PackageComaintainers " + - "SET Priority = ? " + - "WHERE PackageBaseID = ? AND UsersID = ?", - [i, pkgbase_id, userid]) - i += 1 - - for userid in uids_rem: - cur = conn.execute("DELETE FROM PackageComaintainers " + - "WHERE PackageBaseID = ? AND UsersID = ?", - [pkgbase_id, userid]) - subprocess.Popen((notify_cmd, 'comaintainer-remove', - str(pkgbase_id), str(userid))) - - conn.commit() - conn.close() - - -def pkgbase_disown(pkgbase, user, privileged): - pkgbase_id = pkgbase_from_name(pkgbase) - if not pkgbase_id: - die('{:s}: package base not found: {:s}'.format(action, pkgbase)) - - initialized_by_owner = pkgbase_has_full_access(pkgbase, user) - if not privileged and not initialized_by_owner: - die('{:s}: permission denied: {:s}'.format(action, user)) - - # TODO: Support disowning package bases via package request. - # TODO: Scan through pending orphan requests and close them. - - comaintainers = [] - new_maintainer_userid = None - - conn = aurweb.db.Connection() - - # Make the first co-maintainer the new maintainer, unless the action was - # enforced by a Trusted User. - if initialized_by_owner: - comaintainers = pkgbase_get_comaintainers(pkgbase) - if len(comaintainers) > 0: - new_maintainer = comaintainers[0] - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", - [new_maintainer]) - new_maintainer_userid = cur.fetchone()[0] - comaintainers.remove(new_maintainer) - - pkgbase_set_comaintainers(pkgbase, comaintainers, user, privileged) - cur = conn.execute("UPDATE PackageBases SET MaintainerUID = ? " + - "WHERE ID = ?", [new_maintainer_userid, pkgbase_id]) - - conn.commit() - - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - userid = cur.fetchone()[0] - if userid == 0: - die('{:s}: unknown user: {:s}'.format(action, user)) - - subprocess.Popen((notify_cmd, 'disown', str(pkgbase_id), str(userid))) - - conn.close() - - -def pkgbase_set_keywords(pkgbase, keywords): - pkgbase_id = pkgbase_from_name(pkgbase) - if not pkgbase_id: - die('{:s}: package base not found: {:s}'.format(action, pkgbase)) - - conn = aurweb.db.Connection() - - conn.execute("DELETE FROM PackageKeywords WHERE PackageBaseID = ?", - [pkgbase_id]) - for keyword in keywords: - conn.execute("INSERT INTO PackageKeywords (PackageBaseID, Keyword) " + - "VALUES (?, ?)", [pkgbase_id, keyword]) - - conn.commit() - conn.close() - - -def pkgbase_has_write_access(pkgbase, user): - conn = aurweb.db.Connection() - - cur = conn.execute("SELECT COUNT(*) FROM PackageBases " + - "LEFT JOIN PackageComaintainers " + - "ON PackageComaintainers.PackageBaseID = PackageBases.ID " + - "INNER JOIN Users " + - "ON Users.ID = PackageBases.MaintainerUID " + - "OR PackageBases.MaintainerUID IS NULL " + - "OR Users.ID = PackageComaintainers.UsersID " + - "WHERE Name = ? AND Username = ?", [pkgbase, user]) - return cur.fetchone()[0] > 0 - - -def pkgbase_has_full_access(pkgbase, user): - conn = aurweb.db.Connection() - - cur = conn.execute("SELECT COUNT(*) FROM PackageBases " + - "INNER JOIN Users " + - "ON Users.ID = PackageBases.MaintainerUID " + - "WHERE Name = ? AND Username = ?", [pkgbase, user]) - return cur.fetchone()[0] > 0 - - -def die(msg): - sys.stderr.write("{:s}\n".format(msg)) - exit(1) - - -def die_with_help(msg): - die(msg + "\nTry `{:s} help` for a list of commands.".format(ssh_cmdline)) - - -def warn(msg): - sys.stderr.write("warning: {:s}\n".format(msg)) - - -def usage(cmds): - sys.stderr.write("Commands:\n") - colwidth = max([len(cmd) for cmd in cmds.keys()]) + 4 - for key in sorted(cmds): - sys.stderr.write(" " + key.ljust(colwidth) + cmds[key] + "\n") - exit(0) - - -def main(): - user = os.environ.get('AUR_USER') - privileged = (os.environ.get('AUR_PRIVILEGED', '0') == '1') - ssh_cmd = os.environ.get('SSH_ORIGINAL_COMMAND') - ssh_client = os.environ.get('SSH_CLIENT') - - if not ssh_cmd: - die_with_help("Interactive shell is disabled.") - cmdargv = shlex.split(ssh_cmd) - action = cmdargv[0] - remote_addr = ssh_client.split(' ')[0] if ssh_client else None - - if enable_maintenance: - if remote_addr not in maintenance_exc: - die("The AUR is down due to maintenance. We will be back soon.") - - if action == 'git' and cmdargv[1] in ('upload-pack', 'receive-pack'): - action = action + '-' + cmdargv[1] - del cmdargv[1] - - if action == 'git-upload-pack' or action == 'git-receive-pack': - if len(cmdargv) < 2: - die_with_help("{:s}: missing path".format(action)) - - path = cmdargv[1].rstrip('/') - if not path.startswith('/'): - path = '/' + path - if not path.endswith('.git'): - path = path + '.git' - pkgbase = path[1:-4] - if not re.match(repo_regex, pkgbase): - die('{:s}: invalid repository name: {:s}'.format(action, pkgbase)) - - if action == 'git-receive-pack' and pkgbase_exists(pkgbase): - if not privileged and not pkgbase_has_write_access(pkgbase, user): - die('{:s}: permission denied: {:s}'.format(action, user)) - - os.environ["AUR_USER"] = user - os.environ["AUR_PKGBASE"] = pkgbase - os.environ["GIT_NAMESPACE"] = pkgbase - cmd = action + " '" + repo_path + "'" - os.execl(git_shell_cmd, git_shell_cmd, '-c', cmd) - elif action == 'set-keywords': - if len(cmdargv) < 2: - die_with_help("{:s}: missing repository name".format(action)) - pkgbase_set_keywords(cmdargv[1], cmdargv[2:]) - elif action == 'list-repos': - if len(cmdargv) > 1: - die_with_help("{:s}: too many arguments".format(action)) - list_repos(user) - elif action == 'setup-repo': - if len(cmdargv) < 2: - die_with_help("{:s}: missing repository name".format(action)) - if len(cmdargv) > 2: - die_with_help("{:s}: too many arguments".format(action)) - warn('{:s} is deprecated. ' - 'Use `git push` to create new repositories.'.format(action)) - create_pkgbase(cmdargv[1], user) - elif action == 'restore': - if len(cmdargv) < 2: - die_with_help("{:s}: missing repository name".format(action)) - if len(cmdargv) > 2: - die_with_help("{:s}: too many arguments".format(action)) - - pkgbase = cmdargv[1] - if not re.match(repo_regex, pkgbase): - die('{:s}: invalid repository name: {:s}'.format(action, pkgbase)) - - if pkgbase_exists(pkgbase): - die('{:s}: package base exists: {:s}'.format(action, pkgbase)) - create_pkgbase(pkgbase, user) - - os.environ["AUR_USER"] = user - os.environ["AUR_PKGBASE"] = pkgbase - os.execl(git_update_cmd, git_update_cmd, 'restore') - elif action == 'adopt': - if len(cmdargv) < 2: - die_with_help("{:s}: missing repository name".format(action)) - if len(cmdargv) > 2: - die_with_help("{:s}: too many arguments".format(action)) - - pkgbase = cmdargv[1] - pkgbase_adopt(pkgbase, user, privileged) - elif action == 'disown': - if len(cmdargv) < 2: - die_with_help("{:s}: missing repository name".format(action)) - if len(cmdargv) > 2: - die_with_help("{:s}: too many arguments".format(action)) - - pkgbase = cmdargv[1] - pkgbase_disown(pkgbase, user, privileged) - elif action == 'set-comaintainers': - if len(cmdargv) < 2: - die_with_help("{:s}: missing repository name".format(action)) - - pkgbase = cmdargv[1] - userlist = cmdargv[2:] - pkgbase_set_comaintainers(pkgbase, userlist, user, privileged) - elif action == 'help': - cmds = { - "adopt ": "Adopt a package base.", - "disown ": "Disown a package base.", - "help": "Show this help message and exit.", - "list-repos": "List all your repositories.", - "restore ": "Restore a deleted package base.", - "set-comaintainers [...]": "Set package base co-maintainers.", - "set-keywords [...]": "Change package base keywords.", - "setup-repo ": "Create a repository (deprecated).", - "git-receive-pack": "Internal command used with Git.", - "git-upload-pack": "Internal command used with Git.", - } - usage(cmds) - else: - die_with_help("invalid command: {:s}".format(action)) - - -if __name__ == '__main__': - main() diff --git a/git-interface/git-update.py b/git-interface/git-update.py deleted file mode 100755 index 7337341..0000000 --- a/git-interface/git-update.py +++ /dev/null @@ -1,419 +0,0 @@ -#!/usr/bin/python3 - -import os -import pygit2 -import re -import subprocess -import sys -import time - -import srcinfo.parse -import srcinfo.utils - -import aurweb.config -import aurweb.db - -notify_cmd = aurweb.config.get('notifications', 'notify-cmd') - -repo_path = aurweb.config.get('serve', 'repo-path') -repo_regex = aurweb.config.get('serve', 'repo-regex') - -max_blob_size = aurweb.config.getint('update', 'max-blob-size') - - -def size_humanize(num): - for unit in ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB']: - if abs(num) < 2048.0: - if isinstance(num, int): - return "{}{}".format(num, unit) - else: - return "{:.2f}{}".format(num, unit) - num /= 1024.0 - return "{:.2f}{}".format(num, 'YiB') - - -def extract_arch_fields(pkginfo, field): - values = [] - - if field in pkginfo: - for val in pkginfo[field]: - values.append({"value": val, "arch": None}) - - for arch in ['i686', 'x86_64']: - if field + '_' + arch in pkginfo: - for val in pkginfo[field + '_' + arch]: - values.append({"value": val, "arch": arch}) - - return values - - -def parse_dep(depstring): - dep, _, desc = depstring.partition(': ') - depname = re.sub(r'(<|=|>).*', '', dep) - depcond = dep[len(depname):] - - if (desc): - return (depname + ': ' + desc, depcond) - else: - return (depname, depcond) - - -def create_pkgbase(conn, pkgbase, user): - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - userid = cur.fetchone()[0] - - now = int(time.time()) - cur = conn.execute("INSERT INTO PackageBases (Name, SubmittedTS, " + - "ModifiedTS, SubmitterUID, MaintainerUID) VALUES " + - "(?, ?, ?, ?, ?)", [pkgbase, now, now, userid, userid]) - pkgbase_id = cur.lastrowid - - cur = conn.execute("INSERT INTO PackageNotifications " + - "(PackageBaseID, UserID) VALUES (?, ?)", - [pkgbase_id, userid]) - - conn.commit() - - return pkgbase_id - - -def save_metadata(metadata, conn, user): - # Obtain package base ID and previous maintainer. - pkgbase = metadata['pkgbase'] - cur = conn.execute("SELECT ID, MaintainerUID FROM PackageBases " - "WHERE Name = ?", [pkgbase]) - (pkgbase_id, maintainer_uid) = cur.fetchone() - was_orphan = not maintainer_uid - - # Obtain the user ID of the new maintainer. - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - user_id = int(cur.fetchone()[0]) - - # Update package base details and delete current packages. - now = int(time.time()) - conn.execute("UPDATE PackageBases SET ModifiedTS = ?, " + - "PackagerUID = ?, OutOfDateTS = NULL WHERE ID = ?", - [now, user_id, pkgbase_id]) - conn.execute("UPDATE PackageBases SET MaintainerUID = ? " + - "WHERE ID = ? AND MaintainerUID IS NULL", - [user_id, pkgbase_id]) - for table in ('Sources', 'Depends', 'Relations', 'Licenses', 'Groups'): - conn.execute("DELETE FROM Package" + table + " WHERE EXISTS (" + - "SELECT * FROM Packages " + - "WHERE Packages.PackageBaseID = ? AND " + - "Package" + table + ".PackageID = Packages.ID)", - [pkgbase_id]) - conn.execute("DELETE FROM Packages WHERE PackageBaseID = ?", [pkgbase_id]) - - for pkgname in srcinfo.utils.get_package_names(metadata): - pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) - - if 'epoch' in pkginfo and int(pkginfo['epoch']) > 0: - ver = '{:d}:{:s}-{:s}'.format(int(pkginfo['epoch']), - pkginfo['pkgver'], - pkginfo['pkgrel']) - else: - ver = '{:s}-{:s}'.format(pkginfo['pkgver'], pkginfo['pkgrel']) - - for field in ('pkgdesc', 'url'): - if field not in pkginfo: - pkginfo[field] = None - - # Create a new package. - cur = conn.execute("INSERT INTO Packages (PackageBaseID, Name, " + - "Version, Description, URL) " + - "VALUES (?, ?, ?, ?, ?)", - [pkgbase_id, pkginfo['pkgname'], ver, - pkginfo['pkgdesc'], pkginfo['url']]) - conn.commit() - pkgid = cur.lastrowid - - # Add package sources. - for source_info in extract_arch_fields(pkginfo, 'source'): - conn.execute("INSERT INTO PackageSources (PackageID, Source, " + - "SourceArch) VALUES (?, ?, ?)", - [pkgid, source_info['value'], source_info['arch']]) - - # Add package dependencies. - for deptype in ('depends', 'makedepends', - 'checkdepends', 'optdepends'): - cur = conn.execute("SELECT ID FROM DependencyTypes WHERE Name = ?", - [deptype]) - deptypeid = cur.fetchone()[0] - for dep_info in extract_arch_fields(pkginfo, deptype): - depname, depcond = parse_dep(dep_info['value']) - deparch = dep_info['arch'] - conn.execute("INSERT INTO PackageDepends (PackageID, " + - "DepTypeID, DepName, DepCondition, DepArch) " + - "VALUES (?, ?, ?, ?, ?)", - [pkgid, deptypeid, depname, depcond, deparch]) - - # Add package relations (conflicts, provides, replaces). - for reltype in ('conflicts', 'provides', 'replaces'): - cur = conn.execute("SELECT ID FROM RelationTypes WHERE Name = ?", - [reltype]) - reltypeid = cur.fetchone()[0] - for rel_info in extract_arch_fields(pkginfo, reltype): - relname, relcond = parse_dep(rel_info['value']) - relarch = rel_info['arch'] - conn.execute("INSERT INTO PackageRelations (PackageID, " + - "RelTypeID, RelName, RelCondition, RelArch) " + - "VALUES (?, ?, ?, ?, ?)", - [pkgid, reltypeid, relname, relcond, relarch]) - - # Add package licenses. - if 'license' in pkginfo: - for license in pkginfo['license']: - cur = conn.execute("SELECT ID FROM Licenses WHERE Name = ?", - [license]) - row = cur.fetchone() - if row: - licenseid = row[0] - else: - cur = conn.execute("INSERT INTO Licenses (Name) " + - "VALUES (?)", [license]) - conn.commit() - licenseid = cur.lastrowid - conn.execute("INSERT INTO PackageLicenses (PackageID, " + - "LicenseID) VALUES (?, ?)", - [pkgid, licenseid]) - - # Add package groups. - if 'groups' in pkginfo: - for group in pkginfo['groups']: - cur = conn.execute("SELECT ID FROM Groups WHERE Name = ?", - [group]) - row = cur.fetchone() - if row: - groupid = row[0] - else: - cur = conn.execute("INSERT INTO Groups (Name) VALUES (?)", - [group]) - conn.commit() - groupid = cur.lastrowid - conn.execute("INSERT INTO PackageGroups (PackageID, " - "GroupID) VALUES (?, ?)", [pkgid, groupid]) - - # Add user to notification list on adoption. - if was_orphan: - cur = conn.execute("SELECT COUNT(*) FROM PackageNotifications WHERE " + - "PackageBaseID = ? AND UserID = ?", - [pkgbase_id, user_id]) - if cur.fetchone()[0] == 0: - conn.execute("INSERT INTO PackageNotifications " + - "(PackageBaseID, UserID) VALUES (?, ?)", - [pkgbase_id, user_id]) - - conn.commit() - - -def update_notify(conn, user, pkgbase_id): - # Obtain the user ID of the new maintainer. - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - user_id = int(cur.fetchone()[0]) - - # Execute the notification script. - subprocess.Popen((notify_cmd, 'update', str(user_id), str(pkgbase_id))) - - -def die(msg): - sys.stderr.write("error: {:s}\n".format(msg)) - exit(1) - - -def warn(msg): - sys.stderr.write("warning: {:s}\n".format(msg)) - - -def die_commit(msg, commit): - sys.stderr.write("error: The following error " + - "occurred when parsing commit\n") - sys.stderr.write("error: {:s}:\n".format(commit)) - sys.stderr.write("error: {:s}\n".format(msg)) - exit(1) - - -def main(): - repo = pygit2.Repository(repo_path) - - user = os.environ.get("AUR_USER") - pkgbase = os.environ.get("AUR_PKGBASE") - privileged = (os.environ.get("AUR_PRIVILEGED", '0') == '1') - warn_or_die = warn if privileged else die - - if len(sys.argv) == 2 and sys.argv[1] == "restore": - if 'refs/heads/' + pkgbase not in repo.listall_references(): - die('{:s}: repository not found: {:s}'.format(sys.argv[1], - pkgbase)) - refname = "refs/heads/master" - branchref = 'refs/heads/' + pkgbase - sha1_old = sha1_new = repo.lookup_reference(branchref).target - elif len(sys.argv) == 4: - refname, sha1_old, sha1_new = sys.argv[1:4] - else: - die("invalid arguments") - - if refname != "refs/heads/master": - die("pushing to a branch other than master is restricted") - - conn = aurweb.db.Connection() - - # Detect and deny non-fast-forwards. - if sha1_old != "0" * 40 and not privileged: - walker = repo.walk(sha1_old, pygit2.GIT_SORT_TOPOLOGICAL) - walker.hide(sha1_new) - if next(walker, None) is not None: - die("denying non-fast-forward (you should pull first)") - - # Prepare the walker that validates new commits. - walker = repo.walk(sha1_new, pygit2.GIT_SORT_TOPOLOGICAL) - if sha1_old != "0" * 40: - walker.hide(sha1_old) - - # Validate all new commits. - for commit in walker: - for fname in ('.SRCINFO', 'PKGBUILD'): - if fname not in commit.tree: - die_commit("missing {:s}".format(fname), str(commit.id)) - - for treeobj in commit.tree: - blob = repo[treeobj.id] - - if isinstance(blob, pygit2.Tree): - die_commit("the repository must not contain subdirectories", - str(commit.id)) - - if not isinstance(blob, pygit2.Blob): - die_commit("not a blob object: {:s}".format(treeobj), - str(commit.id)) - - if blob.size > max_blob_size: - die_commit("maximum blob size ({:s}) exceeded".format( - size_humanize(max_blob_size)), str(commit.id)) - - metadata_raw = repo[commit.tree['.SRCINFO'].id].data.decode() - (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw) - if errors: - sys.stderr.write("error: The following errors occurred " - "when parsing .SRCINFO in commit\n") - sys.stderr.write("error: {:s}:\n".format(str(commit.id))) - for error in errors: - for err in error['error']: - sys.stderr.write("error: line {:d}: {:s}\n".format( - error['line'], err)) - exit(1) - - metadata_pkgbase = metadata['pkgbase'] - if not re.match(repo_regex, metadata_pkgbase): - die_commit('invalid pkgbase: {:s}'.format(metadata_pkgbase), - str(commit.id)) - - for pkgname in set(metadata['packages'].keys()): - pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) - - for field in ('pkgver', 'pkgrel', 'pkgname'): - if field not in pkginfo: - die_commit('missing mandatory field: {:s}'.format(field), - str(commit.id)) - - if 'epoch' in pkginfo and not pkginfo['epoch'].isdigit(): - die_commit('invalid epoch: {:s}'.format(pkginfo['epoch']), - str(commit.id)) - - if not re.match(r'[a-z0-9][a-z0-9\.+_-]*$', pkginfo['pkgname']): - die_commit('invalid package name: {:s}'.format( - pkginfo['pkgname']), str(commit.id)) - - for field in ('pkgname', 'pkgdesc', 'url'): - if field in pkginfo and len(pkginfo[field]) > 255: - die_commit('{:s} field too long: {:s}'.format(field, - pkginfo[field]), str(commit.id)) - - for field in ('install', 'changelog'): - if field in pkginfo and not pkginfo[field] in commit.tree: - die_commit('missing {:s} file: {:s}'.format(field, - pkginfo[field]), str(commit.id)) - - for field in extract_arch_fields(pkginfo, 'source'): - fname = field['value'] - if "://" in fname or "lp:" in fname: - continue - if fname not in commit.tree: - die_commit('missing source file: {:s}'.format(fname), - str(commit.id)) - - # Display a warning if .SRCINFO is unchanged. - if sha1_old not in ("0000000000000000000000000000000000000000", sha1_new): - srcinfo_id_old = repo[sha1_old].tree['.SRCINFO'].id - srcinfo_id_new = repo[sha1_new].tree['.SRCINFO'].id - if srcinfo_id_old == srcinfo_id_new: - warn(".SRCINFO unchanged. " - "The package database will not be updated!") - - # Read .SRCINFO from the HEAD commit. - metadata_raw = repo[repo[sha1_new].tree['.SRCINFO'].id].data.decode() - (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw) - - # Ensure that the package base name matches the repository name. - metadata_pkgbase = metadata['pkgbase'] - if metadata_pkgbase != pkgbase: - die('invalid pkgbase: {:s}, expected {:s}'.format(metadata_pkgbase, - pkgbase)) - - # Ensure that packages are neither blacklisted nor overwritten. - pkgbase = metadata['pkgbase'] - cur = conn.execute("SELECT ID FROM PackageBases WHERE Name = ?", [pkgbase]) - row = cur.fetchone() - pkgbase_id = row[0] if row else 0 - - cur = conn.execute("SELECT Name FROM PackageBlacklist") - blacklist = [row[0] for row in cur.fetchall()] - - cur = conn.execute("SELECT Name, Repo FROM OfficialProviders") - providers = dict(cur.fetchall()) - - for pkgname in srcinfo.utils.get_package_names(metadata): - pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) - pkgname = pkginfo['pkgname'] - - if pkgname in blacklist: - warn_or_die('package is blacklisted: {:s}'.format(pkgname)) - if pkgname in providers: - warn_or_die('package already provided by [{:s}]: {:s}'.format( - providers[pkgname], pkgname)) - - cur = conn.execute("SELECT COUNT(*) FROM Packages WHERE Name = ? " + - "AND PackageBaseID <> ?", [pkgname, pkgbase_id]) - if cur.fetchone()[0] > 0: - die('cannot overwrite package: {:s}'.format(pkgname)) - - # Create a new package base if it does not exist yet. - if pkgbase_id == 0: - pkgbase_id = create_pkgbase(conn, pkgbase, user) - - # Store package base details in the database. - save_metadata(metadata, conn, user) - - # Create (or update) a branch with the name of the package base for better - # accessibility. - branchref = 'refs/heads/' + pkgbase - repo.create_reference(branchref, sha1_new, True) - - # Work around a Git bug: The HEAD ref is not updated when using - # gitnamespaces. This can be removed once the bug fix is included in Git - # mainline. See - # http://git.661346.n2.nabble.com/PATCH-receive-pack-Create-a-HEAD-ref-for-ref-namespace-td7632149.html - # for details. - headref = 'refs/namespaces/' + pkgbase + '/HEAD' - repo.create_reference(headref, sha1_new, True) - - # Send package update notifications. - update_notify(conn, user, pkgbase_id) - - # Close the database. - cur.close() - conn.close() - - -if __name__ == '__main__': - main() diff --git a/setup.py b/setup.py index 48eb176..b64e71c 100644 --- a/setup.py +++ b/setup.py @@ -17,4 +17,11 @@ setup( name="aurweb", version=version, packages=find_packages(), + entry_points={ + 'console_scripts': [ + 'aurweb-git-auth = aurweb.git.auth:main', + 'aurweb-git-serve = aurweb.git.serve:main', + 'aurweb-git-update = aurweb.git.update:main', + ], + }, ) diff --git a/test/setup.sh b/test/setup.sh index dc9cff2..d02d298 100644 --- a/test/setup.sh +++ b/test/setup.sh @@ -8,9 +8,9 @@ PYTHONPATH="$TOPLEVEL" export PYTHONPATH # Configure paths to the Git interface scripts. -GIT_AUTH="$TOPLEVEL/git-interface/git-auth.py" -GIT_SERVE="$TOPLEVEL/git-interface/git-serve.py" -GIT_UPDATE="$TOPLEVEL/git-interface/git-update.py" +GIT_AUTH="$TOPLEVEL/aurweb/git/auth.py" +GIT_SERVE="$TOPLEVEL/aurweb/git/serve.py" +GIT_UPDATE="$TOPLEVEL/aurweb/git/update.py" MKPKGLISTS="$TOPLEVEL/scripts/mkpkglists.py" TUVOTEREMINDER="$TOPLEVEL/scripts/tuvotereminder.py" PKGMAINT="$TOPLEVEL/scripts/pkgmaint.py" @@ -38,7 +38,7 @@ reply-to = noreply@aur.archlinux.org [auth] valid-keytypes = ssh-rsa ssh-dss ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 ssh-ed25519 username-regex = [a-zA-Z0-9]+[.\-_]?[a-zA-Z0-9]+$ -git-serve-cmd = /srv/http/aurweb/git-interface/git-serve.py +git-serve-cmd = $GIT_SERVE ssh-options = restrict [serve] -- cgit v1.2.3-54-g00ecf From 1492444ecbe68e4498a6f7ae0258c39ebbd47138 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Tue, 11 Oct 2016 08:09:21 +0200 Subject: Make URL columns 8000 characters wide According to RFC 7230, URLs can be up too 8000 characters long. Resize all URL fields accordingly. Also, add a test to verify that URLs with more than 8000 characters are rejected by the update hook. Reported-by: Andreas Linz Signed-off-by: Lukas Fleischer --- aurweb/git/update.py | 5 +++-- schema/aur-schema.sql | 4 ++-- test/t1300-git-update.sh | 16 ++++++++++++++++ upgrading/4.4.0.txt | 12 ++++++++++++ 4 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 upgrading/4.4.0.txt (limited to 'aurweb') diff --git a/aurweb/git/update.py b/aurweb/git/update.py index 7337341..af2dfed 100755 --- a/aurweb/git/update.py +++ b/aurweb/git/update.py @@ -324,8 +324,9 @@ def main(): die_commit('invalid package name: {:s}'.format( pkginfo['pkgname']), str(commit.id)) - for field in ('pkgname', 'pkgdesc', 'url'): - if field in pkginfo and len(pkginfo[field]) > 255: + max_len = {'pkgname': 255, 'pkgdesc': 255, 'url': 8000} + for field in max_len.keys(): + if field in pkginfo and len(pkginfo[field]) > max_len[field]: die_commit('{:s} field too long: {:s}'.format(field, pkginfo[field]), str(commit.id)) diff --git a/schema/aur-schema.sql b/schema/aur-schema.sql index 030370b..30209bd 100644 --- a/schema/aur-schema.sql +++ b/schema/aur-schema.sql @@ -119,7 +119,7 @@ CREATE TABLE Packages ( Name VARCHAR(255) NOT NULL, Version VARCHAR(255) NOT NULL DEFAULT '', Description VARCHAR(255) NULL DEFAULT NULL, - URL VARCHAR(255) NULL DEFAULT NULL, + URL VARCHAR(8000) NULL DEFAULT NULL, PRIMARY KEY (ID), UNIQUE (Name), FOREIGN KEY (PackageBaseID) REFERENCES PackageBases(ID) ON DELETE CASCADE @@ -227,7 +227,7 @@ CREATE INDEX RelationsRelName ON PackageRelations (RelName); -- CREATE TABLE PackageSources ( PackageID INTEGER UNSIGNED NOT NULL, - Source VARCHAR(255) NOT NULL DEFAULT "/dev/null", + Source VARCHAR(8000) NOT NULL DEFAULT "/dev/null", SourceArch VARCHAR(255) NULL DEFAULT NULL, FOREIGN KEY (PackageID) REFERENCES Packages(ID) ON DELETE CASCADE ) ENGINE = InnoDB; diff --git a/test/t1300-git-update.sh b/test/t1300-git-update.sh index b642089..abab7ea 100755 --- a/test/t1300-git-update.sh +++ b/test/t1300-git-update.sh @@ -309,6 +309,22 @@ test_expect_success 'Pushing .SRCINFO with invalid epoch.' ' grep -q "^error: invalid epoch: !$" actual ' +test_expect_success 'Pushing .SRCINFO with too long URL.' ' + old=$(git -C aur.git rev-parse HEAD) && + url="http://$(printf "%7993s" x | sed "s/ /x/g")/" && + test_when_finished "git -C aur.git reset --hard $old" && + ( + cd aur.git && + sed "s#.*url.*#\\0\\nurl = $url#" .SRCINFO >.SRCINFO.new + mv .SRCINFO.new .SRCINFO + git commit -q -am "Change URL" + ) && + new=$(git -C aur.git rev-parse HEAD) && + AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \ + test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 && + grep -q "^error: url field too long: $url\$" actual +' + test_expect_success 'Missing install file.' ' old=$(git -C aur.git rev-parse HEAD) && test_when_finished "git -C aur.git reset --hard $old" && diff --git a/upgrading/4.4.0.txt b/upgrading/4.4.0.txt new file mode 100644 index 0000000..1cc55b3 --- /dev/null +++ b/upgrading/4.4.0.txt @@ -0,0 +1,12 @@ +1. Resize the URL column of the Packages table: + +---- +ALTER TABLE Packages MODIFY URL VARCHAR(8000) NULL DEFAULT NULL; +---- + +2. Resize the Source column of the PackageSources table: + +---- +ALTER TABLE PackageSources + MODIFY Source VARCHAR(8000) NOT NULL DEFAULT "/dev/null"; +---- -- cgit v1.2.3-54-g00ecf From 29a5f94dab27cc64da6262b2f37bd856641ed292 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Tue, 11 Oct 2016 08:22:03 +0200 Subject: git-update: Catch long source URLs Bail out early if the source array contains an entry with more than 8000 characters. Signed-off-by: Lukas Fleischer --- aurweb/git/update.py | 3 +++ test/t1300-git-update.sh | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) (limited to 'aurweb') diff --git a/aurweb/git/update.py b/aurweb/git/update.py index af2dfed..3b84eb5 100755 --- a/aurweb/git/update.py +++ b/aurweb/git/update.py @@ -337,6 +337,9 @@ def main(): for field in extract_arch_fields(pkginfo, 'source'): fname = field['value'] + if len(fname) > 8000: + die_commit('source entry too long: {:s}'.format(fname), + str(commit.id)) if "://" in fname or "lp:" in fname: continue if fname not in commit.tree: diff --git a/test/t1300-git-update.sh b/test/t1300-git-update.sh index abab7ea..a65ca3a 100755 --- a/test/t1300-git-update.sh +++ b/test/t1300-git-update.sh @@ -370,6 +370,22 @@ test_expect_success 'Missing source file.' ' grep -q "^error: missing source file: file$" actual ' +test_expect_success 'Pushing .SRCINFO with too long source URL.' ' + old=$(git -C aur.git rev-parse HEAD) && + url="http://$(printf "%7993s" x | sed "s/ /x/g")/" && + test_when_finished "git -C aur.git reset --hard $old" && + ( + cd aur.git && + sed "s#.*depends.*#\\0\\nsource = $url#" .SRCINFO >.SRCINFO.new + mv .SRCINFO.new .SRCINFO + git commit -q -am "Add huge source URL" + ) && + new=$(git -C aur.git rev-parse HEAD) && + AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \ + test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 && + grep -q "^error: source entry too long: $url\$" actual +' + test_expect_success 'Pushing a blacklisted package.' ' old=$(git -C aur.git rev-parse HEAD) && test_when_finished "git -C aur.git reset --hard $old" && -- cgit v1.2.3-54-g00ecf From fc6dc44295bed8c5d9c8014c2eab2d6bf33d2541 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Tue, 11 Oct 2016 07:44:21 +0200 Subject: git-serve: Close orphan requests upon disown When disowning a package base via the SSH interface, auto-accept all pending orphan requests for the affected package. Also, add a test case that checks whether (only) orphan requests belonging to disowned packages are closed correctly. Signed-off-by: Lukas Fleischer --- aurweb/git/serve.py | 44 +++++++++++++++++++++++++++++++++++++++++++- test/t1200-git-serve.sh | 23 +++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) (limited to 'aurweb') diff --git a/aurweb/git/serve.py b/aurweb/git/serve.py index ebfef94..476aea8 100755 --- a/aurweb/git/serve.py +++ b/aurweb/git/serve.py @@ -183,6 +183,44 @@ def pkgbase_set_comaintainers(pkgbase, userlist, user, privileged): conn.close() +def pkgreq_by_pkgbase(pkgbase_id, reqtype): + conn = aurweb.db.Connection() + + cur = conn.execute("SELECT PackageRequests.ID FROM PackageRequests " + + "INNER JOIN RequestTypes ON " + + "RequestTypes.ID = PackageRequests.ReqTypeID " + + "WHERE PackageRequests.Status = 0 " + + "AND PackageRequests.PackageBaseID = ?" + + "AND RequestTypes.Name = ?", [pkgbase_id, reqtype]) + + return [row[0] for row in cur.fetchall()] + + +def pkgreq_close(reqid, reason, comments, autoclose=False): + statusmap = {'accepted': 2, 'rejected': 3} + if reason not in statusmap: + die('{:s}: invalid reason: {:s}'.format(action, reason)) + status = statusmap[reason] + + conn = aurweb.db.Connection() + + if autoclose: + userid = 0 + else: + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) + userid = cur.fetchone()[0] + if userid == 0: + die('{:s}: unknown user: {:s}'.format(action, user)) + + conn.execute("UPDATE PackageRequests SET Status = ?, ClosureComment = ? " + + "WHERE ID = ?", [status, comments, reqid]) + conn.commit() + conn.close() + + subprocess.Popen((notify_cmd, 'request-close', str(userid), str(reqid), + reason)).wait() + + def pkgbase_disown(pkgbase, user, privileged): pkgbase_id = pkgbase_from_name(pkgbase) if not pkgbase_id: @@ -193,7 +231,11 @@ def pkgbase_disown(pkgbase, user, privileged): die('{:s}: permission denied: {:s}'.format(action, user)) # TODO: Support disowning package bases via package request. - # TODO: Scan through pending orphan requests and close them. + + # Scan through pending orphan requests and close them. + comment = 'The user {:s} disowned the package.'.format(user) + for reqid in pkgreq_by_pkgbase(pkgbase_id, 'orphan'): + pkgreq_close(reqid, 'accepted', comment, True) comaintainers = [] new_maintainer_userid = None diff --git a/test/t1200-git-serve.sh b/test/t1200-git-serve.sh index 2f1926e..5054ce3 100755 --- a/test/t1200-git-serve.sh +++ b/test/t1200-git-serve.sh @@ -317,4 +317,27 @@ test_expect_success "Force-disown a package base and check (co-)maintainer list. test_cmp expected actual ' +test_expect_success "Check whether package requests are closed when disowning." ' + SSH_ORIGINAL_COMMAND="adopt foobar" AUR_USER=user AUR_PRIVILEGED=0 \ + "$GIT_SERVE" 2>&1 && + cat <<-EOD | sqlite3 aur.db && + INSERT INTO PackageRequests (ID, ReqTypeID, PackageBaseID, PackageBaseName, UsersID) VALUES (1, 2, 3, "foobar", 4); + INSERT INTO PackageRequests (ID, ReqTypeID, PackageBaseID, PackageBaseName, UsersID) VALUES (2, 3, 3, "foobar", 5); + INSERT INTO PackageRequests (ID, ReqTypeID, PackageBaseID, PackageBaseName, UsersID) VALUES (3, 2, 2, "foobar2", 6); + EOD + >sendmail.out && + SSH_ORIGINAL_COMMAND="disown foobar" AUR_USER=user AUR_PRIVILEGED=0 \ + "$GIT_SERVE" 2>&1 && + cat <<-EOD >expected && + Subject: [PRQ#1] Request Accepted + EOD + grep "^Subject.*PRQ" sendmail.out >sendmail.parts && + test_cmp sendmail.parts expected && + cat <<-EOD >expected && + 1|2|3|foobar||4||The user user disowned the package.|0|2 + EOD + echo "SELECT * FROM PackageRequests WHERE Status = 2;" | sqlite3 aur.db >actual && + test_cmp actual expected +' + test_done -- cgit v1.2.3-54-g00ecf From 9581069f49fbc305be3a37c2b46db0a24ede0564 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Mon, 17 Oct 2016 15:12:00 +0200 Subject: aurweb/git: Add missing __init__.py file Signed-off-by: Lukas Fleischer --- aurweb/git/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 aurweb/git/__init__.py (limited to 'aurweb') diff --git a/aurweb/git/__init__.py b/aurweb/git/__init__.py new file mode 100644 index 0000000..e69de29 -- cgit v1.2.3-54-g00ecf From 85866796a40923708f6b868c32ddc2f2f4417d1d Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Mon, 17 Oct 2016 15:01:45 +0200 Subject: Move configuration to /etc/aurweb/config Since d4fe77a (Reorganize Git interface scripts, 2016-10-08), the key components of the aurweb SSH interface are installed system-wide. Update the default configuration path to point to a central location. Signed-off-by: Lukas Fleischer --- .gitignore | 1 - INSTALL | 4 ++-- aurweb/config.py | 3 +-- upgrading/4.4.1.txt | 3 +++ web/lib/confparser.inc.php | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 upgrading/4.4.1.txt (limited to 'aurweb') diff --git a/.gitignore b/.gitignore index 2cb5bdc..f0e462d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -conf/config dummy-data.sql* po/*.mo po/*.po~ diff --git a/INSTALL b/INSTALL index 395915a..95cac4c 100644 --- a/INSTALL +++ b/INSTALL @@ -30,8 +30,8 @@ Setup on Arch Linux } } -3) Copy conf/config.proto to conf/config and adjust the configuration (pay - attention to disable_http_login, enable_maintenance and aur_location). +3) Copy conf/config.proto to /etc/aurweb/config and adjust the configuration + (pay attention to disable_http_login, enable_maintenance and aur_location). 4) Create a new MySQL database and a user and import the AUR SQL schema: diff --git a/aurweb/config.py b/aurweb/config.py index aac188b..a52d942 100644 --- a/aurweb/config.py +++ b/aurweb/config.py @@ -12,8 +12,7 @@ def _get_parser(): if 'AUR_CONFIG' in os.environ: path = os.environ.get('AUR_CONFIG') else: - relpath = "/../conf/config" - path = os.path.dirname(os.path.realpath(__file__)) + relpath + path = "/etc/aurweb/config" _parser.read(path) return _parser diff --git a/upgrading/4.4.1.txt b/upgrading/4.4.1.txt new file mode 100644 index 0000000..b06696e --- /dev/null +++ b/upgrading/4.4.1.txt @@ -0,0 +1,3 @@ +1. The default configuration file search path now points to /etc/aurweb/config. + Make sure you copy your aurweb configuration to the new location before + upgrading. diff --git a/web/lib/confparser.inc.php b/web/lib/confparser.inc.php index 6368b86..789300e 100644 --- a/web/lib/confparser.inc.php +++ b/web/lib/confparser.inc.php @@ -4,7 +4,7 @@ function config_load() { global $AUR_CONFIG; if (!isset($AUR_CONFIG)) { - $AUR_CONFIG = parse_ini_file("../../conf/config", true, INI_SCANNER_RAW); + $AUR_CONFIG = parse_ini_file("/etc/aurweb/config", true, INI_SCANNER_RAW); } } -- cgit v1.2.3-54-g00ecf From 37188603b52a3dac23df229ada82c7da0c3d9c00 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Mon, 17 Oct 2016 15:20:29 +0200 Subject: Make maintenance scripts installable Add wrappers for the maintenance scripts to the setuptools configuration. Signed-off-by: Lukas Fleischer --- aurweb/scripts/__init__.py | 0 aurweb/scripts/aurblup.py | 55 +++++ aurweb/scripts/mkpkglists.py | 38 ++++ aurweb/scripts/notify.py | 455 +++++++++++++++++++++++++++++++++++++++ aurweb/scripts/pkgmaint.py | 20 ++ aurweb/scripts/popupdate.py | 26 +++ aurweb/scripts/tuvotereminder.py | 28 +++ scripts/__init__.py | 0 scripts/aurblup.py | 55 ----- scripts/mkpkglists.py | 38 ---- scripts/notify.py | 455 --------------------------------------- scripts/pkgmaint.py | 20 -- scripts/popupdate.py | 26 --- scripts/tuvotereminder.py | 28 --- setup.py | 6 + upgrading/4.4.1.txt | 3 + 16 files changed, 631 insertions(+), 622 deletions(-) create mode 100644 aurweb/scripts/__init__.py create mode 100755 aurweb/scripts/aurblup.py create mode 100755 aurweb/scripts/mkpkglists.py create mode 100755 aurweb/scripts/notify.py create mode 100755 aurweb/scripts/pkgmaint.py create mode 100755 aurweb/scripts/popupdate.py create mode 100755 aurweb/scripts/tuvotereminder.py delete mode 100644 scripts/__init__.py delete mode 100755 scripts/aurblup.py delete mode 100755 scripts/mkpkglists.py delete mode 100755 scripts/notify.py delete mode 100755 scripts/pkgmaint.py delete mode 100755 scripts/popupdate.py delete mode 100755 scripts/tuvotereminder.py (limited to 'aurweb') diff --git a/aurweb/scripts/__init__.py b/aurweb/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aurweb/scripts/aurblup.py b/aurweb/scripts/aurblup.py new file mode 100755 index 0000000..1b6de2f --- /dev/null +++ b/aurweb/scripts/aurblup.py @@ -0,0 +1,55 @@ +#!/usr/bin/python3 + +import pyalpm +import re + +import aurweb.config +import aurweb.db + +db_path = aurweb.config.get('aurblup', 'db-path') +sync_dbs = aurweb.config.get('aurblup', 'sync-dbs').split(' ') +server = aurweb.config.get('aurblup', 'server') + + +def main(): + blacklist = set() + providers = set() + repomap = dict() + + h = pyalpm.Handle("/", db_path) + for sync_db in sync_dbs: + repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL) + repo.servers = [server.replace("%s", sync_db)] + t = h.init_transaction() + repo.update(False) + t.release() + + for pkg in repo.pkgcache: + blacklist.add(pkg.name) + [blacklist.add(x) for x in pkg.replaces] + providers.add((pkg.name, pkg.name)) + repomap[(pkg.name, pkg.name)] = repo.name + for provision in pkg.provides: + provisionname = re.sub(r'(<|=|>).*', '', provision) + providers.add((pkg.name, provisionname)) + repomap[(pkg.name, provisionname)] = repo.name + + conn = aurweb.db.Connection() + + cur = conn.execute("SELECT Name, Provides FROM OfficialProviders") + oldproviders = set(cur.fetchall()) + + for pkg, provides in providers.difference(oldproviders): + repo = repomap[(pkg, provides)] + conn.execute("INSERT INTO OfficialProviders (Name, Repo, Provides) " + "VALUES (?, ?, ?)", [pkg, repo, provides]) + for pkg, provides in oldproviders.difference(providers): + conn.execute("DELETE FROM OfficialProviders " + "WHERE Name = ? AND Provides = ?", [pkg, provides]) + + conn.commit() + conn.close() + + +if __name__ == '__main__': + main() diff --git a/aurweb/scripts/mkpkglists.py b/aurweb/scripts/mkpkglists.py new file mode 100755 index 0000000..8a0f2e9 --- /dev/null +++ b/aurweb/scripts/mkpkglists.py @@ -0,0 +1,38 @@ +#!/usr/bin/python3 + +import datetime +import gzip + +import aurweb.config +import aurweb.db + +packagesfile = aurweb.config.get('mkpkglists', 'packagesfile') +pkgbasefile = aurweb.config.get('mkpkglists', 'pkgbasefile') + + +def main(): + conn = aurweb.db.Connection() + + datestr = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") + pkglist_header = "# AUR package list, generated on " + datestr + pkgbaselist_header = "# AUR package base list, generated on " + datestr + + with gzip.open(packagesfile, "w") as f: + f.write(bytes(pkglist_header + "\n", "UTF-8")) + cur = conn.execute("SELECT Packages.Name FROM Packages " + + "INNER JOIN PackageBases " + + "ON PackageBases.ID = Packages.PackageBaseID " + + "WHERE PackageBases.PackagerUID IS NOT NULL") + f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + + with gzip.open(pkgbasefile, "w") as f: + f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) + cur = conn.execute("SELECT Name FROM PackageBases " + + "WHERE PackagerUID IS NOT NULL") + f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + + conn.close() + + +if __name__ == '__main__': + main() diff --git a/aurweb/scripts/notify.py b/aurweb/scripts/notify.py new file mode 100755 index 0000000..ddd6e49 --- /dev/null +++ b/aurweb/scripts/notify.py @@ -0,0 +1,455 @@ +#!/usr/bin/python3 + +import email.mime.text +import subprocess +import sys +import textwrap + +import aurweb.config +import aurweb.db + +aur_location = aurweb.config.get('options', 'aur_location') +aur_request_ml = aurweb.config.get('options', 'aur_request_ml') + +sendmail = aurweb.config.get('notifications', 'sendmail') +sender = aurweb.config.get('notifications', 'sender') +reply_to = aurweb.config.get('notifications', 'reply-to') + + +def headers_cc(cclist): + return {'Cc': str.join(', ', cclist)} + + +def headers_msgid(thread_id): + return {'Message-ID': thread_id} + + +def headers_reply(thread_id): + return {'In-Reply-To': thread_id, 'References': thread_id} + + +def send_notification(to, subject, body, refs, headers={}): + wrapped = '' + for line in body.splitlines(): + wrapped += textwrap.fill(line, break_long_words=False) + '\n' + if refs: + body = wrapped + '\n' + refs + else: + body = wrapped + + for recipient in to: + msg = email.mime.text.MIMEText(body, 'plain', 'utf-8') + msg['Subject'] = subject + msg['From'] = sender + msg['Reply-to'] = reply_to + msg['To'] = recipient + + for key, value in headers.items(): + msg[key] = value + + p = subprocess.Popen([sendmail, '-t', '-oi'], stdin=subprocess.PIPE) + p.communicate(msg.as_bytes()) + + +def username_from_id(conn, uid): + cur = conn.execute('SELECT UserName FROM Users WHERE ID = ?', [uid]) + return cur.fetchone()[0] + + +def pkgbase_from_id(conn, pkgbase_id): + cur = conn.execute('SELECT Name FROM PackageBases WHERE ID = ?', + [pkgbase_id]) + return cur.fetchone()[0] + + +def pkgbase_from_pkgreq(conn, reqid): + cur = conn.execute('SELECT PackageBaseID FROM PackageRequests ' + + 'WHERE ID = ?', [reqid]) + return cur.fetchone()[0] + + +def get_user_email(conn, uid): + cur = conn.execute('SELECT Email FROM Users WHERE ID = ?', [uid]) + return cur.fetchone()[0] + + +def get_maintainer_email(conn, pkgbase_id): + cur = conn.execute('SELECT Users.Email FROM Users ' + + 'INNER JOIN PackageBases ' + + 'ON PackageBases.MaintainerUID = Users.ID WHERE ' + + 'PackageBases.ID = ?', [pkgbase_id]) + return cur.fetchone()[0] + + +def get_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) + return [row[0] for row in cur.fetchall()] + + +def get_comment_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'Users.CommentNotify = 1 AND ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) + return [row[0] for row in cur.fetchall()] + + +def get_update_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'Users.UpdateNotify = 1 AND ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) + return [row[0] for row in cur.fetchall()] + + +def get_ownership_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'Users.OwnershipNotify = 1 AND ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) + return [row[0] for row in cur.fetchall()] + + +def get_request_recipients(conn, reqid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM PackageRequests ' + + 'INNER JOIN PackageBases ' + + 'ON PackageBases.ID = PackageRequests.PackageBaseID ' + + 'INNER JOIN Users ' + + 'ON Users.ID = PackageRequests.UsersID ' + + 'OR Users.ID = PackageBases.MaintainerUID ' + + 'WHERE PackageRequests.ID = ?', [reqid]) + return [row[0] for row in cur.fetchall()] + + +def get_tu_vote_reminder_recipients(conn, vote_id): + cur = conn.execute('SELECT Users.Email FROM Users ' + + 'WHERE AccountTypeID = 2 ' + + 'EXCEPT SELECT Users.Email FROM Users ' + + 'INNER JOIN TU_Votes ' + + 'ON TU_Votes.UserID = Users.ID ' + + 'WHERE TU_Votes.VoteID = ?', [vote_id]) + return [row[0] for row in cur.fetchall()] + + +def get_comment(conn, comment_id): + cur = conn.execute('SELECT Comments FROM PackageComments WHERE ID = ?', + [comment_id]) + return cur.fetchone()[0] + + +def get_flagger_comment(conn, pkgbase_id): + cur = conn.execute('SELECT FlaggerComment FROM PackageBases WHERE ID = ?', + [pkgbase_id]) + return cur.fetchone()[0] + + +def get_request_comment(conn, reqid): + cur = conn.execute('SELECT Comments FROM PackageRequests WHERE ID = ?', + [reqid]) + return cur.fetchone()[0] + + +def get_request_closure_comment(conn, reqid): + cur = conn.execute('SELECT ClosureComment FROM PackageRequests ' + + 'WHERE ID = ?', [reqid]) + return cur.fetchone()[0] + + +def send_resetkey(conn, uid): + cur = conn.execute('SELECT UserName, Email, ResetKey FROM Users ' + + 'WHERE ID = ?', [uid]) + username, to, resetkey = cur.fetchone() + + subject = 'AUR Password Reset' + body = 'A password reset request was submitted for the account %s ' \ + 'associated with your email address. If you wish to reset your ' \ + 'password follow the link [1] below, otherwise ignore this ' \ + 'message and nothing will happen.' % (username) + refs = '[1] ' + aur_location + '/passreset/?resetkey=' + resetkey + + send_notification([to], subject, body, refs) + + +def welcome(conn, uid): + cur = conn.execute('SELECT UserName, Email, ResetKey FROM Users ' + + 'WHERE ID = ?', [uid]) + username, to, resetkey = cur.fetchone() + + subject = 'Welcome to the Arch User Repository' + body = 'Welcome to the Arch User Repository! In order to set an initial ' \ + 'password for your new account, please click the link [1] below. ' \ + 'If the link does not work, try copying and pasting it into your ' \ + 'browser.' + refs = '[1] ' + aur_location + '/passreset/?resetkey=' + resetkey + + send_notification([to], subject, body, refs) + + +def comment(conn, uid, pkgbase_id, comment_id): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_comment_recipients(conn, pkgbase_id, uid) + text = get_comment(conn, comment_id) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Comment for %s' % (pkgbase) + body = '%s [1] added the following comment to %s [2]:' % (user, pkgbase) + body += '\n\n' + text + '\n\n' + body += 'If you no longer wish to receive notifications about this ' \ + 'package, please go to the package page [2] and select "%s".' % \ + ('Disable notifications') + refs = '[1] ' + user_uri + '\n' + refs += '[2] ' + pkgbase_uri + thread_id = '' + headers = headers_reply(thread_id) + + send_notification(to, subject, body, refs, headers) + + +def update(conn, uid, pkgbase_id): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_update_recipients(conn, pkgbase_id, uid) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Package Update: %s' % (pkgbase) + body = '%s [1] pushed a new commit to %s [2].' % (user, pkgbase) + body += '\n\n' + body += 'If you no longer wish to receive notifications about this ' \ + 'package, please go to the package page [2] and select "%s".' % \ + ('Disable notifications') + refs = '[1] ' + user_uri + '\n' + refs += '[2] ' + pkgbase_uri + thread_id = '' + headers = headers_reply(thread_id) + + send_notification(to, subject, body, refs, headers) + + +def flag(conn, uid, pkgbase_id): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = [get_maintainer_email(conn, pkgbase_id)] + text = get_flagger_comment(conn, pkgbase_id) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Out-of-date Notification for %s' % (pkgbase) + body = 'Your package %s [1] has been flagged out-of-date by %s [2]:' % \ + (pkgbase, user) + body += '\n\n' + text + refs = '[1] ' + pkgbase_uri + '\n' + refs += '[2] ' + user_uri + + send_notification(to, subject, body, refs) + + +def adopt(conn, pkgbase_id, uid): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_ownership_recipients(conn, pkgbase_id, uid) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Ownership Notification for %s' % (pkgbase) + body = 'The package %s [1] was adopted by %s [2].' % (pkgbase, user) + refs = '[1] ' + pkgbase_uri + '\n' + refs += '[2] ' + user_uri + + send_notification(to, subject, body, refs) + + +def disown(conn, pkgbase_id, uid): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_ownership_recipients(conn, pkgbase_id, uid) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Ownership Notification for %s' % (pkgbase) + body = 'The package %s [1] was disowned by %s [2].' % (pkgbase, user) + refs = '[1] ' + pkgbase_uri + '\n' + refs += '[2] ' + user_uri + + send_notification(to, subject, body, refs) + + +def comaintainer_add(conn, pkgbase_id, uid): + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = [get_user_email(conn, uid)] + + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Co-Maintainer Notification for %s' % (pkgbase) + body = 'You were added to the co-maintainer list of %s [1].' % (pkgbase) + refs = '[1] ' + pkgbase_uri + '\n' + + send_notification(to, subject, body, refs) + + +def comaintainer_remove(conn, pkgbase_id, uid): + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = [get_user_email(conn, uid)] + + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Co-Maintainer Notification for %s' % (pkgbase) + body = ('You were removed from the co-maintainer list of %s [1].' % + (pkgbase)) + refs = '[1] ' + pkgbase_uri + '\n' + + send_notification(to, subject, body, refs) + + +def delete(conn, uid, old_pkgbase_id, new_pkgbase_id=None): + user = username_from_id(conn, uid) + old_pkgbase = pkgbase_from_id(conn, old_pkgbase_id) + if new_pkgbase_id: + new_pkgbase = pkgbase_from_id(conn, new_pkgbase_id) + to = get_recipients(conn, old_pkgbase_id, uid) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + old_pkgbase + '/' + + subject = 'AUR Package deleted: %s' % (old_pkgbase) + if new_pkgbase_id: + new_pkgbase_uri = aur_location + '/pkgbase/' + new_pkgbase + '/' + body = '%s [1] merged %s [2] into %s [3].\n\n' \ + 'If you no longer wish receive notifications about the new ' \ + 'package, please go to [3] and click "%s".' %\ + (user, old_pkgbase, new_pkgbase, 'Disable notifications') + refs = '[1] ' + user_uri + '\n' + refs += '[2] ' + pkgbase_uri + '\n' + refs += '[3] ' + new_pkgbase_uri + else: + body = '%s [1] deleted %s [2].\n\n' \ + 'You will no longer receive notifications about this ' \ + 'package.' % (user, old_pkgbase) + refs = '[1] ' + user_uri + '\n' + refs += '[2] ' + pkgbase_uri + + send_notification(to, subject, body, refs) + + +def request_open(conn, uid, reqid, reqtype, pkgbase_id, merge_into=None): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = [aur_request_ml] + cc = get_request_recipients(conn, reqid) + text = get_request_comment(conn, reqid) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = '[PRQ#%d] %s Request for %s' % \ + (int(reqid), reqtype.title(), pkgbase) + if merge_into: + merge_into_uri = aur_location + '/pkgbase/' + merge_into + '/' + body = '%s [1] filed a request to merge %s [2] into %s [3]:' % \ + (user, pkgbase, merge_into) + body += '\n\n' + text + refs = '[1] ' + user_uri + '\n' + refs += '[2] ' + pkgbase_uri + '\n' + refs += '[3] ' + merge_into_uri + else: + body = '%s [1] filed a %s request for %s [2]:' % \ + (user, reqtype, pkgbase) + body += '\n\n' + text + refs = '[1] ' + user_uri + '\n' + refs += '[2] ' + pkgbase_uri + '\n' + thread_id = '' + # Use a deterministic Message-ID for the first email referencing a request. + headers = headers_msgid(thread_id) + headers.update(headers_cc(cc)) + + send_notification(to, subject, body, refs, headers) + + +def request_close(conn, uid, reqid, reason): + to = [aur_request_ml] + cc = get_request_recipients(conn, reqid) + text = get_request_closure_comment(conn, reqid) + + subject = '[PRQ#%d] Request %s' % (int(reqid), reason.title()) + if int(uid): + user = username_from_id(conn, uid) + user_uri = aur_location + '/account/' + user + '/' + body = 'Request #%d has been %s by %s [1]' % (int(reqid), reason, user) + refs = '[1] ' + user_uri + else: + body = 'Request #%d has been %s automatically by the Arch User ' \ + 'Repository package request system' % (int(reqid), reason) + refs = None + if text.strip() == '': + body += '.' + else: + body += ':\n\n' + text + thread_id = '' + headers = headers_reply(thread_id) + headers.update(headers_cc(cc)) + + send_notification(to, subject, body, refs, headers) + + +def tu_vote_reminder(conn, vote_id): + to = get_tu_vote_reminder_recipients(conn, vote_id) + + vote_uri = aur_location + '/tu/?id=' + vote_id + + subject = 'TU Vote Reminder: Proposal %d' % (int(vote_id)) + body = 'Please remember to cast your vote on proposal %d [1]. ' \ + 'The voting period ends in less than 48 hours.' % (int(vote_id)) + refs = '[1] ' + vote_uri + + send_notification(to, subject, body, refs) + + +def main(): + action = sys.argv[1] + action_map = { + 'send-resetkey': send_resetkey, + 'welcome': welcome, + 'comment': comment, + 'update': update, + 'flag': flag, + 'adopt': adopt, + 'disown': disown, + 'comaintainer-add': comaintainer_add, + 'comaintainer-remove': comaintainer_remove, + 'delete': delete, + 'request-open': request_open, + 'request-close': request_close, + 'tu-vote-reminder': tu_vote_reminder, + } + + conn = aurweb.db.Connection() + + action_map[action](conn, *sys.argv[2:]) + + conn.commit() + conn.close() + + +if __name__ == '__main__': + main() diff --git a/aurweb/scripts/pkgmaint.py b/aurweb/scripts/pkgmaint.py new file mode 100755 index 0000000..3ad9ed8 --- /dev/null +++ b/aurweb/scripts/pkgmaint.py @@ -0,0 +1,20 @@ +#!/usr/bin/python3 + +import time + +import aurweb.db + + +def main(): + conn = aurweb.db.Connection() + + limit_to = int(time.time()) - 86400 + conn.execute("DELETE FROM PackageBases WHERE " + + "SubmittedTS < ? AND PackagerUID IS NULL", [limit_to]) + + conn.commit() + conn.close() + + +if __name__ == '__main__': + main() diff --git a/aurweb/scripts/popupdate.py b/aurweb/scripts/popupdate.py new file mode 100755 index 0000000..58cd018 --- /dev/null +++ b/aurweb/scripts/popupdate.py @@ -0,0 +1,26 @@ +#!/usr/bin/python3 + +import time + +import aurweb.db + + +def main(): + conn = aurweb.db.Connection() + + conn.execute("UPDATE PackageBases SET NumVotes = (" + + "SELECT COUNT(*) FROM PackageVotes " + + "WHERE PackageVotes.PackageBaseID = PackageBases.ID)") + + now = int(time.time()) + conn.execute("UPDATE PackageBases SET Popularity = (" + + "SELECT COALESCE(SUM(POWER(0.98, (? - VoteTS) / 86400)), 0.0) " + + "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " + + "PackageBases.ID AND NOT VoteTS IS NULL)", [now]) + + conn.commit() + conn.close() + + +if __name__ == '__main__': + main() diff --git a/aurweb/scripts/tuvotereminder.py b/aurweb/scripts/tuvotereminder.py new file mode 100755 index 0000000..97b1d12 --- /dev/null +++ b/aurweb/scripts/tuvotereminder.py @@ -0,0 +1,28 @@ +#!/usr/bin/python3 + +import subprocess +import time + +import aurweb.config +import aurweb.db + +notify_cmd = aurweb.config.get('notifications', 'notify-cmd') + + +def main(): + conn = aurweb.db.Connection() + + now = int(time.time()) + filter_from = now + 500 + filter_to = now + 172800 + + cur = conn.execute("SELECT ID FROM TU_VoteInfo " + + "WHERE End >= ? AND End <= ?", + [filter_from, filter_to]) + + for vote_id in [row[0] for row in cur.fetchall()]: + subprocess.Popen((notify_cmd, 'tu-vote-reminder', str(vote_id))).wait() + + +if __name__ == '__main__': + main() diff --git a/scripts/__init__.py b/scripts/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/scripts/aurblup.py b/scripts/aurblup.py deleted file mode 100755 index 1b6de2f..0000000 --- a/scripts/aurblup.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/python3 - -import pyalpm -import re - -import aurweb.config -import aurweb.db - -db_path = aurweb.config.get('aurblup', 'db-path') -sync_dbs = aurweb.config.get('aurblup', 'sync-dbs').split(' ') -server = aurweb.config.get('aurblup', 'server') - - -def main(): - blacklist = set() - providers = set() - repomap = dict() - - h = pyalpm.Handle("/", db_path) - for sync_db in sync_dbs: - repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL) - repo.servers = [server.replace("%s", sync_db)] - t = h.init_transaction() - repo.update(False) - t.release() - - for pkg in repo.pkgcache: - blacklist.add(pkg.name) - [blacklist.add(x) for x in pkg.replaces] - providers.add((pkg.name, pkg.name)) - repomap[(pkg.name, pkg.name)] = repo.name - for provision in pkg.provides: - provisionname = re.sub(r'(<|=|>).*', '', provision) - providers.add((pkg.name, provisionname)) - repomap[(pkg.name, provisionname)] = repo.name - - conn = aurweb.db.Connection() - - cur = conn.execute("SELECT Name, Provides FROM OfficialProviders") - oldproviders = set(cur.fetchall()) - - for pkg, provides in providers.difference(oldproviders): - repo = repomap[(pkg, provides)] - conn.execute("INSERT INTO OfficialProviders (Name, Repo, Provides) " - "VALUES (?, ?, ?)", [pkg, repo, provides]) - for pkg, provides in oldproviders.difference(providers): - conn.execute("DELETE FROM OfficialProviders " - "WHERE Name = ? AND Provides = ?", [pkg, provides]) - - conn.commit() - conn.close() - - -if __name__ == '__main__': - main() diff --git a/scripts/mkpkglists.py b/scripts/mkpkglists.py deleted file mode 100755 index 8a0f2e9..0000000 --- a/scripts/mkpkglists.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/python3 - -import datetime -import gzip - -import aurweb.config -import aurweb.db - -packagesfile = aurweb.config.get('mkpkglists', 'packagesfile') -pkgbasefile = aurweb.config.get('mkpkglists', 'pkgbasefile') - - -def main(): - conn = aurweb.db.Connection() - - datestr = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") - pkglist_header = "# AUR package list, generated on " + datestr - pkgbaselist_header = "# AUR package base list, generated on " + datestr - - with gzip.open(packagesfile, "w") as f: - f.write(bytes(pkglist_header + "\n", "UTF-8")) - cur = conn.execute("SELECT Packages.Name FROM Packages " + - "INNER JOIN PackageBases " + - "ON PackageBases.ID = Packages.PackageBaseID " + - "WHERE PackageBases.PackagerUID IS NOT NULL") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - - with gzip.open(pkgbasefile, "w") as f: - f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) - cur = conn.execute("SELECT Name FROM PackageBases " + - "WHERE PackagerUID IS NOT NULL") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - - conn.close() - - -if __name__ == '__main__': - main() diff --git a/scripts/notify.py b/scripts/notify.py deleted file mode 100755 index ddd6e49..0000000 --- a/scripts/notify.py +++ /dev/null @@ -1,455 +0,0 @@ -#!/usr/bin/python3 - -import email.mime.text -import subprocess -import sys -import textwrap - -import aurweb.config -import aurweb.db - -aur_location = aurweb.config.get('options', 'aur_location') -aur_request_ml = aurweb.config.get('options', 'aur_request_ml') - -sendmail = aurweb.config.get('notifications', 'sendmail') -sender = aurweb.config.get('notifications', 'sender') -reply_to = aurweb.config.get('notifications', 'reply-to') - - -def headers_cc(cclist): - return {'Cc': str.join(', ', cclist)} - - -def headers_msgid(thread_id): - return {'Message-ID': thread_id} - - -def headers_reply(thread_id): - return {'In-Reply-To': thread_id, 'References': thread_id} - - -def send_notification(to, subject, body, refs, headers={}): - wrapped = '' - for line in body.splitlines(): - wrapped += textwrap.fill(line, break_long_words=False) + '\n' - if refs: - body = wrapped + '\n' + refs - else: - body = wrapped - - for recipient in to: - msg = email.mime.text.MIMEText(body, 'plain', 'utf-8') - msg['Subject'] = subject - msg['From'] = sender - msg['Reply-to'] = reply_to - msg['To'] = recipient - - for key, value in headers.items(): - msg[key] = value - - p = subprocess.Popen([sendmail, '-t', '-oi'], stdin=subprocess.PIPE) - p.communicate(msg.as_bytes()) - - -def username_from_id(conn, uid): - cur = conn.execute('SELECT UserName FROM Users WHERE ID = ?', [uid]) - return cur.fetchone()[0] - - -def pkgbase_from_id(conn, pkgbase_id): - cur = conn.execute('SELECT Name FROM PackageBases WHERE ID = ?', - [pkgbase_id]) - return cur.fetchone()[0] - - -def pkgbase_from_pkgreq(conn, reqid): - cur = conn.execute('SELECT PackageBaseID FROM PackageRequests ' + - 'WHERE ID = ?', [reqid]) - return cur.fetchone()[0] - - -def get_user_email(conn, uid): - cur = conn.execute('SELECT Email FROM Users WHERE ID = ?', [uid]) - return cur.fetchone()[0] - - -def get_maintainer_email(conn, pkgbase_id): - cur = conn.execute('SELECT Users.Email FROM Users ' + - 'INNER JOIN PackageBases ' + - 'ON PackageBases.MaintainerUID = Users.ID WHERE ' + - 'PackageBases.ID = ?', [pkgbase_id]) - return cur.fetchone()[0] - - -def get_recipients(conn, pkgbase_id, uid): - cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + - 'INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'PackageNotifications.UserID != ? AND ' + - 'PackageNotifications.PackageBaseID = ?', - [uid, pkgbase_id]) - return [row[0] for row in cur.fetchall()] - - -def get_comment_recipients(conn, pkgbase_id, uid): - cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + - 'INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'Users.CommentNotify = 1 AND ' + - 'PackageNotifications.UserID != ? AND ' + - 'PackageNotifications.PackageBaseID = ?', - [uid, pkgbase_id]) - return [row[0] for row in cur.fetchall()] - - -def get_update_recipients(conn, pkgbase_id, uid): - cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + - 'INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'Users.UpdateNotify = 1 AND ' + - 'PackageNotifications.UserID != ? AND ' + - 'PackageNotifications.PackageBaseID = ?', - [uid, pkgbase_id]) - return [row[0] for row in cur.fetchall()] - - -def get_ownership_recipients(conn, pkgbase_id, uid): - cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + - 'INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'Users.OwnershipNotify = 1 AND ' + - 'PackageNotifications.UserID != ? AND ' + - 'PackageNotifications.PackageBaseID = ?', - [uid, pkgbase_id]) - return [row[0] for row in cur.fetchall()] - - -def get_request_recipients(conn, reqid): - cur = conn.execute('SELECT DISTINCT Users.Email FROM PackageRequests ' + - 'INNER JOIN PackageBases ' + - 'ON PackageBases.ID = PackageRequests.PackageBaseID ' + - 'INNER JOIN Users ' + - 'ON Users.ID = PackageRequests.UsersID ' + - 'OR Users.ID = PackageBases.MaintainerUID ' + - 'WHERE PackageRequests.ID = ?', [reqid]) - return [row[0] for row in cur.fetchall()] - - -def get_tu_vote_reminder_recipients(conn, vote_id): - cur = conn.execute('SELECT Users.Email FROM Users ' + - 'WHERE AccountTypeID = 2 ' + - 'EXCEPT SELECT Users.Email FROM Users ' + - 'INNER JOIN TU_Votes ' + - 'ON TU_Votes.UserID = Users.ID ' + - 'WHERE TU_Votes.VoteID = ?', [vote_id]) - return [row[0] for row in cur.fetchall()] - - -def get_comment(conn, comment_id): - cur = conn.execute('SELECT Comments FROM PackageComments WHERE ID = ?', - [comment_id]) - return cur.fetchone()[0] - - -def get_flagger_comment(conn, pkgbase_id): - cur = conn.execute('SELECT FlaggerComment FROM PackageBases WHERE ID = ?', - [pkgbase_id]) - return cur.fetchone()[0] - - -def get_request_comment(conn, reqid): - cur = conn.execute('SELECT Comments FROM PackageRequests WHERE ID = ?', - [reqid]) - return cur.fetchone()[0] - - -def get_request_closure_comment(conn, reqid): - cur = conn.execute('SELECT ClosureComment FROM PackageRequests ' + - 'WHERE ID = ?', [reqid]) - return cur.fetchone()[0] - - -def send_resetkey(conn, uid): - cur = conn.execute('SELECT UserName, Email, ResetKey FROM Users ' + - 'WHERE ID = ?', [uid]) - username, to, resetkey = cur.fetchone() - - subject = 'AUR Password Reset' - body = 'A password reset request was submitted for the account %s ' \ - 'associated with your email address. If you wish to reset your ' \ - 'password follow the link [1] below, otherwise ignore this ' \ - 'message and nothing will happen.' % (username) - refs = '[1] ' + aur_location + '/passreset/?resetkey=' + resetkey - - send_notification([to], subject, body, refs) - - -def welcome(conn, uid): - cur = conn.execute('SELECT UserName, Email, ResetKey FROM Users ' + - 'WHERE ID = ?', [uid]) - username, to, resetkey = cur.fetchone() - - subject = 'Welcome to the Arch User Repository' - body = 'Welcome to the Arch User Repository! In order to set an initial ' \ - 'password for your new account, please click the link [1] below. ' \ - 'If the link does not work, try copying and pasting it into your ' \ - 'browser.' - refs = '[1] ' + aur_location + '/passreset/?resetkey=' + resetkey - - send_notification([to], subject, body, refs) - - -def comment(conn, uid, pkgbase_id, comment_id): - user = username_from_id(conn, uid) - pkgbase = pkgbase_from_id(conn, pkgbase_id) - to = get_comment_recipients(conn, pkgbase_id, uid) - text = get_comment(conn, comment_id) - - user_uri = aur_location + '/account/' + user + '/' - pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' - - subject = 'AUR Comment for %s' % (pkgbase) - body = '%s [1] added the following comment to %s [2]:' % (user, pkgbase) - body += '\n\n' + text + '\n\n' - body += 'If you no longer wish to receive notifications about this ' \ - 'package, please go to the package page [2] and select "%s".' % \ - ('Disable notifications') - refs = '[1] ' + user_uri + '\n' - refs += '[2] ' + pkgbase_uri - thread_id = '' - headers = headers_reply(thread_id) - - send_notification(to, subject, body, refs, headers) - - -def update(conn, uid, pkgbase_id): - user = username_from_id(conn, uid) - pkgbase = pkgbase_from_id(conn, pkgbase_id) - to = get_update_recipients(conn, pkgbase_id, uid) - - user_uri = aur_location + '/account/' + user + '/' - pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' - - subject = 'AUR Package Update: %s' % (pkgbase) - body = '%s [1] pushed a new commit to %s [2].' % (user, pkgbase) - body += '\n\n' - body += 'If you no longer wish to receive notifications about this ' \ - 'package, please go to the package page [2] and select "%s".' % \ - ('Disable notifications') - refs = '[1] ' + user_uri + '\n' - refs += '[2] ' + pkgbase_uri - thread_id = '' - headers = headers_reply(thread_id) - - send_notification(to, subject, body, refs, headers) - - -def flag(conn, uid, pkgbase_id): - user = username_from_id(conn, uid) - pkgbase = pkgbase_from_id(conn, pkgbase_id) - to = [get_maintainer_email(conn, pkgbase_id)] - text = get_flagger_comment(conn, pkgbase_id) - - user_uri = aur_location + '/account/' + user + '/' - pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' - - subject = 'AUR Out-of-date Notification for %s' % (pkgbase) - body = 'Your package %s [1] has been flagged out-of-date by %s [2]:' % \ - (pkgbase, user) - body += '\n\n' + text - refs = '[1] ' + pkgbase_uri + '\n' - refs += '[2] ' + user_uri - - send_notification(to, subject, body, refs) - - -def adopt(conn, pkgbase_id, uid): - user = username_from_id(conn, uid) - pkgbase = pkgbase_from_id(conn, pkgbase_id) - to = get_ownership_recipients(conn, pkgbase_id, uid) - - user_uri = aur_location + '/account/' + user + '/' - pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' - - subject = 'AUR Ownership Notification for %s' % (pkgbase) - body = 'The package %s [1] was adopted by %s [2].' % (pkgbase, user) - refs = '[1] ' + pkgbase_uri + '\n' - refs += '[2] ' + user_uri - - send_notification(to, subject, body, refs) - - -def disown(conn, pkgbase_id, uid): - user = username_from_id(conn, uid) - pkgbase = pkgbase_from_id(conn, pkgbase_id) - to = get_ownership_recipients(conn, pkgbase_id, uid) - - user_uri = aur_location + '/account/' + user + '/' - pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' - - subject = 'AUR Ownership Notification for %s' % (pkgbase) - body = 'The package %s [1] was disowned by %s [2].' % (pkgbase, user) - refs = '[1] ' + pkgbase_uri + '\n' - refs += '[2] ' + user_uri - - send_notification(to, subject, body, refs) - - -def comaintainer_add(conn, pkgbase_id, uid): - pkgbase = pkgbase_from_id(conn, pkgbase_id) - to = [get_user_email(conn, uid)] - - pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' - - subject = 'AUR Co-Maintainer Notification for %s' % (pkgbase) - body = 'You were added to the co-maintainer list of %s [1].' % (pkgbase) - refs = '[1] ' + pkgbase_uri + '\n' - - send_notification(to, subject, body, refs) - - -def comaintainer_remove(conn, pkgbase_id, uid): - pkgbase = pkgbase_from_id(conn, pkgbase_id) - to = [get_user_email(conn, uid)] - - pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' - - subject = 'AUR Co-Maintainer Notification for %s' % (pkgbase) - body = ('You were removed from the co-maintainer list of %s [1].' % - (pkgbase)) - refs = '[1] ' + pkgbase_uri + '\n' - - send_notification(to, subject, body, refs) - - -def delete(conn, uid, old_pkgbase_id, new_pkgbase_id=None): - user = username_from_id(conn, uid) - old_pkgbase = pkgbase_from_id(conn, old_pkgbase_id) - if new_pkgbase_id: - new_pkgbase = pkgbase_from_id(conn, new_pkgbase_id) - to = get_recipients(conn, old_pkgbase_id, uid) - - user_uri = aur_location + '/account/' + user + '/' - pkgbase_uri = aur_location + '/pkgbase/' + old_pkgbase + '/' - - subject = 'AUR Package deleted: %s' % (old_pkgbase) - if new_pkgbase_id: - new_pkgbase_uri = aur_location + '/pkgbase/' + new_pkgbase + '/' - body = '%s [1] merged %s [2] into %s [3].\n\n' \ - 'If you no longer wish receive notifications about the new ' \ - 'package, please go to [3] and click "%s".' %\ - (user, old_pkgbase, new_pkgbase, 'Disable notifications') - refs = '[1] ' + user_uri + '\n' - refs += '[2] ' + pkgbase_uri + '\n' - refs += '[3] ' + new_pkgbase_uri - else: - body = '%s [1] deleted %s [2].\n\n' \ - 'You will no longer receive notifications about this ' \ - 'package.' % (user, old_pkgbase) - refs = '[1] ' + user_uri + '\n' - refs += '[2] ' + pkgbase_uri - - send_notification(to, subject, body, refs) - - -def request_open(conn, uid, reqid, reqtype, pkgbase_id, merge_into=None): - user = username_from_id(conn, uid) - pkgbase = pkgbase_from_id(conn, pkgbase_id) - to = [aur_request_ml] - cc = get_request_recipients(conn, reqid) - text = get_request_comment(conn, reqid) - - user_uri = aur_location + '/account/' + user + '/' - pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' - - subject = '[PRQ#%d] %s Request for %s' % \ - (int(reqid), reqtype.title(), pkgbase) - if merge_into: - merge_into_uri = aur_location + '/pkgbase/' + merge_into + '/' - body = '%s [1] filed a request to merge %s [2] into %s [3]:' % \ - (user, pkgbase, merge_into) - body += '\n\n' + text - refs = '[1] ' + user_uri + '\n' - refs += '[2] ' + pkgbase_uri + '\n' - refs += '[3] ' + merge_into_uri - else: - body = '%s [1] filed a %s request for %s [2]:' % \ - (user, reqtype, pkgbase) - body += '\n\n' + text - refs = '[1] ' + user_uri + '\n' - refs += '[2] ' + pkgbase_uri + '\n' - thread_id = '' - # Use a deterministic Message-ID for the first email referencing a request. - headers = headers_msgid(thread_id) - headers.update(headers_cc(cc)) - - send_notification(to, subject, body, refs, headers) - - -def request_close(conn, uid, reqid, reason): - to = [aur_request_ml] - cc = get_request_recipients(conn, reqid) - text = get_request_closure_comment(conn, reqid) - - subject = '[PRQ#%d] Request %s' % (int(reqid), reason.title()) - if int(uid): - user = username_from_id(conn, uid) - user_uri = aur_location + '/account/' + user + '/' - body = 'Request #%d has been %s by %s [1]' % (int(reqid), reason, user) - refs = '[1] ' + user_uri - else: - body = 'Request #%d has been %s automatically by the Arch User ' \ - 'Repository package request system' % (int(reqid), reason) - refs = None - if text.strip() == '': - body += '.' - else: - body += ':\n\n' + text - thread_id = '' - headers = headers_reply(thread_id) - headers.update(headers_cc(cc)) - - send_notification(to, subject, body, refs, headers) - - -def tu_vote_reminder(conn, vote_id): - to = get_tu_vote_reminder_recipients(conn, vote_id) - - vote_uri = aur_location + '/tu/?id=' + vote_id - - subject = 'TU Vote Reminder: Proposal %d' % (int(vote_id)) - body = 'Please remember to cast your vote on proposal %d [1]. ' \ - 'The voting period ends in less than 48 hours.' % (int(vote_id)) - refs = '[1] ' + vote_uri - - send_notification(to, subject, body, refs) - - -def main(): - action = sys.argv[1] - action_map = { - 'send-resetkey': send_resetkey, - 'welcome': welcome, - 'comment': comment, - 'update': update, - 'flag': flag, - 'adopt': adopt, - 'disown': disown, - 'comaintainer-add': comaintainer_add, - 'comaintainer-remove': comaintainer_remove, - 'delete': delete, - 'request-open': request_open, - 'request-close': request_close, - 'tu-vote-reminder': tu_vote_reminder, - } - - conn = aurweb.db.Connection() - - action_map[action](conn, *sys.argv[2:]) - - conn.commit() - conn.close() - - -if __name__ == '__main__': - main() diff --git a/scripts/pkgmaint.py b/scripts/pkgmaint.py deleted file mode 100755 index 3ad9ed8..0000000 --- a/scripts/pkgmaint.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/python3 - -import time - -import aurweb.db - - -def main(): - conn = aurweb.db.Connection() - - limit_to = int(time.time()) - 86400 - conn.execute("DELETE FROM PackageBases WHERE " + - "SubmittedTS < ? AND PackagerUID IS NULL", [limit_to]) - - conn.commit() - conn.close() - - -if __name__ == '__main__': - main() diff --git a/scripts/popupdate.py b/scripts/popupdate.py deleted file mode 100755 index 58cd018..0000000 --- a/scripts/popupdate.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/python3 - -import time - -import aurweb.db - - -def main(): - conn = aurweb.db.Connection() - - conn.execute("UPDATE PackageBases SET NumVotes = (" + - "SELECT COUNT(*) FROM PackageVotes " + - "WHERE PackageVotes.PackageBaseID = PackageBases.ID)") - - now = int(time.time()) - conn.execute("UPDATE PackageBases SET Popularity = (" + - "SELECT COALESCE(SUM(POWER(0.98, (? - VoteTS) / 86400)), 0.0) " + - "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " + - "PackageBases.ID AND NOT VoteTS IS NULL)", [now]) - - conn.commit() - conn.close() - - -if __name__ == '__main__': - main() diff --git a/scripts/tuvotereminder.py b/scripts/tuvotereminder.py deleted file mode 100755 index 97b1d12..0000000 --- a/scripts/tuvotereminder.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/python3 - -import subprocess -import time - -import aurweb.config -import aurweb.db - -notify_cmd = aurweb.config.get('notifications', 'notify-cmd') - - -def main(): - conn = aurweb.db.Connection() - - now = int(time.time()) - filter_from = now + 500 - filter_to = now + 172800 - - cur = conn.execute("SELECT ID FROM TU_VoteInfo " + - "WHERE End >= ? AND End <= ?", - [filter_from, filter_to]) - - for vote_id in [row[0] for row in cur.fetchall()]: - subprocess.Popen((notify_cmd, 'tu-vote-reminder', str(vote_id))).wait() - - -if __name__ == '__main__': - main() diff --git a/setup.py b/setup.py index b64e71c..99dbfed 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,12 @@ setup( 'aurweb-git-auth = aurweb.git.auth:main', 'aurweb-git-serve = aurweb.git.serve:main', 'aurweb-git-update = aurweb.git.update:main', + 'aurweb-aurblup = aurweb.scripts.aurblup:main', + 'aurweb-mkpkglists = aurweb.scripts.mkpkglists:main', + 'aurweb-notify = aurweb.scripts.notify:main', + 'aurweb-pkgmaint = aurweb.scripts.pkgmaint:main', + 'aurweb-popupdate = aurweb.scripts.popupdate:main', + 'aurweb-tuvotereminder = aurweb.scripts.tuvotereminder:main', ], }, ) diff --git a/upgrading/4.4.1.txt b/upgrading/4.4.1.txt index b06696e..726f9e2 100644 --- a/upgrading/4.4.1.txt +++ b/upgrading/4.4.1.txt @@ -1,3 +1,6 @@ 1. The default configuration file search path now points to /etc/aurweb/config. Make sure you copy your aurweb configuration to the new location before upgrading. + +2. The maintenance scripts have been prefixed by "aurweb-" and can now be + installed using `python3 setup.py install`. -- cgit v1.2.3-54-g00ecf