From 2915abb9d35308150ec107c5f4664e116daaf1de Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Wed, 3 Aug 2016 02:20:40 +0200 Subject: git-interface: Add database abstraction layer Add a new class that connects to the database specified in the configuration file and provides an interface to execute SQL queries. Prepared statements with qmark ("?") placeholders are supported. Replace all direct database accesses with calls to the new abstraction layer. Signed-off-by: Lukas Fleischer --- git-interface/git-auth.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) (limited to 'git-interface/git-auth.py') diff --git a/git-interface/git-auth.py b/git-interface/git-auth.py index 83bd20c..7cd033c 100755 --- a/git-interface/git-auth.py +++ b/git-interface/git-auth.py @@ -1,12 +1,13 @@ #!/usr/bin/python3 import configparser -import mysql.connector import shlex import os import re import sys +import db + def format_command(env_vars, command, ssh_opts, ssh_key): environment = '' @@ -26,12 +27,6 @@ def format_command(env_vars, command, ssh_opts, ssh_key): config = configparser.RawConfigParser() config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") -aur_db_host = config.get('database', 'host') -aur_db_name = config.get('database', 'name') -aur_db_user = config.get('database', 'user') -aur_db_pass = config.get('database', 'password') -aur_db_socket = config.get('database', 'socket') - valid_keytypes = config.get('auth', 'valid-keytypes').split() username_regex = config.get('auth', 'username-regex') git_serve_cmd = config.get('auth', 'git-serve-cmd') @@ -42,15 +37,12 @@ keytext = sys.argv[2] if keytype not in valid_keytypes: exit(1) -db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) +conn = db.Connection() -cur = db.cursor() -cur.execute("SELECT Users.Username, Users.AccountTypeID FROM Users " + - "INNER JOIN SSHPubKeys ON SSHPubKeys.UserID = Users.ID " - "WHERE SSHPubKeys.PubKey = %s AND Users.Suspended = 0", - (keytype + " " + keytext,)) +cur = conn.execute("SELECT Users.Username, Users.AccountTypeID FROM Users " + + "INNER JOIN SSHPubKeys ON SSHPubKeys.UserID = Users.ID " + "WHERE SSHPubKeys.PubKey = ? AND Users.Suspended = 0", + (keytype + " " + keytext,)) if cur.rowcount != 1: exit(1) -- cgit v1.2.3-54-g00ecf From 2f5f5583bec2a0a04424d6bedd763855f308bce6 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Wed, 3 Aug 2016 20:21:40 +0200 Subject: git-interface: Factor out configuration file parsing Add a new module that automatically locates the configuration file and provides methods to obtain the values of configuration options. Use the new module instead of ConfigParser everywhere. Signed-off-by: Lukas Fleischer --- git-interface/config.py | 27 +++++++++++++++++++++++++++ git-interface/db.py | 7 ++----- git-interface/git-auth.py | 6 +----- git-interface/git-serve.py | 5 +---- git-interface/git-update.py | 5 +---- 5 files changed, 32 insertions(+), 18 deletions(-) create mode 100644 git-interface/config.py (limited to 'git-interface/git-auth.py') diff --git a/git-interface/config.py b/git-interface/config.py new file mode 100644 index 0000000..cd6495b --- /dev/null +++ b/git-interface/config.py @@ -0,0 +1,27 @@ +import configparser +import os + +_parser = None + + +def _get_parser(): + global _parser + + if not _parser: + _parser = configparser.RawConfigParser() + path = os.path.dirname(os.path.realpath(__file__)) + "/../conf/config" + _parser.read(path) + + return _parser + + +def get(section, option): + return _get_parser().get(section, option) + + +def getboolean(section, option): + return _get_parser().getboolean(section, option) + + +def getint(section, option): + return _get_parser().getint(section, option) diff --git a/git-interface/db.py b/git-interface/db.py index d3e1e69..c4c7d31 100644 --- a/git-interface/db.py +++ b/git-interface/db.py @@ -1,15 +1,12 @@ -import configparser import mysql.connector -import os + +import config class Connection: _conn = None def __init__(self): - config = configparser.RawConfigParser() - config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") - aur_db_host = config.get('database', 'host') aur_db_name = config.get('database', 'name') aur_db_user = config.get('database', 'user') diff --git a/git-interface/git-auth.py b/git-interface/git-auth.py index 7cd033c..ebdc75c 100755 --- a/git-interface/git-auth.py +++ b/git-interface/git-auth.py @@ -1,11 +1,10 @@ #!/usr/bin/python3 -import configparser import shlex -import os import re import sys +import config import db @@ -24,9 +23,6 @@ def format_command(env_vars, command, ssh_opts, ssh_key): return msg -config = configparser.RawConfigParser() -config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") - valid_keytypes = config.get('auth', 'valid-keytypes').split() username_regex = config.get('auth', 'username-regex') git_serve_cmd = config.get('auth', 'git-serve-cmd') diff --git a/git-interface/git-serve.py b/git-interface/git-serve.py index ab612f0..6377ffc 100755 --- a/git-interface/git-serve.py +++ b/git-interface/git-serve.py @@ -1,16 +1,13 @@ #!/usr/bin/python3 -import configparser import os import re import shlex import sys +import config import db -config = configparser.RawConfigParser() -config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") - repo_path = config.get('serve', 'repo-path') repo_regex = config.get('serve', 'repo-regex') git_shell_cmd = config.get('serve', 'git-shell-cmd') diff --git a/git-interface/git-update.py b/git-interface/git-update.py index b7199e6..9a127a9 100755 --- a/git-interface/git-update.py +++ b/git-interface/git-update.py @@ -1,6 +1,5 @@ #!/usr/bin/python3 -import configparser import os import pygit2 import re @@ -10,11 +9,9 @@ import sys import srcinfo.parse import srcinfo.utils +import config import db -config = configparser.RawConfigParser() -config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") - notify_cmd = config.get('notifications', 'notify-cmd') repo_path = config.get('serve', 'repo-path') -- cgit v1.2.3-54-g00ecf From 27631f1157226bd9ca4d0dbfb6a59c7656e7e361 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Thu, 4 Aug 2016 21:00:50 +0200 Subject: git-interface: Do not use rowcount Avoid using Cursor.rowcount to obtain the number of rows returned by a SELECT statement as this is not guaranteed to be supported by every database engine. Signed-off-by: Lukas Fleischer --- git-interface/git-auth.py | 5 +++-- git-interface/git-update.py | 13 ++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) (limited to 'git-interface/git-auth.py') diff --git a/git-interface/git-auth.py b/git-interface/git-auth.py index ebdc75c..45fd577 100755 --- a/git-interface/git-auth.py +++ b/git-interface/git-auth.py @@ -40,10 +40,11 @@ cur = conn.execute("SELECT Users.Username, Users.AccountTypeID FROM Users " + "WHERE SSHPubKeys.PubKey = ? AND Users.Suspended = 0", (keytype + " " + keytext,)) -if cur.rowcount != 1: +row = cur.fetchone() +if not row or cur.fetchone(): exit(1) -user, account_type = cur.fetchone() +user, account_type = row if not re.match(username_regex, user): exit(1) diff --git a/git-interface/git-update.py b/git-interface/git-update.py index 9a127a9..d6c9f10 100755 --- a/git-interface/git-update.py +++ b/git-interface/git-update.py @@ -140,8 +140,9 @@ def save_metadata(metadata, conn, user): for license in pkginfo['license']: cur = conn.execute("SELECT ID FROM Licenses WHERE Name = ?", [license]) - if cur.rowcount == 1: - licenseid = cur.fetchone()[0] + row = cur.fetchone() + if row: + licenseid = row[0] else: cur = conn.execute("INSERT INTO Licenses (Name) " + "VALUES (?)", [license]) @@ -156,8 +157,9 @@ def save_metadata(metadata, conn, user): for group in pkginfo['groups']: cur = conn.execute("SELECT ID FROM Groups WHERE Name = ?", [group]) - if cur.rowcount == 1: - groupid = cur.fetchone()[0] + row = cur.fetchone() + if row: + groupid = row[0] else: cur = conn.execute("INSERT INTO Groups (Name) VALUES (?)", [group]) @@ -329,7 +331,8 @@ if metadata_pkgbase != pkgbase: # Ensure that packages are neither blacklisted nor overwritten. pkgbase = metadata['pkgbase'] cur = conn.execute("SELECT ID FROM PackageBases WHERE Name = ?", [pkgbase]) -pkgbase_id = cur.fetchone()[0] if cur.rowcount == 1 else 0 +row = cur.fetchone() +pkgbase_id = row[0] if row else 0 cur = conn.execute("SELECT Name FROM PackageBlacklist") blacklist = [row[0] for row in cur.fetchall()] -- cgit v1.2.3-54-g00ecf From 3a352435e95207fd395a9dbd19227da57f243047 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Tue, 20 Sep 2016 08:45:43 +0200 Subject: git-auth: Move entry point to a main() method Move the main program logic of git-auth to a main() method such that it can be used as a module and easily be invoked by setuptools wrapper scripts. Signed-off-by: Lukas Fleischer --- git-interface/git-auth.py | 54 +++++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 25 deletions(-) (limited to 'git-interface/git-auth.py') diff --git a/git-interface/git-auth.py b/git-interface/git-auth.py index 45fd577..d3b0188 100755 --- a/git-interface/git-auth.py +++ b/git-interface/git-auth.py @@ -23,36 +23,40 @@ def format_command(env_vars, command, ssh_opts, ssh_key): return msg -valid_keytypes = config.get('auth', 'valid-keytypes').split() -username_regex = config.get('auth', 'username-regex') -git_serve_cmd = config.get('auth', 'git-serve-cmd') -ssh_opts = config.get('auth', 'ssh-options') +def main(): + valid_keytypes = config.get('auth', 'valid-keytypes').split() + username_regex = config.get('auth', 'username-regex') + git_serve_cmd = config.get('auth', 'git-serve-cmd') + ssh_opts = config.get('auth', 'ssh-options') -keytype = sys.argv[1] -keytext = sys.argv[2] -if keytype not in valid_keytypes: - exit(1) + keytype = sys.argv[1] + keytext = sys.argv[2] + if keytype not in valid_keytypes: + exit(1) -conn = db.Connection() + conn = db.Connection() -cur = conn.execute("SELECT Users.Username, Users.AccountTypeID FROM Users " + - "INNER JOIN SSHPubKeys ON SSHPubKeys.UserID = Users.ID " - "WHERE SSHPubKeys.PubKey = ? AND Users.Suspended = 0", - (keytype + " " + keytext,)) + cur = conn.execute("SELECT Users.Username, Users.AccountTypeID FROM Users " + "INNER JOIN SSHPubKeys ON SSHPubKeys.UserID = Users.ID " + "WHERE SSHPubKeys.PubKey = ? AND Users.Suspended = 0", + (keytype + " " + keytext,)) -row = cur.fetchone() -if not row or cur.fetchone(): - exit(1) + row = cur.fetchone() + if not row or cur.fetchone(): + exit(1) -user, account_type = row -if not re.match(username_regex, user): - exit(1) + user, account_type = row + if not re.match(username_regex, user): + exit(1) + env_vars = { + 'AUR_USER': user, + 'AUR_PRIVILEGED': '1' if account_type > 1 else '0', + } + key = keytype + ' ' + keytext -env_vars = { - 'AUR_USER': user, - 'AUR_PRIVILEGED': '1' if account_type > 1 else '0', -} -key = keytype + ' ' + keytext + print(format_command(env_vars, git_serve_cmd, ssh_opts, key)) -print(format_command(env_vars, git_serve_cmd, ssh_opts, key)) + +if __name__ == '__main__': + main() -- cgit v1.2.3-54-g00ecf From dc3fd60715a5b17b9542ec888c6eaeb14c284e2b Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Tue, 20 Sep 2016 20:18:24 +0200 Subject: Use setuptools to install Python modules Instead of using relative imports, add support for installing the config and db Python modules to a proper location using setuptools. Change all git-interface scripts to access those modules from the search path. Signed-off-by: Lukas Fleischer --- aurweb/__init__.py | 0 aurweb/config.py | 31 +++++++++++++++++++++++++++ aurweb/db.py | 51 +++++++++++++++++++++++++++++++++++++++++++++ git-interface/__init__.py | 0 git-interface/config.py | 31 --------------------------- git-interface/db.py | 51 --------------------------------------------- git-interface/git-auth.py | 14 ++++++------- git-interface/git-serve.py | 40 +++++++++++++++++------------------ git-interface/git-update.py | 14 ++++++------- git-interface/test/setup.sh | 4 ++++ scripts/__init__.py | 0 setup.py | 20 ++++++++++++++++++ 12 files changed, 140 insertions(+), 116 deletions(-) create mode 100644 aurweb/__init__.py create mode 100644 aurweb/config.py create mode 100644 aurweb/db.py create mode 100644 git-interface/__init__.py delete mode 100644 git-interface/config.py delete mode 100644 git-interface/db.py create mode 100644 scripts/__init__.py create mode 100644 setup.py (limited to 'git-interface/git-auth.py') diff --git a/aurweb/__init__.py b/aurweb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aurweb/config.py b/aurweb/config.py new file mode 100644 index 0000000..aac188b --- /dev/null +++ b/aurweb/config.py @@ -0,0 +1,31 @@ +import configparser +import os + +_parser = None + + +def _get_parser(): + global _parser + + if not _parser: + _parser = configparser.RawConfigParser() + if 'AUR_CONFIG' in os.environ: + path = os.environ.get('AUR_CONFIG') + else: + relpath = "/../conf/config" + path = os.path.dirname(os.path.realpath(__file__)) + relpath + _parser.read(path) + + return _parser + + +def get(section, option): + return _get_parser().get(section, option) + + +def getboolean(section, option): + return _get_parser().getboolean(section, option) + + +def getint(section, option): + return _get_parser().getint(section, option) diff --git a/aurweb/db.py b/aurweb/db.py new file mode 100644 index 0000000..0b58197 --- /dev/null +++ b/aurweb/db.py @@ -0,0 +1,51 @@ +import mysql.connector +import sqlite3 + +import aurweb.config + + +class Connection: + _conn = None + _paramstyle = None + + def __init__(self): + aur_db_backend = aurweb.config.get('database', 'backend') + + if aur_db_backend == 'mysql': + aur_db_host = aurweb.config.get('database', 'host') + aur_db_name = aurweb.config.get('database', 'name') + aur_db_user = aurweb.config.get('database', 'user') + aur_db_pass = aurweb.config.get('database', 'password') + aur_db_socket = aurweb.config.get('database', 'socket') + self._conn = mysql.connector.connect(host=aur_db_host, + user=aur_db_user, + passwd=aur_db_pass, + db=aur_db_name, + unix_socket=aur_db_socket, + buffered=True) + self._paramstyle = mysql.connector.paramstyle + elif aur_db_backend == 'sqlite': + aur_db_name = aurweb.config.get('database', 'name') + self._conn = sqlite3.connect(aur_db_name) + self._paramstyle = sqlite3.paramstyle + else: + raise ValueError('unsupported database backend') + + def execute(self, query, params=()): + if self._paramstyle in ('format', 'pyformat'): + query = query.replace('%', '%%').replace('?', '%s') + elif self._paramstyle == 'qmark': + pass + else: + raise ValueError('unsupported paramstyle') + + cur = self._conn.cursor() + cur.execute(query, params) + + return cur + + def commit(self): + self._conn.commit() + + def close(self): + self._conn.close() diff --git a/git-interface/__init__.py b/git-interface/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/git-interface/config.py b/git-interface/config.py deleted file mode 100644 index aac188b..0000000 --- a/git-interface/config.py +++ /dev/null @@ -1,31 +0,0 @@ -import configparser -import os - -_parser = None - - -def _get_parser(): - global _parser - - if not _parser: - _parser = configparser.RawConfigParser() - if 'AUR_CONFIG' in os.environ: - path = os.environ.get('AUR_CONFIG') - else: - relpath = "/../conf/config" - path = os.path.dirname(os.path.realpath(__file__)) + relpath - _parser.read(path) - - return _parser - - -def get(section, option): - return _get_parser().get(section, option) - - -def getboolean(section, option): - return _get_parser().getboolean(section, option) - - -def getint(section, option): - return _get_parser().getint(section, option) diff --git a/git-interface/db.py b/git-interface/db.py deleted file mode 100644 index 75d2283..0000000 --- a/git-interface/db.py +++ /dev/null @@ -1,51 +0,0 @@ -import mysql.connector -import sqlite3 - -import config - - -class Connection: - _conn = None - _paramstyle = None - - def __init__(self): - aur_db_backend = config.get('database', 'backend') - - if aur_db_backend == 'mysql': - aur_db_host = config.get('database', 'host') - aur_db_name = config.get('database', 'name') - aur_db_user = config.get('database', 'user') - aur_db_pass = config.get('database', 'password') - aur_db_socket = config.get('database', 'socket') - self._conn = mysql.connector.connect(host=aur_db_host, - user=aur_db_user, - passwd=aur_db_pass, - db=aur_db_name, - unix_socket=aur_db_socket, - buffered=True) - self._paramstyle = mysql.connector.paramstyle - elif aur_db_backend == 'sqlite': - aur_db_name = config.get('database', 'name') - self._conn = sqlite3.connect(aur_db_name) - self._paramstyle = sqlite3.paramstyle - else: - raise ValueError('unsupported database backend') - - def execute(self, query, params=()): - if self._paramstyle in ('format', 'pyformat'): - query = query.replace('%', '%%').replace('?', '%s') - elif self._paramstyle == 'qmark': - pass - else: - raise ValueError('unsupported paramstyle') - - cur = self._conn.cursor() - cur.execute(query, params) - - return cur - - def commit(self): - self._conn.commit() - - def close(self): - self._conn.close() diff --git a/git-interface/git-auth.py b/git-interface/git-auth.py index d3b0188..022b0ff 100755 --- a/git-interface/git-auth.py +++ b/git-interface/git-auth.py @@ -4,8 +4,8 @@ import shlex import re import sys -import config -import db +import aurweb.config +import aurweb.db def format_command(env_vars, command, ssh_opts, ssh_key): @@ -24,17 +24,17 @@ def format_command(env_vars, command, ssh_opts, ssh_key): def main(): - valid_keytypes = config.get('auth', 'valid-keytypes').split() - username_regex = config.get('auth', 'username-regex') - git_serve_cmd = config.get('auth', 'git-serve-cmd') - ssh_opts = config.get('auth', 'ssh-options') + valid_keytypes = aurweb.config.get('auth', 'valid-keytypes').split() + username_regex = aurweb.config.get('auth', 'username-regex') + git_serve_cmd = aurweb.config.get('auth', 'git-serve-cmd') + ssh_opts = aurweb.config.get('auth', 'ssh-options') keytype = sys.argv[1] keytext = sys.argv[2] if keytype not in valid_keytypes: exit(1) - conn = db.Connection() + conn = aurweb.db.Connection() cur = conn.execute("SELECT Users.Username, Users.AccountTypeID FROM Users " "INNER JOIN SSHPubKeys ON SSHPubKeys.UserID = Users.ID " diff --git a/git-interface/git-serve.py b/git-interface/git-serve.py index 8bcecd2..5f3b26d 100755 --- a/git-interface/git-serve.py +++ b/git-interface/git-serve.py @@ -7,23 +7,23 @@ import subprocess import sys import time -import config -import db +import aurweb.config +import aurweb.db -notify_cmd = config.get('notifications', 'notify-cmd') +notify_cmd = aurweb.config.get('notifications', 'notify-cmd') -repo_path = config.get('serve', 'repo-path') -repo_regex = config.get('serve', 'repo-regex') -git_shell_cmd = config.get('serve', 'git-shell-cmd') -git_update_cmd = config.get('serve', 'git-update-cmd') -ssh_cmdline = config.get('serve', 'ssh-cmdline') +repo_path = aurweb.config.get('serve', 'repo-path') +repo_regex = aurweb.config.get('serve', 'repo-regex') +git_shell_cmd = aurweb.config.get('serve', 'git-shell-cmd') +git_update_cmd = aurweb.config.get('serve', 'git-update-cmd') +ssh_cmdline = aurweb.config.get('serve', 'ssh-cmdline') -enable_maintenance = config.getboolean('options', 'enable-maintenance') -maintenance_exc = config.get('options', 'maintenance-exceptions').split() +enable_maintenance = aurweb.config.getboolean('options', 'enable-maintenance') +maintenance_exc = aurweb.config.get('options', 'maintenance-exceptions').split() def pkgbase_from_name(pkgbase): - conn = db.Connection() + conn = aurweb.db.Connection() cur = conn.execute("SELECT ID FROM PackageBases WHERE Name = ?", [pkgbase]) row = cur.fetchone() @@ -35,7 +35,7 @@ def pkgbase_exists(pkgbase): def list_repos(user): - conn = db.Connection() + conn = aurweb.db.Connection() cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) userid = cur.fetchone()[0] @@ -55,7 +55,7 @@ def create_pkgbase(pkgbase, user): if pkgbase_exists(pkgbase): die('{:s}: package base already exists: {:s}'.format(action, pkgbase)) - conn = db.Connection() + conn = aurweb.db.Connection() cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) userid = cur.fetchone()[0] @@ -81,7 +81,7 @@ def pkgbase_adopt(pkgbase, user, privileged): if not pkgbase_id: die('{:s}: package base not found: {:s}'.format(action, pkgbase)) - conn = db.Connection() + conn = aurweb.db.Connection() cur = conn.execute("SELECT ID FROM PackageBases WHERE ID = ? AND " + "MaintainerUID IS NULL", [pkgbase_id]) @@ -111,7 +111,7 @@ def pkgbase_adopt(pkgbase, user, privileged): def pkgbase_get_comaintainers(pkgbase): - conn = db.Connection() + conn = aurweb.db.Connection() cur = conn.execute("SELECT UserName FROM PackageComaintainers " + "INNER JOIN Users " + @@ -132,7 +132,7 @@ def pkgbase_set_comaintainers(pkgbase, userlist, user, privileged): if not privileged and not pkgbase_has_full_access(pkgbase, user): die('{:s}: permission denied: {:s}'.format(action, user)) - conn = db.Connection() + conn = aurweb.db.Connection() userlist_old = set(pkgbase_get_comaintainers(pkgbase)) @@ -198,7 +198,7 @@ def pkgbase_disown(pkgbase, user, privileged): comaintainers = [] new_maintainer_userid = None - conn = db.Connection() + conn = aurweb.db.Connection() # Make the first co-maintainer the new maintainer, unless the action was # enforced by a Trusted User. @@ -232,7 +232,7 @@ def pkgbase_set_keywords(pkgbase, keywords): if not pkgbase_id: die('{:s}: package base not found: {:s}'.format(action, pkgbase)) - conn = db.Connection() + conn = aurweb.db.Connection() conn.execute("DELETE FROM PackageKeywords WHERE PackageBaseID = ?", [pkgbase_id]) @@ -245,7 +245,7 @@ def pkgbase_set_keywords(pkgbase, keywords): def pkgbase_has_write_access(pkgbase, user): - conn = db.Connection() + conn = aurweb.db.Connection() cur = conn.execute("SELECT COUNT(*) FROM PackageBases " + "LEFT JOIN PackageComaintainers " + @@ -259,7 +259,7 @@ def pkgbase_has_write_access(pkgbase, user): def pkgbase_has_full_access(pkgbase, user): - conn = db.Connection() + conn = aurweb.db.Connection() cur = conn.execute("SELECT COUNT(*) FROM PackageBases " + "INNER JOIN Users " + diff --git a/git-interface/git-update.py b/git-interface/git-update.py index 36c38ae..7337341 100755 --- a/git-interface/git-update.py +++ b/git-interface/git-update.py @@ -10,15 +10,15 @@ import time import srcinfo.parse import srcinfo.utils -import config -import db +import aurweb.config +import aurweb.db -notify_cmd = config.get('notifications', 'notify-cmd') +notify_cmd = aurweb.config.get('notifications', 'notify-cmd') -repo_path = config.get('serve', 'repo-path') -repo_regex = config.get('serve', 'repo-regex') +repo_path = aurweb.config.get('serve', 'repo-path') +repo_regex = aurweb.config.get('serve', 'repo-regex') -max_blob_size = config.getint('update', 'max-blob-size') +max_blob_size = aurweb.config.getint('update', 'max-blob-size') def size_humanize(num): @@ -256,7 +256,7 @@ def main(): if refname != "refs/heads/master": die("pushing to a branch other than master is restricted") - conn = db.Connection() + conn = aurweb.db.Connection() # Detect and deny non-fast-forwards. if sha1_old != "0" * 40 and not privileged: diff --git a/git-interface/test/setup.sh b/git-interface/test/setup.sh index f9c1616..d269af6 100644 --- a/git-interface/test/setup.sh +++ b/git-interface/test/setup.sh @@ -2,6 +2,10 @@ TEST_DIRECTORY="$(pwd)" . ./sharness.sh +# Configure python search path. +PYTHONPATH="$TEST_DIRECTORY/../../" +export PYTHONPATH + # Configure paths to the Git interface scripts. GIT_AUTH="$TEST_DIRECTORY/../git-auth.py" GIT_SERVE="$TEST_DIRECTORY/../git-serve.py" diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..48eb176 --- /dev/null +++ b/setup.py @@ -0,0 +1,20 @@ +import re +from setuptools import setup, find_packages +import sys + +version = None +with open('web/lib/version.inc.php', 'r') as f: + for line in f.readlines(): + match = re.match(r'^define\("AURWEB_VERSION", "v([0-9.]+)"\);$', line) + if match: + version = match.group(1) + +if not version: + sys.stderr.write('error: Failed to parse version file!') + sys.exit(1) + +setup( + name="aurweb", + version=version, + packages=find_packages(), +) -- cgit v1.2.3-54-g00ecf From d4fe77ac572ef0e60c9ffa5f987c9cda448cf9f2 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Sat, 8 Oct 2016 14:19:11 +0200 Subject: Reorganize Git interface scripts Move the Git interface scripts from git-interface/ to aurweb/git/. Use setuptools to automatically create wrappers which can be installed using `python3 setup.py install`. Update the configuration files, the test suite as well as the INSTALL and README files to reflect these changes. Signed-off-by: Lukas Fleischer --- INSTALL | 28 ++- README | 5 +- aurweb/git/auth.py | 62 +++++++ aurweb/git/serve.py | 409 +++++++++++++++++++++++++++++++++++++++++ aurweb/git/update.py | 419 +++++++++++++++++++++++++++++++++++++++++++ conf/config.proto | 4 +- git-interface/Makefile | 18 -- git-interface/__init__.py | 0 git-interface/config.mk | 1 - git-interface/git-auth.py | 62 ------- git-interface/git-auth.sh.in | 3 - git-interface/git-serve.py | 409 ----------------------------------------- git-interface/git-update.py | 419 ------------------------------------------- setup.py | 7 + test/setup.sh | 8 +- 15 files changed, 916 insertions(+), 938 deletions(-) create mode 100755 aurweb/git/auth.py create mode 100755 aurweb/git/serve.py create mode 100755 aurweb/git/update.py delete mode 100644 git-interface/Makefile delete mode 100644 git-interface/__init__.py delete mode 100644 git-interface/config.mk delete mode 100755 git-interface/git-auth.py delete mode 100644 git-interface/git-auth.sh.in delete mode 100755 git-interface/git-serve.py delete mode 100755 git-interface/git-update.py (limited to 'git-interface/git-auth.py') diff --git a/INSTALL b/INSTALL index dab48cc..395915a 100644 --- a/INSTALL +++ b/INSTALL @@ -37,11 +37,16 @@ Setup on Arch Linux $ mysql -uaur -p AUR 1 else '0', + } + key = keytype + ' ' + keytext + + print(format_command(env_vars, git_serve_cmd, ssh_opts, key)) + + +if __name__ == '__main__': + main() diff --git a/aurweb/git/serve.py b/aurweb/git/serve.py new file mode 100755 index 0000000..ebfef94 --- /dev/null +++ b/aurweb/git/serve.py @@ -0,0 +1,409 @@ +#!/usr/bin/python3 + +import os +import re +import shlex +import subprocess +import sys +import time + +import aurweb.config +import aurweb.db + +notify_cmd = aurweb.config.get('notifications', 'notify-cmd') + +repo_path = aurweb.config.get('serve', 'repo-path') +repo_regex = aurweb.config.get('serve', 'repo-regex') +git_shell_cmd = aurweb.config.get('serve', 'git-shell-cmd') +git_update_cmd = aurweb.config.get('serve', 'git-update-cmd') +ssh_cmdline = aurweb.config.get('serve', 'ssh-cmdline') + +enable_maintenance = aurweb.config.getboolean('options', 'enable-maintenance') +maintenance_exc = aurweb.config.get('options', 'maintenance-exceptions').split() + + +def pkgbase_from_name(pkgbase): + conn = aurweb.db.Connection() + cur = conn.execute("SELECT ID FROM PackageBases WHERE Name = ?", [pkgbase]) + + row = cur.fetchone() + return row[0] if row else None + + +def pkgbase_exists(pkgbase): + return pkgbase_from_name(pkgbase) is not None + + +def list_repos(user): + conn = aurweb.db.Connection() + + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) + userid = cur.fetchone()[0] + if userid == 0: + die('{:s}: unknown user: {:s}'.format(action, user)) + + cur = conn.execute("SELECT Name, PackagerUID FROM PackageBases " + + "WHERE MaintainerUID = ?", [userid]) + for row in cur: + print((' ' if row[1] else '*') + row[0]) + conn.close() + + +def create_pkgbase(pkgbase, user): + if not re.match(repo_regex, pkgbase): + die('{:s}: invalid repository name: {:s}'.format(action, pkgbase)) + if pkgbase_exists(pkgbase): + die('{:s}: package base already exists: {:s}'.format(action, pkgbase)) + + conn = aurweb.db.Connection() + + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) + userid = cur.fetchone()[0] + if userid == 0: + die('{:s}: unknown user: {:s}'.format(action, user)) + + now = int(time.time()) + cur = conn.execute("INSERT INTO PackageBases (Name, SubmittedTS, " + + "ModifiedTS, SubmitterUID, MaintainerUID) VALUES " + + "(?, ?, ?, ?, ?)", [pkgbase, now, now, userid, userid]) + pkgbase_id = cur.lastrowid + + cur = conn.execute("INSERT INTO PackageNotifications " + + "(PackageBaseID, UserID) VALUES (?, ?)", + [pkgbase_id, userid]) + + conn.commit() + conn.close() + + +def pkgbase_adopt(pkgbase, user, privileged): + pkgbase_id = pkgbase_from_name(pkgbase) + if not pkgbase_id: + die('{:s}: package base not found: {:s}'.format(action, pkgbase)) + + conn = aurweb.db.Connection() + + cur = conn.execute("SELECT ID FROM PackageBases WHERE ID = ? AND " + + "MaintainerUID IS NULL", [pkgbase_id]) + if not privileged and not cur.fetchone(): + die('{:s}: permission denied: {:s}'.format(action, user)) + + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) + userid = cur.fetchone()[0] + if userid == 0: + die('{:s}: unknown user: {:s}'.format(action, user)) + + cur = conn.execute("UPDATE PackageBases SET MaintainerUID = ? " + + "WHERE ID = ?", [userid, pkgbase_id]) + + cur = conn.execute("SELECT COUNT(*) FROM PackageNotifications WHERE " + + "PackageBaseID = ? AND UserID = ?", + [pkgbase_id, userid]) + if cur.fetchone()[0] == 0: + cur = conn.execute("INSERT INTO PackageNotifications " + + "(PackageBaseID, UserID) VALUES (?, ?)", + [pkgbase_id, userid]) + conn.commit() + + subprocess.Popen((notify_cmd, 'adopt', str(pkgbase_id), str(userid))) + + conn.close() + + +def pkgbase_get_comaintainers(pkgbase): + conn = aurweb.db.Connection() + + cur = conn.execute("SELECT UserName FROM PackageComaintainers " + + "INNER JOIN Users " + + "ON Users.ID = PackageComaintainers.UsersID " + + "INNER JOIN PackageBases " + + "ON PackageBases.ID = PackageComaintainers.PackageBaseID " + + "WHERE PackageBases.Name = ? " + + "ORDER BY Priority ASC", [pkgbase]) + + return [row[0] for row in cur.fetchall()] + + +def pkgbase_set_comaintainers(pkgbase, userlist, user, privileged): + pkgbase_id = pkgbase_from_name(pkgbase) + if not pkgbase_id: + die('{:s}: package base not found: {:s}'.format(action, pkgbase)) + + if not privileged and not pkgbase_has_full_access(pkgbase, user): + die('{:s}: permission denied: {:s}'.format(action, user)) + + conn = aurweb.db.Connection() + + userlist_old = set(pkgbase_get_comaintainers(pkgbase)) + + uids_old = set() + for olduser in userlist_old: + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", + [olduser]) + userid = cur.fetchone()[0] + if userid == 0: + die('{:s}: unknown user: {:s}'.format(action, user)) + uids_old.add(userid) + + uids_new = set() + for newuser in userlist: + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", + [newuser]) + userid = cur.fetchone()[0] + if userid == 0: + die('{:s}: unknown user: {:s}'.format(action, user)) + uids_new.add(userid) + + uids_add = uids_new - uids_old + uids_rem = uids_old - uids_new + + i = 1 + for userid in uids_new: + if userid in uids_add: + cur = conn.execute("INSERT INTO PackageComaintainers " + + "(PackageBaseID, UsersID, Priority) " + + "VALUES (?, ?, ?)", [pkgbase_id, userid, i]) + subprocess.Popen((notify_cmd, 'comaintainer-add', str(pkgbase_id), + str(userid))) + else: + cur = conn.execute("UPDATE PackageComaintainers " + + "SET Priority = ? " + + "WHERE PackageBaseID = ? AND UsersID = ?", + [i, pkgbase_id, userid]) + i += 1 + + for userid in uids_rem: + cur = conn.execute("DELETE FROM PackageComaintainers " + + "WHERE PackageBaseID = ? AND UsersID = ?", + [pkgbase_id, userid]) + subprocess.Popen((notify_cmd, 'comaintainer-remove', + str(pkgbase_id), str(userid))) + + conn.commit() + conn.close() + + +def pkgbase_disown(pkgbase, user, privileged): + pkgbase_id = pkgbase_from_name(pkgbase) + if not pkgbase_id: + die('{:s}: package base not found: {:s}'.format(action, pkgbase)) + + initialized_by_owner = pkgbase_has_full_access(pkgbase, user) + if not privileged and not initialized_by_owner: + die('{:s}: permission denied: {:s}'.format(action, user)) + + # TODO: Support disowning package bases via package request. + # TODO: Scan through pending orphan requests and close them. + + comaintainers = [] + new_maintainer_userid = None + + conn = aurweb.db.Connection() + + # Make the first co-maintainer the new maintainer, unless the action was + # enforced by a Trusted User. + if initialized_by_owner: + comaintainers = pkgbase_get_comaintainers(pkgbase) + if len(comaintainers) > 0: + new_maintainer = comaintainers[0] + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", + [new_maintainer]) + new_maintainer_userid = cur.fetchone()[0] + comaintainers.remove(new_maintainer) + + pkgbase_set_comaintainers(pkgbase, comaintainers, user, privileged) + cur = conn.execute("UPDATE PackageBases SET MaintainerUID = ? " + + "WHERE ID = ?", [new_maintainer_userid, pkgbase_id]) + + conn.commit() + + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) + userid = cur.fetchone()[0] + if userid == 0: + die('{:s}: unknown user: {:s}'.format(action, user)) + + subprocess.Popen((notify_cmd, 'disown', str(pkgbase_id), str(userid))) + + conn.close() + + +def pkgbase_set_keywords(pkgbase, keywords): + pkgbase_id = pkgbase_from_name(pkgbase) + if not pkgbase_id: + die('{:s}: package base not found: {:s}'.format(action, pkgbase)) + + conn = aurweb.db.Connection() + + conn.execute("DELETE FROM PackageKeywords WHERE PackageBaseID = ?", + [pkgbase_id]) + for keyword in keywords: + conn.execute("INSERT INTO PackageKeywords (PackageBaseID, Keyword) " + + "VALUES (?, ?)", [pkgbase_id, keyword]) + + conn.commit() + conn.close() + + +def pkgbase_has_write_access(pkgbase, user): + conn = aurweb.db.Connection() + + cur = conn.execute("SELECT COUNT(*) FROM PackageBases " + + "LEFT JOIN PackageComaintainers " + + "ON PackageComaintainers.PackageBaseID = PackageBases.ID " + + "INNER JOIN Users " + + "ON Users.ID = PackageBases.MaintainerUID " + + "OR PackageBases.MaintainerUID IS NULL " + + "OR Users.ID = PackageComaintainers.UsersID " + + "WHERE Name = ? AND Username = ?", [pkgbase, user]) + return cur.fetchone()[0] > 0 + + +def pkgbase_has_full_access(pkgbase, user): + conn = aurweb.db.Connection() + + cur = conn.execute("SELECT COUNT(*) FROM PackageBases " + + "INNER JOIN Users " + + "ON Users.ID = PackageBases.MaintainerUID " + + "WHERE Name = ? AND Username = ?", [pkgbase, user]) + return cur.fetchone()[0] > 0 + + +def die(msg): + sys.stderr.write("{:s}\n".format(msg)) + exit(1) + + +def die_with_help(msg): + die(msg + "\nTry `{:s} help` for a list of commands.".format(ssh_cmdline)) + + +def warn(msg): + sys.stderr.write("warning: {:s}\n".format(msg)) + + +def usage(cmds): + sys.stderr.write("Commands:\n") + colwidth = max([len(cmd) for cmd in cmds.keys()]) + 4 + for key in sorted(cmds): + sys.stderr.write(" " + key.ljust(colwidth) + cmds[key] + "\n") + exit(0) + + +def main(): + user = os.environ.get('AUR_USER') + privileged = (os.environ.get('AUR_PRIVILEGED', '0') == '1') + ssh_cmd = os.environ.get('SSH_ORIGINAL_COMMAND') + ssh_client = os.environ.get('SSH_CLIENT') + + if not ssh_cmd: + die_with_help("Interactive shell is disabled.") + cmdargv = shlex.split(ssh_cmd) + action = cmdargv[0] + remote_addr = ssh_client.split(' ')[0] if ssh_client else None + + if enable_maintenance: + if remote_addr not in maintenance_exc: + die("The AUR is down due to maintenance. We will be back soon.") + + if action == 'git' and cmdargv[1] in ('upload-pack', 'receive-pack'): + action = action + '-' + cmdargv[1] + del cmdargv[1] + + if action == 'git-upload-pack' or action == 'git-receive-pack': + if len(cmdargv) < 2: + die_with_help("{:s}: missing path".format(action)) + + path = cmdargv[1].rstrip('/') + if not path.startswith('/'): + path = '/' + path + if not path.endswith('.git'): + path = path + '.git' + pkgbase = path[1:-4] + if not re.match(repo_regex, pkgbase): + die('{:s}: invalid repository name: {:s}'.format(action, pkgbase)) + + if action == 'git-receive-pack' and pkgbase_exists(pkgbase): + if not privileged and not pkgbase_has_write_access(pkgbase, user): + die('{:s}: permission denied: {:s}'.format(action, user)) + + os.environ["AUR_USER"] = user + os.environ["AUR_PKGBASE"] = pkgbase + os.environ["GIT_NAMESPACE"] = pkgbase + cmd = action + " '" + repo_path + "'" + os.execl(git_shell_cmd, git_shell_cmd, '-c', cmd) + elif action == 'set-keywords': + if len(cmdargv) < 2: + die_with_help("{:s}: missing repository name".format(action)) + pkgbase_set_keywords(cmdargv[1], cmdargv[2:]) + elif action == 'list-repos': + if len(cmdargv) > 1: + die_with_help("{:s}: too many arguments".format(action)) + list_repos(user) + elif action == 'setup-repo': + if len(cmdargv) < 2: + die_with_help("{:s}: missing repository name".format(action)) + if len(cmdargv) > 2: + die_with_help("{:s}: too many arguments".format(action)) + warn('{:s} is deprecated. ' + 'Use `git push` to create new repositories.'.format(action)) + create_pkgbase(cmdargv[1], user) + elif action == 'restore': + if len(cmdargv) < 2: + die_with_help("{:s}: missing repository name".format(action)) + if len(cmdargv) > 2: + die_with_help("{:s}: too many arguments".format(action)) + + pkgbase = cmdargv[1] + if not re.match(repo_regex, pkgbase): + die('{:s}: invalid repository name: {:s}'.format(action, pkgbase)) + + if pkgbase_exists(pkgbase): + die('{:s}: package base exists: {:s}'.format(action, pkgbase)) + create_pkgbase(pkgbase, user) + + os.environ["AUR_USER"] = user + os.environ["AUR_PKGBASE"] = pkgbase + os.execl(git_update_cmd, git_update_cmd, 'restore') + elif action == 'adopt': + if len(cmdargv) < 2: + die_with_help("{:s}: missing repository name".format(action)) + if len(cmdargv) > 2: + die_with_help("{:s}: too many arguments".format(action)) + + pkgbase = cmdargv[1] + pkgbase_adopt(pkgbase, user, privileged) + elif action == 'disown': + if len(cmdargv) < 2: + die_with_help("{:s}: missing repository name".format(action)) + if len(cmdargv) > 2: + die_with_help("{:s}: too many arguments".format(action)) + + pkgbase = cmdargv[1] + pkgbase_disown(pkgbase, user, privileged) + elif action == 'set-comaintainers': + if len(cmdargv) < 2: + die_with_help("{:s}: missing repository name".format(action)) + + pkgbase = cmdargv[1] + userlist = cmdargv[2:] + pkgbase_set_comaintainers(pkgbase, userlist, user, privileged) + elif action == 'help': + cmds = { + "adopt ": "Adopt a package base.", + "disown ": "Disown a package base.", + "help": "Show this help message and exit.", + "list-repos": "List all your repositories.", + "restore ": "Restore a deleted package base.", + "set-comaintainers [...]": "Set package base co-maintainers.", + "set-keywords [...]": "Change package base keywords.", + "setup-repo ": "Create a repository (deprecated).", + "git-receive-pack": "Internal command used with Git.", + "git-upload-pack": "Internal command used with Git.", + } + usage(cmds) + else: + die_with_help("invalid command: {:s}".format(action)) + + +if __name__ == '__main__': + main() diff --git a/aurweb/git/update.py b/aurweb/git/update.py new file mode 100755 index 0000000..7337341 --- /dev/null +++ b/aurweb/git/update.py @@ -0,0 +1,419 @@ +#!/usr/bin/python3 + +import os +import pygit2 +import re +import subprocess +import sys +import time + +import srcinfo.parse +import srcinfo.utils + +import aurweb.config +import aurweb.db + +notify_cmd = aurweb.config.get('notifications', 'notify-cmd') + +repo_path = aurweb.config.get('serve', 'repo-path') +repo_regex = aurweb.config.get('serve', 'repo-regex') + +max_blob_size = aurweb.config.getint('update', 'max-blob-size') + + +def size_humanize(num): + for unit in ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB']: + if abs(num) < 2048.0: + if isinstance(num, int): + return "{}{}".format(num, unit) + else: + return "{:.2f}{}".format(num, unit) + num /= 1024.0 + return "{:.2f}{}".format(num, 'YiB') + + +def extract_arch_fields(pkginfo, field): + values = [] + + if field in pkginfo: + for val in pkginfo[field]: + values.append({"value": val, "arch": None}) + + for arch in ['i686', 'x86_64']: + if field + '_' + arch in pkginfo: + for val in pkginfo[field + '_' + arch]: + values.append({"value": val, "arch": arch}) + + return values + + +def parse_dep(depstring): + dep, _, desc = depstring.partition(': ') + depname = re.sub(r'(<|=|>).*', '', dep) + depcond = dep[len(depname):] + + if (desc): + return (depname + ': ' + desc, depcond) + else: + return (depname, depcond) + + +def create_pkgbase(conn, pkgbase, user): + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) + userid = cur.fetchone()[0] + + now = int(time.time()) + cur = conn.execute("INSERT INTO PackageBases (Name, SubmittedTS, " + + "ModifiedTS, SubmitterUID, MaintainerUID) VALUES " + + "(?, ?, ?, ?, ?)", [pkgbase, now, now, userid, userid]) + pkgbase_id = cur.lastrowid + + cur = conn.execute("INSERT INTO PackageNotifications " + + "(PackageBaseID, UserID) VALUES (?, ?)", + [pkgbase_id, userid]) + + conn.commit() + + return pkgbase_id + + +def save_metadata(metadata, conn, user): + # Obtain package base ID and previous maintainer. + pkgbase = metadata['pkgbase'] + cur = conn.execute("SELECT ID, MaintainerUID FROM PackageBases " + "WHERE Name = ?", [pkgbase]) + (pkgbase_id, maintainer_uid) = cur.fetchone() + was_orphan = not maintainer_uid + + # Obtain the user ID of the new maintainer. + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) + user_id = int(cur.fetchone()[0]) + + # Update package base details and delete current packages. + now = int(time.time()) + conn.execute("UPDATE PackageBases SET ModifiedTS = ?, " + + "PackagerUID = ?, OutOfDateTS = NULL WHERE ID = ?", + [now, user_id, pkgbase_id]) + conn.execute("UPDATE PackageBases SET MaintainerUID = ? " + + "WHERE ID = ? AND MaintainerUID IS NULL", + [user_id, pkgbase_id]) + for table in ('Sources', 'Depends', 'Relations', 'Licenses', 'Groups'): + conn.execute("DELETE FROM Package" + table + " WHERE EXISTS (" + + "SELECT * FROM Packages " + + "WHERE Packages.PackageBaseID = ? AND " + + "Package" + table + ".PackageID = Packages.ID)", + [pkgbase_id]) + conn.execute("DELETE FROM Packages WHERE PackageBaseID = ?", [pkgbase_id]) + + for pkgname in srcinfo.utils.get_package_names(metadata): + pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) + + if 'epoch' in pkginfo and int(pkginfo['epoch']) > 0: + ver = '{:d}:{:s}-{:s}'.format(int(pkginfo['epoch']), + pkginfo['pkgver'], + pkginfo['pkgrel']) + else: + ver = '{:s}-{:s}'.format(pkginfo['pkgver'], pkginfo['pkgrel']) + + for field in ('pkgdesc', 'url'): + if field not in pkginfo: + pkginfo[field] = None + + # Create a new package. + cur = conn.execute("INSERT INTO Packages (PackageBaseID, Name, " + + "Version, Description, URL) " + + "VALUES (?, ?, ?, ?, ?)", + [pkgbase_id, pkginfo['pkgname'], ver, + pkginfo['pkgdesc'], pkginfo['url']]) + conn.commit() + pkgid = cur.lastrowid + + # Add package sources. + for source_info in extract_arch_fields(pkginfo, 'source'): + conn.execute("INSERT INTO PackageSources (PackageID, Source, " + + "SourceArch) VALUES (?, ?, ?)", + [pkgid, source_info['value'], source_info['arch']]) + + # Add package dependencies. + for deptype in ('depends', 'makedepends', + 'checkdepends', 'optdepends'): + cur = conn.execute("SELECT ID FROM DependencyTypes WHERE Name = ?", + [deptype]) + deptypeid = cur.fetchone()[0] + for dep_info in extract_arch_fields(pkginfo, deptype): + depname, depcond = parse_dep(dep_info['value']) + deparch = dep_info['arch'] + conn.execute("INSERT INTO PackageDepends (PackageID, " + + "DepTypeID, DepName, DepCondition, DepArch) " + + "VALUES (?, ?, ?, ?, ?)", + [pkgid, deptypeid, depname, depcond, deparch]) + + # Add package relations (conflicts, provides, replaces). + for reltype in ('conflicts', 'provides', 'replaces'): + cur = conn.execute("SELECT ID FROM RelationTypes WHERE Name = ?", + [reltype]) + reltypeid = cur.fetchone()[0] + for rel_info in extract_arch_fields(pkginfo, reltype): + relname, relcond = parse_dep(rel_info['value']) + relarch = rel_info['arch'] + conn.execute("INSERT INTO PackageRelations (PackageID, " + + "RelTypeID, RelName, RelCondition, RelArch) " + + "VALUES (?, ?, ?, ?, ?)", + [pkgid, reltypeid, relname, relcond, relarch]) + + # Add package licenses. + if 'license' in pkginfo: + for license in pkginfo['license']: + cur = conn.execute("SELECT ID FROM Licenses WHERE Name = ?", + [license]) + row = cur.fetchone() + if row: + licenseid = row[0] + else: + cur = conn.execute("INSERT INTO Licenses (Name) " + + "VALUES (?)", [license]) + conn.commit() + licenseid = cur.lastrowid + conn.execute("INSERT INTO PackageLicenses (PackageID, " + + "LicenseID) VALUES (?, ?)", + [pkgid, licenseid]) + + # Add package groups. + if 'groups' in pkginfo: + for group in pkginfo['groups']: + cur = conn.execute("SELECT ID FROM Groups WHERE Name = ?", + [group]) + row = cur.fetchone() + if row: + groupid = row[0] + else: + cur = conn.execute("INSERT INTO Groups (Name) VALUES (?)", + [group]) + conn.commit() + groupid = cur.lastrowid + conn.execute("INSERT INTO PackageGroups (PackageID, " + "GroupID) VALUES (?, ?)", [pkgid, groupid]) + + # Add user to notification list on adoption. + if was_orphan: + cur = conn.execute("SELECT COUNT(*) FROM PackageNotifications WHERE " + + "PackageBaseID = ? AND UserID = ?", + [pkgbase_id, user_id]) + if cur.fetchone()[0] == 0: + conn.execute("INSERT INTO PackageNotifications " + + "(PackageBaseID, UserID) VALUES (?, ?)", + [pkgbase_id, user_id]) + + conn.commit() + + +def update_notify(conn, user, pkgbase_id): + # Obtain the user ID of the new maintainer. + cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) + user_id = int(cur.fetchone()[0]) + + # Execute the notification script. + subprocess.Popen((notify_cmd, 'update', str(user_id), str(pkgbase_id))) + + +def die(msg): + sys.stderr.write("error: {:s}\n".format(msg)) + exit(1) + + +def warn(msg): + sys.stderr.write("warning: {:s}\n".format(msg)) + + +def die_commit(msg, commit): + sys.stderr.write("error: The following error " + + "occurred when parsing commit\n") + sys.stderr.write("error: {:s}:\n".format(commit)) + sys.stderr.write("error: {:s}\n".format(msg)) + exit(1) + + +def main(): + repo = pygit2.Repository(repo_path) + + user = os.environ.get("AUR_USER") + pkgbase = os.environ.get("AUR_PKGBASE") + privileged = (os.environ.get("AUR_PRIVILEGED", '0') == '1') + warn_or_die = warn if privileged else die + + if len(sys.argv) == 2 and sys.argv[1] == "restore": + if 'refs/heads/' + pkgbase not in repo.listall_references(): + die('{:s}: repository not found: {:s}'.format(sys.argv[1], + pkgbase)) + refname = "refs/heads/master" + branchref = 'refs/heads/' + pkgbase + sha1_old = sha1_new = repo.lookup_reference(branchref).target + elif len(sys.argv) == 4: + refname, sha1_old, sha1_new = sys.argv[1:4] + else: + die("invalid arguments") + + if refname != "refs/heads/master": + die("pushing to a branch other than master is restricted") + + conn = aurweb.db.Connection() + + # Detect and deny non-fast-forwards. + if sha1_old != "0" * 40 and not privileged: + walker = repo.walk(sha1_old, pygit2.GIT_SORT_TOPOLOGICAL) + walker.hide(sha1_new) + if next(walker, None) is not None: + die("denying non-fast-forward (you should pull first)") + + # Prepare the walker that validates new commits. + walker = repo.walk(sha1_new, pygit2.GIT_SORT_TOPOLOGICAL) + if sha1_old != "0" * 40: + walker.hide(sha1_old) + + # Validate all new commits. + for commit in walker: + for fname in ('.SRCINFO', 'PKGBUILD'): + if fname not in commit.tree: + die_commit("missing {:s}".format(fname), str(commit.id)) + + for treeobj in commit.tree: + blob = repo[treeobj.id] + + if isinstance(blob, pygit2.Tree): + die_commit("the repository must not contain subdirectories", + str(commit.id)) + + if not isinstance(blob, pygit2.Blob): + die_commit("not a blob object: {:s}".format(treeobj), + str(commit.id)) + + if blob.size > max_blob_size: + die_commit("maximum blob size ({:s}) exceeded".format( + size_humanize(max_blob_size)), str(commit.id)) + + metadata_raw = repo[commit.tree['.SRCINFO'].id].data.decode() + (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw) + if errors: + sys.stderr.write("error: The following errors occurred " + "when parsing .SRCINFO in commit\n") + sys.stderr.write("error: {:s}:\n".format(str(commit.id))) + for error in errors: + for err in error['error']: + sys.stderr.write("error: line {:d}: {:s}\n".format( + error['line'], err)) + exit(1) + + metadata_pkgbase = metadata['pkgbase'] + if not re.match(repo_regex, metadata_pkgbase): + die_commit('invalid pkgbase: {:s}'.format(metadata_pkgbase), + str(commit.id)) + + for pkgname in set(metadata['packages'].keys()): + pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) + + for field in ('pkgver', 'pkgrel', 'pkgname'): + if field not in pkginfo: + die_commit('missing mandatory field: {:s}'.format(field), + str(commit.id)) + + if 'epoch' in pkginfo and not pkginfo['epoch'].isdigit(): + die_commit('invalid epoch: {:s}'.format(pkginfo['epoch']), + str(commit.id)) + + if not re.match(r'[a-z0-9][a-z0-9\.+_-]*$', pkginfo['pkgname']): + die_commit('invalid package name: {:s}'.format( + pkginfo['pkgname']), str(commit.id)) + + for field in ('pkgname', 'pkgdesc', 'url'): + if field in pkginfo and len(pkginfo[field]) > 255: + die_commit('{:s} field too long: {:s}'.format(field, + pkginfo[field]), str(commit.id)) + + for field in ('install', 'changelog'): + if field in pkginfo and not pkginfo[field] in commit.tree: + die_commit('missing {:s} file: {:s}'.format(field, + pkginfo[field]), str(commit.id)) + + for field in extract_arch_fields(pkginfo, 'source'): + fname = field['value'] + if "://" in fname or "lp:" in fname: + continue + if fname not in commit.tree: + die_commit('missing source file: {:s}'.format(fname), + str(commit.id)) + + # Display a warning if .SRCINFO is unchanged. + if sha1_old not in ("0000000000000000000000000000000000000000", sha1_new): + srcinfo_id_old = repo[sha1_old].tree['.SRCINFO'].id + srcinfo_id_new = repo[sha1_new].tree['.SRCINFO'].id + if srcinfo_id_old == srcinfo_id_new: + warn(".SRCINFO unchanged. " + "The package database will not be updated!") + + # Read .SRCINFO from the HEAD commit. + metadata_raw = repo[repo[sha1_new].tree['.SRCINFO'].id].data.decode() + (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw) + + # Ensure that the package base name matches the repository name. + metadata_pkgbase = metadata['pkgbase'] + if metadata_pkgbase != pkgbase: + die('invalid pkgbase: {:s}, expected {:s}'.format(metadata_pkgbase, + pkgbase)) + + # Ensure that packages are neither blacklisted nor overwritten. + pkgbase = metadata['pkgbase'] + cur = conn.execute("SELECT ID FROM PackageBases WHERE Name = ?", [pkgbase]) + row = cur.fetchone() + pkgbase_id = row[0] if row else 0 + + cur = conn.execute("SELECT Name FROM PackageBlacklist") + blacklist = [row[0] for row in cur.fetchall()] + + cur = conn.execute("SELECT Name, Repo FROM OfficialProviders") + providers = dict(cur.fetchall()) + + for pkgname in srcinfo.utils.get_package_names(metadata): + pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) + pkgname = pkginfo['pkgname'] + + if pkgname in blacklist: + warn_or_die('package is blacklisted: {:s}'.format(pkgname)) + if pkgname in providers: + warn_or_die('package already provided by [{:s}]: {:s}'.format( + providers[pkgname], pkgname)) + + cur = conn.execute("SELECT COUNT(*) FROM Packages WHERE Name = ? " + + "AND PackageBaseID <> ?", [pkgname, pkgbase_id]) + if cur.fetchone()[0] > 0: + die('cannot overwrite package: {:s}'.format(pkgname)) + + # Create a new package base if it does not exist yet. + if pkgbase_id == 0: + pkgbase_id = create_pkgbase(conn, pkgbase, user) + + # Store package base details in the database. + save_metadata(metadata, conn, user) + + # Create (or update) a branch with the name of the package base for better + # accessibility. + branchref = 'refs/heads/' + pkgbase + repo.create_reference(branchref, sha1_new, True) + + # Work around a Git bug: The HEAD ref is not updated when using + # gitnamespaces. This can be removed once the bug fix is included in Git + # mainline. See + # http://git.661346.n2.nabble.com/PATCH-receive-pack-Create-a-HEAD-ref-for-ref-namespace-td7632149.html + # for details. + headref = 'refs/namespaces/' + pkgbase + '/HEAD' + repo.create_reference(headref, sha1_new, True) + + # Send package update notifications. + update_notify(conn, user, pkgbase_id) + + # Close the database. + cur.close() + conn.close() + + +if __name__ == '__main__': + main() diff --git a/conf/config.proto b/conf/config.proto index 21441a9..96fad80 100644 --- a/conf/config.proto +++ b/conf/config.proto @@ -46,14 +46,14 @@ RSA = SHA256:Ju+yWiMb/2O+gKQ9RJCDqvRg7l+Q95KFAeqM5sr6l2s [auth] valid-keytypes = ssh-rsa ssh-dss ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 ssh-ed25519 username-regex = [a-zA-Z0-9]+[.\-_]?[a-zA-Z0-9]+$ -git-serve-cmd = /srv/http/aurweb/git-interface/git-serve.py +git-serve-cmd = /usr/local/bin/aurweb-git-serve ssh-options = restrict [serve] repo-path = /srv/http/aurweb/aur.git/ repo-regex = [a-z0-9][a-z0-9.+_-]*$ git-shell-cmd = /usr/bin/git-shell -git-update-cmd = /srv/http/aurweb/git-interface/git-update.py +git-update-cmd = /usr/local/bin/aurweb-git-update ssh-cmdline = ssh aur@aur.archlinux.org [update] diff --git a/git-interface/Makefile b/git-interface/Makefile deleted file mode 100644 index 8865790..0000000 --- a/git-interface/Makefile +++ /dev/null @@ -1,18 +0,0 @@ -GIT_INTERFACE_DIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) - -include config.mk - -git-auth.sh: - sed 's#%GIT_INTERFACE_DIR%#$(GIT_INTERFACE_DIR)#' git-auth.sh - chmod +x git-auth.sh - -install: git-auth.sh - install -Dm0755 git-auth.sh "$(DESTDIR)$(PREFIX)/bin/aur-git-auth" - -uninstall: - rm -f "$(DESTDIR)$(PREFIX)/bin/aur-git-auth" - -clean: - rm -f git-auth.sh - -.PHONY: install uninstall clean diff --git a/git-interface/__init__.py b/git-interface/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/git-interface/config.mk b/git-interface/config.mk deleted file mode 100644 index 4d794a1..0000000 --- a/git-interface/config.mk +++ /dev/null @@ -1 +0,0 @@ -PREFIX = /usr/local diff --git a/git-interface/git-auth.py b/git-interface/git-auth.py deleted file mode 100755 index 022b0ff..0000000 --- a/git-interface/git-auth.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/python3 - -import shlex -import re -import sys - -import aurweb.config -import aurweb.db - - -def format_command(env_vars, command, ssh_opts, ssh_key): - environment = '' - for key, var in env_vars.items(): - environment += '{}={} '.format(key, shlex.quote(var)) - - command = shlex.quote(command) - command = '{}{}'.format(environment, command) - - # The command is being substituted into an authorized_keys line below, - # so we need to escape the double quotes. - command = command.replace('"', '\\"') - msg = 'command="{}",{} {}'.format(command, ssh_opts, ssh_key) - return msg - - -def main(): - valid_keytypes = aurweb.config.get('auth', 'valid-keytypes').split() - username_regex = aurweb.config.get('auth', 'username-regex') - git_serve_cmd = aurweb.config.get('auth', 'git-serve-cmd') - ssh_opts = aurweb.config.get('auth', 'ssh-options') - - keytype = sys.argv[1] - keytext = sys.argv[2] - if keytype not in valid_keytypes: - exit(1) - - conn = aurweb.db.Connection() - - cur = conn.execute("SELECT Users.Username, Users.AccountTypeID FROM Users " - "INNER JOIN SSHPubKeys ON SSHPubKeys.UserID = Users.ID " - "WHERE SSHPubKeys.PubKey = ? AND Users.Suspended = 0", - (keytype + " " + keytext,)) - - row = cur.fetchone() - if not row or cur.fetchone(): - exit(1) - - user, account_type = row - if not re.match(username_regex, user): - exit(1) - - env_vars = { - 'AUR_USER': user, - 'AUR_PRIVILEGED': '1' if account_type > 1 else '0', - } - key = keytype + ' ' + keytext - - print(format_command(env_vars, git_serve_cmd, ssh_opts, key)) - - -if __name__ == '__main__': - main() diff --git a/git-interface/git-auth.sh.in b/git-interface/git-auth.sh.in deleted file mode 100644 index 223816a..0000000 --- a/git-interface/git-auth.sh.in +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -%GIT_INTERFACE_DIR%/git-auth.py "$1" "$2" diff --git a/git-interface/git-serve.py b/git-interface/git-serve.py deleted file mode 100755 index ebfef94..0000000 --- a/git-interface/git-serve.py +++ /dev/null @@ -1,409 +0,0 @@ -#!/usr/bin/python3 - -import os -import re -import shlex -import subprocess -import sys -import time - -import aurweb.config -import aurweb.db - -notify_cmd = aurweb.config.get('notifications', 'notify-cmd') - -repo_path = aurweb.config.get('serve', 'repo-path') -repo_regex = aurweb.config.get('serve', 'repo-regex') -git_shell_cmd = aurweb.config.get('serve', 'git-shell-cmd') -git_update_cmd = aurweb.config.get('serve', 'git-update-cmd') -ssh_cmdline = aurweb.config.get('serve', 'ssh-cmdline') - -enable_maintenance = aurweb.config.getboolean('options', 'enable-maintenance') -maintenance_exc = aurweb.config.get('options', 'maintenance-exceptions').split() - - -def pkgbase_from_name(pkgbase): - conn = aurweb.db.Connection() - cur = conn.execute("SELECT ID FROM PackageBases WHERE Name = ?", [pkgbase]) - - row = cur.fetchone() - return row[0] if row else None - - -def pkgbase_exists(pkgbase): - return pkgbase_from_name(pkgbase) is not None - - -def list_repos(user): - conn = aurweb.db.Connection() - - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - userid = cur.fetchone()[0] - if userid == 0: - die('{:s}: unknown user: {:s}'.format(action, user)) - - cur = conn.execute("SELECT Name, PackagerUID FROM PackageBases " + - "WHERE MaintainerUID = ?", [userid]) - for row in cur: - print((' ' if row[1] else '*') + row[0]) - conn.close() - - -def create_pkgbase(pkgbase, user): - if not re.match(repo_regex, pkgbase): - die('{:s}: invalid repository name: {:s}'.format(action, pkgbase)) - if pkgbase_exists(pkgbase): - die('{:s}: package base already exists: {:s}'.format(action, pkgbase)) - - conn = aurweb.db.Connection() - - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - userid = cur.fetchone()[0] - if userid == 0: - die('{:s}: unknown user: {:s}'.format(action, user)) - - now = int(time.time()) - cur = conn.execute("INSERT INTO PackageBases (Name, SubmittedTS, " + - "ModifiedTS, SubmitterUID, MaintainerUID) VALUES " + - "(?, ?, ?, ?, ?)", [pkgbase, now, now, userid, userid]) - pkgbase_id = cur.lastrowid - - cur = conn.execute("INSERT INTO PackageNotifications " + - "(PackageBaseID, UserID) VALUES (?, ?)", - [pkgbase_id, userid]) - - conn.commit() - conn.close() - - -def pkgbase_adopt(pkgbase, user, privileged): - pkgbase_id = pkgbase_from_name(pkgbase) - if not pkgbase_id: - die('{:s}: package base not found: {:s}'.format(action, pkgbase)) - - conn = aurweb.db.Connection() - - cur = conn.execute("SELECT ID FROM PackageBases WHERE ID = ? AND " + - "MaintainerUID IS NULL", [pkgbase_id]) - if not privileged and not cur.fetchone(): - die('{:s}: permission denied: {:s}'.format(action, user)) - - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - userid = cur.fetchone()[0] - if userid == 0: - die('{:s}: unknown user: {:s}'.format(action, user)) - - cur = conn.execute("UPDATE PackageBases SET MaintainerUID = ? " + - "WHERE ID = ?", [userid, pkgbase_id]) - - cur = conn.execute("SELECT COUNT(*) FROM PackageNotifications WHERE " + - "PackageBaseID = ? AND UserID = ?", - [pkgbase_id, userid]) - if cur.fetchone()[0] == 0: - cur = conn.execute("INSERT INTO PackageNotifications " + - "(PackageBaseID, UserID) VALUES (?, ?)", - [pkgbase_id, userid]) - conn.commit() - - subprocess.Popen((notify_cmd, 'adopt', str(pkgbase_id), str(userid))) - - conn.close() - - -def pkgbase_get_comaintainers(pkgbase): - conn = aurweb.db.Connection() - - cur = conn.execute("SELECT UserName FROM PackageComaintainers " + - "INNER JOIN Users " + - "ON Users.ID = PackageComaintainers.UsersID " + - "INNER JOIN PackageBases " + - "ON PackageBases.ID = PackageComaintainers.PackageBaseID " + - "WHERE PackageBases.Name = ? " + - "ORDER BY Priority ASC", [pkgbase]) - - return [row[0] for row in cur.fetchall()] - - -def pkgbase_set_comaintainers(pkgbase, userlist, user, privileged): - pkgbase_id = pkgbase_from_name(pkgbase) - if not pkgbase_id: - die('{:s}: package base not found: {:s}'.format(action, pkgbase)) - - if not privileged and not pkgbase_has_full_access(pkgbase, user): - die('{:s}: permission denied: {:s}'.format(action, user)) - - conn = aurweb.db.Connection() - - userlist_old = set(pkgbase_get_comaintainers(pkgbase)) - - uids_old = set() - for olduser in userlist_old: - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", - [olduser]) - userid = cur.fetchone()[0] - if userid == 0: - die('{:s}: unknown user: {:s}'.format(action, user)) - uids_old.add(userid) - - uids_new = set() - for newuser in userlist: - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", - [newuser]) - userid = cur.fetchone()[0] - if userid == 0: - die('{:s}: unknown user: {:s}'.format(action, user)) - uids_new.add(userid) - - uids_add = uids_new - uids_old - uids_rem = uids_old - uids_new - - i = 1 - for userid in uids_new: - if userid in uids_add: - cur = conn.execute("INSERT INTO PackageComaintainers " + - "(PackageBaseID, UsersID, Priority) " + - "VALUES (?, ?, ?)", [pkgbase_id, userid, i]) - subprocess.Popen((notify_cmd, 'comaintainer-add', str(pkgbase_id), - str(userid))) - else: - cur = conn.execute("UPDATE PackageComaintainers " + - "SET Priority = ? " + - "WHERE PackageBaseID = ? AND UsersID = ?", - [i, pkgbase_id, userid]) - i += 1 - - for userid in uids_rem: - cur = conn.execute("DELETE FROM PackageComaintainers " + - "WHERE PackageBaseID = ? AND UsersID = ?", - [pkgbase_id, userid]) - subprocess.Popen((notify_cmd, 'comaintainer-remove', - str(pkgbase_id), str(userid))) - - conn.commit() - conn.close() - - -def pkgbase_disown(pkgbase, user, privileged): - pkgbase_id = pkgbase_from_name(pkgbase) - if not pkgbase_id: - die('{:s}: package base not found: {:s}'.format(action, pkgbase)) - - initialized_by_owner = pkgbase_has_full_access(pkgbase, user) - if not privileged and not initialized_by_owner: - die('{:s}: permission denied: {:s}'.format(action, user)) - - # TODO: Support disowning package bases via package request. - # TODO: Scan through pending orphan requests and close them. - - comaintainers = [] - new_maintainer_userid = None - - conn = aurweb.db.Connection() - - # Make the first co-maintainer the new maintainer, unless the action was - # enforced by a Trusted User. - if initialized_by_owner: - comaintainers = pkgbase_get_comaintainers(pkgbase) - if len(comaintainers) > 0: - new_maintainer = comaintainers[0] - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", - [new_maintainer]) - new_maintainer_userid = cur.fetchone()[0] - comaintainers.remove(new_maintainer) - - pkgbase_set_comaintainers(pkgbase, comaintainers, user, privileged) - cur = conn.execute("UPDATE PackageBases SET MaintainerUID = ? " + - "WHERE ID = ?", [new_maintainer_userid, pkgbase_id]) - - conn.commit() - - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - userid = cur.fetchone()[0] - if userid == 0: - die('{:s}: unknown user: {:s}'.format(action, user)) - - subprocess.Popen((notify_cmd, 'disown', str(pkgbase_id), str(userid))) - - conn.close() - - -def pkgbase_set_keywords(pkgbase, keywords): - pkgbase_id = pkgbase_from_name(pkgbase) - if not pkgbase_id: - die('{:s}: package base not found: {:s}'.format(action, pkgbase)) - - conn = aurweb.db.Connection() - - conn.execute("DELETE FROM PackageKeywords WHERE PackageBaseID = ?", - [pkgbase_id]) - for keyword in keywords: - conn.execute("INSERT INTO PackageKeywords (PackageBaseID, Keyword) " + - "VALUES (?, ?)", [pkgbase_id, keyword]) - - conn.commit() - conn.close() - - -def pkgbase_has_write_access(pkgbase, user): - conn = aurweb.db.Connection() - - cur = conn.execute("SELECT COUNT(*) FROM PackageBases " + - "LEFT JOIN PackageComaintainers " + - "ON PackageComaintainers.PackageBaseID = PackageBases.ID " + - "INNER JOIN Users " + - "ON Users.ID = PackageBases.MaintainerUID " + - "OR PackageBases.MaintainerUID IS NULL " + - "OR Users.ID = PackageComaintainers.UsersID " + - "WHERE Name = ? AND Username = ?", [pkgbase, user]) - return cur.fetchone()[0] > 0 - - -def pkgbase_has_full_access(pkgbase, user): - conn = aurweb.db.Connection() - - cur = conn.execute("SELECT COUNT(*) FROM PackageBases " + - "INNER JOIN Users " + - "ON Users.ID = PackageBases.MaintainerUID " + - "WHERE Name = ? AND Username = ?", [pkgbase, user]) - return cur.fetchone()[0] > 0 - - -def die(msg): - sys.stderr.write("{:s}\n".format(msg)) - exit(1) - - -def die_with_help(msg): - die(msg + "\nTry `{:s} help` for a list of commands.".format(ssh_cmdline)) - - -def warn(msg): - sys.stderr.write("warning: {:s}\n".format(msg)) - - -def usage(cmds): - sys.stderr.write("Commands:\n") - colwidth = max([len(cmd) for cmd in cmds.keys()]) + 4 - for key in sorted(cmds): - sys.stderr.write(" " + key.ljust(colwidth) + cmds[key] + "\n") - exit(0) - - -def main(): - user = os.environ.get('AUR_USER') - privileged = (os.environ.get('AUR_PRIVILEGED', '0') == '1') - ssh_cmd = os.environ.get('SSH_ORIGINAL_COMMAND') - ssh_client = os.environ.get('SSH_CLIENT') - - if not ssh_cmd: - die_with_help("Interactive shell is disabled.") - cmdargv = shlex.split(ssh_cmd) - action = cmdargv[0] - remote_addr = ssh_client.split(' ')[0] if ssh_client else None - - if enable_maintenance: - if remote_addr not in maintenance_exc: - die("The AUR is down due to maintenance. We will be back soon.") - - if action == 'git' and cmdargv[1] in ('upload-pack', 'receive-pack'): - action = action + '-' + cmdargv[1] - del cmdargv[1] - - if action == 'git-upload-pack' or action == 'git-receive-pack': - if len(cmdargv) < 2: - die_with_help("{:s}: missing path".format(action)) - - path = cmdargv[1].rstrip('/') - if not path.startswith('/'): - path = '/' + path - if not path.endswith('.git'): - path = path + '.git' - pkgbase = path[1:-4] - if not re.match(repo_regex, pkgbase): - die('{:s}: invalid repository name: {:s}'.format(action, pkgbase)) - - if action == 'git-receive-pack' and pkgbase_exists(pkgbase): - if not privileged and not pkgbase_has_write_access(pkgbase, user): - die('{:s}: permission denied: {:s}'.format(action, user)) - - os.environ["AUR_USER"] = user - os.environ["AUR_PKGBASE"] = pkgbase - os.environ["GIT_NAMESPACE"] = pkgbase - cmd = action + " '" + repo_path + "'" - os.execl(git_shell_cmd, git_shell_cmd, '-c', cmd) - elif action == 'set-keywords': - if len(cmdargv) < 2: - die_with_help("{:s}: missing repository name".format(action)) - pkgbase_set_keywords(cmdargv[1], cmdargv[2:]) - elif action == 'list-repos': - if len(cmdargv) > 1: - die_with_help("{:s}: too many arguments".format(action)) - list_repos(user) - elif action == 'setup-repo': - if len(cmdargv) < 2: - die_with_help("{:s}: missing repository name".format(action)) - if len(cmdargv) > 2: - die_with_help("{:s}: too many arguments".format(action)) - warn('{:s} is deprecated. ' - 'Use `git push` to create new repositories.'.format(action)) - create_pkgbase(cmdargv[1], user) - elif action == 'restore': - if len(cmdargv) < 2: - die_with_help("{:s}: missing repository name".format(action)) - if len(cmdargv) > 2: - die_with_help("{:s}: too many arguments".format(action)) - - pkgbase = cmdargv[1] - if not re.match(repo_regex, pkgbase): - die('{:s}: invalid repository name: {:s}'.format(action, pkgbase)) - - if pkgbase_exists(pkgbase): - die('{:s}: package base exists: {:s}'.format(action, pkgbase)) - create_pkgbase(pkgbase, user) - - os.environ["AUR_USER"] = user - os.environ["AUR_PKGBASE"] = pkgbase - os.execl(git_update_cmd, git_update_cmd, 'restore') - elif action == 'adopt': - if len(cmdargv) < 2: - die_with_help("{:s}: missing repository name".format(action)) - if len(cmdargv) > 2: - die_with_help("{:s}: too many arguments".format(action)) - - pkgbase = cmdargv[1] - pkgbase_adopt(pkgbase, user, privileged) - elif action == 'disown': - if len(cmdargv) < 2: - die_with_help("{:s}: missing repository name".format(action)) - if len(cmdargv) > 2: - die_with_help("{:s}: too many arguments".format(action)) - - pkgbase = cmdargv[1] - pkgbase_disown(pkgbase, user, privileged) - elif action == 'set-comaintainers': - if len(cmdargv) < 2: - die_with_help("{:s}: missing repository name".format(action)) - - pkgbase = cmdargv[1] - userlist = cmdargv[2:] - pkgbase_set_comaintainers(pkgbase, userlist, user, privileged) - elif action == 'help': - cmds = { - "adopt ": "Adopt a package base.", - "disown ": "Disown a package base.", - "help": "Show this help message and exit.", - "list-repos": "List all your repositories.", - "restore ": "Restore a deleted package base.", - "set-comaintainers [...]": "Set package base co-maintainers.", - "set-keywords [...]": "Change package base keywords.", - "setup-repo ": "Create a repository (deprecated).", - "git-receive-pack": "Internal command used with Git.", - "git-upload-pack": "Internal command used with Git.", - } - usage(cmds) - else: - die_with_help("invalid command: {:s}".format(action)) - - -if __name__ == '__main__': - main() diff --git a/git-interface/git-update.py b/git-interface/git-update.py deleted file mode 100755 index 7337341..0000000 --- a/git-interface/git-update.py +++ /dev/null @@ -1,419 +0,0 @@ -#!/usr/bin/python3 - -import os -import pygit2 -import re -import subprocess -import sys -import time - -import srcinfo.parse -import srcinfo.utils - -import aurweb.config -import aurweb.db - -notify_cmd = aurweb.config.get('notifications', 'notify-cmd') - -repo_path = aurweb.config.get('serve', 'repo-path') -repo_regex = aurweb.config.get('serve', 'repo-regex') - -max_blob_size = aurweb.config.getint('update', 'max-blob-size') - - -def size_humanize(num): - for unit in ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB']: - if abs(num) < 2048.0: - if isinstance(num, int): - return "{}{}".format(num, unit) - else: - return "{:.2f}{}".format(num, unit) - num /= 1024.0 - return "{:.2f}{}".format(num, 'YiB') - - -def extract_arch_fields(pkginfo, field): - values = [] - - if field in pkginfo: - for val in pkginfo[field]: - values.append({"value": val, "arch": None}) - - for arch in ['i686', 'x86_64']: - if field + '_' + arch in pkginfo: - for val in pkginfo[field + '_' + arch]: - values.append({"value": val, "arch": arch}) - - return values - - -def parse_dep(depstring): - dep, _, desc = depstring.partition(': ') - depname = re.sub(r'(<|=|>).*', '', dep) - depcond = dep[len(depname):] - - if (desc): - return (depname + ': ' + desc, depcond) - else: - return (depname, depcond) - - -def create_pkgbase(conn, pkgbase, user): - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - userid = cur.fetchone()[0] - - now = int(time.time()) - cur = conn.execute("INSERT INTO PackageBases (Name, SubmittedTS, " + - "ModifiedTS, SubmitterUID, MaintainerUID) VALUES " + - "(?, ?, ?, ?, ?)", [pkgbase, now, now, userid, userid]) - pkgbase_id = cur.lastrowid - - cur = conn.execute("INSERT INTO PackageNotifications " + - "(PackageBaseID, UserID) VALUES (?, ?)", - [pkgbase_id, userid]) - - conn.commit() - - return pkgbase_id - - -def save_metadata(metadata, conn, user): - # Obtain package base ID and previous maintainer. - pkgbase = metadata['pkgbase'] - cur = conn.execute("SELECT ID, MaintainerUID FROM PackageBases " - "WHERE Name = ?", [pkgbase]) - (pkgbase_id, maintainer_uid) = cur.fetchone() - was_orphan = not maintainer_uid - - # Obtain the user ID of the new maintainer. - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - user_id = int(cur.fetchone()[0]) - - # Update package base details and delete current packages. - now = int(time.time()) - conn.execute("UPDATE PackageBases SET ModifiedTS = ?, " + - "PackagerUID = ?, OutOfDateTS = NULL WHERE ID = ?", - [now, user_id, pkgbase_id]) - conn.execute("UPDATE PackageBases SET MaintainerUID = ? " + - "WHERE ID = ? AND MaintainerUID IS NULL", - [user_id, pkgbase_id]) - for table in ('Sources', 'Depends', 'Relations', 'Licenses', 'Groups'): - conn.execute("DELETE FROM Package" + table + " WHERE EXISTS (" + - "SELECT * FROM Packages " + - "WHERE Packages.PackageBaseID = ? AND " + - "Package" + table + ".PackageID = Packages.ID)", - [pkgbase_id]) - conn.execute("DELETE FROM Packages WHERE PackageBaseID = ?", [pkgbase_id]) - - for pkgname in srcinfo.utils.get_package_names(metadata): - pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) - - if 'epoch' in pkginfo and int(pkginfo['epoch']) > 0: - ver = '{:d}:{:s}-{:s}'.format(int(pkginfo['epoch']), - pkginfo['pkgver'], - pkginfo['pkgrel']) - else: - ver = '{:s}-{:s}'.format(pkginfo['pkgver'], pkginfo['pkgrel']) - - for field in ('pkgdesc', 'url'): - if field not in pkginfo: - pkginfo[field] = None - - # Create a new package. - cur = conn.execute("INSERT INTO Packages (PackageBaseID, Name, " + - "Version, Description, URL) " + - "VALUES (?, ?, ?, ?, ?)", - [pkgbase_id, pkginfo['pkgname'], ver, - pkginfo['pkgdesc'], pkginfo['url']]) - conn.commit() - pkgid = cur.lastrowid - - # Add package sources. - for source_info in extract_arch_fields(pkginfo, 'source'): - conn.execute("INSERT INTO PackageSources (PackageID, Source, " + - "SourceArch) VALUES (?, ?, ?)", - [pkgid, source_info['value'], source_info['arch']]) - - # Add package dependencies. - for deptype in ('depends', 'makedepends', - 'checkdepends', 'optdepends'): - cur = conn.execute("SELECT ID FROM DependencyTypes WHERE Name = ?", - [deptype]) - deptypeid = cur.fetchone()[0] - for dep_info in extract_arch_fields(pkginfo, deptype): - depname, depcond = parse_dep(dep_info['value']) - deparch = dep_info['arch'] - conn.execute("INSERT INTO PackageDepends (PackageID, " + - "DepTypeID, DepName, DepCondition, DepArch) " + - "VALUES (?, ?, ?, ?, ?)", - [pkgid, deptypeid, depname, depcond, deparch]) - - # Add package relations (conflicts, provides, replaces). - for reltype in ('conflicts', 'provides', 'replaces'): - cur = conn.execute("SELECT ID FROM RelationTypes WHERE Name = ?", - [reltype]) - reltypeid = cur.fetchone()[0] - for rel_info in extract_arch_fields(pkginfo, reltype): - relname, relcond = parse_dep(rel_info['value']) - relarch = rel_info['arch'] - conn.execute("INSERT INTO PackageRelations (PackageID, " + - "RelTypeID, RelName, RelCondition, RelArch) " + - "VALUES (?, ?, ?, ?, ?)", - [pkgid, reltypeid, relname, relcond, relarch]) - - # Add package licenses. - if 'license' in pkginfo: - for license in pkginfo['license']: - cur = conn.execute("SELECT ID FROM Licenses WHERE Name = ?", - [license]) - row = cur.fetchone() - if row: - licenseid = row[0] - else: - cur = conn.execute("INSERT INTO Licenses (Name) " + - "VALUES (?)", [license]) - conn.commit() - licenseid = cur.lastrowid - conn.execute("INSERT INTO PackageLicenses (PackageID, " + - "LicenseID) VALUES (?, ?)", - [pkgid, licenseid]) - - # Add package groups. - if 'groups' in pkginfo: - for group in pkginfo['groups']: - cur = conn.execute("SELECT ID FROM Groups WHERE Name = ?", - [group]) - row = cur.fetchone() - if row: - groupid = row[0] - else: - cur = conn.execute("INSERT INTO Groups (Name) VALUES (?)", - [group]) - conn.commit() - groupid = cur.lastrowid - conn.execute("INSERT INTO PackageGroups (PackageID, " - "GroupID) VALUES (?, ?)", [pkgid, groupid]) - - # Add user to notification list on adoption. - if was_orphan: - cur = conn.execute("SELECT COUNT(*) FROM PackageNotifications WHERE " + - "PackageBaseID = ? AND UserID = ?", - [pkgbase_id, user_id]) - if cur.fetchone()[0] == 0: - conn.execute("INSERT INTO PackageNotifications " + - "(PackageBaseID, UserID) VALUES (?, ?)", - [pkgbase_id, user_id]) - - conn.commit() - - -def update_notify(conn, user, pkgbase_id): - # Obtain the user ID of the new maintainer. - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - user_id = int(cur.fetchone()[0]) - - # Execute the notification script. - subprocess.Popen((notify_cmd, 'update', str(user_id), str(pkgbase_id))) - - -def die(msg): - sys.stderr.write("error: {:s}\n".format(msg)) - exit(1) - - -def warn(msg): - sys.stderr.write("warning: {:s}\n".format(msg)) - - -def die_commit(msg, commit): - sys.stderr.write("error: The following error " + - "occurred when parsing commit\n") - sys.stderr.write("error: {:s}:\n".format(commit)) - sys.stderr.write("error: {:s}\n".format(msg)) - exit(1) - - -def main(): - repo = pygit2.Repository(repo_path) - - user = os.environ.get("AUR_USER") - pkgbase = os.environ.get("AUR_PKGBASE") - privileged = (os.environ.get("AUR_PRIVILEGED", '0') == '1') - warn_or_die = warn if privileged else die - - if len(sys.argv) == 2 and sys.argv[1] == "restore": - if 'refs/heads/' + pkgbase not in repo.listall_references(): - die('{:s}: repository not found: {:s}'.format(sys.argv[1], - pkgbase)) - refname = "refs/heads/master" - branchref = 'refs/heads/' + pkgbase - sha1_old = sha1_new = repo.lookup_reference(branchref).target - elif len(sys.argv) == 4: - refname, sha1_old, sha1_new = sys.argv[1:4] - else: - die("invalid arguments") - - if refname != "refs/heads/master": - die("pushing to a branch other than master is restricted") - - conn = aurweb.db.Connection() - - # Detect and deny non-fast-forwards. - if sha1_old != "0" * 40 and not privileged: - walker = repo.walk(sha1_old, pygit2.GIT_SORT_TOPOLOGICAL) - walker.hide(sha1_new) - if next(walker, None) is not None: - die("denying non-fast-forward (you should pull first)") - - # Prepare the walker that validates new commits. - walker = repo.walk(sha1_new, pygit2.GIT_SORT_TOPOLOGICAL) - if sha1_old != "0" * 40: - walker.hide(sha1_old) - - # Validate all new commits. - for commit in walker: - for fname in ('.SRCINFO', 'PKGBUILD'): - if fname not in commit.tree: - die_commit("missing {:s}".format(fname), str(commit.id)) - - for treeobj in commit.tree: - blob = repo[treeobj.id] - - if isinstance(blob, pygit2.Tree): - die_commit("the repository must not contain subdirectories", - str(commit.id)) - - if not isinstance(blob, pygit2.Blob): - die_commit("not a blob object: {:s}".format(treeobj), - str(commit.id)) - - if blob.size > max_blob_size: - die_commit("maximum blob size ({:s}) exceeded".format( - size_humanize(max_blob_size)), str(commit.id)) - - metadata_raw = repo[commit.tree['.SRCINFO'].id].data.decode() - (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw) - if errors: - sys.stderr.write("error: The following errors occurred " - "when parsing .SRCINFO in commit\n") - sys.stderr.write("error: {:s}:\n".format(str(commit.id))) - for error in errors: - for err in error['error']: - sys.stderr.write("error: line {:d}: {:s}\n".format( - error['line'], err)) - exit(1) - - metadata_pkgbase = metadata['pkgbase'] - if not re.match(repo_regex, metadata_pkgbase): - die_commit('invalid pkgbase: {:s}'.format(metadata_pkgbase), - str(commit.id)) - - for pkgname in set(metadata['packages'].keys()): - pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) - - for field in ('pkgver', 'pkgrel', 'pkgname'): - if field not in pkginfo: - die_commit('missing mandatory field: {:s}'.format(field), - str(commit.id)) - - if 'epoch' in pkginfo and not pkginfo['epoch'].isdigit(): - die_commit('invalid epoch: {:s}'.format(pkginfo['epoch']), - str(commit.id)) - - if not re.match(r'[a-z0-9][a-z0-9\.+_-]*$', pkginfo['pkgname']): - die_commit('invalid package name: {:s}'.format( - pkginfo['pkgname']), str(commit.id)) - - for field in ('pkgname', 'pkgdesc', 'url'): - if field in pkginfo and len(pkginfo[field]) > 255: - die_commit('{:s} field too long: {:s}'.format(field, - pkginfo[field]), str(commit.id)) - - for field in ('install', 'changelog'): - if field in pkginfo and not pkginfo[field] in commit.tree: - die_commit('missing {:s} file: {:s}'.format(field, - pkginfo[field]), str(commit.id)) - - for field in extract_arch_fields(pkginfo, 'source'): - fname = field['value'] - if "://" in fname or "lp:" in fname: - continue - if fname not in commit.tree: - die_commit('missing source file: {:s}'.format(fname), - str(commit.id)) - - # Display a warning if .SRCINFO is unchanged. - if sha1_old not in ("0000000000000000000000000000000000000000", sha1_new): - srcinfo_id_old = repo[sha1_old].tree['.SRCINFO'].id - srcinfo_id_new = repo[sha1_new].tree['.SRCINFO'].id - if srcinfo_id_old == srcinfo_id_new: - warn(".SRCINFO unchanged. " - "The package database will not be updated!") - - # Read .SRCINFO from the HEAD commit. - metadata_raw = repo[repo[sha1_new].tree['.SRCINFO'].id].data.decode() - (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw) - - # Ensure that the package base name matches the repository name. - metadata_pkgbase = metadata['pkgbase'] - if metadata_pkgbase != pkgbase: - die('invalid pkgbase: {:s}, expected {:s}'.format(metadata_pkgbase, - pkgbase)) - - # Ensure that packages are neither blacklisted nor overwritten. - pkgbase = metadata['pkgbase'] - cur = conn.execute("SELECT ID FROM PackageBases WHERE Name = ?", [pkgbase]) - row = cur.fetchone() - pkgbase_id = row[0] if row else 0 - - cur = conn.execute("SELECT Name FROM PackageBlacklist") - blacklist = [row[0] for row in cur.fetchall()] - - cur = conn.execute("SELECT Name, Repo FROM OfficialProviders") - providers = dict(cur.fetchall()) - - for pkgname in srcinfo.utils.get_package_names(metadata): - pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) - pkgname = pkginfo['pkgname'] - - if pkgname in blacklist: - warn_or_die('package is blacklisted: {:s}'.format(pkgname)) - if pkgname in providers: - warn_or_die('package already provided by [{:s}]: {:s}'.format( - providers[pkgname], pkgname)) - - cur = conn.execute("SELECT COUNT(*) FROM Packages WHERE Name = ? " + - "AND PackageBaseID <> ?", [pkgname, pkgbase_id]) - if cur.fetchone()[0] > 0: - die('cannot overwrite package: {:s}'.format(pkgname)) - - # Create a new package base if it does not exist yet. - if pkgbase_id == 0: - pkgbase_id = create_pkgbase(conn, pkgbase, user) - - # Store package base details in the database. - save_metadata(metadata, conn, user) - - # Create (or update) a branch with the name of the package base for better - # accessibility. - branchref = 'refs/heads/' + pkgbase - repo.create_reference(branchref, sha1_new, True) - - # Work around a Git bug: The HEAD ref is not updated when using - # gitnamespaces. This can be removed once the bug fix is included in Git - # mainline. See - # http://git.661346.n2.nabble.com/PATCH-receive-pack-Create-a-HEAD-ref-for-ref-namespace-td7632149.html - # for details. - headref = 'refs/namespaces/' + pkgbase + '/HEAD' - repo.create_reference(headref, sha1_new, True) - - # Send package update notifications. - update_notify(conn, user, pkgbase_id) - - # Close the database. - cur.close() - conn.close() - - -if __name__ == '__main__': - main() diff --git a/setup.py b/setup.py index 48eb176..b64e71c 100644 --- a/setup.py +++ b/setup.py @@ -17,4 +17,11 @@ setup( name="aurweb", version=version, packages=find_packages(), + entry_points={ + 'console_scripts': [ + 'aurweb-git-auth = aurweb.git.auth:main', + 'aurweb-git-serve = aurweb.git.serve:main', + 'aurweb-git-update = aurweb.git.update:main', + ], + }, ) diff --git a/test/setup.sh b/test/setup.sh index dc9cff2..d02d298 100644 --- a/test/setup.sh +++ b/test/setup.sh @@ -8,9 +8,9 @@ PYTHONPATH="$TOPLEVEL" export PYTHONPATH # Configure paths to the Git interface scripts. -GIT_AUTH="$TOPLEVEL/git-interface/git-auth.py" -GIT_SERVE="$TOPLEVEL/git-interface/git-serve.py" -GIT_UPDATE="$TOPLEVEL/git-interface/git-update.py" +GIT_AUTH="$TOPLEVEL/aurweb/git/auth.py" +GIT_SERVE="$TOPLEVEL/aurweb/git/serve.py" +GIT_UPDATE="$TOPLEVEL/aurweb/git/update.py" MKPKGLISTS="$TOPLEVEL/scripts/mkpkglists.py" TUVOTEREMINDER="$TOPLEVEL/scripts/tuvotereminder.py" PKGMAINT="$TOPLEVEL/scripts/pkgmaint.py" @@ -38,7 +38,7 @@ reply-to = noreply@aur.archlinux.org [auth] valid-keytypes = ssh-rsa ssh-dss ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 ssh-ed25519 username-regex = [a-zA-Z0-9]+[.\-_]?[a-zA-Z0-9]+$ -git-serve-cmd = /srv/http/aurweb/git-interface/git-serve.py +git-serve-cmd = $GIT_SERVE ssh-options = restrict [serve] -- cgit v1.2.3-54-g00ecf