Replace check_archlinux.py by check_archlinux/check_packages.py

The old script had several problems so I decided to do a full rewrite.

The improvements include :

* better and safer parsing of PKGBUILDs
It now uses separate parse_pkgbuilds.sh bash script (inspired from namcap)

* much better performance
A python module for calling vercmp natively, and the algorithm for checking
circular dependencies was greatly improved

* more accurate dependency and provision handling
Now versioned dependencies and provisions are handled correctly.

After building the python module and moving it next to the main script, it
should be possible to use it like this :

For core and extra :
./check_packages.py --abs-tree=/home/abs/rsync/i686 --repos=core,extra
./check_packages.py --abs-tree=/home/abs/rsync/x86_64 --repos=core,extra

For community :
./check_packages.py --abs-tree=/home/abs/rsync/i686 --repos=community
./check_packages.py --abs-tree=/home/abs/rsync/x86_64 --repos=community

Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>

5 files changed, 528 insertions, 0 deletions

diff --git a/cron-jobs/check_archlinux/README b/cron-jobs/check_archlinux/README
new file mode 100644
index 0000000..f3a1b90
--- /dev/null
+++ b/cron-jobs/check_archlinux/README
@@ -0,0 +1,8 @@
+1) Build the python module
+$ python setup.py build
+
+2) copy it back to the current working directory
+$ cp build/lib.*/alpm.* .
+
+3) run the script
+$ ./check_packages.py -h
diff --git a/cron-jobs/check_archlinux/alpm.c b/cron-jobs/check_archlinux/alpm.c
new file mode 100644
index 0000000..0b7cd2c
--- /dev/null
+++ b/cron-jobs/check_archlinux/alpm.c
@@ -0,0 +1,40 @@
+#include <Python.h>
+#include <alpm.h>
+
+static PyObject *
+alpm_vercmp(PyObject *self, PyObject *args)
+{
+	const char *v1, *v2;
+	int ret;
+
+	if (!PyArg_ParseTuple(args, "ss", &v1, &v2))
+		return NULL;
+	ret = alpm_pkg_vercmp(v1, v2);
+	return Py_BuildValue("i", ret);
+}
+
+static PyMethodDef AlpmMethods[] = {
+	{"vercmp",  alpm_vercmp, METH_VARARGS,
+		"Execute vercmp."},
+	{NULL, NULL, 0, NULL}        /* Sentinel */
+};
+
+PyMODINIT_FUNC
+initalpm(void)
+{
+	(void) Py_InitModule("alpm", AlpmMethods);
+}
+
+int
+main(int argc, char *argv[])
+{
+	/* Pass argv[0] to the Python interpreter */
+	Py_SetProgramName(argv[0]);
+
+	/* Initialize the Python interpreter.  Required. */
+	Py_Initialize();
+
+	/* Add a static module */
+	initalpm();
+	return 0;
+}
diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
new file mode 100755
index 0000000..2d2efbe
--- /dev/null
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -0,0 +1,392 @@
+#!/usr/bin/python
+#
+# check_archlinux.py
+#
+# Original script by Scott Horowitz <stonecrest@gmail.com>
+# Rewritten by Xavier Chantry <shiningxc@gmail.com>
+#
+# This script currently checks for a number of issues in your ABS tree:
+#   1. Directories with missing PKGBUILDS
+#   2. Invalid PKGBUILDs (bash syntax error for instance)
+#   3. PKGBUILD names that don't match their directory
+#   4. Duplicate PKGBUILDs
+#   5. Valid arch's in PKGBUILDS
+#   6. Missing (make-)dependencies
+#   7. Hierarchy of repos (e.g., that a core package doesn't depend on
+#      a non-core package)
+#   8. Circular dependencies
+
+import os,re,commands,getopt,sys,alpm
+import pdb
+
+packages = {} # pkgname : PacmanPackage
+provisions = {} # provision : PacmanPackage
+pkgdeps,makepkgdeps = {},{} # pkgname : list of the PacmanPackage dependencies
+invalid_pkgbuilds = []
+missing_pkgbuilds = []
+dups = []
+
+mismatches = []
+missing_deps = []
+missing_makedeps = []
+invalid_archs = []
+dep_hierarchy = []
+makedep_hierarchy = []
+circular_deps = [] # pkgname>dep1>dep2>...>pkgname
+checked_deps = []
+
+class PacmanPackage:
+	def __init__(self):
+		self.name,self.version = "",""
+		self.path,self.repo = "",""
+		self.deps,self.makedeps = [],[]
+		self.provides,self.conflicts = [],[]
+		self.archs = []
+
+class Depend:
+	def __init__(self,name,version,mod):
+		self.name = name
+		self.version = version
+		self.mod = mod
+
+def parse_pkgbuilds(repos):
+	oldcwd = os.getcwd()
+	os.chdir(absroot)
+	for repo in repos:
+		data = commands.getoutput(oldcwd + '/parse_pkgbuilds.sh ' + repo)
+		parse_data(repo,data)
+	os.chdir(oldcwd)
+
+def parse_data(repo,data):
+	attrname = None
+
+	for line in data.split('\n'):
+		if line.startswith('%'):
+			attrname = line.strip('%').lower()
+		elif line.strip() == '':
+			attrname = None
+		elif attrname == "invalid":
+			if repo in repos:
+				invalid_pkgbuilds.append(line)
+		elif attrname == "missing":
+			if repo in repos:
+				missing_pkgbuilds.append(line)
+		elif attrname == "name":
+			pkg = PacmanPackage()
+			pkg.name = line
+			pkg.repo = repo
+			dup = None
+			if packages.has_key(pkg.name):
+				dup = packages[pkg.name]
+			packages[pkg.name] = pkg
+		elif attrname == "version":
+			pkg.version = line
+		elif attrname == "path":
+			pkg.path = line
+			if dup != None and (pkg.repo in repos or dup.repo in repos):
+				dups.append(pkg.path + " vs. " + dup.path)
+		elif attrname == "arch":
+			pkg.archs.append(line)
+		elif attrname == "depends":
+			pkg.deps.append(line)
+		elif attrname == "makedepends":
+			pkg.makedeps.append(line)
+		elif attrname == "conflicts":
+			pkg.conflicts.append(line)
+		elif attrname == "provides":
+			pkg.provides.append(line)
+			provname=line.split("=")[0]
+			if not provisions.has_key(provname):
+				provisions[provname] = []
+			provisions[provname].append(pkg)
+
+def splitdep(dep):
+	name = dep
+	version = ""
+	mod = ""
+	for char in (">=", "<=", "=", ">", "<"):
+		pos = dep.find(char)
+		if pos > -1:
+			name = dep[:pos]
+			version = dep[pos:].replace(char, "")
+			mod = char
+			break
+	return Depend(name,version,mod)
+
+def splitprov(prov):
+	name = prov
+	version = ""
+	pos = prov.find("=")
+	if pos > -1:
+		name = prov[:pos]
+		version = prov[pos:].replace("=", "")
+	return (name,version)
+
+def vercmp(v1,mod,v2):
+	res = alpm.vercmp(v1,v2)
+	if res == 0:
+		return (mod.find("=") > -1)
+	elif res < 0:
+		return (mod.find("<") > -1)
+	elif res > 0:
+		return (mod.find(">") > -1)
+	return False
+
+
+def depcmp(name,version,dep):
+	if name != dep.name:
+		return False
+	if dep.version == "" or dep.mod == "":
+		return True
+	if version == "":
+		return False
+	return vercmp(version,dep.mod,dep.version)
+
+def provcmp(pkg,dep):
+	for prov in pkg.provides:
+		(provname,provver) = splitprov(prov)
+		if depcmp(provname,provver,dep):
+			return True
+	return False
+
+def verify_dep(dep):
+	dep = splitdep(dep)
+	if packages.has_key(dep.name):
+		pkg = packages[dep.name]
+		if depcmp(pkg.name,pkg.version,dep):
+			return [pkg]
+	if provisions.has_key(dep.name):
+		provlist = provisions[dep.name]
+		results = []
+		for prov in provlist:
+			if provcmp(prov,dep):
+				results.append(prov)
+		return results
+	return []
+
+def verify_deps(name,repo,deps):
+	pkg_deps = []
+	missdeps = []
+	hierarchy = []
+	for dep in deps:
+		pkglist = verify_dep(dep)
+		if pkglist == []:
+			missdeps.append(name + " --> '" + dep + "'")
+		else:
+			valid_repos = get_repo_hierarchy(repo)
+			pkgdep = None
+			for pkg in pkglist:
+				if pkg.repo in valid_repos:
+					pkgdep = pkg
+					break
+			if not pkgdep:
+				pkgdep = pkglist[0]
+				hierarchy.append(repo + "/" + name + " depends on " + pkgdep.repo + "/" + pkgdep.name)
+			pkg_deps.append(pkgdep)
+
+	return (pkg_deps,missdeps,hierarchy)
+
+def get_repo_hierarchy(repo):
+	repo_hierarchy = {'core': ['core'], \
+		'extra': ['core', 'extra'], \
+		'community': ['core', 'extra', 'community']}
+	if repo_hierarchy.has_key(repo):
+		return repo_hierarchy[repo]
+	else:
+		return ['core','extra','community']
+
+def verify_archs(name,archs):
+	valid_archs = ['i686', 'x86_64']
+	invalid_archs = []
+	for arch in archs:
+		if arch not in valid_archs:
+			invalid_archs.append(name + " --> " + arch)
+	return invalid_archs
+
+def find_scc(packages):
+	# reset all variables
+	global index,S,pkgindex,pkglowlink
+	index = 0
+	S = []
+	pkgindex = {}
+	pkglowlink = {}
+	cycles = []
+	for pkg in packages:
+		tarjan(pkg)
+
+def tarjan(pkg):
+	global index,S,pkgindex,pkglowlink,cycles
+	pkgindex[pkg] = index
+	pkglowlink[pkg] = index
+	index += 1
+	checked_deps.append(pkg)
+	S.append(pkg)
+	if pkgdeps.has_key(pkg):
+		deps = pkgdeps[pkg]
+	else:
+		print pkg.name
+		deps = []
+	for dep in deps:
+		if not pkgindex.has_key(dep):
+			tarjan(dep)
+			pkglowlink[pkg] = min(pkglowlink[pkg],pkglowlink[dep])
+		elif dep in S:
+			pkglowlink[pkg] = min(pkglowlink[pkg],pkgindex[dep])
+	if pkglowlink[pkg] == pkgindex[pkg]:
+		dep = S.pop()
+		if pkg == dep:
+			return
+		path = pkg.name
+		while pkg != dep:
+			path = dep.name + ">" + path
+			dep = S.pop()
+		path = dep.name + ">" + path
+		if pkg.repo in repos:
+			circular_deps.append(path)
+
+def print_heading(heading):
+	print ""
+	print "=" * (len(heading) + 4)
+	print "= " + heading + " ="
+	print "=" * (len(heading) + 4)
+
+def print_subheading(subheading):
+	print ""
+	print subheading
+	print "-" * (len(subheading) + 2)
+
+def print_missdeps(pkgname,missdeps) :
+	for d in missdeps:
+		print pkgname + " : " + d
+
+def print_result(list, subheading):
+	if len(list) > 0:
+		print_subheading(subheading)
+		for item in list:
+			print item
+
+def print_results():
+	print_result(missing_pkgbuilds, "Missing PKGBUILDs")
+	print_result(invalid_pkgbuilds, "Invalid PKGBUILDs")
+	print_result(mismatches, "Mismatched Pkgnames")
+	print_result(dups, "Duplicate PKGBUILDs")
+	print_result(invalid_archs, "Invalid Archs")
+	print_result(missing_deps, "Missing Dependencies")
+	print_result(missing_makedeps, "Missing Makedepends")
+	print_result(dep_hierarchy, "Repo Hierarchy for Dependencies")
+	print_result(makedep_hierarchy, "Repo Hierarchy for Makedepends")
+	print_result(circular_deps, "Circular Dependencies")
+	print_subheading("Summary")
+	print "Missing PKGBUILDs:                    ", len(missing_pkgbuilds)
+	print "Invalid PKGBUILDs:                    ", len(invalid_pkgbuilds)
+	print "Mismatching PKGBUILD names:           ", len(mismatches)
+	print "Duplicate PKGBUILDs:                  ", len(dups)
+	print "Invalid archs:                        ", len(invalid_archs)
+	print "Missing (make)dependencies:           ", len(missing_deps)+len(missing_makedeps)
+	print "Repo hierarchy problems:              ", len(dep_hierarchy)+len(makedep_hierarchy)
+	print "Circular dependencies:                ", len(circular_deps)
+	print ""
+
+def print_usage():
+	print ""
+	print "Usage: ./check_packages.py [OPTION]"
+	print ""
+	print "Options:"
+	print "  --abs-tree=<path>             Check specified tree (default : /var/abs)"
+	print "  --repos=<r1,r2,...>           Check specified repos (default : core,extra)"
+	print "  -h, --help                    Show this help and exit"
+	print ""
+	print "Examples:"
+	print "\n  Check core and extra in existing abs tree:"
+	print "    ./check_packages.py --abs-tree=/var/abs --repos=core,extra"
+	print "\n  Check community:"
+	print "    ./check_packages.py --abs-tree=/var/abs --repos=community"
+	print ""
+
+## Default path to the abs root directory
+absroot = "/var/abs"
+## Default list of repos to check
+repos = ['core', 'extra']
+
+try:
+	opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos="])
+except getopt.GetoptError:
+	print_usage()
+	sys.exit()
+if opts != []:
+	for o, a in opts:
+		if o in ("--abs-tree"):
+			absroot = a
+		elif o in ("--repos"):
+			repos = a.split(",")
+		else:
+			print_usage()
+			sys.exit()
+		if args != []:
+			print_usage()
+			sys.exit()
+
+if not os.path.isdir(absroot):
+	print "Error : the abs tree " + absroot + " does not exist"
+	sys.exit()
+for repo in repos:
+	repopath = absroot + "/" + repo
+	if not os.path.isdir(repopath):
+		print "Error : the repository " + repo + " does not exist in " + absroot
+		sys.exit()
+# repos which need to be loaded
+loadrepos = set([])
+for repo in repos:
+	loadrepos = loadrepos | set(get_repo_hierarchy(repo))
+
+print_heading("Integrity Check")
+print "\nPerforming integrity checks..."
+
+print "==> parsing pkgbuilds"
+parse_pkgbuilds(loadrepos)
+
+repopkgs = {}
+for name,pkg in packages.iteritems():
+	if pkg.repo in repos:
+		repopkgs[name] = pkg
+
+print "==> checking mismatches"
+for name,pkg in repopkgs.iteritems():
+	pkgdirname = pkg.path.split("/")[-1]
+	if name != pkgdirname:
+		mismatches.append(name + " vs. " + pkg.path)
+
+print "==> checking archs"
+for name,pkg in repopkgs.iteritems():
+	archs = verify_archs(name,pkg.archs)
+	invalid_archs.extend(archs)
+
+# ugly hack to strip the weird kblic- deps
+for name,pkg in packages.iteritems():
+	p = re.compile('klibc-[A-Za-z0-9]{20,}|klibc-\*')
+	pkg.deps = [dep for dep in pkg.deps if not p.match(dep)]
+	pkg.makedeps = [dep for dep in pkg.makedeps if not p.match(dep)]
+
+print "==> checking dependencies"
+for name,pkg in repopkgs.iteritems():
+	(deps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.deps)
+	pkgdeps[pkg] = deps
+	missing_deps.extend(missdeps)
+	dep_hierarchy.extend(hierarchy)
+
+print "==> checking makedepends"
+for name,pkg in repopkgs.iteritems():
+	(makedeps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.makedeps)
+	makepkgdeps[pkg] = makedeps
+	missing_makedeps.extend(missdeps)
+	makedep_hierarchy.extend(hierarchy)
+
+print "==> checking for circular dependencies"
+# make sure pkgdeps is filled for every package
+for name,pkg in packages.iteritems():
+	if not pkgdeps.has_key(pkg):
+		(deps,missdeps,_) = verify_deps(name,pkg.repo,pkg.deps)
+		pkgdeps[pkg] = deps
+find_scc(repopkgs.values())
+
+print_results()
diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
new file mode 100755
index 0000000..d4205ae
--- /dev/null
+++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+parse() {
+	unset pkgname pkgver pkgrel
+	unset depends makedepends conflicts provides
+	ret=0
+	dir=$1
+	pkgbuild=$dir/PKGBUILD
+	source $pkgbuild &>/dev/null || ret=$?
+
+	# ensure $pkgname and $pkgver variables were found
+	if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then
+		echo -e "%INVALID%\n$pkgbuild\n"
+		return 1
+	fi 
+
+	echo -e "%NAME%\n$pkgname\n"
+	echo -e "%VERSION%\n$pkgver-$pkgrel\n"
+	echo -e "%PATH%\n$dir\n"
+
+	if [ -n "$arch" ]; then
+		echo "%ARCH%"
+		for i in ${arch[@]}; do echo $i; done
+		echo ""
+	fi
+	if [ -n "$depends" ]; then
+		echo "%DEPENDS%"
+		for i in ${depends[@]}; do
+			echo $i
+		done
+		echo ""
+	fi
+	if [ -n "$makedepends" ]; then
+		echo "%MAKEDEPENDS%"
+		for i in ${makedepends[@]}; do
+			echo $i
+		done
+		echo ""
+	fi
+	if [ -n "$conflicts" ]; then
+		echo "%CONFLICTS%"
+		for i in ${conflicts[@]}; do echo $i; done
+		echo ""
+	fi
+	if [ -n "$provides" ]; then
+		echo "%PROVIDES%"
+		for i in ${provides[@]}; do echo $i; done
+		echo ""
+	fi
+	return 0
+}
+
+find_pkgbuilds() {
+	if [ -f $1/PKGBUILD ]; then
+		parse $1
+		return
+	fi
+	empty=1
+	for dir in $1/*; do
+		if [ -d $dir ]; then
+			find_pkgbuilds $dir
+			unset empty
+		fi
+	done
+	if [ -n "$empty" ]; then
+		echo -e "%MISSING%\n$1\n"
+	fi
+}
+
+if [ -z "$*" ]; then
+	exit 1
+fi
+
+for dir in "$@"; do
+	find_pkgbuilds $dir
+done
+
+exit 0
diff --git a/cron-jobs/check_archlinux/setup.py b/cron-jobs/check_archlinux/setup.py
new file mode 100644
index 0000000..b172752
--- /dev/null
+++ b/cron-jobs/check_archlinux/setup.py
@@ -0,0 +1,10 @@
+from distutils.core import setup, Extension
+
+alpm = Extension('alpm',
+		libraries = ['alpm'],
+		sources = ['alpm.c'])
+
+setup (name = 'Alpm',
+		version = '1.0',
+		description = 'Alpm bindings',
+		ext_modules = [alpm])