Merge branch 'master' into lukeshu/xbs

# Conflicts:
#	config
#	cron-jobs/db-cleanup
#	db-check-nonfree
#	db-import-archlinux-any-to-ours
#	db-import-archlinux-pkg
#	db-import-archlinux-src
#	test/lib/common.inc
#	test/test.d/create-filelists.sh
#	test/test.d/db-update.sh
#	test/test.d/testing2x.sh

15 files changed, 810 insertions, 18 deletions

diff --git a/HACKING.org b/HACKING.org
new file mode 100644
index 0000000..90db1a9
--- /dev/null
+++ b/HACKING.org
@@ -0,0 +1,22 @@
+* README-WORKING-ON.txt
+Message from aurelien.
+
+Working on #bug 5 https://labs.parabola.nu/issues/5
+
+db-update~old is the untouched file
+
+db-update is the testing file
+
+
+db-update-mailer is another (testing) one for the moment. in cron/
+* TODO.txt
+** Test Suite for clean_repo.py
+
+  - Review all repo 
+  - Remove all blacklisted packages
+  - Get pending list right
+  - Extract licenses all right
+
+** Fix db-move
+
+  - Make it use abslibre
diff --git a/TODO b/TODO
deleted file mode 100644
index 9dd4b52..0000000
--- a/TODO
+++ /dev/null
@@ -1,10 +0,0 @@
-* Test Suite for clean_repo.py
-
-  - Review all repo 
-  - Remove all blacklisted packages
-  - Get pending list right
-  - Extract licenses all right
-
-* Fix db-move
-
-  - Make it use abslibre
diff --git a/config b/config
index 79b795c..75eae37 100644
--- a/config
+++ b/config
@@ -1,4 +1,7 @@
 #!/hint/bash
+# Please try to refrain from adding new variables to this file.
+# Instead, create separate ${toolname}.conf files.  Only add a
+# variable here if multiple tools start needing the option.
 
 case "$USER" in
 	db-import-packages) _name=import-packages;;
@@ -8,6 +11,11 @@ esac
 
 FTP_BASE="/srv/repo/main"
 PKGREPOS=()
+# PKGREPOS=(
+# 	'core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing'
+# 	'libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing'
+# 	'~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan'
+# 	'nonsystemd' 'nonsystemd-testing' 'nonprism' 'nonprism-testing' 'pcr' 'kernels' 'cross' 'java')
 PKGPOOL=''
 SRCPOOL=''
 
@@ -24,11 +32,10 @@ SOURCE_CLEANUP_KEEP=14
 REQUIRE_SIGNATURE=true
 
 LOCK_DELAY=10
-LOCK_TIMEOUT=300
 
 [ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging"
 export TMPDIR="${TMPDIR:-/tmp}"
-ARCHES=(i686 x86_64)
+ARCHES=(i686 x86_64 armv7h)
 DBEXT=".db.tar.gz"
 FILESEXT=".files.tar.gz"
 PKGEXT=".pkg.tar.?z"
@@ -42,10 +49,6 @@ SRCEXT=".src.tar.gz"
 LIST="dev@lists.parabola.nu"
 FROM="dbscripts+${_name}@$(hostname -f)"
 
-# Where to send error emails, and who they are from
-LIST="maintenance@lists.parabola.nu"
-FROM="repo@parabola.nu"
-
 # Override default config with config.local
 [ -f "$(dirname "${BASH_SOURCE[0]}")/config.local" ] && . "$(dirname "${BASH_SOURCE[0]}")/config.local"
 [ -f "$(dirname "${BASH_SOURCE[0]}")/config.local.${_name}" ] && . "$(dirname "${BASH_SOURCE[0]}")/config.local.${_name}"
diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index ac0194f..d233bf6 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/python2
 #
 # check_archlinux.py
 #
diff --git a/cron-jobs/db-cleanup b/cron-jobs/db-cleanup
new file mode 100755
index 0000000..12d3615
--- /dev/null
+++ b/cron-jobs/db-cleanup
@@ -0,0 +1,70 @@
+#!/bin/bash
+# Syncs pools against themselves using database contents as filter to cleanup
+# them up
+# License: GPLv3
+
+# Principles
+# * Get repos dbs contents
+# * Make them a include list
+# * Rsync pools against themselves removing excluded files
+# * Instant cleanup!
+
+trap_exit() {
+	echo
+	error "$@"
+	exit 1
+}
+
+source "$(dirname $(dirname "$(readlink -e "$0")"))/config"
+source "$(dirname "$(readlink -e "$0")")/db-cleanup.conf"
+source "$(librelib messages)"
+
+# From makepkg
+set -E
+
+trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT
+trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT
+trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR
+
+EXTRAFLAGS=()
+"${CLEANUP_DRYRUN}" && EXTRAFLAGS+=(--dry-run)
+
+filter=$(mktemp -t "${0##*/}.XXXXXXXXXX")
+trap "rm -f -- $(printf %q "$filter")" EXIT
+
+for _repo in "${PKGREPOS[@]}"; do
+	for _arch in "${ARCHES[@]}"; do
+		msg "Getting %s-%s database" "${_repo}" "${_arch}"
+
+		dbfile="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}"
+
+		if [ ! -r "${dbfile}" ]; then
+			warning "Not found"
+			continue
+		fi
+
+		# Echo the contents into a filter file
+		bsdtar tf "${dbfile}" | \
+			cut -d'/' -f1 | \
+			sort -u | \
+			sed "s|$|*|" >> "$filter"
+
+	done
+done
+
+msg "Removing old files:"
+
+for POOL in "${PKGPOOLS[@]}" "${SRCPOOLS[@]}"; do
+	msg2 '%s' "${POOL}"
+
+	rsync "${EXTRAFLAGS[@]}" -va --delete-excluded \
+		--include-from="$filter" \
+		--exclude="*" \
+		"${FTP_BASE}/${POOL}/" \
+		"${FTP_BASE}/${POOL}/"
+done
+
+msg "Removing dead symlinks:"
+actions=(-print)
+"${CLEANUP_DRYRUN}" || actions+=(-delete)
+find -L "${FTP_BASE}/" -type l "${actions[@]}"
diff --git a/cron-jobs/db-cleanup.conf b/cron-jobs/db-cleanup.conf
new file mode 100644
index 0000000..7c2344c
--- /dev/null
+++ b/cron-jobs/db-cleanup.conf
@@ -0,0 +1,11 @@
+# Directories where packages are shared between repos
+# *relative to FTP_BASE*
+PKGPOOLS=(pool/parabola_gnu+linux # Parabola GNU/Linux-libre
+	  pool/arch_gnu+linux_arm # Arch Linux ARM
+	  pool/{packages,community} # Arch Linux
+	 )
+# Directories where sources are stored
+SRCPOOLS=(src/parabola_gnu+linux # Parabola GNU/Linux-libre
+	  src/arch_gnu+linux_arm # Arch Linux ARM
+	  sources/{packages,community} # Arch Linux
+	 )
diff --git a/db-check-nonfree b/db-check-nonfree
new file mode 100755
index 0000000..ea123fa
--- /dev/null
+++ b/db-check-nonfree
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+. "$(dirname "$(readlink -e "$0")")/config"
+. "$(dirname "$(readlink -e "$0")")/db-check-nonfree.conf"
+. "$(dirname "$(readlink -e "$0")")/db-functions"
+
+if [ $# -ge 1 ]; then
+	error "Calling %s with a specific repository is not supported" "${0##*/}"
+	exit 1
+fi
+
+# TODO: this might lock too much (architectures)
+for repo in "${repos[@]}"; do
+	for pkgarch in "${ARCHES[@]}"; do
+		repo_lock "${repo}" "${pkgarch}" || exit 1
+	done
+done
+
+msg "Check nonfree in repo:"
+nonfree=($(cut -d: -f1 "${BLACKLIST_FILE}" | sort -u))
+for repo in "${ARCHREPOS[@]}"; do
+	for pkgarch in "${ARCHES[@]}"; do
+		msg2 "%s %s" "$repo" "$pkgarch"
+		if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then
+			continue
+		fi
+		unset dbpkgs
+		unset cleanpkgs
+		cleanpkgs=()
+		dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u ))
+		for pkgname in "${dbpkgs[@]}"; do
+			if in_array "${pkgname}" "${nonfree[@]}"; then
+				cleanpkgs+=("${pkgname}")
+			fi
+		done
+		if [ ${#cleanpkgs[@]} -ge 1 ]; then
+			msg2 "Nonfree: %s" "${cleanpkgs[*]}"
+			arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}"
+		fi
+	done
+done
+
+for repo in "${repos[@]}"; do
+	for pkgarch in "${ARCHES[@]}"; do
+		repo_unlock "${repo}" "${pkgarch}"
+	done
+done
diff --git a/db-check-nonfree.conf b/db-check-nonfree.conf
new file mode 100644
index 0000000..ea3330c
--- /dev/null
+++ b/db-check-nonfree.conf
@@ -0,0 +1 @@
+BLACKLIST_FILE="$HOME/blacklist/blacklist.txt"
diff --git a/db-import-archlinux-any-to-ours b/db-import-archlinux-any-to-ours
new file mode 100755
index 0000000..ab9bb77
--- /dev/null
+++ b/db-import-archlinux-any-to-ours
@@ -0,0 +1,72 @@
+#!/bin/bash
+# Releases 'any' packages from Arch arches to ours
+
+trap_exit() {
+	echo
+	error "$@"
+	exit 1
+}
+
+source "$(dirname "$(readlink -e "$0")")/config"
+source "$(dirname "$(readlink -e "$0")")/db-import-archlinux.conf"
+source "$(librelib messages)"
+
+# From makepkg
+set -E
+
+trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT
+trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT
+trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR
+
+# The architecture to compare with
+BASEARCH='x86_64'
+
+# Traverse all Arch repos
+for _repo in "${ARCHREPOS[@]}"; do
+	msg "Processing %s..." "${_repo}"
+
+	# Find 'any' packages
+	# This is hardcoded but it could release other arches...
+	PKGS=($(find "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \
+	             -iname '*-any.pkg.tar.?z' \
+	             -printf "%f "))
+
+	if [ ${#PKGS[@]} -eq 0 ]; then
+		msg2 "No '%s' packages here" any
+		continue
+	fi
+
+	for _arch in "${OURARCHES[@]}"; do
+		msg2 "Syncing %s..." "${_arch}"
+
+		# Sync 'any' only and extract the synced packages
+		SYNCED=($(
+				rsync -av \
+					--include='*-any.pkg.tar.?z' \
+					--include='*-any.pkg.tar.?z.sig' \
+					--exclude='*' \
+					"${FTP_BASE}/${_repo}/os/${BASEARCH}/" \
+					"${FTP_BASE}/${_repo}/os/${_arch}/" 2>&1 | \
+					grep 'any\.pkg\.tar\..z$' | \
+					cut -d ' ' -f 1 ))
+
+		if [ ${#SYNCED[@]} -eq 0 ]; then
+			msg2 "Already synced (or error happened)"
+			continue
+		fi
+
+		msg2 "Synced %d packages: %s" "${#SYNCED[@]}" "${SYNCED[*]}"
+
+		msg2 "Adding to db..."
+
+		pushd "${FTP_BASE}/${_repo}/os/${_arch}/" >/dev/null
+
+		# Add the packages to the db
+		repo-add "${_repo}${DBEXT}" "${SYNCED[@]}"
+
+		popd >/dev/null 
+
+		# Avoid mixups
+		unset SYNCED PKGS
+	done
+done
diff --git a/db-import-archlinux-pkg b/db-import-archlinux-pkg
new file mode 100755
index 0000000..aae7862
--- /dev/null
+++ b/db-import-archlinux-pkg
@@ -0,0 +1,215 @@
+#!/bin/bash
+# Syncs Arch repos based on info contained in repo.db files
+# License: GPLv3
+
+# Principles
+# * Get repo.db from an Arch-like repo
+# * Generate a list of available packages
+# * Create sync whitelist (based on package blacklist)
+# * Get packages
+# * Check package signatures
+# * Check database signatures
+# * Sync repo => repo
+
+# TODO
+# * make a tarball of files used for forensics
+
+set -e
+
+# Run as `V=true db-import-pkg-archlinux` to get verbose output
+VERBOSE=${V}
+extra=()
+${VERBOSE} && extra+=(-v)
+
+WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX")
+trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT
+
+# Returns contents of a repo
+get_repos() {
+	# Exclude everything but db files
+	rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \
+		--include="*.db" \
+		--include="*${DBEXT}" \
+		--include="*.files" \
+		--include="*${FILESEXT}" \
+		--exclude="*" \
+		--delete-after \
+		"rsync://${mirror}/${mirrorpath}/" "$WORKDIR"
+}
+
+get_repo_content() {
+	# Return all contents
+	bsdtar tf "${1}" | \
+		cut -d "/" -f 1 | \
+		sort -u
+}
+
+# Prints blacklisted packages
+get_blacklist() {
+	cut -d ':' -f 1 "${BLACKLIST_FILE}"
+}
+
+# repo
+# arch
+get_repo_file() {
+	echo "${WORKDIR}/${1}/os/${2}/${1}"
+}
+
+# Process the databases and get the libre packages
+init() {
+
+	# Get the blacklisted packages
+	blacklist=($(get_blacklist))
+	# Store all the whitelist files
+	whitelists=()
+
+	msg "%d packages in blacklist" ${#blacklist[@]}
+
+	test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist"
+
+	# Sync the repos databases
+	get_repos
+
+	# Traverse all repo-arch pairs
+	for _repo in "${ARCHREPOS[@]}"; do
+		for _arch in "${ARCHARCHES[@]}"; do
+			msg "Processing %s-%s" "${_repo}" "${_arch}"
+
+			db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT}
+			files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT}
+
+			if [ ! -f "${db_file}" ]; then
+				warning "%s doesn't exist, skipping this repo-arch" "${db_file}"
+				continue
+			fi
+			if [ ! -f "${files_file}" ]; then
+				warning "%s doesn't exist, skipping this repo-arch" "${files_file}"
+				continue
+			fi
+
+			# Remove blacklisted packages and count them
+			# TODO capture all removed packages for printing on debug mode
+			msg2 "Removing blacklisted packages from %s database..." .db
+			LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \
+			|& sed -n 's/-> Removing/	&/p'
+			msg2 "Removing blacklisted packages from %s database..." .files
+			LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \
+			|& sed -n 's/-> Removing/	&/p'
+			# Get db contents
+			db=($(get_repo_content "${db_file}"))
+
+			msg2 "Process clean db for syncing..."
+
+			# Create a whitelist, add * wildcard to end
+			# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if
+			# we aren't syncing them
+			# IMPORTANT: the . in the sed command is needed because an empty
+			# whitelist would consist of a single * allowing any package to
+			# pass through
+			printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist"
+
+			msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)"
+
+			# Sync excluding everything but whitelist
+			# We delete here for cleanup
+			rsync "${extra[@]}" --no-motd -rtlH \
+				--delete-after \
+				--delete-excluded \
+				--delay-updates \
+				--include-from="/tmp/${_repo}-${_arch}.whitelist" \
+				--exclude="*" \
+				"rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/" \
+				"${FTP_BASE}/${_repo}/os/${_arch}/"
+
+			# Add a new whitelist
+			whitelists+=(/tmp/${_repo}-${_arch}.whitelist)
+
+			msg "Putting databases back in place"
+			rsync "${extra[@]}" --no-motd -rtlH \
+				--delay-updates \
+				--safe-links \
+				"${WORKDIR}/${_repo}/os/${_arch}/" \
+				"${FTP_BASE}/${_repo}/os/${_arch}/"
+
+			# Cleanup
+			unset db 
+		done
+	done
+
+
+	msg "Syncing package pool"
+	# Concatenate all whitelists, check for single *s just in case
+	cat "${whitelists[@]}" | grep -v "^\*$" | sort -u > /tmp/any.whitelist
+
+	msg2 "Retrieving %d packages from pool" "$(wc -l /tmp/any.whitelist | cut -d' ' -f1)"
+
+	# Sync
+	# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
+	# packages too
+	local pkgpool
+	for pkgpool in "${ARCHPKGPOOLS[@]}"; do
+		rsync "${extra[@]}" --no-motd -rtlH \
+			--delay-updates \
+			--safe-links \
+			--include-from=/tmp/any.whitelist \
+			--exclude="*" \
+			"rsync://${mirror}/${mirrorpath}/${pkgpool}/" \
+			"${FTP_BASE}/${pkgpool}/"
+	done
+
+	# Sync sources
+	msg "Syncing source pool"
+	#sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist
+	#msg2 "Retrieving %d sources from pool" $(wc -l < /tmp/any-src.whitelist)
+
+	# Sync
+	# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
+	# packages too
+	local srcpool
+	for srcpool in "${ARCHSRCPOOLS[@]}"; do
+		rsync "${extra[@]}" --no-motd -rtlH \
+			--delay-updates \
+			--safe-links \
+			--include-from=/tmp/any.whitelist \
+			--exclude="*" \
+			"rsync://${mirror}/${mirrorpath}/${srcpool}/" \
+			"${FTP_BASE}/${srcpool}/"
+	done
+
+	date -u +%s > "${FTP_BASE}/lastsync"
+
+	# Cleanup
+	unset blacklist whitelists _arch _repo repo_file
+}
+
+trap_exit() {
+	local signal=$1; shift
+	echo
+	error "$@"
+	trap -- "$signal"
+	kill "-$signal" "$$"
+}
+
+fatal_error() {
+	error "$@"
+	exit 1
+}
+
+source "$(dirname "$(readlink -e "$0")")/config"
+source "$(dirname "$(readlink -e "$0")")/db-import-archlinux.conf"
+source "$(librelib messages)"
+
+# Check variables presence
+for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do
+	test -z "${!var}" && fatal_error "Empty %s" "${var}"
+done
+
+# From makepkg
+set -E
+for signal in TERM HUP QUIT; do
+	trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal"
+done
+trap 'trap_exit INT "Aborted by user! Exiting..."' INT
+trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR
+
+init
diff --git a/db-import-archlinux-src b/db-import-archlinux-src
new file mode 100755
index 0000000..72171f1
--- /dev/null
+++ b/db-import-archlinux-src
@@ -0,0 +1,124 @@
+#!/bin/bash
+
+set -e
+
+FTP_BASE=/srv/repo/main
+ABSLIBRE=/srv/abslibre
+ABSGIT=/srv/git/abslibre/abslibre.git
+# Remote
+# ABSGIT=http://projects.parabolagnulinux.org/abslibre.git
+BLACKLIST=/home/repo/blacklist/blacklist.txt
+SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g --quiet'
+BLFILE=/tmp/blacklist.txt
+
+# Variables from abs.conf
+ABSROOT="/srv/abs/"
+# DON'T CHANGE. WE NEED IT FOR ABSLIBRE
+SYNCSERVER="rsync.archlinux.org"
+ARCH="i686"
+MIRRORLIST="/etc/pacman.d/mirrorlist"
+REPOS=(core extra community testing community-testing !staging !community-staging)
+
+# Steps
+# * Sync abs
+# * Download blacklist.txt
+# * Sync abslibre from abs excluding from blacklist
+# * Create repo.abs.tar.gz tarballs
+
+function sync_abs() {
+	for ARCH in any i686 x86_64; do
+		rsync ${SYNCARGS} ${SYNCSERVER}::abs/${ARCH}/ ${ABSROOT}/${ARCH} || return $?
+	done
+
+	# fix some permissions
+	find "${ABSROOT}" -type d -print0 | xargs -0 chmod 755
+	find "${ABSROOT}" -type f -print0 | xargs -0 chmod 644
+}
+
+function get_blacklist() {
+	printf ":: Updating blacklist...\t"
+	cat "${BLACKLIST}" | cut -d':' -f1 | sort -u | \
+		sed "s/^/**\//" > ${BLFILE} || {
+		printf "[FAILED]\n"
+		return 1
+	}
+
+	# Prevent using an empty blacklist
+	[ $(wc -l ${BLFILE} | cut -d " " -f1) -eq 0 ] && return 1
+
+	printf "[OK]\n"
+}
+
+function sync_abs_libre() {
+
+	# Clone ABSLibre git repo
+	rm -rf /tmp/abslibre
+	git clone "$ABSGIT" /tmp/abslibre
+
+	# Sync from ABS and then sync from ABSLibre
+	printf ":: Syncing ABSLibre...\t"
+	(rsync ${SYNCARGS} --delete-excluded \
+		--exclude-from=${BLFILE} \
+		${ABSROOT} \
+		${ABSLIBRE} \
+		&&
+		for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --quiet --exclude=.git/ /tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || {
+		printf "[FAILED]\n"
+		return 1
+	}
+
+	# fix some permissions
+	find "${ABSLIBRE}" -type d -print0 | xargs -0 chmod 755
+	find "${ABSLIBRE}" -type f -print0 | xargs -0 chmod 644
+
+	printf "[OK]\n"
+}
+
+# This part is very hacky and particular to the current setup :P
+sync_pre_mips64el() {
+	pushd /home/fauno/Repos/abslibre-pre-mips64el >/dev/null
+
+	sudo -u fauno sh -c "
+		rsync ${SYNCARGS} \
+			--exclude=.git* \
+			--exclude=community-staging \
+			--exclude=community-testing \
+			--exclude=gnome-unstable \
+			--exclude=kde-unstable \
+			--exclude=multilib \
+			--exclude=multilib-testing \
+			--exclude=multilib-staging \
+			--exclude=staging \
+			--exclude=testing \
+			${ABSLIBRE}/x86_64/ \
+			/home/fauno/Repos/abslibre-pre-mips64el/ &&
+			git add . &&
+			git commit -m \"$(date)\" -a
+		git push origin master
+		git gc
+		"
+}
+
+# Create .abs.tar.gz tarballs
+create_tarballs() {
+	for repo in ${ABSLIBRE}/{i686,x86_64}/*; do
+		baserepo=${repo##*/}
+		arch=$(basename $(dirname $repo))
+
+		# Remove the old one
+		mkdir -p $FTP_BASE/$baserepo/os/$arch/
+		rm -fv $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz
+		# Create a new one joining arch and any
+		# Remove the first part of the path (it could be $repo but any isn't hit)
+		bsdtar -czf $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz \
+			-s ":${ABSLIBRE}/[a-z0-9_]\+/[a-z]\+::" \
+			$repo/* ${ABSLIBRE}/any/${baserepo}/*
+
+	done
+}
+
+sync_abs
+get_blacklist
+sync_abs_libre
+#sync_pre_mips64el
+create_tarballs
diff --git a/db-import-archlinux.conf b/db-import-archlinux.conf
new file mode 100644
index 0000000..525fc58
--- /dev/null
+++ b/db-import-archlinux.conf
@@ -0,0 +1,19 @@
+ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing')
+ARCHPKGPOOLS=(pool/{packages,community})
+ARCHSRCPOOLS=(sources/{packages,community})
+ARCHARCHES=(i686 x86_64)
+OURARCHES=(armv7h)
+
+BLACKLIST_FILE="$HOME/blacklist/blacklist.txt"
+
+mirror="mirrors.kernel.org"
+
+## mirrors without sources folder
+#mirror="mirrors.niyawe.de"
+#mirror="mirror.nl.leaseweb.net"
+#mirror="mirror.one.com"
+#mirror="mirror.us.leaseweb.net"
+#mirror="mirror.bytemark.co.uk"
+#mirror="mirror.de.leaseweb.net"
+
+mirrorpath="archlinux"
diff --git a/db-import-archlinuxarm-pkg b/db-import-archlinuxarm-pkg
new file mode 100755
index 0000000..8d8cb3d
--- /dev/null
+++ b/db-import-archlinuxarm-pkg
@@ -0,0 +1,204 @@
+#!/bin/bash
+# Syncs Arch repos based on info contained in repo.db files
+# License: GPLv3
+
+# Principles
+# * Get repo.db from an Arch-like repo
+# * Generate a list of available packages
+# * Create sync whitelist (based on package blacklist)
+# * Get packages
+# * Check package signatures
+# * Check database signatures
+# * Sync repo => repo
+
+# TODO
+# * make a tarball of files used for forensics
+
+set -e
+
+# Run as `V=true db-import-pkg-archlinux` to get verbose output
+VERBOSE=${V}
+extra=()
+${VERBOSE} && extra+=(-v)
+
+WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX")
+trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT
+
+# Returns contents of a repo
+get_repos() {
+	# Exclude everything but db files
+	rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \
+		--include="*.db" \
+		--include="*${DBEXT}" \
+		--include="*.files" \
+		--include="*${FILESEXT}" \
+		--exclude="*" \
+		--delete-after \
+		"rsync://${mirror}/${mirrorpath}/" "$WORKDIR"
+}
+
+get_repo_content() {
+	# Return all contents
+	bsdtar tf "${1}" | \
+		cut -d "/" -f 1 | \
+		sort -u
+}
+
+# Prints blacklisted packages
+get_blacklist() {
+	cut -d ':' -f 1 "${BLACKLIST_FILE}"
+}
+
+# repo
+# arch
+get_repo_file() {
+	echo "${WORKDIR}/${2}/${1}/${1}"
+}
+
+# Process the databases and get the libre packages
+init() {
+
+	# Get the blacklisted packages
+	blacklist=($(get_blacklist))
+	# Store all the whitelist files
+	whitelists=()
+
+	msg "%d packages in blacklist" ${#blacklist[@]}
+
+	test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist"
+
+	# Sync the repos databases
+	get_repos
+
+	# Traverse all repo-arch pairs
+	for _arch in "${OURARCHES[@]}"; do
+		for _repo in "${ARMREPOS[@]}"; do
+			msg "Processing %s-%s" "${_repo}" "${_arch}"
+
+			db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT}
+			files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT}
+
+			if [ ! -f "${db_file}" ]; then
+				warning "%s doesn't exist, skipping this arch-repo" "${db_file}"
+				continue
+			fi
+			if [ ! -f "${files_file}" ]; then
+				warning "%s doesn't exist, skipping this arch-repo" "${files_file}"
+				continue
+			fi
+
+			# Remove blacklisted packages and count them
+			# TODO capture all removed packages for printing on debug mode
+			msg2 "Removing blacklisted packages from %s database..." .db
+			LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \
+			|& sed -n 's/-> Removing/	&/p'
+			msg2 "Removing blacklisted packages from %s database..." .files
+			LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \
+			|& sed -n 's/-> Removing/	&/p'
+			# Get db contents
+			db=($(get_repo_content "${db_file}"))
+
+			msg2 "Process clean db for syncing..."
+
+			# Create a whitelist, add * wildcard to end
+			# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if
+			# we aren't syncing them
+			# IMPORTANT: the . in the sed command is needed because an empty
+			# whitelist would consist of a single * allowing any package to
+			# pass through
+			printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist"
+
+			msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)"
+
+			msg2 "Retrieving %d packages to pool" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)"
+
+			# Sync excluding everything but whitelist
+			rsync "${extra[@]}" --no-motd -rtlH \
+				--delay-updates \
+				--safe-links \
+				--include-from="/tmp/${_repo}-${_arch}.whitelist" \
+				--exclude="*" \
+				"rsync://${mirror}/${mirrorpath}/${_arch}/${_repo}/" \
+				"${FTP_BASE}/${PKGPOOLARM}/"
+
+			msg "Putting databases back in place"
+			rsync "${extra[@]}" --no-motd -rtlH \
+				--delay-updates \
+				--safe-links \
+				"${WORKDIR}/${_arch}/${_repo}/" \
+				"${FTP_BASE}/${_repo}/os/${_arch}/"
+
+			# Cleanup
+			unset db 
+		done
+	done
+
+
+	msg "Generating symbolic links to pool"
+
+	for _arch in "${OURARCHES[@]}"; do
+		for _repo in "${ARMREPOS[@]}"; do
+			# Modify whitelist to search packages and create symlinks
+			sed -i "s/*/-${_arch}.pkg.tar.xz/g" "/tmp/${_repo}-${_arch}.whitelist"
+
+			msg "Putting symlinks in ${_repo}/os/${_arch}"
+
+			while read _pkgfile; do
+				# Symlink to package
+				if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}" ]; then
+					ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}" \
+					"${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}"
+				elif [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile/${_arch}/any}" ]; then
+					ln -sfv "../../../${PKGPOOLARM}/${_pkgfile/${_arch}/any}" \
+					"${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}"
+				fi
+
+				# Symlink to signature
+				if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}.sig" ]; then
+					ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}.sig" \
+					"${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}.sig"
+				elif [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile/${_arch}/any}.sig" ]; then
+					ln -sfv "../../../${PKGPOOLARM}/${_pkgfile/${_arch}/any}.sig" \
+					"${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}.sig"
+				fi
+			done < "/tmp/${_repo}-${_arch}.whitelist"
+		done
+	done
+
+	date -u +%s > "${FTP_BASE}/lastsync"
+
+	# Cleanup
+	unset blacklist whitelists _arch _repo repo_file _pkgfile
+}
+
+trap_exit() {
+	local signal=$1; shift
+	echo
+	error "$@"
+	trap -- "$signal"
+	kill "-$signal" "$$"
+}
+
+fatal_error() {
+	error "$@"
+	exit 1
+}
+
+source "$(dirname "$(readlink -e "$0")")/config"
+source "$(dirname "$(readlink -e "$0")")/db-import-archlinuxarm.conf"
+source "$(librelib messages)"
+
+# Check variables presence
+for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE; do
+	test -z "${!var}" && fatal_error "Empty %s" "${var}"
+done
+
+# From makepkg
+set -E
+for signal in TERM HUP QUIT; do
+	trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal"
+done
+trap 'trap_exit INT "Aborted by user! Exiting..."' INT
+trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR
+
+init
diff --git a/db-import-archlinuxarm.conf b/db-import-archlinuxarm.conf
new file mode 100644
index 0000000..4a12a25
--- /dev/null
+++ b/db-import-archlinuxarm.conf
@@ -0,0 +1,13 @@
+ARMREPOS=('core' 'extra' 'community')
+PKGPOOLARM='pool/arch_gnu+linux_arm'
+OURARCHES=(armv7h)
+
+BLACKLIST_FILE="$HOME/blacklist/blacklist.txt"
+
+#mirror="mirror.yandex.ru"
+mirror="ftp.halifax.rwth-aachen.de"
+
+## mirrors without sources folder
+## use "archlinuxarm" instead "archlinux-arm" to mirror.yandex.ru
+
+mirrorpath="archlinux-arm"
diff --git a/db-update b/db-update
index c55869e..e98320f 100755
--- a/db-update
+++ b/db-update
@@ -9,7 +9,7 @@ if [ $# -ge 1 ]; then
 fi
 
 # Find repos with packages to release
-staging_repos=($(find  "${STAGING}" -mindepth 1 -type f -name "*${PKGEXT}" -printf '%h\n' | sort -u))
+staging_repos=($(find  "${STAGING}" -mindepth 1 -maxdepth 3 -type f -name "*${PKGEXT}" -printf '%h\n' | sort -u))
 if [ $? -ge 1 ]; then
 	die "Could not read %s" "${STAGING}"
 fi
@@ -20,6 +20,7 @@ for staging_repo in "${staging_repos[@]##*/}"; do
 		repos+=("${staging_repo}")
 	fi
 done
+repos=($(echo "${repos[@]}" | tr " " "\n" | sort -u))
 
 # TODO: this might lock too much (architectures)
 for repo in "${repos[@]}"; do