fix new db-sync

2 files changed, 144 insertions, 328 deletions

diff --git a/db-sync b/db-sync
index 81dee24..c90f89b 100755
--- a/db-sync
+++ b/db-sync
@@ -13,174 +13,181 @@
 
 # TODO
 # * make a tarball of files used for forensics
-# * get files db
 
 # Run as `V=true db-sync` to get verbose output
 VERBOSE=${V}
 ${VERBOSE} && extra="-v"
 
+WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX")
+trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT
+
 # Returns contents of a repo
 get_repos() {
-        mkdir -p ${TMPDIR}/$0.$$.cache
-# Exclude everything but db files
-  rsync ${extra} -mrtlH --no-p --include="*/" \
-        --include="*.db" \
-        --include="*${DBEXT}" \
-        --exclude="*" \
-        --delete-after \
-        rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache
+	# Exclude everything but db files
+	rsync ${extra} --no-motd -mrtlH --no-p --include="*/" \
+		--include="*.db" \
+		--include="*${DBEXT}" \
+		--include="*.files" \
+		--include="*${FILESEXT}" \
+		--exclude="*" \
+		--delete-after \
+		rsync://${mirror}/${mirrorpath}/ "$WORKDIR"
 }
 
 get_repo_content() {
-# Return all contents
-  bsdtar tf ${1} | \
-    cut -d "/" -f 1 | \
-    sort -u
+	# Return all contents
+	bsdtar tf ${1} | \
+		cut -d "/" -f 1 | \
+		sort -u
 }
 
 # Prints blacklisted packages
 get_blacklist() {
-  cut -d ':' -f 1 "${BLACKLIST_FILE}"
+	cut -d ':' -f 1 "${BLACKLIST_FILE}"
 }
 
 # repo
 # arch
 get_repo_file() {
-#  shopt -s nullglob
-
-  echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}${DBEXT}"
+	echo "${WORKDIR}/${1}/os/${2}/${1}"
 }
 
 # Process the databases and get the libre packages
 init() {
 
-# Get the blacklisted packages
-  blacklist=($(get_blacklist))
-# Store all the whitelist files
-  whitelists=()
-
-  msg "${#blacklist[@]} packages in blacklist"
-
-# Sync the repos databases
-  get_repos
-
-# Traverse all repo-arch pairs
-  for _repo in ${ARCHREPOS[@]}; do
-    for _arch in ${ARCHARCHES[@]}; do
-      msg "Processing ${_repo}-${_arch}"
-
-      repo_file=$(get_repo_file ${_repo} ${_arch})
-
-      if [ ! -f "${repo_file}" ]; then
-         warning "${repo_file} doesn't exist, skipping this repo-arch"
-         continue
-      fi
-
-# Remove blacklisted packages and count them
-# TODO capture all removed packages for printing on debug mode
-      msg2 "Removing blacklisted packages: $(
-      LC_ALL=C repo-remove ${repo_file} ${blacklist[@]} 2>&1 | \
-        grep "\-> Removing" 2>/dev/null| wc -l)"
-
-# Get db contents
-      db=($(get_repo_content ${repo_file}))
-
-      msg2 "Process clean db for syncing..."
-
-# Create a whitelist, add * wildcard to end
-# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if
-# we aren't syncing them
-      echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist
-
-      msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist"
-
-# Sync excluding everything but whitelist
-# We delete here for cleanup
-      rsync ${extra} -rtlH \
-            --delete-after \
-            --delete-excluded \
-            --delay-updates \
-            --include-from=/tmp/${_repo}-${_arch}.whitelist \
-            --exclude="*" \
-            rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \
-            ${FTP_BASE}/${_repo}/os/${_arch}/
-
-# Add a new whitelist
-      whitelists+=(/tmp/${_repo}-${_arch}.whitelist)
-
-      msg "Putting databases back in place"
-      rsync ${extra} -rtlH \
-            --delay-updates \
-            --safe-links \
-            ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \
-            ${FTP_BASE}/${_repo}/os/${_arch}/
-
-# Cleanup
-      unset db 
-    done
-  done
-
-
-  msg "Syncing package pool"
-# Concatenate all whitelists
-  cat ${whitelists[@]} | sort -u > /tmp/any.whitelist
-
-  msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool"
-
-# Sync
-# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
-# packages too
-  for PKGPOOL in ${PKGPOOLS[@]}; do
-    rsync ${extra} -rtlH \
-          --delay-updates \
-          --safe-links \
-          --include-from=/tmp/any.whitelist \
-          --exclude="*" \
-          rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \
-          ${FTP_BASE}/${PKGPOOL}/
-  done
-
-# Sync sources
-  msg "Syncing source pool"
-  #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist
-
-  #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool"
-# Sync
-# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
-# packages too
-  for SRCPOOL in ${SRCPOOLS[@]}; do
-		 rsync ${extra} -rtlH \
-		       --delay-updates \
-		       --safe-links \
-		       --include-from=/tmp/any.whitelist \
-		       --exclude="*" \
-		       rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \
-		       ${FTP_BASE}/${SRCPOOL}/
-  done
-        
-
-# Cleanup
-  unset blacklist whitelists _arch _repo repo_file
+	# Get the blacklisted packages
+	blacklist=($(get_blacklist))
+	# Store all the whitelist files
+	whitelists=()
+
+	msg "%d packages in blacklist" ${#blacklist[@]}
+
+	# Sync the repos databases
+	get_repos
+
+	# Traverse all repo-arch pairs
+	for _repo in ${ARCHREPOS[@]}; do
+		for _arch in ${ARCHARCHES[@]}; do
+			msg "Processing ${_repo}-${_arch}"
+
+			db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT}
+			files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT}
+
+			if [ ! -f "${db_file}" ]; then
+				warning "%s doesn't exist, skipping this repo-arch" "${db_file}"
+				continue
+			fi
+			if [ ! -f "${files_file}" ]; then
+				warning "%s doesn't exist, skipping this repo-arch" "${files_file}"
+				continue
+			fi
+
+			# Remove blacklisted packages and count them
+			# TODO capture all removed packages for printing on debug mode
+			msg2 "Removing blacklisted packages from %s database..." .db
+			LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \
+				|& sed -n 's/-> Removing/  &/p'
+			msg2 "Removing blacklisted packages from %s database..." .files
+			LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \
+				|& sed -n 's/-> Removing/  &/p'
+			# Get db contents
+			db=($(get_repo_content ${db_file}))
+
+			msg2 "Process clean db for syncing..."
+
+			# Create a whitelist, add * wildcard to end
+			# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if
+			# we aren't syncing them
+			printf '%s\n' "${db[@]}" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist
+
+			msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist"
+
+			# Sync excluding everything but whitelist
+			# We delete here for cleanup
+			rsync ${extra} --no-motd -rtlH \
+				--delete-after \
+				--delete-excluded \
+				--delay-updates \
+				--include-from=/tmp/${_repo}-${_arch}.whitelist \
+				--exclude="*" \
+				rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \
+				${FTP_BASE}/${_repo}/os/${_arch}/
+
+			# Add a new whitelist
+			whitelists+=(/tmp/${_repo}-${_arch}.whitelist)
+
+			msg "Putting databases back in place"
+			rsync ${extra} --no-motd -rtlH \
+				--delay-updates \
+				--safe-links \
+				${WORKDIR}/${_repo}/os/${_arch}/ \
+				${FTP_BASE}/${_repo}/os/${_arch}/
+
+			# Cleanup
+			unset db 
+		done
+	done
+
+
+	msg "Syncing package pool"
+	# Concatenate all whitelists
+	cat ${whitelists[@]} | sort -u > /tmp/any.whitelist
+
+	msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool"
+
+	# Sync
+	# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
+	# packages too
+	for PKGPOOL in ${PKGPOOLS[@]}; do
+		rsync ${extra} --no-motd -rtlH \
+			--delay-updates \
+			--safe-links \
+			--include-from=/tmp/any.whitelist \
+			--exclude="*" \
+			rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \
+			${FTP_BASE}/${PKGPOOL}/
+	done
+
+	# Sync sources
+	msg "Syncing source pool"
+	#sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist
+	#msg2 "Retrieving %d sources from pool" $(wc -l < /tmp/any-src.whitelist)
+
+	# Sync
+	# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
+	# packages too
+	for SRCPOOL in ${SRCPOOLS[@]}; do
+		rsync ${extra} --no-motd -rtlH \
+			--delay-updates \
+			--safe-links \
+			--include-from=/tmp/any.whitelist \
+			--exclude="*" \
+			rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \
+			${FTP_BASE}/${SRCPOOL}/
+	done
+
+	# Cleanup
+	unset blacklist whitelists _arch _repo repo_file
 }
 
 trap_exit() {
-    echo
-    error "$@"
-    exit 1
+	local signal=$1; shift
+	echo
+	error "$@"
+	trap -- "$signal"
+	kill "-$signal" "$$"
 }
 
-
-source $(dirname $0)/config
-source $(dirname $0)/local_config
-source $(dirname $0)/libremessages
+source "$(dirname "$(readlink -e "$0")")/config"
+source "$(dirname "$(readlink -e "$0")")/local_config"
+source "$(dirname "$(readlink -e "$0")")/libremessages"
 
 # From makepkg
 set -E
-
-trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT
-trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT
-trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR
+for signal in TERM HUP QUIT; do
+    trap "trap_exit $signal '%s signal caught. Exiting...' $signal" $signal
+done
+trap 'trap_exit INT "Aborted by user! Exiting..."' INT
+trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR
 
 init
-
-rm -r ${TMPDIR}/$0.$$.cache
diff --git a/db-sync.orig b/db-sync.orig
deleted file mode 100755
index 0e07757..0000000
--- a/db-sync.orig
+++ /dev/null
@@ -1,191 +0,0 @@
-#!/bin/bash
-# Syncs Arch repos based on info contained in repo.db files
-# License: GPLv3
-
-# Principles
-# * Get repo.db from an Arch-like repo
-# * Generate a list of available packages
-# * Create sync whitelist (based on package blacklist)
-# * Get packages
-# * Check package signatures
-# * Check database signatures
-# * Sync repo => repo
-
-# TODO
-# * make a tarball of files used for forensics
-# * get files db
-
-# Run as `V=true db-sync` to get verbose output
-VERBOSE=${V}
-${VERBOSE} && extra="-v"
-
-# Returns contents of a repo
-get_repos() {
-	mkdir -p ${TMPDIR}/${0##*/}.$$.cache
-	# Exclude everything but db files
-	rsync ${extra} -mrtlH --no-p --include="*/" \
-		--include="*.db" \
-		--include="*${DBEXT}" \
-		--include="*.files" \
-		--include="*${FILESEXT}" \
-		--exclude="*" \
-		--delete-after \
-		rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${0##*/}.$$.cache
-}
-
-get_repo_content() {
-	# Return all contents
-	bsdtar tf ${1} | \
-		cut -d "/" -f 1 | \
-		sort -u
-}
-
-# Prints blacklisted packages
-get_blacklist() {
-	cut -d ':' -f 1 "${BLACKLIST_FILE}"
-}
-
-# repo
-# arch
-get_repo_file() {
-	echo "${TMPDIR}/${0##*/}.$$.cache/${1}/os/${2}/${1}"
-}
-
-# Process the databases and get the libre packages
-init() {
-
-	# Get the blacklisted packages
-	blacklist=($(get_blacklist))
-	# Store all the whitelist files
-	whitelists=()
-
-	msg "${#blacklist[@]} packages in blacklist"
-
-	# Sync the repos databases
-	get_repos
-
-	# Traverse all repo-arch pairs
-	for _repo in ${ARCHREPOS[@]}; do
-		for _arch in ${ARCHARCHES[@]}; do
-			msg "Processing ${_repo}-${_arch}"
-
-			db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT}
-			files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT}
-
-			if [ ! -f "${db_file}" ]; then
-				warning "%s doesn't exist, skipping this repo-arch" "${db_file}"
-				continue
-			fi
-			if [ ! -f "${files_file}" ]; then
-				warning "%s doesn't exist, skipping this repo-arch" "${files_file}"
-				continue
-			fi
-
-			# Remove blacklisted packages and count them
-			# TODO capture all removed packages for printing on debug mode
-			msg2 "Removing blacklisted packages from .db database..."
-			LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}"
-			msg2 "Removing blacklisted packages from .files database..."
-			LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}"
-
-			# Get db contents
-			db=($(get_repo_content ${db_file}))
-
-			msg2 "Process clean db for syncing..."
-
-			# Create a whitelist, add * wildcard to end
-			# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if
-			# we aren't syncing them
-			echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist
-
-			msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist"
-
-			# Sync excluding everything but whitelist
-			# We delete here for cleanup
-			rsync ${extra} -rtlH \
-				--delete-after \
-				--delete-excluded \
-				--delay-updates \
-				--include-from=/tmp/${_repo}-${_arch}.whitelist \
-				--exclude="*" \
-				rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \
-				${FTP_BASE}/${_repo}/os/${_arch}/
-
-			# Add a new whitelist
-			whitelists+=(/tmp/${_repo}-${_arch}.whitelist)
-
-			msg "Putting databases back in place"
-			rsync ${extra} -rtlH \
-				--delay-updates \
-				--safe-links \
-				${TMPDIR}/${0##*/}.$$.cache/${_repo}/os/${_arch}/ \
-				${FTP_BASE}/${_repo}/os/${_arch}/
-
-			# Cleanup
-			unset db 
-		done
-	done
-
-
-	msg "Syncing package pool"
-	# Concatenate all whitelists
-	cat ${whitelists[@]} | sort -u > /tmp/any.whitelist
-
-	msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool"
-
-	# Sync
-	# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
-	# packages too
-	for PKGPOOL in ${PKGPOOLS[@]}; do
-		rsync ${extra} -rtlH \
-			--delay-updates \
-			--safe-links \
-			--include-from=/tmp/any.whitelist \
-			--exclude="*" \
-			rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \
-			${FTP_BASE}/${PKGPOOL}/
-	done
-
-	# Sync sources
-	msg "Syncing source pool"
-	#sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist
-
-	#msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool"
-	# Sync
-	# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
-	# packages too
-	for SRCPOOL in ${SRCPOOLS[@]}; do
-		rsync ${extra} -rtlH \
-			--delay-updates \
-			--safe-links \
-			--include-from=/tmp/any.whitelist \
-			--exclude="*" \
-			rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \
-			${FTP_BASE}/${SRCPOOL}/
-	done
-
-	# Cleanup
-	unset blacklist whitelists _arch _repo repo_file
-}
-
-trap_exit() {
-	echo
-	error "$@"
-	exit 1
-}
-
-
-source "$(dirname "$(readlink -e "$0")")/config"
-source "$(dirname "$(readlink -e "$0")")/local_config"
-source "$(dirname "$(readlink -e "$0")")/libremessages"
-
-# From makepkg
-set -E
-
-trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT
-trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT
-trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR
-
-init
-
-rm -r ${TMPDIR}/${0##*/}.$$.cache