revert to old db-sync until fix it, rename lastest db-sync to db-sync.orig

2 files changed, 321 insertions, 135 deletions

diff --git a/db-sync b/db-sync
index 0e07757..81dee24 100755
--- a/db-sync
+++ b/db-sync
@@ -21,163 +21,158 @@ ${VERBOSE} && extra="-v"
 
 # Returns contents of a repo
 get_repos() {
-	mkdir -p ${TMPDIR}/${0##*/}.$$.cache
-	# Exclude everything but db files
-	rsync ${extra} -mrtlH --no-p --include="*/" \
-		--include="*.db" \
-		--include="*${DBEXT}" \
-		--include="*.files" \
-		--include="*${FILESEXT}" \
-		--exclude="*" \
-		--delete-after \
-		rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${0##*/}.$$.cache
+        mkdir -p ${TMPDIR}/$0.$$.cache
+# Exclude everything but db files
+  rsync ${extra} -mrtlH --no-p --include="*/" \
+        --include="*.db" \
+        --include="*${DBEXT}" \
+        --exclude="*" \
+        --delete-after \
+        rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache
 }
 
 get_repo_content() {
-	# Return all contents
-	bsdtar tf ${1} | \
-		cut -d "/" -f 1 | \
-		sort -u
+# Return all contents
+  bsdtar tf ${1} | \
+    cut -d "/" -f 1 | \
+    sort -u
 }
 
 # Prints blacklisted packages
 get_blacklist() {
-	cut -d ':' -f 1 "${BLACKLIST_FILE}"
+  cut -d ':' -f 1 "${BLACKLIST_FILE}"
 }
 
 # repo
 # arch
 get_repo_file() {
-	echo "${TMPDIR}/${0##*/}.$$.cache/${1}/os/${2}/${1}"
+#  shopt -s nullglob
+
+  echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}${DBEXT}"
 }
 
 # Process the databases and get the libre packages
 init() {
 
-	# Get the blacklisted packages
-	blacklist=($(get_blacklist))
-	# Store all the whitelist files
-	whitelists=()
-
-	msg "${#blacklist[@]} packages in blacklist"
-
-	# Sync the repos databases
-	get_repos
-
-	# Traverse all repo-arch pairs
-	for _repo in ${ARCHREPOS[@]}; do
-		for _arch in ${ARCHARCHES[@]}; do
-			msg "Processing ${_repo}-${_arch}"
-
-			db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT}
-			files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT}
-
-			if [ ! -f "${db_file}" ]; then
-				warning "%s doesn't exist, skipping this repo-arch" "${db_file}"
-				continue
-			fi
-			if [ ! -f "${files_file}" ]; then
-				warning "%s doesn't exist, skipping this repo-arch" "${files_file}"
-				continue
-			fi
-
-			# Remove blacklisted packages and count them
-			# TODO capture all removed packages for printing on debug mode
-			msg2 "Removing blacklisted packages from .db database..."
-			LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}"
-			msg2 "Removing blacklisted packages from .files database..."
-			LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}"
-
-			# Get db contents
-			db=($(get_repo_content ${db_file}))
-
-			msg2 "Process clean db for syncing..."
-
-			# Create a whitelist, add * wildcard to end
-			# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if
-			# we aren't syncing them
-			echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist
-
-			msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist"
-
-			# Sync excluding everything but whitelist
-			# We delete here for cleanup
-			rsync ${extra} -rtlH \
-				--delete-after \
-				--delete-excluded \
-				--delay-updates \
-				--include-from=/tmp/${_repo}-${_arch}.whitelist \
-				--exclude="*" \
-				rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \
-				${FTP_BASE}/${_repo}/os/${_arch}/
-
-			# Add a new whitelist
-			whitelists+=(/tmp/${_repo}-${_arch}.whitelist)
-
-			msg "Putting databases back in place"
-			rsync ${extra} -rtlH \
-				--delay-updates \
-				--safe-links \
-				${TMPDIR}/${0##*/}.$$.cache/${_repo}/os/${_arch}/ \
-				${FTP_BASE}/${_repo}/os/${_arch}/
-
-			# Cleanup
-			unset db 
-		done
-	done
-
-
-	msg "Syncing package pool"
-	# Concatenate all whitelists
-	cat ${whitelists[@]} | sort -u > /tmp/any.whitelist
-
-	msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool"
-
-	# Sync
-	# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
-	# packages too
-	for PKGPOOL in ${PKGPOOLS[@]}; do
-		rsync ${extra} -rtlH \
-			--delay-updates \
-			--safe-links \
-			--include-from=/tmp/any.whitelist \
-			--exclude="*" \
-			rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \
-			${FTP_BASE}/${PKGPOOL}/
-	done
-
-	# Sync sources
-	msg "Syncing source pool"
-	#sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist
-
-	#msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool"
-	# Sync
-	# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
-	# packages too
-	for SRCPOOL in ${SRCPOOLS[@]}; do
-		rsync ${extra} -rtlH \
-			--delay-updates \
-			--safe-links \
-			--include-from=/tmp/any.whitelist \
-			--exclude="*" \
-			rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \
-			${FTP_BASE}/${SRCPOOL}/
-	done
-
-	# Cleanup
-	unset blacklist whitelists _arch _repo repo_file
+# Get the blacklisted packages
+  blacklist=($(get_blacklist))
+# Store all the whitelist files
+  whitelists=()
+
+  msg "${#blacklist[@]} packages in blacklist"
+
+# Sync the repos databases
+  get_repos
+
+# Traverse all repo-arch pairs
+  for _repo in ${ARCHREPOS[@]}; do
+    for _arch in ${ARCHARCHES[@]}; do
+      msg "Processing ${_repo}-${_arch}"
+
+      repo_file=$(get_repo_file ${_repo} ${_arch})
+
+      if [ ! -f "${repo_file}" ]; then
+         warning "${repo_file} doesn't exist, skipping this repo-arch"
+         continue
+      fi
+
+# Remove blacklisted packages and count them
+# TODO capture all removed packages for printing on debug mode
+      msg2 "Removing blacklisted packages: $(
+      LC_ALL=C repo-remove ${repo_file} ${blacklist[@]} 2>&1 | \
+        grep "\-> Removing" 2>/dev/null| wc -l)"
+
+# Get db contents
+      db=($(get_repo_content ${repo_file}))
+
+      msg2 "Process clean db for syncing..."
+
+# Create a whitelist, add * wildcard to end
+# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if
+# we aren't syncing them
+      echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist
+
+      msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist"
+
+# Sync excluding everything but whitelist
+# We delete here for cleanup
+      rsync ${extra} -rtlH \
+            --delete-after \
+            --delete-excluded \
+            --delay-updates \
+            --include-from=/tmp/${_repo}-${_arch}.whitelist \
+            --exclude="*" \
+            rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \
+            ${FTP_BASE}/${_repo}/os/${_arch}/
+
+# Add a new whitelist
+      whitelists+=(/tmp/${_repo}-${_arch}.whitelist)
+
+      msg "Putting databases back in place"
+      rsync ${extra} -rtlH \
+            --delay-updates \
+            --safe-links \
+            ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \
+            ${FTP_BASE}/${_repo}/os/${_arch}/
+
+# Cleanup
+      unset db 
+    done
+  done
+
+
+  msg "Syncing package pool"
+# Concatenate all whitelists
+  cat ${whitelists[@]} | sort -u > /tmp/any.whitelist
+
+  msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool"
+
+# Sync
+# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
+# packages too
+  for PKGPOOL in ${PKGPOOLS[@]}; do
+    rsync ${extra} -rtlH \
+          --delay-updates \
+          --safe-links \
+          --include-from=/tmp/any.whitelist \
+          --exclude="*" \
+          rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \
+          ${FTP_BASE}/${PKGPOOL}/
+  done
+
+# Sync sources
+  msg "Syncing source pool"
+  #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist
+
+  #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool"
+# Sync
+# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
+# packages too
+  for SRCPOOL in ${SRCPOOLS[@]}; do
+		 rsync ${extra} -rtlH \
+		       --delay-updates \
+		       --safe-links \
+		       --include-from=/tmp/any.whitelist \
+		       --exclude="*" \
+		       rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \
+		       ${FTP_BASE}/${SRCPOOL}/
+  done
+        
+
+# Cleanup
+  unset blacklist whitelists _arch _repo repo_file
 }
 
 trap_exit() {
-	echo
-	error "$@"
-	exit 1
+    echo
+    error "$@"
+    exit 1
 }
 
 
-source "$(dirname "$(readlink -e "$0")")/config"
-source "$(dirname "$(readlink -e "$0")")/local_config"
-source "$(dirname "$(readlink -e "$0")")/libremessages"
+source $(dirname $0)/config
+source $(dirname $0)/local_config
+source $(dirname $0)/libremessages
 
 # From makepkg
 set -E
@@ -188,4 +183,4 @@ trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR
 
 init
 
-rm -r ${TMPDIR}/${0##*/}.$$.cache
+rm -r ${TMPDIR}/$0.$$.cache
diff --git a/db-sync.orig b/db-sync.orig
new file mode 100755
index 0000000..0e07757
--- /dev/null
+++ b/db-sync.orig
@@ -0,0 +1,191 @@
+#!/bin/bash
+# Syncs Arch repos based on info contained in repo.db files
+# License: GPLv3
+
+# Principles
+# * Get repo.db from an Arch-like repo
+# * Generate a list of available packages
+# * Create sync whitelist (based on package blacklist)
+# * Get packages
+# * Check package signatures
+# * Check database signatures
+# * Sync repo => repo
+
+# TODO
+# * make a tarball of files used for forensics
+# * get files db
+
+# Run as `V=true db-sync` to get verbose output
+VERBOSE=${V}
+${VERBOSE} && extra="-v"
+
+# Returns contents of a repo
+get_repos() {
+	mkdir -p ${TMPDIR}/${0##*/}.$$.cache
+	# Exclude everything but db files
+	rsync ${extra} -mrtlH --no-p --include="*/" \
+		--include="*.db" \
+		--include="*${DBEXT}" \
+		--include="*.files" \
+		--include="*${FILESEXT}" \
+		--exclude="*" \
+		--delete-after \
+		rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${0##*/}.$$.cache
+}
+
+get_repo_content() {
+	# Return all contents
+	bsdtar tf ${1} | \
+		cut -d "/" -f 1 | \
+		sort -u
+}
+
+# Prints blacklisted packages
+get_blacklist() {
+	cut -d ':' -f 1 "${BLACKLIST_FILE}"
+}
+
+# repo
+# arch
+get_repo_file() {
+	echo "${TMPDIR}/${0##*/}.$$.cache/${1}/os/${2}/${1}"
+}
+
+# Process the databases and get the libre packages
+init() {
+
+	# Get the blacklisted packages
+	blacklist=($(get_blacklist))
+	# Store all the whitelist files
+	whitelists=()
+
+	msg "${#blacklist[@]} packages in blacklist"
+
+	# Sync the repos databases
+	get_repos
+
+	# Traverse all repo-arch pairs
+	for _repo in ${ARCHREPOS[@]}; do
+		for _arch in ${ARCHARCHES[@]}; do
+			msg "Processing ${_repo}-${_arch}"
+
+			db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT}
+			files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT}
+
+			if [ ! -f "${db_file}" ]; then
+				warning "%s doesn't exist, skipping this repo-arch" "${db_file}"
+				continue
+			fi
+			if [ ! -f "${files_file}" ]; then
+				warning "%s doesn't exist, skipping this repo-arch" "${files_file}"
+				continue
+			fi
+
+			# Remove blacklisted packages and count them
+			# TODO capture all removed packages for printing on debug mode
+			msg2 "Removing blacklisted packages from .db database..."
+			LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}"
+			msg2 "Removing blacklisted packages from .files database..."
+			LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}"
+
+			# Get db contents
+			db=($(get_repo_content ${db_file}))
+
+			msg2 "Process clean db for syncing..."
+
+			# Create a whitelist, add * wildcard to end
+			# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if
+			# we aren't syncing them
+			echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist
+
+			msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist"
+
+			# Sync excluding everything but whitelist
+			# We delete here for cleanup
+			rsync ${extra} -rtlH \
+				--delete-after \
+				--delete-excluded \
+				--delay-updates \
+				--include-from=/tmp/${_repo}-${_arch}.whitelist \
+				--exclude="*" \
+				rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \
+				${FTP_BASE}/${_repo}/os/${_arch}/
+
+			# Add a new whitelist
+			whitelists+=(/tmp/${_repo}-${_arch}.whitelist)
+
+			msg "Putting databases back in place"
+			rsync ${extra} -rtlH \
+				--delay-updates \
+				--safe-links \
+				${TMPDIR}/${0##*/}.$$.cache/${_repo}/os/${_arch}/ \
+				${FTP_BASE}/${_repo}/os/${_arch}/
+
+			# Cleanup
+			unset db 
+		done
+	done
+
+
+	msg "Syncing package pool"
+	# Concatenate all whitelists
+	cat ${whitelists[@]} | sort -u > /tmp/any.whitelist
+
+	msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool"
+
+	# Sync
+	# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
+	# packages too
+	for PKGPOOL in ${PKGPOOLS[@]}; do
+		rsync ${extra} -rtlH \
+			--delay-updates \
+			--safe-links \
+			--include-from=/tmp/any.whitelist \
+			--exclude="*" \
+			rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \
+			${FTP_BASE}/${PKGPOOL}/
+	done
+
+	# Sync sources
+	msg "Syncing source pool"
+	#sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist
+
+	#msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool"
+	# Sync
+	# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
+	# packages too
+	for SRCPOOL in ${SRCPOOLS[@]}; do
+		rsync ${extra} -rtlH \
+			--delay-updates \
+			--safe-links \
+			--include-from=/tmp/any.whitelist \
+			--exclude="*" \
+			rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \
+			${FTP_BASE}/${SRCPOOL}/
+	done
+
+	# Cleanup
+	unset blacklist whitelists _arch _repo repo_file
+}
+
+trap_exit() {
+	echo
+	error "$@"
+	exit 1
+}
+
+
+source "$(dirname "$(readlink -e "$0")")/config"
+source "$(dirname "$(readlink -e "$0")")/local_config"
+source "$(dirname "$(readlink -e "$0")")/libremessages"
+
+# From makepkg
+set -E
+
+trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT
+trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT
+trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR
+
+init
+
+rm -r ${TMPDIR}/${0##*/}.$$.cache