Merge branch 'master' of https://projects.parabolagnulinux.org/dbscripts

Conflicts:
	config

1 files changed, 171 insertions, 0 deletions

diff --git a/db-sync b/db-sync
new file mode 100755
index 0000000..1a9e02f
--- /dev/null
+++ b/db-sync
@@ -0,0 +1,171 @@
+#!/bin/bash
+# Syncs Arch repos based on info contained in repo.db files
+# License: GPLv3
+
+# Principles
+# * Get repo.db from an Arch-like repo
+# * Generate a list of available packages
+# * Create sync whitelist (based on package blacklist)
+# * Get packages
+# * Check package signatures
+# * Check database signatures
+# * Sync repo => repo
+
+# TODO
+# * verbose mode
+# * make a tarball of files used for forensics
+# * get files db
+
+# Returns contents of a repo
+get_repos() {
+# Exclude everything but db files
+  rsync -avm --include="*/" \
+        --include="*.db" \
+        --include="*${DBEXT}" \
+        --exclude="*" \
+        --delete-after \
+        rsync://${mirror}/${mirrorpath}/ cache/
+}
+
+get_repo_content() {
+# Return all contents
+  bsdtar tf ${1} | \
+    cut -d "/" -f 1 | \
+    sort -u
+}
+
+# Prints blacklisted packages
+get_blacklist() {
+  cut -d ':' -f 1 "${BLACKLIST_FILE}"
+}
+
+# repo
+# arch
+get_repo_file() {
+  [ ! -r "cache/${1}/os/${2}/${1}${DBEXT}" ] && return 1
+
+  echo "cache/${1}/os/${2}/${1}${DBEXT}"
+}
+
+# Process the databases and get the libre packages
+init() {
+
+# Get the blacklisted packages
+  blacklist=($(get_blacklist))
+# Store all the whitelist files
+  whitelists=()
+
+  msg "${#blacklist[@]} packages in blacklist"
+
+# Sync the repos databases
+  get_repos
+
+# Traverse all repo-arch pairs
+  for _arch in ${ARCHARCHES[@]}; do
+    for _repo in ${ARCHREPOS[@]}; do
+      msg "Processing ${_repo}-${_arch}"
+
+      repo_file=$(get_repo_file ${_repo} ${_arch})
+
+# Remove blacklisted packages and count them
+# TODO capture all removed packages for printing on debug mode
+      msg2 "Removing blacklisted packages: $(
+      LC_ALL=C repo-remove ${repo_file} ${blacklist[@]} 2>&1 | \
+        grep "\-> Removing" 2>/dev/null| wc -l)"
+
+# Get db contents
+      db=($(get_repo_content ${repo_file}))
+
+      msg2 "Process clean db for syncing..."
+
+# Create a whitelist, add * wildcard to end
+# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if
+# we aren't syncing them
+      echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist
+
+      msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist"
+
+# Sync excluding everything but whitelist
+# We delete here for cleanup
+      rsync -vrtlH \
+            --delete-after \
+            --delete-excluded \
+            --delay-updates \
+            --max-delete=1000 \
+            --include-from=/tmp/${_repo}-${_arch}.whitelist \
+            --exclude="*" \
+            rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \
+            ${FTP_BASE}/${_repo}/os/${_arch}/
+
+# Add a new whitelist
+      whitelists+=(/tmp/${_repo}-${_arch}.whitelist)
+
+# Cleanup
+      unset db 
+    done
+  done
+
+  msg "Putting databases back in place"
+  rsync -vrtlH \
+        --delay-updates \
+        --safe-links \
+        cache/ \
+        ${FTP_BASE}/
+
+  msg "Syncing package pool"
+# Concatenate all whitelists
+  cat ${whitelists[@]} | sort -u > /tmp/any.whitelist
+
+  msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool"
+
+# Sync
+# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
+# packages too
+  rsync -vrtlH \
+        --delay-updates \
+        --safe-links \
+        --include-from=/tmp/any.whitelist \
+        --exclude="*" \
+        rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \
+        ${FTP_BASE}/${PKGPOOL}/
+
+# Sync sources
+  msg "Syncing source pool"
+  #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist
+
+  #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool"
+# Sync
+# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
+# packages too
+  rsync -vrtlH \
+        --delay-updates \
+        --safe-links \
+        --include-from=/tmp/any.whitelist \
+        --exclude="*" \
+        rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \
+        ${FTP_BASE}/${SRCPOOL}/
+        
+
+# Cleanup
+  unset blacklist whitelists _arch _repo repo_file
+}
+
+trap_exit() {
+    echo
+    error "$@"
+	exit 1
+}
+
+
+source $(dirname $0)/config
+source $(dirname $0)/local_config
+source $(dirname $0)/libremessages
+
+# From makepkg
+set -E
+
+trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT
+trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT
+trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR
+
+init