Merge branch 'sync-db-first' of https://projects.parabolagnulinux.org/dbscripts into sync-db-first

3 files changed, 187 insertions, 8 deletions

diff --git a/config b/config
index 35c7963..a0849a9 100644
--- a/config
+++ b/config
@@ -1,10 +1,10 @@
 #!/bin/bash
-FTP_BASE="/home/parabolavnx/parabolagnulinux.org/repo"
-ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo"
-SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre"
+FTP_BASE="/tmp/repo"
+ARCH_BASE="/tmp/repo"
+SVNREPO="/var/abs"
 
 # Repos from Arch
-ARCHREPOS=('core' 'extra' 'community'  'testing' 'multilib')
+ARCHREPOS=('core') #'extra' 'community'  'testing' 'multilib')
 # Official Parabola repos
 OURREPOS=('libre' 'libre-testing')
 # User repos
@@ -31,13 +31,13 @@ LOCK_DELAY=10
 LOCK_TIMEOUT=300
 
 STAGING="$FTP_BASE/staging"
-TMPDIR="$HOME/tmp"
-ARCHARCHES=(i686 x86_64)
+TMPDIR="/tmp"
+ARCHARCHES=(i686) # x86_64)
 ARCHES=(${ARCHARCHES[@]} mips64el)
 DBEXT=".db.tar.gz"
 FILESEXT=".files.tar.gz"
 PKGEXT=".pkg.tar.?z"
 SRCEXT=".src.tar.gz"
 
-MAKEPKGCONF="$HOME/etc/makepkg.conf"
-BLACKLIST_FILE="$HOME/blacklist/blacklist.txt"
+MAKEPKGCONF="/etc/makepkg.conf"
+BLACKLIST_FILE="$FTP_BASE/blacklist.txt"
diff --git a/createrepos b/createrepos
new file mode 100755
index 0000000..4ee057b
--- /dev/null
+++ b/createrepos
@@ -0,0 +1,8 @@
+#!/bin/bash
+# Creates the repo structure defined in config
+
+source $(dirname $0)/config
+
+mkdir -p ${FTP_BASE}/{${PKGPOOL},${SRCPOOL}} ${ARCH_BASE} ${CLEANUP_DESTDIR} ${SOURCE_CLEANUP_DESTDIR} ${STAGING}
+
+$(dirname $0)/create-repo ${PKGREPOS[@]}
diff --git a/db-sync b/db-sync
new file mode 100755
index 0000000..0bb79b7
--- /dev/null
+++ b/db-sync
@@ -0,0 +1,171 @@
+#!/bin/bash
+# Syncs Arch repos based on info contained in repo.db files
+# License: GPLv3
+
+# Principles
+# * Get repo.db from an Arch-like repo
+# * Generate a list of available packages
+# * Create sync whitelist (based on package blacklist)
+# * Get packages
+# * Check package signatures
+# * Check database signatures
+# * Sync repo => repo
+
+# TODO
+# * verbose mode
+# * make a tarball of files used for forensics
+# * get files db
+
+# Returns contents of a repo
+get_repos() {
+# Exclude everything but db files
+  rsync -avm --include="*/" \
+        --include="*.db" \
+        --include="*${DBEXT}" \
+        --exclude="*" \
+        --delete-after \
+        rsync://${mirror}/${mirrorpath}/ cache/
+}
+
+get_repo_content() {
+# Return all contents
+  bsdtar tf ${1} | \
+    cut -d "/" -f 1 | \
+    sort -u
+}
+
+# Prints blacklisted packages
+get_blacklist() {
+  cut -d ':' -f 1 "${BLACKLIST_FILE}"
+}
+
+# repo
+# arch
+get_repo_file() {
+  [ ! -r "cache/${1}/os/${2}/${1}${DBEXT}" ] && return 1
+
+  echo "cache/${1}/os/${2}/${1}${DBEXT}"
+}
+
+# Process the databases and get the libre packages
+init() {
+
+# Get the blacklisted packages
+  blacklist=($(get_blacklist))
+# Store all the whitelist files
+  whitelists=()
+
+  msg "${#blacklist[@]} packages in blacklist"
+
+# Sync the repos databases
+  get_repos
+
+# Traverse all repo-arch pairs
+  for _arch in ${ARCHARCHES[@]}; do
+    for _repo in ${ARCHREPOS[@]}; do
+      msg "Processing ${_repo}-${_arch}"
+
+      repo_file=$(get_repo_file ${_repo} ${_arch})
+
+# Remove blacklisted packages and count them
+# TODO capture all removed packages for printing on debug mode
+      msg2 "Removing blacklisted packages: $(
+      LC_ALL=C repo-remove ${repo_file} ${blacklist[@]} 2>&1 | \
+        grep "\-> Removing" 2>/dev/null| wc -l)"
+
+# Get db contents
+      db=($(get_repo_content ${repo_file}))
+
+      msg2 "Process clean db for syncing..."
+
+# Create a whitelist, add * wildcard to end
+# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if
+# we aren't syncing them
+      echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist
+
+      msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist"
+
+# Sync excluding everything but whitelist
+# We delete here for cleanup
+      rsync -vrtlH \
+            --delete-after \
+            --safe-links \
+            --delay-updates \
+            --max-delete=1000 \
+            --include-from=/tmp/${_repo}-${_arch}.whitelist \
+            --exclude="*" \
+            rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \
+            ${FTP_BASE}/${_repo}/os/${_arch}/
+
+# Add a new whitelist
+      whitelists+=(/tmp/${_repo}-${_arch}.whitelist)
+
+# Cleanup
+      unset db 
+    done
+  done
+
+  msg "Putting databases back in place"
+  rsync -vrtlH \
+        --delay-updates \
+        --safe-links \
+        cache/ \
+        ${FTP_BASE}/
+
+  msg "Syncing package pool"
+# Concatenate all whitelists
+  cat ${whitelists[@]} | sort -u > /tmp/any.whitelist
+
+  msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool"
+
+# Sync
+# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
+# packages too
+  rsync -vrtlH \
+        --delay-updates \
+        --safe-links \
+        --include-from=/tmp/any.whitelist \
+        --exclude="*" \
+        rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \
+        ${FTP_BASE}/${PKGPOOL}/
+
+# Sync sources
+  msg "Syncing source pool"
+  #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist
+
+  #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool"
+# Sync
+# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
+# packages too
+  rsync -vrtlH \
+        --delay-updates \
+        --safe-links \
+        --include-from=/tmp/any.whitelist \
+        --exclude="*" \
+        rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \
+        ${FTP_BASE}/${SRCPOOL}/
+        
+
+# Cleanup
+  unset blacklist whitelists _arch _repo repo_file
+}
+
+trap_exit() {
+    echo
+    error "$@"
+	exit 1
+}
+
+
+source $(dirname $0)/config
+source $(dirname $0)/local_config
+source $(dirname $0)/libremessages
+
+# From makepkg
+set -E
+
+trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT
+trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT
+trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR
+
+init