add incomplete sync support, optimize xbs and fix some variables

7 files changed, 180 insertions, 60 deletions

diff --git a/src/bin/db-import b/src/bin/db-import
index f5e445e..13f2b17 100644
--- a/src/bin/db-import
+++ b/src/bin/db-import
@@ -19,4 +19,4 @@ CONFIG_DIR="$(dirname "$(readlink -e "$0")")/etc"
 
 FLAGS=("${@}")
 
-RUN_DIR="xbs import ${FLAGS[@]}"
+"${BIN_DIR}"/xbs import ${FLAGS[@]}"
diff --git a/src/bin/db-init b/src/bin/db-init
index 624fb7f..db7e61f 100644
--- a/src/bin/db-init
+++ b/src/bin/db-init
@@ -19,4 +19,4 @@ CONFIG_DIR="$(dirname "$(readlink -e "$0")")/etc"
 
 FLAGS=("${@}")
 
-RUN_DIR="xbs init ${FLAGS[@]}"
+"${BIN_DIR}"/xbs init ${FLAGS[@]}"
diff --git a/src/bin/db-repo-add b/src/bin/db-repo-add
index 6969746..aa6ac88 100644
--- a/src/bin/db-repo-add
+++ b/src/bin/db-repo-add
@@ -19,4 +19,4 @@ CONFIG_DIR="$(dirname "$(readlink -e "$0")")/etc"
 
 FLAGS=("${@}")
 
-RUN_DIR="xbs add ${FLAGS[@]}"
+"${BIN_DIR}"/xbs add ${FLAGS[@]}"
diff --git a/src/bin/db-repo-remove b/src/bin/db-repo-remove
index e3c29dd..8e3c37a 100644
--- a/src/bin/db-repo-remove
+++ b/src/bin/db-repo-remove
@@ -19,4 +19,4 @@ CONFIG_DIR="$(dirname "$(readlink -e "$0")")/etc"
 
 FLAGS=("${@}")
 
-RUN_DIR="xbs rm ${FLAGS[@]}"
+"${BIN_DIR}"/xbs rm ${FLAGS[@]}"
diff --git a/src/bin/db-sync b/src/bin/db-sync
new file mode 100644
index 0000000..02a1c92
--- /dev/null
+++ b/src/bin/db-sync
@@ -0,0 +1,22 @@
+#!/bin/bash
+# X Build System
+# Copyright (C) 2012-2015  Parabola Hackers <https://www.parabola.nu>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+CONFIG_DIR="$(dirname "$(readlink -e "$0")")/etc"
+
+FLAGS=("${@}")
+
+"${BIN_DIR}"/xbs sync ${FLAGS[@]}"
diff --git a/src/bin/xbs b/src/bin/xbs
index 7b5bea6..68a1ca7 100644
--- a/src/bin/xbs
+++ b/src/bin/xbs
@@ -100,7 +100,127 @@ xbs-rm_check-permission() {
 
 
 xbs-sync() {
-	
+	WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX")
+	trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT
+	# Get the blacklisted packages
+	blacklist=($(cut -d ':' -f 1 "${BLACKLIST_FILE}"))	
+	# Store all the whitelist files
+	whitelists=()
+	msg "%d packages in blacklist" "${#blacklist[@]}"
+	test "${#blacklist[@]}" -eq '0' && fatal_error "Empty blacklist"
+	## Sync the repos databases
+	# Exclude everything but db files
+	rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \
+		--include="*.db" \
+		--include="*${DB_EXT}" \
+		--include="*.files" \
+		--include="*${FILES_EXT}" \
+		--exclude="*" \
+		--delete-after \
+		"rsync://${mirror}/${mirrorpath}/" "$WORKDIR"
+	# Traverse all repo-arch pairs
+	for '_repo' in "${DD_REPOS[@]}"; do
+		for '_arch' in "${ARCHES[@]}"; do
+		xbs-sync_traverse-xdd-repositories
+		done
+	done
+	fi [ "${#MULTILIB_ARCHES[@]}" -gt '0' ]; then
+		for '_repo' in "${DD_MULTILIB_REPOS[@]}"; do
+			for '_arch' in "${MULTILIB_ARCHES[@]}"; do
+			xbs-sync_traverse-xdd-repositories
+			done
+		done
+	fi
+	msg "Syncing package pool"
+	# Concatenate all whitelists, check for single *s just in case
+	cat "${whitelists[@]}" | grep -v "^\*$" | sort -u > /tmp/any.whitelist
+	msg2 "Retrieving %d packages from pool" "$(wc -l /tmp/any.whitelist | cut -d' ' -f1)"
+	# Sync
+	# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
+	# packages too
+	local pkgpool
+	for pkgpool in "${ARCHPKGPOOLS[@]}"; do
+		rsync "${extra[@]}" --no-motd -rtlH \
+			--delay-updates \
+			--safe-links \
+			--include-from=/tmp/any.whitelist \
+			--exclude="*" \
+			"rsync://${mirror}/${mirrorpath}/${pkgpool}/" \
+			"${FTP_BASE}/${pkgpool}/"
+	done
+	# Sync sources
+	msg "Syncing source pool"
+	#sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist
+	#msg2 "Retrieving %d sources from pool" $(wc -l < /tmp/any-src.whitelist)
+	# Sync
+	# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
+	# packages too
+	local srcpool
+	for srcpool in "${ARCHSRCPOOLS[@]}"; do
+		rsync "${extra[@]}" --no-motd -rtlH \
+			--delay-updates \
+			--safe-links \
+			--include-from=/tmp/any.whitelist \
+			--exclude="*" \
+			"rsync://${mirror}/${mirrorpath}/${srcpool}/" \
+			"${FTP_BASE}/${srcpool}/"
+	done
+	date -u +%s > "${FTP_BASE}/lastsync"
+	# Cleanup
+	unset blacklist whitelists _arch _repo repo_file
+}
+
+xbs-sync_traverse-xdd-repositories() {
+	msg "Processing %s-%s" "${_repo}-${_arch}"
+	db_file=$(get_repo_file "${_repo}" "${_arch}")${DB_EXT}
+	files_file=$(get_repo_file "${_repo}" "${_arch}")${FILES_EXT}
+	if [ ! -f "${db_file}" ]; then
+		warning "%s doesn't exist, skipping this repo-derivated" "${db_file}"
+		continue
+	fi
+	if [ ! -f "${files_file}" ]; then
+		warning "%s doesn't exist, skipping this repo-derivated" "${files_file}"
+		continue
+	fi
+	# Remove blacklisted packages and count them
+	# TODO capture all removed packages for printing on debug mode
+	msg2 "Removing blacklisted packages from %s database..." .db
+	LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \
+	|& sed -n 's/-> Removing/	&/p'
+	msg2 "Removing blacklisted packages from %s database..." .files
+	LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \
+	|& sed -n 's/-> Removing/	&/p'
+	# Get db contents
+	db=($(get_repo_content "${db_file}"))
+	msg2 "Process clean db for syncing..."
+	# Create a whitelist, add * wildcard to end
+	# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if
+	# we aren't syncing them
+	# IMPORTANT: the . in the sed command is needed because an empty
+	# whitelist would consist of a single * allowing any package to
+	# pass through
+	printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist"
+	msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)"
+	# Sync excluding everything but whitelist
+	# We delete here for cleanup
+	rsync "${extra[@]}" --no-motd -rtlH \
+		--delete-after \
+		--delete-excluded \
+		--delay-updates \
+		--include-from="/tmp/${_repo}-${_arch}.whitelist" \
+		--exclude="*" \
+		"rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/" \
+		"${FTP_BASE}/${_repo}/os/${_arch}/"
+	# Add a new whitelist
+	whitelists+=(/tmp/${_repo}-${_arch}.whitelist)
+	msg "Putting databases back in place"
+	rsync "${extra[@]}" --no-motd -rtlH \
+		--delay-updates \
+		--safe-links \
+		"${WORKDIR}/${_repo}/os/${_arch}/" \
+		"${FTP_BASE}/${_repo}/os/${_arch}/"
+	# Cleanup
+	unset db
 }
 
 ##############################################################
@@ -117,10 +237,11 @@ fi
 rules=${1}
 
 case "${rules}" in
-	add)
-		# Add package to repository defined in config
-		if [ "${#}" -lt '5' ]; then
-			msg "usage: %s add <platform> <repo> <arch> <pkg_file> ..." "${0##*/}"
+	add|remove|rm)
+		# Add or remove package to repository defined in config
+		
+		if [ "${#}" -lt '4' ]; then
+			msg "usage: %s <add || rm> <platform> <repo> <arch> <pkg_file> ..." "${0##*/}"
 			exit 1
 		}
 		platform="${2}"
@@ -132,71 +253,45 @@ case "${rules}" in
 			xbs_variable_platform-config
 			if  [ -f "${PLATAFORM_CONFIG}" ]; then
 				xbs_source_platform-config
-				xbs-add
+				if [ "${rules}" == 'add' ]; then
+					xbs-add
+				elif [ "${rules}" == 'rm' || "${rules}" == 'remove' ]; then
+					xbs-rm
+				fi
 			else
 				xbs_error_platform-not-found
 			fi
 		done
 		;;
-	init)
-		# Creates the repository structure defined in config
+	init|sync|import)
+		# init: Creates the repository structure defined in config
+		# sync or import: Sync and import derivated repositories based on info contained in repo.db files
 		platforms=("${@:2}")
 		if [ "${#}" -lt '2' ]; then
-			for '_platform' in "${PLATFORMS[@]}"; do
-				xbs_variable_platform-config
-				xbs_source_platform-config
-				xbs-init
-			done
-		else
-			for '_platform' in "${platforms[@]}"; do
-				xbs_rename-variable_platform-config
-				xbs_variable_platform-config
-				if  [ -f "${PLATAFORM_CONFIG}" ]; then
+			if  [ "${rules}" == 'init' ] && [ "${DISTRO_TYPE}" == 'native' ]; then
+				for '_platform' in "${PLATFORMS[@]}"; do
+					xbs_variable_platform-config
 					xbs_source_platform-config
 					xbs-init
-				else
-					xbs_error_platform-not-found
-				fi
-			done
-		fi
-		;;
-	rm|remove)
-		# Remove package to repository defined in config
-		if [ "${#}" -lt '5' ]; then
-			msg "usage: %s rm <platform> <repo> <arch> <pkg_name> ..." "${0##*/}"
-			exit 1
-		fi
-		platform="${2}"
-		repo="${3}"
-		arch="${4}"
-		pkg_names=("${@:5}")
-		for '_platform' in "${platforms[@]}"; do
-			xbs_rename-variable_platform-config
-			xbs_variable_platform-config
-			if  [ -f "${PLATAFORM_CONFIG}" ]; then
-				xbs_source_platform-config
-				xbs-rm
-			else
-				xbs_error_platform-not-found
+				done
+			elif  [ "${rules}" == 'sync' || "${rules}" == 'import' ] && [ "${DISTRO_TYPE}" == 'derivated' ]; then
+				for '_platform' in "${DD_PLATFORMS[@]}"; do
+					xbs_variable_platform-config
+					xbs_source_platform-config
+					xbs-sync
+				done
 			fi
-		done
-		;;
-	sync|import)
-		# Import and sync the derivated repositories based on info contained in repo.db files
-		platforms=("${@:2}")
-		if [ "${#}" -lt '2' ]; then
-			for '_platform' in "${DD_PLATFORMS[@]}"; do
-				xbs_variable_platform-config
-				xbs_source_platform-config
-				xbs-sync
-			done
 		else
 			for '_platform' in "${platforms[@]}"; do
 				xbs_rename-variable_platform-config
 				xbs_variable_platform-config
-				if  [ -f "${PLATAFORM_CONFIG}" && "${DISTRO_TYPE}" == 'derivated' ]; then
+				if  [ -f "${PLATAFORM_CONFIG}" ]; then
 					xbs_source_platform-config
-					xbs-sync
+					if [ "${rules}" == 'init' ]; then
+						xbs-init
+					elif [ "${rules}" == 'sync' || "${rules}" == 'import' ] && [ "${DISTRO_TYPE}" == 'derivated' ]; then
+						xbs-sync
+					fi
 				else
 					xbs_error_platform-not-found
 				fi
diff --git a/src/etc/xbs.cfg b/src/etc/xbs.cfg
index f005a76..5786c33 100644
--- a/src/etc/xbs.cfg
+++ b/src/etc/xbs.cfg
@@ -5,9 +5,12 @@ ND_PLATFORMS=('gnu+linux_rolling') # 'gnu+linux_testing' 'gnu+linux_stable' 'gnu
 DD_PLATFORMS=('gnu+linux_abslibre' 'gnu+linux_abslibre-arm') # 'gnu+hurd_abslibre'
 PLATFORMS=("${ND_PLATFORMS[@]}" "${DD_PLATFORMS[@]}")
 
+PREFIX="$(dirname "$(readlink -e "$0")")"
+BIN_DIR="${PREFIX}/bin"
+CONFIG_DIR="${PREFIX}/etc"
+SHARE_DIR="${PREFIX}/share/xbs"
 SRV_DIR='/srv'
 TMP_DIR='/tmp'
-SHARE_DIR="$(dirname "$(readlink -e "$0")")/share/xbs"
 HTTP_DIR="${SRV_DIR}/repo/http"
 ROOT_DIR="${SRV_DIR}/repo/main"
 GIT_DIR="${SRV_DIR}/git"