From 9fb22ceccd65f03478618de4a68afd3f31e0bf6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?coadde=20=5BM=C3=A1rcio=20Alexandre=20Silva=20Delgado=5D?= Date: Sat, 29 Aug 2015 17:48:39 -0300 Subject: add incomplete sync support, optimize xbs and fix some variables --- src/bin/db-import | 2 +- src/bin/db-init | 2 +- src/bin/db-repo-add | 2 +- src/bin/db-repo-remove | 2 +- src/bin/db-sync | 22 ++++++ src/bin/xbs | 205 ++++++++++++++++++++++++++++++++++++------------- src/etc/xbs.cfg | 5 +- 7 files changed, 180 insertions(+), 60 deletions(-) create mode 100644 src/bin/db-sync diff --git a/src/bin/db-import b/src/bin/db-import index f5e445e..13f2b17 100644 --- a/src/bin/db-import +++ b/src/bin/db-import @@ -19,4 +19,4 @@ CONFIG_DIR="$(dirname "$(readlink -e "$0")")/etc" FLAGS=("${@}") -RUN_DIR="xbs import ${FLAGS[@]}" +"${BIN_DIR}"/xbs import ${FLAGS[@]}" diff --git a/src/bin/db-init b/src/bin/db-init index 624fb7f..db7e61f 100644 --- a/src/bin/db-init +++ b/src/bin/db-init @@ -19,4 +19,4 @@ CONFIG_DIR="$(dirname "$(readlink -e "$0")")/etc" FLAGS=("${@}") -RUN_DIR="xbs init ${FLAGS[@]}" +"${BIN_DIR}"/xbs init ${FLAGS[@]}" diff --git a/src/bin/db-repo-add b/src/bin/db-repo-add index 6969746..aa6ac88 100644 --- a/src/bin/db-repo-add +++ b/src/bin/db-repo-add @@ -19,4 +19,4 @@ CONFIG_DIR="$(dirname "$(readlink -e "$0")")/etc" FLAGS=("${@}") -RUN_DIR="xbs add ${FLAGS[@]}" +"${BIN_DIR}"/xbs add ${FLAGS[@]}" diff --git a/src/bin/db-repo-remove b/src/bin/db-repo-remove index e3c29dd..8e3c37a 100644 --- a/src/bin/db-repo-remove +++ b/src/bin/db-repo-remove @@ -19,4 +19,4 @@ CONFIG_DIR="$(dirname "$(readlink -e "$0")")/etc" FLAGS=("${@}") -RUN_DIR="xbs rm ${FLAGS[@]}" +"${BIN_DIR}"/xbs rm ${FLAGS[@]}" diff --git a/src/bin/db-sync b/src/bin/db-sync new file mode 100644 index 0000000..02a1c92 --- /dev/null +++ b/src/bin/db-sync @@ -0,0 +1,22 @@ +#!/bin/bash +# X Build System +# Copyright (C) 2012-2015 Parabola Hackers + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +CONFIG_DIR="$(dirname "$(readlink -e "$0")")/etc" + +FLAGS=("${@}") + +"${BIN_DIR}"/xbs sync ${FLAGS[@]}" diff --git a/src/bin/xbs b/src/bin/xbs index 7b5bea6..68a1ca7 100644 --- a/src/bin/xbs +++ b/src/bin/xbs @@ -100,7 +100,127 @@ xbs-rm_check-permission() { xbs-sync() { - + WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") + trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT + # Get the blacklisted packages + blacklist=($(cut -d ':' -f 1 "${BLACKLIST_FILE}")) + # Store all the whitelist files + whitelists=() + msg "%d packages in blacklist" "${#blacklist[@]}" + test "${#blacklist[@]}" -eq '0' && fatal_error "Empty blacklist" + ## Sync the repos databases + # Exclude everything but db files + rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \ + --include="*.db" \ + --include="*${DB_EXT}" \ + --include="*.files" \ + --include="*${FILES_EXT}" \ + --exclude="*" \ + --delete-after \ + "rsync://${mirror}/${mirrorpath}/" "$WORKDIR" + # Traverse all repo-arch pairs + for '_repo' in "${DD_REPOS[@]}"; do + for '_arch' in "${ARCHES[@]}"; do + xbs-sync_traverse-xdd-repositories + done + done + fi [ "${#MULTILIB_ARCHES[@]}" -gt '0' ]; then + for '_repo' in "${DD_MULTILIB_REPOS[@]}"; do + for '_arch' in "${MULTILIB_ARCHES[@]}"; do + xbs-sync_traverse-xdd-repositories + done + done + fi + msg "Syncing package pool" + # Concatenate all whitelists, check for single *s just in case + cat "${whitelists[@]}" | grep -v "^\*$" | sort -u > /tmp/any.whitelist + msg2 "Retrieving %d packages from pool" "$(wc -l /tmp/any.whitelist | cut -d' ' -f1)" + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + local pkgpool + for pkgpool in "${ARCHPKGPOOLS[@]}"; do + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + "rsync://${mirror}/${mirrorpath}/${pkgpool}/" \ + "${FTP_BASE}/${pkgpool}/" + done + # Sync sources + msg "Syncing source pool" + #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist + #msg2 "Retrieving %d sources from pool" $(wc -l < /tmp/any-src.whitelist) + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + local srcpool + for srcpool in "${ARCHSRCPOOLS[@]}"; do + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + "rsync://${mirror}/${mirrorpath}/${srcpool}/" \ + "${FTP_BASE}/${srcpool}/" + done + date -u +%s > "${FTP_BASE}/lastsync" + # Cleanup + unset blacklist whitelists _arch _repo repo_file +} + +xbs-sync_traverse-xdd-repositories() { + msg "Processing %s-%s" "${_repo}-${_arch}" + db_file=$(get_repo_file "${_repo}" "${_arch}")${DB_EXT} + files_file=$(get_repo_file "${_repo}" "${_arch}")${FILES_EXT} + if [ ! -f "${db_file}" ]; then + warning "%s doesn't exist, skipping this repo-derivated" "${db_file}" + continue + fi + if [ ! -f "${files_file}" ]; then + warning "%s doesn't exist, skipping this repo-derivated" "${files_file}" + continue + fi + # Remove blacklisted packages and count them + # TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages from %s database..." .db + LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + msg2 "Removing blacklisted packages from %s database..." .files + LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + # Get db contents + db=($(get_repo_content "${db_file}")) + msg2 "Process clean db for syncing..." + # Create a whitelist, add * wildcard to end + # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if + # we aren't syncing them + # IMPORTANT: the . in the sed command is needed because an empty + # whitelist would consist of a single * allowing any package to + # pass through + printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" + msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" + # Sync excluding everything but whitelist + # We delete here for cleanup + rsync "${extra[@]}" --no-motd -rtlH \ + --delete-after \ + --delete-excluded \ + --delay-updates \ + --include-from="/tmp/${_repo}-${_arch}.whitelist" \ + --exclude="*" \ + "rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/" \ + "${FTP_BASE}/${_repo}/os/${_arch}/" + # Add a new whitelist + whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + msg "Putting databases back in place" + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + "${WORKDIR}/${_repo}/os/${_arch}/" \ + "${FTP_BASE}/${_repo}/os/${_arch}/" + # Cleanup + unset db } ############################################################## @@ -117,10 +237,11 @@ fi rules=${1} case "${rules}" in - add) - # Add package to repository defined in config - if [ "${#}" -lt '5' ]; then - msg "usage: %s add ..." "${0##*/}" + add|remove|rm) + # Add or remove package to repository defined in config + + if [ "${#}" -lt '4' ]; then + msg "usage: %s ..." "${0##*/}" exit 1 } platform="${2}" @@ -132,71 +253,45 @@ case "${rules}" in xbs_variable_platform-config if [ -f "${PLATAFORM_CONFIG}" ]; then xbs_source_platform-config - xbs-add + if [ "${rules}" == 'add' ]; then + xbs-add + elif [ "${rules}" == 'rm' || "${rules}" == 'remove' ]; then + xbs-rm + fi else xbs_error_platform-not-found fi done ;; - init) - # Creates the repository structure defined in config + init|sync|import) + # init: Creates the repository structure defined in config + # sync or import: Sync and import derivated repositories based on info contained in repo.db files platforms=("${@:2}") if [ "${#}" -lt '2' ]; then - for '_platform' in "${PLATFORMS[@]}"; do - xbs_variable_platform-config - xbs_source_platform-config - xbs-init - done - else - for '_platform' in "${platforms[@]}"; do - xbs_rename-variable_platform-config - xbs_variable_platform-config - if [ -f "${PLATAFORM_CONFIG}" ]; then + if [ "${rules}" == 'init' ] && [ "${DISTRO_TYPE}" == 'native' ]; then + for '_platform' in "${PLATFORMS[@]}"; do + xbs_variable_platform-config xbs_source_platform-config xbs-init - else - xbs_error_platform-not-found - fi - done - fi - ;; - rm|remove) - # Remove package to repository defined in config - if [ "${#}" -lt '5' ]; then - msg "usage: %s rm ..." "${0##*/}" - exit 1 - fi - platform="${2}" - repo="${3}" - arch="${4}" - pkg_names=("${@:5}") - for '_platform' in "${platforms[@]}"; do - xbs_rename-variable_platform-config - xbs_variable_platform-config - if [ -f "${PLATAFORM_CONFIG}" ]; then - xbs_source_platform-config - xbs-rm - else - xbs_error_platform-not-found + done + elif [ "${rules}" == 'sync' || "${rules}" == 'import' ] && [ "${DISTRO_TYPE}" == 'derivated' ]; then + for '_platform' in "${DD_PLATFORMS[@]}"; do + xbs_variable_platform-config + xbs_source_platform-config + xbs-sync + done fi - done - ;; - sync|import) - # Import and sync the derivated repositories based on info contained in repo.db files - platforms=("${@:2}") - if [ "${#}" -lt '2' ]; then - for '_platform' in "${DD_PLATFORMS[@]}"; do - xbs_variable_platform-config - xbs_source_platform-config - xbs-sync - done else for '_platform' in "${platforms[@]}"; do xbs_rename-variable_platform-config xbs_variable_platform-config - if [ -f "${PLATAFORM_CONFIG}" && "${DISTRO_TYPE}" == 'derivated' ]; then + if [ -f "${PLATAFORM_CONFIG}" ]; then xbs_source_platform-config - xbs-sync + if [ "${rules}" == 'init' ]; then + xbs-init + elif [ "${rules}" == 'sync' || "${rules}" == 'import' ] && [ "${DISTRO_TYPE}" == 'derivated' ]; then + xbs-sync + fi else xbs_error_platform-not-found fi diff --git a/src/etc/xbs.cfg b/src/etc/xbs.cfg index f005a76..5786c33 100644 --- a/src/etc/xbs.cfg +++ b/src/etc/xbs.cfg @@ -5,9 +5,12 @@ ND_PLATFORMS=('gnu+linux_rolling') # 'gnu+linux_testing' 'gnu+linux_stable' 'gnu DD_PLATFORMS=('gnu+linux_abslibre' 'gnu+linux_abslibre-arm') # 'gnu+hurd_abslibre' PLATFORMS=("${ND_PLATFORMS[@]}" "${DD_PLATFORMS[@]}") +PREFIX="$(dirname "$(readlink -e "$0")")" +BIN_DIR="${PREFIX}/bin" +CONFIG_DIR="${PREFIX}/etc" +SHARE_DIR="${PREFIX}/share/xbs" SRV_DIR='/srv' TMP_DIR='/tmp' -SHARE_DIR="$(dirname "$(readlink -e "$0")")/share/xbs" HTTP_DIR="${SRV_DIR}/repo/http" ROOT_DIR="${SRV_DIR}/repo/main" GIT_DIR="${SRV_DIR}/git" -- cgit v1.2.3