From 12b709bd141bf25a97c42034ec77798b3a548105 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?coadde=20=5BM=C3=A1rcio=20Alexandre=20Silva=20Delgado=5D?= Date: Tue, 25 Aug 2015 18:05:37 -0300 Subject: update any-to-ours and fixes --- abslibre | 9 +++--- any-to-ours | 75 +++++++++++++++++++++++++++++++++++-------- config | 11 ------- config_abslibre_gnu+hurd | 17 ++++++++-- config_abslibre_gnu+linux | 17 ++++++++-- config_abslibre_gnu+linux_arm | 17 ++++++++-- config_gnu+hurd | 17 ++++++++-- config_gnu+linux | 17 ++++++++-- create-repo | 2 +- createrepos | 2 +- 10 files changed, 143 insertions(+), 41 deletions(-) diff --git a/abslibre b/abslibre index aab71cc..20d931f 100755 --- a/abslibre +++ b/abslibre @@ -8,7 +8,8 @@ set -e # * Sync bs (abslibre) from bs derivation (abs) excluding from blacklist # * Create repo.abs.tar.gz tarballs -. './config' +platform_name="${platform/\//+}" +source "$(dirname "$(readlink -e "$0")")/config_${platform_name,,}" for 'platform' in "${PLATFORMS[@]}"; do @@ -76,11 +77,11 @@ for 'platform' in "${PLATFORMS[@]}"; do arch="$(basename $(dirname ${repo}))" # Remove the old one - mkdir -p "${FTP_BASE}/${baserepo}/os/${arch}" - rm -fv "${FTP_BASE}/${baserepo}/os/${arch}/${baserepo}.abs.tar.gz" + mkdir -p "${REPO_DIR}/${baserepo}/os/${arch}" + rm -fv "${REPO_DIR}/${baserepo}/os/${arch}/${baserepo}.abs.tar.gz" # Create a new one joining arch and any # Remove the first part of the path (it could be $repo but any isn't hit) - bsdtar -czf "${FTP_BASE}/${baserepo}/os/${arch}/${baserepo}.abs.tar.gz" \ + bsdtar -czf "${REPO_DIR}/${baserepo}/os/${arch}/${baserepo}.abs.tar.gz" \ -s ":${BS_MAIN_DIR}/[a-z0-9_]\+/[a-z]\+::" \ "${repo}"/* "${BS_MAIN_DIR}/any/${baserepo}"/* diff --git a/any-to-ours b/any-to-ours index a901d54..a84c368 100755 --- a/any-to-ours +++ b/any-to-ours @@ -1,5 +1,5 @@ #!/bin/bash -# Releases 'any' packages from Arch arches to ours +# Releases 'any' packages from Arch-derivations arches to ours trap_exit() { echo @@ -7,7 +7,8 @@ trap_exit() { exit 1 } -source "$(dirname "$(readlink -e "$0")")/config" +platform_name="${platform/\//+}" +source "$(dirname "$(readlink -e "$0")")/config_${platform_name,,}" source "$(dirname "$(readlink -e "$0")")/db-libremessages" # From makepkg @@ -17,16 +18,13 @@ trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR -# The architecture to compare with -BASEARCH='x86_64' - # Traverse all Arch repos -for _repo in "${ARCHREPOS[@]}"; do +for '_repo' in "${MAIN_REPOS[@]}"; do msg "Processing %s..." "${_repo}" # Find 'any' packages # This is hardcoded but it could release other arches... - PKGS=($(find "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ + PKGS=($(find "${REPO_DIR}/${_repo}/os/${ARCHES[0]}/" \ -iname '*-any.pkg.tar.?z' \ -printf "%f ")) @@ -35,7 +33,7 @@ for _repo in "${ARCHREPOS[@]}"; do continue fi - for _arch in "${OURARCHES[@]}"; do + for '_arch' in "${ARCHES[@]}"; do msg2 "Syncing %s..." "${_arch}" # Sync 'any' only and extract the synced packages @@ -44,12 +42,12 @@ for _repo in "${ARCHREPOS[@]}"; do --include='*-any.pkg.tar.?z' \ --include='*-any.pkg.tar.?z.sig' \ --exclude='*' \ - "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" 2>&1 | \ + "${REPO_DIR}/${_repo}/os/${ARCHES[0]}/" \ + "${REPO_DIR}/${_repo}/os/${_arch}/" 2>&1 | \ grep 'any\.pkg\.tar\..z$' | \ cut -d ' ' -f 1 )) - if [ ${#SYNCED[@]} -eq 0 ]; then + if [ "${#SYNCED[@]}" -eq 0 ]; then msg2 "Already synced (or error happened)" continue fi @@ -58,14 +56,63 @@ for _repo in "${ARCHREPOS[@]}"; do msg2 "Adding to db..." - pushd "${FTP_BASE}/${_repo}/os/${_arch}/" >/dev/null + pushd "${REPO_DIR}/${_repo}/os/${_arch}/" >/dev/null # Add the packages to the db - repo-add "${_repo}${DBEXT}" "${SYNCED[@]}" + repo-add "${_repo}${DB_EXT}" "${SYNCED[@]}" popd >/dev/null # Avoid mixups - unset SYNCED PKGS + unset 'SYNCED' 'PKGS' done + if [ -n "${MULTILIB}" ]; then + for '_repo_multilib' in "${MULTILIB_REPOS[@]}"; do + msg "Processing %s..." "${_repo}" '(multilib)' + + # Find 'any' packages + # This is hardcoded but it could release other arches... + PKGS_MULTILIB=($(find "${REPO_DIR}/${_repo_multilib}/os/${MULTILIB[0]}/" \ + -iname '*-any.pkg.tar.?z' \ + -printf "%f ")) + + if [ ${#PKGS_MULTILIB[@]} -eq 0 ]; then + msg2 "No '%s' multilib packages here" any + continue + fi + + for '_arch_multilib' in "${MULTILIB[@]}"; do + msg2 "Syncing %s..." "${_arch_multilib}" '(multilib)' + + # Sync 'any' only and extract the synced packages + SYNCED_MULTILIB=($( + rsync -av \ + --include='*-any.pkg.tar.?z' \ + --include='*-any.pkg.tar.?z.sig' \ + --exclude='*' \ + "${REPO_DIR}/${_repo_multilib}/os/${MULTILIB[0]}/" \ + "${REPO_DIR}/${_repo_multilib}/os/${_arch_multilib}/" 2>&1 | \ + grep 'any\.pkg\.tar\..z$' | \ + cut -d ' ' -f 1 )) + + if [ "${#SYNCED_MULTILIB[@]}" -eq 0 ]; then + msg2 "Already synced (or error happened)" + continue + fi + + msg2 "Synced %d packages: %s" "${#SYNCED_MULTILIB[@]}" "${SYNCED_MULTILIB[*]}" + + msg2 "Adding Multilib to db..." + + pushd "${REPO_DIR}/${_repo_multilib}/os/${_arch_multilib}/" >/dev/null + + # Add the packages to the db + repo-add "${_repo_multilib}${DB_EXT}" "${SYNCED_MULTILIB[@]}" + + popd >/dev/null + + # Avoid mixups + unset 'SYNCED_MULTILIB' 'PKGS_MULTILIB' + done + fi done diff --git a/config b/config index 5e13683..d99bd04 100644 --- a/config +++ b/config @@ -15,20 +15,9 @@ SRC_POOL_DIR='sources' SRV_DIR='/srv' TMP_DIR='/tmp' -FTP_BASE="${SRV_DIR}/repo/main" GIT_BASE="${SRV_DIR}/git" SYNC_ARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g --quiet' -CLEANUP_DESTDIR="${FTP_BASE}/old/packages" -CLEANUP_DRYRUN='false' -# Time in days to keep moved packages -CLEANUP_KEEP='30' - -SOURCE_CLEANUP_DESTDIR="${FTP_BASE}/old/sources" -SOURCE_CLEANUP_DRYRUN='true' -# Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP='30' - REQUIRE_SIGNATURE='true' LOCK_DELAY='10' diff --git a/config_abslibre_gnu+hurd b/config_abslibre_gnu+hurd index d1b1b7d..93dc4fa 100644 --- a/config_abslibre_gnu+hurd +++ b/config_abslibre_gnu+hurd @@ -1,6 +1,6 @@ #!/hint/bash -. ./config +source "$(dirname "$(readlink -e "$0")")/config" # Parabola GNU/Hurd configuration with Arch GNU/Hurd derivation DERIVATION='true' @@ -35,7 +35,7 @@ PKG_ANY_REPOS=("${EXTRA_REPOS[@]}" "${PATFORM_REPOS[@]}" "${LIBRE_MAIN_REPOS[@]} PKG_MULTILIB_REPOS=("${EXTRA_REPOS[@]}" "${PATFORM_REPOS[@]}" "${LIBRE_MAIN_REPOS[@]}" "${MAIN_REPOS[@]}" "${LIBRE_MULTILIB_REPOS[@]}" "${MULTILIB_REPOS[@]}" "${COMMUNITY_REPOS[@]}") # Directories where packages are shared between repos -# *relative to FTP_BASE* +# *relative to REPO_DIR* PKG_POOL='abslibre_gnu+hurd' # Directories where sources are stored @@ -53,3 +53,16 @@ BLACKLIST_NAME='blacklist_gnu+hurd.txt' BLACKLIST_FILE="${HOME}/blacklist/${BLACKLIST_NAME}.txt" BLACKLIST_TMP="${TMP_DIR}/${BLACKLIST_NAME}.txt" SYNC_DERIVATION_SERVER='rsync.archhurd.org' + +# Base Repository +REPO_DIR="${SRV_DIR}/repo/abslibre_gnu+hurd" + +CLEANUP_DESTDIR="${REPO_DIR}/old/packages" +CLEANUP_DRYRUN='false' +# Time in days to keep moved packages +CLEANUP_KEEP='30' + +SOURCE_CLEANUP_DESTDIR="${REPO_DIR}/old/sources" +SOURCE_CLEANUP_DRYRUN='true' +# Time in days to keep moved sourcepackages +SOURCE_CLEANUP_KEEP='30' diff --git a/config_abslibre_gnu+linux b/config_abslibre_gnu+linux index bcf8ab0..7a76a02 100644 --- a/config_abslibre_gnu+linux +++ b/config_abslibre_gnu+linux @@ -1,6 +1,6 @@ #!/hint/bash -. ./config +source "$(dirname "$(readlink -e "$0")")/config" # Parabola GNU/Linux-libre configuration with Arch GNU/Linux derivation DERIVATION='true' @@ -35,7 +35,7 @@ PKG_ANY_REPOS=("${EXTRA_REPOS[@]}" "${PATFORM_REPOS[@]}" "${LIBRE_MAIN_REPOS[@]} PKG_MULTILIB_REPOS=("${EXTRA_REPOS[@]}" "${PATFORM_REPOS[@]}" "${LIBRE_MAIN_REPOS[@]}" "${MAIN_REPOS[@]}" "${LIBRE_MULTILIB_REPOS[@]}" "${MULTILIB_REPOS[@]}" "${COMMUNITY_REPOS[@]}") # Directories where packages are shared between repos -# *relative to FTP_BASE* +# *relative to REPO_DIR* PKG_POOL='abslibre_gnu+linux' # Directories where sources are stored @@ -53,3 +53,16 @@ BLACKLIST_NAME='blacklist_gnu+linux.txt' BLACKLIST_FILE="${HOME}/blacklist/${BLACKLIST_NAME}.txt" BLACKLIST_TMP="${TMP_DIR}/${BLACKLIST_NAME}.txt" SYNC_DERIVATION_SERVER='rsync.archlinux.org' + +# Base Repository +REPO_DIR="${SRV_DIR}/repo/abslibre_gnu+linux" + +CLEANUP_DESTDIR="${REPO_DIR}/old/packages" +CLEANUP_DRYRUN='false' +# Time in days to keep moved packages +CLEANUP_KEEP='30' + +SOURCE_CLEANUP_DESTDIR="${REPO_DIR}/old/sources" +SOURCE_CLEANUP_DRYRUN='true' +# Time in days to keep moved sourcepackages +SOURCE_CLEANUP_KEEP='30' diff --git a/config_abslibre_gnu+linux_arm b/config_abslibre_gnu+linux_arm index 7c76002..ad113b4 100644 --- a/config_abslibre_gnu+linux_arm +++ b/config_abslibre_gnu+linux_arm @@ -1,6 +1,6 @@ #!/hint/bash -. ./config +source "$(dirname "$(readlink -e "$0")")/config" # Parabola GNU/Linux-libre configuration with Arch GNU/Linux ARM derivation DERIVATION='true' @@ -35,7 +35,7 @@ PKG_ANY_REPOS=("${EXTRA_REPOS[@]}" "${PATFORM_REPOS[@]}" "${LIBRE_MAIN_REPOS[@]} PKG_MULTILIB_REPOS=("${EXTRA_REPOS[@]}" "${PATFORM_REPOS[@]}" "${LIBRE_MAIN_REPOS[@]}" "${MAIN_REPOS[@]}" "${LIBRE_MULTILIB_REPOS[@]}" "${MULTILIB_REPOS[@]}" "${COMMUNITY_REPOS[@]}") # Directories where packages are shared between repos -# *relative to FTP_BASE* +# *relative to REPO_DIR* PKG_POOL='abslibre_gnu+linux_arm' # Directories where sources are stored @@ -53,3 +53,16 @@ BLACKLIST_NAME='blacklist_gnu+linux_arm.txt' BLACKLIST_FILE="${HOME}/blacklist/${BLACKLIST_NAME}.txt" BLACKLIST_TMP="${TMP_DIR}/${BLACKLIST_NAME}.txt" SYNC_DERIVATION_SERVER='rsync.archlinuxarm.org' + +# Base Repository +REPO_DIR="${SRV_DIR}/repo/abslibre_gnu+linux_arm" + +CLEANUP_DESTDIR="${REPO_DIR}/old/packages" +CLEANUP_DRYRUN='false' +# Time in days to keep moved packages +CLEANUP_KEEP='30' + +SOURCE_CLEANUP_DESTDIR="${REPO_DIR}/old/sources" +SOURCE_CLEANUP_DRYRUN='true' +# Time in days to keep moved sourcepackages +SOURCE_CLEANUP_KEEP='30' diff --git a/config_gnu+hurd b/config_gnu+hurd index dd7f424..9408dee 100644 --- a/config_gnu+hurd +++ b/config_gnu+hurd @@ -1,6 +1,6 @@ #!/hint/bash -. ./config +source "$(dirname "$(readlink -e "$0")")/config" # Parabola GNU/Hurd configuration without any distribution derivation DERIVATION='false' @@ -31,7 +31,7 @@ PKG_ANY_REPOS=("${EXTRA_REPOS[@]}" "${PATFORM_REPOS[@]}" "${MAIN_REPOS[@]}" "${C PKG_MULTILIB_REPOS=("${EXTRA_REPOS[@]}" "${PATFORM_REPOS[@]}" "${MAIN_REPOS[@]}" "${MULTILIB_REPOS[@]}" "${COMMUNITY_REPOS[@]}") # Directories where packages are shared between repos -# *relative to FTP_BASE* +# *relative to REPO_DIR* PKG_POOL='gnu+hurd' # Directories where sources are stored @@ -43,3 +43,16 @@ BS_MAIN_DIR="${SRV_DIR}/${BS_MAIN_NAME}" BS_GIT="${GIT_BASE}/${BS_MAIN_NAME}/pmr.git" # BS_GIT='http://projects.parabola.gnu/${BS_MAIN_NAME}/pmr.git' BS_GIT_TMP="${TMP_DIR}/${BS_MAIN_NAME}" + +# Base Repository +REPO_DIR="${SRV_DIR}/repo/gnu+hurd" + +CLEANUP_DESTDIR="${REPO_DIR}/old/packages" +CLEANUP_DRYRUN='false' +# Time in days to keep moved packages +CLEANUP_KEEP='30' + +SOURCE_CLEANUP_DESTDIR="${REPO_DIR}/old/sources" +SOURCE_CLEANUP_DRYRUN='true' +# Time in days to keep moved sourcepackages +SOURCE_CLEANUP_KEEP='30' diff --git a/config_gnu+linux b/config_gnu+linux index c34b9a6..eee8b11 100644 --- a/config_gnu+linux +++ b/config_gnu+linux @@ -1,6 +1,6 @@ #!/hint/bash -. ./config +source "$(dirname "$(readlink -e "$0")")/config" # Parabola GNU/Linux-libre configuration without any distribution derivation DERIVATION='false' @@ -31,7 +31,7 @@ PKG_ANY_REPOS=("${EXTRA_REPOS[@]}" "${PATFORM_REPOS[@]}" "${MAIN_REPOS[@]}" "${C PKG_MULTILIB_REPOS=("${EXTRA_REPOS[@]}" "${PATFORM_REPOS[@]}" "${MAIN_REPOS[@]}" "${MULTILIB_REPOS[@]}" "${COMMUNITY_REPOS[@]}") # Directories where packages are shared between repos -# *relative to FTP_BASE* +# *relative to REPO_DIR* PKG_POOL='gnu+linux' # Directories where sources are stored @@ -43,3 +43,16 @@ BS_MAIN_DIR="${SRV_DIR}/${BS_MAIN_NAME}" BS_GIT="${GIT_BASE}/${BS_MAIN_NAME}/pmr.git" # BS_GIT='http://projects.parabola.gnu/${BS_MAIN_NAME}/pmr.git' BS_GIT_TMP="${TMP_DIR}/${BS_MAIN_NAME}" + +# Base Repository +REPO_DIR="${SRV_DIR}/repo/gnu+linux" + +CLEANUP_DESTDIR="${REPO_DIR}/old/packages" +CLEANUP_DRYRUN='false' +# Time in days to keep moved packages +CLEANUP_KEEP='30' + +SOURCE_CLEANUP_DESTDIR="${REPO_DIR}/old/sources" +SOURCE_CLEANUP_DRYRUN='true' +# Time in days to keep moved sourcepackages +SOURCE_CLEANUP_KEEP='30' diff --git a/create-repo b/create-repo index 3feb098..dc86722 100755 --- a/create-repo +++ b/create-repo @@ -13,7 +13,7 @@ msg "Creating repos..." for _repo in "$@"; do msg2 "Creating [%s]" "${_repo}" for _arch in "${ARCHES[@]}"; do - mkdir -p "${FTP_BASE}/${_repo}/os/${_arch}" || \ + mkdir -p "${REPO_DIR}/${_repo}/os/${_arch}" || \ error "Failed creating %s dir" "${_arch}" done done diff --git a/createrepos b/createrepos index 8da2455..bee86d2 100755 --- a/createrepos +++ b/createrepos @@ -3,6 +3,6 @@ source "$(dirname "$(readlink -e "$0")")/config" -mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" +mkdir -p -- "${REPO_DIR}"/{"${PKGPOOL}","${SRCPOOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" "$(dirname "$(readlink -e "$0")")/create-repo" "${PKGREPOS[@]}" -- cgit v1.2.3