From 7b4e592c6605f2a5bc0df7b82ccac1f1bffe656e Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 21 Sep 2013 16:26:16 -0400 Subject: librerelease: Avoid locking the staging directory the whole time. https://labs.parabola.nu/issues/410 --- src/abslibre-tools/librerelease | 74 +++++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 22 deletions(-) diff --git a/src/abslibre-tools/librerelease b/src/abslibre-tools/librerelease index 9278a32..2d2a480 100755 --- a/src/abslibre-tools/librerelease +++ b/src/abslibre-tools/librerelease @@ -38,6 +38,18 @@ . libremessages . $(librelib conf.sh) +dryrun="" +readonly rsync_flags=( + --no-group + --no-perms + --copy-links + --hard-links + --partial + --human-readable + --progress + -e ssh +) + usage() { print "Usage: %s [OPTIONS]" "${0##*/}" echo @@ -51,11 +63,15 @@ usage() { flag '-h' "Show this message" } +list0_files() { + find "${WORKDIR}/staging" -L -type f -print0 +} + list_packages() { find "$WORKDIR/staging/" -mindepth 1 -type d -not -empty -printf '%f\n' | sort | while read -r repo; do msg2 "$repo" - find "${WORKDIR}/staging/${repo}" -type f -printf "%f\n" | sort + find "${WORKDIR}/staging/${repo}" -L -type f -printf "%f\n" | sort done } @@ -111,14 +127,29 @@ clean_non_packages() { } # Clean everything if not on dry-run mode -clean() { +clean_files() { + local file_list=$1 + + local rmcmd=(rm -fv) if [[ -n "${dryrun}" ]]; then - : - else - msg "Removing files from local staging directory" - # use '-exec rm' instead of '-delete' to be verbose - find "${WORKDIR}/staging" -type f -exec rm -fv {} + + rmcmd=(printf "$(_ "removed '%s' (dry-run)")\n") fi + + msg "Removing files from local staging directory" + xargs -0 -a "$file_list" "${rmcmd[@]}" +} + +clean() { + lock 10 "${WORKDIR}/staging.lock" \ + 'Waiting for an exclusive lock on the staging directory' + + local file_list="$(mktemp -t ${0##*/}.XXXXXXXXXX)" + trap "$(printf 'rm -f -- %q' "$file_list")" EXIT + list0_files > "$file_list" + + lock_close 10 + + clean_files "$file_list" } main() { @@ -128,7 +159,6 @@ main() { fi # Parse options - local dryrun="" local mode="release_packages" while getopts 'clnh' arg; do case $arg in @@ -154,10 +184,10 @@ main() { check_vars makepkg GPGKEY load_files libretools check_vars libretools WORKDIR REPODEST || return 1 + REPODEST+='/staging/' # The following variables are actually optional #check_vars libretools HOOKPRERELEASE HOOKPOSTRELEASE || return 1 - lock 10 "${WORKDIR}/staging.lock" 'Waiting for an exclusive lock on the staging directory' "$mode" } @@ -167,6 +197,9 @@ release_packages() { bash -c "${HOOKPRERELEASE}" fi + lock 10 "${WORKDIR}/staging.lock" \ + 'Waiting for an exclusive lock on the staging directory' + clean_non_packages sign_packages || return 1 @@ -176,19 +209,16 @@ release_packages() { find ${WORKDIR}/staging -type f -exec chmod 644 {} + find ${WORKDIR}/staging -type d -exec chmod 755 {} + - msg "%s to upload" $(du -h -d 0 ${WORKDIR}/staging | tr "\t" " " | cut -d" " -f1) + local file_list="$(mktemp -t ${0##*/}.XXXXXXXXXX)" + trap "$(printf 'rm -f -- %q' "$file_list")" EXIT + list0_files > "$file_list" + + lock_close 10 + + msg "%s to upload" "$(du -hc --files0-from="$file_list" | sed -n '$s/\t.*//p')" msg "Uploading packages..." - if ! rsync --recursive \ - ${dryrun} \ - --no-group \ - --no-perms \ - --copy-links \ - --hard-links \ - --partial \ - --prune-empty-dirs \ - --human-readable \ - --progress \ - -e "ssh " \ + if ! rsync ${dryrun} "${rsync_flags[@]}" \ + -0 --files-from="$file_list" ${WORKDIR}/staging \ ${REPODEST}/ then @@ -196,7 +226,7 @@ release_packages() { return 1 fi - clean + clean_files "$file_list" msg "Running db-update on repos" ssh ${REPODEST%%:*} dbscripts/db-update -- cgit v1.2.3-54-g00ecf