From 9eb1cd7b9403533c4b60ecfbbbf00a08c211059a Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 15:03:27 +0200 Subject: Move common function to db-functions db-functions now sets an individual $WORKDIR and implements trap functinos that remove locks on exit or error. There are new functions to lock and unlock the running script. misc-scripts/ftpdir-cleanup was renamed to ftpdir-cleanup-repo as the cron-job had the same name. Script names have to be unique when using db-functions. --- misc-scripts/ftpdir-cleanup | 218 --------------------------------------- misc-scripts/ftpdir-cleanup-repo | 207 +++++++++++++++++++++++++++++++++++++ misc-scripts/make-sourceball | 21 +--- misc-scripts/sourceballs-cleanup | 27 +---- 4 files changed, 212 insertions(+), 261 deletions(-) delete mode 100755 misc-scripts/ftpdir-cleanup create mode 100755 misc-scripts/ftpdir-cleanup-repo (limited to 'misc-scripts') diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup deleted file mode 100755 index ac17a8d..0000000 --- a/misc-scripts/ftpdir-cleanup +++ /dev/null @@ -1,218 +0,0 @@ -#!/bin/bash - -if [ $# -ne 1 ]; then - echo "usage: $(basename $0) " - exit 1 -fi - -reponame=$1 - -############################################################ - -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" - -cleanup() { - trap '' 2 15 - for arch in ${ARCHES[@]}; do - repo_unlock $reponame $arch - done - exit 1 -} - -clean_pkgs () { - for pkg in "$@"; do - if [ -h "$pkg" ]; then - rm -f "$pkg" - else - mv "$pkg" "$CLEANUP_DESTDIR" - fi - done -} - -trap cleanup 2 -trap cleanup 15 - -${CLEANUP_DRYRUN} && echo 'dry run mode is active' - -ftppath_base="$FTP_BASE/$reponame/os" - -for arch in ${ARCHES[@]}; do - - repo_lock $reponame $arch $LOCK_TIMEOUT || continue - - CLEANUP_TMPDIR=$(mktemp -d ${TMPDIR}/cleanup-XXXXXX) || exit 1 - ftppath="$ftppath_base/$arch" - MISSINGFILES="" - DELETEFILES="" - DELETESYMLINKS="" - EXTRAFILES="" - - if [ ! -d "$ftppath" ]; then - echo "FTP path '$ftppath' does not exist" - exit 1 - fi - - if ! cd "${CLEANUP_TMPDIR}" ; then - echo "Failed to cd to ${CLEANUP_TMPDIR}" - exit 1 - fi - - if [ ! -f "$ftppath/$reponame$DBEXT" ]; then - echo "" - echo "WARNING: The file \"$ftppath/$reponame$DBEXT\" could not be found, skipping." - echo "" - repo_unlock $reponame $arch - continue - fi - - if ! bsdtar xf "$ftppath/$reponame$DBEXT"; then - echo "" - echo "ERROR: Command failed: bsdtar xf \"$ftppath/$reponame$DBEXT\"" - exit 1 - fi - - cd "$ftppath" - - for pkg in $CLEANUP_TMPDIR/*; do - [ -f "${pkg}" ] || continue - filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) - - if [ ! -e "${filename}" ]; then - MISSINGFILES="${MISSINGFILES} ${filename}" - else - pkgname="$(getpkgname ${filename})" - for otherfile in ${pkgname}-*; do - if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then - if [ -h "${otherfile}" ]; then - DELETESYMLINKS="${DELETESYMLINKS} ${otherfile}" - else - DELETEFILES="${DELETEFILES} ${otherfile}" - fi - fi - done - fi - done - - for pkg in *$PKGEXT; do - if [ ! -e "$pkg" ]; then - continue - fi - pkgname="$(getpkgname $pkg)" - for p in ${CLEANUP_TMPDIR}/${pkgname}-*; do - [ ! -d "${p}" ] || continue 2 - dbpkgname=$(grep -A1 '^%FILENAME%$' "${p}/desc" 2>/dev/null| tail -n1) - if [ "${dbpkgname}" = "${pkgname}" ]; then - continue 2 - fi - done - EXTRAFILES="$EXTRAFILES $pkg" - done - - rm -rf ${CLEANUP_TMPDIR} - - # Do a quick check to see if a missing ARCHINDEPFILE is in the any dir - # If it is, and the file is MISSING, restore it - missfiles="$MISSINGFILES" - MISSINGFILES="" - for mf in $missfiles; do - if [ -e "${ftppath_base}/any/${mf}" ]; then - echo "Restoring missing 'any' symlink: ${mf}" - ${CLEANUP_DRYRUN} || ln -s "../any/${mf}" "${ftppath}" - else - MISSINGFILES="${MISSINGFILES} ${mf}" - fi - done - - repo_unlock $reponame $arch - - #Make sure we've done *something* before outputting anything - if [ -z "$DELETEFILES$DELETESYMLINKS$MISSINGFILES$EXTRAFILES" ]; then - continue - fi - - echo "Scan complete for $reponame ($arch) at ${ftppath}" - - if [ -n "$DELETEFILES" ]; then - echo " The following files are out of date" - for f in $DELETEFILES; do - echo " $f" - done - echo "" - fi - - if [ -n "$DELETESYMLINKS" ]; then - echo " The following symlinks are out of date" - echo " They will be deleted" - for f in $DELETESYMLINKS; do - echo " $f" - done - echo "" - fi - - if [ -n "$MISSINGFILES" ]; then - echo " The following files are missing in the repo" - for f in $MISSINGFILES; do - echo " $f" - done - echo "" - fi - - if [ -n "$EXTRAFILES" ]; then - echo " The following files are in the repo but not the db" - for f in $EXTRAFILES; do - echo " $f" - done - fi - - if [ -n "${DELETEFILES}" ]; then - ${CLEANUP_DRYRUN} || clean_pkgs ${DELETEFILES} - echo "" - fi - - if [ -n "${DELETESYMLINKS}" ]; then - ${CLEANUP_DRYRUN} || clean_pkgs ${DELETESYMLINKS} - echo "" - fi - - if [ -n "${EXTRAFILES}" ]; then - ${CLEANUP_DRYRUN} || clean_pkgs ${EXTRAFILES} - echo "" - fi - -done - -ARCHINDEPFILES="" - -if [ -d "$ftppath_base/any" ]; then - cd "$ftppath_base/any" - for pkg in *$PKGEXT; do - [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" - found=0 - #check for any existing symlinks - for arch in ${ARCHES[@]}; do - if [ -h "$ftppath_base/$arch/$pkg" ]; then - found=1 - break - fi - done - if [ $found -eq 0 ]; then - # We found no symlinks to this, delete it - ARCHINDEPFILES="$ARCHINDEPFILES $pkg" - fi - done -fi - -if [ -n "$ARCHINDEPFILES" ]; then - echo " The following architecture independent packages" - echo " are not symlinked in the architecture repositories." - for f in $ARCHINDEPFILES; do - echo " $f" - done -fi - -if [ -d "$ftppath_base/any" -a -n "${ARCHINDEPFILES}" ]; then - cd "$ftppath_base/any" - ${CLEANUP_DRYRUN} || clean_pkgs ${ARCHINDEPFILES} - echo "" -fi diff --git a/misc-scripts/ftpdir-cleanup-repo b/misc-scripts/ftpdir-cleanup-repo new file mode 100755 index 0000000..12f726c --- /dev/null +++ b/misc-scripts/ftpdir-cleanup-repo @@ -0,0 +1,207 @@ +#!/bin/bash + +if [ $# -ne 1 ]; then + echo "usage: $(basename $0) " + exit 1 +fi + +reponame=$1 + +############################################################ + +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +clean_pkgs () { + for pkg in "$@"; do + if [ -h "$pkg" ]; then + rm -f "$pkg" + else + mv "$pkg" "$CLEANUP_DESTDIR" + fi + done +} + +${CLEANUP_DRYRUN} && echo 'dry run mode is active' + +ftppath_base="$FTP_BASE/$reponame/os" + +for arch in ${ARCHES[@]}; do + + repo_lock $reponame $arch $LOCK_TIMEOUT || continue + + CLEANUP_TMPDIR=$(mktemp -d ${WORKDIR}/cleanup-XXXXXX) || exit 1 + ftppath="$ftppath_base/$arch" + MISSINGFILES="" + DELETEFILES="" + DELETESYMLINKS="" + EXTRAFILES="" + + if [ ! -d "$ftppath" ]; then + echo "FTP path '$ftppath' does not exist" + exit 1 + fi + + if ! cd "${CLEANUP_TMPDIR}" ; then + echo "Failed to cd to ${CLEANUP_TMPDIR}" + exit 1 + fi + + if [ ! -f "$ftppath/$reponame$DBEXT" ]; then + echo "" + echo "WARNING: The file \"$ftppath/$reponame$DBEXT\" could not be found, skipping." + echo "" + repo_unlock $reponame $arch + continue + fi + + if ! bsdtar xf "$ftppath/$reponame$DBEXT"; then + echo "" + echo "ERROR: Command failed: bsdtar xf \"$ftppath/$reponame$DBEXT\"" + exit 1 + fi + + cd "$ftppath" + + for pkg in $CLEANUP_TMPDIR/*; do + [ -f "${pkg}" ] || continue + filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) + + if [ ! -e "${filename}" ]; then + MISSINGFILES="${MISSINGFILES} ${filename}" + else + pkgname="$(getpkgname ${filename})" + for otherfile in ${pkgname}-*; do + if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then + if [ -h "${otherfile}" ]; then + DELETESYMLINKS="${DELETESYMLINKS} ${otherfile}" + else + DELETEFILES="${DELETEFILES} ${otherfile}" + fi + fi + done + fi + done + + for pkg in *$PKGEXT; do + if [ ! -e "$pkg" ]; then + continue + fi + pkgname="$(getpkgname $pkg)" + for p in ${CLEANUP_TMPDIR}/${pkgname}-*; do + [ ! -d "${p}" ] || continue 2 + dbpkgname=$(grep -A1 '^%FILENAME%$' "${p}/desc" 2>/dev/null| tail -n1) + if [ "${dbpkgname}" = "${pkgname}" ]; then + continue 2 + fi + done + EXTRAFILES="$EXTRAFILES $pkg" + done + + rm -rf ${CLEANUP_TMPDIR} + + # Do a quick check to see if a missing ARCHINDEPFILE is in the any dir + # If it is, and the file is MISSING, restore it + missfiles="$MISSINGFILES" + MISSINGFILES="" + for mf in $missfiles; do + if [ -e "${ftppath_base}/any/${mf}" ]; then + echo "Restoring missing 'any' symlink: ${mf}" + ${CLEANUP_DRYRUN} || ln -s "../any/${mf}" "${ftppath}" + else + MISSINGFILES="${MISSINGFILES} ${mf}" + fi + done + + repo_unlock $reponame $arch + + #Make sure we've done *something* before outputting anything + if [ -z "$DELETEFILES$DELETESYMLINKS$MISSINGFILES$EXTRAFILES" ]; then + continue + fi + + echo "Scan complete for $reponame ($arch) at ${ftppath}" + + if [ -n "$DELETEFILES" ]; then + echo " The following files are out of date" + for f in $DELETEFILES; do + echo " $f" + done + echo "" + fi + + if [ -n "$DELETESYMLINKS" ]; then + echo " The following symlinks are out of date" + echo " They will be deleted" + for f in $DELETESYMLINKS; do + echo " $f" + done + echo "" + fi + + if [ -n "$MISSINGFILES" ]; then + echo " The following files are missing in the repo" + for f in $MISSINGFILES; do + echo " $f" + done + echo "" + fi + + if [ -n "$EXTRAFILES" ]; then + echo " The following files are in the repo but not the db" + for f in $EXTRAFILES; do + echo " $f" + done + fi + + if [ -n "${DELETEFILES}" ]; then + ${CLEANUP_DRYRUN} || clean_pkgs ${DELETEFILES} + echo "" + fi + + if [ -n "${DELETESYMLINKS}" ]; then + ${CLEANUP_DRYRUN} || clean_pkgs ${DELETESYMLINKS} + echo "" + fi + + if [ -n "${EXTRAFILES}" ]; then + ${CLEANUP_DRYRUN} || clean_pkgs ${EXTRAFILES} + echo "" + fi + +done + +ARCHINDEPFILES="" + +if [ -d "$ftppath_base/any" ]; then + cd "$ftppath_base/any" + for pkg in *$PKGEXT; do + [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" + found=0 + #check for any existing symlinks + for arch in ${ARCHES[@]}; do + if [ -h "$ftppath_base/$arch/$pkg" ]; then + found=1 + break + fi + done + if [ $found -eq 0 ]; then + # We found no symlinks to this, delete it + ARCHINDEPFILES="$ARCHINDEPFILES $pkg" + fi + done +fi + +if [ -n "$ARCHINDEPFILES" ]; then + echo " The following architecture independent packages" + echo " are not symlinked in the architecture repositories." + for f in $ARCHINDEPFILES; do + echo " $f" + done +fi + +if [ -d "$ftppath_base/any" -a -n "${ARCHINDEPFILES}" ]; then + cd "$ftppath_base/any" + ${CLEANUP_DRYRUN} || clean_pkgs ${ARCHINDEPFILES} + echo "" +fi diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index e216356..6e57385 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -22,21 +22,7 @@ _arch="$3" srcpath="$FTP_BASE/sources/" logpath="/var/log/sourceballs/" -cleanup() { - restore_umask - rm -rf "$WORKDIR" - [ "$1" ] && exit $1 -} - -ctrl_c() { - echo "Interrupted" >&2 - cleanup 0 -} - -die() { - echo -e "$*" >&2 - cleanup 1 -} +script_lock create_srcpackage() { if [ -d "$1" ]; then @@ -71,9 +57,6 @@ create_srcpackage() { fi } -trap ctrl_c 2 -trap cleanup 0 1 - set_umask /bin/mkdir -p "$logpath" cd "$WORKDIR" @@ -91,3 +74,5 @@ else die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'" fi fi + +script_unlock diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup index c87a888..e058c38 100755 --- a/misc-scripts/sourceballs-cleanup +++ b/misc-scripts/sourceballs-cleanup @@ -6,19 +6,7 @@ srcpath="$FTP_BASE/sources/" logfile="$srcpath/cleanup.txt" -LOCKFILE="/tmp/.sourceball-cleanup.lock" - -cleanup () { - restore_umask - rm -rf "$WORKDIR" - rm -f "$LOCKFILE" - exit 0 -} - -ctrl_c() { - echo "Interrupted" >&2 - cleanup 0 -} +script_lock remove_old() { if [ -d "$1" ]; then @@ -52,17 +40,6 @@ remove_old() { fi } -if [ -f "$LOCKFILE" ]; then - owner="$(/usr/bin/stat -c %U $LOCKFILE)" - echo "error: source tarball generation is already in progress (started by $owner)" - exit 1 -fi - -trap cleanup 0 -trap ctrl_c 2 - -/bin/touch "$LOCKFILE" - #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null @@ -91,4 +68,4 @@ for sourceball in "$srcpath"/*$SRCEXT; do fi done -cleanup 0 +script_unlock -- cgit v1.2.3-54-g00ecf