summaryrefslogtreecommitdiff
path: root/cron-jobs/ftpdir-cleanup
diff options
context:
space:
mode:
authorPierre Schmitz <pierre@archlinux.de>2010-08-18 16:20:27 +0200
committerPierre Schmitz <pierre@archlinux.de>2010-08-18 16:20:27 +0200
commit683db123710ff9092ae99929fce232620932fdb0 (patch)
treed27df12e90e3d1d1de5b5d52d9a6c83afd8c2fc5 /cron-jobs/ftpdir-cleanup
parent8647de4f52e05eadc0a4832813e9a57c20a487dd (diff)
Rewrite ftpdir-cleanup
* runtime reduced to < 1 second * uses file lists and the comm command * searches for missing packages * removes empty legacy directories like extra/os/any
Diffstat (limited to 'cron-jobs/ftpdir-cleanup')
-rwxr-xr-xcron-jobs/ftpdir-cleanup116
1 files changed, 92 insertions, 24 deletions
diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup
index 3456e9c..0a43bd1 100755
--- a/cron-jobs/ftpdir-cleanup
+++ b/cron-jobs/ftpdir-cleanup
@@ -3,37 +3,105 @@
. "$(dirname $0)/../db-functions"
. "$(dirname $0)/../config"
-repos="$(get_repos_for_host)"
+clean_pkg() {
+ local pkg
+ local target
+
+ if ! ${CLEANUP_DRYRUN}; then
+ for pkg in "$@"; do
+ if [ -h "$pkg" ]; then
+ rm -f "$pkg"
+ else
+ mv -f "$pkg" "$CLEANUP_DESTDIR"
+ fi
+ done
+ fi
+}
+
+repos=($(get_repos_for_host))
script_lock
-#adjust the nice level to run at a lower priority
-/usr/bin/renice +10 -p $$ > /dev/null
+for repo in ${repos[@]}; do
+ for arch in ${ARCHES[@]}; do
+ repo_lock ${repo} ${arch} || exit 1
+ done
+done
+
+${CLEANUP_DRYRUN} && warning 'dry run mode is active'
-repopaths=''
-for repo in $repos; do
- $(dirname $0)/../misc-scripts/ftpdir-cleanup-repo $repo
- repopaths="${repopaths} ${FTP_BASE}/${repo}/os/"
+for repo in ${repos[@]}; do
+ for arch in ${ARCHES[@]}; do
+ if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then
+ warning "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} not found, skipping"
+ continue
+ fi
+ # get a list of actual available package files
+ find "${FTP_BASE}/${repo}/os/${arch}" -xtype f -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/repo-${repo}-${arch}"
+ # get a list of package files defined in the repo db
+ bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}"
+
+ missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}"))
+ if [ ${#missing_pkgs[@]} -ge 1 ]; then
+ error "Missing packages in [${repo}] (${arch})..."
+ for missing_pkg in ${missing_pkgs[@]}; do
+ msg2 "${missing_pkg}"
+ done
+ fi
+
+ old_pkgs=($(comm -23 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}"))
+ if [ ${#old_pkgs[@]} -ge 1 ]; then
+ msg "Removing old packages from [${repo}] (${arch})..."
+ for old_pkg in ${old_pkgs[@]}; do
+ msg2 "${old_pkg}"
+ clean_pkg "${FTP_BASE}/${repo}/os/${arch}/${old_pkg}"
+ done
+ fi
+ done
done
-to_cleanup=""
-poolpath="$FTP_BASE/$(get_pkgpool_for_host)/"
-pushd $poolpath >/dev/null
-for pkg in *$PKGEXT; do
-[ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz"
-LINKS="$(/usr/bin/find $repopaths -type l -name "$pkg" 2>/dev/null)"
-if [ -z "$LINKS" ]; then
- to_cleanup="$to_cleanup $poolpath/$pkg"
+# get a list of all available packages in the pacakge pool
+find "$FTP_BASE/$(get_pkgpool_for_host)" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/pool"
+# create a list of packages in our db
+cat "${WORKDIR}/db-"* | sort -u > "${WORKDIR}/db"
+
+old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db"))
+if [ ${#old_pkgs[@]} -ge 1 ]; then
+ msg "Removing old packages from package pool..."
+ for old_pkg in ${old_pkgs[@]}; do
+ msg2 "${old_pkg}"
+ clean_pkg "$FTP_BASE/$(get_pkgpool_for_host)/${old_pkg}"
+ done
fi
+
+# cleanup of legacy $repo/os/any directories
+for repo in ${repos[@]}; do
+ if [ ! -d "${FTP_BASE}/${repo}/os/any" ]; then
+ continue
+ fi
+ if [ -n "$(find "${FTP_BASE}/${repo}/os/any" -type d -empty)" ]; then
+ msg "Removing empty legacy directory ${repo}/os/any"
+ ${CLEANUP_DRYRUN} && rmdir "${FTP_BASE}/${repo}/os/any"
+ continue
+ fi
+ find "${FTP_BASE}/${repo}/os/any" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/any-${repo}"
+ cat "${WORKDIR}/db-${repo}-"* | sort -u > "${WORKDIR}/all-${repo}"
+
+ old_pkgs=($(comm -23 "${WORKDIR}/any-${repo}" "${WORKDIR}/all-${repo}"))
+ if [ ${#old_pkgs[@]} -ge 1 ]; then
+ msg "Removing old packages from [${repo}] (any)..."
+ for old_pkg in ${old_pkgs[@]}; do
+ msg2 "${old_pkg}"
+ clean_pkg "${FTP_BASE}/${repo}/os/any/${old_pkg}"
+ done
+ fi
+done
+
+
+for repo in ${repos[@]}; do
+ for arch in ${ARCHES[@]}; do
+ repo_unlock ${repo} ${arch}
+ done
done
-popd >/dev/null
-
-if [ -n "$to_cleanup" ]; then
- msg "The following packages are no longer in any repo"
- for f in $to_cleanup; do
- msg2 "$(basename "$f")"
- done
- ${CLEANUP_DRYRUN} || mv $to_cleanup "$CLEANUP_DESTDIR"
-fi
script_unlock