From 64effec1bf3c96fb75fe1aa6ddefa5a43db3e230 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sat, 16 Apr 2011 11:11:51 -0700 Subject: Using own db-update and ftpdir-cleanup --- cron-jobs/ftpdir-cleanup | 113 +++++++++++++++++++++++++++++++++++++++++++++++ db-update | 76 +++++++++++++++++++++++++++++++ repo-update | 2 +- 3 files changed, 190 insertions(+), 1 deletion(-) create mode 100755 cron-jobs/ftpdir-cleanup create mode 100755 db-update diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup new file mode 100755 index 0000000..09e8a49 --- /dev/null +++ b/cron-jobs/ftpdir-cleanup @@ -0,0 +1,113 @@ +#!/bin/bash + +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +clean_pkg() { + local pkg + local target + + if ! ${CLEANUP_DRYRUN}; then + for pkg in "$@"; do + if [ -h "$pkg" ]; then + rm -f "$pkg" + else + mv -f "$pkg" "$CLEANUP_DESTDIR" + touch "${CLEANUP_DESTDIR}/$(basename ${pkg})" + fi + done + fi +} + +script_lock + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_lock ${repo} ${arch} || exit 1 + done +done + +${CLEANUP_DRYRUN} && warning 'dry run mode is active' + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + continue + fi + # get a list of actual available package files + find "${FTP_BASE}/${repo}/os/${arch}" -xtype f -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/repo-${repo}-${arch}" + # get a list of package files defined in the repo db + bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}" + + missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) + if [ ${#missing_pkgs[@]} -ge 1 ]; then + error "Missing packages in [${repo}] (${arch})..." + for missing_pkg in ${missing_pkgs[@]}; do + msg2 "${missing_pkg}" + done + fi + + old_pkgs=($(comm -23 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) + if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old packages from [${repo}] (${arch})..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + clean_pkg "${FTP_BASE}/${repo}/os/${arch}/${old_pkg}" + done + fi + done +done + +# get a list of all available packages in the pacakge pool +find "$FTP_BASE/${PKGPOOL}" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/pool" +# create a list of packages in our db +cat "${WORKDIR}/db-"* | sort -u > "${WORKDIR}/db" + +old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db")) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old packages from package pool..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + clean_pkg "$FTP_BASE/${PKGPOOL}/${old_pkg}" + done +fi + +# cleanup of legacy $repo/os/any directories +for repo in ${PKGREPOS[@]}; do + if [ ! -d "${FTP_BASE}/${repo}/os/any" ]; then + continue + fi + if [ -n "$(find "${FTP_BASE}/${repo}/os/any" -type d -empty)" ]; then + msg "Removing empty legacy directory ${repo}/os/any" + ${CLEANUP_DRYRUN} || rmdir "${FTP_BASE}/${repo}/os/any" + continue + fi + find "${FTP_BASE}/${repo}/os/any" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/any-${repo}" + cat "${WORKDIR}/db-${repo}-"* | sort -u > "${WORKDIR}/all-${repo}" + + old_pkgs=($(comm -23 "${WORKDIR}/any-${repo}" "${WORKDIR}/all-${repo}")) + if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old packages from [${repo}] (any)..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + clean_pkg "${FTP_BASE}/${repo}/os/any/${old_pkg}" + done + fi +done + +old_pkgs=($(find ${CLEANUP_DESTDIR} -type f -name "*${PKGEXT}" -mtime +${CLEANUP_KEEP} -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old packages from the cleanup directory..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${CLEANUP_DRYRUN} || rm -f "${CLEANUP_DESTDIR}/${old_pkg}" + done +fi + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_unlock ${repo} ${arch} + done +done + +script_unlock diff --git a/db-update b/db-update new file mode 100755 index 0000000..4740809 --- /dev/null +++ b/db-update @@ -0,0 +1,76 @@ +#!/bin/bash + +. "$(dirname $0)/db-functions" +. "$(dirname $0)/config" + +if [ $# -ge 1 ]; then + warning "Calling $(basename $0) with a specific repository is no longer supported" + exit 1 +fi + +# Find repos with packages to release +repos=($(find "${STAGING}" -mindepth 1 -type d ! -empty -printf '%f ' 2>/dev/null)) +if [ $? -ge 1 ]; then + die "Could not read ${STAGING}" +fi + +# TODO: this might lock too much (architectures) +for repo in ${repos[@]}; do + for pkgarch in ${ARCHES[@]}; do + repo_lock ${repo} ${pkgarch} || exit 1 + done +done + +# check if packages are valid +for repo in ${repos[@]}; do + if ! check_repo_permission "${repo}"; then + die "You don't have permission to update packages in ${repo}" + fi + pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) + if [ $? -eq 0 ]; then + for pkg in ${pkgs[@]}; do + if [ -h "${pkg}" ]; then + die "Package ${repo}/$(basename ${pkg}) is a symbolic link" + fi + if ! check_pkgfile "${pkg}"; then + die "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data" + fi + #if ! check_pkgrepos "${pkg}"; then + # die "Package ${repo}/$(basename ${pkg}) already exists in another repository" + #fi + done + if ! check_splitpkgs ${repo} ${pkgs[@]}; then + die "Missing split packages for ${repo}" + fi + else + die "Could not read ${STAGING}" + fi +done + +for repo in ${repos[@]}; do + msg "Updating [${repo}]..." + any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null)) + for pkgarch in ${ARCHES[@]}; do + add_pkgs=() + arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-${pkgarch}${PKGEXT} 2>/dev/null)) + for pkg in ${arch_pkgs[@]} ${any_pkgs[@]}; do + pkgfile="$(basename ${pkg})" + msg2 "${pkgfile} (${pkgarch})" + # any packages might have been moved by the previous run + if [ -f "${pkg}" ]; then + mv "${pkg}" "$FTP_BASE/${PKGPOOL}" + fi + ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" + add_pkgs[${#add_pkgs[*]}]=${pkgfile} + done + if [ ${#add_pkgs[@]} -ge 1 ]; then + arch_repo_add "${repo}" "${pkgarch}" ${add_pkgs[@]} + fi + done +done + +for repo in ${repos[@]}; do + for pkgarch in ${ARCHES[@]}; do + repo_unlock ${repo} ${pkgarch} + done +done diff --git a/repo-update b/repo-update index 7038789..ff7f8e0 100755 --- a/repo-update +++ b/repo-update @@ -4,7 +4,7 @@ source ~/.bashrc # the PATH is updated here # Updates free/ databases and releases packages on free/staging # Also moves old packages to the old/packages dir -db-update +~/repm/db-update ~/repm/cron-jobs/ftpdir-cleanup # Syncs repos -- cgit v1.2.3