From acaa58d1833510973cf41fa8349fd3be39c596d8 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 19:11:28 -0400 Subject: Backport changes from the lukeshu/xbs branch. Notable, remove the files.db stuff that became extraneous with pacman 5. --- config | 4 ++ cron-jobs/devlist-mailer | 5 +-- cron-jobs/ftpdir-cleanup | 10 ++++- cron-jobs/sourceballs | 11 +++-- cron-jobs/sourceballs.skip | 24 ++++++++--- db-functions | 105 +++++++-------------------------------------- db-move | 4 +- 7 files changed, 57 insertions(+), 106 deletions(-) diff --git a/config b/config index 041c9eb..43bef5e 100644 --- a/config +++ b/config @@ -34,3 +34,7 @@ DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" + +# Where to send error emails, and who they are from +LIST="maintenance@lists.parabola.nu" +FROM="repo@parabola.nu" diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index 1a05521..7f298b9 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -2,9 +2,8 @@ #Dummy helper to send email to arch-dev # It does nothing if no output -LIST="arch-dev-public@archlinux.org" -#LIST="aaronmgriffin@gmail.com" -FROM="repomaint@archlinux.org" +# Load $LIST and $FROM from the config file +. "$(dirname "$(readlink -e "$0")")/../config" SUBJECT="Repository Maintenance $(date +"%d-%m-%Y")" if [ $# -ge 1 ]; then diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 4a2b418..4063c09 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -22,6 +22,14 @@ clean_pkg() { fi } +script_lock + +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + repo_lock "${repo}" "${arch}" || exit 1 + done +done + "${CLEANUP_DRYRUN}" && warning 'dry run mode is active' for repo in "${PKGREPOS[@]}"; do @@ -56,7 +64,7 @@ done # get a list of all available packages in the pacakge pool find "$FTP_BASE/${PKGPOOL}" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/pool" # create a list of packages in our db -cat "${WORKDIR}/db-"* | sort -u > "${WORKDIR}/db" +find "${WORKDIR}" -maxdepth 1 -type f -name 'db-*' -exec cat {} \; | sort -u > "${WORKDIR}/db" old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db")) if [ ${#old_pkgs[@]} -ge 1 ]; then diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index c12a128..c02912a 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -63,11 +63,10 @@ for repo in "${PKGREPOS[@]}"; do if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then continue fi - # Commenting out, we'll sourceball everything # Check if the license or .force file does not enforce creating a source package -# if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then -# continue -# fi + if ! ([[ -z ${ALLOWED_LICENSES[*]} ]] || chk_license "${pkglicense[@]}" || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then + continue + fi # Store the expected file name of the source package echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" @@ -118,8 +117,8 @@ for repo in "${PKGREPOS[@]}"; do done # Cleanup old source packages -cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" -cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +find "${WORKDIR}" -maxdepth 1 -type f -name 'expected-src-pkgs' -exec cat {} \; | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +find "${WORKDIR}" -maxdepth 1 -type f -name 'available-src-pkgs' -exec cat {} \; | sort -u > "${WORKDIR}/available-src-pkgs.sort" old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) if [ ${#old_pkgs[@]} -ge 1 ]; then diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip index 14d6f4b..0e1731c 100644 --- a/cron-jobs/sourceballs.skip +++ b/cron-jobs/sourceballs.skip @@ -1,14 +1,28 @@ -nexuiz-data +0ad-data +alienarena-data +blobwars-data +btanks-data +dangerdeep-data +egoboo-data +fillets-ng-data +flightgear-data +frogatto-data +gcompris-data +naev-data +openarena-data +rocksndiamonds-data +smc-data +speed-dreams-data torcs-data tremulous-data ufoai-data -frogatto-data vdrift-data -naev-data -btanks-data +warmux-data wesnoth-data -texlive-bin +widelands-data +xonotic-data texlive-bibtexextra +texlive-bin texlive-core texlive-fontsextra texlive-formatsextra diff --git a/db-functions b/db-functions index d76aa41..885e825 100644 --- a/db-functions +++ b/db-functions @@ -1,7 +1,7 @@ #!/hint/bash # Some PKGBUILDs need CARCH to be set -CARCH="x86_64" +CARCH=$(. "$(librelib conf.sh)"; load_files makepkg; echo "$CARCH") # Useful functions UMASK="" @@ -26,73 +26,21 @@ mv_acl() { # set up general environment WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") +if [ -n "${SVNUSER}" ]; then + setfacl -m u:"${SVNUSER}":rwx "${WORKDIR}" + setfacl -m d:u:"${USER}":rwx "${WORKDIR}" + setfacl -m d:u:"${SVNUSER}":rwx "${WORKDIR}" +fi LOCKS=() REPO_MODIFIED=0 -# check if messages are to be printed using color -unset ALL_OFF BOLD BLUE GREEN RED YELLOW -if [[ -t 2 ]]; then - ALL_OFF="$(tput sgr0)" - BOLD="$(tput bold)" - BLUE="${BOLD}$(tput setaf 4)" - GREEN="${BOLD}$(tput setaf 2)" - RED="${BOLD}$(tput setaf 1)" - YELLOW="${BOLD}$(tput setaf 3)" -fi -readonly ALL_OFF BOLD BLUE GREEN RED YELLOW - -plain() { - local mesg=$1; shift - printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" -} - -msg() { - local mesg=$1; shift - printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" -} - -msg2() { - local mesg=$1; shift - printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" -} - -warning() { - local mesg=$1; shift - printf "${YELLOW}==> WARNING:${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -error() { - local mesg=$1; shift - printf "${RED}==> ERROR${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -## -# usage : in_array( $needle, $haystack ) -# return : 0 - found -# 1 - not found -## -in_array() { - local needle=$1; shift - [[ -z $1 ]] && return 1 # Not Found - local item - for item in "$@"; do - [[ $item = "$needle" ]] && return 0 # Found - done - return 1 # Not Found -} - -## -# usage : get_full_version( $epoch, $pkgver, $pkgrel ) -# return : full version spec, including epoch (if necessary), pkgver, pkgrel -## -get_full_version() { - if [[ $1 -eq 0 ]]; then - # zero epoch case, don't include it in version - echo "$2-$3" - else - echo "$1:$2-$3" - fi -} +# Used: plain, msg, msg2, warning, error, in_array, get_full_version, abort, die +# Overwritten: cleanup +# Ignored: stat_busy, stat_done, +# setup_workdir, trap_abort, trap_exit, +# lock, slock, lock_close +# pkgver_equal, find_cached_package, check_root +. "$(librelib common)" script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" @@ -141,22 +89,11 @@ cleanup() { if (( REPO_MODIFIED )); then date +%s > "${FTP_BASE}/lastupdate" - date -u +%s > "${FTP_BASE}/lastsync" fi [ "$1" ] && exit "$1" } -abort() { - msg 'Aborting...' - cleanup 0 -} - -die() { - error "$@" - cleanup 1 -} - trap abort INT QUIT TERM HUP trap cleanup EXIT @@ -165,7 +102,6 @@ trap cleanup EXIT repo_lock () { local LOCKDIR="$TMPDIR/.repolock.$1.$2" local DBLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${DBEXT}.lck" - local FILESLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${FILESEXT}.lck" local _count local _trial local _timeout @@ -177,10 +113,6 @@ repo_lock () { error "Repo [%s] (%s) is already locked by repo-{add,remove} process %s" "$1" "$2" "$(<"$DBLOCKFILE")" return 1 fi - if [ -f "${FILESLOCKFILE}" ]; then - error "Repo [%s] (%s) is already locked by repo-{add,remove} process %s" "$1" "$2" "$(<"$FILESLOCKFILE")" - return 1 - fi if [ $# -eq 2 ]; then _lockblock=true @@ -340,7 +272,7 @@ check_pkgfile() { in_array "${pkgarch}" "${ARCHES[@]}" 'any' || return 1 - if echo "${pkgfile##*/}" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then + if echo "${pkgfile##*/}" | grep "${pkgname}-${pkgver}-${pkgarch}" &>/dev/null; then return 0 else return 1 @@ -492,10 +424,8 @@ arch_repo_add() { printf -v pkgs_str -- '%q ' "${pkgs[@]}" # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - /usr/bin/repo-add -q "${repo}${DBEXT}" "${pkgs[@]}" >/dev/null \ + /usr/bin/repo-add -q "${repo}${DBEXT}" "${pkgs[@]}" \ || error 'repo-add %q %s' "${repo}${DBEXT}" "${pkgs_str% }" - /usr/bin/repo-add -f -q "${repo}${FILESEXT}" "${pkgs[@]}" \ - || error 'repo-add -f %q %s' "${repo}${FILESEXT}" "${pkgs_str% }" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -507,17 +437,14 @@ arch_repo_remove() { local arch=$2 local pkgs=("${@:3}") local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" - local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" if [ ! -f "${dbfile}" ]; then error "No database found at '%s'" "${dbfile}" return 1 fi printf -v pkgs_str -- '%q ' "${pkgs[@]}" - /usr/bin/repo-remove -q "${dbfile}" "${pkgs[@]}" >/dev/null \ + /usr/bin/repo-remove -q "${dbfile}" "${pkgs[@]}" \ || error 'repo-remove %q %s' "${dbfile}" "${pkgs_str% }" - /usr/bin/repo-remove -q "${filesfile}" "${pkgs[@]}" \ - || error 'repo-remove %q %s' "${filesfile}" "${pkgs_str% }" set_repo_permission "${repo}" "${arch}" REPO_MODIFIED=1 diff --git a/db-move b/db-move index 275a11a..82403c1 100755 --- a/db-move +++ b/db-move @@ -34,7 +34,7 @@ for pkgbase in "${args[@]:2}"; do die "Could not read pkgname" fi - pkgver=$(. "${xbsrepo_from}/PKGBUILD"; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}") + pkgver=$(. "${xbsrepo_from}/PKGBUILD"; get_full_version) if [ -z "${pkgver}" ]; then die "Could not read pkgver" fi @@ -74,7 +74,7 @@ for pkgbase in "${args[@]:2}"; do fi msg2 '%s (%s)' "${pkgbase}" "${tarches[*]}" pkgnames=($(. "${xbsrepo_to}/PKGBUILD"; echo "${pkgname[@]}")) - pkgver=$(. "${xbsrepo_to}/PKGBUILD"; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}") + pkgver=$(. "${xbsrepo_to}/PKGBUILD"; get_full_version) for pkgname in "${pkgnames[@]}"; do for tarch in "${tarches[@]}"; do -- cgit v1.2.3