diff options
Diffstat (limited to 'cron-jobs')
| -rwxr-xr-x | cron-jobs/check_archlinux/check_packages.py | 2 | ||||
| -rwxr-xr-x | cron-jobs/check_archlinux/parse_pkgbuilds.sh | 64 | ||||
| -rwxr-xr-x | cron-jobs/devlist-mailer | 13 | ||||
| -rwxr-xr-x | cron-jobs/ftpdir-cleanup | 59 | ||||
| -rwxr-xr-x | cron-jobs/integrity-check | 8 | ||||
| -rwxr-xr-x | cron-jobs/make_repo_torrents | 70 | ||||
| -rw-r--r-- | cron-jobs/makepkg.conf | 121 | ||||
| -rwxr-xr-x | cron-jobs/sourceballs | 76 | ||||
| -rwxr-xr-x | cron-jobs/update-web-db | 78 | ||||
| l--------- | cron-jobs/update-web-files-db | 1 |
10 files changed, 176 insertions, 316 deletions
diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index d233bf6..ac0194f 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2 +#!/usr/bin/env python2 # # check_archlinux.py # diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index 3f92169..b857ac8 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -6,23 +6,23 @@ exit() { return; } splitpkg_overrides=('depends' 'optdepends' 'provides' 'conflicts') -variables=('pkgname' 'pkgbase' 'epoch' 'pkgver' 'pkgrel' 'makedepends' 'arch' ${splitpkg_overrides[@]}) +variables=('pkgname' 'pkgbase' 'epoch' 'pkgver' 'pkgrel' 'makedepends' 'arch' "${splitpkg_overrides[@]}") readonly -a variables splitpkg_overrides backup_package_variables() { - for var in ${splitpkg_overrides[@]}; do + for var in "${splitpkg_overrides[@]}"; do indirect="${var}_backup" - eval "${indirect}=(\${$var[@]})" + eval "${indirect}=(\"\${$var[@]}\")" done } restore_package_variables() { - for var in ${splitpkg_overrides[@]}; do + for var in "${splitpkg_overrides[@]}"; do indirect="${var}_backup" if [ -n "${!indirect}" ]; then - eval "${var}=(\${$indirect[@]})" + eval "${var}=(\"\${$indirect[@]}\")" else - unset ${var} + unset "${var}" fi done } @@ -42,31 +42,31 @@ print_info() { if [ -n "$arch" ]; then echo "%ARCH%" - for i in ${arch[@]}; do echo $i; done + for i in "${arch[@]}"; do echo "$i"; done echo "" fi if [ -n "$depends" ]; then echo "%DEPENDS%" - for i in ${depends[@]}; do - echo $i + for i in "${depends[@]}"; do + echo "$i" done echo "" fi if [ -n "$makedepends" ]; then echo "%MAKEDEPENDS%" - for i in ${makedepends[@]}; do - echo $i + for i in "${makedepends[@]}"; do + echo "$i" done echo "" fi if [ -n "$conflicts" ]; then echo "%CONFLICTS%" - for i in ${conflicts[@]}; do echo $i; done + for i in "${conflicts[@]}"; do echo "$i"; done echo "" fi if [ -n "$provides" ]; then echo "%PROVIDES%" - for i in ${provides[@]}; do echo $i; done + for i in "${provides[@]}"; do echo "$i"; done echo "" fi } @@ -75,10 +75,10 @@ source_pkgbuild() { ret=0 dir=$1 pkgbuild=$dir/PKGBUILD - for var in ${variables[@]}; do - unset ${var} + for var in "${variables[@]}"; do + unset "${var}" done - source $pkgbuild &>/dev/null || ret=$? + source "$pkgbuild" &>/dev/null || ret=$? # ensure $pkgname and $pkgver variables were found if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then @@ -88,8 +88,8 @@ source_pkgbuild() { if [ "${#pkgname[@]}" -gt "1" ]; then pkgbase=${pkgbase:-${pkgname[0]}} - for pkg in ${pkgname[@]}; do - if [ "$(type -t package_${pkg})" != "function" ]; then + for pkg in "${pkgname[@]}"; do + if [ "$(type -t "package_${pkg}")" != "function" ]; then echo -e "%INVALID%\n$pkgbuild\n" return 1 else @@ -98,13 +98,13 @@ source_pkgbuild() { while IFS= read -r line; do var=${line%%=*} var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters - for realvar in ${variables[@]}; do + for realvar in "${variables[@]}"; do if [ "$var" == "$realvar" ]; then eval $line break fi done - done < <(type package_${pkg}) + done < <(type "package_${pkg}") print_info restore_package_variables fi @@ -118,20 +118,20 @@ source_pkgbuild() { } find_pkgbuilds() { - #Skip over some dirs - local d="${1##*/}" - if [ "$d" = "CVS" -o "$d" = ".svn" ]; then - return - fi - - if [ -f $1/PKGBUILD ]; then - source_pkgbuild $1 + #Skip over some dirs + local d="${1##*/}" + if [ "$d" = "CVS" -o "$d" = ".svn" ]; then + return + fi + + if [ -f "$1/PKGBUILD" ]; then + source_pkgbuild "$1" return fi empty=1 - for dir in $1/*; do - if [ -d $dir ]; then - find_pkgbuilds $dir + for dir in "$1"/*; do + if [ -d "$dir" ]; then + find_pkgbuilds "$dir" unset empty fi done @@ -147,7 +147,7 @@ fi CARCH=$1 shift for dir in "$@"; do - find_pkgbuilds $dir + find_pkgbuilds "$dir" done exit 0 diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index ca2e46b..7f298b9 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -2,27 +2,26 @@ #Dummy helper to send email to arch-dev # It does nothing if no output -LIST="arch-dev-public@archlinux.org" -#LIST="aaronmgriffin@gmail.com" -FROM="repomaint@archlinux.org" +# Load $LIST and $FROM from the config file +. "$(dirname "$(readlink -e "$0")")/../config" SUBJECT="Repository Maintenance $(date +"%d-%m-%Y")" if [ $# -ge 1 ]; then - SUBJECT="$1 $(date +"%d-%m-%Y")" + SUBJECT="$1 $(date +"%d-%m-%Y")" fi if [ $# -ge 2 ]; then - LIST="$2" + LIST="$2" fi stdin="$(cat)" #echo used to strip whitespace for checking for actual data if [ -n "$(echo $stdin)" ]; then -echo "Subject: $SUBJECT + echo "Subject: $SUBJECT To: $LIST From: $FROM -$stdin" | /usr/sbin/sendmail -F$FROM "$LIST" +$stdin" | /usr/sbin/sendmail -F"$FROM" "$LIST" fi diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index e1294bd..4063c09 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,22 +1,13 @@ #!/bin/bash -. "$(dirname $0)/../config" -. "$(dirname $0)/../db-functions" - -# just like mv -f, but we touch the file and then copy the content so -# default ACLs in the target dir will be applied -mv_acl() { - rm -f "$2" - touch "$2" - cat "$1" >"$2" || return 1 - rm -f "$1" -} +. "$(dirname "$(readlink -e "$0")")/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" clean_pkg() { local pkg local target - if ! ${CLEANUP_DRYRUN}; then + if ! "${CLEANUP_DRYRUN}"; then for pkg in "$@"; do if [ -h "$pkg" ]; then rm -f "$pkg" "$pkg.sig" @@ -33,16 +24,16 @@ clean_pkg() { script_lock -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do - repo_lock ${repo} ${arch} || exit 1 +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + repo_lock "${repo}" "${arch}" || exit 1 done done -${CLEANUP_DRYRUN} && warning 'dry run mode is active' +"${CLEANUP_DRYRUN}" && warning 'dry run mode is active' -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi @@ -53,17 +44,17 @@ for repo in ${PKGREPOS[@]}; do missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) if [ ${#missing_pkgs[@]} -ge 1 ]; then - error "Missing packages in [${repo}] (${arch})..." - for missing_pkg in ${missing_pkgs[@]}; do - msg2 "${missing_pkg}" + error "Missing packages in [%s] (%s)..." "${repo}" "${arch}" + for missing_pkg in "${missing_pkgs[@]}"; do + msg2 '%s' "${missing_pkg}" done fi old_pkgs=($(comm -23 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) if [ ${#old_pkgs[@]} -ge 1 ]; then - msg "Removing old packages from [${repo}] (${arch})..." - for old_pkg in ${old_pkgs[@]}; do - msg2 "${old_pkg}" + msg "Removing old packages from [%s] (%s)..." "${repo}" "${arch}" + for old_pkg in "${old_pkgs[@]}"; do + msg2 '%s' "${old_pkg}" clean_pkg "${FTP_BASE}/${repo}/os/${arch}/${old_pkg}" done fi @@ -73,32 +64,32 @@ done # get a list of all available packages in the pacakge pool find "$FTP_BASE/${PKGPOOL}" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/pool" # create a list of packages in our db -cat "${WORKDIR}/db-"* | sort -u > "${WORKDIR}/db" +find "${WORKDIR}" -maxdepth 1 -type f -name 'db-*' -exec cat {} \; | sort -u > "${WORKDIR}/db" old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db")) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old packages from package pool..." - for old_pkg in ${old_pkgs[@]}; do - msg2 "${old_pkg}" + for old_pkg in "${old_pkgs[@]}"; do + msg2 '%s' "${old_pkg}" clean_pkg "$FTP_BASE/${PKGPOOL}/${old_pkg}" done fi -old_pkgs=($(find ${CLEANUP_DESTDIR} -type f -name "*${PKGEXT}" -mtime +${CLEANUP_KEEP} -printf '%f\n')) +old_pkgs=($(find "${CLEANUP_DESTDIR}" -type f -name "*${PKGEXT}" -mtime +"${CLEANUP_KEEP}" -printf '%f\n')) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old packages from the cleanup directory..." - for old_pkg in ${old_pkgs[@]}; do - msg2 "${old_pkg}" - if ! ${CLEANUP_DRYRUN}; then + for old_pkg in "${old_pkgs[@]}"; do + msg2 '%s' "${old_pkg}" + if ! "${CLEANUP_DRYRUN}"; then rm -f "${CLEANUP_DESTDIR}/${old_pkg}" rm -f "${CLEANUP_DESTDIR}/${old_pkg}.sig" fi done fi -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do - repo_unlock ${repo} ${arch} +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + repo_unlock "${repo}" "${arch}" done done diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index f6c26cf..7459380 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -1,6 +1,6 @@ #!/bin/bash -dirname="$(dirname $0)" +dirname="$(dirname "$(readlink -e "$0")")" . "${dirname}/../config" . "${dirname}/../db-functions" @@ -8,17 +8,17 @@ dirname="$(dirname $0)" script_lock if [ $# -ne 1 ]; then - die "usage: ${0##*/} <mailto>" + die "usage: %s <mailto>" "${0##*/}" fi mailto=$1 check() { - ${dirname}/check_archlinux/check_packages.py \ + "${dirname}"/check_archlinux/check_packages.py \ --repos="${repos}" \ --abs-tree="/srv/abs/rsync/${arch},/srv/abs/rsync/any" \ --repo-dir="${FTP_BASE}" \ --arch="${arch}" \ - 2>&1 | ${dirname}/devlist-mailer "Integrity Check ${arch}: ${repos}" "${mailto}" + 2>&1 | "${dirname}"/devlist-mailer "Integrity Check ${arch}: ${repos}" "${mailto}" } repos='core,extra,community' diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents new file mode 100755 index 0000000..2eb0978 --- /dev/null +++ b/cron-jobs/make_repo_torrents @@ -0,0 +1,70 @@ +#!/bin/bash +# Copyright (C) 2014 Joseph Graham <joseph@t67.eu> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# This script finds any updated packages and calls +# `make_indivudual_torrent' for each of them. + +username=$( id -un ) + +case "${username}" in + repo | root ) + true + ;; + * ) + echo "This script must be run as repo user or root user." + echo "ByeBye!" + exit 1 + ;; +esac + +# pacman doesn't support multiple different packages of the same name, +# so it's OK to just stuff all the torrents into a single directory. +script_directory="$(dirname "$(readlink -e "$0")")/.." +. "$(dirname "$(readlink -e "$0")")/../config" +public_location="$FTP_BASE/" +torrent_location="$FTP_BASE/torrents/" + +cd "${torrent_location}" + +# Find any directories that might have packages in then +find "${public_location}" -name 'os' -type 'd' | +while read dir +do + # Find any packages + find "${dir}" -name '*\.pkg\.tar\.xz' | + while read pkg + do + pkg_name="${pkg##*/}" + + if [[ -h "${pkg}" ]] # check if it's a symbolic link + then + # We get the target of the symlink + pkg=$( readlink -f "${pkg}" ) + fi + + # If a .torrent file does not already exist for this package, we call + # `make_individual_torrent' to make it. + if ! [[ -f "${torrent_location}${pkg_name}.torrent" ]] + then + "$script_directory/make_individual_torrent" "${pkg}" "${public_location}" + fi + done +done + +if [[ "${username}" == root ]] +then + chown repo * +fi diff --git a/cron-jobs/makepkg.conf b/cron-jobs/makepkg.conf deleted file mode 100644 index 603edba..0000000 --- a/cron-jobs/makepkg.conf +++ /dev/null @@ -1,121 +0,0 @@ -# -# /etc/makepkg.conf -# - -######################################################################### -# SOURCE ACQUISITION -######################################################################### -# -#-- The download utilities that makepkg should use to acquire sources -# Format: 'protocol::agent' -DLAGENTS=('ftp::/usr/bin/curl -sS -fC - --ftp-pasv --retry 3 --retry-delay 3 -o %o %u' - 'http::/usr/bin/curl -sS -fLC - --retry 3 --retry-delay 3 -o %o %u' - 'https::/usr/bin/curl -sS -fLC - --retry 3 --retry-delay 3 -o %o %u' - 'rsync::/usr/bin/rsync -q --no-motd -z %u %o' - 'scp::/usr/bin/scp -q -C %u %o') - -# Other common tools: -# /usr/bin/snarf -# /usr/bin/lftpget -c -# /usr/bin/wget - -######################################################################### -# ARCHITECTURE, COMPILE FLAGS -######################################################################### -# -CARCH="x86_64" -CHOST="x86_64-unknown-linux-gnu" - -#-- Compiler and Linker Flags -# -march (or -mcpu) builds exclusively for an architecture -# -mtune optimizes for an architecture, but builds for whole processor family -CFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fstack-protector --param=ssp-buffer-size=4 -D_FORTIFY_SOURCE=2" -CXXFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fstack-protector --param=ssp-buffer-size=4 -D_FORTIFY_SOURCE=2" -LDFLAGS="-Wl,-O1,--sort-common,--as-needed,-z,relro,--hash-style=gnu" -#-- Make Flags: change this for DistCC/SMP systems -#MAKEFLAGS="-j2" - -######################################################################### -# BUILD ENVIRONMENT -######################################################################### -# -# Defaults: BUILDENV=(fakeroot !distcc color !ccache check !sign) -# A negated environment option will do the opposite of the comments below. -# -#-- fakeroot: Allow building packages as a non-root user -#-- distcc: Use the Distributed C/C++/ObjC compiler -#-- color: Colorize output messages -#-- ccache: Use ccache to cache compilation -#-- check: Run the check() function if present in the PKGBUILD -#-- sign: Generate PGP signature file -# -BUILDENV=(fakeroot !distcc color !ccache check !sign) -# -#-- If using DistCC, your MAKEFLAGS will also need modification. In addition, -#-- specify a space-delimited list of hosts running in the DistCC cluster. -#DISTCC_HOSTS="" -# -#-- Specify a directory for package building. -#BUILDDIR=/tmp/makepkg - -######################################################################### -# GLOBAL PACKAGE OPTIONS -# These are default values for the options=() settings -######################################################################### -# -# Default: OPTIONS=(strip docs libtool emptydirs zipman purge !upx) -# A negated option will do the opposite of the comments below. -# -#-- strip: Strip symbols from binaries/libraries -#-- docs: Save doc directories specified by DOC_DIRS -#-- libtool: Leave libtool (.la) files in packages -#-- emptydirs: Leave empty directories in packages -#-- zipman: Compress manual (man and info) pages in MAN_DIRS with gzip -#-- purge: Remove files specified by PURGE_TARGETS -#-- upx: Compress binary executable files using UPX -# -OPTIONS=(strip docs libtool emptydirs zipman purge !upx) - -#-- File integrity checks to use. Valid: md5, sha1, sha256, sha384, sha512 -INTEGRITY_CHECK=(md5) -#-- Options to be used when stripping binaries. See `man strip' for details. -STRIP_BINARIES="--strip-all" -#-- Options to be used when stripping shared libraries. See `man strip' for details. -STRIP_SHARED="--strip-unneeded" -#-- Options to be used when stripping static libraries. See `man strip' for details. -STRIP_STATIC="--strip-debug" -#-- Manual (man and info) directories to compress (if zipman is specified) -MAN_DIRS=({usr{,/local}{,/share},opt/*}/{man,info}) -#-- Doc directories to remove (if !docs is specified) -DOC_DIRS=(usr/{,local/}{,share/}{doc,gtk-doc} opt/*/{doc,gtk-doc}) -#-- Files to be removed from all packages (if purge is specified) -PURGE_TARGETS=(usr/{,share}/info/dir .packlist *.pod) - -######################################################################### -# PACKAGE OUTPUT -######################################################################### -# -# Default: put built package and cached source in build directory -# -#-- Destination: specify a fixed directory where all packages will be placed -#PKGDEST=/home/packages -#-- Source cache: specify a fixed directory where source files will be cached -#SRCDEST=/home/sources -#-- Source packages: specify a fixed directory where all src packages will be placed -#SRCPKGDEST=/home/srcpackages -#-- Packager: name/email of the person or organization building packages -#PACKAGER="John Doe <john@doe.com>" -#-- Specify a key to use for package signing -#GPGKEY="" - -######################################################################### -# EXTENSION DEFAULTS -######################################################################### -# -# WARNING: Do NOT modify these variables unless you know what you are -# doing. -# -PKGEXT='.pkg.tar.xz' -SRCEXT='.src.tar.gz' - -# vim: set ft=sh ts=2 sw=2 et: diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 6f75ccc..c02912a 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,15 +1,15 @@ #!/bin/bash -dirname="$(dirname $(readlink -e $0))" +dirname="$(dirname "$(readlink -e "$0")")" . "${dirname}/../config" . "${dirname}/../db-functions" pushd "${WORKDIR}" >/dev/null script_lock -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do - repo_lock ${repo} ${arch} || exit 1 +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + repo_lock "${repo}" "${arch}" || exit 1 done done @@ -18,8 +18,8 @@ renice +10 -p $$ > /dev/null # Create a readable file for each repo with the following format # <pkgbase|pkgname> <pkgver>-<pkgrel> <arch> <license>[ <license>] -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do # Repo does not exist; skip it if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue @@ -39,9 +39,9 @@ for repo in ${PKGREPOS[@]}; do done | sort -u > "${WORKDIR}/db-${repo}" done -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do - repo_unlock ${repo} ${arch} +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + repo_unlock "${repo}" "${arch}" done done @@ -49,22 +49,22 @@ done find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package -for repo in ${PKGREPOS[@]}; do +for repo in "${PKGREPOS[@]}"; do newpkgs=() failedpkgs=() while read line; do - pkginfo=(${line}) + pkginfo=("${line}") pkgbase=${pkginfo[0]} pkgver=${pkginfo[1]} pkgarch=${pkginfo[2]} - pkglicense=(${pkginfo[@]:3}) + pkglicense=("${pkginfo[@]:3}") # Should this package be skipped? if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then continue fi # Check if the license or .force file does not enforce creating a source package - if ! ([[ -z ${ALLOWED_LICENSES[@]} ]] || chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then + if ! ([[ -z ${ALLOWED_LICENSES[*]} ]] || chk_license "${pkglicense[@]}" || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then continue fi # Store the expected file name of the source package @@ -73,29 +73,29 @@ for repo in ${PKGREPOS[@]}; do # Build the source package if its not already there if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then # Check if we had failed before - if in_array "${pkgbase}-${pkgver}${SRCEXT}" ${failedpkgs[@]}; then + if in_array "${pkgbase}-${pkgver}${SRCEXT}" "${failedpkgs[@]}"; then continue fi - # Get the sources from svn + # Get the sources from xbs mkdir -p -m0770 "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" - arch_svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ + cp -a "$(xbs releasepath "${pkgbase}" "${repo}" "${pkgarch}")" \ "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 if [ $? -ge 1 ]; then - failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" + failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") continue fi # Build the actual source package pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null - makepkg --nocolor --allsource --ignorearch --skippgpcheck --config "${dirname}/makepkg.conf" >"${WORKDIR}/${pkgbase}.log" 2>&1 + SRCPKGDEST=. makepkg --nocolor --allsource --ignorearch --skippgpcheck >"${WORKDIR}/${pkgbase}.log" 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then - mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + mv_acl "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}/${pkgbase}-${pkgver}${SRCEXT}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" - newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" + newpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") else - failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" + failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") cat "${WORKDIR}/${pkgbase}.log" >> "${WORKDIR}/makepkg-fail.log" fi popd >/dev/null @@ -103,42 +103,42 @@ for repo in ${PKGREPOS[@]}; do done < "${WORKDIR}/db-${repo}" if [ ${#newpkgs[@]} -ge 1 ]; then - msg "Adding source packages for [${repo}]..." - for new_pkg in ${newpkgs[@]}; do - msg2 "${new_pkg}" + msg "Adding source packages for [%s]..." "${repo}" + for new_pkg in "${newpkgs[@]}"; do + msg2 '%s' "${new_pkg}" done fi if [ ${#failedpkgs[@]} -ge 1 ]; then - msg "Failed to create source packages for [${repo}]..." - for failed_pkg in ${failedpkgs[@]}; do - msg2 "${failed_pkg}" + msg "Failed to create source packages for [%s]..." "${repo}" + for failed_pkg in "${failedpkgs[@]}"; do + msg2 '%s' "${failed_pkg}" done fi done # Cleanup old source packages -cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" -cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +find "${WORKDIR}" -maxdepth 1 -type f -name 'expected-src-pkgs' -exec cat {} \; | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +find "${WORKDIR}" -maxdepth 1 -type f -name 'available-src-pkgs' -exec cat {} \; | sort -u > "${WORKDIR}/available-src-pkgs.sort" old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages..." - ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' - for old_pkg in ${old_pkgs[@]}; do - msg2 "${old_pkg}" - if ! ${SOURCE_CLEANUP_DRYRUN}; then - mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + "${SOURCE_CLEANUP_DRYRUN}" && warning 'dry run mode is active' + for old_pkg in "${old_pkgs[@]}"; do + msg2 '%s' "${old_pkg}" + if ! "${SOURCE_CLEANUP_DRYRUN}"; then + mv_acl "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi done fi -old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) +old_pkgs=($(find "${SOURCE_CLEANUP_DESTDIR}" -type f -name "*${SRCEXT}" -mtime +"${SOURCE_CLEANUP_KEEP}" -printf '%f\n')) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages from the cleanup directory..." - for old_pkg in ${old_pkgs[@]}; do - msg2 "${old_pkg}" - ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + for old_pkg in "${old_pkgs[@]}"; do + msg2 '%s' "${old_pkg}" + "${SOURCE_CLEANUP_DRYRUN}" || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" done fi diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db deleted file mode 100755 index 195d1fc..0000000 --- a/cron-jobs/update-web-db +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -. "$(dirname $0)/../config" -. "$(dirname $0)/../db-functions" - -# setup paths -SPATH="/srv/http/archweb" -ENVPATH="/srv/http/archweb-env/bin/activate" - -# having "more important repos" last should make [core] trickle to the top of -# the updates list each hour rather than being overwhelmed by big [extra] and -# [community] updates -REPOS=('community-testing' 'multilib-testing' 'multilib' 'community' 'extra' 'testing' 'core') -LOGOUT="/tmp/archweb_update.log" - -# figure out what operation to perform -cmd="${0##*/}" -if [[ $cmd != "update-web-db" && $cmd != "update-web-files-db" ]]; then - die "Invalid command name '$cmd' specified!" -fi - -script_lock - -# run at nice 5. it can churn quite a bit of cpu after all. -renice +5 -p $$ > /dev/null - -echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" - -# source our virtualenv if it exists -if [ -f "$ENVPATH" ]; then - . "$ENVPATH" -fi - -case "$cmd" in - update-web-db) - dbfileext="${DBEXT}" - flags="" - ;; - update-web-files-db) - dbfileext="${FILESEXT}" - flags="--filesonly" - ;; -esac - -# Lock the repos and get a copy of the db files to work on -for repo in ${REPOS[@]}; do - for arch in ${ARCHES[@]}; do - repo_lock ${repo} ${arch} || exit 1 - dbfile="/srv/ftp/${repo}/os/${arch}/${repo}${dbfileext}" - if [ -f "${dbfile}" ]; then - mkdir -p "${WORKDIR}/${repo}/${arch}" - cp "${dbfile}" "${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" - fi - repo_unlock ${repo} ${arch} - done -done - -# Run reporead on our db copy -pushd $SPATH >/dev/null -for repo in ${REPOS[@]}; do - for arch in ${ARCHES[@]}; do - dbcopy="${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" - if [ -f "${dbcopy}" ]; then - echo "Updating ${repo}-${arch}" >> "${LOGOUT}" - ./manage.py reporead ${flags} ${arch} "${dbcopy}" >> "${LOGOUT}" 2>&1 - echo "" >> "${LOGOUT}" - fi - done -done -popd >/dev/null -echo "" >> "${LOGOUT}" - -# rotate the file if it is getting big (> 10M), overwriting any old backup -if [[ $(stat -c%s "${LOGOUT}") -gt 10485760 ]]; then - mv "${LOGOUT}" "${LOGOUT}.old" -fi - -script_unlock diff --git a/cron-jobs/update-web-files-db b/cron-jobs/update-web-files-db deleted file mode 120000 index 0c2c4fa..0000000 --- a/cron-jobs/update-web-files-db +++ /dev/null @@ -1 +0,0 @@ -update-web-db
\ No newline at end of file |