diff options
Diffstat (limited to 'cron-jobs')
| -rwxr-xr-x | cron-jobs/repo-sanity-check | 56 | ||||
| -rwxr-xr-x | cron-jobs/sourceballs | 30 | ||||
| -rwxr-xr-x | cron-jobs/sourceballs2 | 91 | ||||
| -rwxr-xr-x | cron-jobs/update-abs-tarballs | 7 |
4 files changed, 172 insertions, 12 deletions
diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check new file mode 100755 index 0000000..1ba90a6 --- /dev/null +++ b/cron-jobs/repo-sanity-check @@ -0,0 +1,56 @@ +#!/bin/bash +# Solves issue165 + +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +# Traverse all repos +for _repo in ${PKGREPOS[@]}; do + msg "Cleaning up [${_repo}]" + +# Find all pkgnames on this repo's abs + on_abs=($( + find ${SVNREPO}/${_repo} -name PKGBUILD | \ + while read pkgbuild; do + source ${pkgbuild} >/dev/null 2>&1 +# cleanup to save memory + unset build package source md5sums pkgdesc pkgver pkgrel epoch \ + url license arch depends makedepends optdepends options \ + >/dev/null 2>&1 + +# also cleanup package functions + for _pkg in ${pkgname[@]}; do + unset package_${pkg} >/dev/null 2>&1 + done + +# this fills the on_abs array + echo ${pkgname[@]} + done + )) + +# quit if abs is empty + if [ ${#on_abs[*]} -eq 0 ]; then + warning "[${_repo}]'s ABS tree is empty, skipping" + break + fi + +# Find all pkgnames on repos + on_repo=($( + find ${FTP_BASE}/${_repo} -name "*.pkg.tar.?z" -printf "%f\n" | \ + sed "s/^\(.\+\)-[^-]\+-[^-]\+-[^-]\+$/\1/" + )) + +# Compares them, whatever is on repos but not on abs should be removed + remove=($(comm -13 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \ + <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u))) + +# Remove them from databases, ftpdir-cleanup will take care of the rest + find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -exec \ + repo-remove {} ${remove[@]} >/dev/null 2>&1 \; + + msg2 "Removed the following packages:" + plain "$(echo ${remove[@]} | tr ' ' "\n")" + +done + +exit $? diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b55de05..ee074bd 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -21,10 +21,10 @@ renice +10 -p $$ > /dev/null for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do # Repo does not exist; skip it - if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + if [ ! -f "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi - bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + bsdtar -xOf "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ | awk '/^%NAME%/ { getline b }; /^%BASE%/ { getline b }; /^%VERSION%/ { getline v }; @@ -46,7 +46,7 @@ for repo in ${PKGREPOS[@]}; do done # Create a list of all available source package file names -find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do @@ -59,14 +59,15 @@ for repo in ${PKGREPOS[@]}; do pkgarch=${pkginfo[2]} pkglicense=(${pkginfo[@]:3}) - # Should this package be skipped? + # Should this packages be skipped? if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then continue fi + # Commenting out, we'll sourceball everything # Check if the license or .force file does not enforce creating a source package - if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then - continue - fi +# if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then +# continue +# fi # Store the expected file name of the source package echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" @@ -79,8 +80,13 @@ for repo in ${PKGREPOS[@]}; do # Get the sources from svn mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" - svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ - "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 + #svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ + # "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 + + # If it's on official repos, nor [libre], nor [libre-testing] + cp -r "${SVNREPO}/$repo/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ + cp -r "${SVNREPO}/libre/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ + cp -r "${SVNREPO}/libre-testing/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 if [ $? -ge 1 ]; then failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" continue @@ -88,9 +94,9 @@ for repo in ${PKGREPOS[@]}; do # Build the actual source package pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null - makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1 + makepkg --nocolor --allsource --ignorearch # >/dev/null 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then - mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + mv "${pkgbase}-${pkgver}${SRCEXT}" "${ARCH_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" @@ -126,7 +132,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then - mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi done diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 new file mode 100755 index 0000000..5e228fc --- /dev/null +++ b/cron-jobs/sourceballs2 @@ -0,0 +1,91 @@ +#!/bin/bash +# Steps +# Traverse ABSLibre +# Makepkg --allsource every package +# Remove the old sourceballs + +dirname="$(dirname $(readlink -e $0))" +. "${dirname}/../db-functions" +. "${dirname}/../config" +. "${MAKEPKGCONF}" + +pushd "${WORKDIR}" >/dev/null + +script_lock + +#adjust the nice level to run at a lower priority +renice +10 -p $$ > /dev/null + +# Create a list of all available source package file names +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" + +pushd "${SVNREPO}" >/dev/null + +for repo in ${PKGREPOS[@]}; do + msg "Sourceballing [${repo}]" + + pushd $repo >/dev/null + find -maxdepth 1 -type d | while read pkg; do + pushd "${SVNREPO}/$repo/$pkg" >/dev/null + + [[ ! -e PKGBUILD ]] && { + warning "$repo/$pkg is not a package" + continue + } + +# Unset the previous data + unset pkgbase pkgname pkgver pkgrel + source PKGBUILD + + unset build package url pkgdesc source md5sums depends makedepends \ + optdepends license arch options check mksource + + for _pkg in ${pkgname[@]}; do + unset package_${_pkg} >/dev/null 2>&1 + done + + pkgbase=${pkgbase:-$pkgname} + srcfile="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" + + echo "${srcfile}" >> "${WORKDIR}/expected-src-pkgs" + + # Skip already sourceballed + [ -e "${SRCPKGDEST}/${srcfile}" ] && continue + + makepkg --allsource --ignorearch -c >/dev/null 2>&1 + + [ $? -ne 0 ] && plain ${srcfile} + + done # end find pkgs + popd >/dev/null + +done # end repos + +# Cleanup old source packages +cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) + +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages..." + ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + if ! ${SOURCE_CLEANUP_DRYRUN}; then + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + fi + done +fi + +old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages from the cleanup directory..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + done +fi + +script_unlock + diff --git a/cron-jobs/update-abs-tarballs b/cron-jobs/update-abs-tarballs new file mode 100755 index 0000000..824ac34 --- /dev/null +++ b/cron-jobs/update-abs-tarballs @@ -0,0 +1,7 @@ +#!/bin/bash + +. "$(dirname $0)/../config" + +rsync -av --exclude=staging/ parabolagnulinux.org::abstar/ ${FTP_BASE}/ + +exit $? |