diff options
| -rw-r--r-- | config | 9 | ||||
| -rwxr-xr-x | create-repo | 24 | ||||
| -rwxr-xr-x | cron-jobs/check_archlinux/parse_pkgbuilds.sh | 8 | ||||
| -rwxr-xr-x | cron-jobs/repo-sanity-check | 56 | ||||
| -rwxr-xr-x | cron-jobs/sourceballs2 | 43 | ||||
| -rwxr-xr-x | cron-jobs/update-abs-tarballs | 7 | ||||
| -rwxr-xr-x | db-check-nonfree | 12 | ||||
| -rwxr-xr-x | db-remove | 15 | ||||
| -rwxr-xr-x | repo-restore-to-normal | 58 | ||||
| -rwxr-xr-x | repo-update | 17 | ||||
| -rw-r--r-- | test/lib/common.inc | 2 | ||||
| -rwxr-xr-x | test/test.d/packages.sh | 4 |
12 files changed, 197 insertions, 58 deletions
@@ -3,8 +3,15 @@ FTP_BASE="/home/repo/public" ARCH_BASE="/home/repo/public" SVNREPO="/home/repo/abslibre" +# Repos from Arch ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') -PKGREPOS=(${ARCHREPOS[@]} 'libre' 'libre-testing' 'social' 'sugar') +# Official Parabola repos +OURREPOS=('libre' 'libre-testing') +# User repos +USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm') +# Community project repos +PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'social') +PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) PKGPOOL='pool/packages' SRCPOOL='sources/packages' diff --git a/create-repo b/create-repo new file mode 100755 index 0000000..58842c3 --- /dev/null +++ b/create-repo @@ -0,0 +1,24 @@ +#!/bin/bash +# Creates repository structure + +. "$(dirname $0)/db-functions" +. "$(dirname $0)/config" + +if [ $# -eq 0 ]; then + msg "Usage: $0 repo1 [repo2 ... repoX]" + exit 1 +fi + +msg "Creating repos..." +for _repo in $@; do + msg2 "Creating [${_repo}]" + mkdir -p "${FTP_BASE}/staging/${_repo}" || \ + error "Failed creating staging dir" + + for _arch in ${ARCHES[@]}; do + mkdir -p "${FTP_BASE}/${_repo}/os/${_arch}" || \ + error "Failed creating ${_arch} dir" + done +done + +msg "Don't forget to add them to the PKGREPOS array on $(dirname $0)/config" diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index 5cd17e4..3f92169 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -6,7 +6,7 @@ exit() { return; } splitpkg_overrides=('depends' 'optdepends' 'provides' 'conflicts') -variables=('pkgname' 'pkgbase' 'pkgver' 'pkgrel' 'makedepends' 'arch' ${splitpkg_overrides[@]}) +variables=('pkgname' 'pkgbase' 'epoch' 'pkgver' 'pkgrel' 'makedepends' 'arch' ${splitpkg_overrides[@]}) readonly -a variables splitpkg_overrides backup_package_variables() { @@ -29,7 +29,11 @@ restore_package_variables() { print_info() { echo -e "%NAME%\n$pkgname\n" - echo -e "%VERSION%\n$pkgver-$pkgrel\n" + if [ -n "$epoch" ]; then + echo -e "%VERSION%\n$epoch:$pkgver-$pkgrel\n" + else + echo -e "%VERSION%\n$pkgver-$pkgrel\n" + fi echo -e "%PATH%\n$dir\n" if [ -n "$pkgbase" ]; then diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check new file mode 100755 index 0000000..1ba90a6 --- /dev/null +++ b/cron-jobs/repo-sanity-check @@ -0,0 +1,56 @@ +#!/bin/bash +# Solves issue165 + +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +# Traverse all repos +for _repo in ${PKGREPOS[@]}; do + msg "Cleaning up [${_repo}]" + +# Find all pkgnames on this repo's abs + on_abs=($( + find ${SVNREPO}/${_repo} -name PKGBUILD | \ + while read pkgbuild; do + source ${pkgbuild} >/dev/null 2>&1 +# cleanup to save memory + unset build package source md5sums pkgdesc pkgver pkgrel epoch \ + url license arch depends makedepends optdepends options \ + >/dev/null 2>&1 + +# also cleanup package functions + for _pkg in ${pkgname[@]}; do + unset package_${pkg} >/dev/null 2>&1 + done + +# this fills the on_abs array + echo ${pkgname[@]} + done + )) + +# quit if abs is empty + if [ ${#on_abs[*]} -eq 0 ]; then + warning "[${_repo}]'s ABS tree is empty, skipping" + break + fi + +# Find all pkgnames on repos + on_repo=($( + find ${FTP_BASE}/${_repo} -name "*.pkg.tar.?z" -printf "%f\n" | \ + sed "s/^\(.\+\)-[^-]\+-[^-]\+-[^-]\+$/\1/" + )) + +# Compares them, whatever is on repos but not on abs should be removed + remove=($(comm -13 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \ + <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u))) + +# Remove them from databases, ftpdir-cleanup will take care of the rest + find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -exec \ + repo-remove {} ${remove[@]} >/dev/null 2>&1 \; + + msg2 "Removed the following packages:" + plain "$(echo ${remove[@]} | tr ' ' "\n")" + +done + +exit $? diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index b29b396..5e228fc 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -22,10 +22,10 @@ find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort pushd "${SVNREPO}" >/dev/null for repo in ${PKGREPOS[@]}; do - failedpkgs=() + msg "Sourceballing [${repo}]" pushd $repo >/dev/null - find . -maxdepth 1 -type d | while read pkg; do + find -maxdepth 1 -type d | while read pkg; do pushd "${SVNREPO}/$repo/$pkg" >/dev/null [[ ! -e PKGBUILD ]] && { @@ -33,35 +33,33 @@ for repo in ${PKGREPOS[@]}; do continue } - unset pkgbase pkgname +# Unset the previous data + unset pkgbase pkgname pkgver pkgrel source PKGBUILD + + unset build package url pkgdesc source md5sums depends makedepends \ + optdepends license arch options check mksource + + for _pkg in ${pkgname[@]}; do + unset package_${_pkg} >/dev/null 2>&1 + done + pkgbase=${pkgbase:-$pkgname} + srcfile="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" - echo "${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" + echo "${srcfile}" >> "${WORKDIR}/expected-src-pkgs" # Skip already sourceballed - [[ -e "${SRCPKGDEST}/${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" ]] && \ - continue + [ -e "${SRCPKGDEST}/${srcfile}" ] && continue - msg2 "$pkgbase-$pkgver-$pkgrel..." - makepkg --allsource --ignorearch -c makepkg --allsource --ignorearch -c >/dev/null 2>&1 - [[ $? -ne 0 ]] && { - warning "Failed." - failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" - } + [ $? -ne 0 ] && plain ${srcfile} - done + done # end find pkgs popd >/dev/null - if [ ${#failedpkgs[@]} -ge 1 ]; then - msg "Failed to create source packages for [${repo}]..." - for failed_pkg in ${failedpkgs[@]}; do - msg2 "${failed_pkg}" - done - fi -done +done # end repos # Cleanup old source packages cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" @@ -89,10 +87,5 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then done fi -msg "Failed" -for _fail in ${failedpkgs[@]}; do - msg2 "$_fail" -done - script_unlock diff --git a/cron-jobs/update-abs-tarballs b/cron-jobs/update-abs-tarballs new file mode 100755 index 0000000..824ac34 --- /dev/null +++ b/cron-jobs/update-abs-tarballs @@ -0,0 +1,7 @@ +#!/bin/bash + +. "$(dirname $0)/../config" + +rsync -av --exclude=staging/ parabolagnulinux.org::abstar/ ${FTP_BASE}/ + +exit $? diff --git a/db-check-nonfree b/db-check-nonfree index d8f34f0..ab6491d 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -23,11 +23,17 @@ for repo in ${ARCHREPOS[@]}; do if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then continue fi + unset dbpkgs unset cleanpkgs - cmd_="$(dirname $0)/list_nonfree_in_db.py -k ${BLACKLIST_FILE} \ - -b ${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" - cleanpkgs=($(${cmd_})) + cleanpkgs=() + dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) + for pkgname in ${dbpkgs[@]}; do + if in_array ${pkgname} ${nonfree[@]}; then + cleanpkgs+=(${pkgname}) + fi + done if [ ${#cleanpkgs[@]} -ge 1 ]; then + msg2 "Unfree: ${cleanpkgs[@]}" arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} fi done @@ -12,9 +12,6 @@ pkgbase="$1" repo="$2" arch="$3" -ftppath="$FTP_BASE/$repo/os" -svnrepo="$repo-$arch" - if ! check_repo_permission $repo; then die "You don't have permission to remove packages from ${repo}" fi @@ -30,14 +27,14 @@ for tarch in ${tarches[@]}; do done msg "Removing $pkgbase from [$repo]..." -/usr/bin/svn checkout -q "${SVNREPO}/${pkgbase}" "${WORKDIR}/svn/${pkgbase}" >/dev/null -if [ -d "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" ]; then - pkgnames=($(. "${WORKDIR}/svn/$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgname[@]})) - /usr/bin/svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" - /usr/bin/svn commit -q "${WORKDIR}/svn/$pkgbase" -m "$(basename $0): $pkgbase removed by $(id -un)" +if [ -d "${SVNREPO}/$repo/$pkgbase" ]; then + pkgnames=($(. "${SVNREPO}/$repo/$pkgbase/PKGBUILD"; echo ${pkgname[@]})) else - warning "$pkgbase not found in $svnrepo" + warning "$pkgbase not found in $repo" + warning "Removing only $pkgbase from the repo" + warning "If it was a split package you have to remove the others yourself!" + pkgnames=($pkgbase) fi for tarch in ${tarches[@]}; do diff --git a/repo-restore-to-normal b/repo-restore-to-normal new file mode 100755 index 0000000..9463731 --- /dev/null +++ b/repo-restore-to-normal @@ -0,0 +1,58 @@ +#!/bin/bash +# Solves issue165 + +. "$(dirname $0)/db-functions" +. "$(dirname $0)/config" + +CLEANUP_DESTDIR=/home/parabolavnx/repo/pool/restore +PKGREPOS=(community) + +# Find all pkgnames on old with pkgver-pkgrels +#on_repo=($(find ${CLEANUP_DESTDIR} -name "*.pkg.tar.?z" -printf "%f\n" | \ +# sed "s/^\(.\+-[^-]\+-[^-]\+\)-[^-]\+$/\1/")) + +# Traverse all repos +for _repo in ${PKGREPOS[@]}; do + msg "Restoring [${_repo}]" + +# Find all pkgnames on this repo's abs + on_abs=($( + find ${SVNREPO}/${_repo} -name PKGBUILD | \ + while read pkgbuild; do + unset pkgname pkgver pkgrel + source ${pkgbuild} >/dev/null 2>&1 +# cleanup to save memory + unset build package source md5sums pkgdesc epoch \ + url license arch depends makedepends optdepends options \ + >/dev/null 2>&1 + +# also cleanup package functions + for _pkg in ${pkgname[@]}; do + unset package_${pkg} >/dev/null 2>&1 +# this fills the on_abs array + echo ${_pkg}-${pkgver}-${pkgrel} + done + + done + )) + +# quit if abs is empty + if [ ${#on_abs[*]} -eq 0 ]; then + warning "[${_repo}]'s ABS tree is empty, skipping" + continue + fi + +# Compares them, whatever is on abs should be restored +# restore=($(comm -12 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \ +# <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u))) + + msg2 "Restoring the following packages:" +# plain "$(echo ${restore[@]} | tr ' ' "\n")" + + for _pkg in ${on_abs[@]}; do + find ${CLEANUP_DESTDIR} -name "${_pkg}*" -exec cp -v '{}' ${STAGING}/${_repo} \; + done + +done + +exit $? diff --git a/repo-update b/repo-update index 12c6d0c..a44ae87 100755 --- a/repo-update +++ b/repo-update @@ -5,41 +5,28 @@ source $(dirname $0)/config source $(dirname $0)/local_config source $(dirname $0)/libremessages -msg "Updating your-freedom" -$(dirname $0)/yf-update - -msg "Global exclude-list" -# To not downgrade repos by accident -find ${repodir}/old/packages -name "*${PKGEXT}" \ - -fprintf ${rsync_not_needed}2 '%f\n' - for repo in ${ARCHREPOS[@]}; do msg "Syncing ${repo}" for arch in ${ARCHARCHES[@]} 'any'; do msg2 "${repo} ${arch}" # makes a file containing rsync output for filter.py - plain "Checking packages..." ${rsync_list_command} \ rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/staging/${repo}/ > ${rsout_file} 2&>/dev/null + ${repodir}/staging/${repo}/ > ${rsout_file} # reads blacklist and rsout_file and makes an rsync exclude-from # list - plain "Excluding nonfree..." filter.py -r ${rsync_blacklist} -k ${blacklist} \ -f ${rsout_file} # list files in ${repodir}/${repo} and write their names on # rsync_not_needed for using as an rsync exclude-from - plain "Excluding our packages" find ${repodir}/${repo} -name "*${PKGEXT}" \ -fprintf ${rsync_not_needed} '%f\n' # Actual rsync command - plain "Syncing..." ${rsync_update_command} \ --exclude-from=${rsync_blacklist} \ --exclude-from=${rsync_not_needed} \ - --exclude-from=${rsync_not_needed}2 \ rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/staging/${repo}/ 2&>/dev/null + ${repodir}/staging/${repo}/ done for arch in ${ARCHARCHES[@]}; do msg2 "Making pending list for $repo $arch" diff --git a/test/lib/common.inc b/test/lib/common.inc index eb46508..c96e677 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -11,7 +11,7 @@ oneTimeSetUp() { local pkgarch local pkgversion local build - pkgdir="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)" + pkgdir="$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX)" cp -Lr $(dirname ${BASH_SOURCE[0]})/../packages/* "${pkgdir}" msg 'Building packages...' for d in "${pkgdir}"/*; do diff --git a/test/test.d/packages.sh b/test/test.d/packages.sh index 324f73a..488cb15 100755 --- a/test/test.d/packages.sh +++ b/test/test.d/packages.sh @@ -4,8 +4,8 @@ curdir=$(readlink -e $(dirname $0)) . "${curdir}/../lib/common.inc" testPackages() { - # TODO: namcap -r depends fails with i686 packages - find "${pkgdir}" -name "*${PKGEXT}" -exec namcap -e depends {} + || fail 'namcap failed' + # TODO: namcap -r sodepends fails with i686 packages + find "${pkgdir}" -name "*${PKGEXT}" -exec namcap -e sodepends,pkgnameindesc {} + || fail 'namcap failed' } . "${curdir}/../lib/shunit2" |