summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--config9
-rwxr-xr-xcreate-repo24
-rwxr-xr-xcron-jobs/check_archlinux/parse_pkgbuilds.sh8
-rwxr-xr-xcron-jobs/repo-sanity-check56
-rwxr-xr-xcron-jobs/sourceballs243
-rwxr-xr-xcron-jobs/update-abs-tarballs7
-rwxr-xr-xdb-check-nonfree12
-rwxr-xr-xdb-remove15
-rwxr-xr-xrepo-restore-to-normal58
-rwxr-xr-xrepo-update17
-rw-r--r--test/lib/common.inc2
-rwxr-xr-xtest/test.d/packages.sh4
12 files changed, 197 insertions, 58 deletions
diff --git a/config b/config
index 4f4d81d..c8b3f0b 100644
--- a/config
+++ b/config
@@ -3,8 +3,15 @@ FTP_BASE="/home/repo/public"
ARCH_BASE="/home/repo/public"
SVNREPO="/home/repo/abslibre"
+# Repos from Arch
ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib')
-PKGREPOS=(${ARCHREPOS[@]} 'libre' 'libre-testing' 'social' 'sugar')
+# Official Parabola repos
+OURREPOS=('libre' 'libre-testing')
+# User repos
+USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm')
+# Community project repos
+PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'social')
+PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]})
PKGPOOL='pool/packages'
SRCPOOL='sources/packages'
diff --git a/create-repo b/create-repo
new file mode 100755
index 0000000..58842c3
--- /dev/null
+++ b/create-repo
@@ -0,0 +1,24 @@
+#!/bin/bash
+# Creates repository structure
+
+. "$(dirname $0)/db-functions"
+. "$(dirname $0)/config"
+
+if [ $# -eq 0 ]; then
+ msg "Usage: $0 repo1 [repo2 ... repoX]"
+ exit 1
+fi
+
+msg "Creating repos..."
+for _repo in $@; do
+ msg2 "Creating [${_repo}]"
+ mkdir -p "${FTP_BASE}/staging/${_repo}" || \
+ error "Failed creating staging dir"
+
+ for _arch in ${ARCHES[@]}; do
+ mkdir -p "${FTP_BASE}/${_repo}/os/${_arch}" || \
+ error "Failed creating ${_arch} dir"
+ done
+done
+
+msg "Don't forget to add them to the PKGREPOS array on $(dirname $0)/config"
diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
index 5cd17e4..3f92169 100755
--- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh
+++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
@@ -6,7 +6,7 @@
exit() { return; }
splitpkg_overrides=('depends' 'optdepends' 'provides' 'conflicts')
-variables=('pkgname' 'pkgbase' 'pkgver' 'pkgrel' 'makedepends' 'arch' ${splitpkg_overrides[@]})
+variables=('pkgname' 'pkgbase' 'epoch' 'pkgver' 'pkgrel' 'makedepends' 'arch' ${splitpkg_overrides[@]})
readonly -a variables splitpkg_overrides
backup_package_variables() {
@@ -29,7 +29,11 @@ restore_package_variables() {
print_info() {
echo -e "%NAME%\n$pkgname\n"
- echo -e "%VERSION%\n$pkgver-$pkgrel\n"
+ if [ -n "$epoch" ]; then
+ echo -e "%VERSION%\n$epoch:$pkgver-$pkgrel\n"
+ else
+ echo -e "%VERSION%\n$pkgver-$pkgrel\n"
+ fi
echo -e "%PATH%\n$dir\n"
if [ -n "$pkgbase" ]; then
diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check
new file mode 100755
index 0000000..1ba90a6
--- /dev/null
+++ b/cron-jobs/repo-sanity-check
@@ -0,0 +1,56 @@
+#!/bin/bash
+# Solves issue165
+
+. "$(dirname $0)/../db-functions"
+. "$(dirname $0)/../config"
+
+# Traverse all repos
+for _repo in ${PKGREPOS[@]}; do
+ msg "Cleaning up [${_repo}]"
+
+# Find all pkgnames on this repo's abs
+ on_abs=($(
+ find ${SVNREPO}/${_repo} -name PKGBUILD | \
+ while read pkgbuild; do
+ source ${pkgbuild} >/dev/null 2>&1
+# cleanup to save memory
+ unset build package source md5sums pkgdesc pkgver pkgrel epoch \
+ url license arch depends makedepends optdepends options \
+ >/dev/null 2>&1
+
+# also cleanup package functions
+ for _pkg in ${pkgname[@]}; do
+ unset package_${pkg} >/dev/null 2>&1
+ done
+
+# this fills the on_abs array
+ echo ${pkgname[@]}
+ done
+ ))
+
+# quit if abs is empty
+ if [ ${#on_abs[*]} -eq 0 ]; then
+ warning "[${_repo}]'s ABS tree is empty, skipping"
+ break
+ fi
+
+# Find all pkgnames on repos
+ on_repo=($(
+ find ${FTP_BASE}/${_repo} -name "*.pkg.tar.?z" -printf "%f\n" | \
+ sed "s/^\(.\+\)-[^-]\+-[^-]\+-[^-]\+$/\1/"
+ ))
+
+# Compares them, whatever is on repos but not on abs should be removed
+ remove=($(comm -13 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \
+ <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u)))
+
+# Remove them from databases, ftpdir-cleanup will take care of the rest
+ find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -exec \
+ repo-remove {} ${remove[@]} >/dev/null 2>&1 \;
+
+ msg2 "Removed the following packages:"
+ plain "$(echo ${remove[@]} | tr ' ' "\n")"
+
+done
+
+exit $?
diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2
index b29b396..5e228fc 100755
--- a/cron-jobs/sourceballs2
+++ b/cron-jobs/sourceballs2
@@ -22,10 +22,10 @@ find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort
pushd "${SVNREPO}" >/dev/null
for repo in ${PKGREPOS[@]}; do
- failedpkgs=()
+ msg "Sourceballing [${repo}]"
pushd $repo >/dev/null
- find . -maxdepth 1 -type d | while read pkg; do
+ find -maxdepth 1 -type d | while read pkg; do
pushd "${SVNREPO}/$repo/$pkg" >/dev/null
[[ ! -e PKGBUILD ]] && {
@@ -33,35 +33,33 @@ for repo in ${PKGREPOS[@]}; do
continue
}
- unset pkgbase pkgname
+# Unset the previous data
+ unset pkgbase pkgname pkgver pkgrel
source PKGBUILD
+
+ unset build package url pkgdesc source md5sums depends makedepends \
+ optdepends license arch options check mksource
+
+ for _pkg in ${pkgname[@]}; do
+ unset package_${_pkg} >/dev/null 2>&1
+ done
+
pkgbase=${pkgbase:-$pkgname}
+ srcfile="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}"
- echo "${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs"
+ echo "${srcfile}" >> "${WORKDIR}/expected-src-pkgs"
# Skip already sourceballed
- [[ -e "${SRCPKGDEST}/${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" ]] && \
- continue
+ [ -e "${SRCPKGDEST}/${srcfile}" ] && continue
- msg2 "$pkgbase-$pkgver-$pkgrel..."
- makepkg --allsource --ignorearch -c
makepkg --allsource --ignorearch -c >/dev/null 2>&1
- [[ $? -ne 0 ]] && {
- warning "Failed."
- failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}"
- }
+ [ $? -ne 0 ] && plain ${srcfile}
- done
+ done # end find pkgs
popd >/dev/null
- if [ ${#failedpkgs[@]} -ge 1 ]; then
- msg "Failed to create source packages for [${repo}]..."
- for failed_pkg in ${failedpkgs[@]}; do
- msg2 "${failed_pkg}"
- done
- fi
-done
+done # end repos
# Cleanup old source packages
cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort"
@@ -89,10 +87,5 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then
done
fi
-msg "Failed"
-for _fail in ${failedpkgs[@]}; do
- msg2 "$_fail"
-done
-
script_unlock
diff --git a/cron-jobs/update-abs-tarballs b/cron-jobs/update-abs-tarballs
new file mode 100755
index 0000000..824ac34
--- /dev/null
+++ b/cron-jobs/update-abs-tarballs
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. "$(dirname $0)/../config"
+
+rsync -av --exclude=staging/ parabolagnulinux.org::abstar/ ${FTP_BASE}/
+
+exit $?
diff --git a/db-check-nonfree b/db-check-nonfree
index d8f34f0..ab6491d 100755
--- a/db-check-nonfree
+++ b/db-check-nonfree
@@ -23,11 +23,17 @@ for repo in ${ARCHREPOS[@]}; do
if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then
continue
fi
+ unset dbpkgs
unset cleanpkgs
- cmd_="$(dirname $0)/list_nonfree_in_db.py -k ${BLACKLIST_FILE} \
- -b ${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}"
- cleanpkgs=($(${cmd_}))
+ cleanpkgs=()
+ dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u ))
+ for pkgname in ${dbpkgs[@]}; do
+ if in_array ${pkgname} ${nonfree[@]}; then
+ cleanpkgs+=(${pkgname})
+ fi
+ done
if [ ${#cleanpkgs[@]} -ge 1 ]; then
+ msg2 "Unfree: ${cleanpkgs[@]}"
arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]}
fi
done
diff --git a/db-remove b/db-remove
index 292af5d..ccfeb36 100755
--- a/db-remove
+++ b/db-remove
@@ -12,9 +12,6 @@ pkgbase="$1"
repo="$2"
arch="$3"
-ftppath="$FTP_BASE/$repo/os"
-svnrepo="$repo-$arch"
-
if ! check_repo_permission $repo; then
die "You don't have permission to remove packages from ${repo}"
fi
@@ -30,14 +27,14 @@ for tarch in ${tarches[@]}; do
done
msg "Removing $pkgbase from [$repo]..."
-/usr/bin/svn checkout -q "${SVNREPO}/${pkgbase}" "${WORKDIR}/svn/${pkgbase}" >/dev/null
-if [ -d "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" ]; then
- pkgnames=($(. "${WORKDIR}/svn/$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgname[@]}))
- /usr/bin/svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo"
- /usr/bin/svn commit -q "${WORKDIR}/svn/$pkgbase" -m "$(basename $0): $pkgbase removed by $(id -un)"
+if [ -d "${SVNREPO}/$repo/$pkgbase" ]; then
+ pkgnames=($(. "${SVNREPO}/$repo/$pkgbase/PKGBUILD"; echo ${pkgname[@]}))
else
- warning "$pkgbase not found in $svnrepo"
+ warning "$pkgbase not found in $repo"
+ warning "Removing only $pkgbase from the repo"
+ warning "If it was a split package you have to remove the others yourself!"
+ pkgnames=($pkgbase)
fi
for tarch in ${tarches[@]}; do
diff --git a/repo-restore-to-normal b/repo-restore-to-normal
new file mode 100755
index 0000000..9463731
--- /dev/null
+++ b/repo-restore-to-normal
@@ -0,0 +1,58 @@
+#!/bin/bash
+# Solves issue165
+
+. "$(dirname $0)/db-functions"
+. "$(dirname $0)/config"
+
+CLEANUP_DESTDIR=/home/parabolavnx/repo/pool/restore
+PKGREPOS=(community)
+
+# Find all pkgnames on old with pkgver-pkgrels
+#on_repo=($(find ${CLEANUP_DESTDIR} -name "*.pkg.tar.?z" -printf "%f\n" | \
+# sed "s/^\(.\+-[^-]\+-[^-]\+\)-[^-]\+$/\1/"))
+
+# Traverse all repos
+for _repo in ${PKGREPOS[@]}; do
+ msg "Restoring [${_repo}]"
+
+# Find all pkgnames on this repo's abs
+ on_abs=($(
+ find ${SVNREPO}/${_repo} -name PKGBUILD | \
+ while read pkgbuild; do
+ unset pkgname pkgver pkgrel
+ source ${pkgbuild} >/dev/null 2>&1
+# cleanup to save memory
+ unset build package source md5sums pkgdesc epoch \
+ url license arch depends makedepends optdepends options \
+ >/dev/null 2>&1
+
+# also cleanup package functions
+ for _pkg in ${pkgname[@]}; do
+ unset package_${pkg} >/dev/null 2>&1
+# this fills the on_abs array
+ echo ${_pkg}-${pkgver}-${pkgrel}
+ done
+
+ done
+ ))
+
+# quit if abs is empty
+ if [ ${#on_abs[*]} -eq 0 ]; then
+ warning "[${_repo}]'s ABS tree is empty, skipping"
+ continue
+ fi
+
+# Compares them, whatever is on abs should be restored
+# restore=($(comm -12 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \
+# <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u)))
+
+ msg2 "Restoring the following packages:"
+# plain "$(echo ${restore[@]} | tr ' ' "\n")"
+
+ for _pkg in ${on_abs[@]}; do
+ find ${CLEANUP_DESTDIR} -name "${_pkg}*" -exec cp -v '{}' ${STAGING}/${_repo} \;
+ done
+
+done
+
+exit $?
diff --git a/repo-update b/repo-update
index 12c6d0c..a44ae87 100755
--- a/repo-update
+++ b/repo-update
@@ -5,41 +5,28 @@ source $(dirname $0)/config
source $(dirname $0)/local_config
source $(dirname $0)/libremessages
-msg "Updating your-freedom"
-$(dirname $0)/yf-update
-
-msg "Global exclude-list"
-# To not downgrade repos by accident
-find ${repodir}/old/packages -name "*${PKGEXT}" \
- -fprintf ${rsync_not_needed}2 '%f\n'
-
for repo in ${ARCHREPOS[@]}; do
msg "Syncing ${repo}"
for arch in ${ARCHARCHES[@]} 'any'; do
msg2 "${repo} ${arch}"
# makes a file containing rsync output for filter.py
- plain "Checking packages..."
${rsync_list_command} \
rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \
- ${repodir}/staging/${repo}/ > ${rsout_file} 2&>/dev/null
+ ${repodir}/staging/${repo}/ > ${rsout_file}
# reads blacklist and rsout_file and makes an rsync exclude-from
# list
- plain "Excluding nonfree..."
filter.py -r ${rsync_blacklist} -k ${blacklist} \
-f ${rsout_file}
# list files in ${repodir}/${repo} and write their names on
# rsync_not_needed for using as an rsync exclude-from
- plain "Excluding our packages"
find ${repodir}/${repo} -name "*${PKGEXT}" \
-fprintf ${rsync_not_needed} '%f\n'
# Actual rsync command
- plain "Syncing..."
${rsync_update_command} \
--exclude-from=${rsync_blacklist} \
--exclude-from=${rsync_not_needed} \
- --exclude-from=${rsync_not_needed}2 \
rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \
- ${repodir}/staging/${repo}/ 2&>/dev/null
+ ${repodir}/staging/${repo}/
done
for arch in ${ARCHARCHES[@]}; do
msg2 "Making pending list for $repo $arch"
diff --git a/test/lib/common.inc b/test/lib/common.inc
index eb46508..c96e677 100644
--- a/test/lib/common.inc
+++ b/test/lib/common.inc
@@ -11,7 +11,7 @@ oneTimeSetUp() {
local pkgarch
local pkgversion
local build
- pkgdir="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)"
+ pkgdir="$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX)"
cp -Lr $(dirname ${BASH_SOURCE[0]})/../packages/* "${pkgdir}"
msg 'Building packages...'
for d in "${pkgdir}"/*; do
diff --git a/test/test.d/packages.sh b/test/test.d/packages.sh
index 324f73a..488cb15 100755
--- a/test/test.d/packages.sh
+++ b/test/test.d/packages.sh
@@ -4,8 +4,8 @@ curdir=$(readlink -e $(dirname $0))
. "${curdir}/../lib/common.inc"
testPackages() {
- # TODO: namcap -r depends fails with i686 packages
- find "${pkgdir}" -name "*${PKGEXT}" -exec namcap -e depends {} + || fail 'namcap failed'
+ # TODO: namcap -r sodepends fails with i686 packages
+ find "${pkgdir}" -name "*${PKGEXT}" -exec namcap -e sodepends,pkgnameindesc {} + || fail 'namcap failed'
}
. "${curdir}/../lib/shunit2"