Merge branch 'master' of https://projects.parabolagnulinux.org/dbscripts

Conflicts:
	config

12 files changed, 197 insertions, 58 deletions

diff --git a/config b/config
index 4f4d81d..c8b3f0b 100644
--- a/config
+++ b/config
@@ -3,8 +3,15 @@ FTP_BASE="/home/repo/public"
 ARCH_BASE="/home/repo/public"
 SVNREPO="/home/repo/abslibre"
 
+# Repos from Arch
 ARCHREPOS=('core' 'extra' 'community'  'testing' 'multilib')
-PKGREPOS=(${ARCHREPOS[@]} 'libre' 'libre-testing' 'social' 'sugar')
+# Official Parabola repos
+OURREPOS=('libre' 'libre-testing')
+# User repos
+USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm')
+# Community project repos
+PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'social')
+PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]})
 PKGPOOL='pool/packages'
 SRCPOOL='sources/packages'
 
diff --git a/create-repo b/create-repo
new file mode 100755
index 0000000..58842c3
--- /dev/null
+++ b/create-repo
@@ -0,0 +1,24 @@
+#!/bin/bash
+# Creates repository structure
+
+. "$(dirname $0)/db-functions"
+. "$(dirname $0)/config"
+
+if [ $# -eq 0 ]; then
+    msg "Usage: $0 repo1 [repo2 ... repoX]"
+    exit 1
+fi
+
+msg "Creating repos..."
+for _repo in $@; do
+    msg2 "Creating [${_repo}]"
+    mkdir -p "${FTP_BASE}/staging/${_repo}" || \
+        error "Failed creating staging dir"
+
+    for _arch in ${ARCHES[@]}; do
+        mkdir -p "${FTP_BASE}/${_repo}/os/${_arch}" || \
+            error "Failed creating ${_arch} dir"
+    done
+done
+
+msg "Don't forget to add them to the PKGREPOS array on $(dirname $0)/config"
diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
index 5cd17e4..3f92169 100755
--- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh
+++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
@@ -6,7 +6,7 @@
 exit() { return; }
 
 splitpkg_overrides=('depends' 'optdepends' 'provides' 'conflicts')
-variables=('pkgname' 'pkgbase' 'pkgver' 'pkgrel' 'makedepends' 'arch' ${splitpkg_overrides[@]})
+variables=('pkgname' 'pkgbase' 'epoch' 'pkgver' 'pkgrel' 'makedepends' 'arch' ${splitpkg_overrides[@]})
 readonly -a variables splitpkg_overrides
 
 backup_package_variables() {
@@ -29,7 +29,11 @@ restore_package_variables() {
 
 print_info() {
 	echo -e "%NAME%\n$pkgname\n"
-	echo -e "%VERSION%\n$pkgver-$pkgrel\n"
+	if [ -n "$epoch" ]; then
+		echo -e "%VERSION%\n$epoch:$pkgver-$pkgrel\n"
+	else
+		echo -e "%VERSION%\n$pkgver-$pkgrel\n"
+	fi
 	echo -e "%PATH%\n$dir\n"
 
 	if [ -n "$pkgbase" ]; then
diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check
new file mode 100755
index 0000000..1ba90a6
--- /dev/null
+++ b/cron-jobs/repo-sanity-check
@@ -0,0 +1,56 @@
+#!/bin/bash
+# Solves issue165
+
+. "$(dirname $0)/../db-functions"
+. "$(dirname $0)/../config"
+
+# Traverse all repos
+for _repo in ${PKGREPOS[@]}; do
+    msg "Cleaning up [${_repo}]"
+
+# Find all pkgnames on this repo's abs
+    on_abs=($(
+        find ${SVNREPO}/${_repo} -name PKGBUILD | \
+        while read pkgbuild; do
+            source ${pkgbuild} >/dev/null 2>&1
+# cleanup to save memory
+            unset build package source md5sums pkgdesc pkgver pkgrel epoch \
+                  url license arch depends makedepends optdepends options \
+                  >/dev/null 2>&1
+
+# also cleanup package functions
+            for _pkg in ${pkgname[@]}; do
+                unset package_${pkg} >/dev/null 2>&1
+            done
+            
+# this fills the on_abs array
+            echo ${pkgname[@]}
+        done
+    ))
+
+# quit if abs is empty
+    if [ ${#on_abs[*]} -eq 0 ]; then
+        warning "[${_repo}]'s ABS tree is empty, skipping"
+        break
+    fi
+
+# Find all pkgnames on repos
+    on_repo=($(
+        find ${FTP_BASE}/${_repo} -name "*.pkg.tar.?z" -printf "%f\n" | \
+        sed "s/^\(.\+\)-[^-]\+-[^-]\+-[^-]\+$/\1/"
+    ))
+
+# Compares them, whatever is on repos but not on abs should be removed
+    remove=($(comm -13 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \
+             <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u)))
+
+# Remove them from databases, ftpdir-cleanup will take care of the rest
+    find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -exec \
+        repo-remove {} ${remove[@]} >/dev/null 2>&1 \;
+
+    msg2 "Removed the following packages:"
+    plain "$(echo ${remove[@]} | tr ' ' "\n")"
+
+done
+
+exit $?
diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2
index b29b396..5e228fc 100755
--- a/cron-jobs/sourceballs2
+++ b/cron-jobs/sourceballs2
@@ -22,10 +22,10 @@ find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort
 pushd "${SVNREPO}" >/dev/null
 
 for repo in ${PKGREPOS[@]}; do
-    failedpkgs=()
+    msg "Sourceballing [${repo}]"
 
     pushd $repo >/dev/null
-    find . -maxdepth 1 -type d | while read pkg; do
+    find -maxdepth 1 -type d | while read pkg; do
         pushd "${SVNREPO}/$repo/$pkg" >/dev/null
 
         [[ ! -e PKGBUILD ]] && {
@@ -33,35 +33,33 @@ for repo in ${PKGREPOS[@]}; do
             continue
         }
 
-        unset pkgbase pkgname
+# Unset the previous data
+        unset pkgbase pkgname pkgver pkgrel
         source PKGBUILD
+
+        unset build package url pkgdesc source md5sums depends makedepends \
+              optdepends license arch options check mksource
+
+        for _pkg in ${pkgname[@]}; do
+            unset package_${_pkg} >/dev/null 2>&1
+        done
+
         pkgbase=${pkgbase:-$pkgname}
+        srcfile="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}"
 
-		echo "${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs"
+		echo "${srcfile}" >> "${WORKDIR}/expected-src-pkgs"
 
         # Skip already sourceballed
-        [[ -e "${SRCPKGDEST}/${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" ]] && \
-            continue
+        [ -e "${SRCPKGDEST}/${srcfile}" ] && continue
 
-        msg2 "$pkgbase-$pkgver-$pkgrel..."
-        makepkg --allsource --ignorearch -c
         makepkg --allsource --ignorearch -c >/dev/null 2>&1
 
-        [[ $? -ne 0 ]] && {
-            warning "Failed."
-            failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}"
-        }
+        [ $? -ne 0 ] && plain ${srcfile}
 
-    done
+    done # end find pkgs
     popd >/dev/null
 
-    if [ ${#failedpkgs[@]} -ge 1 ]; then
-        msg "Failed to create source packages for [${repo}]..."
-        for failed_pkg in ${failedpkgs[@]}; do
-            msg2 "${failed_pkg}"
-        done
-    fi
-done
+done # end repos
 
 # Cleanup old source packages
 cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort"
@@ -89,10 +87,5 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then
 	done
 fi
 
-msg "Failed"
-for _fail in ${failedpkgs[@]}; do
-    msg2 "$_fail"
-done
-
 script_unlock
 
diff --git a/cron-jobs/update-abs-tarballs b/cron-jobs/update-abs-tarballs
new file mode 100755
index 0000000..824ac34
--- /dev/null
+++ b/cron-jobs/update-abs-tarballs
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. "$(dirname $0)/../config"
+
+rsync -av --exclude=staging/ parabolagnulinux.org::abstar/ ${FTP_BASE}/
+
+exit $?
diff --git a/db-check-nonfree b/db-check-nonfree
index d8f34f0..ab6491d 100755
--- a/db-check-nonfree
+++ b/db-check-nonfree
@@ -23,11 +23,17 @@ for repo in ${ARCHREPOS[@]}; do
 	if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then
 	    continue
 	fi
+	unset dbpkgs
 	unset cleanpkgs
-        cmd_="$(dirname $0)/list_nonfree_in_db.py -k ${BLACKLIST_FILE} \
-            -b ${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}"
-	cleanpkgs=($(${cmd_}))
+	cleanpkgs=()
+	dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u ))
+	for pkgname in ${dbpkgs[@]}; do
+	    if in_array ${pkgname} ${nonfree[@]}; then
+            cleanpkgs+=(${pkgname})
+	    fi
+	done
 	if [ ${#cleanpkgs[@]} -ge 1 ]; then
+        msg2 "Unfree: ${cleanpkgs[@]}"
 	    arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]}
 	fi
     done
diff --git a/db-remove b/db-remove
index 292af5d..ccfeb36 100755
--- a/db-remove
+++ b/db-remove
@@ -12,9 +12,6 @@ pkgbase="$1"
 repo="$2"
 arch="$3"
 
-ftppath="$FTP_BASE/$repo/os"
-svnrepo="$repo-$arch"
-
 if ! check_repo_permission $repo; then
 	die "You don't have permission to remove packages from ${repo}"
 fi
@@ -30,14 +27,14 @@ for tarch in ${tarches[@]}; do
 done
 
 msg "Removing $pkgbase from [$repo]..."
-/usr/bin/svn checkout -q "${SVNREPO}/${pkgbase}" "${WORKDIR}/svn/${pkgbase}" >/dev/null
 
-if [ -d "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" ]; then
-	pkgnames=($(. "${WORKDIR}/svn/$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgname[@]}))
-	/usr/bin/svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo"
-	/usr/bin/svn commit -q "${WORKDIR}/svn/$pkgbase" -m "$(basename $0): $pkgbase removed by $(id -un)"
+if [ -d "${SVNREPO}/$repo/$pkgbase" ]; then
+	pkgnames=($(. "${SVNREPO}/$repo/$pkgbase/PKGBUILD"; echo ${pkgname[@]}))
 else
-	warning "$pkgbase not found in $svnrepo"
+	warning "$pkgbase not found in $repo"
+	warning "Removing only $pkgbase from the repo"
+	warning "If it was a split package you have to remove the others yourself!"
+	pkgnames=($pkgbase)
 fi
 
 for tarch in ${tarches[@]}; do
diff --git a/repo-restore-to-normal b/repo-restore-to-normal
new file mode 100755
index 0000000..9463731
--- /dev/null
+++ b/repo-restore-to-normal
@@ -0,0 +1,58 @@
+#!/bin/bash
+# Solves issue165
+
+. "$(dirname $0)/db-functions"
+. "$(dirname $0)/config"
+
+CLEANUP_DESTDIR=/home/parabolavnx/repo/pool/restore
+PKGREPOS=(community)
+
+# Find all pkgnames on old with pkgver-pkgrels
+#on_repo=($(find ${CLEANUP_DESTDIR} -name "*.pkg.tar.?z" -printf "%f\n" | \
+#    sed "s/^\(.\+-[^-]\+-[^-]\+\)-[^-]\+$/\1/"))
+
+# Traverse all repos
+for _repo in ${PKGREPOS[@]}; do
+    msg "Restoring [${_repo}]"
+
+# Find all pkgnames on this repo's abs
+    on_abs=($(
+        find ${SVNREPO}/${_repo} -name PKGBUILD | \
+        while read pkgbuild; do
+            unset pkgname pkgver pkgrel
+            source ${pkgbuild} >/dev/null 2>&1
+# cleanup to save memory
+            unset build package source md5sums pkgdesc epoch \
+                  url license arch depends makedepends optdepends options \
+                  >/dev/null 2>&1
+
+# also cleanup package functions
+            for _pkg in ${pkgname[@]}; do
+                unset package_${pkg} >/dev/null 2>&1
+# this fills the on_abs array
+                echo ${_pkg}-${pkgver}-${pkgrel}
+            done
+            
+        done
+    ))
+
+# quit if abs is empty
+    if [ ${#on_abs[*]} -eq 0 ]; then
+        warning "[${_repo}]'s ABS tree is empty, skipping"
+        continue
+    fi
+
+# Compares them, whatever is on abs should be restored
+#    restore=($(comm -12 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \
+#             <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u)))
+
+    msg2 "Restoring the following packages:"
+#    plain "$(echo ${restore[@]} | tr ' ' "\n")"
+
+    for _pkg in ${on_abs[@]}; do
+        find ${CLEANUP_DESTDIR} -name "${_pkg}*" -exec cp -v '{}' ${STAGING}/${_repo} \;
+    done
+
+done
+
+exit $?
diff --git a/repo-update b/repo-update
index 12c6d0c..a44ae87 100755
--- a/repo-update
+++ b/repo-update
@@ -5,41 +5,28 @@ source $(dirname $0)/config
 source $(dirname $0)/local_config
 source $(dirname $0)/libremessages
 
-msg "Updating your-freedom"
-$(dirname $0)/yf-update
-
-msg "Global exclude-list"
-# To not downgrade repos by accident
-find ${repodir}/old/packages -name "*${PKGEXT}" \
-    -fprintf ${rsync_not_needed}2 '%f\n'
-
 for repo in ${ARCHREPOS[@]}; do
     msg "Syncing ${repo}"
     for arch in ${ARCHARCHES[@]} 'any'; do
 	msg2 "${repo} ${arch}"
 	# makes a file containing rsync output for filter.py
-        plain "Checking packages..."
 	${rsync_list_command} \
 	    rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \
-	    ${repodir}/staging/${repo}/ > ${rsout_file} 2&>/dev/null
+	    ${repodir}/staging/${repo}/ > ${rsout_file}
 	# reads blacklist and rsout_file and makes an rsync exclude-from
 	# list
-        plain "Excluding nonfree..."
 	filter.py -r ${rsync_blacklist} -k ${blacklist} \
 	    -f ${rsout_file}
 	# list files in ${repodir}/${repo} and write their names on 
 	# rsync_not_needed for using as an rsync exclude-from
-        plain "Excluding our packages"
 	find ${repodir}/${repo} -name "*${PKGEXT}" \
 	    -fprintf ${rsync_not_needed} '%f\n'
 	# Actual rsync command
-        plain "Syncing..."
 	${rsync_update_command} \
 	    --exclude-from=${rsync_blacklist} \
 	    --exclude-from=${rsync_not_needed} \
-	    --exclude-from=${rsync_not_needed}2 \
 	    rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \
-	    ${repodir}/staging/${repo}/ 2&>/dev/null
+	    ${repodir}/staging/${repo}/
     done
     for arch in ${ARCHARCHES[@]}; do
 	msg2 "Making pending list for $repo $arch"
diff --git a/test/lib/common.inc b/test/lib/common.inc
index eb46508..c96e677 100644
--- a/test/lib/common.inc
+++ b/test/lib/common.inc
@@ -11,7 +11,7 @@ oneTimeSetUp() {
 	local pkgarch
 	local pkgversion
 	local build
-	pkgdir="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)"
+	pkgdir="$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX)"
 	cp -Lr $(dirname ${BASH_SOURCE[0]})/../packages/* "${pkgdir}"
 	msg 'Building packages...'
 	for d in "${pkgdir}"/*; do
diff --git a/test/test.d/packages.sh b/test/test.d/packages.sh
index 324f73a..488cb15 100755
--- a/test/test.d/packages.sh
+++ b/test/test.d/packages.sh
@@ -4,8 +4,8 @@ curdir=$(readlink -e $(dirname $0))
 . "${curdir}/../lib/common.inc"
 
 testPackages() {
-	# TODO: namcap -r depends fails with i686 packages
-	find "${pkgdir}" -name "*${PKGEXT}" -exec namcap -e depends {} + || fail 'namcap failed'
+	# TODO: namcap -r sodepends fails with i686 packages
+	find "${pkgdir}" -name "*${PKGEXT}" -exec namcap -e sodepends,pkgnameindesc {} + || fail 'namcap failed'
 }
 
 . "${curdir}/../lib/shunit2"