From 11db46275d14c9a9c2e59019fed3de4b8803f9f4 Mon Sep 17 00:00:00 2001 From: Parabola Date: Fri, 18 Feb 2011 15:42:59 -0800 Subject: Parabola specific changes --- config | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'config') diff --git a/config b/config index 53191e0..720d110 100644 --- a/config +++ b/config @@ -1,25 +1,25 @@ -FTP_BASE="/srv/ftp" -SVNREPO='' -PKGREPOS=() -PKGPOOL='' -SRCPOOL='' +FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" +SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" +PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') +PKGPOOL='pool/packages' +SRCPOOL='sources/packages' -CLEANUP_DESTDIR="/srv/package-cleanup" +CLEANUP_DESTDIR="$FTP_BASE/old/packages" CLEANUP_DRYRUN=false # Time in days to keep moved packages CLEANUP_KEEP=30 -SOURCE_CLEANUP_DESTDIR="/srv/source-cleanup" +SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" SOURCE_CLEANUP_DRYRUN=false # Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP=14 +SOURCE_CLEANUP_KEEP=30 LOCK_DELAY=10 LOCK_TIMEOUT=300 -STAGING="$HOME/staging" -TMPDIR="/srv/tmp" -ARCHES=(i686 x86_64) +STAGING="$FTP_BASE/staging" +TMPDIR="$HOME/tmp" +ARCHES=(i686 x86_64 mipsel) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.*" @@ -29,4 +29,4 @@ SRCEXT=".src.tar.gz" ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') # Override default config with config.local -[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" +#[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" -- cgit v1.2.3-54-g00ecf From c6516eca2803b780a01a6b2bf3e081c8bd570301 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 20 Feb 2011 17:10:40 -0800 Subject: sourceballs --- config | 1 + cron-jobs/sourceballs | 12 ++++++------ db-functions | 5 ++++- 3 files changed, 11 insertions(+), 7 deletions(-) (limited to 'config') diff --git a/config b/config index 720d110..12bdaaa 100644 --- a/config +++ b/config @@ -1,4 +1,5 @@ FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" +ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') PKGPOOL='pool/packages' diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 0df007e..53d43ce 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -21,10 +21,10 @@ renice +10 -p $$ > /dev/null for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do # Repo does not exist; skip it - if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + if [ ! -f "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi - bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + bsdtar -xOf "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ | awk '/^%NAME%/ { getline b }; /^%BASE%/ { getline b }; /^%VERSION%/ { getline v }; @@ -46,7 +46,7 @@ for repo in ${PKGREPOS[@]}; do done # Create a list of all available source package file names -find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do @@ -91,9 +91,9 @@ for repo in ${PKGREPOS[@]}; do # Build the actual source package pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null - makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1 + makepkg --nocolor --allsource --ignorearch # >/dev/null 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then - mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + mv "${pkgbase}-${pkgver}${SRCEXT}" "${ARCH_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" @@ -129,7 +129,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then - mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi done diff --git a/db-functions b/db-functions index 0553188..e790fb6 100644 --- a/db-functions +++ b/db-functions @@ -16,7 +16,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) +WORKDIR=$(mktemp -d /home/parabolavnx/tmp/$(basename $0).XXXXXXXXXX) LOCKS=() # check if messages are to be printed using color @@ -376,6 +376,9 @@ check_splitpkgs() { echo "Failed $_pkgbase-$_pkgver-$_pkgarch" return 1 } + + sleep 20s + fi local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) -- cgit v1.2.3-54-g00ecf From 229a9c504cbd733c93cf91399dc54bedf5160cc5 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 6 Mar 2011 15:22:21 -0800 Subject: Fixes --- config | 8 ++--- cron-jobs/sourceballs2 | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++ db-functions | 4 +-- 3 files changed, 90 insertions(+), 9 deletions(-) create mode 100755 cron-jobs/sourceballs2 (limited to 'config') diff --git a/config b/config index 12bdaaa..217b627 100644 --- a/config +++ b/config @@ -20,14 +20,10 @@ LOCK_TIMEOUT=300 STAGING="$FTP_BASE/staging" TMPDIR="$HOME/tmp" -ARCHES=(i686 x86_64 mipsel) +ARCHES=(i686 x86_64 mips64el) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.*" SRCEXT=".src.tar.gz" -# Allowed licenses: get sourceballs only for licenses in this array -ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') - -# Override default config with config.local -#[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" +MAKEPKGCONF="$HOME/etc/makepkg.conf" diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 new file mode 100755 index 0000000..452208e --- /dev/null +++ b/cron-jobs/sourceballs2 @@ -0,0 +1,87 @@ +#!/bin/bash + +dirname="$(dirname $(readlink -e $0))" +. "${dirname}/../db-functions" +. "${dirname}/../config" +. "${MAKEPKGCONF}" + +pushd "${WORKDIR}" >/dev/null + +script_lock + +#adjust the nice level to run at a lower priority +renice +10 -p $$ > /dev/null + +# Create a list of all available source package file names +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" + +# Steps +# Traverse the ABSLibre +# Makepkg --allsource every package +# Remove the old packages +pushd "${SVNREPO}" >/dev/null + +failedpkgs=() +for repo in ${PKGREPOS[@]}; do + pushd $repo >/dev/null + find . -maxdepth 1 -type d | while read pkg; do + pushd "${SVNREPO}/$repo/$pkg" >/dev/null + + [[ ! -e PKGBUILD ]] && { + warning "$repo/$pkg is not a package" + continue + } + + unset pkgbase pkgname + source PKGBUILD + pkgbase=${pkgbase:-$pkgname} + + echo "${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" + + # Skip already sourceballed + [[ -e "${SRCPKGDEST}/${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" ]] && \ + continue + + makepkg --allsource --ignorearch -c >/dev/null 2>&1 + + [[ $? -ne 0 ]] && \ + failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" + + done + popd >/dev/null +done + +# Cleanup old source packages +cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) + +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages..." + ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + if ! ${SOURCE_CLEANUP_DRYRUN}; then + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + fi + done +fi + +old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages from the cleanup directory..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + done +fi + +msg "Failed" +for _fail in ${failedpkgs[@]}; do + msg2 "$_fail" +done + + +script_unlock + diff --git a/db-functions b/db-functions index e2731cb..b900669 100644 --- a/db-functions +++ b/db-functions @@ -1,7 +1,5 @@ #!/bin/bash -. config - # Some PKGBUILDs need CARCH to be set CARCH="x86_64" @@ -18,7 +16,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -d ${TMPDIR}/$(basename $0).XXXXXXXXXX) +WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) LOCKS=() # check if messages are to be printed using color -- cgit v1.2.3-54-g00ecf From 374c8e2a5183cdbaefe9f54184603ad8d09e30c2 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sat, 16 Apr 2011 03:15:01 -0500 Subject: * bash-repo for testing * merged dbscripts with repm --- changelog | 2 -- clean_repo.py | 3 ++- config | 2 +- config.sh | 26 ---------------------- cptobin.sh | 2 -- filter.py | 5 +++-- get_license.sh | 50 +++++++++++++++++------------------------- local_config.example | 18 ++++++--------- main.sh | 62 +++++++++++++++++++++++----------------------------- 9 files changed, 60 insertions(+), 110 deletions(-) delete mode 100644 changelog delete mode 100755 config.sh delete mode 100755 cptobin.sh (limited to 'config') diff --git a/changelog b/changelog deleted file mode 100644 index 0e89f2f..0000000 --- a/changelog +++ /dev/null @@ -1,2 +0,0 @@ -repo-maintainer 1.0 - * Known working version diff --git a/clean_repo.py b/clean_repo.py index 5423c4d..e1a17c2 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -22,7 +22,8 @@ def mkpending(packages_iterable, pending_file, blacklisted_names, pkgs=[pkg for pkg in pkgs if pkg["name"] not in search and "custom" in pkg["license"]] fsock=open(pending_file, "w") - fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] + fsock.write("\n".join([pkg["name"] + ":" + pkg["location"] + + ":" + pkg["license"] for pkg in pkgs]) + "\n") except(IOError): raise NonValidFile("Can't read or write %s" % pending_file) diff --git a/config b/config index 217b627..d9bad13 100644 --- a/config +++ b/config @@ -1,7 +1,7 @@ FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" -PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') +PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing' 'multilib') PKGPOOL='pool/packages' SRCPOOL='sources/packages' diff --git a/config.sh b/config.sh deleted file mode 100755 index 9a44f50..0000000 --- a/config.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -# -*- coding: utf-8 -*- -source local_config - -function run_python_cmd { - env \ - mirror=${mirror} \ - mirrorpath=${mirrorpath} \ - logname=${logname} \ - tempdir=${tempdir} \ - archdb=${archdb} \ - repodir=${repodir} \ - blacklist=${blacklist} \ - whitelist=${whitelist} \ - pending=${pending} \ - rsync_blacklist=${rsync_blacklist} \ - repo_list=${repo_list} \ - dir_list=${dir_list} \ - arch_list=${arch_list} \ - other=${other} \ - output=${output} \ - debug=${debug} \ - $1 -} - -source libremessages \ No newline at end of file diff --git a/cptobin.sh b/cptobin.sh deleted file mode 100755 index ce4b3bd..0000000 --- a/cptobin.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -cp -f clean_repo.py config.py config.sh get_license.sh main.sh filter.py local_config ~/usr/bin/ \ No newline at end of file diff --git a/filter.py b/filter.py index add0235..4abdf38 100755 --- a/filter.py +++ b/filter.py @@ -189,9 +189,10 @@ if __name__ == "__main__": help="File containing blacklisted names", required=True,) parser.add_argument("-c", "--rsync-command", type=str, - help="This command will be run to get a pkg list") + help="This command will be run to get a pkg list", + required=True,) args=parser.parse_args() - rsout=check_output(args.rsync_command) + rsout=check_output(args.rsync_command.split()) packages=pkginfo_from_rsync_output(rsout) rsyncBlaclist_from_blacklist(packages, listado(args.blacklist_file), args.rsync_exclude_file) diff --git a/get_license.sh b/get_license.sh index 0da58cb..024876c 100755 --- a/get_license.sh +++ b/get_license.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # -*- coding: utf-8 -*- # get_license.sh @@ -20,36 +20,26 @@ # You should have received a copy of the GNU General Public License # along with Parabola. If not, see . - -docs="/home/parabolavnx/parabolagnulinux.org/docs" -repo="/home/parabolavnx/parabolagnulinux.org/repo" -dir="$docs/pending-licenses" - -echo "Cleaning $dir" -rm -rf $dir/* - -tempdir=$(mktemp -d) -cd $tempdir - -pending=($(cut -d: -f1 $docs/pending*.txt)) -echo ${pending[@]} - -for pkg in ${pending[@]}; do - pkg_in_repo=( $(ls ${repo}/*/os/*/${pkg}*) ) - for y in ${pkg_in_repo[@]}; do - echo "chmod +r $y" - chmod +r $y - echo "tar -xf $y usr/share/licenses" - bsdtar -xf $y usr/share/licenses - echo "chmod -r $y" - chmod -r $y +source ./config +source ./local_config +source ./libremessages + +msg "Creating pending licenses list" +pushd ${licenses_dir} +rm -rf ${licenses_dir}/* + +for repo in ${PKGREPOS[@]}; do + msg2 "Extracting licenses in ${repo}" + pending=($(cut -d: -f2 ${docs_dir}/pending-${repo})) + pushd ${repodir}/${repo} + for pkg in ${pending[@]}; do + plain "${pkg}" + bsdtar -xf ${pkg} usr/share/licenses || { + error "${pkg} has no licenses" + } + chmod -r ${pkg} done done -mv usr/share/licenses/* $dir - -cd - -rm -rf $tempdir - +popd exit 0 \ No newline at end of file diff --git a/local_config.example b/local_config.example index 99a1fdb..2c0ef3e 100644 --- a/local_config.example +++ b/local_config.example @@ -4,21 +4,21 @@ mirrorpath="::mirrors/archlinux" # Directories: they should end without / ## Optionals -paraboladir=~/parabolagnulinux.org -logtime=$(date -u +%Y%m%d-%H:%M) +paraboladir= ## Must be defined tempdir=~/tmp archdb=${tempdir}/db docs_dir=${paraboladir}/docs -repodir=${paraboladir}/repo +repodir=${paraboladir}/repo/staging +licenses_dir=${docs_dir}/pending_licenses # End Directories # Files -logname=${paraboladir}/log/${logtime}-repo-maintainer.log +logname=${paraboladir}/log/$(date -u +%Y%m%d-%H:%M)-repo-maintainer.log blacklist=${docs_dir}/blacklist.txt whitelist=${docs_dir}/whitelist.txt -pending=${docs_dir}/pending rsync_blacklist=${docs_dir}/rsyncBlacklist +rsync_not_needed=${tmp}/rsync_not_needed # Repos, arches, and dirs for repo repo_list="core:extra:community:testing:community-testing:multilib" @@ -26,10 +26,6 @@ dir_list="pool" arch_list="i686:x86_64" other="any" -# Output options -output="True" -debug="False" - # Rsync commands -rsync_update_command="rsync -nav --delay-updates --exclude='*.{abs|db}.tar.*' " -rsync_post_command="rsync -nav --delete --exclude='*.abs.tar.*' " \ No newline at end of file +rsync_list_command="rsync\ -ptgoL\ --list-only\ " +rsync_update_command="rsync -ptgoL --exclude='*.abs.tar.*'" diff --git a/main.sh b/main.sh index d15f736..b3ecb92 100755 --- a/main.sh +++ b/main.sh @@ -1,43 +1,35 @@ #!/bin/bash # -*- coding: utf-8 -*- -source config.sh +source ./config +source ./local_config +source ./libremessages -function mkrsexclude { - local error=1 - while ${error}; do - run_python_cmd "filter.py" - error=$? - done -} - -msg "Cleaning $tempdir" -stdnull "rm -r $tempdir/* " - -msg "Generating exclude list for rsync" -mkrsexclude - -msg "Syncing repos without delete" -# rsync_update_command does not sync db or abs -${rsync_update_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ','),\ - $(echo ${dir_list} | tr ':' ',')} ${repodir} - -msg "Syncing each repo and cleaning" -msg2 "Remove pending files" -stdnull "rm -rf ${pending}*" -for repo in $(echo ${repo_list} | tr ':' ' '); do - for arch in $(echo ${arch_list} | tr ':' ' '); do - msg2 "Syncing ${repo} ${arch}" - cmd=$(echo ${rsync_post_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/${repo} ${repodir}/${repo}) - plain "${cmd}" - ${cmd} - msg2 "Cleaning ${repo} ${arch}" - # This also generates pending lists - run_python_cmd "clean_repo.py -b ${repodir}/${repo}/os/${arch}/${repo}.db.tar.gz -d ${repodir}/${repo}/os/${arch}/" +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]} 'any'; do + msg "Syncing ${repo} ${arch}" + filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ + \"${rsync_list_command}\ \ + ${mirror}${mirrorpath}/${repo}/os/${arch}\ \ + ${repodir}/${repo}/\" + find ${repodir}/${repo} -name *${PKGEXT} -print \ + > ${rsync_not_needed} + ${rsync_update_command} \ + ${mirror}${mirrorpath}/${repo}/os/${arch} \ + ${repodir}/${repo} \ + --exclude-from=${rsync_blacklist} \ + --exclude-from=${rsync_not_needed} done + for arch in ${ARCHES[@]}; do + if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then + clean_repo.py -k ${blacklist} -w ${whitelist} \ + -p ${docs_dir}/pending-${repo} \ + -b ${repodir}/${repo}/${repo}${DBEXT} + fi + clean_repo.py -k ${blacklist} -d ${repodir}/${repo} done -msg "Checking licenses" +db-update +ftpdir-cleanup + get_license.sh -- cgit v1.2.3-54-g00ecf