diff options
| -rw-r--r-- | changelog | 2 | ||||
| -rwxr-xr-x | clean_repo.py | 3 | ||||
| -rw-r--r-- | config | 2 | ||||
| -rwxr-xr-x | config.sh | 26 | ||||
| -rwxr-xr-x | cptobin.sh | 2 | ||||
| -rwxr-xr-x | filter.py | 5 | ||||
| -rwxr-xr-x | get_license.sh | 50 | ||||
| -rw-r--r-- | local_config.example | 18 | ||||
| -rwxr-xr-x | main.sh | 62 |
9 files changed, 60 insertions, 110 deletions
diff --git a/changelog b/changelog deleted file mode 100644 index 0e89f2f..0000000 --- a/changelog +++ /dev/null @@ -1,2 +0,0 @@ -repo-maintainer 1.0 - * Known working version diff --git a/clean_repo.py b/clean_repo.py index 5423c4d..e1a17c2 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -22,7 +22,8 @@ def mkpending(packages_iterable, pending_file, blacklisted_names, pkgs=[pkg for pkg in pkgs if pkg["name"] not in search and "custom" in pkg["license"]] fsock=open(pending_file, "w") - fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] + fsock.write("\n".join([pkg["name"] + ":" + pkg["location"] + + ":" + pkg["license"] for pkg in pkgs]) + "\n") except(IOError): raise NonValidFile("Can't read or write %s" % pending_file) @@ -1,7 +1,7 @@ FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" -PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') +PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing' 'multilib') PKGPOOL='pool/packages' SRCPOOL='sources/packages' diff --git a/config.sh b/config.sh deleted file mode 100755 index 9a44f50..0000000 --- a/config.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -# -*- coding: utf-8 -*- -source local_config - -function run_python_cmd { - env \ - mirror=${mirror} \ - mirrorpath=${mirrorpath} \ - logname=${logname} \ - tempdir=${tempdir} \ - archdb=${archdb} \ - repodir=${repodir} \ - blacklist=${blacklist} \ - whitelist=${whitelist} \ - pending=${pending} \ - rsync_blacklist=${rsync_blacklist} \ - repo_list=${repo_list} \ - dir_list=${dir_list} \ - arch_list=${arch_list} \ - other=${other} \ - output=${output} \ - debug=${debug} \ - $1 -} - -source libremessages
\ No newline at end of file diff --git a/cptobin.sh b/cptobin.sh deleted file mode 100755 index ce4b3bd..0000000 --- a/cptobin.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -cp -f clean_repo.py config.py config.sh get_license.sh main.sh filter.py local_config ~/usr/bin/
\ No newline at end of file @@ -189,9 +189,10 @@ if __name__ == "__main__": help="File containing blacklisted names", required=True,) parser.add_argument("-c", "--rsync-command", type=str, - help="This command will be run to get a pkg list") + help="This command will be run to get a pkg list", + required=True,) args=parser.parse_args() - rsout=check_output(args.rsync_command) + rsout=check_output(args.rsync_command.split()) packages=pkginfo_from_rsync_output(rsout) rsyncBlaclist_from_blacklist(packages, listado(args.blacklist_file), args.rsync_exclude_file) diff --git a/get_license.sh b/get_license.sh index 0da58cb..024876c 100755 --- a/get_license.sh +++ b/get_license.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # -*- coding: utf-8 -*- # get_license.sh @@ -20,36 +20,26 @@ # You should have received a copy of the GNU General Public License # along with Parabola. If not, see <http://www.gnu.org/licenses/>. - -docs="/home/parabolavnx/parabolagnulinux.org/docs" -repo="/home/parabolavnx/parabolagnulinux.org/repo" -dir="$docs/pending-licenses" - -echo "Cleaning $dir" -rm -rf $dir/* - -tempdir=$(mktemp -d) -cd $tempdir - -pending=($(cut -d: -f1 $docs/pending*.txt)) -echo ${pending[@]} - -for pkg in ${pending[@]}; do - pkg_in_repo=( $(ls ${repo}/*/os/*/${pkg}*) ) - for y in ${pkg_in_repo[@]}; do - echo "chmod +r $y" - chmod +r $y - echo "tar -xf $y usr/share/licenses" - bsdtar -xf $y usr/share/licenses - echo "chmod -r $y" - chmod -r $y +source ./config +source ./local_config +source ./libremessages + +msg "Creating pending licenses list" +pushd ${licenses_dir} +rm -rf ${licenses_dir}/* + +for repo in ${PKGREPOS[@]}; do + msg2 "Extracting licenses in ${repo}" + pending=($(cut -d: -f2 ${docs_dir}/pending-${repo})) + pushd ${repodir}/${repo} + for pkg in ${pending[@]}; do + plain "${pkg}" + bsdtar -xf ${pkg} usr/share/licenses || { + error "${pkg} has no licenses" + } + chmod -r ${pkg} done done -mv usr/share/licenses/* $dir - -cd - -rm -rf $tempdir - +popd exit 0
\ No newline at end of file diff --git a/local_config.example b/local_config.example index 99a1fdb..2c0ef3e 100644 --- a/local_config.example +++ b/local_config.example @@ -4,21 +4,21 @@ mirrorpath="::mirrors/archlinux" # Directories: they should end without / ## Optionals -paraboladir=~/parabolagnulinux.org -logtime=$(date -u +%Y%m%d-%H:%M) +paraboladir= ## Must be defined tempdir=~/tmp archdb=${tempdir}/db docs_dir=${paraboladir}/docs -repodir=${paraboladir}/repo +repodir=${paraboladir}/repo/staging +licenses_dir=${docs_dir}/pending_licenses # End Directories # Files -logname=${paraboladir}/log/${logtime}-repo-maintainer.log +logname=${paraboladir}/log/$(date -u +%Y%m%d-%H:%M)-repo-maintainer.log blacklist=${docs_dir}/blacklist.txt whitelist=${docs_dir}/whitelist.txt -pending=${docs_dir}/pending rsync_blacklist=${docs_dir}/rsyncBlacklist +rsync_not_needed=${tmp}/rsync_not_needed # Repos, arches, and dirs for repo repo_list="core:extra:community:testing:community-testing:multilib" @@ -26,10 +26,6 @@ dir_list="pool" arch_list="i686:x86_64" other="any" -# Output options -output="True" -debug="False" - # Rsync commands -rsync_update_command="rsync -nav --delay-updates --exclude='*.{abs|db}.tar.*' " -rsync_post_command="rsync -nav --delete --exclude='*.abs.tar.*' "
\ No newline at end of file +rsync_list_command="rsync\ -ptgoL\ --list-only\ " +rsync_update_command="rsync -ptgoL --exclude='*.abs.tar.*'" @@ -1,43 +1,35 @@ #!/bin/bash # -*- coding: utf-8 -*- -source config.sh +source ./config +source ./local_config +source ./libremessages -function mkrsexclude { - local error=1 - while ${error}; do - run_python_cmd "filter.py" - error=$? - done -} - -msg "Cleaning $tempdir" -stdnull "rm -r $tempdir/* " - -msg "Generating exclude list for rsync" -mkrsexclude - -msg "Syncing repos without delete" -# rsync_update_command does not sync db or abs -${rsync_update_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ','),\ - $(echo ${dir_list} | tr ':' ',')} ${repodir} - -msg "Syncing each repo and cleaning" -msg2 "Remove pending files" -stdnull "rm -rf ${pending}*" -for repo in $(echo ${repo_list} | tr ':' ' '); do - for arch in $(echo ${arch_list} | tr ':' ' '); do - msg2 "Syncing ${repo} ${arch}" - cmd=$(echo ${rsync_post_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/${repo} ${repodir}/${repo}) - plain "${cmd}" - ${cmd} - msg2 "Cleaning ${repo} ${arch}" - # This also generates pending lists - run_python_cmd "clean_repo.py -b ${repodir}/${repo}/os/${arch}/${repo}.db.tar.gz -d ${repodir}/${repo}/os/${arch}/" +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]} 'any'; do + msg "Syncing ${repo} ${arch}" + filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ + \"${rsync_list_command}\ \ + ${mirror}${mirrorpath}/${repo}/os/${arch}\ \ + ${repodir}/${repo}/\" + find ${repodir}/${repo} -name *${PKGEXT} -print \ + > ${rsync_not_needed} + ${rsync_update_command} \ + ${mirror}${mirrorpath}/${repo}/os/${arch} \ + ${repodir}/${repo} \ + --exclude-from=${rsync_blacklist} \ + --exclude-from=${rsync_not_needed} done + for arch in ${ARCHES[@]}; do + if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then + clean_repo.py -k ${blacklist} -w ${whitelist} \ + -p ${docs_dir}/pending-${repo} \ + -b ${repodir}/${repo}/${repo}${DBEXT} + fi + clean_repo.py -k ${blacklist} -d ${repodir}/${repo} done -msg "Checking licenses" +db-update +ftpdir-cleanup + get_license.sh |