From e341449526be63b061e6d2661e5f539b959e8c07 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sat, 16 Apr 2011 14:49:51 -0500 Subject: Will this solve problems? --- repo-update | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/repo-update b/repo-update index 59a0658..495f515 100755 --- a/repo-update +++ b/repo-update @@ -8,7 +8,7 @@ source ./libremessages for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]} 'any'; do msg "Syncing ${repo} ${arch}" - filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ + python filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ \"${rsync_list_command}\ \ ${mirror}${mirrorpath}/${repo}/os/${arch}\ \ ${repodir}/${repo}/\" @@ -22,11 +22,11 @@ for repo in ${PKGREPOS[@]}; do done for arch in ${ARCHES[@]}; do if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then - clean_repo.py -k ${blacklist} -w ${whitelist} \ + python clean_repo.py -k ${blacklist} -w ${whitelist} \ -p ${docs_dir}/pending-${repo} \ -b ${repodir}/${repo}/${repo}${DBEXT} fi - clean_repo.py -k ${blacklist} -d ${repodir}/${repo} + python clean_repo.py -k ${blacklist} -d ${repodir}/${repo} done done -- cgit v1.2.3-54-g00ecf From c44ba7fa5154e71921d64f2065a71a4c6755c61b Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 19 Apr 2011 19:14:41 -0500 Subject: Next fix --- .gitignore | 2 ++ clean_repo.py | 12 ++++++++---- filter.py | 16 +++++++++++----- local_config.example | 13 ++++++++----- repo-update | 19 ++++++++++--------- 5 files changed, 39 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index fb6241a..45688ab 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ local_config /config.local test/packages/*/*.pkg.tar.?z +#*# +.#* \ No newline at end of file diff --git a/clean_repo.py b/clean_repo.py index e1a17c2..bc401b5 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -48,10 +48,13 @@ def remove_from_blacklist(path_to_db, blacklisted_names): def cleanup_nonfree_in_dir(directory, blacklisted_names): if "~" in directory: directory=(os.path.expanduser(directory)) + pkglist=list() pkgs=pkginfo_from_files_in_dir(directory) for package in pkgs: if package["name"] in blacklisted_names: os.remove(package["location"]) + pkglist.append(package) + return pkglist if __name__ == "__main__": parser = argparse.ArgumentParser( @@ -64,10 +67,10 @@ if __name__ == "__main__": group_dir=parser.add_argument_group("Clean non-free packages in dir") group_dir.add_argument("-d", "--directory", type=str, - help="directory to clean",) + help="directory to clean") group_db=parser.add_argument_group("Clean non-free packages in db", - "All arguments need to be specified") + "All arguments need to be specified for db cleaning") group_db.add_argument("-b", "--database", type=str, help="dabatase to clean") group_db.add_argument("-p", "--pending-file", type=str, @@ -82,8 +85,8 @@ if __name__ == "__main__": elif not args.pending_file or not args.whitelist_file \ and args.database: parser.print_help() - else: - blacklisted=listado(args.blacklist_file) + + blacklisted=listado(args.blacklist_file) if args.database: whitelisted=listado(args.whitelist_file) @@ -94,3 +97,4 @@ if __name__ == "__main__": if args.directory: cleanup_nonfree_in_dir(args.directory, blacklisted) + diff --git a/filter.py b/filter.py index c71f54d..5d90bdd 100755 --- a/filter.py +++ b/filter.py @@ -1,4 +1,4 @@ - #! /usr/bin/python +#! /usr/bin/python #-*- encoding: utf-8 -*- from glob import glob from repm.config import * @@ -188,11 +188,17 @@ if __name__ == "__main__": parser.add_argument("-k", "--blacklist-file", type=str, help="File containing blacklisted names", required=True,) - parser.add_argument("-c", "--rsync-command", type=str, - help="This command will be run to get a pkg list", + parser.add_argument("-f", "--rsout-file", type=str, + help="This file will be read to get a pkg list", required=True,) args=parser.parse_args() - rsout=check_output(args.rsync_command.split()) + try: + fsock=open(args.rsout_file, "r") + rsout=fsock.read() + except IOError: + print("%s is not readable" % args.rsout_file) + finally: + fsock.close() packages=pkginfo_from_rsync_output(rsout) - rsyncBlaclist_from_blacklist(packages, listado(args.blacklist_file), + rsyncBlacklist_from_blacklist(packages, listado(args.blacklist_file), args.rsync_exclude_file) diff --git a/local_config.example b/local_config.example index 0a827f4..05694e9 100644 --- a/local_config.example +++ b/local_config.example @@ -1,6 +1,6 @@ # Mirror options mirror="mirrors.eu.kernel.org" -mirrorpath="::mirrors/archlinux" +mirrorpath="mirrors/archlinux" # Directories: they should end without / paraboladir=~/parabolagnulinux.org @@ -13,11 +13,14 @@ licenses_dir=${docs_dir}/pending_licenses # Files logname=${paraboladir}/log/$(date -u +%Y%m%d-%H:%M)-repo-maintainer.log +rsout_file=${tempdir}/rsout +rsync_not_needed=${tempdir}/rsync_not_needed + +rsync_blacklist=${docs_dir}/rsyncBlacklist + blacklist=${docs_dir}/blacklist.txt whitelist=${docs_dir}/whitelist.txt -rsync_blacklist=${docs_dir}/rsyncBlacklist -rsync_not_needed=${tmp}/rsync_not_needed # Rsync commands -rsync_list_command="rsync\ -ptgoL\ --list-only\ " -rsync_update_command="rsync -ptgoL --exclude='*.abs.tar.*'" +rsync_list_command="rsync -rptgoL --exclude='*.abs.tar.*' --list-only --no-motd " +rsync_update_command="rsync -rptgoL --exclude='*.abs.tar.*' --no-motd " diff --git a/repo-update b/repo-update index 495f515..9b36d75 100755 --- a/repo-update +++ b/repo-update @@ -8,21 +8,22 @@ source ./libremessages for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]} 'any'; do msg "Syncing ${repo} ${arch}" - python filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ - \"${rsync_list_command}\ \ - ${mirror}${mirrorpath}/${repo}/os/${arch}\ \ - ${repodir}/${repo}/\" - find ${repodir}/${repo} -name *${PKGEXT} -print \ - > ${rsync_not_needed} + ${rsync_list_command} \ + ${mirror}::${mirrorpath}/${repo}/os/${arch}/ \ + ${repodir}/${repo}/ > ${rsout_file} + filter.py -r ${rsync_blacklist} -k ${blacklist} \ + -f ${rsout_file} + find ${repodir}/${repo} -name *${PKGEXT} \ + -fprint ${rsync_not_needed} ${rsync_update_command} \ - ${mirror}${mirrorpath}/${repo}/os/${arch} \ - ${repodir}/${repo} \ --exclude-from=${rsync_blacklist} \ --exclude-from=${rsync_not_needed} + ${mirror}::${mirrorpath}/${repo}/os/${arch}/ \ + ${repodir}/${repo}/ done for arch in ${ARCHES[@]}; do if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then - python clean_repo.py -k ${blacklist} -w ${whitelist} \ + clean_repo.py -k ${blacklist} -w ${whitelist} \ -p ${docs_dir}/pending-${repo} \ -b ${repodir}/${repo}/${repo}${DBEXT} fi -- cgit v1.2.3-54-g00ecf From 79e48771c2d64b50db5c8dadb4636e98fab94480 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 19 Apr 2011 19:23:37 -0500 Subject: Next fix --- repo-update | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/repo-update b/repo-update index 9b36d75..2354e0a 100755 --- a/repo-update +++ b/repo-update @@ -1,9 +1,9 @@ #!/bin/bash # -*- coding: utf-8 -*- -source ./config -source ./local_config -source ./libremessages +source config +source local_config +source libremessages for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]} 'any'; do -- cgit v1.2.3-54-g00ecf