From 3fb5f8efef231dd7784be880934cd106603ab6f1 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 11 Apr 2011 01:09:25 -0500 Subject: bash-port ready for testing. --- clean_repo.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 clean_repo.py (limited to 'clean_repo.py') diff --git a/clean_repo.py b/clean_repo.py new file mode 100644 index 0000000..29d446d --- /dev/null +++ b/clean_repo.py @@ -0,0 +1,45 @@ +#! /usr/bin/python +#-*- encoding: utf-8 -*- +from repm.filter import * +import argparse + +def remove_from_blacklist(path_to_db, blacklisted_names, + debug=config["debug"]): + """ Check the blacklist and remove packages on the db""" + + pkgs=[pkg for pkg in pkginfo_from_db(path_to_db) if + pkg["name"] in blacklisted_names] + if pkgs: + lista=" ".join(pkgs) + cmd = "repo-remove " + path_to_db + " " + lista + printf(cmd) + a = check_output(cmd) + if debug: + printf(a) + return pkgs, cmd + +def cleanup_nonfree_in_dir(directory, blacklisted_names): + pkgs=pkginfo_from_files_in_dir(directory) + for package in pkgs: + if package["name"] in blacklisted_names: + os.remove(package["location"]) + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Clean a repo db and packages") + parser.add_argument("-b", "--database", type=str, + help="dabatase to clean") + parser.add_argument("-d", "--directory", type=str, + help="directory to clean") + args=parser.parse_args() + + if args.directory: + cleanup_nonfree_in_dir(args.database, listado(config["blacklist"])) + + if args.database: + pkgs=pkginfo_from_db(args.database) + remove_from_blacklist(args.database, pkgs, + tuple(listado(config["blacklist"]) + + listado(config["pending"]))) + if not args.directory and not args.database: + parser.print_help() -- cgit v1.2.3-54-g00ecf From f6184ee3c9422c40effb0265527ee5a31b027a1a Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernandez Date: Sun, 10 Apr 2011 23:44:01 -0700 Subject: chmod +x --- .gitignore | 3 ++- clean_repo.py | 0 config.py | 0 filter.py | 0 main.sh | 2 +- mkpending.py | 0 6 files changed, 3 insertions(+), 2 deletions(-) mode change 100644 => 100755 clean_repo.py mode change 100644 => 100755 config.py mode change 100644 => 100755 filter.py mode change 100644 => 100755 main.sh mode change 100644 => 100755 mkpending.py (limited to 'clean_repo.py') diff --git a/.gitignore b/.gitignore index e645833..cfefcc3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *~ -*.pyc \ No newline at end of file +*.pyc +cptobin.sh diff --git a/clean_repo.py b/clean_repo.py old mode 100644 new mode 100755 diff --git a/config.py b/config.py old mode 100644 new mode 100755 diff --git a/filter.py b/filter.py old mode 100644 new mode 100755 diff --git a/main.sh b/main.sh old mode 100644 new mode 100755 index cf7bcd8..886ea38 --- a/main.sh +++ b/main.sh @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/bin/bash # -*- coding: utf-8 -*- source config.sh diff --git a/mkpending.py b/mkpending.py old mode 100644 new mode 100755 -- cgit v1.2.3-54-g00ecf From 67038ba1840d0f57b0ce49fdabd3dfa8057e2451 Mon Sep 17 00:00:00 2001 From: Joshua Haase Date: Mon, 11 Apr 2011 00:22:06 -0700 Subject: fixed some errors --- clean_repo.py | 2 +- config.py | 4 ++-- config.sh | 2 ++ cptobin.sh | 2 +- filter.py | 8 ++++---- mkpending.py | 4 ++-- 6 files changed, 12 insertions(+), 10 deletions(-) (limited to 'clean_repo.py') diff --git a/clean_repo.py b/clean_repo.py index 29d446d..b0d306f 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -34,7 +34,7 @@ if __name__ == "__main__": args=parser.parse_args() if args.directory: - cleanup_nonfree_in_dir(args.database, listado(config["blacklist"])) + cleanup_nonfree_in_dir(args.directory, listado(config["blacklist"])) if args.database: pkgs=pkginfo_from_db(args.database) diff --git a/config.py b/config.py index 24ecfaf..f4089f9 100755 --- a/config.py +++ b/config.py @@ -6,8 +6,8 @@ except(ImportError): from commands import getoutput as check_output import os -stringvars=("mirror", "mirrorpath", "logname", "tempdir", "docs_dir", - "repodir", "rsync_blacklist") +stringvars=("mirror", "mirrorpath", "logname", "tempdir", "archdb", "docs_dir", + "repodir", "blacklist", "whitelist", "pending", "rsync_blacklist") listvars=("repo_list", "dir_list", "arch_list", "other",) boolvars=("output", "debug",) diff --git a/config.sh b/config.sh index 741dee4..4f42e0b 100755 --- a/config.sh +++ b/config.sh @@ -12,6 +12,7 @@ logtime=$(date -u +%Y%m%d-%H:%M) ## Must be defined logname=${paraboladir}/${logtime}-repo-maintainer.log tempdir=~/tmp/ +archdb=${tempdir}/db docs_dir=${paraboladir}/docs repodir=${paraboladir}/repo # End Directories @@ -42,6 +43,7 @@ function run_python_cmd { mirrorpath=${mirrorpath} \ logname=${logname} \ tempdir=${tempdir} \ + archdb=${archdb} \ docs_dir=${docs_dir} \ repodir=${repodir} \ blacklist=${blacklist} \ diff --git a/cptobin.sh b/cptobin.sh index 25a22d4..e803132 100755 --- a/cptobin.sh +++ b/cptobin.sh @@ -1,2 +1,2 @@ #!/bin/bash -cp -f clean_repo.py config.py config.sh main.sh mkpending.py filter.py ~/usr/bin/ \ No newline at end of file +cp -f clean_repo.py config.py config.sh get_license.sh main.sh mkpending.py filter.py ~/usr/bin/ \ No newline at end of file diff --git a/filter.py b/filter.py index 1a0fa6f..78ad410 100755 --- a/filter.py +++ b/filter.py @@ -137,15 +137,15 @@ def pkginfo_from_db(path_to_db): if not os.path.isfile(path_to_db): raise NonValidFile(path_to_db + "is not a file") - check_output("mkdir -p " + archdb) + check_output("mkdir -p " + config["archdb"]) try: - db_open_tar = tarfile.open(db_tar_file, 'r:gz') + db_open_tar = tarfile.open(path_to_db, 'r:gz') except tarfile.ReadError: - printf("No valid db_file %s or not readable" % db_tar_file) + printf("No valid db_file %s or not readable" % path_to_db) return(tuple()) else: - printf("No db_file %s" % db_tar_file) + printf("No db_file %s" % path_to_db) return(tuple()) for file in db_open_tar.getmembers(): diff --git a/mkpending.py b/mkpending.py index 43a5fb2..2b255a8 100755 --- a/mkpending.py +++ b/mkpending.py @@ -32,9 +32,9 @@ def write_pending(packages_iterable, repo, prefix=config["pending"]): if __name__ == "__main__": parser = argparse.ArgumentParser( description="Clean a repo db and packages") - parser.add_argument("-b", "--dababase", type=str, required=True + parser.add_argument("-b", "--dababase", type=str, required=True, help="database to check") - parser.add_argument("-r", "--repo", type=str, required=True + parser.add_argument("-r", "--repo", type=str, required=True, help="repo of database") args=parser.parse_args() -- cgit v1.2.3-54-g00ecf From 3e27d11f68571bce92138f6cbfcaecac75fa1644 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 11 Apr 2011 12:39:40 -0500 Subject: Fixed some errors --- clean_repo.py | 6 +++++- config.py | 35 +++++++++++++++++++++++++++++------ config.sh | 5 ----- cptobin.sh | 2 +- filter.py | 32 +++++++++++++++++--------------- local_config.example | 10 +++++++--- main.sh | 3 ++- mkpending.py | 35 ++++++++++++++++++++--------------- pato2.py | 10 +--------- 9 files changed, 82 insertions(+), 56 deletions(-) (limited to 'clean_repo.py') diff --git a/clean_repo.py b/clean_repo.py index b0d306f..d4e06fc 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -6,7 +6,9 @@ import argparse def remove_from_blacklist(path_to_db, blacklisted_names, debug=config["debug"]): """ Check the blacklist and remove packages on the db""" - + if "~" in path_to_db: + path_to_db=(os.path.expanduser(path_to_db)) + pkgs=[pkg for pkg in pkginfo_from_db(path_to_db) if pkg["name"] in blacklisted_names] if pkgs: @@ -19,6 +21,8 @@ def remove_from_blacklist(path_to_db, blacklisted_names, return pkgs, cmd def cleanup_nonfree_in_dir(directory, blacklisted_names): + if "~" in directory: + directory=(os.path.expanduser(directory)) pkgs=pkginfo_from_files_in_dir(directory) for package in pkgs: if package["name"] in blacklisted_names: diff --git a/config.py b/config.py index f4089f9..8b48c66 100755 --- a/config.py +++ b/config.py @@ -3,11 +3,15 @@ try: from subprocess import check_output except(ImportError): - from commands import getoutput as check_output + from commands import getoutput + def check_output(*popenargs,**kwargs): + cmd=" ".join(*popenargs) + return getoutput(cmd) import os -stringvars=("mirror", "mirrorpath", "logname", "tempdir", "archdb", "docs_dir", - "repodir", "blacklist", "whitelist", "pending", "rsync_blacklist") +stringvars=("mirror", "mirrorpath", "logname", "tempdir", "archdb", + "repodir", "blacklist", "whitelist", "pending", + "rsync_blacklist",) listvars=("repo_list", "dir_list", "arch_list", "other",) boolvars=("output", "debug",) @@ -37,10 +41,29 @@ for var in boolvars: # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " +def printf(text,output=config["output"]): + """Guarda el texto en la variable log y puede imprimir en pantalla.""" + log_file = open(config["logname"], 'a') + log_file.write("\n" + str(text) + "\n") + log_file.close() + if output: + print (str(text) + "\n") + +del exit_if_none + # Classes and Exceptions -class NonValidFile(ValueError): pass -class NonValidDir(ValueError): pass -class NonValidCommand(ValueError): pass +class NonValidFile(ValueError): + def __init__(self): + ValueError.__init__(self) + printf(self.message) +class NonValidDir(ValueError): + def __init__(self): + ValueError.__init__(self) + printf(self.message) +class NonValidCommand(ValueError): + def __init__(self): + ValueError.__init__(self) + printf(self.message) class Package: """ An object that has information about a package. """ diff --git a/config.sh b/config.sh index 75d4b73..9a44f50 100755 --- a/config.sh +++ b/config.sh @@ -2,10 +2,6 @@ # -*- coding: utf-8 -*- source local_config -# Rsync commands -rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " -rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " - function run_python_cmd { env \ mirror=${mirror} \ @@ -13,7 +9,6 @@ function run_python_cmd { logname=${logname} \ tempdir=${tempdir} \ archdb=${archdb} \ - docs_dir=${docs_dir} \ repodir=${repodir} \ blacklist=${blacklist} \ whitelist=${whitelist} \ diff --git a/cptobin.sh b/cptobin.sh index e803132..068d765 100755 --- a/cptobin.sh +++ b/cptobin.sh @@ -1,2 +1,2 @@ #!/bin/bash -cp -f clean_repo.py config.py config.sh get_license.sh main.sh mkpending.py filter.py ~/usr/bin/ \ No newline at end of file +cp -f clean_repo.py config.py config.sh get_license.sh main.sh mkpending.py filter.py local_config ~/usr/bin/ \ No newline at end of file diff --git a/filter.py b/filter.py index 78ad410..48e2d93 100755 --- a/filter.py +++ b/filter.py @@ -4,12 +4,17 @@ from glob import glob from repm.config import * from repm.pato2 import * -def listado(filename): +def listado(filename,start=0,end=None): """Obtiene una lista de paquetes de un archivo.""" - archivo = open(filename,"r") - lista = archivo.read().split("\n") - archivo.close() - return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] + fsock = open(filename,"r") + lista = fsock.read().split("\n") + fsock.close() + if end is not None: + return [pkg.split(":")[start:end].rstrip() + for pkg in lista if pkg] + else: + return [pkg.split(":")[start].rstrip() + for pkg in lista if pkg] def pkginfo_from_filename(filename): """ Generates a Package object with info from a filename, @@ -135,29 +140,26 @@ def pkginfo_from_db(path_to_db): package_list=list() if not os.path.isfile(path_to_db): - raise NonValidFile(path_to_db + "is not a file") + raise NonValidFile(path_to_db + " is not a file") - check_output("mkdir -p " + config["archdb"]) + check_output(["mkdir", "-p", config["archdb"]]) try: db_open_tar = tarfile.open(path_to_db, 'r:gz') except tarfile.ReadError: - printf("No valid db_file %s or not readable" % path_to_db) - return(tuple()) - else: - printf("No db_file %s" % path_to_db) + raise NonValidFile("No valid db_file %s or not readable" % path_to_db) return(tuple()) for file in db_open_tar.getmembers(): - db_open_tar.extract(file, archdb) + db_open_tar.extract(file, config["archdb"]) db_open_tar.close() # Get info from file - for dir_ in glob(archdb + "/*"): + for dir_ in glob(config["archdb"] + "/*"): if isdir(dir_) and isfile(dir_ + "/desc"): package_list.append(pkginfo_from_desc( os.path.join(dir_,"desc"))) - check_output("rm -r %s/*" % archdb) - if verbose_: + check_output(["rm", "-r", config["archdb"]]) + if config["debug"]: printf(package_list) return package_list diff --git a/local_config.example b/local_config.example index 7edfde7..8015ee2 100644 --- a/local_config.example +++ b/local_config.example @@ -2,19 +2,19 @@ mirror="mirrors.eu.kernel.org" mirrorpath="::mirrors/archlinux" -# Directories +# Directories: they should end without / ## Optionals paraboladir=~/parabolagnulinux.org logtime=$(date -u +%Y%m%d-%H:%M) ## Must be defined -logname=${paraboladir}/${logtime}-repo-maintainer.log -tempdir=~/tmp/ +tempdir=~/tmp archdb=${tempdir}/db docs_dir=${paraboladir}/docs repodir=${paraboladir}/repo # End Directories # Files +logname=${paraboladir}/log/${logtime}-repo-maintainer.log blacklist=${docs_dir}/blacklist.txt whitelist=${docs_dir}/whitelist.txt pending=${docs_dir}/pending @@ -29,3 +29,7 @@ other="any" # Output options output="True" debug="False" + +# Rsync commands +rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " +rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " \ No newline at end of file diff --git a/main.sh b/main.sh index 886ea38..46f2f75 100755 --- a/main.sh +++ b/main.sh @@ -20,7 +20,8 @@ mkrsexclude msg "Syncing repos without delete" # rsync_update_command does not sync db or abs ${rsync_update_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ',')} ${repodir} + ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ','),\ + $(echo ${dir_list} | tr ':' ',')} ${repodir} msg "Syncing each repo and cleaning" msg2 "Remove pending files" diff --git a/mkpending.py b/mkpending.py index 2b255a8..6022206 100755 --- a/mkpending.py +++ b/mkpending.py @@ -1,9 +1,27 @@ #!/usr/bin/python # -*- coding: utf-8 -*- from repm.filter import * +import argparse -def make_pending(path_to_db): +def make_pending(path_to_db, repo, prefix=config["pending"]): """ Determine wich packages are pending for license auditing.""" + filename=prefix + "-" + repo + ".txt" + try: + fsock=open(filename, "rw") + if os.path.isfile(filename): + pkgs=[pkg for pkg in packages_iterable if pkg["name"] not in + listado(filename)] + fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] + for pkg in pkgs]) + "\n") + except(IOError): + print("Can't read %s" % filename) + exit(1) + finally: + fsock.close() + + if "~" in path_to_db: + path_to_db=(os.path.expanduser(path_to_db)) + packages_iterable=pkginfo_from_db(path_to_db) search = tuple(listado(config["blacklist"]) + listado(config["whitelist"])) @@ -15,24 +33,11 @@ def make_pending(path_to_db): def write_pending(packages_iterable, repo, prefix=config["pending"]): """ Write a pending file with the info of the packages """ - filename=prefix + "-" + repo + ".txt" - try: - fsock=open(filename, "a") - except(IOError): - print("Can't read %s" % filename) - finally: - fsock.close() - if os.path.isfile(filename): - pkgs=[pkg for pkg in packages_iterable if pkg["name"] not in - listado(filename)] - fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] - for pkg in pkgs]) + "\n") - fsock.close() if __name__ == "__main__": parser = argparse.ArgumentParser( description="Clean a repo db and packages") - parser.add_argument("-b", "--dababase", type=str, required=True, + parser.add_argument("-b", "--database", type=str, required=True, help="database to check") parser.add_argument("-r", "--repo", type=str, required=True, help="repo of database") diff --git a/pato2.py b/pato2.py index 4cdb536..6daa8b8 100644 --- a/pato2.py +++ b/pato2.py @@ -29,14 +29,6 @@ from repm.filter import * import tarfile from os.path import isdir, isfile -def printf(text,output=config["output"]): - """Guarda el texto en la variable log y puede imprimir en pantalla.""" - log_file = open(config["logname"], 'a') - log_file.write("\n" + str(text) + "\n") - log_file.close() - if output_: - print (str(text) + "\n") - def generate_rsync_command(base_command, dir_list=(config["repo_list"] + config["dir_list"]), @@ -68,4 +60,4 @@ def run_rsync(command,debug=config["debug"]): """ Runs rsync and gets returns it's output """ if debug: printf("rsync_command: " + command) - return check_output(command) + return check_output(command.split()) -- cgit v1.2.3-54-g00ecf From deab65fad4ced009fb31f7033b1db8ef0af78aee Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Fri, 15 Apr 2011 04:53:27 -0500 Subject: Python parts ready for bash usage in python 2 and 3 --- clean_repo.py | 78 ++++++++++++++++++++++++++++++----------------------- config.py | 65 +++++++++----------------------------------- filter.py | 48 ++++++++++++++++++++------------- pato2.py | 63 ------------------------------------------- test/test_filter.py | 3 ++- 5 files changed, 88 insertions(+), 169 deletions(-) delete mode 100644 pato2.py (limited to 'clean_repo.py') diff --git a/clean_repo.py b/clean_repo.py index eccfd01..5423c4d 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -3,21 +3,16 @@ from repm.filter import * import argparse -def mkpending(path_to_db, repo, prefix=config["pending"]): +def mkpending(packages_iterable, pending_file, blacklisted_names, + whitelisted_names): """ Determine wich packages are pending for license auditing.""" - if "~" in path_to_db: - path_to_db=(os.path.expanduser(path_to_db)) - - search = tuple(listado(config["blacklist"]) + - listado(config["whitelist"])) + search = tuple(blacklisted_names + + whitelisted_names) - pkgs=list(pkginfo_from_db(path_to_db)) - - filename=prefix + "-" + repo + ".txt" try: - fsock=open(filename, "rw") - pkgs=[pkg for pkg in pkginfo_from_db(path_to_db) - if pkg["name"] not in listado(filename)] + fsock=open(pending_file, "r") + pkgs=[pkg for pkg in packages_iterable + if pkg["name"] not in listado(pending_file)] for line in fsock.readlines(): if line: pkg=Package() @@ -26,16 +21,16 @@ def mkpending(path_to_db, repo, prefix=config["pending"]): pkgs.append(pkg) pkgs=[pkg for pkg in pkgs if pkg["name"] not in search and "custom" in pkg["license"]] + fsock=open(pending_file, "w") fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] for pkg in pkgs]) + "\n") except(IOError): - raise NonValidFile("Can't read or write %s" % filename) + raise NonValidFile("Can't read or write %s" % pending_file) finally: fsock.close() return pkgs -def remove_from_blacklist(path_to_db, blacklisted_names, - debug=config["debug"]): +def remove_from_blacklist(path_to_db, blacklisted_names): """ Check the blacklist and remove packages on the db""" if "~" in path_to_db: path_to_db=(os.path.expanduser(path_to_db)) @@ -47,9 +42,7 @@ def remove_from_blacklist(path_to_db, blacklisted_names, cmd = "repo-remove " + path_to_db + " " + lista printf(cmd) a = check_output(cmd) - if debug: - printf(a) - return pkgs, cmd + return pkgs def cleanup_nonfree_in_dir(directory, blacklisted_names): if "~" in directory: @@ -61,25 +54,42 @@ def cleanup_nonfree_in_dir(directory, blacklisted_names): if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Clean a repo db and packages") - parser.add_argument("-b", "--database", type=str, - help="dabatase to clean") - parser.add_argument("-d", "--directory", type=str, - help="directory to clean") + prog="clean_repo", + description="Clean a repo db and packages",) + + parser.add_argument("-k", "--blacklist-file", type=str, + help="File containing blacklisted names", + required=True,) + + group_dir=parser.add_argument_group("Clean non-free packages in dir") + group_dir.add_argument("-d", "--directory", type=str, + help="directory to clean",) + + group_db=parser.add_argument_group("Clean non-free packages in db", + "All arguments need to be specified") + group_db.add_argument("-b", "--database", type=str, + help="dabatase to clean") + group_db.add_argument("-p", "--pending-file", type=str, + help="File in which to write pending list") + group_db.add_argument("-w", "--whitelist-file", type=str, + help="File containing whitelisted names") + args=parser.parse_args() + if not args.directory and not args.database: + parser.print_help() + elif not args.pending_file or not args.whitelist_file \ + and args.database: + parser.print_help() + else: + blacklisted=listado(args.blacklist_file) + if args.database: - repo=os.path.basename(args.database).split(".")[0] + whitelisted=listado(args.whitelist_file) pkgs=pkginfo_from_db(args.database) - remove_from_blacklist(args.database, pkgs, - tuple(listado(config["blacklist"]) + - listado(config["pending"] + - "-" + repo + ".txt"))) - mkpending(args.database, args.repo) + remove_from_blacklist(args.database, blacklisted) + mkpending(pkgs, args.pending_file, + blacklisted, whitelisted) if args.directory: - cleanup_nonfree_in_dir(args.directory, - listado(config["blacklist"])) - - if not args.directory and not args.database: - parser.print_help() + cleanup_nonfree_in_dir(args.directory, blacklisted) diff --git a/config.py b/config.py index 8b48c66..4e218a5 100755 --- a/config.py +++ b/config.py @@ -9,61 +9,26 @@ except(ImportError): return getoutput(cmd) import os -stringvars=("mirror", "mirrorpath", "logname", "tempdir", "archdb", - "repodir", "blacklist", "whitelist", "pending", - "rsync_blacklist",) -listvars=("repo_list", "dir_list", "arch_list", "other",) -boolvars=("output", "debug",) - -config=dict() - -def exit_if_none(var): - if os.environ.get(var) is None: - exit("%s is not defined" % var) - -for var in stringvars: - exit_if_none(var) - config[var]=os.environ.get(var) - -for var in listvars: - exit_if_none(var) - config[var]=tuple(os.environ.get(var).split(":")) - -for var in boolvars: - exit_if_none(var) - if os.environ.get(var) == "True": - config[var]=True - elif os.environ.get(var) =="False": - config[var]=False - else: - print('%s is not True or False' % var) # Rsync commands -rsync_list_command="rsync -a --no-motd --list-only " -def printf(text,output=config["output"]): +def printf(text, logfile=False): """Guarda el texto en la variable log y puede imprimir en pantalla.""" - log_file = open(config["logname"], 'a') - log_file.write("\n" + str(text) + "\n") - log_file.close() - if output: - print (str(text) + "\n") + print (str(text) + "\n") + if logfile: + try: + log = open(logfile, 'a') + log.write("\n" + str(text) + "\n") + except: + print("Can't open %s" % logfile) + finally: + log.close() -del exit_if_none # Classes and Exceptions -class NonValidFile(ValueError): - def __init__(self): - ValueError.__init__(self) - printf(self.message) -class NonValidDir(ValueError): - def __init__(self): - ValueError.__init__(self) - printf(self.message) -class NonValidCommand(ValueError): - def __init__(self): - ValueError.__init__(self) - printf(self.message) +class NonValidFile(ValueError): pass +class NonValidDir(ValueError): pass +class NonValidCommand(ValueError): pass class Package: """ An object that has information about a package. """ @@ -101,7 +66,3 @@ class Package: return False else: return True - -if __name__=="__main__": - for key in config.keys(): - print("%s : %s" % (key,config[key])) diff --git a/filter.py b/filter.py index 4b9603d..add0235 100755 --- a/filter.py +++ b/filter.py @@ -2,7 +2,7 @@ #-*- encoding: utf-8 -*- from glob import glob from repm.config import * -from repm.pato2 import * +import tarfile def listado(filename,start=0,end=None): """Obtiene una lista de paquetes de un archivo.""" @@ -28,7 +28,7 @@ def pkginfo_from_filename(filename): ---------- pkg -> Package object""" if ".pkg.tar." not in filename: - raise NonValidFile + raise NonValidFile("File is not a pacman package") pkg = Package() pkg["location"] = filename fileattrs = os.path.basename(filename).split("-") @@ -140,19 +140,18 @@ def pkginfo_from_db(path_to_db): desc_files=[desc for desc in dbsock.getnames() if "/desc" in desc] for name in desc_files: - desc=dbsock.extractfile(name) - package_list.append(pkginfo_from_desc(desc.read())) + desc=dbsock.extractfile(name).read().decode("UTF-8") + package_list.append(pkginfo_from_desc(desc)) except tarfile.ReadError: raise NonValidFile("No valid db_file %s or not readable" % path_to_db) - return(tuple()) finally: dbsock.close() return package_list def rsyncBlacklist_from_blacklist(packages_iterable, blacklisted_names, - exclude_file=config["rsync_blacklist"]): + exclude_file): """ Generate an exclude list for rsync Parameters: @@ -168,20 +167,31 @@ def rsyncBlacklist_from_blacklist(packages_iterable, pkgs=[pkg["location"] for pkg in packages_iterable if isinstance(pkg, Package) and pkg["name"] in blacklisted_names] - - try: - fsock = open(exclude_file,"w") - fsock.write("\n".join(pkgs) + "\n") - except IOError: - printf("%s wasnt written" % exclude_file) - exit(1) - finally: - fsock.close() + if exclude_file: + try: + fsock = open(exclude_file,"w") + fsock.write("\n".join(pkgs) + "\n") + except IOError: + printf("%s wasnt written" % exclude_file) + exit(1) + finally: + fsock.close() return pkgs if __name__ == "__main__": - cmd=generate_rsync_command(rsync_list_command) - a=run_rsync(cmd) - packages=pkginfo_from_rsync_output(a) - rsyncBlaclist_from_blacklist(packages,listado(blacklist)) + import argparse + parser=argparse.ArgumentParser() + parser.add_argument("-r", "--rsync-exclude-file", type=str, + help="File in which to generate exclude list", + required=True,) + parser.add_argument("-k", "--blacklist-file", type=str, + help="File containing blacklisted names", + required=True,) + parser.add_argument("-c", "--rsync-command", type=str, + help="This command will be run to get a pkg list") + args=parser.parse_args() + rsout=check_output(args.rsync_command) + packages=pkginfo_from_rsync_output(rsout) + rsyncBlaclist_from_blacklist(packages, listado(args.blacklist_file), + args.rsync_exclude_file) diff --git a/pato2.py b/pato2.py deleted file mode 100644 index 6daa8b8..0000000 --- a/pato2.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -""" - parabola.py - Copyright 2009 Rafik Mas'ad - Copyright 2010 Joshua Ismael Haase Hernández - - ---------- GNU General Public License 3 ---------- - - This file is part of Parabola. - - Parabola is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - Parabola is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with Parabola. If not, see . - - -""" -from repm.config import * -from repm.filter import * -import tarfile -from os.path import isdir, isfile - -def generate_rsync_command(base_command, - dir_list=(config["repo_list"] + - config["dir_list"]), - destdir=config["repodir"], - source=config["mirror"] +config["mirrorpath"]): - """ Generates an rsync command for executing - it by combining all parameters. - - Parameters: - ---------- - base_command -> str - dir_list -> list or tuple - destdir -> str Path to dir, dir must exist. - source -> str The source for rsync - blacklist_file -> False or str Path to file, file must exist. - - Return: - ---------- - rsync_command -> str """ - if not os.path.isdir(destdir): - print(destdir + " is not a directory") - raise NonValidDir - - dir_list="{" + ",".join(dir_list) + "}" - return " ".join((base_command, os.path.join(source, dir_list), - destdir)) - -def run_rsync(command,debug=config["debug"]): - """ Runs rsync and gets returns it's output """ - if debug: - printf("rsync_command: " + command) - return check_output(command.split()) diff --git a/test/test_filter.py b/test/test_filter.py index b6d5766..d8006f9 100644 --- a/test/test_filter.py +++ b/test/test_filter.py @@ -143,7 +143,8 @@ class generateRsyncBlacklist(unittest.TestCase): def testExcludeFiles(self): a=rsyncBlacklist_from_blacklist(self.example_package_list, - listado("blacklist_sample")) + listado("blacklist_sample"), + False) b=[self.example_package_list[0]["location"],self.example_package_list[2]["location"]] self.assertEqual(a,b) -- cgit v1.2.3-54-g00ecf From 374c8e2a5183cdbaefe9f54184603ad8d09e30c2 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sat, 16 Apr 2011 03:15:01 -0500 Subject: * bash-repo for testing * merged dbscripts with repm --- changelog | 2 -- clean_repo.py | 3 ++- config | 2 +- config.sh | 26 ---------------------- cptobin.sh | 2 -- filter.py | 5 +++-- get_license.sh | 50 +++++++++++++++++------------------------- local_config.example | 18 ++++++--------- main.sh | 62 +++++++++++++++++++++++----------------------------- 9 files changed, 60 insertions(+), 110 deletions(-) delete mode 100644 changelog delete mode 100755 config.sh delete mode 100755 cptobin.sh (limited to 'clean_repo.py') diff --git a/changelog b/changelog deleted file mode 100644 index 0e89f2f..0000000 --- a/changelog +++ /dev/null @@ -1,2 +0,0 @@ -repo-maintainer 1.0 - * Known working version diff --git a/clean_repo.py b/clean_repo.py index 5423c4d..e1a17c2 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -22,7 +22,8 @@ def mkpending(packages_iterable, pending_file, blacklisted_names, pkgs=[pkg for pkg in pkgs if pkg["name"] not in search and "custom" in pkg["license"]] fsock=open(pending_file, "w") - fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] + fsock.write("\n".join([pkg["name"] + ":" + pkg["location"] + + ":" + pkg["license"] for pkg in pkgs]) + "\n") except(IOError): raise NonValidFile("Can't read or write %s" % pending_file) diff --git a/config b/config index 217b627..d9bad13 100644 --- a/config +++ b/config @@ -1,7 +1,7 @@ FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" -PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') +PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing' 'multilib') PKGPOOL='pool/packages' SRCPOOL='sources/packages' diff --git a/config.sh b/config.sh deleted file mode 100755 index 9a44f50..0000000 --- a/config.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -# -*- coding: utf-8 -*- -source local_config - -function run_python_cmd { - env \ - mirror=${mirror} \ - mirrorpath=${mirrorpath} \ - logname=${logname} \ - tempdir=${tempdir} \ - archdb=${archdb} \ - repodir=${repodir} \ - blacklist=${blacklist} \ - whitelist=${whitelist} \ - pending=${pending} \ - rsync_blacklist=${rsync_blacklist} \ - repo_list=${repo_list} \ - dir_list=${dir_list} \ - arch_list=${arch_list} \ - other=${other} \ - output=${output} \ - debug=${debug} \ - $1 -} - -source libremessages \ No newline at end of file diff --git a/cptobin.sh b/cptobin.sh deleted file mode 100755 index ce4b3bd..0000000 --- a/cptobin.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -cp -f clean_repo.py config.py config.sh get_license.sh main.sh filter.py local_config ~/usr/bin/ \ No newline at end of file diff --git a/filter.py b/filter.py index add0235..4abdf38 100755 --- a/filter.py +++ b/filter.py @@ -189,9 +189,10 @@ if __name__ == "__main__": help="File containing blacklisted names", required=True,) parser.add_argument("-c", "--rsync-command", type=str, - help="This command will be run to get a pkg list") + help="This command will be run to get a pkg list", + required=True,) args=parser.parse_args() - rsout=check_output(args.rsync_command) + rsout=check_output(args.rsync_command.split()) packages=pkginfo_from_rsync_output(rsout) rsyncBlaclist_from_blacklist(packages, listado(args.blacklist_file), args.rsync_exclude_file) diff --git a/get_license.sh b/get_license.sh index 0da58cb..024876c 100755 --- a/get_license.sh +++ b/get_license.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # -*- coding: utf-8 -*- # get_license.sh @@ -20,36 +20,26 @@ # You should have received a copy of the GNU General Public License # along with Parabola. If not, see . - -docs="/home/parabolavnx/parabolagnulinux.org/docs" -repo="/home/parabolavnx/parabolagnulinux.org/repo" -dir="$docs/pending-licenses" - -echo "Cleaning $dir" -rm -rf $dir/* - -tempdir=$(mktemp -d) -cd $tempdir - -pending=($(cut -d: -f1 $docs/pending*.txt)) -echo ${pending[@]} - -for pkg in ${pending[@]}; do - pkg_in_repo=( $(ls ${repo}/*/os/*/${pkg}*) ) - for y in ${pkg_in_repo[@]}; do - echo "chmod +r $y" - chmod +r $y - echo "tar -xf $y usr/share/licenses" - bsdtar -xf $y usr/share/licenses - echo "chmod -r $y" - chmod -r $y +source ./config +source ./local_config +source ./libremessages + +msg "Creating pending licenses list" +pushd ${licenses_dir} +rm -rf ${licenses_dir}/* + +for repo in ${PKGREPOS[@]}; do + msg2 "Extracting licenses in ${repo}" + pending=($(cut -d: -f2 ${docs_dir}/pending-${repo})) + pushd ${repodir}/${repo} + for pkg in ${pending[@]}; do + plain "${pkg}" + bsdtar -xf ${pkg} usr/share/licenses || { + error "${pkg} has no licenses" + } + chmod -r ${pkg} done done -mv usr/share/licenses/* $dir - -cd - -rm -rf $tempdir - +popd exit 0 \ No newline at end of file diff --git a/local_config.example b/local_config.example index 99a1fdb..2c0ef3e 100644 --- a/local_config.example +++ b/local_config.example @@ -4,21 +4,21 @@ mirrorpath="::mirrors/archlinux" # Directories: they should end without / ## Optionals -paraboladir=~/parabolagnulinux.org -logtime=$(date -u +%Y%m%d-%H:%M) +paraboladir= ## Must be defined tempdir=~/tmp archdb=${tempdir}/db docs_dir=${paraboladir}/docs -repodir=${paraboladir}/repo +repodir=${paraboladir}/repo/staging +licenses_dir=${docs_dir}/pending_licenses # End Directories # Files -logname=${paraboladir}/log/${logtime}-repo-maintainer.log +logname=${paraboladir}/log/$(date -u +%Y%m%d-%H:%M)-repo-maintainer.log blacklist=${docs_dir}/blacklist.txt whitelist=${docs_dir}/whitelist.txt -pending=${docs_dir}/pending rsync_blacklist=${docs_dir}/rsyncBlacklist +rsync_not_needed=${tmp}/rsync_not_needed # Repos, arches, and dirs for repo repo_list="core:extra:community:testing:community-testing:multilib" @@ -26,10 +26,6 @@ dir_list="pool" arch_list="i686:x86_64" other="any" -# Output options -output="True" -debug="False" - # Rsync commands -rsync_update_command="rsync -nav --delay-updates --exclude='*.{abs|db}.tar.*' " -rsync_post_command="rsync -nav --delete --exclude='*.abs.tar.*' " \ No newline at end of file +rsync_list_command="rsync\ -ptgoL\ --list-only\ " +rsync_update_command="rsync -ptgoL --exclude='*.abs.tar.*'" diff --git a/main.sh b/main.sh index d15f736..b3ecb92 100755 --- a/main.sh +++ b/main.sh @@ -1,43 +1,35 @@ #!/bin/bash # -*- coding: utf-8 -*- -source config.sh +source ./config +source ./local_config +source ./libremessages -function mkrsexclude { - local error=1 - while ${error}; do - run_python_cmd "filter.py" - error=$? - done -} - -msg "Cleaning $tempdir" -stdnull "rm -r $tempdir/* " - -msg "Generating exclude list for rsync" -mkrsexclude - -msg "Syncing repos without delete" -# rsync_update_command does not sync db or abs -${rsync_update_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ','),\ - $(echo ${dir_list} | tr ':' ',')} ${repodir} - -msg "Syncing each repo and cleaning" -msg2 "Remove pending files" -stdnull "rm -rf ${pending}*" -for repo in $(echo ${repo_list} | tr ':' ' '); do - for arch in $(echo ${arch_list} | tr ':' ' '); do - msg2 "Syncing ${repo} ${arch}" - cmd=$(echo ${rsync_post_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/${repo} ${repodir}/${repo}) - plain "${cmd}" - ${cmd} - msg2 "Cleaning ${repo} ${arch}" - # This also generates pending lists - run_python_cmd "clean_repo.py -b ${repodir}/${repo}/os/${arch}/${repo}.db.tar.gz -d ${repodir}/${repo}/os/${arch}/" +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]} 'any'; do + msg "Syncing ${repo} ${arch}" + filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ + \"${rsync_list_command}\ \ + ${mirror}${mirrorpath}/${repo}/os/${arch}\ \ + ${repodir}/${repo}/\" + find ${repodir}/${repo} -name *${PKGEXT} -print \ + > ${rsync_not_needed} + ${rsync_update_command} \ + ${mirror}${mirrorpath}/${repo}/os/${arch} \ + ${repodir}/${repo} \ + --exclude-from=${rsync_blacklist} \ + --exclude-from=${rsync_not_needed} done + for arch in ${ARCHES[@]}; do + if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then + clean_repo.py -k ${blacklist} -w ${whitelist} \ + -p ${docs_dir}/pending-${repo} \ + -b ${repodir}/${repo}/${repo}${DBEXT} + fi + clean_repo.py -k ${blacklist} -d ${repodir}/${repo} done -msg "Checking licenses" +db-update +ftpdir-cleanup + get_license.sh -- cgit v1.2.3-54-g00ecf