From 2321a53d35a9736b8c5c715ca40a9af69b1bf64e Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 01:14:47 -0600 Subject: Rearranged functions --- pato2.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) (limited to 'pato2.py') diff --git a/pato2.py b/pato2.py index cfdd859..125ae8c 100644 --- a/pato2.py +++ b/pato2.py @@ -166,6 +166,46 @@ def get_licenses(verbose_=verbose): a=commands.getoutput(cmd_) if verbose_: printf(a) +def generate_rsync_command(base_command, dir_list, destdir=repodir, + source=mirror+mirrorpath, blacklist_file=False): + """ Generates an rsync command for executing it by combining all parameters. + + Parameters: + ---------- + base_command -> str + dir_list -> list or tuple + destdir -> str Path to dir, dir must exist. + source -> str The source for rsync + blacklist_file -> False or str Path to file, file must exist. + + Return: + ---------- + rsync_command -> str """ + from os.path import isfile, isdir + + if blacklist_file and not isfile(blacklist_file): + print(blacklist_file + " is not a file") + raise NonValidFile + + if not os.path.isdir(destdir): + print(destdir + " is not a directory") + raise NonValidDir + + dir_list="{" + ",".join(dir_list) + "}" + + if blacklist_file: + return " ".join((base_command, "--exclude-from-file="+blacklist_file, + mirror_name + mirror_path + dir_list, destdir)) + return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) + +def run_rsync(base_for_rsync=rsync_list_command, dir_list_for_rsync=(repo_list + dir_list), + debug=verbose): + """ Runs rsync and gets returns it's output """ + cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) + if debug: + printf("rsync_command" + cmd) + return commands.getoutput(cmd) + if __name__ == "__main__": from time import time start_time = time() -- cgit v1.2.3-54-g00ecf From 6203dc2fc926781db694ca2383b1e44e2c5469c7 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 01:29:33 -0600 Subject: - filter.py: * Makes a blacklist for rsync - pato2.py: * Has more functions --- filter.py | 6 +++++- pato2.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'pato2.py') diff --git a/filter.py b/filter.py index b51dba8..14ae31e 100644 --- a/filter.py +++ b/filter.py @@ -1,4 +1,4 @@ -#! /usr/bin/python + #! /usr/bin/python #-*- encoding: utf-8 -*- import commands import os @@ -79,3 +79,7 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, except IOError: printf("%s wasnt written" % blacklist_file) +if name == "__main__": + a=run_rsync(rsync_list_command) + packages=get_file_list_from_rsync_output(a) + generate_exclude_list_from_blacklist(packages,listado(blacklist)) diff --git a/pato2.py b/pato2.py index 125ae8c..97eec9d 100644 --- a/pato2.py +++ b/pato2.py @@ -198,7 +198,7 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name + mirror_path + dir_list, destdir)) return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) -def run_rsync(base_for_rsync=rsync_list_command, dir_list_for_rsync=(repo_list + dir_list), +def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), debug=verbose): """ Runs rsync and gets returns it's output """ cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) -- cgit v1.2.3-54-g00ecf From 7f40b89e8cd0e9b2cfd92a8b74c4063d8b9aa3e3 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 01:38:59 -0600 Subject: Corrected funcions --- pato2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'pato2.py') diff --git a/pato2.py b/pato2.py index 97eec9d..55e63e7 100644 --- a/pato2.py +++ b/pato2.py @@ -195,13 +195,13 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, if blacklist_file: return " ".join((base_command, "--exclude-from-file="+blacklist_file, - mirror_name + mirror_path + dir_list, destdir)) - return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) + source + dir_list, destdir)) + return " ".join((base_command, source + dir_list, destdir)) def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), debug=verbose): """ Runs rsync and gets returns it's output """ - cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) + cmd = str(generate_rsync_command(base_for_rsync, (repo_list + dir_list))) if debug: printf("rsync_command" + cmd) return commands.getoutput(cmd) -- cgit v1.2.3-54-g00ecf From 9fecb003deaae2364e0d9d41055ca33a7d1a2131 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 17:09:16 -0600 Subject: * Changed format of run_rsync debug output. --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'pato2.py') diff --git a/pato2.py b/pato2.py index 55e63e7..13242f7 100644 --- a/pato2.py +++ b/pato2.py @@ -203,7 +203,7 @@ def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), """ Runs rsync and gets returns it's output """ cmd = str(generate_rsync_command(base_for_rsync, (repo_list + dir_list))) if debug: - printf("rsync_command" + cmd) + printf("rsync_command: " + cmd) return commands.getoutput(cmd) if __name__ == "__main__": -- cgit v1.2.3-54-g00ecf From e4ffbeff4bbdeb3f57c39c6f6dac3ce902db9b4c Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 17:14:37 -0600 Subject: * Fixed generate_rsync_command --- pato2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'pato2.py') diff --git a/pato2.py b/pato2.py index 13242f7..74d18be 100644 --- a/pato2.py +++ b/pato2.py @@ -195,8 +195,8 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, if blacklist_file: return " ".join((base_command, "--exclude-from-file="+blacklist_file, - source + dir_list, destdir)) - return " ".join((base_command, source + dir_list, destdir)) + os.join(source, dir_list), destdir)) + return " ".join((base_command, os.join(source, dir_list), destdir)) def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), debug=verbose): -- cgit v1.2.3-54-g00ecf From e5b869eade3fbf47cf9ac1b7c6be34dfee14b2c1 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 17:17:31 -0600 Subject: * Fixed generate_rsync_command --- pato2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'pato2.py') diff --git a/pato2.py b/pato2.py index 74d18be..e32a4dd 100644 --- a/pato2.py +++ b/pato2.py @@ -195,8 +195,8 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, if blacklist_file: return " ".join((base_command, "--exclude-from-file="+blacklist_file, - os.join(source, dir_list), destdir)) - return " ".join((base_command, os.join(source, dir_list), destdir)) + os.path.join(source, dir_list), destdir)) + return " ".join((base_command, os.path.join(source, dir_list), destdir)) def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), debug=verbose): -- cgit v1.2.3-54-g00ecf From 9de298a7b88f7f36aec4c9344356a935c67cfeb3 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 23:27:46 -0600 Subject: * Added test for generate_exclude_list_from_blacklist * Fixed listado to strip spaces --- filter.py | 1 - pato2.py | 2 +- test/blacklist_sample | 2 ++ test/test_pkginfo_from_rsync_output.py | 17 +++++++++++------ 4 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 test/blacklist_sample (limited to 'pato2.py') diff --git a/filter.py b/filter.py index f8182e3..f91ec68 100644 --- a/filter.py +++ b/filter.py @@ -107,7 +107,6 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, a.append(package["location"]) if debug: - printf(a) return a try: fsock = open(exclude_file,"w") diff --git a/pato2.py b/pato2.py index e32a4dd..5bc97a9 100644 --- a/pato2.py +++ b/pato2.py @@ -42,7 +42,7 @@ def listado(filename_): archivo = open(filename_,"r") lista = archivo.read().split("\n") archivo.close() - return [pkg.split(":")[0] for pkg in lista if pkg] + return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] def db(repo_,arch_): """Construye un nombre para sincronizar una base de datos.""" diff --git a/test/blacklist_sample b/test/blacklist_sample new file mode 100644 index 0000000..2a02af6 --- /dev/null +++ b/test/blacklist_sample @@ -0,0 +1,2 @@ +alex:alex-libre: Aquí va un comentario +gmime22 ::Non free dependencies \ No newline at end of file diff --git a/test/test_pkginfo_from_rsync_output.py b/test/test_pkginfo_from_rsync_output.py index 5ed5dd8..aca49e8 100644 --- a/test/test_pkginfo_from_rsync_output.py +++ b/test/test_pkginfo_from_rsync_output.py @@ -47,14 +47,19 @@ class pkginfoFromRsyncOutput(unittest.TestCase): self.assertIsInstance(pkg,Package) def testPackageInfo(self): - if self.pkglist: - first_package=self.pkglist[0] - else: - self.fail(self.pkglist) - self.assertEqual(first_package,example_package_list[0]) + if not self.pkglist: + self.fail("Pkglist doesn't exist: " + str(self.pkglist)) + self.assertEqual(self.pkglist,example_package_list) class generateRsyncBlacklist(unittest.TestCase): """ Test Blacklist generation """ - + def testListado(self): + self.assertEqual(listado("blacklist_sample"),["alex","gmime22"]) + + def testExcludeFiles(self): + a=generate_exclude_list_from_blacklist(example_package_list,listado("blacklist_sample"),debug=True) + b=[example_package_list[0]["location"],example_package_list[2]["location"]] + self.assertEqual(a,b) + if __name__ == "__main__": unittest.main() -- cgit v1.2.3-54-g00ecf From cf3253774c64e6430d4e3afb826cc6c8b0e4b91b Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 15 Feb 2011 00:13:37 -0600 Subject: * Upgraded sync_all_repo --- config.py | 2 +- pato2.py | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) (limited to 'pato2.py') diff --git a/config.py b/config.py index 465a8a8..36b54df 100644 --- a/config.py +++ b/config.py @@ -44,7 +44,7 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " - +rsync_update_command="rsync -av --delete-after --delay-updates " # Classes and Exceptions class NonValidFile(ValueError): pass diff --git a/pato2.py b/pato2.py index 5bc97a9..9b902f4 100644 --- a/pato2.py +++ b/pato2.py @@ -25,7 +25,7 @@ """ from repm.config import * - +from repm.filter import * import tarfile from glob import glob from os.path import isdir, isfile @@ -52,12 +52,14 @@ def packages(repo_, arch_, expr="*"): """ Get packages on a repo, arch folder """ return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) ) -def sync_all_repo(verbose_=verbose): - folders = ",".join(repo_list + dir_list) - cmd_ = "rsync -av --delete-after --delay-updates " + mirror + mirrorpath + "/{" + folders + "} " + repodir - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose_: printf(a) +def sync_all_repo(debug=verbose): + cmd=generate_rsync_command(rsync_list_command) + rsout=run_rsync(cmd) + pkgs=pkginfo_from_rsync_output(rsout) + generate_exclude_list_from_blacklist(pkgs,listado(blacklist)) + cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) + a=run_rsync(cmd) + if debug: printf(a) def get_from_desc(desc, var,db_tar_file=False): """ Get a var from desc file """ @@ -166,7 +168,7 @@ def get_licenses(verbose_=verbose): a=commands.getoutput(cmd_) if verbose_: printf(a) -def generate_rsync_command(base_command, dir_list, destdir=repodir, +def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdir=repodir, source=mirror+mirrorpath, blacklist_file=False): """ Generates an rsync command for executing it by combining all parameters. @@ -198,10 +200,8 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, os.path.join(source, dir_list), destdir)) return " ".join((base_command, os.path.join(source, dir_list), destdir)) -def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), - debug=verbose): +def run_rsync(command,debug=verbose): """ Runs rsync and gets returns it's output """ - cmd = str(generate_rsync_command(base_for_rsync, (repo_list + dir_list))) if debug: printf("rsync_command: " + cmd) return commands.getoutput(cmd) -- cgit v1.2.3-54-g00ecf From fb247388925beabe01162f28bfcba7fcf8809254 Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 14 Feb 2011 22:41:29 -0800 Subject: * Fixed for rsync to run --- pato2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'pato2.py') diff --git a/pato2.py b/pato2.py index 9b902f4..c598c2f 100644 --- a/pato2.py +++ b/pato2.py @@ -196,15 +196,15 @@ def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdi dir_list="{" + ",".join(dir_list) + "}" if blacklist_file: - return " ".join((base_command, "--exclude-from-file="+blacklist_file, + return " ".join((base_command, "--exclude-from="+blacklist_file, os.path.join(source, dir_list), destdir)) return " ".join((base_command, os.path.join(source, dir_list), destdir)) def run_rsync(command,debug=verbose): """ Runs rsync and gets returns it's output """ if debug: - printf("rsync_command: " + cmd) - return commands.getoutput(cmd) + printf("rsync_command: " + command) + return commands.getoutput(command) if __name__ == "__main__": from time import time -- cgit v1.2.3-54-g00ecf From 7bb59afd65871ed2949b86ec6cef805ea2fd3dd2 Mon Sep 17 00:00:00 2001 From: Nicolás Reynolds Date: Tue, 15 Feb 2011 19:25:51 -0800 Subject: Fixed add_free_repo --- config.py | 1 + pato2.py | 295 ++++++++++++++++++++++++++++++++------------------------------ 2 files changed, 152 insertions(+), 144 deletions(-) (limited to 'pato2.py') diff --git a/config.py b/config.py index c60e8f3..6420355 100644 --- a/config.py +++ b/config.py @@ -19,6 +19,7 @@ logdir = path + "/log" ## Must be defined logname= logdir + "/" + time__ + "-repo-maintainer.log" +freedir= path + "/free/" repodir= path + "/repo" tmp = home + "/tmp" archdb = tmp + "/db" diff --git a/pato2.py b/pato2.py index c598c2f..f5deb48 100644 --- a/pato2.py +++ b/pato2.py @@ -31,142 +31,149 @@ from glob import glob from os.path import isdir, isfile def printf(text,output_=output): - """Guarda el texto en la variable log y puede imprimir en pantalla.""" - log_file = open(logname, 'a') - log_file.write("\n" + str(text) + "\n") - log_file.close() - if output_: print (str(text) + "\n") + """Guarda el texto en la variable log y puede imprimir en pantalla.""" + log_file = open(logname, 'a') + log_file.write("\n" + str(text) + "\n") + log_file.close() + if output_: print (str(text) + "\n") def listado(filename_): - """Obtiene una lista de paquetes de un archivo.""" - archivo = open(filename_,"r") - lista = archivo.read().split("\n") - archivo.close() - return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] + """Obtiene una lista de paquetes de un archivo.""" + archivo = open(filename_,"r") + lista = archivo.read().split("\n") + archivo.close() + return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] def db(repo_,arch_): - """Construye un nombre para sincronizar una base de datos.""" - return "/%s/os/%s/%s.db.tar.gz" % (repo_, arch_, repo_) + """Construye un nombre para sincronizar una base de datos.""" + return "/%s/os/%s/%s.db.tar.gz" % (repo_, arch_, repo_) def packages(repo_, arch_, expr="*"): - """ Get packages on a repo, arch folder """ - return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) ) + """ Get packages on a repo, arch folder """ + return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) ) def sync_all_repo(debug=verbose): - cmd=generate_rsync_command(rsync_list_command) - rsout=run_rsync(cmd) - pkgs=pkginfo_from_rsync_output(rsout) - generate_exclude_list_from_blacklist(pkgs,listado(blacklist)) - cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) - a=run_rsync(cmd) - if debug: printf(a) + cmd=generate_rsync_command(rsync_list_command) + rsout=run_rsync(cmd) + pkgs=pkginfo_from_rsync_output(rsout) + generate_exclude_list_from_blacklist(pkgs,listado(blacklist)) + cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) + a=run_rsync(cmd) + if debug: printf(a) def get_from_desc(desc, var,db_tar_file=False): - """ Get a var from desc file """ - desc = desc.split("\n") - return desc[desc.index(var)+1] + """ Get a var from desc file """ + desc = desc.split("\n") + return desc[desc.index(var)+1] def get_info(repo_,arch_,db_tar_file=False,verbose_=verbose): - """ Makes a list of package name, file and license """ - info=list() - # Extract DB tar.gz - commands.getoutput("mkdir -p " + archdb) - if not db_tar_file: - db_tar_file = repodir + db(repo_,arch_) - if isfile(db_tar_file): - try: - db_open_tar = tarfile.open(db_tar_file, 'r:gz') - except tarfile.ReadError: - printf("No valid db_file %s" % db_tar_file) - return(tuple()) - else: - printf("No db_file %s" % db_tar_file) - return(tuple()) - for file in db_open_tar.getmembers(): - db_open_tar.extract(file, archdb) - db_open_tar.close() - # Get info from file - for dir_ in glob(archdb + "/*"): - if isdir(dir_) and isfile(dir_ + "/desc"): - pkg_desc_file = open(dir_ + "/desc", "r") - desc = pkg_desc_file.read() - pkg_desc_file.close() - info.append(( get_from_desc(desc,"%NAME%"), - dir_.split("/")[-1], - get_from_desc(desc,"%LICENSE%") )) - if verbose_: printf(info) - commands.getoutput("rm -r %s/*" % archdb) - return tuple(info) + """ Makes a list of package name, file and license """ + info=list() + # Extract DB tar.gz + commands.getoutput("mkdir -p " + archdb) + if not db_tar_file: + db_tar_file = repodir + db(repo_,arch_) + if isfile(db_tar_file): + try: + db_open_tar = tarfile.open(db_tar_file, 'r:gz') + except tarfile.ReadError: + printf("No valid db_file %s" % db_tar_file) + return(tuple()) + else: + printf("No db_file %s" % db_tar_file) + return(tuple()) + for file in db_open_tar.getmembers(): + db_open_tar.extract(file, archdb) + db_open_tar.close() + # Get info from file + for dir_ in glob(archdb + "/*"): + if isdir(dir_) and isfile(dir_ + "/desc"): + pkg_desc_file = open(dir_ + "/desc", "r") + desc = pkg_desc_file.read() + pkg_desc_file.close() + info.append(( get_from_desc(desc,"%NAME%"), + dir_.split("/")[-1], + get_from_desc(desc,"%LICENSE%") )) + if verbose_: printf(info) + commands.getoutput("rm -r %s/*" % archdb) + return tuple(info) def make_pending(repo_,arch_,info_): - """ Si los paquetes no están en blacklist ni whitelist y la licencia contiene "custom" los agrega a pending""" - search = tuple( listado(blacklist) + listado (whitelist) ) - if verbose: printf("blaclist + whitelist= " + str(search) ) - lista_=list() - for (name,pkg_,license_) in info_: - if "custom" in license_: - if name not in search: - lista_.append( (name, license_ ) ) - elif not name: - printf( pkg_ + " package has no %NAME% attibute " ) - if verbose: printf( lista_ ) - a=open( pending + "-" + repo_ + ".txt", "w" ).write( - "\n".join([name + ":" + license_ for (name,license_) in lista_]) ) + """ Si los paquetes no están en blacklist ni whitelist y la licencia contiene "custom" los agrega a pending""" + search = tuple( listado(blacklist) + listado (whitelist) ) + if verbose: printf("blaclist + whitelist= " + str(search) ) + lista_=list() + for (name,pkg_,license_) in info_: + if "custom" in license_: + if name not in search: + lista_.append( (name, license_ ) ) + elif not name: + printf( pkg_ + " package has no %NAME% attibute " ) + if verbose: printf( lista_ ) + a=open( pending + "-" + repo_ + ".txt", "w" ).write( + "\n".join([name + ":" + license_ for (name,license_) in lista_]) ) def remove_from_blacklist(repo_,arch_,info_,blacklist_): - """ Check the blacklist and remove packages on the db""" - lista_=list() - pack_=list() - for (name_, pkg_, license_) in info_: - if name_ in blacklist_: - lista_.append(name_) - for p in packages(repo_,arch_,pkg_ + "*"): - pack_.append(p) - if lista_: - lista_=" ".join(lista_) - com_ = "repo-remove " + repodir + db(repo_,arch_) + " " + lista_ - printf(com_) - a = commands.getoutput(com_) - if verbose: printf(a) - if pack_: - pack_=" ".join(pack_) - com_="chmod a-r " + pack_ - printf(com_) - a=commands.getoutput(com_) - if verbose: printf(a) + """ Check the blacklist and remove packages on the db""" + lista_=list() + pack_=list() + for (name_, pkg_, license_) in info_: + if name_ in blacklist_: + lista_.append(name_) + for p in packages(repo_,arch_,pkg_ + "*"): + pack_.append(p) + if lista_: + lista_=" ".join(lista_) + com_ = "repo-remove " + repodir + db(repo_,arch_) + " " + lista_ + printf(com_) + a = commands.getoutput(com_) + if verbose: printf(a) + if pack_: + pack_=" ".join(pack_) + com_="chmod a-r " + pack_ + printf(com_) + a=commands.getoutput(com_) + if verbose: printf(a) def link(repo_,arch_,file_): - """ Makes a link in the repo for the package """ - cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ - a=commands.getoutput(cmd_) - if verbose: - printf(cmd_ + a) + """ Makes a link in the repo for the package """ + cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ + a=commands.getoutput(cmd_) + if verbose: + printf(cmd_ + a) def add_free_repo(verbose_=verbose): - for repo_ in repo_list: - for arch_ in arch_list: - lista_=list() - for file_ in glob(free_path + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"): - lista_.append(file_) - link(repo_,arch_,file_) - for dir_ in other: - for file_ in glob(free_path + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"): - lista_.append(file_) - link(repo_,arch_,file_) - if lista_: - lista_=" ".join(lista_) - if verbose: printf(lista_) - cmd_="repo-add " + repodir + db(repo_,arch_) + " " + lista_ - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose: printf(a) + cmd_=home + "/usr/bin/sync-free" + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose_: printf(a) + for repo_ in repo_list: + for arch_ in arch_list: + lista_=list() + for file_ in glob(freedir + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"): + lista_.append(file_) + # link(repo_,arch_,file_) + for dir_ in other: + for file_ in glob(freedir + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"): + lista_.append(file_) + # link(repo_,arch_,file_) + + printf(lista_) + + if lista_: + lista_=" ".join(lista_) + if verbose: printf(lista_) + cmd_="repo-add " + repodir + db(repo_,arch_) + " " + lista_ + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose: printf(a) def get_licenses(verbose_=verbose): - """ Extract the license from packages in repo_,arch_ and in pending_ file""" - cmd_=home + "/usr/bin/get_license.sh" - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose_: printf(a) + """ Extract the license from packages in repo_,arch_ and in pending_ file""" + cmd_=home + "/usr/bin/get_license.sh" + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose_: printf(a) def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdir=repodir, source=mirror+mirrorpath, blacklist_file=False): @@ -207,33 +214,33 @@ def run_rsync(command,debug=verbose): return commands.getoutput(command) if __name__ == "__main__": - from time import time - start_time = time() - def minute(): - return str(round((time() - start_time)/60, 1)) - - printf(" Cleaning %s folder " % (tmp) ) - commands.getoutput("rm -r %s/*" % tmp) - printf(" Syncing repo") - sync_all_repo(True) - - printf(" Updating databases and pending files lists: minute %s \n" % minute() ) - for repo in repo_list: - for arch in arch_list: - printf( "\n" + repo + "-" + arch + "\n" ) - printf( "Get info: minute %s " % minute() ) - info=get_info(repo,arch) - printf( "Make pending: minute %s" % minute() ) - make_pending(repo,arch,info) - printf( "Update DB: minute %s" % minute() ) - remove_from_blacklist( - repo, arch, info, tuple( listado(blacklist) + listado(pending + "-" + repo + ".txt") ) ) - - printf("Adding Parabola Packages: minute %s\n" % minute() ) - add_free_repo(True) - - printf("Extracting licenses in pending: minute %s" % minute() ) - get_licenses() - - printf("\n\nDelay: %s minutes \n" % minute()) - + from time import time + start_time = time() + def minute(): + return str(round((time() - start_time)/60, 1)) + + printf(" Cleaning %s folder " % (tmp) ) + commands.getoutput("rm -r %s/*" % tmp) + printf(" Syncing repo") + sync_all_repo(True) + + printf(" Updating databases and pending files lists: minute %s \n" % minute() ) + for repo in repo_list: + for arch in arch_list: + printf( "\n" + repo + "-" + arch + "\n" ) + printf( "Get info: minute %s " % minute() ) + info=get_info(repo,arch) + printf( "Make pending: minute %s" % minute() ) + make_pending(repo,arch,info) + printf( "Update DB: minute %s" % minute() ) + remove_from_blacklist( + repo, arch, info, tuple( listado(blacklist) + listado(pending + "-" + repo + ".txt") ) ) + + printf("Adding Parabola Packages: minute %s\n" % minute() ) + add_free_repo(True) + + printf("Extracting licenses in pending: minute %s" % minute() ) + get_licenses() + + printf("\n\nDelay: %s minutes \n" % minute()) + -- cgit v1.2.3-54-g00ecf From 9793aaf4e9a716f126e71a1db412cbd32f161593 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Thu, 3 Mar 2011 20:11:33 -0600 Subject: * repo-maintainer doesn't take out reading permission from pending files * Removed commented code --- pato2.py | 8 -------- 1 file changed, 8 deletions(-) (limited to 'pato2.py') diff --git a/pato2.py b/pato2.py index f5deb48..21dde16 100644 --- a/pato2.py +++ b/pato2.py @@ -128,12 +128,6 @@ def remove_from_blacklist(repo_,arch_,info_,blacklist_): printf(com_) a = commands.getoutput(com_) if verbose: printf(a) - if pack_: - pack_=" ".join(pack_) - com_="chmod a-r " + pack_ - printf(com_) - a=commands.getoutput(com_) - if verbose: printf(a) def link(repo_,arch_,file_): """ Makes a link in the repo for the package """ @@ -152,11 +146,9 @@ def add_free_repo(verbose_=verbose): lista_=list() for file_ in glob(freedir + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"): lista_.append(file_) - # link(repo_,arch_,file_) for dir_ in other: for file_ in glob(freedir + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"): lista_.append(file_) - # link(repo_,arch_,file_) printf(lista_) -- cgit v1.2.3-54-g00ecf From 8c9ff0c97d0802fadc96fe9eacaef7eaa2db5ad2 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sun, 6 Mar 2011 16:37:36 -0600 Subject: * Using os.path.join for joining path instead of str + str to avoid errors --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'pato2.py') diff --git a/pato2.py b/pato2.py index 21dde16..936e97a 100644 --- a/pato2.py +++ b/pato2.py @@ -137,7 +137,7 @@ def link(repo_,arch_,file_): printf(cmd_ + a) def add_free_repo(verbose_=verbose): - cmd_=home + "/usr/bin/sync-free" + cmd_=os.path.join(home,"/usr/bin/sync-free") printf(cmd_) a=commands.getoutput(cmd_) if verbose_: printf(a) -- cgit v1.2.3-54-g00ecf From e2385920015311376c1631a42a1ca8f1c45dae4f Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sun, 6 Mar 2011 18:08:25 -0600 Subject: * Fixed error that leaked non-free files to repo --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'pato2.py') diff --git a/pato2.py b/pato2.py index 936e97a..336c6fb 100644 --- a/pato2.py +++ b/pato2.py @@ -56,7 +56,7 @@ def sync_all_repo(debug=verbose): cmd=generate_rsync_command(rsync_list_command) rsout=run_rsync(cmd) pkgs=pkginfo_from_rsync_output(rsout) - generate_exclude_list_from_blacklist(pkgs,listado(blacklist)) + generate_exclude_list_from_blacklist(pkgs,listado(blacklist),debug=False) cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) a=run_rsync(cmd) if debug: printf(a) -- cgit v1.2.3-54-g00ecf From 125df5a815e0a327739e928cc765ffdcf4bd0aea Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sun, 6 Mar 2011 18:19:03 -0600 Subject: * Added function for cleanup dir --- pato2.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'pato2.py') diff --git a/pato2.py b/pato2.py index 336c6fb..03afebd 100644 --- a/pato2.py +++ b/pato2.py @@ -129,6 +129,12 @@ def remove_from_blacklist(repo_,arch_,info_,blacklist_): a = commands.getoutput(com_) if verbose: printf(a) +def cleanup_nonfree_in_dir(directory,blacklisted_names): + pkgs=pkginfo_from_files_in_dir(directory) + for package in pkgs: + if package["name"] in blacklisted_names: + os.remove(package["location"]) + def link(repo_,arch_,file_): """ Makes a link in the repo for the package """ cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ -- cgit v1.2.3-54-g00ecf From a41f5f44a8f812dfa2aacaf359bb51782d7a1633 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sun, 6 Mar 2011 19:02:14 -0600 Subject: * Fixed blacklist for rsync_command at sync_all_repo --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'pato2.py') diff --git a/pato2.py b/pato2.py index 03afebd..18ace85 100644 --- a/pato2.py +++ b/pato2.py @@ -57,7 +57,7 @@ def sync_all_repo(debug=verbose): rsout=run_rsync(cmd) pkgs=pkginfo_from_rsync_output(rsout) generate_exclude_list_from_blacklist(pkgs,listado(blacklist),debug=False) - cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) + cmd=generate_rsync_command(rsync_update_command,blacklist_file=rsync_blacklist) a=run_rsync(cmd) if debug: printf(a) -- cgit v1.2.3-54-g00ecf From 0889827f252c9f6c21c1d4f2afc704b9b303e083 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Thu, 10 Mar 2011 13:59:35 -0600 Subject: * Fixed issue 71: pending list ends with newline --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'pato2.py') diff --git a/pato2.py b/pato2.py index 18ace85..27a3962 100644 --- a/pato2.py +++ b/pato2.py @@ -111,7 +111,7 @@ def make_pending(repo_,arch_,info_): printf( pkg_ + " package has no %NAME% attibute " ) if verbose: printf( lista_ ) a=open( pending + "-" + repo_ + ".txt", "w" ).write( - "\n".join([name + ":" + license_ for (name,license_) in lista_]) ) + "\n".join([name + ":" + license_ for (name,license_) in lista_]) + "\n") def remove_from_blacklist(repo_,arch_,info_,blacklist_): """ Check the blacklist and remove packages on the db""" -- cgit v1.2.3-54-g00ecf From 47045b1934932b0695dd301a9c76b9dab1b03023 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 21 Mar 2011 17:30:40 -0600 Subject: * Changed sync_all_repo to adress bug83 --- config.py | 3 ++- filter.py | 3 ++- pato2.py | 6 +++++- test/core.db.tar.gz | Bin 1637 -> 1345 bytes test/test_filter.py | 23 +++++++++++++++++++++++ 5 files changed, 32 insertions(+), 3 deletions(-) (limited to 'pato2.py') diff --git a/config.py b/config.py index ada17f0..c218ddd 100644 --- a/config.py +++ b/config.py @@ -46,7 +46,8 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " -rsync_update_command="rsync -av --delete-after --delay-updates " +rsync_update_command="rsync -av --delay-updates --exclude='*.db.tar.{gz|xz}' " +rsync_post_command="rsync -av --delete " # Classes and Exceptions class NonValidFile(ValueError): pass diff --git a/filter.py b/filter.py index b6e8105..668822b 100644 --- a/filter.py +++ b/filter.py @@ -115,7 +115,8 @@ def pkginfo_from_files_in_dir(directory): package_list.append(pkginfo_from_filename(filename)) return tuple(package_list) - +def pkginfo_from_db(path_to_db): + """ """ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, exclude_file=rsync_blacklist, debug=verbose): diff --git a/pato2.py b/pato2.py index 27a3962..0d77d6b 100644 --- a/pato2.py +++ b/pato2.py @@ -59,7 +59,11 @@ def sync_all_repo(debug=verbose): generate_exclude_list_from_blacklist(pkgs,listado(blacklist),debug=False) cmd=generate_rsync_command(rsync_update_command,blacklist_file=rsync_blacklist) a=run_rsync(cmd) - if debug: printf(a) + cmd=generate_rsync_command(rsync_post_command,blacklist_file=rsync_blacklist) + b=run_rsync(cmd) + if debug: + printf(a) + printf(b) def get_from_desc(desc, var,db_tar_file=False): """ Get a var from desc file """ diff --git a/test/core.db.tar.gz b/test/core.db.tar.gz index a28ea64..5eb2081 100644 Binary files a/test/core.db.tar.gz and b/test/core.db.tar.gz differ diff --git a/test/test_filter.py b/test/test_filter.py index 5127b75..1906b87 100644 --- a/test/test_filter.py +++ b/test/test_filter.py @@ -163,7 +163,30 @@ class pkginfo_from_descKnownValues(unittest.TestCase): class pkginfo_from_db(unittest.TestCase): archdb = os.path.join("./workdir") + example_package_list=(Package(),Package(),Package()) + example_package_list[0].package_info={ "name" : "acl", + "version" : "2.2.49", + "release" : "2", + "arch" : "x86_64", + "license" : ("LGPL",), + "location": "acl-2.2.49-2-x86_64.pkg.tar.xz" + "depends" : ("attr>=2.4.41"),} + example_package_list[1].package_info={ "name" : "glibc", + "version" : "2.13", + "release" : "4", + "arch" : "x86_64", + "license" : ("GPL","LGPL"), + "location": "glibc-2.13-4-x86_64.pkg.tar.xz" + "depends" : ("linux-api-headers>=2.6.37","tzdata",),} + example_package_list[2].package_info={ "name" : "", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "" + "depends" : False,} if __name__ == "__main__": unittest.main() + -- cgit v1.2.3-54-g00ecf