From 49a0260b06f9d24c790631cb4e2a4f63fce3a8c3 Mon Sep 17 00:00:00 2001 From: Parabola Date: Fri, 17 Sep 2010 09:55:19 -0700 Subject: Upload version --- TODO | 0 __init__.py | 0 changelog | 2 + get_license.sh | 55 ++++++++++++++ pato2.py | 232 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ repo-state | 11 +++ 6 files changed, 300 insertions(+) create mode 100644 TODO create mode 100644 __init__.py create mode 100644 changelog create mode 100755 get_license.sh create mode 100644 pato2.py create mode 100755 repo-state diff --git a/TODO b/TODO new file mode 100644 index 0000000..e69de29 diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/changelog b/changelog new file mode 100644 index 0000000..0e89f2f --- /dev/null +++ b/changelog @@ -0,0 +1,2 @@ +repo-maintainer 1.0 + * Known working version diff --git a/get_license.sh b/get_license.sh new file mode 100755 index 0000000..a7241a1 --- /dev/null +++ b/get_license.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# -*- coding: utf-8 -*- + + # get_license.sh + # Copyright 2010 Joshua Ismael Haase Hernández + + # ---------- GNU General Public License 3 ---------- + + # This file is part of Parabola. + + # Parabola is free software: you can redistribute it and/or modify + # it under the terms of the GNU General Public License as published by + # the Free Software Foundation, either version 3 of the License, or + # (at your option) any later version. + + # Parabola is distributed in the hope that it will be useful, + # but WITHOUT ANY WARRANTY; without even the implied warranty of + # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + # GNU General Public License for more details. + + # You should have received a copy of the GNU General Public License + # along with Parabola. If not, see . + +docs="/home/parabolavnx/parabolagnulinux.org/docs" +repo="/home/parabolavnx/parabolagnulinux.org/repo" +dir="$docs/pending-licenses" + +echo "Cleaning $dir" +rm -rf $dir/* + +tempdir=$(mktemp -d) +cd $tempdir + +a=($(cut -d: -f1 $docs/pending*.txt)) +echo ${a[@]} + +for x in ${a[@]}; do + b=( $(ls $repo/*/os/*/$x*) ) + for y in ${b[@]}; do + echo "chmod +r $y" + chmod +r $y + echo "tar -xf $y usr/share/licenses" + bsdtar -xf $y usr/share/licenses + echo "chmod -r $y" + chmod -r $y + done +done + +mv usr/share/licenses/* $dir + +cd + +rm -rf $tempdir + +exit 0 \ No newline at end of file diff --git a/pato2.py b/pato2.py new file mode 100644 index 0000000..6d8a981 --- /dev/null +++ b/pato2.py @@ -0,0 +1,232 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +""" + parabola.py + Copyright 2009 Rafik Mas'ad + Copyright 2010 Joshua Ismael Haase Hernández + + ---------- GNU General Public License 3 ---------- + + This file is part of Parabola. + + Parabola is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Parabola is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Parabola. If not, see . + + +""" + +import tarfile, commands +from glob import glob +from user import home +from os.path import isdir, isfile, realpath +# ---------- Config Variables Start Here ---------- # + +time__ = commands.getoutput("date +%Y%m%d-%H:%M") + +# Mirror Parameters +mirror = "mirrors.kernel.org" +mirrorpath = "::mirrors/archlinux" + +# Directories and files +## Optionals +path = home + "/parabolagnulinux.org" +docs = path + "/docs" +logdir = path + "/log" +## Must be defined +logname= logdir + "/" + time__ + "-repo-maintainer.log" +repodir= path + "/repo" +tmp = home + "/tmp" +archdb = tmp + "/db" +emptydb= path + "/files/repo-empty.db.tar.gz" + +# Repo, arch, and other folders to use for repo +repo_list = ("core", "extra", "community","multilib") +dir_list = ("pool",) +arch_list = ("i686", "x86_64") +other = ("any",) + +# Output +output = True +verbose = False + +# Files +blacklist = docs + "/blacklist.txt" +whitelist = docs + "/whitelist.txt" +pending = docs + "/pending" +rsyncBlacklist = docs + "/rsyncBlacklist" + +# ---------- Config Variables End Here---------- # + +def printf(text,output_=output): + """Guarda el texto en la variable log y puede imprimir en pantalla.""" + log_file = open(logname, 'a') + log_file.write("\n" + str(text) + "\n") + log_file.close() + if output_: print (str(text) + "\n") + +def listado(filename_): + """Obtiene una lista de paquetes de un archivo.""" + archivo = open(filename_,"r") + lista = archivo.read().split("\n") + archivo.close() + return [pkg.split(":")[0] for pkg in lista if pkg] + +def db(repo_,arch_): + """Construye un nombre para sincronizar una base de datos.""" + return "/%s/os/%s/%s.db.tar.gz" % (repo_, arch_, repo_) + +def packages(repo_, arch_, expr="*"): + """ Get packages on a repo, arch folder """ + return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) ) + +def sync_all_repo(verbose_=verbose): + folders = ",".join(repo_list + dir_list) + cmd_ = "rsync -av --delete-after --delay-updates " + mirror + mirrorpath + "/{" + folders + "} " + repodir + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose_: printf(a) + +def get_from_desc(desc, var,db_tar_file=False): + """ Get a var from desc file """ + desc = desc.split("\n") + return desc[desc.index(var)+1] + +def get_info(repo_,arch_,db_tar_file=False,verbose_=verbose): + """ Makes a list of package name, file and license """ + info=list() + # Extract DB tar.gz + commands.getoutput("mkdir -p " + archdb) + if not db_tar_file: + db_tar_file = repodir + db(repo_,arch_) + if isfile(db_tar_file): + db_open_tar = tarfile.open(db_tar_file, 'r:gz') + else: + printf("No db_file %s" % db_tar_file) + return(tuple()) + for file in db_open_tar.getmembers(): + db_open_tar.extract(file, archdb) + db_open_tar.close() + # Get info from file + for dir_ in glob(archdb + "/*"): + if isdir(dir_) and isfile(dir_ + "/desc"): + pkg_desc_file = open(dir_ + "/desc", "r") + desc = pkg_desc_file.read() + pkg_desc_file.close() + info.append(( get_from_desc(desc,"%NAME%"), + dir_.split("/")[-1], + get_from_desc(desc,"%LICENSE%") )) + if verbose_: printf(info) + commands.getoutput("rm -r %s/*" % archdb) + return tuple(info) + +def make_pending(repo_,arch_,info_): + """ Si los paquetes no están en blacklist ni whitelist y la licencia contiene "custom" los agrega a pending""" + search = tuple( listado(blacklist) + listado (whitelist) ) + if verbose: printf("blaclist + whitelist= " + str(search) ) + lista_=list() + for (name,pkg_,license_) in info_: + if "custom" in license_: + if name not in search: + lista_.append( (name, license_ ) ) + elif not name: + printf( pkg_ + " package has no %NAME% attibute " ) + if verbose: printf( lista_ ) + a=open( pending + "-" + repo_ + ".txt", "w" ).write( + "\n".join([name + ":" + license_ for (name,license_) in lista_]) ) + +def remove_from_blacklist(repo_,arch_,info_,blacklist_): + """ Check the blacklist and remove packages on the db""" + lista_=list() + pack_=list() + for (name_, pkg_, license_) in info_: + if name_ in blacklist_: + lista_.append(name_) + for p in packages(repo_,arch_,pkg_ + "*"): + pack_.append(p) + if lista_: + lista_=" ".join(lista_) + com_ = "repo-remove " + repodir + db(repo_,arch_) + " " + lista_ + printf(com_) + a = commands.getoutput(com_) + if verbose: printf(a) + if pack_: + pack_=" ".join(pack_) + com_="chmod a-r " + pack_ + printf(com_) + a=commands.getoutput(com_) + if verbose: printf(a) + +def link(repo_,arch_,file_): + """ Makes a link in the repo for the package """ + cmd_="ln -sf " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ + a=commands.getoutput(cmd_) + if verbose: + printf(cmd_ + a) + +def add_free_repo(verbose_=verbose): + for repo_ in repo_list: + for arch_ in arch_list: + lista_=list() + for file_ in glob(repodir + "/free/" + repo_ + "/os/" + arch_ + "/*"): + lista_.append(file_) + link(repo_,arch_,file_) + for dir_ in other: + for file_ in glob(repodir + "/free/" + repo_ + "/os/" + dir_ + "/*"): + lista_.append(file_) + link(repo_,arch_,file_) + if lista_: + lista_=" ".join(lista_) + if verbose: printf(lista_) + cmd_="repo-add " + repodir + db(repo_,arch_) + " " + lista_ + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose: printf(a) + +def get_licenses(verbose_=verbose): + """ Extract the license from packages in repo_,arch_ and in pending_ file""" + cmd_=home + "/usr/bin/get_license.sh" + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose_: printf(a) + +if __name__ == "__main__": + from time import time + start_time = time() + def minute(): + return str(round((time() - start_time)/60, 1)) + + printf(" Cleaning %s folder " % (tmp) ) + commands.getoutput("rm -r %s/*" % tmp) + printf(" Syncing repo") + sync_all_repo(True) + + printf(" Updating databases and pending files lists: minute %s \n" % minute() ) + for repo in repo_list: + for arch in arch_list: + printf( "\n" + repo + "-" + arch + "\n" ) + printf( "Get info: minute %s " % minute() ) + info=get_info(repo,arch) + printf( "Make pending: minute %s" % minute() ) + make_pending(repo,arch,info) + printf( "Update DB: minute %s" % minute() ) + remove_from_blacklist( + repo, arch, info, tuple( listado(blacklist) + listado(pending + "-" + repo + ".txt") ) ) + + printf("Adding Parabola Packages: minute %s\n" % minute() ) + add_free_repo(True) + + printf("Extracting licenses in pending: minute %s" % minute() ) + get_licenses() + + printf("\n\nDelay: %s minutes \n" % minute()) + diff --git a/repo-state b/repo-state new file mode 100755 index 0000000..319db83 --- /dev/null +++ b/repo-state @@ -0,0 +1,11 @@ +#!/bin/sh + +a=$(date +%Y%m%d+%H%M) +mkdir -p $a +cd $a + +. /home/joshpar/programas/arch2parabola/repo-list-diff +# scp parabolavnx@parabolagnulinux.org:~/tmp/rsyncBlacklist.txt ./ + +cd .. +exit 0 \ No newline at end of file -- cgit v1.2.3-54-g00ecf From 847679d7a5e594efce5f8df73ca6751fab9ce591 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Fri, 17 Sep 2010 12:34:02 -0500 Subject: Added TODO list --- TODO | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/TODO b/TODO index e69de29..11ef51a 100644 --- a/TODO +++ b/TODO @@ -0,0 +1,6 @@ +* Test Suite + + - Review all repo efectively + - Remove all blacklisted packages + - Get pending list right + - Extract licenses all right \ No newline at end of file -- cgit v1.2.3-54-g00ecf From c6787b4d56fc585aca5b5f5422aea0f2c85a5db1 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 28 Sep 2010 21:21:05 -0500 Subject: Added sources to the repo syncing command --- .gitignore | 1 + pato2.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e4e5f6c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*~ \ No newline at end of file diff --git a/pato2.py b/pato2.py index 6d8a981..df81b73 100644 --- a/pato2.py +++ b/pato2.py @@ -51,7 +51,7 @@ emptydb= path + "/files/repo-empty.db.tar.gz" # Repo, arch, and other folders to use for repo repo_list = ("core", "extra", "community","multilib") -dir_list = ("pool",) +dir_list = ("pool","sources") arch_list = ("i686", "x86_64") other = ("any",) -- cgit v1.2.3-54-g00ecf From 6c1729bf9d94407b36ac5717c254dacf8dffef3b Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 12 Oct 2010 11:51:43 -0500 Subject: If it can't open db_tar_file, print a warning and continue. --- pato2.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pato2.py b/pato2.py index df81b73..58e03f8 100644 --- a/pato2.py +++ b/pato2.py @@ -109,7 +109,11 @@ def get_info(repo_,arch_,db_tar_file=False,verbose_=verbose): if not db_tar_file: db_tar_file = repodir + db(repo_,arch_) if isfile(db_tar_file): - db_open_tar = tarfile.open(db_tar_file, 'r:gz') + try: + db_open_tar = tarfile.open(db_tar_file, 'r:gz') + except tarfile.ReadError: + printf("No valid db_file %s" % db_tar_file) + return(tuple()) else: printf("No db_file %s" % db_tar_file) return(tuple()) -- cgit v1.2.3-54-g00ecf From 473de5be358e3bc1c0c490622e9b2c47d2bfbdf5 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 12 Oct 2010 12:33:33 -0500 Subject: Deleted unused variables --- pato2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pato2.py b/pato2.py index 58e03f8..78d8c55 100644 --- a/pato2.py +++ b/pato2.py @@ -47,7 +47,6 @@ logname= logdir + "/" + time__ + "-repo-maintainer.log" repodir= path + "/repo" tmp = home + "/tmp" archdb = tmp + "/db" -emptydb= path + "/files/repo-empty.db.tar.gz" # Repo, arch, and other folders to use for repo repo_list = ("core", "extra", "community","multilib") -- cgit v1.2.3-54-g00ecf From b8619985289cde1591aa40c7df2c948fdbb8c30a Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 31 Jan 2011 09:15:56 -0800 Subject: changed mirror --- pato2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pato2.py b/pato2.py index 78d8c55..cf40809 100644 --- a/pato2.py +++ b/pato2.py @@ -34,7 +34,7 @@ from os.path import isdir, isfile, realpath time__ = commands.getoutput("date +%Y%m%d-%H:%M") # Mirror Parameters -mirror = "mirrors.kernel.org" +mirror = "mirrors.eu.kernel.org" mirrorpath = "::mirrors/archlinux" # Directories and files @@ -49,7 +49,7 @@ tmp = home + "/tmp" archdb = tmp + "/db" # Repo, arch, and other folders to use for repo -repo_list = ("core", "extra", "community","multilib") +repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") dir_list = ("pool","sources") arch_list = ("i686", "x86_64") other = ("any",) @@ -90,7 +90,7 @@ def packages(repo_, arch_, expr="*"): def sync_all_repo(verbose_=verbose): folders = ",".join(repo_list + dir_list) - cmd_ = "rsync -av --delete-after --delay-updates " + mirror + mirrorpath + "/{" + folders + "} " + repodir + cmd_ = "rsync -av --progress --delete-after --delay-updates " + mirror + mirrorpath + "/{" + folders + "} " + repodir printf(cmd_) a=commands.getoutput(cmd_) if verbose_: printf(a) -- cgit v1.2.3-54-g00ecf From a5872f3b259b97125089866169d06aa9a4d78e23 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 7 Feb 2011 12:30:30 -0600 Subject: moved config out from pato2 --- config.py | 32 ++++++++++++++++++++++++++++++++ pato2.py | 37 +------------------------------------ 2 files changed, 33 insertions(+), 36 deletions(-) create mode 100644 config.py diff --git a/config.py b/config.py new file mode 100644 index 0000000..e2fc350 --- /dev/null +++ b/config.py @@ -0,0 +1,32 @@ +time__ = commands.getoutput("date +%Y%m%d-%H:%M") + +# Mirror Parameters +mirror = "mirrors.eu.kernel.org" +mirrorpath = "::mirrors/archlinux" + +# Directories and files +## Optionals +path = home + "/parabolagnulinux.org" +docs = path + "/docs" +logdir = path + "/log" +## Must be defined +logname= logdir + "/" + time__ + "-repo-maintainer.log" +repodir= path + "/repo" +tmp = home + "/tmp" +archdb = tmp + "/db" + +# Repo, arch, and other folders to use for repo +repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") +dir_list = ("pool","sources") +arch_list = ("i686", "x86_64") +other = ("any",) + +# Output +output = True +verbose = False + +# Files +blacklist = docs + "/blacklist.txt" +whitelist = docs + "/whitelist.txt" +pending = docs + "/pending" +rsyncBlacklist = docs + "/rsyncBlacklist" diff --git a/pato2.py b/pato2.py index cf40809..3ed57b2 100644 --- a/pato2.py +++ b/pato2.py @@ -24,47 +24,12 @@ """ +from repo_maintainer.config import * import tarfile, commands from glob import glob from user import home from os.path import isdir, isfile, realpath -# ---------- Config Variables Start Here ---------- # - -time__ = commands.getoutput("date +%Y%m%d-%H:%M") - -# Mirror Parameters -mirror = "mirrors.eu.kernel.org" -mirrorpath = "::mirrors/archlinux" - -# Directories and files -## Optionals -path = home + "/parabolagnulinux.org" -docs = path + "/docs" -logdir = path + "/log" -## Must be defined -logname= logdir + "/" + time__ + "-repo-maintainer.log" -repodir= path + "/repo" -tmp = home + "/tmp" -archdb = tmp + "/db" - -# Repo, arch, and other folders to use for repo -repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") -dir_list = ("pool","sources") -arch_list = ("i686", "x86_64") -other = ("any",) - -# Output -output = True -verbose = False - -# Files -blacklist = docs + "/blacklist.txt" -whitelist = docs + "/whitelist.txt" -pending = docs + "/pending" -rsyncBlacklist = docs + "/rsyncBlacklist" - -# ---------- Config Variables End Here---------- # def printf(text,output_=output): """Guarda el texto en la variable log y puede imprimir en pantalla.""" -- cgit v1.2.3-54-g00ecf From 44dc101393495d876a4fab40689c1d6c8b307c20 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 7 Feb 2011 12:32:16 -0600 Subject: Added headers to config.py --- config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config.py b/config.py index e2fc350..df837d4 100644 --- a/config.py +++ b/config.py @@ -1,3 +1,6 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + time__ = commands.getoutput("date +%Y%m%d-%H:%M") # Mirror Parameters -- cgit v1.2.3-54-g00ecf From 937bf24bacdd29629b5efd2c949c0a6e4d17b163 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 7 Feb 2011 17:38:33 -0600 Subject: Making hardlinks instead of symbolic --- config.py | 2 ++ pato2.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index df837d4..8a55cf7 100644 --- a/config.py +++ b/config.py @@ -8,10 +8,12 @@ mirror = "mirrors.eu.kernel.org" mirrorpath = "::mirrors/archlinux" # Directories and files + ## Optionals path = home + "/parabolagnulinux.org" docs = path + "/docs" logdir = path + "/log" + ## Must be defined logname= logdir + "/" + time__ + "-repo-maintainer.log" repodir= path + "/repo" diff --git a/pato2.py b/pato2.py index 3ed57b2..cd08ca0 100644 --- a/pato2.py +++ b/pato2.py @@ -24,7 +24,7 @@ """ -from repo_maintainer.config import * +from repm.config import * import tarfile, commands from glob import glob @@ -136,7 +136,7 @@ def remove_from_blacklist(repo_,arch_,info_,blacklist_): def link(repo_,arch_,file_): """ Makes a link in the repo for the package """ - cmd_="ln -sf " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ + cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ a=commands.getoutput(cmd_) if verbose: printf(cmd_ + a) -- cgit v1.2.3-54-g00ecf From 100970b35ed77a8f617a7071aba2618e038923ba Mon Sep 17 00:00:00 2001 From: Joshua Haase Date: Mon, 7 Feb 2011 16:51:06 -0800 Subject: Use free repo on new location --- pato2.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pato2.py b/pato2.py index cf40809..9cbd3cb 100644 --- a/pato2.py +++ b/pato2.py @@ -40,6 +40,7 @@ mirrorpath = "::mirrors/archlinux" # Directories and files ## Optionals path = home + "/parabolagnulinux.org" +free_path = path + "/free/" docs = path + "/docs" logdir = path + "/log" ## Must be defined @@ -56,7 +57,7 @@ other = ("any",) # Output output = True -verbose = False +verbose = True # Files blacklist = docs + "/blacklist.txt" @@ -180,11 +181,11 @@ def add_free_repo(verbose_=verbose): for repo_ in repo_list: for arch_ in arch_list: lista_=list() - for file_ in glob(repodir + "/free/" + repo_ + "/os/" + arch_ + "/*"): + for file_ in glob(free_path + repo_ + "/os/" + arch_ + "/*"): lista_.append(file_) link(repo_,arch_,file_) for dir_ in other: - for file_ in glob(repodir + "/free/" + repo_ + "/os/" + dir_ + "/*"): + for file_ in glob(free_path + repo_ + "/os/" + dir_ + "/*"): lista_.append(file_) link(repo_,arch_,file_) if lista_: -- cgit v1.2.3-54-g00ecf From 8c4813edd80c08e03c70ada2cb46a3597e493aaf Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 7 Feb 2011 16:54:47 -0800 Subject: 'Configuration out of pato' --- pato2.py | 40 ++-------------------------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/pato2.py b/pato2.py index 9cbd3cb..dd32403 100644 --- a/pato2.py +++ b/pato2.py @@ -24,48 +24,12 @@ """ +from repm.config import * import tarfile, commands from glob import glob from user import home from os.path import isdir, isfile, realpath -# ---------- Config Variables Start Here ---------- # - -time__ = commands.getoutput("date +%Y%m%d-%H:%M") - -# Mirror Parameters -mirror = "mirrors.eu.kernel.org" -mirrorpath = "::mirrors/archlinux" - -# Directories and files -## Optionals -path = home + "/parabolagnulinux.org" -free_path = path + "/free/" -docs = path + "/docs" -logdir = path + "/log" -## Must be defined -logname= logdir + "/" + time__ + "-repo-maintainer.log" -repodir= path + "/repo" -tmp = home + "/tmp" -archdb = tmp + "/db" - -# Repo, arch, and other folders to use for repo -repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") -dir_list = ("pool","sources") -arch_list = ("i686", "x86_64") -other = ("any",) - -# Output -output = True -verbose = True - -# Files -blacklist = docs + "/blacklist.txt" -whitelist = docs + "/whitelist.txt" -pending = docs + "/pending" -rsyncBlacklist = docs + "/rsyncBlacklist" - -# ---------- Config Variables End Here---------- # def printf(text,output_=output): """Guarda el texto en la variable log y puede imprimir en pantalla.""" @@ -172,7 +136,7 @@ def remove_from_blacklist(repo_,arch_,info_,blacklist_): def link(repo_,arch_,file_): """ Makes a link in the repo for the package """ - cmd_="ln -sf " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ + cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ a=commands.getoutput(cmd_) if verbose: printf(cmd_ + a) -- cgit v1.2.3-54-g00ecf From 317c0b412642b5268e8408f9d87e19261370cd32 Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 7 Feb 2011 18:25:37 -0800 Subject: fixed errors --- config.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 config.py diff --git a/config.py b/config.py new file mode 100644 index 0000000..9889c3f --- /dev/null +++ b/config.py @@ -0,0 +1,41 @@ + +#!/usr/bin/python +# -*- coding: utf-8 -*- +from user import home +import commands + +time__ = commands.getoutput("date +%Y%m%d-%H:%M") + +# Mirror Parameters +mirror = "mirrors.eu.kernel.org" +mirrorpath = "::mirrors/archlinux" + +# Directories and files + +## Optionals +path = home + "/parabolagnulinux.org" +docs = path + "/docs" +logdir = path + "/log" + +## Must be defined +logname= logdir + "/" + time__ + "-repo-maintainer.log" +repodir= path + "/repo" +tmp = home + "/tmp" +archdb = tmp + "/db" +free_path= path + "/free/" + +# Repo, arch, and other folders to use for repo +repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") +dir_list = ("pool","sources") +arch_list = ("i686", "x86_64") +other = ("any",) + +# Output +output = True +verbose = False + +# Files +blacklist = docs + "/blacklist.txt" +whitelist = docs + "/whitelist.txt" +pending = docs + "/pending" +rsyncBlacklist = docs + "/rsyncBlacklist" -- cgit v1.2.3-54-g00ecf From f837844242142599d1dee84309468624f6dd2048 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 8 Feb 2011 00:31:56 -0600 Subject: Added classes and exceptions --- config.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/config.py b/config.py index 223257d..6dcd348 100644 --- a/config.py +++ b/config.py @@ -39,3 +39,27 @@ blacklist = docs + "/blacklist.txt" whitelist = docs + "/whitelist.txt" pending = docs + "/pending" rsyncBlacklist = docs + "/rsyncBlacklist" + +# Classes and Exceptions +class NonValidFile(ValueError): pass +class NonValidDir(ValueError): pass +class NonValidCommand(ValueError): pass + +class Package: + """ An object that has information about a package. """ + package_info={ "name" : False, + "version" : False, + "arch" : False, + "license" : False, + "location": False} + + def __setitem__(self, key, item): + return self.package_info.__setitem__(key, item) + + def __getitem__(self, key): + return self.package_info.__getitem__(key) + + def __unicode__(self): + return str(self.package_info) + + -- cgit v1.2.3-54-g00ecf From 039c18a9ab6259a13285a22d84390bba2c35642e Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 8 Feb 2011 00:37:29 -0600 Subject: Removed extra import commands: * commands and home were loaded on repm.config * realpath is not used --- pato2.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pato2.py b/pato2.py index dd32403..236f2ea 100644 --- a/pato2.py +++ b/pato2.py @@ -26,10 +26,9 @@ """ from repm.config import * -import tarfile, commands +import tarfile from glob import glob -from user import home -from os.path import isdir, isfile, realpath +from os.path import isdir, isfile def printf(text,output_=output): """Guarda el texto en la variable log y puede imprimir en pantalla.""" -- cgit v1.2.3-54-g00ecf From 80402467cf38f32373ac14d992d2daee4736ad82 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 8 Feb 2011 00:42:33 -0600 Subject: Added primitive testcase --- test/directory_list | 2 ++ test/link_list | 2 ++ test/package_list | 3 +++ test/test1.py | 22 ++++++++++++++++++++++ 4 files changed, 29 insertions(+) create mode 100644 test/directory_list create mode 100644 test/link_list create mode 100644 test/package_list create mode 100644 test/test1.py diff --git a/test/directory_list b/test/directory_list new file mode 100644 index 0000000..c0f7f47 --- /dev/null +++ b/test/directory_list @@ -0,0 +1,2 @@ +drwxrwxr-x 15 2010/09/11 11:28:50 community-staging +drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os \ No newline at end of file diff --git a/test/link_list b/test/link_list new file mode 100644 index 0000000..d015364 --- /dev/null +++ b/test/link_list @@ -0,0 +1,2 @@ +lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz +lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz \ No newline at end of file diff --git a/test/package_list b/test/package_list new file mode 100644 index 0000000..6ffa1de --- /dev/null +++ b/test/package_list @@ -0,0 +1,3 @@ +-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz +-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz +-rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz \ No newline at end of file diff --git a/test/test1.py b/test/test1.py new file mode 100644 index 0000000..8e5cd1e --- /dev/null +++ b/test/test1.py @@ -0,0 +1,22 @@ +""" """ + +__author__ = "Joshua Ismael Haase Hernández " +__version__ = "$Revision: 1.1 $" +__date__ = "$Date: 2011/02/08 $" +__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" +__license__ = "GPL3+" + +import repm.filter +import unittest +import commands + +class KnownValues(unittest.TestCase): + + def testDirectoryOutput(self): + """get_file_list_from_rsync_output should ignore directories""" + output=commands.getoutput("cat ./directory_list") + result=get_file_list_from_rsync_output(output) + self.assertEqual(tuple(), result) + +if __name__ == "__main__": + unittest.main() -- cgit v1.2.3-54-g00ecf From 0bdbdeaf186a9ab73f596b60748b6ae8a25ebaeb Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 8 Feb 2011 01:18:18 -0600 Subject: Added some testcases. --- test/test1.py | 48 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/test/test1.py b/test/test1.py index 8e5cd1e..a42e9a5 100644 --- a/test/test1.py +++ b/test/test1.py @@ -6,17 +6,59 @@ __date__ = "$Date: 2011/02/08 $" __copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" __license__ = "GPL3+" -import repm.filter +from repm.config import * +from repm.filter import * import unittest -import commands class KnownValues(unittest.TestCase): + directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", + "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os") + # (output, name, version, arch) + link_list=( + ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686"), + ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686"), + ) + package_list=( + ("-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz", + "abuse","0.7.1","x86_64"), + ("-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz", + "acetoneiso2","2.3","i686"), + ("-rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz", + "acetoneiso2","2.3","x86_64") + ) + + def generate_results(example_tuple): + a=list() + for output, name, version, arch in example_tuple: + pkg=Packages() + pkg["name"] = name + pkg["version"] = version + pkg["arch"] = arch + a.append(pkg) + return tuple(a) def testDirectoryOutput(self): """get_file_list_from_rsync_output should ignore directories""" - output=commands.getoutput("cat ./directory_list") + rsync_out="\n".join(directory_list) result=get_file_list_from_rsync_output(output) self.assertEqual(tuple(), result) + def testLinkOutput(self): + """get_file_list_from_rsync_output should make a Package Object + from links """ + correct_result=generate_results(link_list) + rsync_out="\n".join([a for a,b,c,d in link_list]) + result=get_file_list_from_rsync_output(rsync_out) + self.assertEqual(correct_result, result) + + + def testPackageOutput(self): + """get_file_list_from_rsync_output should make a Package Object + from links """ + correct_result=generate_results(package_list) + rsync_out="\n".join([a for a,b,c,d in package_list]) + result=get_file_list_from_rsync_output(rsync_out) + self.assertEqual(correct_result, result) + if __name__ == "__main__": unittest.main() -- cgit v1.2.3-54-g00ecf From 3633ea3c4ec93436b2344dba79b06ff00c7ae528 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 8 Feb 2011 01:30:35 -0600 Subject: Corrected first testcases. --- test/test1.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/test1.py b/test/test1.py index a42e9a5..495ba05 100644 --- a/test/test1.py +++ b/test/test1.py @@ -1,3 +1,4 @@ +# -*- encoding: utf-8 -*- """ """ __author__ = "Joshua Ismael Haase Hernández " @@ -27,10 +28,10 @@ class KnownValues(unittest.TestCase): "acetoneiso2","2.3","x86_64") ) - def generate_results(example_tuple): + def generate_results(self, example_tuple): a=list() for output, name, version, arch in example_tuple: - pkg=Packages() + pkg=Package() pkg["name"] = name pkg["version"] = version pkg["arch"] = arch @@ -39,24 +40,23 @@ class KnownValues(unittest.TestCase): def testDirectoryOutput(self): """get_file_list_from_rsync_output should ignore directories""" - rsync_out="\n".join(directory_list) + rsync_out="\n".join(self.directory_list) result=get_file_list_from_rsync_output(output) self.assertEqual(tuple(), result) def testLinkOutput(self): """get_file_list_from_rsync_output should make a Package Object from links """ - correct_result=generate_results(link_list) - rsync_out="\n".join([a for a,b,c,d in link_list]) + correct_result=self.generate_results(self.link_list) + rsync_out="\n".join([a for a,b,c,d in self.link_list]) result=get_file_list_from_rsync_output(rsync_out) self.assertEqual(correct_result, result) - def testPackageOutput(self): """get_file_list_from_rsync_output should make a Package Object from links """ - correct_result=generate_results(package_list) - rsync_out="\n".join([a for a,b,c,d in package_list]) + correct_result=self.generate_results(self.package_list) + rsync_out="\n".join([a for a,b,c,d in self.package_list]) result=get_file_list_from_rsync_output(rsync_out) self.assertEqual(correct_result, result) -- cgit v1.2.3-54-g00ecf From 61f5ecfa6b57f31c14430a7de5099640fcbe3525 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 8 Feb 2011 03:13:39 -0600 Subject: * Updated Package class * Corrected test1.py -> for get_package_list_from_rsync_output --- config.py | 1 + test/test1.py | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/config.py b/config.py index 6dcd348..a7087c4 100644 --- a/config.py +++ b/config.py @@ -49,6 +49,7 @@ class Package: """ An object that has information about a package. """ package_info={ "name" : False, "version" : False, + "release" : False, "arch" : False, "license" : False, "location": False} diff --git a/test/test1.py b/test/test1.py index 495ba05..db87f51 100644 --- a/test/test1.py +++ b/test/test1.py @@ -14,41 +14,43 @@ import unittest class KnownValues(unittest.TestCase): directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os") - # (output, name, version, arch) + # (output, name, version, arch, release, location) link_list=( - ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686"), - ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686"), + ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686", "2", "community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz"), + ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686", "2", "community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz"), ) package_list=( ("-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz", - "abuse","0.7.1","x86_64"), + "abuse","0.7.1","x86_64","1","pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz"), ("-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz", - "acetoneiso2","2.3","i686"), + "acetoneiso2","2.3","i686", "2", "pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz"), ("-rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz", - "acetoneiso2","2.3","x86_64") + "acetoneiso2","2.3","x86_64","2","pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz") ) def generate_results(self, example_tuple): a=list() - for output, name, version, arch in example_tuple: + for output, name, version, arch, release, location in example_tuple: pkg=Package() pkg["name"] = name pkg["version"] = version pkg["arch"] = arch + pkg["release"] = release + pkg["location"] = location a.append(pkg) return tuple(a) def testDirectoryOutput(self): """get_file_list_from_rsync_output should ignore directories""" rsync_out="\n".join(self.directory_list) - result=get_file_list_from_rsync_output(output) + result=get_file_list_from_rsync_output(rsync_out) self.assertEqual(tuple(), result) def testLinkOutput(self): """get_file_list_from_rsync_output should make a Package Object from links """ correct_result=self.generate_results(self.link_list) - rsync_out="\n".join([a for a,b,c,d in self.link_list]) + rsync_out="\n".join([a for a,b,c,d,e,f in self.link_list]) result=get_file_list_from_rsync_output(rsync_out) self.assertEqual(correct_result, result) @@ -56,7 +58,7 @@ class KnownValues(unittest.TestCase): """get_file_list_from_rsync_output should make a Package Object from links """ correct_result=self.generate_results(self.package_list) - rsync_out="\n".join([a for a,b,c,d in self.package_list]) + rsync_out="\n".join([a for a,b,c,d,e,f in self.package_list]) result=get_file_list_from_rsync_output(rsync_out) self.assertEqual(correct_result, result) -- cgit v1.2.3-54-g00ecf From 8fcf2f7f622de21b580ee808de53cffcc9200639 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 8 Feb 2011 03:41:12 -0600 Subject: Better testcases --- test/test1.py | 60 +++++++++++++++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/test/test1.py b/test/test1.py index db87f51..4ec72d3 100644 --- a/test/test1.py +++ b/test/test1.py @@ -14,12 +14,10 @@ import unittest class KnownValues(unittest.TestCase): directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os") - # (output, name, version, arch, release, location) - link_list=( + # (rsync_out, name, version, arch, release, location) + examples=( ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686", "2", "community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz"), ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686", "2", "community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz"), - ) - package_list=( ("-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz", "abuse","0.7.1","x86_64","1","pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz"), ("-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz", @@ -29,16 +27,8 @@ class KnownValues(unittest.TestCase): ) def generate_results(self, example_tuple): - a=list() - for output, name, version, arch, release, location in example_tuple: - pkg=Package() - pkg["name"] = name - pkg["version"] = version - pkg["arch"] = arch - pkg["release"] = release - pkg["location"] = location - a.append(pkg) - return tuple(a) + rsync_out="\n".join([a for a,b,c,d,e,f in example_tuple]) + return get_file_list_from_rsync_output(rsync_out) def testDirectoryOutput(self): """get_file_list_from_rsync_output should ignore directories""" @@ -46,21 +36,35 @@ class KnownValues(unittest.TestCase): result=get_file_list_from_rsync_output(rsync_out) self.assertEqual(tuple(), result) - def testLinkOutput(self): - """get_file_list_from_rsync_output should make a Package Object - from links """ - correct_result=self.generate_results(self.link_list) - rsync_out="\n".join([a for a,b,c,d,e,f in self.link_list]) - result=get_file_list_from_rsync_output(rsync_out) - self.assertEqual(correct_result, result) + def testNames(self): + results=self.generate_results(self.examples) + var =[name for rsync_out, name, version, arch, release, location in self.examples] + for i in range(len(results)): + self.assertEqual(results[i]["name"], var[i]) - def testPackageOutput(self): - """get_file_list_from_rsync_output should make a Package Object - from links """ - correct_result=self.generate_results(self.package_list) - rsync_out="\n".join([a for a,b,c,d,e,f in self.package_list]) - result=get_file_list_from_rsync_output(rsync_out) - self.assertEqual(correct_result, result) + def testVersions(self): + results=self.generate_results(self.examples) + var = [version for rsync_out, name, version, arch, release, location in self.examples] + for i in range(len(results)): + self.assertEqual(results[i]["name"], var[i]) + + def testArchs(self): + results=self.generate_results(self.examples) + var = [arch for rsync_out, name, version, arch, release, location in self.examples] + for i in range(len(results)): + self.assertEqual(results[i]["name"], var[i]) + + def testReleases(self): + results=self.generate_results(self.examples) + var = [release for rsync_out, name, version, arch, release, location in self.examples] + for i in range(len(results)): + self.assertEqual(results[i]["name"], var[i]) + + def testLocations(self): + results=self.generate_results(self.examples) + var = [location for rsync_out, name, version, arch, release, location in self.examples] + for i in range(len(results)): + self.assertEqual(results[i]["name"], var[i]) if __name__ == "__main__": unittest.main() -- cgit v1.2.3-54-g00ecf From 748ef10d89e5898c1ebffb08165f18cd3829119e Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 8 Feb 2011 03:43:07 -0600 Subject: Fixed stupid error. --- test/test1.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test1.py b/test/test1.py index 4ec72d3..ea488e9 100644 --- a/test/test1.py +++ b/test/test1.py @@ -46,25 +46,25 @@ class KnownValues(unittest.TestCase): results=self.generate_results(self.examples) var = [version for rsync_out, name, version, arch, release, location in self.examples] for i in range(len(results)): - self.assertEqual(results[i]["name"], var[i]) + self.assertEqual(results[i]["version"], var[i]) def testArchs(self): results=self.generate_results(self.examples) var = [arch for rsync_out, name, version, arch, release, location in self.examples] for i in range(len(results)): - self.assertEqual(results[i]["name"], var[i]) + self.assertEqual(results[i]["arch"], var[i]) def testReleases(self): results=self.generate_results(self.examples) var = [release for rsync_out, name, version, arch, release, location in self.examples] for i in range(len(results)): - self.assertEqual(results[i]["name"], var[i]) + self.assertEqual(results[i]["release"], var[i]) def testLocations(self): results=self.generate_results(self.examples) var = [location for rsync_out, name, version, arch, release, location in self.examples] for i in range(len(results)): - self.assertEqual(results[i]["name"], var[i]) + self.assertEqual(results[i]["location"], var[i]) if __name__ == "__main__": unittest.main() -- cgit v1.2.3-54-g00ecf From df79a07c092880259a851d4ebf9febe9a8e19880 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 8 Feb 2011 10:47:55 -0600 Subject: Fixed test1.py --- test/test1.py | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/test/test1.py b/test/test1.py index ea488e9..61b0651 100644 --- a/test/test1.py +++ b/test/test1.py @@ -26,9 +26,9 @@ class KnownValues(unittest.TestCase): "acetoneiso2","2.3","x86_64","2","pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz") ) - def generate_results(self, example_tuple): - rsync_out="\n".join([a for a,b,c,d,e,f in example_tuple]) - return get_file_list_from_rsync_output(rsync_out) + def generate_results(self, example_tuple, attr): + rsync_out, name, version, arch, release, location = example_tuple + return get_file_list_from_rsync_output(rsync_out)[0][attr], locals()[attr] def testDirectoryOutput(self): """get_file_list_from_rsync_output should ignore directories""" @@ -37,34 +37,29 @@ class KnownValues(unittest.TestCase): self.assertEqual(tuple(), result) def testNames(self): - results=self.generate_results(self.examples) - var =[name for rsync_out, name, version, arch, release, location in self.examples] - for i in range(len(results)): - self.assertEqual(results[i]["name"], var[i]) + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="name") + self.assertEqual(k, v) def testVersions(self): - results=self.generate_results(self.examples) - var = [version for rsync_out, name, version, arch, release, location in self.examples] - for i in range(len(results)): - self.assertEqual(results[i]["version"], var[i]) + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="version") + self.assertEqual(k, v) def testArchs(self): - results=self.generate_results(self.examples) - var = [arch for rsync_out, name, version, arch, release, location in self.examples] - for i in range(len(results)): - self.assertEqual(results[i]["arch"], var[i]) + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="arch") + self.assertEqual(k, v) def testReleases(self): - results=self.generate_results(self.examples) - var = [release for rsync_out, name, version, arch, release, location in self.examples] - for i in range(len(results)): - self.assertEqual(results[i]["release"], var[i]) + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="release") + self.assertEqual(k, v) def testLocations(self): - results=self.generate_results(self.examples) - var = [location for rsync_out, name, version, arch, release, location in self.examples] - for i in range(len(results)): - self.assertEqual(results[i]["location"], var[i]) + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="location") + self.assertEqual(k, v) if __name__ == "__main__": unittest.main() -- cgit v1.2.3-54-g00ecf From 9c8b6615d537a9c7bb483457cd8a300d7f9c597e Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 8 Feb 2011 10:55:33 -0600 Subject: * Added filter.py * Added __init__.py for test module * Removed unused files --- filter.py | 90 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ test/__init__.py | 0 test/link_list | 2 -- test/package_list | 3 -- 4 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 filter.py create mode 100644 test/__init__.py delete mode 100644 test/link_list delete mode 100644 test/package_list diff --git a/filter.py b/filter.py new file mode 100644 index 0000000..5229e63 --- /dev/null +++ b/filter.py @@ -0,0 +1,90 @@ +#! /usr/bin/python +#-*- encoding: utf-8 -*- +import commands +import os +import re +from repm.config import * +from repm.pato2 import * + +rsync_list_command="rsync -av --no-motd --list-only " + +def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name=mirror, + mirror_path=mirrorpath, blacklist_file=False): + """ Generates an rsync command for executing it by combining all parameters. + + Parameters: + ---------- + base_command -> str + mirror_name -> str + mirror_path -> str + dir_list -> list or tuple + destdir -> str Must be a dir + blacklist_file -> False or str File must exist + + Return: + ---------- + rsync_command -> str """ + from os.path import isfile, isdir + + if blacklist_file and not isfile(blacklist_file): + print(blacklist_file + " is not a file") + raise NonValidFile + + if not os.path.isdir(destdir): + print(destdir + " is not a directory") + raise NonValidDir + + dir_list="{" + ",".join(dir_list) + "}" + + if blacklist_file: + return " ".join((base_command, "--exclude-from-file="+blacklist_file, + mirror_name + mirror_path + dir_list, destdir)) + return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) + +def run_rsync(base_for_rsync=rsync_list_command, dir_list_for_rsync=(repo_list + dir_list), + debug=verbose): + """ Runs rsync and gets returns it's output """ + cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) + if debug: + printf("rsync_command" + cmd) + return commands.getoutput(cmd) + +def get_file_list_from_rsync_output(rsync_output): + """ Generates a list of packages and versions from an rsync output using --list-only --no-motd. + + Parameters: + ---------- + rsync_output -> str containing output from rsync + + Returns: + ---------- + package_list -> tuple of Package objects. """ + a=list() + + def directory(line): + pass + + def package_or_link(line): + """ Take info out of filename """ + location_field = 4 + pkg = Package() + pkg["location"] = line.rsplit()[location_field] + fileattrs = pkg["location"].split("/")[-1].split("-") + pkg["arch"] = fileattrs.pop(-1).split(".")[0] + pkg["release"] = fileattrs.pop(-1) + pkg["version"] = fileattrs.pop(-1) + pkg["name"] = "-".join(fileattrs) + return pkg + + options = { "d": directory, + "l": package_or_link, + "-": package_or_link} + + for line in rsync_output.split("\n"): + if ".pkg.tar.gz" or ".pkg.tar.xz" in line: + pkginfo=options[line[0]](line) + if pkginfo: + a.append(pkginfo) + + return tuple(a) + diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/link_list b/test/link_list deleted file mode 100644 index d015364..0000000 --- a/test/link_list +++ /dev/null @@ -1,2 +0,0 @@ -lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz -lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz \ No newline at end of file diff --git a/test/package_list b/test/package_list deleted file mode 100644 index 6ffa1de..0000000 --- a/test/package_list +++ /dev/null @@ -1,3 +0,0 @@ --rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz --rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz --rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz \ No newline at end of file -- cgit v1.2.3-54-g00ecf From 1c178e144ae404c6ee332e25c9f3ae37fb9abb23 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 8 Feb 2011 10:56:31 -0600 Subject: * Modified gitignore for ignoring pyc files --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e4e5f6c..e645833 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -*~ \ No newline at end of file +*~ +*.pyc \ No newline at end of file -- cgit v1.2.3-54-g00ecf From 5f08ca1edd02121f950a829745a512a2bc58b016 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 8 Feb 2011 11:05:26 -0600 Subject: * Changed variable name for consistency. --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index a7087c4..6206607 100644 --- a/config.py +++ b/config.py @@ -38,7 +38,7 @@ verbose = False blacklist = docs + "/blacklist.txt" whitelist = docs + "/whitelist.txt" pending = docs + "/pending" -rsyncBlacklist = docs + "/rsyncBlacklist" +rsync_blacklist = docs + "/rsyncBlacklist" # Classes and Exceptions class NonValidFile(ValueError): pass -- cgit v1.2.3-54-g00ecf From d9ac9d17a92607b09fb9afa91ff96aeae38dbdf3 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 8 Feb 2011 11:55:25 -0600 Subject: Delete unneeded file. --- test/directory_list | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 test/directory_list diff --git a/test/directory_list b/test/directory_list deleted file mode 100644 index c0f7f47..0000000 --- a/test/directory_list +++ /dev/null @@ -1,2 +0,0 @@ -drwxrwxr-x 15 2010/09/11 11:28:50 community-staging -drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os \ No newline at end of file -- cgit v1.2.3-54-g00ecf From 2cae61ff561561e405f0cd0c150dbcd77ce25b82 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 8 Feb 2011 13:26:54 -0600 Subject: config py: * Make Package class init, restrict keys. filter.py * Corrected function * Added generate_rsync_exclude --- config.py | 31 ++++++++++++++++++++++++------- filter.py | 53 ++++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 68 insertions(+), 16 deletions(-) diff --git a/config.py b/config.py index 6206607..bd8ef1b 100644 --- a/config.py +++ b/config.py @@ -47,15 +47,21 @@ class NonValidCommand(ValueError): pass class Package: """ An object that has information about a package. """ - package_info={ "name" : False, - "version" : False, - "release" : False, - "arch" : False, - "license" : False, - "location": False} + package_info=dict() + def __init__(self): + self.package_info={ "name" : False, + "version" : False, + "release" : False, + "arch" : False, + "license" : False, + "location": False} + def __setitem__(self, key, item): - return self.package_info.__setitem__(key, item) + if key in self.package_info.keys(): + return self.package_info.__setitem__(key, item) + else: + raise ValueError("Package has no %s attribute" % key) def __getitem__(self, key): return self.package_info.__getitem__(key) @@ -63,4 +69,15 @@ class Package: def __unicode__(self): return str(self.package_info) + def __repr__(self): + return str(self.package_info) + + def __eq__(self,x): + if not isinstance(x, Package): + return False + for key in self.package_info.keys(): + if x[key] != self[key]: + return False + else: + return True diff --git a/filter.py b/filter.py index 5229e63..817a228 100644 --- a/filter.py +++ b/filter.py @@ -6,7 +6,7 @@ import re from repm.config import * from repm.pato2 import * -rsync_list_command="rsync -av --no-motd --list-only " +rsync_list_command="rsync -a --no-motd --list-only " def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name=mirror, mirror_path=mirrorpath, blacklist_file=False): @@ -18,8 +18,8 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name= mirror_name -> str mirror_path -> str dir_list -> list or tuple - destdir -> str Must be a dir - blacklist_file -> False or str File must exist + destdir -> str Path to dir, dir must exist. + blacklist_file -> False or str Path to file, file must exist. Return: ---------- @@ -54,11 +54,11 @@ def get_file_list_from_rsync_output(rsync_output): Parameters: ---------- - rsync_output -> str containing output from rsync + rsync_output -> str Contains output from rsync Returns: ---------- - package_list -> tuple of Package objects. """ + package_list -> tuple Contains Package objects. """ a=list() def directory(line): @@ -81,10 +81,45 @@ def get_file_list_from_rsync_output(rsync_output): "-": package_or_link} for line in rsync_output.split("\n"): - if ".pkg.tar.gz" or ".pkg.tar.xz" in line: - pkginfo=options[line[0]](line) - if pkginfo: - a.append(pkginfo) + if ".pkg.tar" in line: + pkginfo=options[line[0]](line) + if pkginfo: + a.append(pkginfo) return tuple(a) +def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, + blacklist_file=rsync_blacklist, debug=verbose): + """ Generate an exclude list for rsync + + Parameters: + ---------- + package_iterable -> list or tuple Contains Package objects + blacklisted_names-> list or tuple Contains blacklisted names + blacklist_file -> str Path to file + debug -> bool + + Output: + ---------- + if debug == False -> None + if debug == True -> blacklist """ + a=list() + + for package in packages_iterable: + if not isinstance(package, Package): + raise ValueError(" %s is not a Package object " % package) + if package["name"] in blacklisted_names: + a.append(package["location"]) + + if debug: + printf(a) + + try: + fsock = open(blacklist_file,"w") + try: + fsock.write("\n".join(a)) + finally: + fsock.close() + except IOError: + printf("%s wasnt written" % blacklist_file) + -- cgit v1.2.3-54-g00ecf From 072787627a2339c3d5aca541086c2e4545bbad8d Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 8 Feb 2011 14:06:58 -0600 Subject: * Changed variable name in filter.py * Added example in test1.py --- filter.py | 15 +++++++-------- test/test1.py | 5 +++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/filter.py b/filter.py index 817a228..ffc0965 100644 --- a/filter.py +++ b/filter.py @@ -89,20 +89,19 @@ def get_file_list_from_rsync_output(rsync_output): return tuple(a) def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, - blacklist_file=rsync_blacklist, debug=verbose): + exclude_file=rsync_blacklist, debug=verbose): """ Generate an exclude list for rsync Parameters: ---------- - package_iterable -> list or tuple Contains Package objects - blacklisted_names-> list or tuple Contains blacklisted names - blacklist_file -> str Path to file - debug -> bool + package_iterable -> list or tuple Contains Package objects + blacklisted_names-> list or tuple Contains blacklisted names + exclude_file -> str Path to file + debug -> bool If True, file list gets logged Output: ---------- - if debug == False -> None - if debug == True -> blacklist """ + None """ a=list() for package in packages_iterable: @@ -115,7 +114,7 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, printf(a) try: - fsock = open(blacklist_file,"w") + fsock = open(exclude_file,"w") try: fsock.write("\n".join(a)) finally: diff --git a/test/test1.py b/test/test1.py index 61b0651..13b5660 100644 --- a/test/test1.py +++ b/test/test1.py @@ -13,7 +13,8 @@ import unittest class KnownValues(unittest.TestCase): directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", - "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os") + "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os", + 'dr-xr-sr-x 4096 2010/09/11 11:37:10 .') # (rsync_out, name, version, arch, release, location) examples=( ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686", "2", "community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz"), @@ -60,6 +61,6 @@ class KnownValues(unittest.TestCase): for i in self.examples: k,v = self.generate_results(example_tuple=i,attr="location") self.assertEqual(k, v) - + if __name__ == "__main__": unittest.main() -- cgit v1.2.3-54-g00ecf From 6a37d25e51228a0a83d004b9acda63c528972a26 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 13 Feb 2011 14:26:40 -0800 Subject: Fixed glob expression --- pato2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pato2.py b/pato2.py index 236f2ea..cfdd859 100644 --- a/pato2.py +++ b/pato2.py @@ -54,7 +54,7 @@ def packages(repo_, arch_, expr="*"): def sync_all_repo(verbose_=verbose): folders = ",".join(repo_list + dir_list) - cmd_ = "rsync -av --progress --delete-after --delay-updates " + mirror + mirrorpath + "/{" + folders + "} " + repodir + cmd_ = "rsync -av --delete-after --delay-updates " + mirror + mirrorpath + "/{" + folders + "} " + repodir printf(cmd_) a=commands.getoutput(cmd_) if verbose_: printf(a) @@ -144,11 +144,11 @@ def add_free_repo(verbose_=verbose): for repo_ in repo_list: for arch_ in arch_list: lista_=list() - for file_ in glob(free_path + repo_ + "/os/" + arch_ + "/*"): + for file_ in glob(free_path + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"): lista_.append(file_) link(repo_,arch_,file_) for dir_ in other: - for file_ in glob(free_path + repo_ + "/os/" + dir_ + "/*"): + for file_ in glob(free_path + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"): lista_.append(file_) link(repo_,arch_,file_) if lista_: -- cgit v1.2.3-54-g00ecf From 2321a53d35a9736b8c5c715ca40a9af69b1bf64e Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 01:14:47 -0600 Subject: Rearranged functions --- config.py | 5 +++++ filter.py | 43 ------------------------------------------- pato2.py | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 43 deletions(-) diff --git a/config.py b/config.py index bd8ef1b..4854e0d 100644 --- a/config.py +++ b/config.py @@ -40,6 +40,11 @@ whitelist = docs + "/whitelist.txt" pending = docs + "/pending" rsync_blacklist = docs + "/rsyncBlacklist" +# Rsync commands + +rsync_list_command="rsync -a --no-motd --list-only " + + # Classes and Exceptions class NonValidFile(ValueError): pass class NonValidDir(ValueError): pass diff --git a/filter.py b/filter.py index ffc0965..b51dba8 100644 --- a/filter.py +++ b/filter.py @@ -6,49 +6,6 @@ import re from repm.config import * from repm.pato2 import * -rsync_list_command="rsync -a --no-motd --list-only " - -def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name=mirror, - mirror_path=mirrorpath, blacklist_file=False): - """ Generates an rsync command for executing it by combining all parameters. - - Parameters: - ---------- - base_command -> str - mirror_name -> str - mirror_path -> str - dir_list -> list or tuple - destdir -> str Path to dir, dir must exist. - blacklist_file -> False or str Path to file, file must exist. - - Return: - ---------- - rsync_command -> str """ - from os.path import isfile, isdir - - if blacklist_file and not isfile(blacklist_file): - print(blacklist_file + " is not a file") - raise NonValidFile - - if not os.path.isdir(destdir): - print(destdir + " is not a directory") - raise NonValidDir - - dir_list="{" + ",".join(dir_list) + "}" - - if blacklist_file: - return " ".join((base_command, "--exclude-from-file="+blacklist_file, - mirror_name + mirror_path + dir_list, destdir)) - return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) - -def run_rsync(base_for_rsync=rsync_list_command, dir_list_for_rsync=(repo_list + dir_list), - debug=verbose): - """ Runs rsync and gets returns it's output """ - cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) - if debug: - printf("rsync_command" + cmd) - return commands.getoutput(cmd) - def get_file_list_from_rsync_output(rsync_output): """ Generates a list of packages and versions from an rsync output using --list-only --no-motd. diff --git a/pato2.py b/pato2.py index cfdd859..125ae8c 100644 --- a/pato2.py +++ b/pato2.py @@ -166,6 +166,46 @@ def get_licenses(verbose_=verbose): a=commands.getoutput(cmd_) if verbose_: printf(a) +def generate_rsync_command(base_command, dir_list, destdir=repodir, + source=mirror+mirrorpath, blacklist_file=False): + """ Generates an rsync command for executing it by combining all parameters. + + Parameters: + ---------- + base_command -> str + dir_list -> list or tuple + destdir -> str Path to dir, dir must exist. + source -> str The source for rsync + blacklist_file -> False or str Path to file, file must exist. + + Return: + ---------- + rsync_command -> str """ + from os.path import isfile, isdir + + if blacklist_file and not isfile(blacklist_file): + print(blacklist_file + " is not a file") + raise NonValidFile + + if not os.path.isdir(destdir): + print(destdir + " is not a directory") + raise NonValidDir + + dir_list="{" + ",".join(dir_list) + "}" + + if blacklist_file: + return " ".join((base_command, "--exclude-from-file="+blacklist_file, + mirror_name + mirror_path + dir_list, destdir)) + return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) + +def run_rsync(base_for_rsync=rsync_list_command, dir_list_for_rsync=(repo_list + dir_list), + debug=verbose): + """ Runs rsync and gets returns it's output """ + cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) + if debug: + printf("rsync_command" + cmd) + return commands.getoutput(cmd) + if __name__ == "__main__": from time import time start_time = time() -- cgit v1.2.3-54-g00ecf From 6203dc2fc926781db694ca2383b1e44e2c5469c7 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 01:29:33 -0600 Subject: - filter.py: * Makes a blacklist for rsync - pato2.py: * Has more functions --- filter.py | 6 +++++- pato2.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/filter.py b/filter.py index b51dba8..14ae31e 100644 --- a/filter.py +++ b/filter.py @@ -1,4 +1,4 @@ -#! /usr/bin/python + #! /usr/bin/python #-*- encoding: utf-8 -*- import commands import os @@ -79,3 +79,7 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, except IOError: printf("%s wasnt written" % blacklist_file) +if name == "__main__": + a=run_rsync(rsync_list_command) + packages=get_file_list_from_rsync_output(a) + generate_exclude_list_from_blacklist(packages,listado(blacklist)) diff --git a/pato2.py b/pato2.py index 125ae8c..97eec9d 100644 --- a/pato2.py +++ b/pato2.py @@ -198,7 +198,7 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name + mirror_path + dir_list, destdir)) return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) -def run_rsync(base_for_rsync=rsync_list_command, dir_list_for_rsync=(repo_list + dir_list), +def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), debug=verbose): """ Runs rsync and gets returns it's output """ cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) -- cgit v1.2.3-54-g00ecf From b27cbc08447fe5605bf877ee0991888d5ecf6382 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 01:32:48 -0600 Subject: Corrected filter.py to run --- filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filter.py b/filter.py index 14ae31e..c789cd5 100644 --- a/filter.py +++ b/filter.py @@ -79,7 +79,7 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, except IOError: printf("%s wasnt written" % blacklist_file) -if name == "__main__": +if __name__ == "__main__": a=run_rsync(rsync_list_command) packages=get_file_list_from_rsync_output(a) generate_exclude_list_from_blacklist(packages,listado(blacklist)) -- cgit v1.2.3-54-g00ecf From 931daf34033714e64fb4f98a101dd804e4e6363e Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 01:34:35 -0600 Subject: Added os module to config file --- config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/config.py b/config.py index 4854e0d..bd46688 100644 --- a/config.py +++ b/config.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- from user import home import commands +import os time__ = commands.getoutput("date +%Y%m%d-%H:%M") -- cgit v1.2.3-54-g00ecf From 7f40b89e8cd0e9b2cfd92a8b74c4063d8b9aa3e3 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 01:38:59 -0600 Subject: Corrected funcions --- pato2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pato2.py b/pato2.py index 97eec9d..55e63e7 100644 --- a/pato2.py +++ b/pato2.py @@ -195,13 +195,13 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, if blacklist_file: return " ".join((base_command, "--exclude-from-file="+blacklist_file, - mirror_name + mirror_path + dir_list, destdir)) - return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) + source + dir_list, destdir)) + return " ".join((base_command, source + dir_list, destdir)) def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), debug=verbose): """ Runs rsync and gets returns it's output """ - cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) + cmd = str(generate_rsync_command(base_for_rsync, (repo_list + dir_list))) if debug: printf("rsync_command" + cmd) return commands.getoutput(cmd) -- cgit v1.2.3-54-g00ecf From cbe877f2ba25b398ad32b92d22dc6f5a108ef59f Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 02:28:01 -0600 Subject: * Changed get_file_list_ for pkginfo in function names * Added pkginfo_from_... funcions --- filter.py | 57 ++++++++++++++++++++++++++++++++++++++------------------- test/test1.py | 8 ++++---- 2 files changed, 42 insertions(+), 23 deletions(-) diff --git a/filter.py b/filter.py index c789cd5..232319f 100644 --- a/filter.py +++ b/filter.py @@ -1,13 +1,34 @@ #! /usr/bin/python #-*- encoding: utf-8 -*- import commands -import os import re from repm.config import * from repm.pato2 import * -def get_file_list_from_rsync_output(rsync_output): - """ Generates a list of packages and versions from an rsync output using --list-only --no-motd. +def pkginfo_from_filename(filename): + """ Generates a Package object with info from a filename, + filename can be relative or absolute + + Parameters: + ---------- + filename -> str + + Returns: + ---------- + pkg -> Package object""" + pkg = Package() + pkg["location"] = filename + fileattrs = os.path.basename(filename).split("-") + pkg["arch"] = fileattrs.pop(-1).split(".")[0] + pkg["release"] = fileattrs.pop(-1) + pkg["version"] = fileattrs.pop(-1) + pkg["name"] = "-".join(fileattrs) + return pkg + + +def pkginfo_from_rsync_output(rsync_output): + """ Generates a list of packages and versions from an rsync output + wich uses --list-only and --no-motd options. Parameters: ---------- @@ -18,33 +39,30 @@ def get_file_list_from_rsync_output(rsync_output): package_list -> tuple Contains Package objects. """ a=list() - def directory(line): - pass - def package_or_link(line): """ Take info out of filename """ location_field = 4 - pkg = Package() - pkg["location"] = line.rsplit()[location_field] - fileattrs = pkg["location"].split("/")[-1].split("-") - pkg["arch"] = fileattrs.pop(-1).split(".")[0] - pkg["release"] = fileattrs.pop(-1) - pkg["version"] = fileattrs.pop(-1) - pkg["name"] = "-".join(fileattrs) - return pkg - + pkginfo_from_filename(line.rsplit()[location_field]) + + def directory(line): + pass + options = { "d": directory, "l": package_or_link, "-": package_or_link} for line in rsync_output.split("\n"): if ".pkg.tar" in line: - pkginfo=options[line[0]](line) - if pkginfo: - a.append(pkginfo) + pkginfo_=options[line[0]](line) + if pkginfo_: + a.append(pkginfo_) return tuple(a) +def pkginfo_from_files_in_dir(directory): + """ Returns pkginfo from filenames of packages in dir + wich has .pkg.tar.{xz,gz} on them """ + def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, exclude_file=rsync_blacklist, debug=verbose): """ Generate an exclude list for rsync @@ -78,8 +96,9 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, fsock.close() except IOError: printf("%s wasnt written" % blacklist_file) + if __name__ == "__main__": a=run_rsync(rsync_list_command) - packages=get_file_list_from_rsync_output(a) + packages=pkginfo_from_rsync_output(a) generate_exclude_list_from_blacklist(packages,listado(blacklist)) diff --git a/test/test1.py b/test/test1.py index 13b5660..9e94352 100644 --- a/test/test1.py +++ b/test/test1.py @@ -29,12 +29,12 @@ class KnownValues(unittest.TestCase): def generate_results(self, example_tuple, attr): rsync_out, name, version, arch, release, location = example_tuple - return get_file_list_from_rsync_output(rsync_out)[0][attr], locals()[attr] + return pkginfo_from_rsync_output(rsync_out)[0][attr], locals()[attr] def testDirectoryOutput(self): - """get_file_list_from_rsync_output should ignore directories""" + """pkginfo_from_rsync_output should ignore directories""" rsync_out="\n".join(self.directory_list) - result=get_file_list_from_rsync_output(rsync_out) + result=pkginfo_from_rsync_output(rsync_out) self.assertEqual(tuple(), result) def testNames(self): @@ -62,5 +62,5 @@ class KnownValues(unittest.TestCase): k,v = self.generate_results(example_tuple=i,attr="location") self.assertEqual(k, v) -if __name__ == "__main__": +if __name__ == "__mpain__": unittest.main() -- cgit v1.2.3-54-g00ecf From 4e7508d9a61c3653b46da8fa513513756bb3b6f1 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 16:46:33 -0600 Subject: * pkginfo functions --- filter.py | 44 +++++++++++++++++++++++++++++++------------- test/test1.py | 2 +- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/filter.py b/filter.py index 232319f..ad3b33d 100644 --- a/filter.py +++ b/filter.py @@ -1,7 +1,6 @@ #! /usr/bin/python #-*- encoding: utf-8 -*- -import commands -import re +import glob from repm.config import * from repm.pato2 import * @@ -11,11 +10,13 @@ def pkginfo_from_filename(filename): Parameters: ---------- - filename -> str + filename -> str Must contain .pkg.tar. Returns: ---------- pkg -> Package object""" + if ".pkg.tar." not in filename: + raise NonValidFile pkg = Package() pkg["location"] = filename fileattrs = os.path.basename(filename).split("-") @@ -37,31 +38,49 @@ def pkginfo_from_rsync_output(rsync_output): Returns: ---------- package_list -> tuple Contains Package objects. """ - a=list() + package_list=list() def package_or_link(line): """ Take info out of filename """ location_field = 4 pkginfo_from_filename(line.rsplit()[location_field]) - def directory(line): + def do_nothing(): pass - - options = { "d": directory, + + options = { "d": do_nothing, "l": package_or_link, - "-": package_or_link} + "-": package_or_link, + " ": do_nothing} for line in rsync_output.split("\n"): if ".pkg.tar" in line: - pkginfo_=options[line[0]](line) + pkginfo=options[line[0]](line) if pkginfo_: - a.append(pkginfo_) + package_list.append(pkginfo) - return tuple(a) + return tuple(package_list) def pkginfo_from_files_in_dir(directory): """ Returns pkginfo from filenames of packages in dir - wich has .pkg.tar.{xz,gz} on them """ + wich has .pkg.tar. on them + + Parameters: + ---------- + directory -> str Directory must exist + + Returns: + ---------- + package_list -> tuple Contains Package objects """ + package_list=list() + + if not os.path.isdir(directory): + raise NonValidDir + + for filename in glob(os.path.join(directory,"*")): + if ".pkg.tar." in filename: + package_list.append(pkginfo_from_filename(filename)) + return tuple(package_list) def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, exclude_file=rsync_blacklist, debug=verbose): @@ -97,7 +116,6 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, except IOError: printf("%s wasnt written" % blacklist_file) - if __name__ == "__main__": a=run_rsync(rsync_list_command) packages=pkginfo_from_rsync_output(a) diff --git a/test/test1.py b/test/test1.py index 9e94352..b810e9a 100644 --- a/test/test1.py +++ b/test/test1.py @@ -62,5 +62,5 @@ class KnownValues(unittest.TestCase): k,v = self.generate_results(example_tuple=i,attr="location") self.assertEqual(k, v) -if __name__ == "__mpain__": +if __name__ == "__main__": unittest.main() -- cgit v1.2.3-54-g00ecf From 1db8ccbc9221be6c30884172c00caefa217f66a5 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 16:47:03 -0600 Subject: Added test for pkginfo_from_file --- test/test_pkginfo_from_file.py | 76 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 test/test_pkginfo_from_file.py diff --git a/test/test_pkginfo_from_file.py b/test/test_pkginfo_from_file.py new file mode 100644 index 0000000..0d7ede6 --- /dev/null +++ b/test/test_pkginfo_from_file.py @@ -0,0 +1,76 @@ +#! /usr/bin/python +# -*- encoding: utf-8 -*- +""" """ + +__author__ = "Joshua Ismael Haase Hernández " +__version__ = "$Revision: 1.1 $" +__date__ = "$Date: 2011/02/08 $" +__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" +__license__ = "GPL3+" + +from repm.config import * +from repm.filter import * +import unittest + +class KnownValues(unittest.TestCase): + # (filename, name, version, release, arch) + # filename is location + known=( + ("community-testing/os/i686/inputattach-1.24-3-i686.pkg.tar.xz","inputattach","1.24","3","i686"), + ("community-testing/os/i686/ngspice-22-1-i686.pkg.tar.xz","ngspice","22","1","i686"), + ("community-testing/os/i686/tmux-1.4-2-i686.pkg.tar.xz","tmux","1.4","2","i686"), + ("community-testing/os/i686/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), + ("../../../pool/community/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), + ("community-testing/os/x86_64/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), + ("../../../pool/community/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), + ("tor-0.2.1.29-2-x86_64.pkg.tar.xz","tor","0.2.1.29","2","x86_64"), + ) + + def generate_results(self, example_tuple, attr): + location, name, version, release, arch = example_tuple + return pkginfo_from_filename(location)[attr], locals()[attr] + + def testReturnPackageObject(self): + for i in self.known: + location, name, version, release, arch = i + self.assertIsInstance(pkginfo_from_filename(location),Package) + + def testNames(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="name") + self.assertEqual(k, v) + + def testVersions(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="version") + self.assertEqual(k, v) + + def testArchs(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="arch") + self.assertEqual(k, v) + + def testReleases(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="release") + self.assertEqual(k, v) + + def testLocations(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="location") + self.assertEqual(k, v) + +class BadInput(unittest.TestCase): + bad=("community-testing/os/i686/community-testing.db", + "community-testing/os/i686/community-testing.db.tar.gz", + "community-testing/os/i686/community-testing.db.tar.gz.old", + "community-testing/os/i686/community-testing.files", + "community-testing/os/i686/community-testing.files.tar.gz", + "community-testing/os/x86_64") + + def testBadInput(self): + for i in self.bad: + self.assertRaises(NonValidFile,pkginfo_from_filename,i) + +if __name__ == "__main__": + unittest.main() -- cgit v1.2.3-54-g00ecf From b525eeca86d3c5a25f971a12ceeae44fadd6fa94 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 16:59:28 -0600 Subject: renamed test1 to be informative --- test/test_pkginfo_from_rsync_output.py | 66 ++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 test/test_pkginfo_from_rsync_output.py diff --git a/test/test_pkginfo_from_rsync_output.py b/test/test_pkginfo_from_rsync_output.py new file mode 100644 index 0000000..b810e9a --- /dev/null +++ b/test/test_pkginfo_from_rsync_output.py @@ -0,0 +1,66 @@ +# -*- encoding: utf-8 -*- +""" """ + +__author__ = "Joshua Ismael Haase Hernández " +__version__ = "$Revision: 1.1 $" +__date__ = "$Date: 2011/02/08 $" +__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" +__license__ = "GPL3+" + +from repm.config import * +from repm.filter import * +import unittest + +class KnownValues(unittest.TestCase): + directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", + "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os", + 'dr-xr-sr-x 4096 2010/09/11 11:37:10 .') + # (rsync_out, name, version, arch, release, location) + examples=( + ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686", "2", "community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz"), + ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686", "2", "community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz"), + ("-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz", + "abuse","0.7.1","x86_64","1","pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz"), + ("-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz", + "acetoneiso2","2.3","i686", "2", "pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz"), + ("-rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz", + "acetoneiso2","2.3","x86_64","2","pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz") + ) + + def generate_results(self, example_tuple, attr): + rsync_out, name, version, arch, release, location = example_tuple + return pkginfo_from_rsync_output(rsync_out)[0][attr], locals()[attr] + + def testDirectoryOutput(self): + """pkginfo_from_rsync_output should ignore directories""" + rsync_out="\n".join(self.directory_list) + result=pkginfo_from_rsync_output(rsync_out) + self.assertEqual(tuple(), result) + + def testNames(self): + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="name") + self.assertEqual(k, v) + + def testVersions(self): + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="version") + self.assertEqual(k, v) + + def testArchs(self): + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="arch") + self.assertEqual(k, v) + + def testReleases(self): + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="release") + self.assertEqual(k, v) + + def testLocations(self): + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="location") + self.assertEqual(k, v) + +if __name__ == "__main__": + unittest.main() -- cgit v1.2.3-54-g00ecf From a20a7fc689fd0e8680cbc13adfa6fdc2849e4397 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 17:00:13 -0600 Subject: idem --- test/test1.py | 66 ----------------------------------------------------------- 1 file changed, 66 deletions(-) delete mode 100644 test/test1.py diff --git a/test/test1.py b/test/test1.py deleted file mode 100644 index b810e9a..0000000 --- a/test/test1.py +++ /dev/null @@ -1,66 +0,0 @@ -# -*- encoding: utf-8 -*- -""" """ - -__author__ = "Joshua Ismael Haase Hernández " -__version__ = "$Revision: 1.1 $" -__date__ = "$Date: 2011/02/08 $" -__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" -__license__ = "GPL3+" - -from repm.config import * -from repm.filter import * -import unittest - -class KnownValues(unittest.TestCase): - directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", - "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os", - 'dr-xr-sr-x 4096 2010/09/11 11:37:10 .') - # (rsync_out, name, version, arch, release, location) - examples=( - ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686", "2", "community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz"), - ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686", "2", "community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz"), - ("-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz", - "abuse","0.7.1","x86_64","1","pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz"), - ("-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz", - "acetoneiso2","2.3","i686", "2", "pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz"), - ("-rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz", - "acetoneiso2","2.3","x86_64","2","pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz") - ) - - def generate_results(self, example_tuple, attr): - rsync_out, name, version, arch, release, location = example_tuple - return pkginfo_from_rsync_output(rsync_out)[0][attr], locals()[attr] - - def testDirectoryOutput(self): - """pkginfo_from_rsync_output should ignore directories""" - rsync_out="\n".join(self.directory_list) - result=pkginfo_from_rsync_output(rsync_out) - self.assertEqual(tuple(), result) - - def testNames(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="name") - self.assertEqual(k, v) - - def testVersions(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="version") - self.assertEqual(k, v) - - def testArchs(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="arch") - self.assertEqual(k, v) - - def testReleases(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="release") - self.assertEqual(k, v) - - def testLocations(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="location") - self.assertEqual(k, v) - -if __name__ == "__main__": - unittest.main() -- cgit v1.2.3-54-g00ecf From 9fecb003deaae2364e0d9d41055ca33a7d1a2131 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 17:09:16 -0600 Subject: * Changed format of run_rsync debug output. --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pato2.py b/pato2.py index 55e63e7..13242f7 100644 --- a/pato2.py +++ b/pato2.py @@ -203,7 +203,7 @@ def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), """ Runs rsync and gets returns it's output """ cmd = str(generate_rsync_command(base_for_rsync, (repo_list + dir_list))) if debug: - printf("rsync_command" + cmd) + printf("rsync_command: " + cmd) return commands.getoutput(cmd) if __name__ == "__main__": -- cgit v1.2.3-54-g00ecf From e4ffbeff4bbdeb3f57c39c6f6dac3ce902db9b4c Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 17:14:37 -0600 Subject: * Fixed generate_rsync_command --- pato2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pato2.py b/pato2.py index 13242f7..74d18be 100644 --- a/pato2.py +++ b/pato2.py @@ -195,8 +195,8 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, if blacklist_file: return " ".join((base_command, "--exclude-from-file="+blacklist_file, - source + dir_list, destdir)) - return " ".join((base_command, source + dir_list, destdir)) + os.join(source, dir_list), destdir)) + return " ".join((base_command, os.join(source, dir_list), destdir)) def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), debug=verbose): -- cgit v1.2.3-54-g00ecf From e5b869eade3fbf47cf9ac1b7c6be34dfee14b2c1 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 17:17:31 -0600 Subject: * Fixed generate_rsync_command --- pato2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pato2.py b/pato2.py index 74d18be..e32a4dd 100644 --- a/pato2.py +++ b/pato2.py @@ -195,8 +195,8 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, if blacklist_file: return " ".join((base_command, "--exclude-from-file="+blacklist_file, - os.join(source, dir_list), destdir)) - return " ".join((base_command, os.join(source, dir_list), destdir)) + os.path.join(source, dir_list), destdir)) + return " ".join((base_command, os.path.join(source, dir_list), destdir)) def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), debug=verbose): -- cgit v1.2.3-54-g00ecf From f923ab304017a71136642ed7b9780441edcaf441 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 19:36:26 -0600 Subject: * TestCase for pkginfo_from_rsync_output * Corrected __eq__ method in Package class * Corrected pkginfo_from_rsync_output --- config.py | 2 +- filter.py | 18 ++++---- test/rsync_output_sample | 14 +++++++ test/test_pkginfo_from_rsync_output.py | 76 ++++++++++++---------------------- 4 files changed, 52 insertions(+), 58 deletions(-) create mode 100644 test/rsync_output_sample diff --git a/config.py b/config.py index bd46688..465a8a8 100644 --- a/config.py +++ b/config.py @@ -82,7 +82,7 @@ class Package: if not isinstance(x, Package): return False for key in self.package_info.keys(): - if x[key] != self[key]: + if x[key] != self.package_info[key]: return False else: return True diff --git a/filter.py b/filter.py index ad3b33d..55ab94a 100644 --- a/filter.py +++ b/filter.py @@ -38,26 +38,28 @@ def pkginfo_from_rsync_output(rsync_output): Returns: ---------- package_list -> tuple Contains Package objects. """ - package_list=list() def package_or_link(line): """ Take info out of filename """ location_field = 4 - pkginfo_from_filename(line.rsplit()[location_field]) + return pkginfo_from_filename(line.rsplit()[location_field]) def do_nothing(): - pass + """""" options = { "d": do_nothing, "l": package_or_link, "-": package_or_link, " ": do_nothing} + + package_list=list() - for line in rsync_output.split("\n"): - if ".pkg.tar" in line: - pkginfo=options[line[0]](line) - if pkginfo_: - package_list.append(pkginfo) + lines=[x for x in rsync_output.split("\n") if ".pkg.tar" in x] + + for line in lines: + pkginfo=options[line[0]](line) + if pkginfo: + package_list.append(pkginfo) return tuple(package_list) diff --git a/test/rsync_output_sample b/test/rsync_output_sample new file mode 100644 index 0000000..72d9cd0 --- /dev/null +++ b/test/rsync_output_sample @@ -0,0 +1,14 @@ +dr-xr-sr-x 4096 2010/09/11 11:37:10 . +-rw-r--r-- 11 2011/02/08 00:00:01 lastsync +drwxrwxr-x 15 2010/09/11 11:28:50 community-staging +drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os +drwxrwxr-x 8192 2011/02/07 17:00:01 community-staging/os/i686 +lrwxrwxrwx 52 2010/12/23 16:51:01 community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz -> ../../../pool/community/alex-2.3.4-1-i686.pkg.tar.xz +lrwxrwxrwx 27 2011/02/07 14:02:54 community-staging/os/i686/community-staging.db -> community-staging.db.tar.gz +-rw-rw-r-- 2237 2011/02/07 14:02:54 community-staging/os/i686/community-staging.db.tar.gz +-rw-rw-r-- 3209 2011/02/07 14:00:13 community-staging/os/i686/community-staging.db.tar.gz.old +drwxrwxr-x 15 2009/07/22 15:07:56 community +drwxrwxr-x 40 2009/08/04 15:57:42 community/os +drwxrwsr-x 36864 2011/02/03 05:00:01 community/os/any +-rw-rw-r-- 303336 2010/07/16 10:06:28 community/os/any/any2dvd-0.34-4-any.pkg.tar.xz +-rw-rw-r-- 221664 2010/03/28 15:55:48 community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz diff --git a/test/test_pkginfo_from_rsync_output.py b/test/test_pkginfo_from_rsync_output.py index b810e9a..81f14d9 100644 --- a/test/test_pkginfo_from_rsync_output.py +++ b/test/test_pkginfo_from_rsync_output.py @@ -12,55 +12,33 @@ from repm.filter import * import unittest class KnownValues(unittest.TestCase): - directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", - "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os", - 'dr-xr-sr-x 4096 2010/09/11 11:37:10 .') - # (rsync_out, name, version, arch, release, location) - examples=( - ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686", "2", "community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz"), - ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686", "2", "community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz"), - ("-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz", - "abuse","0.7.1","x86_64","1","pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz"), - ("-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz", - "acetoneiso2","2.3","i686", "2", "pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz"), - ("-rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz", - "acetoneiso2","2.3","x86_64","2","pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz") - ) - - def generate_results(self, example_tuple, attr): - rsync_out, name, version, arch, release, location = example_tuple - return pkginfo_from_rsync_output(rsync_out)[0][attr], locals()[attr] - - def testDirectoryOutput(self): - """pkginfo_from_rsync_output should ignore directories""" - rsync_out="\n".join(self.directory_list) - result=pkginfo_from_rsync_output(rsync_out) - self.assertEqual(tuple(), result) - - def testNames(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="name") - self.assertEqual(k, v) - - def testVersions(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="version") - self.assertEqual(k, v) - - def testArchs(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="arch") - self.assertEqual(k, v) - - def testReleases(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="release") - self.assertEqual(k, v) - - def testLocations(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="location") - self.assertEqual(k, v) + try: + output_file = open("rsync_output_sample") + rsync_out= output_file.read() + output_file.close() + except IOError: print("There is no rsync_output_sample file") + + pkglist = pkginfo_from_rsync_output(rsync_out) + + def testOutputArePackages(self): + if not self.pkglist: + self.fail("not pkglist:" + str(self.pkglist)) + for pkg in self.pkglist: + self.assertIsInstance(pkg,Package) + + def testFirstPkg(self): + first_package_info=Package() + first_package_info.package_info={ "name" : "alex", + "version" : "2.3.4", + "release" : "1", + "arch" : "i686", + "license" : False, + "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz"} + if self.pkglist: + first_package=self.pkglist[0] + else: + self.fail(self.pkglist) + self.assertEqual(first_package,first_package_info) if __name__ == "__main__": unittest.main() -- cgit v1.2.3-54-g00ecf From 3866f148dae1f6c90fc6d903784b65e452c048c6 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 22:45:11 -0600 Subject: * Added debug action to generate_exclude_list_from_blacklist * Added test data to test_pkginfo_from_rsync_output.py --- filter.py | 2 +- test/test_pkginfo_from_rsync_output.py | 38 ++++++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/filter.py b/filter.py index 55ab94a..f8182e3 100644 --- a/filter.py +++ b/filter.py @@ -108,7 +108,7 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, if debug: printf(a) - + return a try: fsock = open(exclude_file,"w") try: diff --git a/test/test_pkginfo_from_rsync_output.py b/test/test_pkginfo_from_rsync_output.py index 81f14d9..5ed5dd8 100644 --- a/test/test_pkginfo_from_rsync_output.py +++ b/test/test_pkginfo_from_rsync_output.py @@ -11,7 +11,27 @@ from repm.config import * from repm.filter import * import unittest -class KnownValues(unittest.TestCase): +example_package_list=(Package(),Package(),Package()) +example_package_list[0].package_info={ "name" : "alex", + "version" : "2.3.4", + "release" : "1", + "arch" : "i686", + "license" : False, + "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz"} +example_package_list[1].package_info={ "name" : "any2dvd", + "version" : "0.34", + "release" : "4", + "arch" : "any", + "license" : False, + "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz"} +example_package_list[2].package_info={ "name" : "gmime22", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz"} + +class pkginfoFromRsyncOutput(unittest.TestCase): try: output_file = open("rsync_output_sample") rsync_out= output_file.read() @@ -26,19 +46,15 @@ class KnownValues(unittest.TestCase): for pkg in self.pkglist: self.assertIsInstance(pkg,Package) - def testFirstPkg(self): - first_package_info=Package() - first_package_info.package_info={ "name" : "alex", - "version" : "2.3.4", - "release" : "1", - "arch" : "i686", - "license" : False, - "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz"} + def testPackageInfo(self): if self.pkglist: first_package=self.pkglist[0] else: self.fail(self.pkglist) - self.assertEqual(first_package,first_package_info) - + self.assertEqual(first_package,example_package_list[0]) + +class generateRsyncBlacklist(unittest.TestCase): + """ Test Blacklist generation """ + if __name__ == "__main__": unittest.main() -- cgit v1.2.3-54-g00ecf From 9de298a7b88f7f36aec4c9344356a935c67cfeb3 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 14 Feb 2011 23:27:46 -0600 Subject: * Added test for generate_exclude_list_from_blacklist * Fixed listado to strip spaces --- filter.py | 1 - pato2.py | 2 +- test/blacklist_sample | 2 ++ test/test_pkginfo_from_rsync_output.py | 17 +++++++++++------ 4 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 test/blacklist_sample diff --git a/filter.py b/filter.py index f8182e3..f91ec68 100644 --- a/filter.py +++ b/filter.py @@ -107,7 +107,6 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, a.append(package["location"]) if debug: - printf(a) return a try: fsock = open(exclude_file,"w") diff --git a/pato2.py b/pato2.py index e32a4dd..5bc97a9 100644 --- a/pato2.py +++ b/pato2.py @@ -42,7 +42,7 @@ def listado(filename_): archivo = open(filename_,"r") lista = archivo.read().split("\n") archivo.close() - return [pkg.split(":")[0] for pkg in lista if pkg] + return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] def db(repo_,arch_): """Construye un nombre para sincronizar una base de datos.""" diff --git a/test/blacklist_sample b/test/blacklist_sample new file mode 100644 index 0000000..2a02af6 --- /dev/null +++ b/test/blacklist_sample @@ -0,0 +1,2 @@ +alex:alex-libre: Aquí va un comentario +gmime22 ::Non free dependencies \ No newline at end of file diff --git a/test/test_pkginfo_from_rsync_output.py b/test/test_pkginfo_from_rsync_output.py index 5ed5dd8..aca49e8 100644 --- a/test/test_pkginfo_from_rsync_output.py +++ b/test/test_pkginfo_from_rsync_output.py @@ -47,14 +47,19 @@ class pkginfoFromRsyncOutput(unittest.TestCase): self.assertIsInstance(pkg,Package) def testPackageInfo(self): - if self.pkglist: - first_package=self.pkglist[0] - else: - self.fail(self.pkglist) - self.assertEqual(first_package,example_package_list[0]) + if not self.pkglist: + self.fail("Pkglist doesn't exist: " + str(self.pkglist)) + self.assertEqual(self.pkglist,example_package_list) class generateRsyncBlacklist(unittest.TestCase): """ Test Blacklist generation """ - + def testListado(self): + self.assertEqual(listado("blacklist_sample"),["alex","gmime22"]) + + def testExcludeFiles(self): + a=generate_exclude_list_from_blacklist(example_package_list,listado("blacklist_sample"),debug=True) + b=[example_package_list[0]["location"],example_package_list[2]["location"]] + self.assertEqual(a,b) + if __name__ == "__main__": unittest.main() -- cgit v1.2.3-54-g00ecf From cf3253774c64e6430d4e3afb826cc6c8b0e4b91b Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Tue, 15 Feb 2011 00:13:37 -0600 Subject: * Upgraded sync_all_repo --- config.py | 2 +- pato2.py | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/config.py b/config.py index 465a8a8..36b54df 100644 --- a/config.py +++ b/config.py @@ -44,7 +44,7 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " - +rsync_update_command="rsync -av --delete-after --delay-updates " # Classes and Exceptions class NonValidFile(ValueError): pass diff --git a/pato2.py b/pato2.py index 5bc97a9..9b902f4 100644 --- a/pato2.py +++ b/pato2.py @@ -25,7 +25,7 @@ """ from repm.config import * - +from repm.filter import * import tarfile from glob import glob from os.path import isdir, isfile @@ -52,12 +52,14 @@ def packages(repo_, arch_, expr="*"): """ Get packages on a repo, arch folder """ return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) ) -def sync_all_repo(verbose_=verbose): - folders = ",".join(repo_list + dir_list) - cmd_ = "rsync -av --delete-after --delay-updates " + mirror + mirrorpath + "/{" + folders + "} " + repodir - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose_: printf(a) +def sync_all_repo(debug=verbose): + cmd=generate_rsync_command(rsync_list_command) + rsout=run_rsync(cmd) + pkgs=pkginfo_from_rsync_output(rsout) + generate_exclude_list_from_blacklist(pkgs,listado(blacklist)) + cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) + a=run_rsync(cmd) + if debug: printf(a) def get_from_desc(desc, var,db_tar_file=False): """ Get a var from desc file """ @@ -166,7 +168,7 @@ def get_licenses(verbose_=verbose): a=commands.getoutput(cmd_) if verbose_: printf(a) -def generate_rsync_command(base_command, dir_list, destdir=repodir, +def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdir=repodir, source=mirror+mirrorpath, blacklist_file=False): """ Generates an rsync command for executing it by combining all parameters. @@ -198,10 +200,8 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, os.path.join(source, dir_list), destdir)) return " ".join((base_command, os.path.join(source, dir_list), destdir)) -def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), - debug=verbose): +def run_rsync(command,debug=verbose): """ Runs rsync and gets returns it's output """ - cmd = str(generate_rsync_command(base_for_rsync, (repo_list + dir_list))) if debug: printf("rsync_command: " + cmd) return commands.getoutput(cmd) -- cgit v1.2.3-54-g00ecf From 8975a9021c8b04fb81d9b9358a55b89ecdb373b7 Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 14 Feb 2011 22:18:36 -0800 Subject: changed config --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 36b54df..c60e8f3 100644 --- a/config.py +++ b/config.py @@ -33,7 +33,7 @@ other = ("any",) # Output output = True -verbose = False +verbose = True # Files blacklist = docs + "/blacklist.txt" -- cgit v1.2.3-54-g00ecf From fb247388925beabe01162f28bfcba7fcf8809254 Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 14 Feb 2011 22:41:29 -0800 Subject: * Fixed for rsync to run --- pato2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pato2.py b/pato2.py index 9b902f4..c598c2f 100644 --- a/pato2.py +++ b/pato2.py @@ -196,15 +196,15 @@ def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdi dir_list="{" + ",".join(dir_list) + "}" if blacklist_file: - return " ".join((base_command, "--exclude-from-file="+blacklist_file, + return " ".join((base_command, "--exclude-from="+blacklist_file, os.path.join(source, dir_list), destdir)) return " ".join((base_command, os.path.join(source, dir_list), destdir)) def run_rsync(command,debug=verbose): """ Runs rsync and gets returns it's output """ if debug: - printf("rsync_command: " + cmd) - return commands.getoutput(cmd) + printf("rsync_command: " + command) + return commands.getoutput(command) if __name__ == "__main__": from time import time -- cgit v1.2.3-54-g00ecf From 7bb59afd65871ed2949b86ec6cef805ea2fd3dd2 Mon Sep 17 00:00:00 2001 From: Nicolás Reynolds Date: Tue, 15 Feb 2011 19:25:51 -0800 Subject: Fixed add_free_repo --- config.py | 1 + pato2.py | 295 ++++++++++++++++++++++++++++++++------------------------------ 2 files changed, 152 insertions(+), 144 deletions(-) diff --git a/config.py b/config.py index c60e8f3..6420355 100644 --- a/config.py +++ b/config.py @@ -19,6 +19,7 @@ logdir = path + "/log" ## Must be defined logname= logdir + "/" + time__ + "-repo-maintainer.log" +freedir= path + "/free/" repodir= path + "/repo" tmp = home + "/tmp" archdb = tmp + "/db" diff --git a/pato2.py b/pato2.py index c598c2f..f5deb48 100644 --- a/pato2.py +++ b/pato2.py @@ -31,142 +31,149 @@ from glob import glob from os.path import isdir, isfile def printf(text,output_=output): - """Guarda el texto en la variable log y puede imprimir en pantalla.""" - log_file = open(logname, 'a') - log_file.write("\n" + str(text) + "\n") - log_file.close() - if output_: print (str(text) + "\n") + """Guarda el texto en la variable log y puede imprimir en pantalla.""" + log_file = open(logname, 'a') + log_file.write("\n" + str(text) + "\n") + log_file.close() + if output_: print (str(text) + "\n") def listado(filename_): - """Obtiene una lista de paquetes de un archivo.""" - archivo = open(filename_,"r") - lista = archivo.read().split("\n") - archivo.close() - return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] + """Obtiene una lista de paquetes de un archivo.""" + archivo = open(filename_,"r") + lista = archivo.read().split("\n") + archivo.close() + return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] def db(repo_,arch_): - """Construye un nombre para sincronizar una base de datos.""" - return "/%s/os/%s/%s.db.tar.gz" % (repo_, arch_, repo_) + """Construye un nombre para sincronizar una base de datos.""" + return "/%s/os/%s/%s.db.tar.gz" % (repo_, arch_, repo_) def packages(repo_, arch_, expr="*"): - """ Get packages on a repo, arch folder """ - return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) ) + """ Get packages on a repo, arch folder """ + return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) ) def sync_all_repo(debug=verbose): - cmd=generate_rsync_command(rsync_list_command) - rsout=run_rsync(cmd) - pkgs=pkginfo_from_rsync_output(rsout) - generate_exclude_list_from_blacklist(pkgs,listado(blacklist)) - cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) - a=run_rsync(cmd) - if debug: printf(a) + cmd=generate_rsync_command(rsync_list_command) + rsout=run_rsync(cmd) + pkgs=pkginfo_from_rsync_output(rsout) + generate_exclude_list_from_blacklist(pkgs,listado(blacklist)) + cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) + a=run_rsync(cmd) + if debug: printf(a) def get_from_desc(desc, var,db_tar_file=False): - """ Get a var from desc file """ - desc = desc.split("\n") - return desc[desc.index(var)+1] + """ Get a var from desc file """ + desc = desc.split("\n") + return desc[desc.index(var)+1] def get_info(repo_,arch_,db_tar_file=False,verbose_=verbose): - """ Makes a list of package name, file and license """ - info=list() - # Extract DB tar.gz - commands.getoutput("mkdir -p " + archdb) - if not db_tar_file: - db_tar_file = repodir + db(repo_,arch_) - if isfile(db_tar_file): - try: - db_open_tar = tarfile.open(db_tar_file, 'r:gz') - except tarfile.ReadError: - printf("No valid db_file %s" % db_tar_file) - return(tuple()) - else: - printf("No db_file %s" % db_tar_file) - return(tuple()) - for file in db_open_tar.getmembers(): - db_open_tar.extract(file, archdb) - db_open_tar.close() - # Get info from file - for dir_ in glob(archdb + "/*"): - if isdir(dir_) and isfile(dir_ + "/desc"): - pkg_desc_file = open(dir_ + "/desc", "r") - desc = pkg_desc_file.read() - pkg_desc_file.close() - info.append(( get_from_desc(desc,"%NAME%"), - dir_.split("/")[-1], - get_from_desc(desc,"%LICENSE%") )) - if verbose_: printf(info) - commands.getoutput("rm -r %s/*" % archdb) - return tuple(info) + """ Makes a list of package name, file and license """ + info=list() + # Extract DB tar.gz + commands.getoutput("mkdir -p " + archdb) + if not db_tar_file: + db_tar_file = repodir + db(repo_,arch_) + if isfile(db_tar_file): + try: + db_open_tar = tarfile.open(db_tar_file, 'r:gz') + except tarfile.ReadError: + printf("No valid db_file %s" % db_tar_file) + return(tuple()) + else: + printf("No db_file %s" % db_tar_file) + return(tuple()) + for file in db_open_tar.getmembers(): + db_open_tar.extract(file, archdb) + db_open_tar.close() + # Get info from file + for dir_ in glob(archdb + "/*"): + if isdir(dir_) and isfile(dir_ + "/desc"): + pkg_desc_file = open(dir_ + "/desc", "r") + desc = pkg_desc_file.read() + pkg_desc_file.close() + info.append(( get_from_desc(desc,"%NAME%"), + dir_.split("/")[-1], + get_from_desc(desc,"%LICENSE%") )) + if verbose_: printf(info) + commands.getoutput("rm -r %s/*" % archdb) + return tuple(info) def make_pending(repo_,arch_,info_): - """ Si los paquetes no están en blacklist ni whitelist y la licencia contiene "custom" los agrega a pending""" - search = tuple( listado(blacklist) + listado (whitelist) ) - if verbose: printf("blaclist + whitelist= " + str(search) ) - lista_=list() - for (name,pkg_,license_) in info_: - if "custom" in license_: - if name not in search: - lista_.append( (name, license_ ) ) - elif not name: - printf( pkg_ + " package has no %NAME% attibute " ) - if verbose: printf( lista_ ) - a=open( pending + "-" + repo_ + ".txt", "w" ).write( - "\n".join([name + ":" + license_ for (name,license_) in lista_]) ) + """ Si los paquetes no están en blacklist ni whitelist y la licencia contiene "custom" los agrega a pending""" + search = tuple( listado(blacklist) + listado (whitelist) ) + if verbose: printf("blaclist + whitelist= " + str(search) ) + lista_=list() + for (name,pkg_,license_) in info_: + if "custom" in license_: + if name not in search: + lista_.append( (name, license_ ) ) + elif not name: + printf( pkg_ + " package has no %NAME% attibute " ) + if verbose: printf( lista_ ) + a=open( pending + "-" + repo_ + ".txt", "w" ).write( + "\n".join([name + ":" + license_ for (name,license_) in lista_]) ) def remove_from_blacklist(repo_,arch_,info_,blacklist_): - """ Check the blacklist and remove packages on the db""" - lista_=list() - pack_=list() - for (name_, pkg_, license_) in info_: - if name_ in blacklist_: - lista_.append(name_) - for p in packages(repo_,arch_,pkg_ + "*"): - pack_.append(p) - if lista_: - lista_=" ".join(lista_) - com_ = "repo-remove " + repodir + db(repo_,arch_) + " " + lista_ - printf(com_) - a = commands.getoutput(com_) - if verbose: printf(a) - if pack_: - pack_=" ".join(pack_) - com_="chmod a-r " + pack_ - printf(com_) - a=commands.getoutput(com_) - if verbose: printf(a) + """ Check the blacklist and remove packages on the db""" + lista_=list() + pack_=list() + for (name_, pkg_, license_) in info_: + if name_ in blacklist_: + lista_.append(name_) + for p in packages(repo_,arch_,pkg_ + "*"): + pack_.append(p) + if lista_: + lista_=" ".join(lista_) + com_ = "repo-remove " + repodir + db(repo_,arch_) + " " + lista_ + printf(com_) + a = commands.getoutput(com_) + if verbose: printf(a) + if pack_: + pack_=" ".join(pack_) + com_="chmod a-r " + pack_ + printf(com_) + a=commands.getoutput(com_) + if verbose: printf(a) def link(repo_,arch_,file_): - """ Makes a link in the repo for the package """ - cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ - a=commands.getoutput(cmd_) - if verbose: - printf(cmd_ + a) + """ Makes a link in the repo for the package """ + cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ + a=commands.getoutput(cmd_) + if verbose: + printf(cmd_ + a) def add_free_repo(verbose_=verbose): - for repo_ in repo_list: - for arch_ in arch_list: - lista_=list() - for file_ in glob(free_path + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"): - lista_.append(file_) - link(repo_,arch_,file_) - for dir_ in other: - for file_ in glob(free_path + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"): - lista_.append(file_) - link(repo_,arch_,file_) - if lista_: - lista_=" ".join(lista_) - if verbose: printf(lista_) - cmd_="repo-add " + repodir + db(repo_,arch_) + " " + lista_ - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose: printf(a) + cmd_=home + "/usr/bin/sync-free" + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose_: printf(a) + for repo_ in repo_list: + for arch_ in arch_list: + lista_=list() + for file_ in glob(freedir + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"): + lista_.append(file_) + # link(repo_,arch_,file_) + for dir_ in other: + for file_ in glob(freedir + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"): + lista_.append(file_) + # link(repo_,arch_,file_) + + printf(lista_) + + if lista_: + lista_=" ".join(lista_) + if verbose: printf(lista_) + cmd_="repo-add " + repodir + db(repo_,arch_) + " " + lista_ + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose: printf(a) def get_licenses(verbose_=verbose): - """ Extract the license from packages in repo_,arch_ and in pending_ file""" - cmd_=home + "/usr/bin/get_license.sh" - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose_: printf(a) + """ Extract the license from packages in repo_,arch_ and in pending_ file""" + cmd_=home + "/usr/bin/get_license.sh" + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose_: printf(a) def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdir=repodir, source=mirror+mirrorpath, blacklist_file=False): @@ -207,33 +214,33 @@ def run_rsync(command,debug=verbose): return commands.getoutput(command) if __name__ == "__main__": - from time import time - start_time = time() - def minute(): - return str(round((time() - start_time)/60, 1)) - - printf(" Cleaning %s folder " % (tmp) ) - commands.getoutput("rm -r %s/*" % tmp) - printf(" Syncing repo") - sync_all_repo(True) - - printf(" Updating databases and pending files lists: minute %s \n" % minute() ) - for repo in repo_list: - for arch in arch_list: - printf( "\n" + repo + "-" + arch + "\n" ) - printf( "Get info: minute %s " % minute() ) - info=get_info(repo,arch) - printf( "Make pending: minute %s" % minute() ) - make_pending(repo,arch,info) - printf( "Update DB: minute %s" % minute() ) - remove_from_blacklist( - repo, arch, info, tuple( listado(blacklist) + listado(pending + "-" + repo + ".txt") ) ) - - printf("Adding Parabola Packages: minute %s\n" % minute() ) - add_free_repo(True) - - printf("Extracting licenses in pending: minute %s" % minute() ) - get_licenses() - - printf("\n\nDelay: %s minutes \n" % minute()) - + from time import time + start_time = time() + def minute(): + return str(round((time() - start_time)/60, 1)) + + printf(" Cleaning %s folder " % (tmp) ) + commands.getoutput("rm -r %s/*" % tmp) + printf(" Syncing repo") + sync_all_repo(True) + + printf(" Updating databases and pending files lists: minute %s \n" % minute() ) + for repo in repo_list: + for arch in arch_list: + printf( "\n" + repo + "-" + arch + "\n" ) + printf( "Get info: minute %s " % minute() ) + info=get_info(repo,arch) + printf( "Make pending: minute %s" % minute() ) + make_pending(repo,arch,info) + printf( "Update DB: minute %s" % minute() ) + remove_from_blacklist( + repo, arch, info, tuple( listado(blacklist) + listado(pending + "-" + repo + ".txt") ) ) + + printf("Adding Parabola Packages: minute %s\n" % minute() ) + add_free_repo(True) + + printf("Extracting licenses in pending: minute %s" % minute() ) + get_licenses() + + printf("\n\nDelay: %s minutes \n" % minute()) + -- cgit v1.2.3-54-g00ecf From 11db46275d14c9a9c2e59019fed3de4b8803f9f4 Mon Sep 17 00:00:00 2001 From: Parabola Date: Fri, 18 Feb 2011 15:42:59 -0800 Subject: Parabola specific changes --- config | 24 ++++++++++++------------ db-functions | 33 ++++++++++++++++++++++++--------- db-update | 9 +++------ 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/config b/config index 53191e0..720d110 100644 --- a/config +++ b/config @@ -1,25 +1,25 @@ -FTP_BASE="/srv/ftp" -SVNREPO='' -PKGREPOS=() -PKGPOOL='' -SRCPOOL='' +FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" +SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" +PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') +PKGPOOL='pool/packages' +SRCPOOL='sources/packages' -CLEANUP_DESTDIR="/srv/package-cleanup" +CLEANUP_DESTDIR="$FTP_BASE/old/packages" CLEANUP_DRYRUN=false # Time in days to keep moved packages CLEANUP_KEEP=30 -SOURCE_CLEANUP_DESTDIR="/srv/source-cleanup" +SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" SOURCE_CLEANUP_DRYRUN=false # Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP=14 +SOURCE_CLEANUP_KEEP=30 LOCK_DELAY=10 LOCK_TIMEOUT=300 -STAGING="$HOME/staging" -TMPDIR="/srv/tmp" -ARCHES=(i686 x86_64) +STAGING="$FTP_BASE/staging" +TMPDIR="$HOME/tmp" +ARCHES=(i686 x86_64 mipsel) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.*" @@ -29,4 +29,4 @@ SRCEXT=".src.tar.gz" ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') # Override default config with config.local -[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" +#[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" diff --git a/db-functions b/db-functions index 7d431fc..0553188 100644 --- a/db-functions +++ b/db-functions @@ -74,7 +74,7 @@ in_array() { script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - local _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + local _owner="$(stat -c %U $LOCKDIR)" error "Script $(basename $0) is already locked by $_owner." exit 1 else @@ -160,7 +160,7 @@ repo_lock () { _count=0 while [ $_count -le $_trial ] || $_lockblock ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + _owner="$(stat -c %U $LOCKDIR)" warning "Repo [${1}] (${2}) is already locked by $_owner. " msg2 "Retrying in $LOCK_DELAY seconds..." else @@ -193,7 +193,7 @@ repo_unlock () { #repo_unlock _grep_pkginfo() { local _ret - _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^${2} = ")" + _ret="$(bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^${2} = ")" echo "${_ret#${2} = }" } @@ -358,9 +358,24 @@ check_splitpkgs() { if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ - "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null - [ $? -ge 1 ] && return 1 + +# Decide whether to look for PKGBUILD on [libre] or [libre-testing] + case $repo in + testing) + altrepo=libre-testing + ;; + *) + altrepo=libre + ;; + esac + + cp -r ${SVNREPO}/$repo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ + cp -r ${SVNREPO}/$altrepo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 + + [[ $? -ge 1 ]] && { + echo "Failed $_pkgbase-$_pkgver-$_pkgarch" + return 1 + } fi local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) @@ -443,7 +458,7 @@ set_repo_permission() { local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" if [ -w "${dbfile}" ]; then - local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") + local group=$(stat --printf='%G' "$(dirname "${dbfile}")") chgrp $group "${dbfile}" || error "Could not change group of ${dbfile} to $group" chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}" else @@ -458,7 +473,7 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ + repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -474,7 +489,7 @@ arch_repo_remove() { error "No database found at '${dbfile}'" return 1 fi - /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ + repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ || error "repo-remove ${dbfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" } diff --git a/db-update b/db-update index 5bdce5e..4740809 100755 --- a/db-update +++ b/db-update @@ -35,12 +35,9 @@ for repo in ${repos[@]}; do if ! check_pkgfile "${pkg}"; then die "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data" fi - if ! check_pkgsvn "${pkg}" "${repo}"; then - die "Package ${repo}/$(basename ${pkg}) is not consistent with svn repository" - fi - if ! check_pkgrepos "${pkg}"; then - die "Package ${repo}/$(basename ${pkg}) already exists in another repository" - fi + #if ! check_pkgrepos "${pkg}"; then + # die "Package ${repo}/$(basename ${pkg}) already exists in another repository" + #fi done if ! check_splitpkgs ${repo} ${pkgs[@]}; then die "Missing split packages for ${repo}" -- cgit v1.2.3-54-g00ecf From c6516eca2803b780a01a6b2bf3e081c8bd570301 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 20 Feb 2011 17:10:40 -0800 Subject: sourceballs --- config | 1 + cron-jobs/sourceballs | 12 ++++++------ db-functions | 5 ++++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/config b/config index 720d110..12bdaaa 100644 --- a/config +++ b/config @@ -1,4 +1,5 @@ FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" +ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') PKGPOOL='pool/packages' diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 0df007e..53d43ce 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -21,10 +21,10 @@ renice +10 -p $$ > /dev/null for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do # Repo does not exist; skip it - if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + if [ ! -f "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi - bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + bsdtar -xOf "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ | awk '/^%NAME%/ { getline b }; /^%BASE%/ { getline b }; /^%VERSION%/ { getline v }; @@ -46,7 +46,7 @@ for repo in ${PKGREPOS[@]}; do done # Create a list of all available source package file names -find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do @@ -91,9 +91,9 @@ for repo in ${PKGREPOS[@]}; do # Build the actual source package pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null - makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1 + makepkg --nocolor --allsource --ignorearch # >/dev/null 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then - mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + mv "${pkgbase}-${pkgver}${SRCEXT}" "${ARCH_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" @@ -129,7 +129,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then - mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi done diff --git a/db-functions b/db-functions index 0553188..e790fb6 100644 --- a/db-functions +++ b/db-functions @@ -16,7 +16,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) +WORKDIR=$(mktemp -d /home/parabolavnx/tmp/$(basename $0).XXXXXXXXXX) LOCKS=() # check if messages are to be printed using color @@ -376,6 +376,9 @@ check_splitpkgs() { echo "Failed $_pkgbase-$_pkgver-$_pkgarch" return 1 } + + sleep 20s + fi local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) -- cgit v1.2.3-54-g00ecf From 9793aaf4e9a716f126e71a1db412cbd32f161593 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Thu, 3 Mar 2011 20:11:33 -0600 Subject: * repo-maintainer doesn't take out reading permission from pending files * Removed commented code --- pato2.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pato2.py b/pato2.py index f5deb48..21dde16 100644 --- a/pato2.py +++ b/pato2.py @@ -128,12 +128,6 @@ def remove_from_blacklist(repo_,arch_,info_,blacklist_): printf(com_) a = commands.getoutput(com_) if verbose: printf(a) - if pack_: - pack_=" ".join(pack_) - com_="chmod a-r " + pack_ - printf(com_) - a=commands.getoutput(com_) - if verbose: printf(a) def link(repo_,arch_,file_): """ Makes a link in the repo for the package """ @@ -152,11 +146,9 @@ def add_free_repo(verbose_=verbose): lista_=list() for file_ in glob(freedir + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"): lista_.append(file_) - # link(repo_,arch_,file_) for dir_ in other: for file_ in glob(freedir + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"): lista_.append(file_) - # link(repo_,arch_,file_) printf(lista_) -- cgit v1.2.3-54-g00ecf From a3694fd2824f6aab2905f52f64a138437f7e004b Mon Sep 17 00:00:00 2001 From: Parabola Date: Sat, 5 Mar 2011 10:56:49 -0800 Subject: Taken out «sources» from dir_list for rsync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index 6420355..b198450 100644 --- a/config.py +++ b/config.py @@ -27,9 +27,10 @@ archdb = tmp + "/db" free_path= path + "/free/" # Repo, arch, and other folders to use for repo -repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") -dir_list = ("pool","sources") -arch_list = ("i686", "x86_64") +# This are tuples, so **always keep a comma before closing parenthesis ** +repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib",) +dir_list = ("pool",) +arch_list = ("i686", "x86_64",) other = ("any",) # Output -- cgit v1.2.3-54-g00ecf From 5a4480f1f3e2e7d0aec965b21f72f4454618333e Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sat, 5 Mar 2011 13:13:41 -0600 Subject: * Added «depends» field to Package class. * Merged tests for filter functions in test_filter.py * Added testfiles for test_filter.py in test directory * Added pkginfo_from_desc function to filter.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 3 +- filter.py | 35 ++++++- test/core.db.tar.gz | Bin 0 -> 1637 bytes test/depends | 4 + test/desc | 39 ++++++++ test/test_filter.py | 169 +++++++++++++++++++++++++++++++++ test/test_pkginfo_from_file.py | 76 --------------- test/test_pkginfo_from_rsync_output.py | 65 ------------- 8 files changed, 248 insertions(+), 143 deletions(-) create mode 100644 test/core.db.tar.gz create mode 100644 test/depends create mode 100644 test/desc create mode 100644 test/test_filter.py delete mode 100644 test/test_pkginfo_from_file.py delete mode 100644 test/test_pkginfo_from_rsync_output.py diff --git a/config.py b/config.py index 6420355..688d6b5 100644 --- a/config.py +++ b/config.py @@ -62,7 +62,8 @@ class Package: "release" : False, "arch" : False, "license" : False, - "location": False} + "location": False, + "depends" : False,} def __setitem__(self, key, item): if key in self.package_info.keys(): diff --git a/filter.py b/filter.py index f91ec68..2fe61f0 100644 --- a/filter.py +++ b/filter.py @@ -26,6 +26,37 @@ def pkginfo_from_filename(filename): pkg["name"] = "-".join(fileattrs) return pkg +def pkginfo_from_desc(filename): + """ Returns pkginfo from desc file. + + Parameters: + ---------- + filename -> str File must exist + + Returns: + ---------- + pkg -> Package object""" + if not os.path.isfile(filename): + raise NonValidFile + try: + f=open(filename) + info=f.read().rsplit() + finally: + f.close() + pkg = Package() + info_map={"name" :("%NAME%" , None), + "version" :("%VERSION%" , 0 ), + "release" :("%VERSION%" , 1 ), + "arch" :("%ARCH%" , None), + "license" :("%LICENSE%" , None), + "location":("%FILENAME%", None),} + + for key in info_map.keys(): + field,pos=info_map[key] + pkg[key]=info[info.index(field)+1] + if pos is not None: + pkg[key]=pkg[key].split("-")[pos] + return pkg def pkginfo_from_rsync_output(rsync_output): """ Generates a list of packages and versions from an rsync output @@ -45,7 +76,7 @@ def pkginfo_from_rsync_output(rsync_output): return pkginfo_from_filename(line.rsplit()[location_field]) def do_nothing(): - """""" + pass options = { "d": do_nothing, "l": package_or_link, @@ -84,6 +115,8 @@ def pkginfo_from_files_in_dir(directory): package_list.append(pkginfo_from_filename(filename)) return tuple(package_list) + + def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, exclude_file=rsync_blacklist, debug=verbose): """ Generate an exclude list for rsync diff --git a/test/core.db.tar.gz b/test/core.db.tar.gz new file mode 100644 index 0000000..a28ea64 Binary files /dev/null and b/test/core.db.tar.gz differ diff --git a/test/depends b/test/depends new file mode 100644 index 0000000..7ff3ad4 --- /dev/null +++ b/test/depends @@ -0,0 +1,4 @@ +%DEPENDS% +glibc>=2.13 +zlib + diff --git a/test/desc b/test/desc new file mode 100644 index 0000000..abba644 --- /dev/null +++ b/test/desc @@ -0,0 +1,39 @@ +%FILENAME% +binutils-2.21-4-x86_64.pkg.tar.xz + +%NAME% +binutils + +%VERSION% +2.21-4 + +%DESC% +A set of programs to assemble and manipulate binary and object files + +%GROUPS% +base + +%CSIZE% +3412892 + +%ISIZE% +17571840 + +%MD5SUM% +4e666f87c78998f4839f33dc06d2043a + +%URL% +http://www.gnu.org/software/binutils/ + +%LICENSE% +GPL + +%ARCH% +x86_64 + +%BUILDDATE% +1297240369 + +%PACKAGER% +Allan McRae + diff --git a/test/test_filter.py b/test/test_filter.py new file mode 100644 index 0000000..5127b75 --- /dev/null +++ b/test/test_filter.py @@ -0,0 +1,169 @@ +# -*- encoding: utf-8 -*- +""" """ + +__author__ = "Joshua Ismael Haase Hernández " +__version__ = "$Revision: 1.1 $" +__date__ = "$Date: 2011/02/08 $" +__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" +__license__ = "GPL3+" + +from repm.config import * +from repm.filter import * +import unittest + +class pkginfo_from_file_KnownValues(unittest.TestCase): + # (filename, name, version, release, arch) + # filename is location + known=( + ("community-testing/os/i686/inputattach-1.24-3-i686.pkg.tar.xz","inputattach","1.24","3","i686"), + ("community-testing/os/i686/ngspice-22-1-i686.pkg.tar.xz","ngspice","22","1","i686"), + ("community-testing/os/i686/tmux-1.4-2-i686.pkg.tar.xz","tmux","1.4","2","i686"), + ("community-testing/os/i686/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), + ("../../../pool/community/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), + ("community-testing/os/x86_64/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), + ("../../../pool/community/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), + ("tor-0.2.1.29-2-x86_64.pkg.tar.xz","tor","0.2.1.29","2","x86_64"), + ) + + def generate_results(self, example_tuple, attr): + location, name, version, release, arch = example_tuple + return pkginfo_from_filename(location)[attr], locals()[attr] + + def testReturnPackageObject(self): + for i in self.known: + location, name, version, release, arch = i + self.assertIsInstance(pkginfo_from_filename(location),Package) + + def testNames(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="name") + self.assertEqual(k, v) + + def testVersions(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="version") + self.assertEqual(k, v) + + def testArchs(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="arch") + self.assertEqual(k, v) + + def testReleases(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="release") + self.assertEqual(k, v) + + def testLocations(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="location") + self.assertEqual(k, v) + +class pkginfo_from_file_BadInput(unittest.TestCase): + bad=("community-testing/os/i686/community-testing.db", + "community-testing/os/i686/community-testing.db.tar.gz", + "community-testing/os/i686/community-testing.db.tar.gz.old", + "community-testing/os/i686/community-testing.files", + "community-testing/os/i686/community-testing.files.tar.gz", + "community-testing/os/x86_64") + + def testBadInput(self): + for i in self.bad: + self.assertRaises(NonValidFile,pkginfo_from_filename,i) + +class pkginfoFromRsyncOutput(unittest.TestCase): + example_package_list=(Package(),Package(),Package()) + example_package_list[0].package_info={ "name" : "alex", + "version" : "2.3.4", + "release" : "1", + "arch" : "i686", + "license" : False, + "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz", + "depends" : False,} + example_package_list[1].package_info={ "name" : "any2dvd", + "version" : "0.34", + "release" : "4", + "arch" : "any", + "license" : False, + "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz", + "depends" : False,} + example_package_list[2].package_info={ "name" : "gmime22", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz", + "depends" : False,} + + try: + output_file = open("rsync_output_sample") + rsync_out= output_file.read() + output_file.close() + except IOError: print("There is no rsync_output_sample file") + + pkglist = pkginfo_from_rsync_output(rsync_out) + + def testOutputArePackages(self): + if not self.pkglist: + self.fail("not pkglist:" + str(self.pkglist)) + for pkg in self.pkglist: + self.assertIsInstance(pkg,Package) + + def testPackageInfo(self): + if not self.pkglist: + self.fail("Pkglist doesn't exist: " + str(self.pkglist)) + self.assertEqual(self.pkglist,self.example_package_list) + +class generateRsyncBlacklist(unittest.TestCase): + example_package_list=(Package(),Package(),Package()) + example_package_list[0].package_info={ "name" : "alex", + "version" : "2.3.4", + "release" : "1", + "arch" : "i686", + "license" : False, + "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz", + "depends" : False,} + example_package_list[1].package_info={ "name" : "any2dvd", + "version" : "0.34", + "release" : "4", + "arch" : "any", + "license" : False, + "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz", + "depends" : False,} + example_package_list[2].package_info={ "name" : "gmime22", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz", + "depends" : False,} + + def testListado(self): + self.assertEqual(listado("blacklist_sample"),["alex","gmime22"]) + + def testExcludeFiles(self): + a=generate_exclude_list_from_blacklist(self.example_package_list,listado("blacklist_sample"),debug=True) + b=[self.example_package_list[0]["location"],self.example_package_list[2]["location"]] + self.assertEqual(a,b) + +class pkginfo_from_descKnownValues(unittest.TestCase): + pkgsample=Package() + pkgsample.package_info={"name" : "binutils", + "version" : "2.21", + "release" : "4", + "arch" : "x86_64", + "license" : "GPL", + "location": "binutils-2.21-4-x86_64.pkg.tar.xz", + "depends" : False,} + pkggen=pkginfo_from_desc("desc") + def testPkginfoFromDesc(self): + if self.pkggen is None: + self.fail("return value is None") + self.assertEqual(self.pkgsample,self.pkggen) + +class pkginfo_from_db(unittest.TestCase): + archdb = os.path.join("./workdir") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_pkginfo_from_file.py b/test/test_pkginfo_from_file.py deleted file mode 100644 index 0d7ede6..0000000 --- a/test/test_pkginfo_from_file.py +++ /dev/null @@ -1,76 +0,0 @@ -#! /usr/bin/python -# -*- encoding: utf-8 -*- -""" """ - -__author__ = "Joshua Ismael Haase Hernández " -__version__ = "$Revision: 1.1 $" -__date__ = "$Date: 2011/02/08 $" -__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" -__license__ = "GPL3+" - -from repm.config import * -from repm.filter import * -import unittest - -class KnownValues(unittest.TestCase): - # (filename, name, version, release, arch) - # filename is location - known=( - ("community-testing/os/i686/inputattach-1.24-3-i686.pkg.tar.xz","inputattach","1.24","3","i686"), - ("community-testing/os/i686/ngspice-22-1-i686.pkg.tar.xz","ngspice","22","1","i686"), - ("community-testing/os/i686/tmux-1.4-2-i686.pkg.tar.xz","tmux","1.4","2","i686"), - ("community-testing/os/i686/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), - ("../../../pool/community/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), - ("community-testing/os/x86_64/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), - ("../../../pool/community/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), - ("tor-0.2.1.29-2-x86_64.pkg.tar.xz","tor","0.2.1.29","2","x86_64"), - ) - - def generate_results(self, example_tuple, attr): - location, name, version, release, arch = example_tuple - return pkginfo_from_filename(location)[attr], locals()[attr] - - def testReturnPackageObject(self): - for i in self.known: - location, name, version, release, arch = i - self.assertIsInstance(pkginfo_from_filename(location),Package) - - def testNames(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="name") - self.assertEqual(k, v) - - def testVersions(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="version") - self.assertEqual(k, v) - - def testArchs(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="arch") - self.assertEqual(k, v) - - def testReleases(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="release") - self.assertEqual(k, v) - - def testLocations(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="location") - self.assertEqual(k, v) - -class BadInput(unittest.TestCase): - bad=("community-testing/os/i686/community-testing.db", - "community-testing/os/i686/community-testing.db.tar.gz", - "community-testing/os/i686/community-testing.db.tar.gz.old", - "community-testing/os/i686/community-testing.files", - "community-testing/os/i686/community-testing.files.tar.gz", - "community-testing/os/x86_64") - - def testBadInput(self): - for i in self.bad: - self.assertRaises(NonValidFile,pkginfo_from_filename,i) - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_pkginfo_from_rsync_output.py b/test/test_pkginfo_from_rsync_output.py deleted file mode 100644 index aca49e8..0000000 --- a/test/test_pkginfo_from_rsync_output.py +++ /dev/null @@ -1,65 +0,0 @@ -# -*- encoding: utf-8 -*- -""" """ - -__author__ = "Joshua Ismael Haase Hernández " -__version__ = "$Revision: 1.1 $" -__date__ = "$Date: 2011/02/08 $" -__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" -__license__ = "GPL3+" - -from repm.config import * -from repm.filter import * -import unittest - -example_package_list=(Package(),Package(),Package()) -example_package_list[0].package_info={ "name" : "alex", - "version" : "2.3.4", - "release" : "1", - "arch" : "i686", - "license" : False, - "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz"} -example_package_list[1].package_info={ "name" : "any2dvd", - "version" : "0.34", - "release" : "4", - "arch" : "any", - "license" : False, - "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz"} -example_package_list[2].package_info={ "name" : "gmime22", - "version" : "2.2.26", - "release" : "1", - "arch" : "x86_64", - "license" : False, - "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz"} - -class pkginfoFromRsyncOutput(unittest.TestCase): - try: - output_file = open("rsync_output_sample") - rsync_out= output_file.read() - output_file.close() - except IOError: print("There is no rsync_output_sample file") - - pkglist = pkginfo_from_rsync_output(rsync_out) - - def testOutputArePackages(self): - if not self.pkglist: - self.fail("not pkglist:" + str(self.pkglist)) - for pkg in self.pkglist: - self.assertIsInstance(pkg,Package) - - def testPackageInfo(self): - if not self.pkglist: - self.fail("Pkglist doesn't exist: " + str(self.pkglist)) - self.assertEqual(self.pkglist,example_package_list) - -class generateRsyncBlacklist(unittest.TestCase): - """ Test Blacklist generation """ - def testListado(self): - self.assertEqual(listado("blacklist_sample"),["alex","gmime22"]) - - def testExcludeFiles(self): - a=generate_exclude_list_from_blacklist(example_package_list,listado("blacklist_sample"),debug=True) - b=[example_package_list[0]["location"],example_package_list[2]["location"]] - self.assertEqual(a,b) - -if __name__ == "__main__": - unittest.main() -- cgit v1.2.3-54-g00ecf From 8c9ff0c97d0802fadc96fe9eacaef7eaa2db5ad2 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sun, 6 Mar 2011 16:37:36 -0600 Subject: * Using os.path.join for joining path instead of str + str to avoid errors --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pato2.py b/pato2.py index 21dde16..936e97a 100644 --- a/pato2.py +++ b/pato2.py @@ -137,7 +137,7 @@ def link(repo_,arch_,file_): printf(cmd_ + a) def add_free_repo(verbose_=verbose): - cmd_=home + "/usr/bin/sync-free" + cmd_=os.path.join(home,"/usr/bin/sync-free") printf(cmd_) a=commands.getoutput(cmd_) if verbose_: printf(a) -- cgit v1.2.3-54-g00ecf From 31a0d9e077ed10c5a4b9f461bbde8979f127bdf7 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sun, 6 Mar 2011 16:43:18 -0600 Subject: * Changed rsync-update-command to not delete (delete made by ftp-dir-cleanup) --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index ada17f0..37bebc0 100644 --- a/config.py +++ b/config.py @@ -46,7 +46,7 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " -rsync_update_command="rsync -av --delete-after --delay-updates " +rsync_update_command="rsync -av --delay-updates " # Classes and Exceptions class NonValidFile(ValueError): pass -- cgit v1.2.3-54-g00ecf From 229a9c504cbd733c93cf91399dc54bedf5160cc5 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 6 Mar 2011 15:22:21 -0800 Subject: Fixes --- config | 8 ++--- cron-jobs/sourceballs2 | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++ db-functions | 4 +-- 3 files changed, 90 insertions(+), 9 deletions(-) create mode 100755 cron-jobs/sourceballs2 diff --git a/config b/config index 12bdaaa..217b627 100644 --- a/config +++ b/config @@ -20,14 +20,10 @@ LOCK_TIMEOUT=300 STAGING="$FTP_BASE/staging" TMPDIR="$HOME/tmp" -ARCHES=(i686 x86_64 mipsel) +ARCHES=(i686 x86_64 mips64el) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.*" SRCEXT=".src.tar.gz" -# Allowed licenses: get sourceballs only for licenses in this array -ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') - -# Override default config with config.local -#[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" +MAKEPKGCONF="$HOME/etc/makepkg.conf" diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 new file mode 100755 index 0000000..452208e --- /dev/null +++ b/cron-jobs/sourceballs2 @@ -0,0 +1,87 @@ +#!/bin/bash + +dirname="$(dirname $(readlink -e $0))" +. "${dirname}/../db-functions" +. "${dirname}/../config" +. "${MAKEPKGCONF}" + +pushd "${WORKDIR}" >/dev/null + +script_lock + +#adjust the nice level to run at a lower priority +renice +10 -p $$ > /dev/null + +# Create a list of all available source package file names +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" + +# Steps +# Traverse the ABSLibre +# Makepkg --allsource every package +# Remove the old packages +pushd "${SVNREPO}" >/dev/null + +failedpkgs=() +for repo in ${PKGREPOS[@]}; do + pushd $repo >/dev/null + find . -maxdepth 1 -type d | while read pkg; do + pushd "${SVNREPO}/$repo/$pkg" >/dev/null + + [[ ! -e PKGBUILD ]] && { + warning "$repo/$pkg is not a package" + continue + } + + unset pkgbase pkgname + source PKGBUILD + pkgbase=${pkgbase:-$pkgname} + + echo "${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" + + # Skip already sourceballed + [[ -e "${SRCPKGDEST}/${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" ]] && \ + continue + + makepkg --allsource --ignorearch -c >/dev/null 2>&1 + + [[ $? -ne 0 ]] && \ + failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" + + done + popd >/dev/null +done + +# Cleanup old source packages +cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) + +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages..." + ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + if ! ${SOURCE_CLEANUP_DRYRUN}; then + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + fi + done +fi + +old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages from the cleanup directory..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + done +fi + +msg "Failed" +for _fail in ${failedpkgs[@]}; do + msg2 "$_fail" +done + + +script_unlock + diff --git a/db-functions b/db-functions index e2731cb..b900669 100644 --- a/db-functions +++ b/db-functions @@ -1,7 +1,5 @@ #!/bin/bash -. config - # Some PKGBUILDs need CARCH to be set CARCH="x86_64" @@ -18,7 +16,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -d ${TMPDIR}/$(basename $0).XXXXXXXXXX) +WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) LOCKS=() # check if messages are to be printed using color -- cgit v1.2.3-54-g00ecf From e2385920015311376c1631a42a1ca8f1c45dae4f Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sun, 6 Mar 2011 18:08:25 -0600 Subject: * Fixed error that leaked non-free files to repo --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pato2.py b/pato2.py index 936e97a..336c6fb 100644 --- a/pato2.py +++ b/pato2.py @@ -56,7 +56,7 @@ def sync_all_repo(debug=verbose): cmd=generate_rsync_command(rsync_list_command) rsout=run_rsync(cmd) pkgs=pkginfo_from_rsync_output(rsout) - generate_exclude_list_from_blacklist(pkgs,listado(blacklist)) + generate_exclude_list_from_blacklist(pkgs,listado(blacklist),debug=False) cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) a=run_rsync(cmd) if debug: printf(a) -- cgit v1.2.3-54-g00ecf From 58a67fb771fbbf433fe3ab2dafb9194e6f635f13 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sun, 6 Mar 2011 18:09:52 -0600 Subject: * Added --delete to rsync-update-command until ftp-dir-cleanup can do delete --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 37bebc0..ada17f0 100644 --- a/config.py +++ b/config.py @@ -46,7 +46,7 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " -rsync_update_command="rsync -av --delay-updates " +rsync_update_command="rsync -av --delete-after --delay-updates " # Classes and Exceptions class NonValidFile(ValueError): pass -- cgit v1.2.3-54-g00ecf From 125df5a815e0a327739e928cc765ffdcf4bd0aea Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sun, 6 Mar 2011 18:19:03 -0600 Subject: * Added function for cleanup dir --- pato2.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pato2.py b/pato2.py index 336c6fb..03afebd 100644 --- a/pato2.py +++ b/pato2.py @@ -129,6 +129,12 @@ def remove_from_blacklist(repo_,arch_,info_,blacklist_): a = commands.getoutput(com_) if verbose: printf(a) +def cleanup_nonfree_in_dir(directory,blacklisted_names): + pkgs=pkginfo_from_files_in_dir(directory) + for package in pkgs: + if package["name"] in blacklisted_names: + os.remove(package["location"]) + def link(repo_,arch_,file_): """ Makes a link in the repo for the package """ cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ -- cgit v1.2.3-54-g00ecf From 15ed2078e8743f87efcd63ac62cc0bfd5b39dc96 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sun, 6 Mar 2011 18:34:28 -0600 Subject: * Corrected import glob function --- filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filter.py b/filter.py index 2fe61f0..b6e8105 100644 --- a/filter.py +++ b/filter.py @@ -1,6 +1,6 @@ #! /usr/bin/python #-*- encoding: utf-8 -*- -import glob +from glob import glob from repm.config import * from repm.pato2 import * -- cgit v1.2.3-54-g00ecf From a41f5f44a8f812dfa2aacaf359bb51782d7a1633 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sun, 6 Mar 2011 19:02:14 -0600 Subject: * Fixed blacklist for rsync_command at sync_all_repo --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pato2.py b/pato2.py index 03afebd..18ace85 100644 --- a/pato2.py +++ b/pato2.py @@ -57,7 +57,7 @@ def sync_all_repo(debug=verbose): rsout=run_rsync(cmd) pkgs=pkginfo_from_rsync_output(rsout) generate_exclude_list_from_blacklist(pkgs,listado(blacklist),debug=False) - cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) + cmd=generate_rsync_command(rsync_update_command,blacklist_file=rsync_blacklist) a=run_rsync(cmd) if debug: printf(a) -- cgit v1.2.3-54-g00ecf From 0889827f252c9f6c21c1d4f2afc704b9b303e083 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Thu, 10 Mar 2011 13:59:35 -0600 Subject: * Fixed issue 71: pending list ends with newline --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pato2.py b/pato2.py index 18ace85..27a3962 100644 --- a/pato2.py +++ b/pato2.py @@ -111,7 +111,7 @@ def make_pending(repo_,arch_,info_): printf( pkg_ + " package has no %NAME% attibute " ) if verbose: printf( lista_ ) a=open( pending + "-" + repo_ + ".txt", "w" ).write( - "\n".join([name + ":" + license_ for (name,license_) in lista_]) ) + "\n".join([name + ":" + license_ for (name,license_) in lista_]) + "\n") def remove_from_blacklist(repo_,arch_,info_,blacklist_): """ Check the blacklist and remove packages on the db""" -- cgit v1.2.3-54-g00ecf From 47045b1934932b0695dd301a9c76b9dab1b03023 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 21 Mar 2011 17:30:40 -0600 Subject: * Changed sync_all_repo to adress bug83 --- config.py | 3 ++- filter.py | 3 ++- pato2.py | 6 +++++- test/core.db.tar.gz | Bin 1637 -> 1345 bytes test/test_filter.py | 23 +++++++++++++++++++++++ 5 files changed, 32 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index ada17f0..c218ddd 100644 --- a/config.py +++ b/config.py @@ -46,7 +46,8 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " -rsync_update_command="rsync -av --delete-after --delay-updates " +rsync_update_command="rsync -av --delay-updates --exclude='*.db.tar.{gz|xz}' " +rsync_post_command="rsync -av --delete " # Classes and Exceptions class NonValidFile(ValueError): pass diff --git a/filter.py b/filter.py index b6e8105..668822b 100644 --- a/filter.py +++ b/filter.py @@ -115,7 +115,8 @@ def pkginfo_from_files_in_dir(directory): package_list.append(pkginfo_from_filename(filename)) return tuple(package_list) - +def pkginfo_from_db(path_to_db): + """ """ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, exclude_file=rsync_blacklist, debug=verbose): diff --git a/pato2.py b/pato2.py index 27a3962..0d77d6b 100644 --- a/pato2.py +++ b/pato2.py @@ -59,7 +59,11 @@ def sync_all_repo(debug=verbose): generate_exclude_list_from_blacklist(pkgs,listado(blacklist),debug=False) cmd=generate_rsync_command(rsync_update_command,blacklist_file=rsync_blacklist) a=run_rsync(cmd) - if debug: printf(a) + cmd=generate_rsync_command(rsync_post_command,blacklist_file=rsync_blacklist) + b=run_rsync(cmd) + if debug: + printf(a) + printf(b) def get_from_desc(desc, var,db_tar_file=False): """ Get a var from desc file """ diff --git a/test/core.db.tar.gz b/test/core.db.tar.gz index a28ea64..5eb2081 100644 Binary files a/test/core.db.tar.gz and b/test/core.db.tar.gz differ diff --git a/test/test_filter.py b/test/test_filter.py index 5127b75..1906b87 100644 --- a/test/test_filter.py +++ b/test/test_filter.py @@ -163,7 +163,30 @@ class pkginfo_from_descKnownValues(unittest.TestCase): class pkginfo_from_db(unittest.TestCase): archdb = os.path.join("./workdir") + example_package_list=(Package(),Package(),Package()) + example_package_list[0].package_info={ "name" : "acl", + "version" : "2.2.49", + "release" : "2", + "arch" : "x86_64", + "license" : ("LGPL",), + "location": "acl-2.2.49-2-x86_64.pkg.tar.xz" + "depends" : ("attr>=2.4.41"),} + example_package_list[1].package_info={ "name" : "glibc", + "version" : "2.13", + "release" : "4", + "arch" : "x86_64", + "license" : ("GPL","LGPL"), + "location": "glibc-2.13-4-x86_64.pkg.tar.xz" + "depends" : ("linux-api-headers>=2.6.37","tzdata",),} + example_package_list[2].package_info={ "name" : "", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "" + "depends" : False,} if __name__ == "__main__": unittest.main() + -- cgit v1.2.3-54-g00ecf From fa11d216d567cec3c96c964829c51bd923b493c5 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 21 Mar 2011 17:49:24 -0600 Subject: * Don't update *.abs.tar.* --- config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index c218ddd..c643bb5 100644 --- a/config.py +++ b/config.py @@ -46,8 +46,8 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " -rsync_update_command="rsync -av --delay-updates --exclude='*.db.tar.{gz|xz}' " -rsync_post_command="rsync -av --delete " +rsync_update_command="rsync -av --delay-updates --exclude=*.{abs|db}.tar.* " +rsync_post_command="rsync -av --delete --exclude=*.abs.tar.* " # Classes and Exceptions class NonValidFile(ValueError): pass -- cgit v1.2.3-54-g00ecf From 72d3e1102acf8e90a1646d5c9afa6fdd01b2aeca Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Fri, 8 Apr 2011 11:40:32 -0500 Subject: Config ported to bash --- config.sh | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100755 config.sh diff --git a/config.sh b/config.sh new file mode 100755 index 0000000..aae3287 --- /dev/null +++ b/config.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# -*- coding: utf-8 -*- + +# Mirror options +mirror="mirrors.eu.kernel.org" +mirrorpath="::mirrors/archlinux" + +# Directories and files + +## Optionals +paraboladir=~/parabolagnulinux.org +logtime=$(date -u +%Y%m%d-%H:%M) + +## Must be defined +logname=${paraboladir}/${logtime}-repo-maintainer.log +tempdir=~/tmp/ +docs_dir=${paraboladir}/docs +repodir=${paraboladir}/repo + +# Repos, arches, and dirs for repo +repolist="core:extra:community:testing:community-testing:multilib" +dir_list="pool" +arch_list="i686:x86_64" +other="any" + +# Output options +output="True" +debug="False" + +# Rsync commands +rsync_list_command="rsync -a --no-motd --list-only " +rsync_update_command="rsync -av --delay-updates --exclude=*.{abs|db}.tar.* " +rsync_post_command="rsync -av --delete --exclude=*.abs.tar.* " + + +function run_python_cmd { + env \ + mirror=${mirror} \ + mirrorpath=${mirrorpath} \ + logname=${logname} \ + tempdir=${tempdir} \ + docs_dir=${docs_dir} \ + repodir=${repodir} \ + repolist=${repolist} \ + dir_list=${dir_list} \ + arch_list=${arch_list} \ + other=${other} \ + output=${output} \ + debug=${debug} \ + $1 +} \ No newline at end of file -- cgit v1.2.3-54-g00ecf From 2b721124f5122faae5709e5e2dd8035f392c1361 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sun, 10 Apr 2011 15:18:17 -0500 Subject: config.py import config.sh settings --- config.py | 72 ++++++++++++++++++++++++++++++--------------------------------- config.sh | 6 +++--- 2 files changed, 37 insertions(+), 41 deletions(-) diff --git a/config.py b/config.py index bd8ef1b..56a7c81 100644 --- a/config.py +++ b/config.py @@ -1,44 +1,36 @@ -#!/usr/bin/python +#!/usr/bin/pythonn # -*- coding: utf-8 -*- -from user import home -import commands +try: + from subprocess import check_output +except(ImportError): + from commands import getoutput as check_output +import os -time__ = commands.getoutput("date +%Y%m%d-%H:%M") +stringvars=("mirror", "mirrorpath", "logname", "tempdir", "docs_dir", + "repodir",) +listvars=("repo_list", "dir_list", "arch_list", "other",) +boolvars=("output", "debug",) -# Mirror Parameters -mirror = "mirrors.eu.kernel.org" -mirrorpath = "::mirrors/archlinux" +config=dict() -# Directories and files +def exit_if_none: + if os.environ.get(var) is None: + exit("%s is not defined" % var) -## Optionals -path = home + "/parabolagnulinux.org" -docs = path + "/docs" -logdir = path + "/log" - -## Must be defined -logname= logdir + "/" + time__ + "-repo-maintainer.log" -repodir= path + "/repo" -tmp = home + "/tmp" -archdb = tmp + "/db" - -free_path= path + "/free/" - -# Repo, arch, and other folders to use for repo -repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") -dir_list = ("pool","sources") -arch_list = ("i686", "x86_64") -other = ("any",) - -# Output -output = True -verbose = False - -# Files -blacklist = docs + "/blacklist.txt" -whitelist = docs + "/whitelist.txt" -pending = docs + "/pending" -rsync_blacklist = docs + "/rsyncBlacklist" +for var in stringvars: + exit_if_none(var) + config[var]=os.environ.get(var) +for var in listvars: + exit_if_none(var) + config[var]=tuple(os.environ.get(var).split(":")) +for var in boolvars: + exit_if_none(var) + if os.environ.get(var) == "True": + config[var]=True + elif os.environ.get(var) =="False": + config[var]=False + else: + print('%s is not True or False' % var) # Classes and Exceptions class NonValidFile(ValueError): pass @@ -55,7 +47,8 @@ class Package: "release" : False, "arch" : False, "license" : False, - "location": False} + "location": False, + "depends" : False,} def __setitem__(self, key, item): if key in self.package_info.keys(): @@ -76,8 +69,11 @@ class Package: if not isinstance(x, Package): return False for key in self.package_info.keys(): - if x[key] != self[key]: + if x[key] != self.package_info[key]: return False else: return True +if __name__=="__main__": + for key in config.keys(): + print("%s : %s" % (key,config[key])) diff --git a/config.sh b/config.sh index aae3287..66be477 100755 --- a/config.sh +++ b/config.sh @@ -1,6 +1,7 @@ #!/bin/sh # -*- coding: utf-8 -*- + # Mirror options mirror="mirrors.eu.kernel.org" mirrorpath="::mirrors/archlinux" @@ -18,7 +19,7 @@ docs_dir=${paraboladir}/docs repodir=${paraboladir}/repo # Repos, arches, and dirs for repo -repolist="core:extra:community:testing:community-testing:multilib" +repo_list="core:extra:community:testing:community-testing:multilib" dir_list="pool" arch_list="i686:x86_64" other="any" @@ -28,7 +29,6 @@ output="True" debug="False" # Rsync commands -rsync_list_command="rsync -a --no-motd --list-only " rsync_update_command="rsync -av --delay-updates --exclude=*.{abs|db}.tar.* " rsync_post_command="rsync -av --delete --exclude=*.abs.tar.* " @@ -41,7 +41,7 @@ function run_python_cmd { tempdir=${tempdir} \ docs_dir=${docs_dir} \ repodir=${repodir} \ - repolist=${repolist} \ + repo_list=${repo_list} \ dir_list=${dir_list} \ arch_list=${arch_list} \ other=${other} \ -- cgit v1.2.3-54-g00ecf From 69d84cc6edab249037522d00d3685939dc38495a Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sun, 10 Apr 2011 17:59:47 -0500 Subject: Firs main deliver --- config.py | 2 +- config.sh | 10 +++++--- libremessages | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ main.sh | 21 ++++++++++++++++ 4 files changed, 106 insertions(+), 4 deletions(-) create mode 100755 libremessages create mode 100644 main.sh diff --git a/config.py b/config.py index 39f512a..8cf07a2 100644 --- a/config.py +++ b/config.py @@ -7,7 +7,7 @@ except(ImportError): import os stringvars=("mirror", "mirrorpath", "logname", "tempdir", "docs_dir", - "repodir",) + "repodir", "rsync_blacklist") listvars=("repo_list", "dir_list", "arch_list", "other",) boolvars=("output", "debug",) diff --git a/config.sh b/config.sh index 66be477..60bd4ea 100755 --- a/config.sh +++ b/config.sh @@ -17,6 +17,7 @@ logname=${paraboladir}/${logtime}-repo-maintainer.log tempdir=~/tmp/ docs_dir=${paraboladir}/docs repodir=${paraboladir}/repo +rsync_blacklist=${docs_dir}/rsyncBlacklist # Repos, arches, and dirs for repo repo_list="core:extra:community:testing:community-testing:multilib" @@ -29,8 +30,8 @@ output="True" debug="False" # Rsync commands -rsync_update_command="rsync -av --delay-updates --exclude=*.{abs|db}.tar.* " -rsync_post_command="rsync -av --delete --exclude=*.abs.tar.* " +rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " +rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " function run_python_cmd { @@ -39,6 +40,7 @@ function run_python_cmd { mirrorpath=${mirrorpath} \ logname=${logname} \ tempdir=${tempdir} \ + rsync_blacklist=${rsync_blacklist} \ docs_dir=${docs_dir} \ repodir=${repodir} \ repo_list=${repo_list} \ @@ -48,4 +50,6 @@ function run_python_cmd { output=${output} \ debug=${debug} \ $1 -} \ No newline at end of file +} + +source libremessages \ No newline at end of file diff --git a/libremessages b/libremessages new file mode 100755 index 0000000..9fbbc2b --- /dev/null +++ b/libremessages @@ -0,0 +1,77 @@ +# Copyright (c) 2006-2010 Pacman Development Team +# Copyright (c) 2002-2006 by Judd Vinet +# Copyright (c) 2005 by Aurelien Foret +# Copyright (c) 2006 by Miklos Vajna +# Copyright (c) 2005 by Christian Hamar +# Copyright (c) 2006 by Alex Smith +# Copyright (c) 2006 by Andras Voroskoi +# Copyright (c) 2011 by Joshua Haase +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# gettext initialization +export TEXTDOMAIN='libretools' +export TEXTDOMAINDIR='/usr/share/locale' + +# check if messages are to be printed using color +unset ALL_OFF BOLD BLUE GREEN RED YELLOW + +if tput setaf 0 &>/dev/null; then + ALL_OFF="$(tput sgr0)" + BOLD="$(tput bold)" + BLUE="${BOLD}$(tput setaf 4)" + GREEN="${BOLD}$(tput setaf 2)" + RED="${BOLD}$(tput setaf 1)" + YELLOW="${BOLD}$(tput setaf 3)" + PURPLE="${ALL_OFF}$(tput setaf 5)" +else + ALL_OFF="\033[1;0m" + BOLD="\033[1;1m" + BLUE="${BOLD}\033[1;34m" + GREEN="${BOLD}\033[1;32m" + RED="${BOLD}\033[1;31m" + YELLOW="${BOLD}\033[1;33m" + PURPLE="${BOLD}\033[1;30;40m" +fi + +stdnull() { + local action=$1; + eval "${action} >/dev/null 2>&1" +} + +plain() { + local mesg=$1; shift + printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +msg() { + local mesg=$1; shift + printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +msg2() { + local mesg=$1; shift + printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +warning() { + local mesg=$1; shift + printf "${YELLOW}==> $(gettext "WARNING:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +error() { + local mesg=$1; shift + printf "${RED}==> $(gettext "ERROR:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + diff --git a/main.sh b/main.sh new file mode 100644 index 0000000..ac504a4 --- /dev/null +++ b/main.sh @@ -0,0 +1,21 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +source config.sh + +function mkrsexclude { + error=1 + while ${error}; do + run_python_cmd "filter.py" + error=$? + done +} + +msg "Cleaning $tempdir" +stdnull "rm -r $tempdir/* " + +msg "Generating exclude list for rsync" +mkrsexclude + +msg "Syncing repo files without delete" +${rsync_update_command} --exclude-from=${rsync_blacklist} ${mirror}${mirropath}/ \ No newline at end of file -- cgit v1.2.3-54-g00ecf From 61ee5da8b31b44e80e008619a32ca886d8799646 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sun, 10 Apr 2011 19:27:12 -0500 Subject: Replaced pato2.py functionality in main.sh --- main.sh | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/main.sh b/main.sh index ac504a4..2d59094 100644 --- a/main.sh +++ b/main.sh @@ -4,7 +4,7 @@ source config.sh function mkrsexclude { - error=1 + local error=1 while ${error}; do run_python_cmd "filter.py" error=$? @@ -17,5 +17,19 @@ stdnull "rm -r $tempdir/* " msg "Generating exclude list for rsync" mkrsexclude -msg "Syncing repo files without delete" -${rsync_update_command} --exclude-from=${rsync_blacklist} ${mirror}${mirropath}/ \ No newline at end of file +msg "Syncing repos without delete" +# rsync_update_command does not sync db or abs +${rsync_update_command} --exclude-from=${rsync_blacklist} \ + ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ',')} ${repodir} + +msg "Syncing each repo and cleaning" +for repo in $(echo ${repo_list} | tr ':' ' '); do + msg2 "Syncing ${repo}" + ${rsync_post_command} --exclude-from=${rsync_blacklist} \ + ${mirror}${mirropath}/${repo} ${repodir}/${repo} + msg2 "Cleaning ${repo}" + clean-repo.py -d ${repodir}/${repo} \ + -b ${repodir}/${repo}/${repo}.db.tar.gz + msg2 "Making pending list for ${repo}" + run_python_cmd "mkpending.py -r ${repo} -d ${repodir}/${repo}" +done -- cgit v1.2.3-54-g00ecf From 3fb5f8efef231dd7784be880934cd106603ab6f1 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 11 Apr 2011 01:09:25 -0500 Subject: bash-port ready for testing. --- clean_repo.py | 45 +++++++++++ config.py | 10 ++- config.sh | 17 +++-- filter.py | 68 +++++++++++++---- get_license.sh | 10 +-- main.sh | 20 +++-- mkpending.py | 45 +++++++++++ pato2.py | 209 ++++------------------------------------------------ test/test_filter.py | 6 +- 9 files changed, 198 insertions(+), 232 deletions(-) create mode 100644 clean_repo.py create mode 100644 mkpending.py diff --git a/clean_repo.py b/clean_repo.py new file mode 100644 index 0000000..29d446d --- /dev/null +++ b/clean_repo.py @@ -0,0 +1,45 @@ +#! /usr/bin/python +#-*- encoding: utf-8 -*- +from repm.filter import * +import argparse + +def remove_from_blacklist(path_to_db, blacklisted_names, + debug=config["debug"]): + """ Check the blacklist and remove packages on the db""" + + pkgs=[pkg for pkg in pkginfo_from_db(path_to_db) if + pkg["name"] in blacklisted_names] + if pkgs: + lista=" ".join(pkgs) + cmd = "repo-remove " + path_to_db + " " + lista + printf(cmd) + a = check_output(cmd) + if debug: + printf(a) + return pkgs, cmd + +def cleanup_nonfree_in_dir(directory, blacklisted_names): + pkgs=pkginfo_from_files_in_dir(directory) + for package in pkgs: + if package["name"] in blacklisted_names: + os.remove(package["location"]) + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Clean a repo db and packages") + parser.add_argument("-b", "--database", type=str, + help="dabatase to clean") + parser.add_argument("-d", "--directory", type=str, + help="directory to clean") + args=parser.parse_args() + + if args.directory: + cleanup_nonfree_in_dir(args.database, listado(config["blacklist"])) + + if args.database: + pkgs=pkginfo_from_db(args.database) + remove_from_blacklist(args.database, pkgs, + tuple(listado(config["blacklist"]) + + listado(config["pending"]))) + if not args.directory and not args.database: + parser.print_help() diff --git a/config.py b/config.py index 8cf07a2..24ecfaf 100644 --- a/config.py +++ b/config.py @@ -1,4 +1,4 @@ -#!/usr/bin/pythonn +#!/usr/bin/python # -*- coding: utf-8 -*- try: from subprocess import check_output @@ -7,22 +7,24 @@ except(ImportError): import os stringvars=("mirror", "mirrorpath", "logname", "tempdir", "docs_dir", - "repodir", "rsync_blacklist") + "repodir", "rsync_blacklist") listvars=("repo_list", "dir_list", "arch_list", "other",) boolvars=("output", "debug",) config=dict() -def exit_if_none: +def exit_if_none(var): if os.environ.get(var) is None: exit("%s is not defined" % var) for var in stringvars: exit_if_none(var) - config[var]=os.environ.get(var) + config[var]=os.environ.get(var) + for var in listvars: exit_if_none(var) config[var]=tuple(os.environ.get(var).split(":")) + for var in boolvars: exit_if_none(var) if os.environ.get(var) == "True": diff --git a/config.sh b/config.sh index 60bd4ea..741dee4 100755 --- a/config.sh +++ b/config.sh @@ -1,22 +1,25 @@ #!/bin/sh # -*- coding: utf-8 -*- - # Mirror options mirror="mirrors.eu.kernel.org" mirrorpath="::mirrors/archlinux" -# Directories and files - +# Directories ## Optionals paraboladir=~/parabolagnulinux.org logtime=$(date -u +%Y%m%d-%H:%M) - ## Must be defined logname=${paraboladir}/${logtime}-repo-maintainer.log tempdir=~/tmp/ docs_dir=${paraboladir}/docs repodir=${paraboladir}/repo +# End Directories + +# Files +blacklist=${docs_dir}/blacklist.txt +whitelist=${docs_dir}/whitelist.txt +pending=${docs_dir}/pending rsync_blacklist=${docs_dir}/rsyncBlacklist # Repos, arches, and dirs for repo @@ -33,16 +36,18 @@ debug="False" rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " - function run_python_cmd { env \ mirror=${mirror} \ mirrorpath=${mirrorpath} \ logname=${logname} \ tempdir=${tempdir} \ - rsync_blacklist=${rsync_blacklist} \ docs_dir=${docs_dir} \ repodir=${repodir} \ + blacklist=${blacklist} \ + whitelist=${whitelist} \ + pending=${pending} \ + rsync_blacklist=${rsync_blacklist} \ repo_list=${repo_list} \ dir_list=${dir_list} \ arch_list=${arch_list} \ diff --git a/filter.py b/filter.py index 668822b..1a0fa6f 100644 --- a/filter.py +++ b/filter.py @@ -4,6 +4,13 @@ from glob import glob from repm.config import * from repm.pato2 import * +def listado(filename): + """Obtiene una lista de paquetes de un archivo.""" + archivo = open(filename,"r") + lista = archivo.read().split("\n") + archivo.close() + return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] + def pkginfo_from_filename(filename): """ Generates a Package object with info from a filename, filename can be relative or absolute @@ -116,10 +123,48 @@ def pkginfo_from_files_in_dir(directory): return tuple(package_list) def pkginfo_from_db(path_to_db): - """ """ + """ Get PKGINFO from db. + + Parameters: + ---------- + path_to_db -> str Path to file -def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, - exclude_file=rsync_blacklist, debug=verbose): + Output: + ---------- + None """ + package_list=list() + + if not os.path.isfile(path_to_db): + raise NonValidFile(path_to_db + "is not a file") + + check_output("mkdir -p " + archdb) + + try: + db_open_tar = tarfile.open(db_tar_file, 'r:gz') + except tarfile.ReadError: + printf("No valid db_file %s or not readable" % db_tar_file) + return(tuple()) + else: + printf("No db_file %s" % db_tar_file) + return(tuple()) + + for file in db_open_tar.getmembers(): + db_open_tar.extract(file, archdb) + db_open_tar.close() + # Get info from file + for dir_ in glob(archdb + "/*"): + if isdir(dir_) and isfile(dir_ + "/desc"): + package_list.append(pkginfo_from_desc( + os.path.join(dir_,"desc"))) + check_output("rm -r %s/*" % archdb) + if verbose_: + printf(package_list) + return package_list + +def generate_exclude_list_from_blacklist(packages_iterable, + blacklisted_names, + exclude_file=config["rsync_blacklist"], + debug=config["debug"]): """ Generate an exclude list for rsync Parameters: @@ -132,16 +177,12 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, Output: ---------- None """ - a=list() - - for package in packages_iterable: - if not isinstance(package, Package): - raise ValueError(" %s is not a Package object " % package) - if package["name"] in blacklisted_names: - a.append(package["location"]) + pkgs=[pkg["location"] for pkg in packages_iterable + if isinstance(pkg, Package) + and pkg["name"] in blacklisted_names] if debug: - return a + return pkgs try: fsock = open(exclude_file,"w") try: @@ -149,9 +190,10 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, finally: fsock.close() except IOError: - printf("%s wasnt written" % blacklist_file) + printf("%s wasnt written" % exclude_file) if __name__ == "__main__": - a=run_rsync(rsync_list_command) + cmd=generate_rsync_command(rsync_list_command) + a=run_rsync(cmd) packages=pkginfo_from_rsync_output(a) generate_exclude_list_from_blacklist(packages,listado(blacklist)) diff --git a/get_license.sh b/get_license.sh index a7241a1..0da58cb 100755 --- a/get_license.sh +++ b/get_license.sh @@ -31,12 +31,12 @@ rm -rf $dir/* tempdir=$(mktemp -d) cd $tempdir -a=($(cut -d: -f1 $docs/pending*.txt)) -echo ${a[@]} +pending=($(cut -d: -f1 $docs/pending*.txt)) +echo ${pending[@]} -for x in ${a[@]}; do - b=( $(ls $repo/*/os/*/$x*) ) - for y in ${b[@]}; do +for pkg in ${pending[@]}; do + pkg_in_repo=( $(ls ${repo}/*/os/*/${pkg}*) ) + for y in ${pkg_in_repo[@]}; do echo "chmod +r $y" chmod +r $y echo "tar -xf $y usr/share/licenses" diff --git a/main.sh b/main.sh index 2d59094..1a2c6c4 100644 --- a/main.sh +++ b/main.sh @@ -23,13 +23,17 @@ ${rsync_update_command} --exclude-from=${rsync_blacklist} \ ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ',')} ${repodir} msg "Syncing each repo and cleaning" +msg2 "Remove pending files" +stdnull "rm -rf ${pending}*" for repo in $(echo ${repo_list} | tr ':' ' '); do - msg2 "Syncing ${repo}" - ${rsync_post_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/${repo} ${repodir}/${repo} - msg2 "Cleaning ${repo}" - clean-repo.py -d ${repodir}/${repo} \ - -b ${repodir}/${repo}/${repo}.db.tar.gz - msg2 "Making pending list for ${repo}" - run_python_cmd "mkpending.py -r ${repo} -d ${repodir}/${repo}" + for arch in $(echo ${arch_list} | tr ':' ' '); do + msg2 "Syncing ${repo} ${arch}" + ${rsync_post_command} --exclude-from=${rsync_blacklist} \ + ${mirror}${mirropath}/${repo} ${repodir}/${repo} + msg2 "Making pending list for ${repo} ${arch}" + run_python_cmd "mkpending.py -r ${repo} -b ${repodir}/${repo}/os/${arch}" + msg2 "Cleaning ${repo} ${arch}" + run_python_cmd "clean-repo.py -b ${repodir}/${repo}/os/${arch}/${repo}.db.tar.gz -d ${repodir}/${repo}/os/${arch}/" + get_license.sh + done done diff --git a/mkpending.py b/mkpending.py new file mode 100644 index 0000000..43a5fb2 --- /dev/null +++ b/mkpending.py @@ -0,0 +1,45 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +from repm.filter import * + +def make_pending(path_to_db): + """ Determine wich packages are pending for license auditing.""" + packages_iterable=pkginfo_from_db(path_to_db) + search = tuple(listado(config["blacklist"]) + + listado(config["whitelist"])) + + pkgs=[pkg for pkg in packages_iterable + if "custom" in pkg["license"] + and pkg["name"] not in search] + return pkgs + +def write_pending(packages_iterable, repo, prefix=config["pending"]): + """ Write a pending file with the info of the packages """ + filename=prefix + "-" + repo + ".txt" + try: + fsock=open(filename, "a") + except(IOError): + print("Can't read %s" % filename) + finally: + fsock.close() + if os.path.isfile(filename): + pkgs=[pkg for pkg in packages_iterable if pkg["name"] not in + listado(filename)] + fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] + for pkg in pkgs]) + "\n") + fsock.close() + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Clean a repo db and packages") + parser.add_argument("-b", "--dababase", type=str, required=True + help="database to check") + parser.add_argument("-r", "--repo", type=str, required=True + help="repo of database") + args=parser.parse_args() + + if args.database and args.repo: + pkgs=make_pending(args.database) + write_pending(pkgs, args.repo) + else: + parser.print_help() diff --git a/pato2.py b/pato2.py index 0d77d6b..4cdb536 100644 --- a/pato2.py +++ b/pato2.py @@ -27,159 +27,23 @@ from repm.config import * from repm.filter import * import tarfile -from glob import glob from os.path import isdir, isfile -def printf(text,output_=output): +def printf(text,output=config["output"]): """Guarda el texto en la variable log y puede imprimir en pantalla.""" - log_file = open(logname, 'a') + log_file = open(config["logname"], 'a') log_file.write("\n" + str(text) + "\n") log_file.close() - if output_: print (str(text) + "\n") - -def listado(filename_): - """Obtiene una lista de paquetes de un archivo.""" - archivo = open(filename_,"r") - lista = archivo.read().split("\n") - archivo.close() - return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] - -def db(repo_,arch_): - """Construye un nombre para sincronizar una base de datos.""" - return "/%s/os/%s/%s.db.tar.gz" % (repo_, arch_, repo_) - -def packages(repo_, arch_, expr="*"): - """ Get packages on a repo, arch folder """ - return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) ) - -def sync_all_repo(debug=verbose): - cmd=generate_rsync_command(rsync_list_command) - rsout=run_rsync(cmd) - pkgs=pkginfo_from_rsync_output(rsout) - generate_exclude_list_from_blacklist(pkgs,listado(blacklist),debug=False) - cmd=generate_rsync_command(rsync_update_command,blacklist_file=rsync_blacklist) - a=run_rsync(cmd) - cmd=generate_rsync_command(rsync_post_command,blacklist_file=rsync_blacklist) - b=run_rsync(cmd) - if debug: - printf(a) - printf(b) - -def get_from_desc(desc, var,db_tar_file=False): - """ Get a var from desc file """ - desc = desc.split("\n") - return desc[desc.index(var)+1] - -def get_info(repo_,arch_,db_tar_file=False,verbose_=verbose): - """ Makes a list of package name, file and license """ - info=list() - # Extract DB tar.gz - commands.getoutput("mkdir -p " + archdb) - if not db_tar_file: - db_tar_file = repodir + db(repo_,arch_) - if isfile(db_tar_file): - try: - db_open_tar = tarfile.open(db_tar_file, 'r:gz') - except tarfile.ReadError: - printf("No valid db_file %s" % db_tar_file) - return(tuple()) - else: - printf("No db_file %s" % db_tar_file) - return(tuple()) - for file in db_open_tar.getmembers(): - db_open_tar.extract(file, archdb) - db_open_tar.close() - # Get info from file - for dir_ in glob(archdb + "/*"): - if isdir(dir_) and isfile(dir_ + "/desc"): - pkg_desc_file = open(dir_ + "/desc", "r") - desc = pkg_desc_file.read() - pkg_desc_file.close() - info.append(( get_from_desc(desc,"%NAME%"), - dir_.split("/")[-1], - get_from_desc(desc,"%LICENSE%") )) - if verbose_: printf(info) - commands.getoutput("rm -r %s/*" % archdb) - return tuple(info) - -def make_pending(repo_,arch_,info_): - """ Si los paquetes no están en blacklist ni whitelist y la licencia contiene "custom" los agrega a pending""" - search = tuple( listado(blacklist) + listado (whitelist) ) - if verbose: printf("blaclist + whitelist= " + str(search) ) - lista_=list() - for (name,pkg_,license_) in info_: - if "custom" in license_: - if name not in search: - lista_.append( (name, license_ ) ) - elif not name: - printf( pkg_ + " package has no %NAME% attibute " ) - if verbose: printf( lista_ ) - a=open( pending + "-" + repo_ + ".txt", "w" ).write( - "\n".join([name + ":" + license_ for (name,license_) in lista_]) + "\n") - -def remove_from_blacklist(repo_,arch_,info_,blacklist_): - """ Check the blacklist and remove packages on the db""" - lista_=list() - pack_=list() - for (name_, pkg_, license_) in info_: - if name_ in blacklist_: - lista_.append(name_) - for p in packages(repo_,arch_,pkg_ + "*"): - pack_.append(p) - if lista_: - lista_=" ".join(lista_) - com_ = "repo-remove " + repodir + db(repo_,arch_) + " " + lista_ - printf(com_) - a = commands.getoutput(com_) - if verbose: printf(a) - -def cleanup_nonfree_in_dir(directory,blacklisted_names): - pkgs=pkginfo_from_files_in_dir(directory) - for package in pkgs: - if package["name"] in blacklisted_names: - os.remove(package["location"]) - -def link(repo_,arch_,file_): - """ Makes a link in the repo for the package """ - cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ - a=commands.getoutput(cmd_) - if verbose: - printf(cmd_ + a) - -def add_free_repo(verbose_=verbose): - cmd_=os.path.join(home,"/usr/bin/sync-free") - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose_: printf(a) - for repo_ in repo_list: - for arch_ in arch_list: - lista_=list() - for file_ in glob(freedir + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"): - lista_.append(file_) - for dir_ in other: - for file_ in glob(freedir + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"): - lista_.append(file_) - - printf(lista_) - - if lista_: - lista_=" ".join(lista_) - if verbose: printf(lista_) - cmd_="repo-add " + repodir + db(repo_,arch_) + " " + lista_ - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose: printf(a) - -def get_licenses(verbose_=verbose): - """ Extract the license from packages in repo_,arch_ and in pending_ file""" - cmd_=home + "/usr/bin/get_license.sh" - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose_: printf(a) - -def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdir=repodir, - source=mirror+mirrorpath, blacklist_file=False): - """ Generates an rsync command for executing it by combining all parameters. + if output_: + print (str(text) + "\n") + +def generate_rsync_command(base_command, + dir_list=(config["repo_list"] + + config["dir_list"]), + destdir=config["repodir"], + source=config["mirror"] +config["mirrorpath"]): + """ Generates an rsync command for executing + it by combining all parameters. Parameters: ---------- @@ -192,57 +56,16 @@ def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdi Return: ---------- rsync_command -> str """ - from os.path import isfile, isdir - - if blacklist_file and not isfile(blacklist_file): - print(blacklist_file + " is not a file") - raise NonValidFile - if not os.path.isdir(destdir): print(destdir + " is not a directory") raise NonValidDir dir_list="{" + ",".join(dir_list) + "}" + return " ".join((base_command, os.path.join(source, dir_list), + destdir)) - if blacklist_file: - return " ".join((base_command, "--exclude-from="+blacklist_file, - os.path.join(source, dir_list), destdir)) - return " ".join((base_command, os.path.join(source, dir_list), destdir)) - -def run_rsync(command,debug=verbose): +def run_rsync(command,debug=config["debug"]): """ Runs rsync and gets returns it's output """ if debug: printf("rsync_command: " + command) - return commands.getoutput(command) - -if __name__ == "__main__": - from time import time - start_time = time() - def minute(): - return str(round((time() - start_time)/60, 1)) - - printf(" Cleaning %s folder " % (tmp) ) - commands.getoutput("rm -r %s/*" % tmp) - printf(" Syncing repo") - sync_all_repo(True) - - printf(" Updating databases and pending files lists: minute %s \n" % minute() ) - for repo in repo_list: - for arch in arch_list: - printf( "\n" + repo + "-" + arch + "\n" ) - printf( "Get info: minute %s " % minute() ) - info=get_info(repo,arch) - printf( "Make pending: minute %s" % minute() ) - make_pending(repo,arch,info) - printf( "Update DB: minute %s" % minute() ) - remove_from_blacklist( - repo, arch, info, tuple( listado(blacklist) + listado(pending + "-" + repo + ".txt") ) ) - - printf("Adding Parabola Packages: minute %s\n" % minute() ) - add_free_repo(True) - - printf("Extracting licenses in pending: minute %s" % minute() ) - get_licenses() - - printf("\n\nDelay: %s minutes \n" % minute()) - + return check_output(command) diff --git a/test/test_filter.py b/test/test_filter.py index 1906b87..5601d57 100644 --- a/test/test_filter.py +++ b/test/test_filter.py @@ -169,21 +169,21 @@ class pkginfo_from_db(unittest.TestCase): "release" : "2", "arch" : "x86_64", "license" : ("LGPL",), - "location": "acl-2.2.49-2-x86_64.pkg.tar.xz" + "location": "acl-2.2.49-2-x86_64.pkg.tar.xz", "depends" : ("attr>=2.4.41"),} example_package_list[1].package_info={ "name" : "glibc", "version" : "2.13", "release" : "4", "arch" : "x86_64", "license" : ("GPL","LGPL"), - "location": "glibc-2.13-4-x86_64.pkg.tar.xz" + "location": "glibc-2.13-4-x86_64.pkg.tar.xz", "depends" : ("linux-api-headers>=2.6.37","tzdata",),} example_package_list[2].package_info={ "name" : "", "version" : "2.2.26", "release" : "1", "arch" : "x86_64", "license" : False, - "location": "" + "location": "", "depends" : False,} -- cgit v1.2.3-54-g00ecf From e0e4837f17f22b7bfafafc8d0e6f7a351249417e Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 11 Apr 2011 01:19:32 -0500 Subject: FixedFixed a little main.sh --- main.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/main.sh b/main.sh index 1a2c6c4..cf7bcd8 100644 --- a/main.sh +++ b/main.sh @@ -33,7 +33,9 @@ for repo in $(echo ${repo_list} | tr ':' ' '); do msg2 "Making pending list for ${repo} ${arch}" run_python_cmd "mkpending.py -r ${repo} -b ${repodir}/${repo}/os/${arch}" msg2 "Cleaning ${repo} ${arch}" - run_python_cmd "clean-repo.py -b ${repodir}/${repo}/os/${arch}/${repo}.db.tar.gz -d ${repodir}/${repo}/os/${arch}/" - get_license.sh + run_python_cmd "clean_repo.py -b ${repodir}/${repo}/os/${arch}/${repo}.db.tar.gz -d ${repodir}/${repo}/os/${arch}/" done done + +msg "Checking licenses" +get_license.sh -- cgit v1.2.3-54-g00ecf From f6184ee3c9422c40effb0265527ee5a31b027a1a Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernandez Date: Sun, 10 Apr 2011 23:44:01 -0700 Subject: chmod +x --- .gitignore | 3 ++- clean_repo.py | 0 config.py | 0 filter.py | 0 main.sh | 2 +- mkpending.py | 0 6 files changed, 3 insertions(+), 2 deletions(-) mode change 100644 => 100755 clean_repo.py mode change 100644 => 100755 config.py mode change 100644 => 100755 filter.py mode change 100644 => 100755 main.sh mode change 100644 => 100755 mkpending.py diff --git a/.gitignore b/.gitignore index e645833..cfefcc3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *~ -*.pyc \ No newline at end of file +*.pyc +cptobin.sh diff --git a/clean_repo.py b/clean_repo.py old mode 100644 new mode 100755 diff --git a/config.py b/config.py old mode 100644 new mode 100755 diff --git a/filter.py b/filter.py old mode 100644 new mode 100755 diff --git a/main.sh b/main.sh old mode 100644 new mode 100755 index cf7bcd8..886ea38 --- a/main.sh +++ b/main.sh @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/bin/bash # -*- coding: utf-8 -*- source config.sh diff --git a/mkpending.py b/mkpending.py old mode 100644 new mode 100755 -- cgit v1.2.3-54-g00ecf From a5f24954011da556977859d8a306ddb39c1ec2e4 Mon Sep 17 00:00:00 2001 From: Joshua Haase Date: Sun, 10 Apr 2011 23:49:50 -0700 Subject: Added cptobin --- .gitignore | 2 +- cptobin.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100755 cptobin.sh diff --git a/.gitignore b/.gitignore index cfefcc3..eef29c1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ *~ *.pyc -cptobin.sh + diff --git a/cptobin.sh b/cptobin.sh new file mode 100755 index 0000000..25a22d4 --- /dev/null +++ b/cptobin.sh @@ -0,0 +1,2 @@ +#!/bin/bash +cp -f clean_repo.py config.py config.sh main.sh mkpending.py filter.py ~/usr/bin/ \ No newline at end of file -- cgit v1.2.3-54-g00ecf From 67038ba1840d0f57b0ce49fdabd3dfa8057e2451 Mon Sep 17 00:00:00 2001 From: Joshua Haase Date: Mon, 11 Apr 2011 00:22:06 -0700 Subject: fixed some errors --- clean_repo.py | 2 +- config.py | 4 ++-- config.sh | 2 ++ cptobin.sh | 2 +- filter.py | 8 ++++---- mkpending.py | 4 ++-- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/clean_repo.py b/clean_repo.py index 29d446d..b0d306f 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -34,7 +34,7 @@ if __name__ == "__main__": args=parser.parse_args() if args.directory: - cleanup_nonfree_in_dir(args.database, listado(config["blacklist"])) + cleanup_nonfree_in_dir(args.directory, listado(config["blacklist"])) if args.database: pkgs=pkginfo_from_db(args.database) diff --git a/config.py b/config.py index 24ecfaf..f4089f9 100755 --- a/config.py +++ b/config.py @@ -6,8 +6,8 @@ except(ImportError): from commands import getoutput as check_output import os -stringvars=("mirror", "mirrorpath", "logname", "tempdir", "docs_dir", - "repodir", "rsync_blacklist") +stringvars=("mirror", "mirrorpath", "logname", "tempdir", "archdb", "docs_dir", + "repodir", "blacklist", "whitelist", "pending", "rsync_blacklist") listvars=("repo_list", "dir_list", "arch_list", "other",) boolvars=("output", "debug",) diff --git a/config.sh b/config.sh index 741dee4..4f42e0b 100755 --- a/config.sh +++ b/config.sh @@ -12,6 +12,7 @@ logtime=$(date -u +%Y%m%d-%H:%M) ## Must be defined logname=${paraboladir}/${logtime}-repo-maintainer.log tempdir=~/tmp/ +archdb=${tempdir}/db docs_dir=${paraboladir}/docs repodir=${paraboladir}/repo # End Directories @@ -42,6 +43,7 @@ function run_python_cmd { mirrorpath=${mirrorpath} \ logname=${logname} \ tempdir=${tempdir} \ + archdb=${archdb} \ docs_dir=${docs_dir} \ repodir=${repodir} \ blacklist=${blacklist} \ diff --git a/cptobin.sh b/cptobin.sh index 25a22d4..e803132 100755 --- a/cptobin.sh +++ b/cptobin.sh @@ -1,2 +1,2 @@ #!/bin/bash -cp -f clean_repo.py config.py config.sh main.sh mkpending.py filter.py ~/usr/bin/ \ No newline at end of file +cp -f clean_repo.py config.py config.sh get_license.sh main.sh mkpending.py filter.py ~/usr/bin/ \ No newline at end of file diff --git a/filter.py b/filter.py index 1a0fa6f..78ad410 100755 --- a/filter.py +++ b/filter.py @@ -137,15 +137,15 @@ def pkginfo_from_db(path_to_db): if not os.path.isfile(path_to_db): raise NonValidFile(path_to_db + "is not a file") - check_output("mkdir -p " + archdb) + check_output("mkdir -p " + config["archdb"]) try: - db_open_tar = tarfile.open(db_tar_file, 'r:gz') + db_open_tar = tarfile.open(path_to_db, 'r:gz') except tarfile.ReadError: - printf("No valid db_file %s or not readable" % db_tar_file) + printf("No valid db_file %s or not readable" % path_to_db) return(tuple()) else: - printf("No db_file %s" % db_tar_file) + printf("No db_file %s" % path_to_db) return(tuple()) for file in db_open_tar.getmembers(): diff --git a/mkpending.py b/mkpending.py index 43a5fb2..2b255a8 100755 --- a/mkpending.py +++ b/mkpending.py @@ -32,9 +32,9 @@ def write_pending(packages_iterable, repo, prefix=config["pending"]): if __name__ == "__main__": parser = argparse.ArgumentParser( description="Clean a repo db and packages") - parser.add_argument("-b", "--dababase", type=str, required=True + parser.add_argument("-b", "--dababase", type=str, required=True, help="database to check") - parser.add_argument("-r", "--repo", type=str, required=True + parser.add_argument("-r", "--repo", type=str, required=True, help="repo of database") args=parser.parse_args() -- cgit v1.2.3-54-g00ecf From 748600bf8dfba34b336ad642a6b26dae674db85f Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 11 Apr 2011 00:25:53 -0700 Subject: Will rsync be better now? --- config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index c643bb5..3e00eb0 100644 --- a/config.py +++ b/config.py @@ -46,8 +46,8 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " -rsync_update_command="rsync -av --delay-updates --exclude=*.{abs|db}.tar.* " -rsync_post_command="rsync -av --delete --exclude=*.abs.tar.* " +rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " +rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " # Classes and Exceptions class NonValidFile(ValueError): pass -- cgit v1.2.3-54-g00ecf From 857ecbe794c714919612b533303e7a9ef781c75b Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 11 Apr 2011 10:26:40 -0500 Subject: Use non tracked file local_config for config. --- .gitignore | 2 +- config.sh | 33 +-------------------------------- local_config.example | 31 +++++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 33 deletions(-) create mode 100644 local_config.example diff --git a/.gitignore b/.gitignore index eef29c1..69dec40 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ *~ *.pyc - +local_config diff --git a/config.sh b/config.sh index 4f42e0b..75d4b73 100755 --- a/config.sh +++ b/config.sh @@ -1,37 +1,6 @@ #!/bin/sh # -*- coding: utf-8 -*- - -# Mirror options -mirror="mirrors.eu.kernel.org" -mirrorpath="::mirrors/archlinux" - -# Directories -## Optionals -paraboladir=~/parabolagnulinux.org -logtime=$(date -u +%Y%m%d-%H:%M) -## Must be defined -logname=${paraboladir}/${logtime}-repo-maintainer.log -tempdir=~/tmp/ -archdb=${tempdir}/db -docs_dir=${paraboladir}/docs -repodir=${paraboladir}/repo -# End Directories - -# Files -blacklist=${docs_dir}/blacklist.txt -whitelist=${docs_dir}/whitelist.txt -pending=${docs_dir}/pending -rsync_blacklist=${docs_dir}/rsyncBlacklist - -# Repos, arches, and dirs for repo -repo_list="core:extra:community:testing:community-testing:multilib" -dir_list="pool" -arch_list="i686:x86_64" -other="any" - -# Output options -output="True" -debug="False" +source local_config # Rsync commands rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " diff --git a/local_config.example b/local_config.example new file mode 100644 index 0000000..7edfde7 --- /dev/null +++ b/local_config.example @@ -0,0 +1,31 @@ +# Mirror options +mirror="mirrors.eu.kernel.org" +mirrorpath="::mirrors/archlinux" + +# Directories +## Optionals +paraboladir=~/parabolagnulinux.org +logtime=$(date -u +%Y%m%d-%H:%M) +## Must be defined +logname=${paraboladir}/${logtime}-repo-maintainer.log +tempdir=~/tmp/ +archdb=${tempdir}/db +docs_dir=${paraboladir}/docs +repodir=${paraboladir}/repo +# End Directories + +# Files +blacklist=${docs_dir}/blacklist.txt +whitelist=${docs_dir}/whitelist.txt +pending=${docs_dir}/pending +rsync_blacklist=${docs_dir}/rsyncBlacklist + +# Repos, arches, and dirs for repo +repo_list="core:extra:community:testing:community-testing:multilib" +dir_list="pool" +arch_list="i686:x86_64" +other="any" + +# Output options +output="True" +debug="False" -- cgit v1.2.3-54-g00ecf From 3e27d11f68571bce92138f6cbfcaecac75fa1644 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Mon, 11 Apr 2011 12:39:40 -0500 Subject: Fixed some errors --- clean_repo.py | 6 +++++- config.py | 35 +++++++++++++++++++++++++++++------ config.sh | 5 ----- cptobin.sh | 2 +- filter.py | 32 +++++++++++++++++--------------- local_config.example | 10 +++++++--- main.sh | 3 ++- mkpending.py | 35 ++++++++++++++++++++--------------- pato2.py | 10 +--------- 9 files changed, 82 insertions(+), 56 deletions(-) diff --git a/clean_repo.py b/clean_repo.py index b0d306f..d4e06fc 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -6,7 +6,9 @@ import argparse def remove_from_blacklist(path_to_db, blacklisted_names, debug=config["debug"]): """ Check the blacklist and remove packages on the db""" - + if "~" in path_to_db: + path_to_db=(os.path.expanduser(path_to_db)) + pkgs=[pkg for pkg in pkginfo_from_db(path_to_db) if pkg["name"] in blacklisted_names] if pkgs: @@ -19,6 +21,8 @@ def remove_from_blacklist(path_to_db, blacklisted_names, return pkgs, cmd def cleanup_nonfree_in_dir(directory, blacklisted_names): + if "~" in directory: + directory=(os.path.expanduser(directory)) pkgs=pkginfo_from_files_in_dir(directory) for package in pkgs: if package["name"] in blacklisted_names: diff --git a/config.py b/config.py index f4089f9..8b48c66 100755 --- a/config.py +++ b/config.py @@ -3,11 +3,15 @@ try: from subprocess import check_output except(ImportError): - from commands import getoutput as check_output + from commands import getoutput + def check_output(*popenargs,**kwargs): + cmd=" ".join(*popenargs) + return getoutput(cmd) import os -stringvars=("mirror", "mirrorpath", "logname", "tempdir", "archdb", "docs_dir", - "repodir", "blacklist", "whitelist", "pending", "rsync_blacklist") +stringvars=("mirror", "mirrorpath", "logname", "tempdir", "archdb", + "repodir", "blacklist", "whitelist", "pending", + "rsync_blacklist",) listvars=("repo_list", "dir_list", "arch_list", "other",) boolvars=("output", "debug",) @@ -37,10 +41,29 @@ for var in boolvars: # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " +def printf(text,output=config["output"]): + """Guarda el texto en la variable log y puede imprimir en pantalla.""" + log_file = open(config["logname"], 'a') + log_file.write("\n" + str(text) + "\n") + log_file.close() + if output: + print (str(text) + "\n") + +del exit_if_none + # Classes and Exceptions -class NonValidFile(ValueError): pass -class NonValidDir(ValueError): pass -class NonValidCommand(ValueError): pass +class NonValidFile(ValueError): + def __init__(self): + ValueError.__init__(self) + printf(self.message) +class NonValidDir(ValueError): + def __init__(self): + ValueError.__init__(self) + printf(self.message) +class NonValidCommand(ValueError): + def __init__(self): + ValueError.__init__(self) + printf(self.message) class Package: """ An object that has information about a package. """ diff --git a/config.sh b/config.sh index 75d4b73..9a44f50 100755 --- a/config.sh +++ b/config.sh @@ -2,10 +2,6 @@ # -*- coding: utf-8 -*- source local_config -# Rsync commands -rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " -rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " - function run_python_cmd { env \ mirror=${mirror} \ @@ -13,7 +9,6 @@ function run_python_cmd { logname=${logname} \ tempdir=${tempdir} \ archdb=${archdb} \ - docs_dir=${docs_dir} \ repodir=${repodir} \ blacklist=${blacklist} \ whitelist=${whitelist} \ diff --git a/cptobin.sh b/cptobin.sh index e803132..068d765 100755 --- a/cptobin.sh +++ b/cptobin.sh @@ -1,2 +1,2 @@ #!/bin/bash -cp -f clean_repo.py config.py config.sh get_license.sh main.sh mkpending.py filter.py ~/usr/bin/ \ No newline at end of file +cp -f clean_repo.py config.py config.sh get_license.sh main.sh mkpending.py filter.py local_config ~/usr/bin/ \ No newline at end of file diff --git a/filter.py b/filter.py index 78ad410..48e2d93 100755 --- a/filter.py +++ b/filter.py @@ -4,12 +4,17 @@ from glob import glob from repm.config import * from repm.pato2 import * -def listado(filename): +def listado(filename,start=0,end=None): """Obtiene una lista de paquetes de un archivo.""" - archivo = open(filename,"r") - lista = archivo.read().split("\n") - archivo.close() - return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] + fsock = open(filename,"r") + lista = fsock.read().split("\n") + fsock.close() + if end is not None: + return [pkg.split(":")[start:end].rstrip() + for pkg in lista if pkg] + else: + return [pkg.split(":")[start].rstrip() + for pkg in lista if pkg] def pkginfo_from_filename(filename): """ Generates a Package object with info from a filename, @@ -135,29 +140,26 @@ def pkginfo_from_db(path_to_db): package_list=list() if not os.path.isfile(path_to_db): - raise NonValidFile(path_to_db + "is not a file") + raise NonValidFile(path_to_db + " is not a file") - check_output("mkdir -p " + config["archdb"]) + check_output(["mkdir", "-p", config["archdb"]]) try: db_open_tar = tarfile.open(path_to_db, 'r:gz') except tarfile.ReadError: - printf("No valid db_file %s or not readable" % path_to_db) - return(tuple()) - else: - printf("No db_file %s" % path_to_db) + raise NonValidFile("No valid db_file %s or not readable" % path_to_db) return(tuple()) for file in db_open_tar.getmembers(): - db_open_tar.extract(file, archdb) + db_open_tar.extract(file, config["archdb"]) db_open_tar.close() # Get info from file - for dir_ in glob(archdb + "/*"): + for dir_ in glob(config["archdb"] + "/*"): if isdir(dir_) and isfile(dir_ + "/desc"): package_list.append(pkginfo_from_desc( os.path.join(dir_,"desc"))) - check_output("rm -r %s/*" % archdb) - if verbose_: + check_output(["rm", "-r", config["archdb"]]) + if config["debug"]: printf(package_list) return package_list diff --git a/local_config.example b/local_config.example index 7edfde7..8015ee2 100644 --- a/local_config.example +++ b/local_config.example @@ -2,19 +2,19 @@ mirror="mirrors.eu.kernel.org" mirrorpath="::mirrors/archlinux" -# Directories +# Directories: they should end without / ## Optionals paraboladir=~/parabolagnulinux.org logtime=$(date -u +%Y%m%d-%H:%M) ## Must be defined -logname=${paraboladir}/${logtime}-repo-maintainer.log -tempdir=~/tmp/ +tempdir=~/tmp archdb=${tempdir}/db docs_dir=${paraboladir}/docs repodir=${paraboladir}/repo # End Directories # Files +logname=${paraboladir}/log/${logtime}-repo-maintainer.log blacklist=${docs_dir}/blacklist.txt whitelist=${docs_dir}/whitelist.txt pending=${docs_dir}/pending @@ -29,3 +29,7 @@ other="any" # Output options output="True" debug="False" + +# Rsync commands +rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " +rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " \ No newline at end of file diff --git a/main.sh b/main.sh index 886ea38..46f2f75 100755 --- a/main.sh +++ b/main.sh @@ -20,7 +20,8 @@ mkrsexclude msg "Syncing repos without delete" # rsync_update_command does not sync db or abs ${rsync_update_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ',')} ${repodir} + ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ','),\ + $(echo ${dir_list} | tr ':' ',')} ${repodir} msg "Syncing each repo and cleaning" msg2 "Remove pending files" diff --git a/mkpending.py b/mkpending.py index 2b255a8..6022206 100755 --- a/mkpending.py +++ b/mkpending.py @@ -1,9 +1,27 @@ #!/usr/bin/python # -*- coding: utf-8 -*- from repm.filter import * +import argparse -def make_pending(path_to_db): +def make_pending(path_to_db, repo, prefix=config["pending"]): """ Determine wich packages are pending for license auditing.""" + filename=prefix + "-" + repo + ".txt" + try: + fsock=open(filename, "rw") + if os.path.isfile(filename): + pkgs=[pkg for pkg in packages_iterable if pkg["name"] not in + listado(filename)] + fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] + for pkg in pkgs]) + "\n") + except(IOError): + print("Can't read %s" % filename) + exit(1) + finally: + fsock.close() + + if "~" in path_to_db: + path_to_db=(os.path.expanduser(path_to_db)) + packages_iterable=pkginfo_from_db(path_to_db) search = tuple(listado(config["blacklist"]) + listado(config["whitelist"])) @@ -15,24 +33,11 @@ def make_pending(path_to_db): def write_pending(packages_iterable, repo, prefix=config["pending"]): """ Write a pending file with the info of the packages """ - filename=prefix + "-" + repo + ".txt" - try: - fsock=open(filename, "a") - except(IOError): - print("Can't read %s" % filename) - finally: - fsock.close() - if os.path.isfile(filename): - pkgs=[pkg for pkg in packages_iterable if pkg["name"] not in - listado(filename)] - fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] - for pkg in pkgs]) + "\n") - fsock.close() if __name__ == "__main__": parser = argparse.ArgumentParser( description="Clean a repo db and packages") - parser.add_argument("-b", "--dababase", type=str, required=True, + parser.add_argument("-b", "--database", type=str, required=True, help="database to check") parser.add_argument("-r", "--repo", type=str, required=True, help="repo of database") diff --git a/pato2.py b/pato2.py index 4cdb536..6daa8b8 100644 --- a/pato2.py +++ b/pato2.py @@ -29,14 +29,6 @@ from repm.filter import * import tarfile from os.path import isdir, isfile -def printf(text,output=config["output"]): - """Guarda el texto en la variable log y puede imprimir en pantalla.""" - log_file = open(config["logname"], 'a') - log_file.write("\n" + str(text) + "\n") - log_file.close() - if output_: - print (str(text) + "\n") - def generate_rsync_command(base_command, dir_list=(config["repo_list"] + config["dir_list"]), @@ -68,4 +60,4 @@ def run_rsync(command,debug=config["debug"]): """ Runs rsync and gets returns it's output """ if debug: printf("rsync_command: " + command) - return check_output(command) + return check_output(command.split()) -- cgit v1.2.3-54-g00ecf From aca64420db93c1d1ab26ad27640458d3a1f09f3d Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Wed, 13 Apr 2011 00:58:19 -0500 Subject: cptobin.sh not copy inexistent files --- cptobin.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cptobin.sh b/cptobin.sh index 068d765..ce4b3bd 100755 --- a/cptobin.sh +++ b/cptobin.sh @@ -1,2 +1,2 @@ #!/bin/bash -cp -f clean_repo.py config.py config.sh get_license.sh main.sh mkpending.py filter.py local_config ~/usr/bin/ \ No newline at end of file +cp -f clean_repo.py config.py config.sh get_license.sh main.sh filter.py local_config ~/usr/bin/ \ No newline at end of file -- cgit v1.2.3-54-g00ecf From ddacb9a98b90fc2b51551fb73ab8513634f4f185 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Wed, 13 Apr 2011 01:01:57 -0500 Subject: filter.py to close correct file --- filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filter.py b/filter.py index 1d70a63..c7f3f18 100755 --- a/filter.py +++ b/filter.py @@ -147,7 +147,7 @@ def pkginfo_from_db(path_to_db): % path_to_db) return(tuple()) finally: - db_open_tar.close() + dbsock.close() return package_list def rsyncBlacklist_from_blacklist(packages_iterable, -- cgit v1.2.3-54-g00ecf From 062b7a2d9b0347e4e867e588826f052d26a3bb30 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Wed, 13 Apr 2011 01:11:20 -0500 Subject: For testing --- filter.py | 2 +- local_config.example | 4 ++-- main.sh | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/filter.py b/filter.py index c7f3f18..4b9603d 100755 --- a/filter.py +++ b/filter.py @@ -137,7 +137,7 @@ def pkginfo_from_db(path_to_db): try: dbsock = tarfile.open(path_to_db, 'r:gz') - desc_files=[desc for desc in db_open_tar.getnames() + desc_files=[desc for desc in dbsock.getnames() if "/desc" in desc] for name in desc_files: desc=dbsock.extractfile(name) diff --git a/local_config.example b/local_config.example index 8015ee2..99a1fdb 100644 --- a/local_config.example +++ b/local_config.example @@ -31,5 +31,5 @@ output="True" debug="False" # Rsync commands -rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " -rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " \ No newline at end of file +rsync_update_command="rsync -nav --delay-updates --exclude='*.{abs|db}.tar.*' " +rsync_post_command="rsync -nav --delete --exclude='*.abs.tar.*' " \ No newline at end of file diff --git a/main.sh b/main.sh index 9f41a95..d15f736 100755 --- a/main.sh +++ b/main.sh @@ -29,8 +29,10 @@ stdnull "rm -rf ${pending}*" for repo in $(echo ${repo_list} | tr ':' ' '); do for arch in $(echo ${arch_list} | tr ':' ' '); do msg2 "Syncing ${repo} ${arch}" - ${rsync_post_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/${repo} ${repodir}/${repo} + cmd=$(echo ${rsync_post_command} --exclude-from=${rsync_blacklist} \ + ${mirror}${mirropath}/${repo} ${repodir}/${repo}) + plain "${cmd}" + ${cmd} msg2 "Cleaning ${repo} ${arch}" # This also generates pending lists run_python_cmd "clean_repo.py -b ${repodir}/${repo}/os/${arch}/${repo}.db.tar.gz -d ${repodir}/${repo}/os/${arch}/" -- cgit v1.2.3-54-g00ecf From deab65fad4ced009fb31f7033b1db8ef0af78aee Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Fri, 15 Apr 2011 04:53:27 -0500 Subject: Python parts ready for bash usage in python 2 and 3 --- clean_repo.py | 78 ++++++++++++++++++++++++++++++----------------------- config.py | 65 +++++++++----------------------------------- filter.py | 48 ++++++++++++++++++++------------- pato2.py | 63 ------------------------------------------- test/test_filter.py | 3 ++- 5 files changed, 88 insertions(+), 169 deletions(-) delete mode 100644 pato2.py diff --git a/clean_repo.py b/clean_repo.py index eccfd01..5423c4d 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -3,21 +3,16 @@ from repm.filter import * import argparse -def mkpending(path_to_db, repo, prefix=config["pending"]): +def mkpending(packages_iterable, pending_file, blacklisted_names, + whitelisted_names): """ Determine wich packages are pending for license auditing.""" - if "~" in path_to_db: - path_to_db=(os.path.expanduser(path_to_db)) - - search = tuple(listado(config["blacklist"]) + - listado(config["whitelist"])) + search = tuple(blacklisted_names + + whitelisted_names) - pkgs=list(pkginfo_from_db(path_to_db)) - - filename=prefix + "-" + repo + ".txt" try: - fsock=open(filename, "rw") - pkgs=[pkg for pkg in pkginfo_from_db(path_to_db) - if pkg["name"] not in listado(filename)] + fsock=open(pending_file, "r") + pkgs=[pkg for pkg in packages_iterable + if pkg["name"] not in listado(pending_file)] for line in fsock.readlines(): if line: pkg=Package() @@ -26,16 +21,16 @@ def mkpending(path_to_db, repo, prefix=config["pending"]): pkgs.append(pkg) pkgs=[pkg for pkg in pkgs if pkg["name"] not in search and "custom" in pkg["license"]] + fsock=open(pending_file, "w") fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] for pkg in pkgs]) + "\n") except(IOError): - raise NonValidFile("Can't read or write %s" % filename) + raise NonValidFile("Can't read or write %s" % pending_file) finally: fsock.close() return pkgs -def remove_from_blacklist(path_to_db, blacklisted_names, - debug=config["debug"]): +def remove_from_blacklist(path_to_db, blacklisted_names): """ Check the blacklist and remove packages on the db""" if "~" in path_to_db: path_to_db=(os.path.expanduser(path_to_db)) @@ -47,9 +42,7 @@ def remove_from_blacklist(path_to_db, blacklisted_names, cmd = "repo-remove " + path_to_db + " " + lista printf(cmd) a = check_output(cmd) - if debug: - printf(a) - return pkgs, cmd + return pkgs def cleanup_nonfree_in_dir(directory, blacklisted_names): if "~" in directory: @@ -61,25 +54,42 @@ def cleanup_nonfree_in_dir(directory, blacklisted_names): if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Clean a repo db and packages") - parser.add_argument("-b", "--database", type=str, - help="dabatase to clean") - parser.add_argument("-d", "--directory", type=str, - help="directory to clean") + prog="clean_repo", + description="Clean a repo db and packages",) + + parser.add_argument("-k", "--blacklist-file", type=str, + help="File containing blacklisted names", + required=True,) + + group_dir=parser.add_argument_group("Clean non-free packages in dir") + group_dir.add_argument("-d", "--directory", type=str, + help="directory to clean",) + + group_db=parser.add_argument_group("Clean non-free packages in db", + "All arguments need to be specified") + group_db.add_argument("-b", "--database", type=str, + help="dabatase to clean") + group_db.add_argument("-p", "--pending-file", type=str, + help="File in which to write pending list") + group_db.add_argument("-w", "--whitelist-file", type=str, + help="File containing whitelisted names") + args=parser.parse_args() + if not args.directory and not args.database: + parser.print_help() + elif not args.pending_file or not args.whitelist_file \ + and args.database: + parser.print_help() + else: + blacklisted=listado(args.blacklist_file) + if args.database: - repo=os.path.basename(args.database).split(".")[0] + whitelisted=listado(args.whitelist_file) pkgs=pkginfo_from_db(args.database) - remove_from_blacklist(args.database, pkgs, - tuple(listado(config["blacklist"]) + - listado(config["pending"] + - "-" + repo + ".txt"))) - mkpending(args.database, args.repo) + remove_from_blacklist(args.database, blacklisted) + mkpending(pkgs, args.pending_file, + blacklisted, whitelisted) if args.directory: - cleanup_nonfree_in_dir(args.directory, - listado(config["blacklist"])) - - if not args.directory and not args.database: - parser.print_help() + cleanup_nonfree_in_dir(args.directory, blacklisted) diff --git a/config.py b/config.py index 8b48c66..4e218a5 100755 --- a/config.py +++ b/config.py @@ -9,61 +9,26 @@ except(ImportError): return getoutput(cmd) import os -stringvars=("mirror", "mirrorpath", "logname", "tempdir", "archdb", - "repodir", "blacklist", "whitelist", "pending", - "rsync_blacklist",) -listvars=("repo_list", "dir_list", "arch_list", "other",) -boolvars=("output", "debug",) - -config=dict() - -def exit_if_none(var): - if os.environ.get(var) is None: - exit("%s is not defined" % var) - -for var in stringvars: - exit_if_none(var) - config[var]=os.environ.get(var) - -for var in listvars: - exit_if_none(var) - config[var]=tuple(os.environ.get(var).split(":")) - -for var in boolvars: - exit_if_none(var) - if os.environ.get(var) == "True": - config[var]=True - elif os.environ.get(var) =="False": - config[var]=False - else: - print('%s is not True or False' % var) # Rsync commands -rsync_list_command="rsync -a --no-motd --list-only " -def printf(text,output=config["output"]): +def printf(text, logfile=False): """Guarda el texto en la variable log y puede imprimir en pantalla.""" - log_file = open(config["logname"], 'a') - log_file.write("\n" + str(text) + "\n") - log_file.close() - if output: - print (str(text) + "\n") + print (str(text) + "\n") + if logfile: + try: + log = open(logfile, 'a') + log.write("\n" + str(text) + "\n") + except: + print("Can't open %s" % logfile) + finally: + log.close() -del exit_if_none # Classes and Exceptions -class NonValidFile(ValueError): - def __init__(self): - ValueError.__init__(self) - printf(self.message) -class NonValidDir(ValueError): - def __init__(self): - ValueError.__init__(self) - printf(self.message) -class NonValidCommand(ValueError): - def __init__(self): - ValueError.__init__(self) - printf(self.message) +class NonValidFile(ValueError): pass +class NonValidDir(ValueError): pass +class NonValidCommand(ValueError): pass class Package: """ An object that has information about a package. """ @@ -101,7 +66,3 @@ class Package: return False else: return True - -if __name__=="__main__": - for key in config.keys(): - print("%s : %s" % (key,config[key])) diff --git a/filter.py b/filter.py index 4b9603d..add0235 100755 --- a/filter.py +++ b/filter.py @@ -2,7 +2,7 @@ #-*- encoding: utf-8 -*- from glob import glob from repm.config import * -from repm.pato2 import * +import tarfile def listado(filename,start=0,end=None): """Obtiene una lista de paquetes de un archivo.""" @@ -28,7 +28,7 @@ def pkginfo_from_filename(filename): ---------- pkg -> Package object""" if ".pkg.tar." not in filename: - raise NonValidFile + raise NonValidFile("File is not a pacman package") pkg = Package() pkg["location"] = filename fileattrs = os.path.basename(filename).split("-") @@ -140,19 +140,18 @@ def pkginfo_from_db(path_to_db): desc_files=[desc for desc in dbsock.getnames() if "/desc" in desc] for name in desc_files: - desc=dbsock.extractfile(name) - package_list.append(pkginfo_from_desc(desc.read())) + desc=dbsock.extractfile(name).read().decode("UTF-8") + package_list.append(pkginfo_from_desc(desc)) except tarfile.ReadError: raise NonValidFile("No valid db_file %s or not readable" % path_to_db) - return(tuple()) finally: dbsock.close() return package_list def rsyncBlacklist_from_blacklist(packages_iterable, blacklisted_names, - exclude_file=config["rsync_blacklist"]): + exclude_file): """ Generate an exclude list for rsync Parameters: @@ -168,20 +167,31 @@ def rsyncBlacklist_from_blacklist(packages_iterable, pkgs=[pkg["location"] for pkg in packages_iterable if isinstance(pkg, Package) and pkg["name"] in blacklisted_names] - - try: - fsock = open(exclude_file,"w") - fsock.write("\n".join(pkgs) + "\n") - except IOError: - printf("%s wasnt written" % exclude_file) - exit(1) - finally: - fsock.close() + if exclude_file: + try: + fsock = open(exclude_file,"w") + fsock.write("\n".join(pkgs) + "\n") + except IOError: + printf("%s wasnt written" % exclude_file) + exit(1) + finally: + fsock.close() return pkgs if __name__ == "__main__": - cmd=generate_rsync_command(rsync_list_command) - a=run_rsync(cmd) - packages=pkginfo_from_rsync_output(a) - rsyncBlaclist_from_blacklist(packages,listado(blacklist)) + import argparse + parser=argparse.ArgumentParser() + parser.add_argument("-r", "--rsync-exclude-file", type=str, + help="File in which to generate exclude list", + required=True,) + parser.add_argument("-k", "--blacklist-file", type=str, + help="File containing blacklisted names", + required=True,) + parser.add_argument("-c", "--rsync-command", type=str, + help="This command will be run to get a pkg list") + args=parser.parse_args() + rsout=check_output(args.rsync_command) + packages=pkginfo_from_rsync_output(rsout) + rsyncBlaclist_from_blacklist(packages, listado(args.blacklist_file), + args.rsync_exclude_file) diff --git a/pato2.py b/pato2.py deleted file mode 100644 index 6daa8b8..0000000 --- a/pato2.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -""" - parabola.py - Copyright 2009 Rafik Mas'ad - Copyright 2010 Joshua Ismael Haase Hernández - - ---------- GNU General Public License 3 ---------- - - This file is part of Parabola. - - Parabola is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - Parabola is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with Parabola. If not, see . - - -""" -from repm.config import * -from repm.filter import * -import tarfile -from os.path import isdir, isfile - -def generate_rsync_command(base_command, - dir_list=(config["repo_list"] + - config["dir_list"]), - destdir=config["repodir"], - source=config["mirror"] +config["mirrorpath"]): - """ Generates an rsync command for executing - it by combining all parameters. - - Parameters: - ---------- - base_command -> str - dir_list -> list or tuple - destdir -> str Path to dir, dir must exist. - source -> str The source for rsync - blacklist_file -> False or str Path to file, file must exist. - - Return: - ---------- - rsync_command -> str """ - if not os.path.isdir(destdir): - print(destdir + " is not a directory") - raise NonValidDir - - dir_list="{" + ",".join(dir_list) + "}" - return " ".join((base_command, os.path.join(source, dir_list), - destdir)) - -def run_rsync(command,debug=config["debug"]): - """ Runs rsync and gets returns it's output """ - if debug: - printf("rsync_command: " + command) - return check_output(command.split()) diff --git a/test/test_filter.py b/test/test_filter.py index b6d5766..d8006f9 100644 --- a/test/test_filter.py +++ b/test/test_filter.py @@ -143,7 +143,8 @@ class generateRsyncBlacklist(unittest.TestCase): def testExcludeFiles(self): a=rsyncBlacklist_from_blacklist(self.example_package_list, - listado("blacklist_sample")) + listado("blacklist_sample"), + False) b=[self.example_package_list[0]["location"],self.example_package_list[2]["location"]] self.assertEqual(a,b) -- cgit v1.2.3-54-g00ecf From 374c8e2a5183cdbaefe9f54184603ad8d09e30c2 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sat, 16 Apr 2011 03:15:01 -0500 Subject: * bash-repo for testing * merged dbscripts with repm --- changelog | 2 -- clean_repo.py | 3 ++- config | 2 +- config.sh | 26 ---------------------- cptobin.sh | 2 -- filter.py | 5 +++-- get_license.sh | 50 +++++++++++++++++------------------------- local_config.example | 18 ++++++--------- main.sh | 62 +++++++++++++++++++++++----------------------------- 9 files changed, 60 insertions(+), 110 deletions(-) delete mode 100644 changelog delete mode 100755 config.sh delete mode 100755 cptobin.sh diff --git a/changelog b/changelog deleted file mode 100644 index 0e89f2f..0000000 --- a/changelog +++ /dev/null @@ -1,2 +0,0 @@ -repo-maintainer 1.0 - * Known working version diff --git a/clean_repo.py b/clean_repo.py index 5423c4d..e1a17c2 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -22,7 +22,8 @@ def mkpending(packages_iterable, pending_file, blacklisted_names, pkgs=[pkg for pkg in pkgs if pkg["name"] not in search and "custom" in pkg["license"]] fsock=open(pending_file, "w") - fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] + fsock.write("\n".join([pkg["name"] + ":" + pkg["location"] + + ":" + pkg["license"] for pkg in pkgs]) + "\n") except(IOError): raise NonValidFile("Can't read or write %s" % pending_file) diff --git a/config b/config index 217b627..d9bad13 100644 --- a/config +++ b/config @@ -1,7 +1,7 @@ FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" -PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') +PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing' 'multilib') PKGPOOL='pool/packages' SRCPOOL='sources/packages' diff --git a/config.sh b/config.sh deleted file mode 100755 index 9a44f50..0000000 --- a/config.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -# -*- coding: utf-8 -*- -source local_config - -function run_python_cmd { - env \ - mirror=${mirror} \ - mirrorpath=${mirrorpath} \ - logname=${logname} \ - tempdir=${tempdir} \ - archdb=${archdb} \ - repodir=${repodir} \ - blacklist=${blacklist} \ - whitelist=${whitelist} \ - pending=${pending} \ - rsync_blacklist=${rsync_blacklist} \ - repo_list=${repo_list} \ - dir_list=${dir_list} \ - arch_list=${arch_list} \ - other=${other} \ - output=${output} \ - debug=${debug} \ - $1 -} - -source libremessages \ No newline at end of file diff --git a/cptobin.sh b/cptobin.sh deleted file mode 100755 index ce4b3bd..0000000 --- a/cptobin.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -cp -f clean_repo.py config.py config.sh get_license.sh main.sh filter.py local_config ~/usr/bin/ \ No newline at end of file diff --git a/filter.py b/filter.py index add0235..4abdf38 100755 --- a/filter.py +++ b/filter.py @@ -189,9 +189,10 @@ if __name__ == "__main__": help="File containing blacklisted names", required=True,) parser.add_argument("-c", "--rsync-command", type=str, - help="This command will be run to get a pkg list") + help="This command will be run to get a pkg list", + required=True,) args=parser.parse_args() - rsout=check_output(args.rsync_command) + rsout=check_output(args.rsync_command.split()) packages=pkginfo_from_rsync_output(rsout) rsyncBlaclist_from_blacklist(packages, listado(args.blacklist_file), args.rsync_exclude_file) diff --git a/get_license.sh b/get_license.sh index 0da58cb..024876c 100755 --- a/get_license.sh +++ b/get_license.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # -*- coding: utf-8 -*- # get_license.sh @@ -20,36 +20,26 @@ # You should have received a copy of the GNU General Public License # along with Parabola. If not, see . - -docs="/home/parabolavnx/parabolagnulinux.org/docs" -repo="/home/parabolavnx/parabolagnulinux.org/repo" -dir="$docs/pending-licenses" - -echo "Cleaning $dir" -rm -rf $dir/* - -tempdir=$(mktemp -d) -cd $tempdir - -pending=($(cut -d: -f1 $docs/pending*.txt)) -echo ${pending[@]} - -for pkg in ${pending[@]}; do - pkg_in_repo=( $(ls ${repo}/*/os/*/${pkg}*) ) - for y in ${pkg_in_repo[@]}; do - echo "chmod +r $y" - chmod +r $y - echo "tar -xf $y usr/share/licenses" - bsdtar -xf $y usr/share/licenses - echo "chmod -r $y" - chmod -r $y +source ./config +source ./local_config +source ./libremessages + +msg "Creating pending licenses list" +pushd ${licenses_dir} +rm -rf ${licenses_dir}/* + +for repo in ${PKGREPOS[@]}; do + msg2 "Extracting licenses in ${repo}" + pending=($(cut -d: -f2 ${docs_dir}/pending-${repo})) + pushd ${repodir}/${repo} + for pkg in ${pending[@]}; do + plain "${pkg}" + bsdtar -xf ${pkg} usr/share/licenses || { + error "${pkg} has no licenses" + } + chmod -r ${pkg} done done -mv usr/share/licenses/* $dir - -cd - -rm -rf $tempdir - +popd exit 0 \ No newline at end of file diff --git a/local_config.example b/local_config.example index 99a1fdb..2c0ef3e 100644 --- a/local_config.example +++ b/local_config.example @@ -4,21 +4,21 @@ mirrorpath="::mirrors/archlinux" # Directories: they should end without / ## Optionals -paraboladir=~/parabolagnulinux.org -logtime=$(date -u +%Y%m%d-%H:%M) +paraboladir= ## Must be defined tempdir=~/tmp archdb=${tempdir}/db docs_dir=${paraboladir}/docs -repodir=${paraboladir}/repo +repodir=${paraboladir}/repo/staging +licenses_dir=${docs_dir}/pending_licenses # End Directories # Files -logname=${paraboladir}/log/${logtime}-repo-maintainer.log +logname=${paraboladir}/log/$(date -u +%Y%m%d-%H:%M)-repo-maintainer.log blacklist=${docs_dir}/blacklist.txt whitelist=${docs_dir}/whitelist.txt -pending=${docs_dir}/pending rsync_blacklist=${docs_dir}/rsyncBlacklist +rsync_not_needed=${tmp}/rsync_not_needed # Repos, arches, and dirs for repo repo_list="core:extra:community:testing:community-testing:multilib" @@ -26,10 +26,6 @@ dir_list="pool" arch_list="i686:x86_64" other="any" -# Output options -output="True" -debug="False" - # Rsync commands -rsync_update_command="rsync -nav --delay-updates --exclude='*.{abs|db}.tar.*' " -rsync_post_command="rsync -nav --delete --exclude='*.abs.tar.*' " \ No newline at end of file +rsync_list_command="rsync\ -ptgoL\ --list-only\ " +rsync_update_command="rsync -ptgoL --exclude='*.abs.tar.*'" diff --git a/main.sh b/main.sh index d15f736..b3ecb92 100755 --- a/main.sh +++ b/main.sh @@ -1,43 +1,35 @@ #!/bin/bash # -*- coding: utf-8 -*- -source config.sh +source ./config +source ./local_config +source ./libremessages -function mkrsexclude { - local error=1 - while ${error}; do - run_python_cmd "filter.py" - error=$? - done -} - -msg "Cleaning $tempdir" -stdnull "rm -r $tempdir/* " - -msg "Generating exclude list for rsync" -mkrsexclude - -msg "Syncing repos without delete" -# rsync_update_command does not sync db or abs -${rsync_update_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ','),\ - $(echo ${dir_list} | tr ':' ',')} ${repodir} - -msg "Syncing each repo and cleaning" -msg2 "Remove pending files" -stdnull "rm -rf ${pending}*" -for repo in $(echo ${repo_list} | tr ':' ' '); do - for arch in $(echo ${arch_list} | tr ':' ' '); do - msg2 "Syncing ${repo} ${arch}" - cmd=$(echo ${rsync_post_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/${repo} ${repodir}/${repo}) - plain "${cmd}" - ${cmd} - msg2 "Cleaning ${repo} ${arch}" - # This also generates pending lists - run_python_cmd "clean_repo.py -b ${repodir}/${repo}/os/${arch}/${repo}.db.tar.gz -d ${repodir}/${repo}/os/${arch}/" +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]} 'any'; do + msg "Syncing ${repo} ${arch}" + filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ + \"${rsync_list_command}\ \ + ${mirror}${mirrorpath}/${repo}/os/${arch}\ \ + ${repodir}/${repo}/\" + find ${repodir}/${repo} -name *${PKGEXT} -print \ + > ${rsync_not_needed} + ${rsync_update_command} \ + ${mirror}${mirrorpath}/${repo}/os/${arch} \ + ${repodir}/${repo} \ + --exclude-from=${rsync_blacklist} \ + --exclude-from=${rsync_not_needed} done + for arch in ${ARCHES[@]}; do + if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then + clean_repo.py -k ${blacklist} -w ${whitelist} \ + -p ${docs_dir}/pending-${repo} \ + -b ${repodir}/${repo}/${repo}${DBEXT} + fi + clean_repo.py -k ${blacklist} -d ${repodir}/${repo} done -msg "Checking licenses" +db-update +ftpdir-cleanup + get_license.sh -- cgit v1.2.3-54-g00ecf From bbb2155b1bb0dd442be86e05d1be6400e62be5d0 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sat, 16 Apr 2011 03:40:54 -0500 Subject: Updated TODO list. --- TODO | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/TODO b/TODO index 11ef51a..e50cefa 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,10 @@ -* Test Suite +* Test Suite for clean_repo.py - - Review all repo efectively - - Remove all blacklisted packages - - Get pending list right - - Extract licenses all right \ No newline at end of file + - Review all repo + - Remove all blacklisted packages + - Get pending list right + - Extract licenses all right + +* Fix db-move + + - Make it use abslibre \ No newline at end of file -- cgit v1.2.3-54-g00ecf From bef1e554380b797a4dfff5950c5aea59d093b4be Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sat, 16 Apr 2011 12:16:21 -0500 Subject: main.sh renamed to repo-update --- repo-state | 11 ----------- repo-update | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 11 deletions(-) delete mode 100755 repo-state create mode 100755 repo-update diff --git a/repo-state b/repo-state deleted file mode 100755 index 319db83..0000000 --- a/repo-state +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -a=$(date +%Y%m%d+%H%M) -mkdir -p $a -cd $a - -. /home/joshpar/programas/arch2parabola/repo-list-diff -# scp parabolavnx@parabolagnulinux.org:~/tmp/rsyncBlacklist.txt ./ - -cd .. -exit 0 \ No newline at end of file diff --git a/repo-update b/repo-update new file mode 100755 index 0000000..b3ecb92 --- /dev/null +++ b/repo-update @@ -0,0 +1,35 @@ +#!/bin/bash +# -*- coding: utf-8 -*- + +source ./config +source ./local_config +source ./libremessages + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]} 'any'; do + msg "Syncing ${repo} ${arch}" + filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ + \"${rsync_list_command}\ \ + ${mirror}${mirrorpath}/${repo}/os/${arch}\ \ + ${repodir}/${repo}/\" + find ${repodir}/${repo} -name *${PKGEXT} -print \ + > ${rsync_not_needed} + ${rsync_update_command} \ + ${mirror}${mirrorpath}/${repo}/os/${arch} \ + ${repodir}/${repo} \ + --exclude-from=${rsync_blacklist} \ + --exclude-from=${rsync_not_needed} + done + for arch in ${ARCHES[@]}; do + if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then + clean_repo.py -k ${blacklist} -w ${whitelist} \ + -p ${docs_dir}/pending-${repo} \ + -b ${repodir}/${repo}/${repo}${DBEXT} + fi + clean_repo.py -k ${blacklist} -d ${repodir}/${repo} +done + +db-update +ftpdir-cleanup + +get_license.sh -- cgit v1.2.3-54-g00ecf From fbc8eec9e5cb3d598fb84579811f16ff8c2b71d8 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sat, 16 Apr 2011 12:26:13 -0500 Subject: Fixed error in repo-update --- repo-update | 1 + 1 file changed, 1 insertion(+) diff --git a/repo-update b/repo-update index b3ecb92..59a0658 100755 --- a/repo-update +++ b/repo-update @@ -27,6 +27,7 @@ for repo in ${PKGREPOS[@]}; do -b ${repodir}/${repo}/${repo}${DBEXT} fi clean_repo.py -k ${blacklist} -d ${repodir}/${repo} + done done db-update -- cgit v1.2.3-54-g00ecf From e0aad034cd3b7e91137f85584f9b53cdc871f44f Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernández Date: Sat, 16 Apr 2011 12:42:00 -0500 Subject: Modified local_config.example --- filter.py | 4 ++-- local_config.example | 10 +--------- main.sh | 35 ----------------------------------- 3 files changed, 3 insertions(+), 46 deletions(-) delete mode 100755 main.sh diff --git a/filter.py b/filter.py index 4abdf38..c71f54d 100755 --- a/filter.py +++ b/filter.py @@ -4,9 +4,9 @@ from glob import glob from repm.config import * import tarfile -def listado(filename,start=0,end=None): +def listado(filename, start=0, end=None): """Obtiene una lista de paquetes de un archivo.""" - fsock = open(filename,"r") + fsock = open(filename, "r") lista = fsock.read().split("\n") fsock.close() if end is not None: diff --git a/local_config.example b/local_config.example index 2c0ef3e..0a827f4 100644 --- a/local_config.example +++ b/local_config.example @@ -3,9 +3,7 @@ mirror="mirrors.eu.kernel.org" mirrorpath="::mirrors/archlinux" # Directories: they should end without / -## Optionals -paraboladir= -## Must be defined +paraboladir=~/parabolagnulinux.org tempdir=~/tmp archdb=${tempdir}/db docs_dir=${paraboladir}/docs @@ -20,12 +18,6 @@ whitelist=${docs_dir}/whitelist.txt rsync_blacklist=${docs_dir}/rsyncBlacklist rsync_not_needed=${tmp}/rsync_not_needed -# Repos, arches, and dirs for repo -repo_list="core:extra:community:testing:community-testing:multilib" -dir_list="pool" -arch_list="i686:x86_64" -other="any" - # Rsync commands rsync_list_command="rsync\ -ptgoL\ --list-only\ " rsync_update_command="rsync -ptgoL --exclude='*.abs.tar.*'" diff --git a/main.sh b/main.sh deleted file mode 100755 index b3ecb92..0000000 --- a/main.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# -*- coding: utf-8 -*- - -source ./config -source ./local_config -source ./libremessages - -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]} 'any'; do - msg "Syncing ${repo} ${arch}" - filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ - \"${rsync_list_command}\ \ - ${mirror}${mirrorpath}/${repo}/os/${arch}\ \ - ${repodir}/${repo}/\" - find ${repodir}/${repo} -name *${PKGEXT} -print \ - > ${rsync_not_needed} - ${rsync_update_command} \ - ${mirror}${mirrorpath}/${repo}/os/${arch} \ - ${repodir}/${repo} \ - --exclude-from=${rsync_blacklist} \ - --exclude-from=${rsync_not_needed} - done - for arch in ${ARCHES[@]}; do - if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then - clean_repo.py -k ${blacklist} -w ${whitelist} \ - -p ${docs_dir}/pending-${repo} \ - -b ${repodir}/${repo}/${repo}${DBEXT} - fi - clean_repo.py -k ${blacklist} -d ${repodir}/${repo} -done - -db-update -ftpdir-cleanup - -get_license.sh -- cgit v1.2.3-54-g00ecf