summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--config.py1
-rw-r--r--pato2.py295
2 files changed, 152 insertions, 144 deletions
diff --git a/config.py b/config.py
index c60e8f3..6420355 100644
--- a/config.py
+++ b/config.py
@@ -19,6 +19,7 @@ logdir = path + "/log"
## Must be defined
logname= logdir + "/" + time__ + "-repo-maintainer.log"
+freedir= path + "/free/"
repodir= path + "/repo"
tmp = home + "/tmp"
archdb = tmp + "/db"
diff --git a/pato2.py b/pato2.py
index c598c2f..f5deb48 100644
--- a/pato2.py
+++ b/pato2.py
@@ -31,142 +31,149 @@ from glob import glob
from os.path import isdir, isfile
def printf(text,output_=output):
- """Guarda el texto en la variable log y puede imprimir en pantalla."""
- log_file = open(logname, 'a')
- log_file.write("\n" + str(text) + "\n")
- log_file.close()
- if output_: print (str(text) + "\n")
+ """Guarda el texto en la variable log y puede imprimir en pantalla."""
+ log_file = open(logname, 'a')
+ log_file.write("\n" + str(text) + "\n")
+ log_file.close()
+ if output_: print (str(text) + "\n")
def listado(filename_):
- """Obtiene una lista de paquetes de un archivo."""
- archivo = open(filename_,"r")
- lista = archivo.read().split("\n")
- archivo.close()
- return [pkg.split(":")[0].rstrip() for pkg in lista if pkg]
+ """Obtiene una lista de paquetes de un archivo."""
+ archivo = open(filename_,"r")
+ lista = archivo.read().split("\n")
+ archivo.close()
+ return [pkg.split(":")[0].rstrip() for pkg in lista if pkg]
def db(repo_,arch_):
- """Construye un nombre para sincronizar una base de datos."""
- return "/%s/os/%s/%s.db.tar.gz" % (repo_, arch_, repo_)
+ """Construye un nombre para sincronizar una base de datos."""
+ return "/%s/os/%s/%s.db.tar.gz" % (repo_, arch_, repo_)
def packages(repo_, arch_, expr="*"):
- """ Get packages on a repo, arch folder """
- return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) )
+ """ Get packages on a repo, arch folder """
+ return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) )
def sync_all_repo(debug=verbose):
- cmd=generate_rsync_command(rsync_list_command)
- rsout=run_rsync(cmd)
- pkgs=pkginfo_from_rsync_output(rsout)
- generate_exclude_list_from_blacklist(pkgs,listado(blacklist))
- cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist)
- a=run_rsync(cmd)
- if debug: printf(a)
+ cmd=generate_rsync_command(rsync_list_command)
+ rsout=run_rsync(cmd)
+ pkgs=pkginfo_from_rsync_output(rsout)
+ generate_exclude_list_from_blacklist(pkgs,listado(blacklist))
+ cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist)
+ a=run_rsync(cmd)
+ if debug: printf(a)
def get_from_desc(desc, var,db_tar_file=False):
- """ Get a var from desc file """
- desc = desc.split("\n")
- return desc[desc.index(var)+1]
+ """ Get a var from desc file """
+ desc = desc.split("\n")
+ return desc[desc.index(var)+1]
def get_info(repo_,arch_,db_tar_file=False,verbose_=verbose):
- """ Makes a list of package name, file and license """
- info=list()
- # Extract DB tar.gz
- commands.getoutput("mkdir -p " + archdb)
- if not db_tar_file:
- db_tar_file = repodir + db(repo_,arch_)
- if isfile(db_tar_file):
- try:
- db_open_tar = tarfile.open(db_tar_file, 'r:gz')
- except tarfile.ReadError:
- printf("No valid db_file %s" % db_tar_file)
- return(tuple())
- else:
- printf("No db_file %s" % db_tar_file)
- return(tuple())
- for file in db_open_tar.getmembers():
- db_open_tar.extract(file, archdb)
- db_open_tar.close()
- # Get info from file
- for dir_ in glob(archdb + "/*"):
- if isdir(dir_) and isfile(dir_ + "/desc"):
- pkg_desc_file = open(dir_ + "/desc", "r")
- desc = pkg_desc_file.read()
- pkg_desc_file.close()
- info.append(( get_from_desc(desc,"%NAME%"),
- dir_.split("/")[-1],
- get_from_desc(desc,"%LICENSE%") ))
- if verbose_: printf(info)
- commands.getoutput("rm -r %s/*" % archdb)
- return tuple(info)
+ """ Makes a list of package name, file and license """
+ info=list()
+ # Extract DB tar.gz
+ commands.getoutput("mkdir -p " + archdb)
+ if not db_tar_file:
+ db_tar_file = repodir + db(repo_,arch_)
+ if isfile(db_tar_file):
+ try:
+ db_open_tar = tarfile.open(db_tar_file, 'r:gz')
+ except tarfile.ReadError:
+ printf("No valid db_file %s" % db_tar_file)
+ return(tuple())
+ else:
+ printf("No db_file %s" % db_tar_file)
+ return(tuple())
+ for file in db_open_tar.getmembers():
+ db_open_tar.extract(file, archdb)
+ db_open_tar.close()
+ # Get info from file
+ for dir_ in glob(archdb + "/*"):
+ if isdir(dir_) and isfile(dir_ + "/desc"):
+ pkg_desc_file = open(dir_ + "/desc", "r")
+ desc = pkg_desc_file.read()
+ pkg_desc_file.close()
+ info.append(( get_from_desc(desc,"%NAME%"),
+ dir_.split("/")[-1],
+ get_from_desc(desc,"%LICENSE%") ))
+ if verbose_: printf(info)
+ commands.getoutput("rm -r %s/*" % archdb)
+ return tuple(info)
def make_pending(repo_,arch_,info_):
- """ Si los paquetes no están en blacklist ni whitelist y la licencia contiene "custom" los agrega a pending"""
- search = tuple( listado(blacklist) + listado (whitelist) )
- if verbose: printf("blaclist + whitelist= " + str(search) )
- lista_=list()
- for (name,pkg_,license_) in info_:
- if "custom" in license_:
- if name not in search:
- lista_.append( (name, license_ ) )
- elif not name:
- printf( pkg_ + " package has no %NAME% attibute " )
- if verbose: printf( lista_ )
- a=open( pending + "-" + repo_ + ".txt", "w" ).write(
- "\n".join([name + ":" + license_ for (name,license_) in lista_]) )
+ """ Si los paquetes no están en blacklist ni whitelist y la licencia contiene "custom" los agrega a pending"""
+ search = tuple( listado(blacklist) + listado (whitelist) )
+ if verbose: printf("blaclist + whitelist= " + str(search) )
+ lista_=list()
+ for (name,pkg_,license_) in info_:
+ if "custom" in license_:
+ if name not in search:
+ lista_.append( (name, license_ ) )
+ elif not name:
+ printf( pkg_ + " package has no %NAME% attibute " )
+ if verbose: printf( lista_ )
+ a=open( pending + "-" + repo_ + ".txt", "w" ).write(
+ "\n".join([name + ":" + license_ for (name,license_) in lista_]) )
def remove_from_blacklist(repo_,arch_,info_,blacklist_):
- """ Check the blacklist and remove packages on the db"""
- lista_=list()
- pack_=list()
- for (name_, pkg_, license_) in info_:
- if name_ in blacklist_:
- lista_.append(name_)
- for p in packages(repo_,arch_,pkg_ + "*"):
- pack_.append(p)
- if lista_:
- lista_=" ".join(lista_)
- com_ = "repo-remove " + repodir + db(repo_,arch_) + " " + lista_
- printf(com_)
- a = commands.getoutput(com_)
- if verbose: printf(a)
- if pack_:
- pack_=" ".join(pack_)
- com_="chmod a-r " + pack_
- printf(com_)
- a=commands.getoutput(com_)
- if verbose: printf(a)
+ """ Check the blacklist and remove packages on the db"""
+ lista_=list()
+ pack_=list()
+ for (name_, pkg_, license_) in info_:
+ if name_ in blacklist_:
+ lista_.append(name_)
+ for p in packages(repo_,arch_,pkg_ + "*"):
+ pack_.append(p)
+ if lista_:
+ lista_=" ".join(lista_)
+ com_ = "repo-remove " + repodir + db(repo_,arch_) + " " + lista_
+ printf(com_)
+ a = commands.getoutput(com_)
+ if verbose: printf(a)
+ if pack_:
+ pack_=" ".join(pack_)
+ com_="chmod a-r " + pack_
+ printf(com_)
+ a=commands.getoutput(com_)
+ if verbose: printf(a)
def link(repo_,arch_,file_):
- """ Makes a link in the repo for the package """
- cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_
- a=commands.getoutput(cmd_)
- if verbose:
- printf(cmd_ + a)
+ """ Makes a link in the repo for the package """
+ cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_
+ a=commands.getoutput(cmd_)
+ if verbose:
+ printf(cmd_ + a)
def add_free_repo(verbose_=verbose):
- for repo_ in repo_list:
- for arch_ in arch_list:
- lista_=list()
- for file_ in glob(free_path + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"):
- lista_.append(file_)
- link(repo_,arch_,file_)
- for dir_ in other:
- for file_ in glob(free_path + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"):
- lista_.append(file_)
- link(repo_,arch_,file_)
- if lista_:
- lista_=" ".join(lista_)
- if verbose: printf(lista_)
- cmd_="repo-add " + repodir + db(repo_,arch_) + " " + lista_
- printf(cmd_)
- a=commands.getoutput(cmd_)
- if verbose: printf(a)
+ cmd_=home + "/usr/bin/sync-free"
+ printf(cmd_)
+ a=commands.getoutput(cmd_)
+ if verbose_: printf(a)
+ for repo_ in repo_list:
+ for arch_ in arch_list:
+ lista_=list()
+ for file_ in glob(freedir + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"):
+ lista_.append(file_)
+ # link(repo_,arch_,file_)
+ for dir_ in other:
+ for file_ in glob(freedir + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"):
+ lista_.append(file_)
+ # link(repo_,arch_,file_)
+
+ printf(lista_)
+
+ if lista_:
+ lista_=" ".join(lista_)
+ if verbose: printf(lista_)
+ cmd_="repo-add " + repodir + db(repo_,arch_) + " " + lista_
+ printf(cmd_)
+ a=commands.getoutput(cmd_)
+ if verbose: printf(a)
def get_licenses(verbose_=verbose):
- """ Extract the license from packages in repo_,arch_ and in pending_ file"""
- cmd_=home + "/usr/bin/get_license.sh"
- printf(cmd_)
- a=commands.getoutput(cmd_)
- if verbose_: printf(a)
+ """ Extract the license from packages in repo_,arch_ and in pending_ file"""
+ cmd_=home + "/usr/bin/get_license.sh"
+ printf(cmd_)
+ a=commands.getoutput(cmd_)
+ if verbose_: printf(a)
def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdir=repodir,
source=mirror+mirrorpath, blacklist_file=False):
@@ -207,33 +214,33 @@ def run_rsync(command,debug=verbose):
return commands.getoutput(command)
if __name__ == "__main__":
- from time import time
- start_time = time()
- def minute():
- return str(round((time() - start_time)/60, 1))
-
- printf(" Cleaning %s folder " % (tmp) )
- commands.getoutput("rm -r %s/*" % tmp)
- printf(" Syncing repo")
- sync_all_repo(True)
-
- printf(" Updating databases and pending files lists: minute %s \n" % minute() )
- for repo in repo_list:
- for arch in arch_list:
- printf( "\n" + repo + "-" + arch + "\n" )
- printf( "Get info: minute %s " % minute() )
- info=get_info(repo,arch)
- printf( "Make pending: minute %s" % minute() )
- make_pending(repo,arch,info)
- printf( "Update DB: minute %s" % minute() )
- remove_from_blacklist(
- repo, arch, info, tuple( listado(blacklist) + listado(pending + "-" + repo + ".txt") ) )
-
- printf("Adding Parabola Packages: minute %s\n" % minute() )
- add_free_repo(True)
-
- printf("Extracting licenses in pending: minute %s" % minute() )
- get_licenses()
-
- printf("\n\nDelay: %s minutes \n" % minute())
-
+ from time import time
+ start_time = time()
+ def minute():
+ return str(round((time() - start_time)/60, 1))
+
+ printf(" Cleaning %s folder " % (tmp) )
+ commands.getoutput("rm -r %s/*" % tmp)
+ printf(" Syncing repo")
+ sync_all_repo(True)
+
+ printf(" Updating databases and pending files lists: minute %s \n" % minute() )
+ for repo in repo_list:
+ for arch in arch_list:
+ printf( "\n" + repo + "-" + arch + "\n" )
+ printf( "Get info: minute %s " % minute() )
+ info=get_info(repo,arch)
+ printf( "Make pending: minute %s" % minute() )
+ make_pending(repo,arch,info)
+ printf( "Update DB: minute %s" % minute() )
+ remove_from_blacklist(
+ repo, arch, info, tuple( listado(blacklist) + listado(pending + "-" + repo + ".txt") ) )
+
+ printf("Adding Parabola Packages: minute %s\n" % minute() )
+ add_free_repo(True)
+
+ printf("Extracting licenses in pending: minute %s" % minute() )
+ get_licenses()
+
+ printf("\n\nDelay: %s minutes \n" % minute())
+