summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xclean_repo.py78
-rwxr-xr-xconfig.py65
-rwxr-xr-xfilter.py48
-rw-r--r--pato2.py63
-rw-r--r--test/test_filter.py3
5 files changed, 88 insertions, 169 deletions
diff --git a/clean_repo.py b/clean_repo.py
index eccfd01..5423c4d 100755
--- a/clean_repo.py
+++ b/clean_repo.py
@@ -3,21 +3,16 @@
from repm.filter import *
import argparse
-def mkpending(path_to_db, repo, prefix=config["pending"]):
+def mkpending(packages_iterable, pending_file, blacklisted_names,
+ whitelisted_names):
""" Determine wich packages are pending for license auditing."""
- if "~" in path_to_db:
- path_to_db=(os.path.expanduser(path_to_db))
-
- search = tuple(listado(config["blacklist"]) +
- listado(config["whitelist"]))
+ search = tuple(blacklisted_names +
+ whitelisted_names)
- pkgs=list(pkginfo_from_db(path_to_db))
-
- filename=prefix + "-" + repo + ".txt"
try:
- fsock=open(filename, "rw")
- pkgs=[pkg for pkg in pkginfo_from_db(path_to_db)
- if pkg["name"] not in listado(filename)]
+ fsock=open(pending_file, "r")
+ pkgs=[pkg for pkg in packages_iterable
+ if pkg["name"] not in listado(pending_file)]
for line in fsock.readlines():
if line:
pkg=Package()
@@ -26,16 +21,16 @@ def mkpending(path_to_db, repo, prefix=config["pending"]):
pkgs.append(pkg)
pkgs=[pkg for pkg in pkgs if pkg["name"] not in search
and "custom" in pkg["license"]]
+ fsock=open(pending_file, "w")
fsock.write("\n".join([pkg["name"] + ":" + pkg["license"]
for pkg in pkgs]) + "\n")
except(IOError):
- raise NonValidFile("Can't read or write %s" % filename)
+ raise NonValidFile("Can't read or write %s" % pending_file)
finally:
fsock.close()
return pkgs
-def remove_from_blacklist(path_to_db, blacklisted_names,
- debug=config["debug"]):
+def remove_from_blacklist(path_to_db, blacklisted_names):
""" Check the blacklist and remove packages on the db"""
if "~" in path_to_db:
path_to_db=(os.path.expanduser(path_to_db))
@@ -47,9 +42,7 @@ def remove_from_blacklist(path_to_db, blacklisted_names,
cmd = "repo-remove " + path_to_db + " " + lista
printf(cmd)
a = check_output(cmd)
- if debug:
- printf(a)
- return pkgs, cmd
+ return pkgs
def cleanup_nonfree_in_dir(directory, blacklisted_names):
if "~" in directory:
@@ -61,25 +54,42 @@ def cleanup_nonfree_in_dir(directory, blacklisted_names):
if __name__ == "__main__":
parser = argparse.ArgumentParser(
- description="Clean a repo db and packages")
- parser.add_argument("-b", "--database", type=str,
- help="dabatase to clean")
- parser.add_argument("-d", "--directory", type=str,
- help="directory to clean")
+ prog="clean_repo",
+ description="Clean a repo db and packages",)
+
+ parser.add_argument("-k", "--blacklist-file", type=str,
+ help="File containing blacklisted names",
+ required=True,)
+
+ group_dir=parser.add_argument_group("Clean non-free packages in dir")
+ group_dir.add_argument("-d", "--directory", type=str,
+ help="directory to clean",)
+
+ group_db=parser.add_argument_group("Clean non-free packages in db",
+ "All arguments need to be specified")
+ group_db.add_argument("-b", "--database", type=str,
+ help="dabatase to clean")
+ group_db.add_argument("-p", "--pending-file", type=str,
+ help="File in which to write pending list")
+ group_db.add_argument("-w", "--whitelist-file", type=str,
+ help="File containing whitelisted names")
+
args=parser.parse_args()
+ if not args.directory and not args.database:
+ parser.print_help()
+ elif not args.pending_file or not args.whitelist_file \
+ and args.database:
+ parser.print_help()
+ else:
+ blacklisted=listado(args.blacklist_file)
+
if args.database:
- repo=os.path.basename(args.database).split(".")[0]
+ whitelisted=listado(args.whitelist_file)
pkgs=pkginfo_from_db(args.database)
- remove_from_blacklist(args.database, pkgs,
- tuple(listado(config["blacklist"]) +
- listado(config["pending"] +
- "-" + repo + ".txt")))
- mkpending(args.database, args.repo)
+ remove_from_blacklist(args.database, blacklisted)
+ mkpending(pkgs, args.pending_file,
+ blacklisted, whitelisted)
if args.directory:
- cleanup_nonfree_in_dir(args.directory,
- listado(config["blacklist"]))
-
- if not args.directory and not args.database:
- parser.print_help()
+ cleanup_nonfree_in_dir(args.directory, blacklisted)
diff --git a/config.py b/config.py
index 8b48c66..4e218a5 100755
--- a/config.py
+++ b/config.py
@@ -9,61 +9,26 @@ except(ImportError):
return getoutput(cmd)
import os
-stringvars=("mirror", "mirrorpath", "logname", "tempdir", "archdb",
- "repodir", "blacklist", "whitelist", "pending",
- "rsync_blacklist",)
-listvars=("repo_list", "dir_list", "arch_list", "other",)
-boolvars=("output", "debug",)
-
-config=dict()
-
-def exit_if_none(var):
- if os.environ.get(var) is None:
- exit("%s is not defined" % var)
-
-for var in stringvars:
- exit_if_none(var)
- config[var]=os.environ.get(var)
-
-for var in listvars:
- exit_if_none(var)
- config[var]=tuple(os.environ.get(var).split(":"))
-
-for var in boolvars:
- exit_if_none(var)
- if os.environ.get(var) == "True":
- config[var]=True
- elif os.environ.get(var) =="False":
- config[var]=False
- else:
- print('%s is not True or False' % var)
# Rsync commands
-rsync_list_command="rsync -a --no-motd --list-only "
-def printf(text,output=config["output"]):
+def printf(text, logfile=False):
"""Guarda el texto en la variable log y puede imprimir en pantalla."""
- log_file = open(config["logname"], 'a')
- log_file.write("\n" + str(text) + "\n")
- log_file.close()
- if output:
- print (str(text) + "\n")
+ print (str(text) + "\n")
+ if logfile:
+ try:
+ log = open(logfile, 'a')
+ log.write("\n" + str(text) + "\n")
+ except:
+ print("Can't open %s" % logfile)
+ finally:
+ log.close()
-del exit_if_none
# Classes and Exceptions
-class NonValidFile(ValueError):
- def __init__(self):
- ValueError.__init__(self)
- printf(self.message)
-class NonValidDir(ValueError):
- def __init__(self):
- ValueError.__init__(self)
- printf(self.message)
-class NonValidCommand(ValueError):
- def __init__(self):
- ValueError.__init__(self)
- printf(self.message)
+class NonValidFile(ValueError): pass
+class NonValidDir(ValueError): pass
+class NonValidCommand(ValueError): pass
class Package:
""" An object that has information about a package. """
@@ -101,7 +66,3 @@ class Package:
return False
else:
return True
-
-if __name__=="__main__":
- for key in config.keys():
- print("%s : %s" % (key,config[key]))
diff --git a/filter.py b/filter.py
index 4b9603d..add0235 100755
--- a/filter.py
+++ b/filter.py
@@ -2,7 +2,7 @@
#-*- encoding: utf-8 -*-
from glob import glob
from repm.config import *
-from repm.pato2 import *
+import tarfile
def listado(filename,start=0,end=None):
"""Obtiene una lista de paquetes de un archivo."""
@@ -28,7 +28,7 @@ def pkginfo_from_filename(filename):
----------
pkg -> Package object"""
if ".pkg.tar." not in filename:
- raise NonValidFile
+ raise NonValidFile("File is not a pacman package")
pkg = Package()
pkg["location"] = filename
fileattrs = os.path.basename(filename).split("-")
@@ -140,19 +140,18 @@ def pkginfo_from_db(path_to_db):
desc_files=[desc for desc in dbsock.getnames()
if "/desc" in desc]
for name in desc_files:
- desc=dbsock.extractfile(name)
- package_list.append(pkginfo_from_desc(desc.read()))
+ desc=dbsock.extractfile(name).read().decode("UTF-8")
+ package_list.append(pkginfo_from_desc(desc))
except tarfile.ReadError:
raise NonValidFile("No valid db_file %s or not readable"
% path_to_db)
- return(tuple())
finally:
dbsock.close()
return package_list
def rsyncBlacklist_from_blacklist(packages_iterable,
blacklisted_names,
- exclude_file=config["rsync_blacklist"]):
+ exclude_file):
""" Generate an exclude list for rsync
Parameters:
@@ -168,20 +167,31 @@ def rsyncBlacklist_from_blacklist(packages_iterable,
pkgs=[pkg["location"] for pkg in packages_iterable
if isinstance(pkg, Package)
and pkg["name"] in blacklisted_names]
-
- try:
- fsock = open(exclude_file,"w")
- fsock.write("\n".join(pkgs) + "\n")
- except IOError:
- printf("%s wasnt written" % exclude_file)
- exit(1)
- finally:
- fsock.close()
+ if exclude_file:
+ try:
+ fsock = open(exclude_file,"w")
+ fsock.write("\n".join(pkgs) + "\n")
+ except IOError:
+ printf("%s wasnt written" % exclude_file)
+ exit(1)
+ finally:
+ fsock.close()
return pkgs
if __name__ == "__main__":
- cmd=generate_rsync_command(rsync_list_command)
- a=run_rsync(cmd)
- packages=pkginfo_from_rsync_output(a)
- rsyncBlaclist_from_blacklist(packages,listado(blacklist))
+ import argparse
+ parser=argparse.ArgumentParser()
+ parser.add_argument("-r", "--rsync-exclude-file", type=str,
+ help="File in which to generate exclude list",
+ required=True,)
+ parser.add_argument("-k", "--blacklist-file", type=str,
+ help="File containing blacklisted names",
+ required=True,)
+ parser.add_argument("-c", "--rsync-command", type=str,
+ help="This command will be run to get a pkg list")
+ args=parser.parse_args()
+ rsout=check_output(args.rsync_command)
+ packages=pkginfo_from_rsync_output(rsout)
+ rsyncBlaclist_from_blacklist(packages, listado(args.blacklist_file),
+ args.rsync_exclude_file)
diff --git a/pato2.py b/pato2.py
deleted file mode 100644
index 6daa8b8..0000000
--- a/pato2.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-"""
- parabola.py
- Copyright 2009 Rafik Mas'ad
- Copyright 2010 Joshua Ismael Haase Hernández
-
- ---------- GNU General Public License 3 ----------
-
- This file is part of Parabola.
-
- Parabola is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- Parabola is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with Parabola. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-from repm.config import *
-from repm.filter import *
-import tarfile
-from os.path import isdir, isfile
-
-def generate_rsync_command(base_command,
- dir_list=(config["repo_list"] +
- config["dir_list"]),
- destdir=config["repodir"],
- source=config["mirror"] +config["mirrorpath"]):
- """ Generates an rsync command for executing
- it by combining all parameters.
-
- Parameters:
- ----------
- base_command -> str
- dir_list -> list or tuple
- destdir -> str Path to dir, dir must exist.
- source -> str The source for rsync
- blacklist_file -> False or str Path to file, file must exist.
-
- Return:
- ----------
- rsync_command -> str """
- if not os.path.isdir(destdir):
- print(destdir + " is not a directory")
- raise NonValidDir
-
- dir_list="{" + ",".join(dir_list) + "}"
- return " ".join((base_command, os.path.join(source, dir_list),
- destdir))
-
-def run_rsync(command,debug=config["debug"]):
- """ Runs rsync and gets returns it's output """
- if debug:
- printf("rsync_command: " + command)
- return check_output(command.split())
diff --git a/test/test_filter.py b/test/test_filter.py
index b6d5766..d8006f9 100644
--- a/test/test_filter.py
+++ b/test/test_filter.py
@@ -143,7 +143,8 @@ class generateRsyncBlacklist(unittest.TestCase):
def testExcludeFiles(self):
a=rsyncBlacklist_from_blacklist(self.example_package_list,
- listado("blacklist_sample"))
+ listado("blacklist_sample"),
+ False)
b=[self.example_package_list[0]["location"],self.example_package_list[2]["location"]]
self.assertEqual(a,b)