diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | TODO | 6 | ||||
| -rw-r--r-- | __init__.py | 0 | ||||
| -rw-r--r-- | changelog | 2 | ||||
| -rwxr-xr-x | clean_repo.py | 95 | ||||
| -rwxr-xr-x | config.py | 68 | ||||
| -rwxr-xr-x | config.sh | 26 | ||||
| -rwxr-xr-x | cptobin.sh | 2 | ||||
| -rw-r--r-- | db-functions | 1 | ||||
| -rwxr-xr-x | filter.py | 197 | ||||
| -rwxr-xr-x | get_license.sh | 55 | ||||
| -rwxr-xr-x | libremessages | 77 | ||||
| -rw-r--r-- | local_config.example | 35 | ||||
| -rwxr-xr-x | main.sh | 43 | ||||
| -rwxr-xr-x | repo-state | 11 | ||||
| -rw-r--r-- | test/__init__.py | 0 | ||||
| -rw-r--r-- | test/blacklist_sample | 2 | ||||
| -rw-r--r-- | test/core.db.tar.gz | bin | 0 -> 1345 bytes | |||
| -rw-r--r-- | test/depends | 4 | ||||
| -rw-r--r-- | test/desc | 39 | ||||
| -rw-r--r-- | test/rsync_output_sample | 14 | ||||
| -rw-r--r-- | test/test_filter.py | 196 |
22 files changed, 875 insertions, 0 deletions
@@ -1,3 +1,5 @@ *~ +*.pyc +local_config /config.local test/packages/*/*.pkg.tar.* @@ -0,0 +1,6 @@ +* Test Suite + + - Review all repo efectively + - Remove all blacklisted packages + - Get pending list right + - Extract licenses all right
\ No newline at end of file diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/__init__.py diff --git a/changelog b/changelog new file mode 100644 index 0000000..0e89f2f --- /dev/null +++ b/changelog @@ -0,0 +1,2 @@ +repo-maintainer 1.0 + * Known working version diff --git a/clean_repo.py b/clean_repo.py new file mode 100755 index 0000000..5423c4d --- /dev/null +++ b/clean_repo.py @@ -0,0 +1,95 @@ +#! /usr/bin/python +#-*- encoding: utf-8 -*- +from repm.filter import * +import argparse + +def mkpending(packages_iterable, pending_file, blacklisted_names, + whitelisted_names): + """ Determine wich packages are pending for license auditing.""" + search = tuple(blacklisted_names + + whitelisted_names) + + try: + fsock=open(pending_file, "r") + pkgs=[pkg for pkg in packages_iterable + if pkg["name"] not in listado(pending_file)] + for line in fsock.readlines(): + if line: + pkg=Package() + pkg["name"]=line.split(":")[0] + pkg["license"]=":".join(line.split(":")[1:]) + pkgs.append(pkg) + pkgs=[pkg for pkg in pkgs if pkg["name"] not in search + and "custom" in pkg["license"]] + fsock=open(pending_file, "w") + fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] + for pkg in pkgs]) + "\n") + except(IOError): + raise NonValidFile("Can't read or write %s" % pending_file) + finally: + fsock.close() + return pkgs + +def remove_from_blacklist(path_to_db, blacklisted_names): + """ Check the blacklist and remove packages on the db""" + if "~" in path_to_db: + path_to_db=(os.path.expanduser(path_to_db)) + + pkgs=[pkg for pkg in pkginfo_from_db(path_to_db) if + pkg["name"] in blacklisted_names] + if pkgs: + lista=" ".join(pkgs) + cmd = "repo-remove " + path_to_db + " " + lista + printf(cmd) + a = check_output(cmd) + return pkgs + +def cleanup_nonfree_in_dir(directory, blacklisted_names): + if "~" in directory: + directory=(os.path.expanduser(directory)) + pkgs=pkginfo_from_files_in_dir(directory) + for package in pkgs: + if package["name"] in blacklisted_names: + os.remove(package["location"]) + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="clean_repo", + description="Clean a repo db and packages",) + + parser.add_argument("-k", "--blacklist-file", type=str, + help="File containing blacklisted names", + required=True,) + + group_dir=parser.add_argument_group("Clean non-free packages in dir") + group_dir.add_argument("-d", "--directory", type=str, + help="directory to clean",) + + group_db=parser.add_argument_group("Clean non-free packages in db", + "All arguments need to be specified") + group_db.add_argument("-b", "--database", type=str, + help="dabatase to clean") + group_db.add_argument("-p", "--pending-file", type=str, + help="File in which to write pending list") + group_db.add_argument("-w", "--whitelist-file", type=str, + help="File containing whitelisted names") + + args=parser.parse_args() + + if not args.directory and not args.database: + parser.print_help() + elif not args.pending_file or not args.whitelist_file \ + and args.database: + parser.print_help() + else: + blacklisted=listado(args.blacklist_file) + + if args.database: + whitelisted=listado(args.whitelist_file) + pkgs=pkginfo_from_db(args.database) + remove_from_blacklist(args.database, blacklisted) + mkpending(pkgs, args.pending_file, + blacklisted, whitelisted) + + if args.directory: + cleanup_nonfree_in_dir(args.directory, blacklisted) diff --git a/config.py b/config.py new file mode 100755 index 0000000..4e218a5 --- /dev/null +++ b/config.py @@ -0,0 +1,68 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +try: + from subprocess import check_output +except(ImportError): + from commands import getoutput + def check_output(*popenargs,**kwargs): + cmd=" ".join(*popenargs) + return getoutput(cmd) +import os + + +# Rsync commands + +def printf(text, logfile=False): + """Guarda el texto en la variable log y puede imprimir en pantalla.""" + print (str(text) + "\n") + if logfile: + try: + log = open(logfile, 'a') + log.write("\n" + str(text) + "\n") + except: + print("Can't open %s" % logfile) + finally: + log.close() + + +# Classes and Exceptions +class NonValidFile(ValueError): pass +class NonValidDir(ValueError): pass +class NonValidCommand(ValueError): pass + +class Package: + """ An object that has information about a package. """ + package_info=dict() + + def __init__(self): + self.package_info={ "name" : False, + "version" : False, + "release" : False, + "arch" : False, + "license" : False, + "location": False, + "depends" : False,} + + def __setitem__(self, key, item): + if key in self.package_info.keys(): + return self.package_info.__setitem__(key, item) + else: + raise ValueError("Package has no %s attribute" % key) + + def __getitem__(self, key): + return self.package_info.__getitem__(key) + + def __unicode__(self): + return str(self.package_info) + + def __repr__(self): + return str(self.package_info) + + def __eq__(self,x): + if not isinstance(x, Package): + return False + for key in self.package_info.keys(): + if x[key] != self.package_info[key]: + return False + else: + return True diff --git a/config.sh b/config.sh new file mode 100755 index 0000000..9a44f50 --- /dev/null +++ b/config.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# -*- coding: utf-8 -*- +source local_config + +function run_python_cmd { + env \ + mirror=${mirror} \ + mirrorpath=${mirrorpath} \ + logname=${logname} \ + tempdir=${tempdir} \ + archdb=${archdb} \ + repodir=${repodir} \ + blacklist=${blacklist} \ + whitelist=${whitelist} \ + pending=${pending} \ + rsync_blacklist=${rsync_blacklist} \ + repo_list=${repo_list} \ + dir_list=${dir_list} \ + arch_list=${arch_list} \ + other=${other} \ + output=${output} \ + debug=${debug} \ + $1 +} + +source libremessages
\ No newline at end of file diff --git a/cptobin.sh b/cptobin.sh new file mode 100755 index 0000000..ce4b3bd --- /dev/null +++ b/cptobin.sh @@ -0,0 +1,2 @@ +#!/bin/bash +cp -f clean_repo.py config.py config.sh get_license.sh main.sh filter.py local_config ~/usr/bin/
\ No newline at end of file diff --git a/db-functions b/db-functions index b900669..1780e4f 100644 --- a/db-functions +++ b/db-functions @@ -351,6 +351,7 @@ check_splitpkgs() { for pkgfile in ${pkgfiles[@]}; do issplitpkg "${pkgfile}" || continue local _pkgbase="$(getpkgbase ${pkgfile})" + msg2 "Checking $_pkgbase" local _pkgname="$(getpkgname ${pkgfile})" local _pkgarch="$(getpkgarch ${pkgfile})" mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" diff --git a/filter.py b/filter.py new file mode 100755 index 0000000..add0235 --- /dev/null +++ b/filter.py @@ -0,0 +1,197 @@ + #! /usr/bin/python +#-*- encoding: utf-8 -*- +from glob import glob +from repm.config import * +import tarfile + +def listado(filename,start=0,end=None): + """Obtiene una lista de paquetes de un archivo.""" + fsock = open(filename,"r") + lista = fsock.read().split("\n") + fsock.close() + if end is not None: + return [pkg.split(":")[start:end].rstrip() + for pkg in lista if pkg] + else: + return [pkg.split(":")[start].rstrip() + for pkg in lista if pkg] + +def pkginfo_from_filename(filename): + """ Generates a Package object with info from a filename, + filename can be relative or absolute + + Parameters: + ---------- + filename -> str Must contain .pkg.tar. + + Returns: + ---------- + pkg -> Package object""" + if ".pkg.tar." not in filename: + raise NonValidFile("File is not a pacman package") + pkg = Package() + pkg["location"] = filename + fileattrs = os.path.basename(filename).split("-") + pkg["arch"] = fileattrs.pop(-1).split(".")[0] + pkg["release"] = fileattrs.pop(-1) + pkg["version"] = fileattrs.pop(-1) + pkg["name"] = "-".join(fileattrs) + return pkg + +def pkginfo_from_desc(info_from_desc, pkg=Package()): + """ Returns pkginfo from desc file. + + Parameters: + ---------- + filename -> str File must exist + + Returns: + ---------- + pkg -> Package object""" + info=info_from_desc.rsplit() + info_map={"name" :("%NAME%" , None), + "version" :("%VERSION%" , 0 ), + "release" :("%VERSION%" , 1 ), + "arch" :("%ARCH%" , None), + "license" :("%LICENSE%" , None), + "location":("%FILENAME%", None),} + + for key in info_map.keys(): + field,pos=info_map[key] + pkg[key]=info[info.index(field)+1] + if pos is not None: + pkg[key]=pkg[key].split("-")[pos] + return pkg + +def pkginfo_from_rsync_output(rsync_output): + """ Generates a list of packages and versions from an rsync output + wich uses --list-only and --no-motd options. + + Parameters: + ---------- + rsync_output -> str Contains output from rsync + + Returns: + ---------- + package_list -> tuple Contains Package objects. """ + + def package_or_link(line): + """ Take info out of filename """ + location_field = 4 + return pkginfo_from_filename(line.rsplit()[location_field]) + + def do_nothing(): + pass + + options = { "d": do_nothing, + "l": package_or_link, + "-": package_or_link, + " ": do_nothing} + + package_list=list() + + lines=[x for x in rsync_output.split("\n") if ".pkg.tar" in x] + + for line in lines: + pkginfo=options[line[0]](line) + if pkginfo: + package_list.append(pkginfo) + + return tuple(package_list) + +def pkginfo_from_files_in_dir(directory): + """ Returns pkginfo from filenames of packages in dir + wich has .pkg.tar. on them + + Parameters: + ---------- + directory -> str Directory must exist + + Returns: + ---------- + package_list -> tuple Contains Package objects """ + package_list=list() + + if not os.path.isdir(directory): + raise NonValidDir + + for filename in glob(os.path.join(directory,"*")): + if ".pkg.tar." in filename: + package_list.append(pkginfo_from_filename(filename)) + return tuple(package_list) + +def pkginfo_from_db(path_to_db): + """ Get pkginfo from db. + + Parameters: + ---------- + path_to_db -> str Path to file + + Output: + ---------- + package_list -> tuple of Package objects""" + package_list=list() + + if not os.path.isfile(path_to_db): + raise NonValidFile(path_to_db + " is not a file") + + try: + dbsock = tarfile.open(path_to_db, 'r:gz') + desc_files=[desc for desc in dbsock.getnames() + if "/desc" in desc] + for name in desc_files: + desc=dbsock.extractfile(name).read().decode("UTF-8") + package_list.append(pkginfo_from_desc(desc)) + except tarfile.ReadError: + raise NonValidFile("No valid db_file %s or not readable" + % path_to_db) + finally: + dbsock.close() + return package_list + +def rsyncBlacklist_from_blacklist(packages_iterable, + blacklisted_names, + exclude_file): + """ Generate an exclude list for rsync + + Parameters: + ---------- + package_iterable -> list or tuple Contains Package objects + blacklisted_names-> list or tuple Contains blacklisted names + exclude_file -> str Path to file + debug -> bool If True, file list gets logged + + Output: + ---------- + None """ + pkgs=[pkg["location"] for pkg in packages_iterable + if isinstance(pkg, Package) + and pkg["name"] in blacklisted_names] + if exclude_file: + try: + fsock = open(exclude_file,"w") + fsock.write("\n".join(pkgs) + "\n") + except IOError: + printf("%s wasnt written" % exclude_file) + exit(1) + finally: + fsock.close() + return pkgs + + +if __name__ == "__main__": + import argparse + parser=argparse.ArgumentParser() + parser.add_argument("-r", "--rsync-exclude-file", type=str, + help="File in which to generate exclude list", + required=True,) + parser.add_argument("-k", "--blacklist-file", type=str, + help="File containing blacklisted names", + required=True,) + parser.add_argument("-c", "--rsync-command", type=str, + help="This command will be run to get a pkg list") + args=parser.parse_args() + rsout=check_output(args.rsync_command) + packages=pkginfo_from_rsync_output(rsout) + rsyncBlaclist_from_blacklist(packages, listado(args.blacklist_file), + args.rsync_exclude_file) diff --git a/get_license.sh b/get_license.sh new file mode 100755 index 0000000..0da58cb --- /dev/null +++ b/get_license.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# -*- coding: utf-8 -*- + + # get_license.sh + # Copyright 2010 Joshua Ismael Haase Hernández + + # ---------- GNU General Public License 3 ---------- + + # This file is part of Parabola. + + # Parabola is free software: you can redistribute it and/or modify + # it under the terms of the GNU General Public License as published by + # the Free Software Foundation, either version 3 of the License, or + # (at your option) any later version. + + # Parabola is distributed in the hope that it will be useful, + # but WITHOUT ANY WARRANTY; without even the implied warranty of + # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + # GNU General Public License for more details. + + # You should have received a copy of the GNU General Public License + # along with Parabola. If not, see <http://www.gnu.org/licenses/>. + +docs="/home/parabolavnx/parabolagnulinux.org/docs" +repo="/home/parabolavnx/parabolagnulinux.org/repo" +dir="$docs/pending-licenses" + +echo "Cleaning $dir" +rm -rf $dir/* + +tempdir=$(mktemp -d) +cd $tempdir + +pending=($(cut -d: -f1 $docs/pending*.txt)) +echo ${pending[@]} + +for pkg in ${pending[@]}; do + pkg_in_repo=( $(ls ${repo}/*/os/*/${pkg}*) ) + for y in ${pkg_in_repo[@]}; do + echo "chmod +r $y" + chmod +r $y + echo "tar -xf $y usr/share/licenses" + bsdtar -xf $y usr/share/licenses + echo "chmod -r $y" + chmod -r $y + done +done + +mv usr/share/licenses/* $dir + +cd + +rm -rf $tempdir + +exit 0
\ No newline at end of file diff --git a/libremessages b/libremessages new file mode 100755 index 0000000..9fbbc2b --- /dev/null +++ b/libremessages @@ -0,0 +1,77 @@ +# Copyright (c) 2006-2010 Pacman Development Team <pacman-dev@archlinux.org> +# Copyright (c) 2002-2006 by Judd Vinet <jvinet@zeroflux.org> +# Copyright (c) 2005 by Aurelien Foret <orelien@chez.com> +# Copyright (c) 2006 by Miklos Vajna <vmiklos@frugalware.org> +# Copyright (c) 2005 by Christian Hamar <krics@linuxforum.hu> +# Copyright (c) 2006 by Alex Smith <alex@alex-smith.me.uk> +# Copyright (c) 2006 by Andras Voroskoi <voroskoi@frugalware.org> +# Copyright (c) 2011 by Joshua Haase <hahj87@gmail.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# gettext initialization +export TEXTDOMAIN='libretools' +export TEXTDOMAINDIR='/usr/share/locale' + +# check if messages are to be printed using color +unset ALL_OFF BOLD BLUE GREEN RED YELLOW + +if tput setaf 0 &>/dev/null; then + ALL_OFF="$(tput sgr0)" + BOLD="$(tput bold)" + BLUE="${BOLD}$(tput setaf 4)" + GREEN="${BOLD}$(tput setaf 2)" + RED="${BOLD}$(tput setaf 1)" + YELLOW="${BOLD}$(tput setaf 3)" + PURPLE="${ALL_OFF}$(tput setaf 5)" +else + ALL_OFF="\033[1;0m" + BOLD="\033[1;1m" + BLUE="${BOLD}\033[1;34m" + GREEN="${BOLD}\033[1;32m" + RED="${BOLD}\033[1;31m" + YELLOW="${BOLD}\033[1;33m" + PURPLE="${BOLD}\033[1;30;40m" +fi + +stdnull() { + local action=$1; + eval "${action} >/dev/null 2>&1" +} + +plain() { + local mesg=$1; shift + printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +msg() { + local mesg=$1; shift + printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +msg2() { + local mesg=$1; shift + printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +warning() { + local mesg=$1; shift + printf "${YELLOW}==> $(gettext "WARNING:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +error() { + local mesg=$1; shift + printf "${RED}==> $(gettext "ERROR:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + diff --git a/local_config.example b/local_config.example new file mode 100644 index 0000000..99a1fdb --- /dev/null +++ b/local_config.example @@ -0,0 +1,35 @@ +# Mirror options +mirror="mirrors.eu.kernel.org" +mirrorpath="::mirrors/archlinux" + +# Directories: they should end without / +## Optionals +paraboladir=~/parabolagnulinux.org +logtime=$(date -u +%Y%m%d-%H:%M) +## Must be defined +tempdir=~/tmp +archdb=${tempdir}/db +docs_dir=${paraboladir}/docs +repodir=${paraboladir}/repo +# End Directories + +# Files +logname=${paraboladir}/log/${logtime}-repo-maintainer.log +blacklist=${docs_dir}/blacklist.txt +whitelist=${docs_dir}/whitelist.txt +pending=${docs_dir}/pending +rsync_blacklist=${docs_dir}/rsyncBlacklist + +# Repos, arches, and dirs for repo +repo_list="core:extra:community:testing:community-testing:multilib" +dir_list="pool" +arch_list="i686:x86_64" +other="any" + +# Output options +output="True" +debug="False" + +# Rsync commands +rsync_update_command="rsync -nav --delay-updates --exclude='*.{abs|db}.tar.*' " +rsync_post_command="rsync -nav --delete --exclude='*.abs.tar.*' "
\ No newline at end of file @@ -0,0 +1,43 @@ +#!/bin/bash +# -*- coding: utf-8 -*- + +source config.sh + +function mkrsexclude { + local error=1 + while ${error}; do + run_python_cmd "filter.py" + error=$? + done +} + +msg "Cleaning $tempdir" +stdnull "rm -r $tempdir/* " + +msg "Generating exclude list for rsync" +mkrsexclude + +msg "Syncing repos without delete" +# rsync_update_command does not sync db or abs +${rsync_update_command} --exclude-from=${rsync_blacklist} \ + ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ','),\ + $(echo ${dir_list} | tr ':' ',')} ${repodir} + +msg "Syncing each repo and cleaning" +msg2 "Remove pending files" +stdnull "rm -rf ${pending}*" +for repo in $(echo ${repo_list} | tr ':' ' '); do + for arch in $(echo ${arch_list} | tr ':' ' '); do + msg2 "Syncing ${repo} ${arch}" + cmd=$(echo ${rsync_post_command} --exclude-from=${rsync_blacklist} \ + ${mirror}${mirropath}/${repo} ${repodir}/${repo}) + plain "${cmd}" + ${cmd} + msg2 "Cleaning ${repo} ${arch}" + # This also generates pending lists + run_python_cmd "clean_repo.py -b ${repodir}/${repo}/os/${arch}/${repo}.db.tar.gz -d ${repodir}/${repo}/os/${arch}/" + done +done + +msg "Checking licenses" +get_license.sh diff --git a/repo-state b/repo-state new file mode 100755 index 0000000..319db83 --- /dev/null +++ b/repo-state @@ -0,0 +1,11 @@ +#!/bin/sh + +a=$(date +%Y%m%d+%H%M) +mkdir -p $a +cd $a + +. /home/joshpar/programas/arch2parabola/repo-list-diff +# scp parabolavnx@parabolagnulinux.org:~/tmp/rsyncBlacklist.txt ./ + +cd .. +exit 0
\ No newline at end of file diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/__init__.py diff --git a/test/blacklist_sample b/test/blacklist_sample new file mode 100644 index 0000000..2a02af6 --- /dev/null +++ b/test/blacklist_sample @@ -0,0 +1,2 @@ +alex:alex-libre: Aquí va un comentario +gmime22 ::Non free dependencies
\ No newline at end of file diff --git a/test/core.db.tar.gz b/test/core.db.tar.gz Binary files differnew file mode 100644 index 0000000..5eb2081 --- /dev/null +++ b/test/core.db.tar.gz diff --git a/test/depends b/test/depends new file mode 100644 index 0000000..7ff3ad4 --- /dev/null +++ b/test/depends @@ -0,0 +1,4 @@ +%DEPENDS% +glibc>=2.13 +zlib + diff --git a/test/desc b/test/desc new file mode 100644 index 0000000..abba644 --- /dev/null +++ b/test/desc @@ -0,0 +1,39 @@ +%FILENAME% +binutils-2.21-4-x86_64.pkg.tar.xz + +%NAME% +binutils + +%VERSION% +2.21-4 + +%DESC% +A set of programs to assemble and manipulate binary and object files + +%GROUPS% +base + +%CSIZE% +3412892 + +%ISIZE% +17571840 + +%MD5SUM% +4e666f87c78998f4839f33dc06d2043a + +%URL% +http://www.gnu.org/software/binutils/ + +%LICENSE% +GPL + +%ARCH% +x86_64 + +%BUILDDATE% +1297240369 + +%PACKAGER% +Allan McRae <allan@archlinux.org> + diff --git a/test/rsync_output_sample b/test/rsync_output_sample new file mode 100644 index 0000000..72d9cd0 --- /dev/null +++ b/test/rsync_output_sample @@ -0,0 +1,14 @@ +dr-xr-sr-x 4096 2010/09/11 11:37:10 . +-rw-r--r-- 11 2011/02/08 00:00:01 lastsync +drwxrwxr-x 15 2010/09/11 11:28:50 community-staging +drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os +drwxrwxr-x 8192 2011/02/07 17:00:01 community-staging/os/i686 +lrwxrwxrwx 52 2010/12/23 16:51:01 community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz -> ../../../pool/community/alex-2.3.4-1-i686.pkg.tar.xz +lrwxrwxrwx 27 2011/02/07 14:02:54 community-staging/os/i686/community-staging.db -> community-staging.db.tar.gz +-rw-rw-r-- 2237 2011/02/07 14:02:54 community-staging/os/i686/community-staging.db.tar.gz +-rw-rw-r-- 3209 2011/02/07 14:00:13 community-staging/os/i686/community-staging.db.tar.gz.old +drwxrwxr-x 15 2009/07/22 15:07:56 community +drwxrwxr-x 40 2009/08/04 15:57:42 community/os +drwxrwsr-x 36864 2011/02/03 05:00:01 community/os/any +-rw-rw-r-- 303336 2010/07/16 10:06:28 community/os/any/any2dvd-0.34-4-any.pkg.tar.xz +-rw-rw-r-- 221664 2010/03/28 15:55:48 community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz diff --git a/test/test_filter.py b/test/test_filter.py new file mode 100644 index 0000000..d8006f9 --- /dev/null +++ b/test/test_filter.py @@ -0,0 +1,196 @@ +# -*- encoding: utf-8 -*- +""" """ + +__author__ = "Joshua Ismael Haase Hernández <hahj87@gmail.com>" +__version__ = "$Revision: 1.1 $" +__date__ = "$Date: 2011/02/08 $" +__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" +__license__ = "GPL3+" + +from repm.config import * +from repm.filter import * +import unittest + +class pkginfo_from_file_KnownValues(unittest.TestCase): + # (filename, name, version, release, arch) + # filename is location + known=( + ("community-testing/os/i686/inputattach-1.24-3-i686.pkg.tar.xz","inputattach","1.24","3","i686"), + ("community-testing/os/i686/ngspice-22-1-i686.pkg.tar.xz","ngspice","22","1","i686"), + ("community-testing/os/i686/tmux-1.4-2-i686.pkg.tar.xz","tmux","1.4","2","i686"), + ("community-testing/os/i686/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), + ("../../../pool/community/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), + ("community-testing/os/x86_64/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), + ("../../../pool/community/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), + ("tor-0.2.1.29-2-x86_64.pkg.tar.xz","tor","0.2.1.29","2","x86_64"), + ) + + def generate_results(self, example_tuple, attr): + location, name, version, release, arch = example_tuple + return pkginfo_from_filename(location)[attr], locals()[attr] + + def testReturnPackageObject(self): + for i in self.known: + location, name, version, release, arch = i + self.assertIsInstance(pkginfo_from_filename(location),Package) + + def testNames(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="name") + self.assertEqual(k, v) + + def testVersions(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="version") + self.assertEqual(k, v) + + def testArchs(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="arch") + self.assertEqual(k, v) + + def testReleases(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="release") + self.assertEqual(k, v) + + def testLocations(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="location") + self.assertEqual(k, v) + +class pkginfo_from_file_BadInput(unittest.TestCase): + bad=("community-testing/os/i686/community-testing.db", + "community-testing/os/i686/community-testing.db.tar.gz", + "community-testing/os/i686/community-testing.db.tar.gz.old", + "community-testing/os/i686/community-testing.files", + "community-testing/os/i686/community-testing.files.tar.gz", + "community-testing/os/x86_64") + + def testBadInput(self): + for i in self.bad: + self.assertRaises(NonValidFile,pkginfo_from_filename,i) + +class pkginfoFromRsyncOutput(unittest.TestCase): + example_package_list=(Package(),Package(),Package()) + example_package_list[0].package_info={ "name" : "alex", + "version" : "2.3.4", + "release" : "1", + "arch" : "i686", + "license" : False, + "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz", + "depends" : False,} + example_package_list[1].package_info={ "name" : "any2dvd", + "version" : "0.34", + "release" : "4", + "arch" : "any", + "license" : False, + "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz", + "depends" : False,} + example_package_list[2].package_info={ "name" : "gmime22", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz", + "depends" : False,} + + try: + output_file = open("rsync_output_sample") + rsync_out= output_file.read() + output_file.close() + except IOError: print("There is no rsync_output_sample file") + + pkglist = pkginfo_from_rsync_output(rsync_out) + + def testOutputArePackages(self): + if not self.pkglist: + self.fail("not pkglist:" + str(self.pkglist)) + for pkg in self.pkglist: + self.assertIsInstance(pkg,Package) + + def testPackageInfo(self): + if not self.pkglist: + self.fail("Pkglist doesn't exist: " + str(self.pkglist)) + self.assertEqual(self.pkglist,self.example_package_list) + +class generateRsyncBlacklist(unittest.TestCase): + example_package_list=(Package(),Package(),Package()) + example_package_list[0].package_info={ "name" : "alex", + "version" : "2.3.4", + "release" : "1", + "arch" : "i686", + "license" : False, + "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz", + "depends" : False,} + example_package_list[1].package_info={ "name" : "any2dvd", + "version" : "0.34", + "release" : "4", + "arch" : "any", + "license" : False, + "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz", + "depends" : False,} + example_package_list[2].package_info={ "name" : "gmime22", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz", + "depends" : False,} + + def testListado(self): + self.assertEqual(listado("blacklist_sample"),["alex","gmime22"]) + + def testExcludeFiles(self): + a=rsyncBlacklist_from_blacklist(self.example_package_list, + listado("blacklist_sample"), + False) + b=[self.example_package_list[0]["location"],self.example_package_list[2]["location"]] + self.assertEqual(a,b) + +class pkginfo_from_descKnownValues(unittest.TestCase): + pkgsample=Package() + pkgsample.package_info={"name" : "binutils", + "version" : "2.21", + "release" : "4", + "arch" : "x86_64", + "license" : "GPL", + "location": "binutils-2.21-4-x86_64.pkg.tar.xz", + "depends" : False,} + fsock=open("desc") + pkggen=pkginfo_from_desc(fsock.read()) + fsock.close() + def testPkginfoFromDesc(self): + if self.pkggen is None: + self.fail("return value is None") + self.assertEqual(self.pkgsample,self.pkggen) + +class pkginfo_from_db(unittest.TestCase): + archdb = os.path.join("./workdir") + example_package_list=(Package(),Package(),Package()) + example_package_list[0].package_info={ "name" : "acl", + "version" : "2.2.49", + "release" : "2", + "arch" : "x86_64", + "license" : ("LGPL",), + "location": "acl-2.2.49-2-x86_64.pkg.tar.xz", + "depends" : ("attr>=2.4.41"),} + example_package_list[1].package_info={ "name" : "glibc", + "version" : "2.13", + "release" : "4", + "arch" : "x86_64", + "license" : ("GPL","LGPL"), + "location": "glibc-2.13-4-x86_64.pkg.tar.xz", + "depends" : ("linux-api-headers>=2.6.37","tzdata",),} + example_package_list[2].package_info={ "name" : "", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "", + "depends" : False,} + + +if __name__ == "__main__": + unittest.main() + |