From a6e67e220e638ee69b63dd9a96af7e608c27531c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?coadde=20=5BM=C3=A1rcio=20Alexandre=20Silva=20Delgado=5D?= Date: Tue, 25 Aug 2015 16:01:19 -0300 Subject: add more configuration and update abslibre script --- abslibre | 199 +++++++++++++++++------------------------- config | 30 ++++--- config_abslibre_gnu+hurd | 24 ++++- config_abslibre_gnu+linux | 16 ++++ config_abslibre_gnu+linux_arm | 55 ++++++++++++ config_gnu+hurd | 10 +++ config_gnu+linux | 12 ++- 7 files changed, 211 insertions(+), 135 deletions(-) create mode 100644 config_abslibre_gnu+linux_arm diff --git a/abslibre b/abslibre index 6b21d24..aab71cc 100755 --- a/abslibre +++ b/abslibre @@ -2,128 +2,93 @@ set -e -FTP_BASE=/srv/repo/main -ABSLIBRE=/srv/abslibre -ABSGIT=/srv/git/abslibre/abslibre.git -# Remote -# ABSGIT=http://projects.parabolagnulinux.org/abslibre.git -BLACKLIST=/home/repo/blacklist/blacklist.txt -SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g --quiet' -BLFILE=/tmp/blacklist.txt - -# Variables from abs.conf -ABSROOT="/srv/abs/" -# DON'T CHANGE. WE NEED IT FOR ABSLIBRE -SYNCSERVER="rsync.archlinux.org" -ARCH="i686" -MIRRORLIST="/etc/pacman.d/mirrorlist" -REPOS=(core extra community testing community-testing !staging !community-staging) - # Steps -# * Sync abs +# * Sync bs derivation (abs) # * Download blacklist.txt -# * Sync abslibre from abs excluding from blacklist +# * Sync bs (abslibre) from bs derivation (abs) excluding from blacklist # * Create repo.abs.tar.gz tarballs -function sync_abs() { - for ARCH in any i686 x86_64; do - rsync ${SYNCARGS} ${SYNCSERVER}::abs/${ARCH}/ ${ABSROOT}/${ARCH} || return $? - done - - # fix some permissions - find "${ABSROOT}" -type d -print0 | xargs -0 chmod 755 - find "${ABSROOT}" -type f -print0 | xargs -0 chmod 644 -} - -function get_blacklist() { - printf ":: Updating blacklist...\t" - cat "${BLACKLIST}" | cut -d':' -f1 | sort -u | \ - sed "s/^/**\//" > ${BLFILE} || { - printf "[FAILED]\n" - return 1 +. './config' + +for 'platform' in "${PLATFORMS[@]}"; do + + platform_name="${platform/\//+}" + . "./config_${platform_name,,}" + + function sync_bs_derivation() { + for 'ARCH' in 'any' "${ARCHES[@]}"; do + rsync "${SYNC_ARGS}" "${SYNC_DERIVATION_SERVER}::${BS_DERIVATION_NAME}/${ARCH}" "${BS_DERIVATION_DIR}/${ARCH}" || return $? + done + + # fix some permissions + find "${BS_DERIVATION_DIR}" -type d -print0 | xargs -0 chmod 755 + find "${BS_DERIVATION_DIR}" -type f -print0 | xargs -0 chmod 644 + } + + function get_blacklist() { + printf ":: Updating blacklist...\t" + cat "${BLACKLIST_FILE}" | cut -d':' -f1 | sort -u | \ + sed "s/^/**\//" > "${BLACKLIST_TMP}" || { + printf "[FAILED]\n" + return 1 + } + + # Prevent using an empty blacklist + [ $(wc -l "${BLACKLIST_TMP}" | cut -d " " -f1) -eq 0 ] && return 1 + + printf "[OK]\n" + } + + function sync_bs() { + + # Clone BS git repo + if [ -d "${BS_GIT_TMP}"/.git ]; then + pushd "${BS_GIT_TMP}" >/dev/null 2>&1 + git pull + popd >/dev/null 2>&1 + else + git clone "${BS_GIT}" "${BS_GIT_TMP}" + fi + + # Sync from BS_DERIVATION and then sync from BS + printf ":: Syncing ${BS_MAIN_NAME}...\t" + (rsync "${SYNC_ARGS}" --delete-excluded \ + --exclude-from="${BLACKLIST_TMP}" \ + "${BS_DERIVATION_DIR}" \ + "${BS_MAIN_DIR}" \ + && + for 'ARCH' in "${ARCHES[@]}"; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --quiet --exclude=.git/ "${BS_GIT_TMP}" "${BS_MAIN_DIR}/${ARCH}/"; done) || { + printf "[FAILED]\n" + return 1 + } + + # fix some permissions + find "${BS_MAIN_DIR}" -type d -print0 | xargs -0 chmod 755 + find "${BS_MAIN_DIR}" -type f -print0 | xargs -0 chmod 644 + + printf "[OK]\n" } - # Prevent using an empty blacklist - [ $(wc -l ${BLFILE} | cut -d " " -f1) -eq 0 ] && return 1 - - printf "[OK]\n" -} - -function sync_abs_libre() { - - # Clone ABSLibre git repo - if [ -d /tmp/abslibre/.git ]; then - pushd /tmp/abslibre >/dev/null 2>&1 - git pull - popd >/dev/null 2>&1 - else - git clone "$ABSGIT" /tmp/abslibre - fi - - # Sync from ABS and then sync from ABSLibre - printf ":: Syncing ABSLibre...\t" - (rsync ${SYNCARGS} --delete-excluded \ - --exclude-from=${BLFILE} \ - ${ABSROOT} \ - ${ABSLIBRE} \ - && - for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --quiet --exclude=.git/ /tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { - printf "[FAILED]\n" - return 1 + # Create .abs.tar.gz tarballs + create_tarballs() { + for 'repo' in "${BS_MAIN_DIR}/${ARCHES[@]}/"*; do + baserepo="${repo##*/}" + arch="$(basename $(dirname ${repo}))" + + # Remove the old one + mkdir -p "${FTP_BASE}/${baserepo}/os/${arch}" + rm -fv "${FTP_BASE}/${baserepo}/os/${arch}/${baserepo}.abs.tar.gz" + # Create a new one joining arch and any + # Remove the first part of the path (it could be $repo but any isn't hit) + bsdtar -czf "${FTP_BASE}/${baserepo}/os/${arch}/${baserepo}.abs.tar.gz" \ + -s ":${BS_MAIN_DIR}/[a-z0-9_]\+/[a-z]\+::" \ + "${repo}"/* "${BS_MAIN_DIR}/any/${baserepo}"/* + + done } - # fix some permissions - find "${ABSLIBRE}" -type d -print0 | xargs -0 chmod 755 - find "${ABSLIBRE}" -type f -print0 | xargs -0 chmod 644 - - printf "[OK]\n" -} - -# This part is very hacky and particular to the current setup :P -sync_pre_mips64el() { - pushd /home/fauno/Repos/abslibre-pre-mips64el >/dev/null - - sudo -u fauno sh -c " - rsync ${SYNCARGS} \ - --exclude=.git* \ - --exclude=community-staging \ - --exclude=community-testing \ - --exclude=gnome-unstable \ - --exclude=kde-unstable \ - --exclude=multilib \ - --exclude=multilib-testing \ - --exclude=multilib-staging \ - --exclude=staging \ - --exclude=testing \ - ${ABSLIBRE}/x86_64/ \ - /home/fauno/Repos/abslibre-pre-mips64el/ && - git add . && - git commit -m \"$(date)\" -a - git push origin master - git gc - " -} - -# Create .abs.tar.gz tarballs -create_tarballs() { - for repo in ${ABSLIBRE}/{i686,x86_64}/*; do - baserepo=${repo##*/} - arch=$(basename $(dirname $repo)) - - # Remove the old one - mkdir -p $FTP_BASE/$baserepo/os/$arch/ - rm -fv $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz - # Create a new one joining arch and any - # Remove the first part of the path (it could be $repo but any isn't hit) - bsdtar -czf $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz \ - -s ":${ABSLIBRE}/[a-z0-9_]\+/[a-z]\+::" \ - $repo/* ${ABSLIBRE}/any/${baserepo}/* - - done -} - -sync_abs -get_blacklist -sync_abs_libre -#sync_pre_mips64el -create_tarballs + sync_bs_derivation + get_blacklist + sync_bs + create_tarballs +done diff --git a/config b/config index 8dbf8fc..5e13683 100644 --- a/config +++ b/config @@ -1,7 +1,5 @@ #!/hint/bash -FTP_BASE='/srv/repo/main' - # Parabola Platforms PLATFORMS=('ABSLibre_GNU/linux') # 'GNU/Hurd' 'GNU/Linux' 'ABSLibre_GNU/Hurd' @@ -15,12 +13,18 @@ STABLE_NAME='minicat-v1' PKG_POOL_DIR='pools' SRC_POOL_DIR='sources' -CLEANUP_DESTDIR="$FTP_BASE/old/packages" +SRV_DIR='/srv' +TMP_DIR='/tmp' +FTP_BASE="${SRV_DIR}/repo/main" +GIT_BASE="${SRV_DIR}/git" +SYNC_ARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g --quiet' + +CLEANUP_DESTDIR="${FTP_BASE}/old/packages" CLEANUP_DRYRUN='false' # Time in days to keep moved packages CLEANUP_KEEP='30' -SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" +SOURCE_CLEANUP_DESTDIR="${FTP_BASE}/old/sources" SOURCE_CLEANUP_DRYRUN='true' # Time in days to keep moved sourcepackages SOURCE_CLEANUP_KEEP='30' @@ -30,15 +34,15 @@ REQUIRE_SIGNATURE='true' LOCK_DELAY='10' LOCK_TIMEOUT='300' -[ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging" -TMPDIR='/tmp' -DBEXT='.db.tar.gz' -FILESEXT='.files.tar.gz' -PKGEXT=".pkg.tar.?z" -SRCEXT='.src.tar.gz' +[ -n "${STAGING:-}" ] || STAGING="${HOME}/staging/unknown/staging" +DB_EXT='.db.tar.gz' +FILES_EXT='.files.tar.gz' +PKG_EXT=".pkg.tar.?z" +SRC_EXT='.src.tar.gz' -MAKEPKGCONF="~/.makepkg.conf" -BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" +MIRROR_LIST='/etc/pacman.d/mirrorlist' +MAKEPKG_CONF='/etc/makepkg.conf' +MAKEPKG_ARCH="$(uname -m)" # parabolaweb root -WEB_DIR='/srv/http/www.parabola.nu/web' +WEB_DIR="${SRV_DIR}/http/www.parabola.nu/web" diff --git a/config_abslibre_gnu+hurd b/config_abslibre_gnu+hurd index 90b412e..d1b1b7d 100644 --- a/config_abslibre_gnu+hurd +++ b/config_abslibre_gnu+hurd @@ -1,6 +1,9 @@ #!/hint/bash +. ./config + # Parabola GNU/Hurd configuration with Arch GNU/Hurd derivation +DERIVATION='true' # Archictectures ARCHES=('i686') @@ -8,14 +11,14 @@ ARCHES=('i686') # Multilib Support MULTILIB=() -# Main Repositories from Arch GNU/Linux +# Main Repositories from Arch GNU/Hurd MAIN_REPOS=('core' 'extra' 'testing' 'staging-core' 'staging-extra') -# Libre Repositories from Parabola GNU/Linux-libre +# Libre Repositories from Parabola GNU/Hurd LIBRE_MAIN_REPOS=('libre' 'libre-testing') -# Multilib Repositories from Arch GNU/Linux +# Multilib Repositories from Arch GNU/Hurd MULTILIB_REPOS=('multilib' 'multilib-testing') -# Libre Multilib Repositories from Parabola GNU/Linux-libre +# Libre Multilib Repositories from Parabola GNU/Hurd LIBRE_MULTILIB_REPOS=('libre-multilib' 'libre-multilib-testing') # Extra Repositories @@ -37,3 +40,16 @@ PKG_POOL='abslibre_gnu+hurd' # Directories where sources are stored SRC_POOL='abslibre_gnu+hurd' + +# Build System and Repository scripts +BS_MAIN_NAME='abslibre_gnu+hurd' +BS_MAIN_DIR="${SRV_DIR}/${BS_MAIN_NAME}" +BS_DERIVATION_NAME='abs_gnu+hurd' +BS_DERIVATION_DIR="${SRV_DIR}/${BS_DERIVATION_NAME}" +BS_GIT="${GIT_BASE}/abslibre/${BS_MAIN_NAME}.git" +# BS_GIT='http://projects.parabola.gnu/abslibre/${BS_MAIN_NAME}.git' +BS_GIT_TMP="${TMP_DIR}/${BS_MAIN_NAME}" +BLACKLIST_NAME='blacklist_gnu+hurd.txt' +BLACKLIST_FILE="${HOME}/blacklist/${BLACKLIST_NAME}.txt" +BLACKLIST_TMP="${TMP_DIR}/${BLACKLIST_NAME}.txt" +SYNC_DERIVATION_SERVER='rsync.archhurd.org' diff --git a/config_abslibre_gnu+linux b/config_abslibre_gnu+linux index e157b9c..bcf8ab0 100644 --- a/config_abslibre_gnu+linux +++ b/config_abslibre_gnu+linux @@ -1,6 +1,9 @@ #!/hint/bash +. ./config + # Parabola GNU/Linux-libre configuration with Arch GNU/Linux derivation +DERIVATION='true' # Archictectures ARCHES=('i686' 'x86_64') @@ -37,3 +40,16 @@ PKG_POOL='abslibre_gnu+linux' # Directories where sources are stored SRC_POOL='abslibre_gnu+linux' + +# Build System and Repository scripts +BS_MAIN_NAME='abslibre_gnu+linux' +BS_MAIN_DIR="${SRV_DIR}/${BS_MAIN_NAME}" +BS_DERIVATION_NAME='abs_gnu+linux' +BS_DERIVATION_DIR="${SRV_DIR}/${BS_DERIVATION_NAME}" +BS_GIT="${GIT_BASE}/abslibre/${BS_MAIN_NAME}.git" +# BS_GIT='http://projects.parabola.gnu/abslibre/${BS_MAIN_NAME}.git' +BS_GIT_TMP="${TMP_DIR}/${BS_MAIN_NAME}" +BLACKLIST_NAME='blacklist_gnu+linux.txt' +BLACKLIST_FILE="${HOME}/blacklist/${BLACKLIST_NAME}.txt" +BLACKLIST_TMP="${TMP_DIR}/${BLACKLIST_NAME}.txt" +SYNC_DERIVATION_SERVER='rsync.archlinux.org' diff --git a/config_abslibre_gnu+linux_arm b/config_abslibre_gnu+linux_arm new file mode 100644 index 0000000..7c76002 --- /dev/null +++ b/config_abslibre_gnu+linux_arm @@ -0,0 +1,55 @@ +#!/hint/bash + +. ./config + +# Parabola GNU/Linux-libre configuration with Arch GNU/Linux ARM derivation +DERIVATION='true' + +# Archictectures +ARCHES=('arm' 'armv6h' 'armv7h' 'aarch64') + +# Multilib Support +MULTILIB=() + +# Main Repositories from Arch GNU/Linux ARM +MAIN_REPOS=('core' 'extra' 'community' 'aur' 'alarm') +# Libre Repositories from Parabola GNU/Linux-libre +LIBRE_MAIN_REPOS=('libre' 'libre-testing') + +# Multilib Repositories from Arch GNU/Linux ARM +MULTILIB_REPOS=('multilib') +# Libre Multilib Repositories from Parabola GNU/Linux-libre +LIBRE_MULTILIB_REPOS=('libre-multilib' 'libre-multilib-testing') + +# Extra Repositories +EXTRA_REPOS=('nonprism' 'nonprism-testing' 'kernels' 'kernels-testing' 'cross' 'java') + +# Community Repositories +COMMUNITY_REPOS=('pcr' 'pcr-testing' 'pur' '~aurelien' '~brendan' '~coadde' '~drtan' '~emulatorman' '~jorginho' '~lukeshu' '~smv' '~xihh') + +# Platform Repositories +PATFORM_REPOS=('nonsystemd' 'nonsystemd-testing') + +# Remote Repositories +PKG_ANY_REPOS=("${EXTRA_REPOS[@]}" "${PATFORM_REPOS[@]}" "${LIBRE_MAIN_REPOS[@]}" "${MAIN_REPOS[@]}" "${COMMUNITY_REPOS[@]}") +PKG_MULTILIB_REPOS=("${EXTRA_REPOS[@]}" "${PATFORM_REPOS[@]}" "${LIBRE_MAIN_REPOS[@]}" "${MAIN_REPOS[@]}" "${LIBRE_MULTILIB_REPOS[@]}" "${MULTILIB_REPOS[@]}" "${COMMUNITY_REPOS[@]}") + +# Directories where packages are shared between repos +# *relative to FTP_BASE* +PKG_POOL='abslibre_gnu+linux_arm' + +# Directories where sources are stored +SRC_POOL='abslibre_gnu+linux_arm' + +# Build System and Repository scripts +BS_MAIN_NAME='abslibre_gnu+linux_arm' +BS_MAIN_DIR="${SRV_DIR}/${BS_MAIN_NAME}" +BS_DERIVATION_NAME='abs_gnu+linux_arm' +BS_DERIVATION_DIR="${SRV_DIR}/${BS_DERIVATION_NAME}" +BS_GIT="${GIT_BASE}/abslibre/${BS_MAIN_NAME}.git" +# BS_GIT='http://projects.parabola.gnu/abslibre/${BS_MAIN_NAME}.git' +BS_GIT_TMP="${TMP_DIR}/${BS_MAIN_NAME}" +BLACKLIST_NAME='blacklist_gnu+linux_arm.txt' +BLACKLIST_FILE="${HOME}/blacklist/${BLACKLIST_NAME}.txt" +BLACKLIST_TMP="${TMP_DIR}/${BLACKLIST_NAME}.txt" +SYNC_DERIVATION_SERVER='rsync.archlinuxarm.org' diff --git a/config_gnu+hurd b/config_gnu+hurd index 45ee43f..dd7f424 100644 --- a/config_gnu+hurd +++ b/config_gnu+hurd @@ -1,6 +1,9 @@ #!/hint/bash +. ./config + # Parabola GNU/Hurd configuration without any distribution derivation +DERIVATION='false' # Archictectures ARCHES=('i686') @@ -33,3 +36,10 @@ PKG_POOL='gnu+hurd' # Directories where sources are stored SRC_POOL='gnu+hurd' + +# Build System and Repository scripts +BS_MAIN_NAME='pbs_gnu+hurd' +BS_MAIN_DIR="${SRV_DIR}/${BS_MAIN_NAME}" +BS_GIT="${GIT_BASE}/${BS_MAIN_NAME}/pmr.git" +# BS_GIT='http://projects.parabola.gnu/${BS_MAIN_NAME}/pmr.git' +BS_GIT_TMP="${TMP_DIR}/${BS_MAIN_NAME}" diff --git a/config_gnu+linux b/config_gnu+linux index 479061c..c34b9a6 100644 --- a/config_gnu+linux +++ b/config_gnu+linux @@ -1,9 +1,12 @@ #!/hint/bash +. ./config + # Parabola GNU/Linux-libre configuration without any distribution derivation +DERIVATION='false' # Archictectures -ARCHES=('i686' 'x86_64') +ARCHES=('i686' 'x86_64' 'mips64el') # Multilib Support MULTILIB=('x86_64') @@ -33,3 +36,10 @@ PKG_POOL='gnu+linux' # Directories where sources are stored SRC_POOL='gnu+linux' + +# Build System and Repository scripts +BS_MAIN_NAME='pbs_gnu+linux' +BS_MAIN_DIR="${SRV_DIR}/${BS_MAIN_NAME}" +BS_GIT="${GIT_BASE}/${BS_MAIN_NAME}/pmr.git" +# BS_GIT='http://projects.parabola.gnu/${BS_MAIN_NAME}/pmr.git' +BS_GIT_TMP="${TMP_DIR}/${BS_MAIN_NAME}" -- cgit v1.2.3