summaryrefslogtreecommitdiff
path: root/parabola-mirror-repos
diff options
context:
space:
mode:
authorEsteban Carnevale <alfplayer@mailoo.org>2014-11-05 11:34:41 -0300
committerEsteban Carnevale <alfplayer@mailoo.org>2014-11-05 11:34:41 -0300
commite633ce6e35bd840c88719b42252e10a243294257 (patch)
tree4074a841feac6d9f18c208f6bb8694cc747ec9eb /parabola-mirror-repos
parentb22abc87dfd204f5667a418fd251863226a00655 (diff)
parabola-mirror-repos: Add script
Diffstat (limited to 'parabola-mirror-repos')
-rwxr-xr-xparabola-mirror-repos421
1 files changed, 421 insertions, 0 deletions
diff --git a/parabola-mirror-repos b/parabola-mirror-repos
new file mode 100755
index 0000000..7052bc8
--- /dev/null
+++ b/parabola-mirror-repos
@@ -0,0 +1,421 @@
+#!/bin/bash
+
+# alfplayer
+# 2014-06-12
+
+# Bash script to mirror only some repositories of Parabola.
+#
+# Synchronizing first from Arch (maybe using db-sync and any-to-ours of dbscripts), can be
+# recommended to pull packages from the local Arch mirror afterwards using this script.
+
+script_filename="$(basename "$0")"
+
+# Create temporary log file
+log_tmp="$(mktemp -p /var/tmp)"
+
+# Default configuration values.
+# They can be overridden by setting the variables in the calling environment.
+# 1 to set, 0 to unset
+: ${project:=parabola}
+#: ${SERVER:=rsync://repo.parabola.nu:875/repos}
+: ${SERVER:=rsync://alfplayer.com/repos/parabola} # remote rsync directory
+#: ${SERVER:=rsync://parabolagnulinux.mirrors.linux.ro/parabolagnulinux}
+: ${base_dir:=/srv/http}
+: ${LOCAL:=${base_dir}/${project}} # symlink to the last snapshot
+: ${REPOS:=core extra community multilib libre kernels libre-multilib libre-multilib-testing libre-testing nonprism nonprism-testing java cross pcr}
+ # mips64el is also excluded in some rsync invocations in this script
+: ${MAXDELETE:=10000} # maximum amount of files to delete in the local directory
+[[ ! ${BW_LIMIT} ]] && \
+ : ${BW_LIMIT:=1300} # KB/s
+: ${log_file:=${script_filename}.log}
+: ${OPTIONS:=-rltvH --no-p --no-g --max-delete=$MAXDELETE --exclude=.* --bwlimit=${BW_LIMIT} --no-motd --chmod=Dug=srwx,Fug=rw --log-file=${log_tmp}}
+: ${oldest_version:=100} # delete versions older than this version
+: ${link_dest[0]:=${base_dir}/archlinux}
+#: ${link_dest[1]:=${base_dir}/parabola.secondary_mirror.tmp}
+ # link-dest does not support any network URI like rsync://...
+[[ ! ${mail_to} ]] && \
+ : ${mail_to:=your@mail.com} # set to enable mailing log file, or comment out to disable it
+: ${date_exclude:=2014.06.19} # disable running this script this date (see next line)
+: ${forcerun:=0} # set to 1 to force running on excluded date
+: ${terminal:=1} # outputs to stdout using rsync --progress (and logs to a file)
+: ${alternative_mirror:=0} # use alternative mirror; forces no_snapshot_delete and disables snapshot_symlink_update; synchronized files end up in ${LOCAL}.tmp
+: ${no_snapshot_delete:=0} # disable deletion of oldest snapshots; alternative_mirror enables it forcefully
+: ${leave_tmp:=0} # leave updated tree in {project}.tmp instead of creating a dated snapshot directory
+: ${snapshot_symlink_update:=1} # update symlink to last snapshot
+ # does nothing if leave_tmp is enabled
+: ${TZ:=UTC} # set timezone to UTC (affects rsync log output)
+: ${db_and_symlinks_update:=1} # update DB files and package symlinks
+ # disabling this uses existing file /tmp/parabola-mirror which can be left over the last invocation of parabola-mirror-repos with this option set
+: ${pools_update:=1} # update files in package pools
+: ${no_file_delete:=1} # do not delete files
+: ${link_dest_snapshot_count=3} # number of snapshots that are passed as --link-dest to rsync
+[[ ! {path_list} ]] && \
+ : ${path_list:=docs sources mirrorlist.txt lastupdate} # list of extra paths to synchronize
+
+export TZ
+
+if [[ ${alternative_mirror} == 1 ]] ; then
+ no_snapshot_delete=1
+ leave_tmp=1
+ #SERVER="rsync://repo.parabolagnulinux.org:875/repos"
+ SERVER="rsync://mirror.yandex.ru/mirrors/parabola"
+ LOCAL="${base_dir}/${project}.secondary_mirror"
+ #link_dest=("${base_dir}/archlinux" "${base_dir}/parabola.tmp")
+ link_dest+=("${base_dir}/${project}.tmp")
+ for dir in "$LOCAL" "$LOCAL".tmp ${LOCAL}.tmp/pool ; do
+ if [[ ! -d "$dir" ]] ; then
+ mkdir -pv "$dir"
+ chmod -v 2770 "$dir"
+ fi
+ done
+else
+ #link_dest+=("${base_dir}/${project}.tmp")
+ link_dest+=("${base_dir}/${project}.secondary_mirror")
+fi
+
+
+REMOTE_POOL_FILES="/tmp/${project}-remote-files"
+LOCAL_POOL_FILES="/tmp/${project}-local-files"
+POOL_FILES_TO_DELETE="/tmp/${project}-to-delete-files"
+
+# Lock with flock (provided by util-linux), save to log file and send email on exit
+LOCKFILE="/var/lock/${script_filename}"
+LOCKFD=99
+
+_lock() { flock -$1 $LOCKFD; }
+
+# Wait until there is no process writing to ${log_tmp}
+# Waiting can be necessary if command groups are not executed until the end
+_wait_log_tmp() {
+ if [[ -e "${log_tmp}" ]] ; then
+ while fuser -s "${log_tmp}" ; do
+ sleep 0.2
+ done
+ fi
+}
+
+_no_more_locking() {
+ set +e
+
+ # Save exit status
+ es=$?
+
+ _wait_log_tmp
+ cat "${log_tmp}" >> "${log_file}"
+
+ if [[ $? == 0 ]] ; then
+ log_written=1
+ else
+ echo "=> ERROR: Failed to write to log file: ${log_file}" >&2
+ fi
+
+ if [[ -e ${LOCAL}.tmp ]] ; then
+ echo "=> WARNING: Temporary directory ${LOCAL}.tmp remains in file system"
+ fi
+
+ if [[ ${mail_to} ]] ; then
+ echo "=> Sending output to ${mail_to}"
+ mail -s "[$(hostname)] ${script_filename}" "${mail_to}" < "${log_tmp}"
+ fi
+
+ if [[ ${log_written} == 1 ]] ; then
+ rm -f "${log_tmp}"
+ fi
+
+ if [[ $es != 0 ]] ; then
+ echo "=> WARNING: Unsuccessful script termination. Exit status: $es"
+ if [[ ${mail_to} ]] ; then
+ echo "=> Sending error notification to ${mail_to}"
+ mail -s "[$(hostname)] ${script_filename} failed. See logged output." "${mail_to}" < /dev/null
+ fi
+ fi
+
+ _lock u
+ _lock xn && \
+ rm -f $LOCKFILE
+}
+
+_prepare_locking() { eval "exec $LOCKFD>\"$LOCKFILE\""; trap _no_more_locking EXIT; }
+
+_prepare_locking
+
+# Lock now. The lock is disabled automatically when the script exits (with any error code).
+if ! _lock xn ; then
+ echo "=> ERROR: Could not obtain lock. Exiting." >&2
+ exit 1
+fi
+
+DATE="$(date +%Y.%m.%d)"
+current="${LOCAL}-${DATE}"
+current_component="${current##*/}"
+for (( link_dest_count=1 ; link_dest_count <= ${link_dest_snapshot_count} ; link_dest_count++ )) ; do
+ date_count="$(date -d @$(( $(date +"%s") - ${link_dest_count} * 86400)) +"%Y.%m.%d")"
+ link_dest+=("${base_dir}/${project}-${date_count}")
+done
+current_exists=0
+tmp_exists=0
+local_useful=0
+date_exact=""
+first_run=0
+if [[ ${BW_LIMIT} ]] ; then
+ OPTIONS+=" --bwlimit=${BW_LIMIT}"
+fi
+path_list_array=(${path_list})
+repos_array=(${REPOS})
+
+error() {
+ echo "$@" >&2
+ exit 1
+}
+
+{
+
+if [[ ${date_exclude} && ${forcerun} != 1 ]] ; then
+ if [[ $DATE == ${date_exclude} ]] ; then
+ echo "Manually disabled: ${DATE}. Exiting."
+ exit 0
+ fi
+fi
+
+# Parse options.
+# -t or --terminal enables rsync option --progress
+if [[ $# == 1 ]] ; then
+ if [[ $1 == -t || $1 == --terminal ]] ; then
+ terminal=1
+ elif [[ $1 == -h ]] ; then
+ echo "Available options: -h, -t (rsync --progress)"
+ else
+ echo "=> Wrong argument: $1"
+ fi
+elif [[ $# -gt 1 ]] ; then
+ echo "=> ${script_filename} has a wrong number of arguments"
+fi
+
+if [[ $terminal == 1 ]] ; then
+ OPTIONS+=" --progress"
+fi
+
+for i in ${!link_dest[@]}; do
+ if [[ -d ${link_dest[$i]} ]] ; then
+ link_dest_option[i]="${link_dest[@]/#/--link-dest=}"
+ else
+ echo "=> WARNING: Argument to rsync option --link-dest is not an existing directory: ${link_dest[$i]}"
+ fi
+done
+
+# Test if ${current} exists
+if [[ -e ${current} ]] ; then
+ current_exists=1
+fi
+
+# Test if ${LOCAL}.tmp exists
+if [[ -e ${LOCAL}.tmp ]] ; then
+ tmp_exists=1
+fi
+
+# Test if ${LOCAL} is an existing symlink pointing to an existing directory
+if [[ -h ${LOCAL} ]] ; then
+ last_path="$(readlink -f "${LOCAL}")"
+ if [[ -d ${last_path} ]] ; then
+ last="${last_path##*/}"
+ local_useful=1
+ else
+ error "=> ERROR: ${LOCAL} is a symlink which does not point to an existing directory."
+ fi
+else
+ if [[ -e ${LOCAL} ]] || stat -t ${LOCAL}-* >/dev/null 2>&1 ; then
+ error "=> ERROR: ${LOCAL} exists but is not a symlink, or a file (or directory) ${LOCAL}-* was found. Fix this before running ${script_filename} again."
+ else
+ echo "=> WARNING: ${LOCAL} does not exist or is not a symlink, and no snapshot directories were found, so it is assumed this is the first time ${script_filename} is run using \"${base_dir}\" as the base directory."
+ first_run=1
+ fi
+fi
+
+# Check the current tree and issue warnings and errors based on the current tree state
+# Also, it sets up the temporary directory
+if [[ ${current_exists} == 1 ]] ; then
+ echo "=> WARNING: ${current} already exists. It will be preserved."
+ no_snapshot_delete=1
+ if [[ ${local_useful} == 1 ]] ; then
+ if [[ ${tmp_exists} == 0 ]] ; then
+ echo "=> WARNING: ${LOCAL}.tmp does not exist."
+ cp -al "${current}" "${LOCAL}".tmp
+ fi
+ date_exact="$(date +%Y.%m.%d-%T)"
+ echo "=> WARNING: Snapshot ${LOCAL}-${date_exact} will be created because ${current} already exists"
+ else
+ echo "=> WARNING: ${LOCAL} does not exist. It will be created."
+ if [[ ${tmp_exists} == 0 ]] ; then
+ echo "=> WARNING: As ${LOCAL}.tmp does not exist, data transfer will start from ${current}"
+ cp -al "${current}" "${LOCAL}".tmp
+ else
+ echo "=> WARNING: Data transfer will start from ${LOCAL}.tmp"
+ fi
+ fi
+elif [[ ${local_useful} == 0 ]] ; then
+ if [[ ${tmp_exists} == 1 ]] ; then
+ echo "=> ${LOCAL} is not useful but ${LOCAL}.tmp exists. Resuming from ${LOCAL}.tmp"
+ echo "=> WARNING: Symlink ${LOCAL} does not exist. It will be created."
+ else
+ if [[ ${first_run} == 0 ]] ; then
+ error "=> ERROR: ${LOCAL}.tmp does not exist and ${LOCAL} is not useful. Exiting."
+ else
+ mkdir "${LOCAL}".tmp
+ fi
+ fi
+else
+ if [[ ${tmp_exists} == 1 ]] ; then
+ echo "=> ${LOCAL}.tmp already exists. Symlink \"${project}\" currently points to ${last_path}."
+ else
+ cp -al "${last_path}" "${LOCAL}".tmp
+ fi
+fi
+
+echo
+echo "=> Creating snapshot for date ${DATE}"
+
+# Change to the temporary directory
+cd "${LOCAL}".tmp
+
+if [[ ${db_and_symlinks_update} == 1 ]] ; then
+
+ # Delete temporary files that may be left over by a previous invocation of parabola-mirror-repos
+ rm -f "$REMOTE_POOL_FILES" "$LOCAL_POOL_FILES" "$POOL_FILES_TO_DELETE" || true
+
+ remote_repo_dirs="${repos_array[@]/#/${SERVER}/}"
+ remote_repo_dirs="${remote_repo_dirs[@]/%//os}"
+
+ echo "=> Getting pool path list"
+ rsync -lrv --out-format="%L" --dry-run --exclude 'mips64el' --exclude '*mips64el.pkg.tar.*' ${remote_repo_dirs[@]} "/tmp/${script_filename}.unexistent_filename" \
+ | grep -- "->" \
+ | grep "pool.*[^/]$" \
+ | sed -e "s#.*\(pool/.*/.*\)#\1#" \
+ >> "$REMOTE_POOL_FILES" || error " => ERROR: Failed with error code: $?"
+
+ if [[ ${first_run} == 0 ]] ; then
+ echo "=> Getting local pool package file list"
+ # Build a list of local packages
+ find pool -mindepth 2 >> "$LOCAL_POOL_FILES"
+
+ echo "=> Building list of local pool package files to delete"
+
+ # Avoid duplicates (comes from -any packages present in both i686/ and x86_64/)
+ sort -u -o "$REMOTE_POOL_FILES" "$REMOTE_POOL_FILES"
+
+ sort -o "$LOCAL_POOL_FILES" "$LOCAL_POOL_FILES"
+
+ # Keep lines that only appears in LOCAL_POOL_FILES
+ comm -13 "$REMOTE_POOL_FILES" "$LOCAL_POOL_FILES" > "$POOL_FILES_TO_DELETE"
+
+ NUMBER_TO_DELETE="$(wc -l $POOL_FILES_TO_DELETE | cut -d ' ' -f 1)"
+
+ if [[ "$NUMBER_TO_DELETE" -gt "$MAXDELETE" ]] ; then
+ error " => ERROR: The number of pool package files to be deleted is ${NUMBER_TO_DELETE}, greater than the specified maximum which is ${MAXDELETE}"
+ fi
+
+ if [[ $NUMBER_TO_DELETE -gt 0 ]] ; then
+ if [[ ${no_file_delete} == 1 ]] ; then
+ echo "=> Deleting ${NUMBER_TO_DELETE} old pool package files"
+ find $(cat "$POOL_FILES_TO_DELETE") -print -exec rm -f {} \;
+ else
+ echo "=> Deletion of pool package files (${NUMBER_TO_DELETE}) is disabled. List of files:"
+ printf '%s\n' $(cat "$POOL_FILES_TO_DELETE")
+ fi
+ fi
+ fi
+fi
+
+echo "=> List of local repositories. Existing files will be hard linked from these instead of being fetched from the remote server."
+printf -- '%s\n' "${link_dest[@]}"
+
+} &> >(tee -a "${log_tmp}")
+
+_wait_log_tmp
+
+if [[ ${db_and_symlinks_update} == 1 ]] ; then
+
+echo "=> Starting to synchronize repository directories (symlinks and db.* files)"
+
+rsync $OPTIONS --stats --exclude 'mips64el' --delete-after --safe-links "${link_dest_option[@]}" --link-dest="$LOCAL" "${repos_array[@]/#/${SERVER}/}" "$LOCAL".tmp || error " => ERROR: rsync terminated with an error code: $?"
+
+fi
+
+{
+
+if [[ ${pools_update} == 1 ]] ; then
+
+echo "=> Starting to synchronize package pools from remote server $SERVER"
+
+fi # end "if [[ ${pools_update} == 1 ]]"
+
+} &> >(tee -a "${log_tmp}")
+
+_wait_log_tmp
+
+if [[ ${pools_update} == 1 ]] ; then
+
+ rsync $OPTIONS --stats --exclude '*-mips64el.pkg.tar.*' --safe-links --files-from="$REMOTE_POOL_FILES" "${link_dest_option[@]}" --link-dest="$LOCAL" $SERVER "${LOCAL}".tmp
+
+fi # end "if [[ ${pools_update} == 1 ]]"
+
+if [[ ${path_list} ]] ; then
+ echo "=> Synchronizing extra paths"
+ rsync $OPTIONS --stats --safe-links "${link_dest_option[@]/%//$path}" --link-dest="$LOCAL"/"$path" ${path_list_array[@]/#/${SERVER}/} "${LOCAL}".tmp/ || error " => ERROR: rsync terminated with an error code: $?"
+fi
+
+{
+
+if [[ ${no_snapshot_delete} != 1 ]] ; then
+
+echo "=> DRY-RUN: Delete versions older than the version number: ${oldest_version}."
+delete_list=$(find ${base_dir} -regextype sed -maxdepth 1 -regex "${LOCAL}-[0-9]\{4\}\.[0-9]\{2\}.[0-9]\{2\}" | head -n -"${oldest_version}") && \
+ if [[ ${delete_list} ]] ; then
+ echo "DRY_RUN: rm -rf ${delete_list}"
+ else
+ echo "=> Nothing to delete"
+ fi
+
+fi # closes [[ ${no_snapshot_delete} == 0 ]]
+
+if [[ ${leave_tmp} == 0 ]] ; then
+ echo "=> Starting to serve the new repository version"
+
+ if [[ ${date_exact} ]] ; then
+ echo " => Renaming ${LOCAL}.tmp to ${LOCAL}"-"${date_exact}"
+ mv "${LOCAL}".tmp "${LOCAL}"-"${date_exact}"
+ else
+ echo " => Renaming ${LOCAL}.tmp to ${current}"
+ mv "${LOCAL}".tmp "${current}"
+ fi
+
+ if [[ ${snapshot_symlink_update} == 1 ]] ; then
+ cd ${base_dir}
+
+ if [[ ${local_useful} == 1 ]] ; then
+ echo " => Deleting symlink $LOCAL"
+ rm -rf "${LOCAL}"
+ fi
+
+ # Create symlink
+ if [[ ${date_exact} ]] ; then
+ #echo " => Creating symlink ${LOCAL}-${date_exact} to ${current_component}"
+ #ln -s ${current_component}-"${date_exact}" ${LOCAL}-"${date_exact}"
+ echo " => Creating symlink ${LOCAL} to ${project}-${date_exact}"
+ ln -s ${project}-"${date_exact}" ${LOCAL}
+ else
+ echo " => Creating symlink \"${project}\" to ${current_component}"
+ ln -s ${current_component} ${project}
+ fi
+ fi
+fi
+
+rm -f "$REMOTE_POOL_FILES" "$LOCAL_POOL_FILES" "$POOL_FILES_TO_DELETE" "$LOCAL".old || true
+
+echo "=> Disk space report"
+df -h "${base_dir}"
+
+echo "=> ${script_filename} finished successfully. Finish time: $(date --rfc-3339=seconds)"
+
+} &> >(tee -a "${log_tmp}")
+
+_wait_log_tmp