From 71dc3b30449a4574455ff7eb9c390e30ad52ac4a Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 15 Dec 2016 16:01:03 -0500 Subject: fix (wip) --- git-mirror | 86 +++++++++++++++++++++++++++++++++------------------ git-mirror-backend.rb | 9 ++++-- git-mirror-cgit | 13 ++++---- git-mirror-gitlab-ce | 16 ++++++++-- git-mirror-gitlab-ee | 7 +++-- 5 files changed, 86 insertions(+), 45 deletions(-) diff --git a/git-mirror b/git-mirror index d9e1932..113379a 100755 --- a/git-mirror +++ b/git-mirror @@ -5,6 +5,8 @@ # terms of the Do What The Fuck You Want To Public License, Version 2, # as published by Sam Hocevar. See the COPYING file for more details. +# Depends 4.4+, as it uses the '-d' flag to 'mapfile'. +# # Depends on the 'gitget' and 'libremessages' commands. On Parabola # GNU/Linux-libre, those are the 'gitget' and 'librelib' packages, # respectively. @@ -12,8 +14,7 @@ # For other systems, they both live at: # https://git.parabola.nu/packages/libretools.git/ -set -o pipefail -set -e +set -o pipefail -euE . libremessages usage() { @@ -27,18 +28,16 @@ main() { fi declare -g cfg_file="$1" - local r=0 while read -r repo; do - handle-repo "$repo" || r=$? + handle-repo "$repo" done < <(cfg-list-repos) - return $r } handle-repo() { [[ $# == 1 ]] || panic local repo=$1 - local local upstream downstreams downstream r=0 - + local local upstream downstreams downstream + # read configuration local="$(cfg-get "repo.$repo.local")" upstream="$(cfg-get "repo.$repo.upstream")" || true @@ -46,7 +45,7 @@ handle-repo() { # download if [[ -n "$upstream" ]]; then - download "$upstream" "$local" + download "$repo" "$upstream" "$local" fi # ensure that $local exists @@ -54,25 +53,29 @@ handle-repo() { # upload for downstream in "${downstreams[@]}"; do - upload "$local" "$downstream" || r=$? + upload "$repo" "$local" "$downstream" done - - return $r } download() { [[ $# == 3 ]] || panic local repo=$1 - local remote=$2 + local _remote=$2 local local=$3 + local IFS='&' + local remote=${_remote%%#*} + local params=(${_remote#"$remote"}) + # download the repository local url + url="$(remote "$remote" pull-url)" - gitget -f -n "$repo" "$url" "$local" + gitget -f -n "$repo" bare "$url" "$local" # download the metadata - remote "$remote" get-meta > "$local/git-mirror.tmp" - git config --file "$local/config" --rename-section git-mirror git-mirror-bak + msg2 "Fetching metadata for repo '%s'" "$repo" + get-meta "$remote" > "$local/git-mirror.tmp" + git config --file "$local/config" --rename-section git-mirror git-mirror-bak 2>/dev/null || true local IFS='=' while read -r key val; do git config --file "$local/config" --add git-mirror."$key" "$val" @@ -81,20 +84,19 @@ download() { git config --file "$local/config" --remove-section git-mirror-bak } -upload() { - [[ $# == 2 ]] || panic - local local=$1 - local remote=${2%%#*} - local IFS='&' - local params=(${2#"$remote"}) +upload() ( + [[ $# == 3 ]] || panic + local repo=$1 + local local=$2 + local remote=$3 # push metadata - { - printf '%q ' set-meta "${remote#*:}" - git config --file "$local/config" --get-regexp '^git-mirror[.]' -z|sed -z 's/ /=/'|xargs -0r printf '%q ' - printf '%q ' "${params[@]}" - } | account "${remote%%:*}" + msg2 "Pushing metadata for repo '%s'" "$repo" + local meta + mapfile -d '' meta < <(git config --file "$local/config" -z --get-regexp '^git-mirror[.]'|sed -z 's/ /=/') + set-meta "$remote" "${meta[@]}" # push repository + msg2 "Pushing repo '%s'" "repo" local repo_mode repo_mode=$(remote "$remote" repo-mode) if [[ $repo_mode == passive ]]; then @@ -102,6 +104,30 @@ upload() { push_url="$(remote "$remote" push-url)" cd "$local" && git push --mirror "$push_url" fi +) + +get-meta() ( + [[ $# = 1 ]] || panic + local _remote=$1 + + local IFS='&' + local remote=${_remote%%#*} + local params=(${_remote#"$remote"}) + + remote "$remote" get-meta + [[ ${#params[@]} = 0 ]] || printf '%s\n' "${params[@]}" +) + +set-meta() { + [[ $# -ge 1 ]] || panic + local _remote=$1 + local args=("${@:2}") + + local IFS='&' + local remote=${_remote%%#*} + local params=(${_remote#"$remote"}) + + remote "$remote" set-meta "${args[@]}" "${params[@]}" } # Spawn an 'account.type' helper. It will read commands from stdin. @@ -109,11 +135,11 @@ account() { [[ $# == 1 ]] || panic local account=$1 - local account_type - account_type="$(cfg-get "account.$account.type")" + local type + type="$(cfg-get "account.$account.type")" { - cfg --list -z|sed -zn "s=^account[.]$account[.]=config =p"|grep -z -v '^type='|xargs -r0 printf '%s\n' + cfg --list|sed -n "s=^account[.]$account[.]==p"|grep -v '^type='|xargs -r printf 'config %q\n' cat } | git mirror-"$type" "$account" } @@ -146,7 +172,7 @@ cfg-get-all() { cfg-list-repos() { [[ $# == 0 ]] || panic - cfg --name-only --get-regexp '^repo[.].*[.].*$' -z|sed -z -e 's|^repo[.]||' -e 's|[.][^.]*$||'|sort -zu|xargs -r0 printf '%s\n' + cfg --name-only -z --get-regexp '^repo[.].*[.].*$' |sed -z -e 's|^repo[.]||' -e 's|[.][^.]*$||'|sort -zu|xargs -r0 printf '%s\n' } main "$@" diff --git a/git-mirror-backend.rb b/git-mirror-backend.rb index 25f7c3e..2be8830 100644 --- a/git-mirror-backend.rb +++ b/git-mirror-backend.rb @@ -4,11 +4,14 @@ require 'shellwords' class GitMirrorBackend def repl(accountName) - @accountName = account_name + @accountName = accountName $stdin.each do |line| args = line.shellsplit - out = self.send('cmd_'+args[0].gsub('-', '_'), *args[1..0]) - if out.is_a? String + args[0] = 'cmd_'+args[0].gsub('-', '_') + out = self.send(*args) + if out.nil? + # do nothing + elsif out.is_a? String puts out elsif out.is_a? Array out.each do |outline| diff --git a/git-mirror-cgit b/git-mirror-cgit index 8c1b799..3a953b2 100755 --- a/git-mirror-cgit +++ b/git-mirror-cgit @@ -23,15 +23,16 @@ class Cgit < GitMirrorBackend end @config[key] = val end + return nil end def cmd_get_meta(path) - doc = Nokogiri::HTML(open(@config[url]+path)) - head = open(@config[url]+path+'/HEAD').read + doc = Nokogiri::HTML(open(@config['url']+path)) + head = open(@config['url']+(path+'/HEAD')).read return { 'description' => doc.css('#header .sub')[0].text, 'owner' => doc.css('#header .sub')[1].text, - 'default_branch' => head.split("\n")[0].sub(/^ref: /, '').sub(/^refs\/heads\//, ''), + 'default-branch' => head.split("\n")[0].sub(/^ref: /, '').sub(/^refs\/heads\//, ''), } end @@ -40,20 +41,20 @@ class Cgit < GitMirrorBackend end def urls(path) - doc = Nokogiri::HTML(open(@config[url]+path)) + doc = Nokogiri::HTML(open(@config['url']+path)) return doc.css('a[rel="vcs-git"]').map{|a| a['href']} end def cmd_push_url(path) prefs = ['ssh', 'https', 'http', 'git'] - return self.urls.sort_by{|url| prefs.index(url)} + return self.urls(path).sort_by{|u| prefs.index(u.split(':', 2)[0])}.first end def cmd_pull_url(path) # prefer https ahead of git because of a bug in git-daemon # with '~'. prefs = ['https', 'git', 'http', 'ssh'] - return self.urls.sort_by{|url| prefs.index(url)} + return self.urls(path).sort_by{|u| prefs.index(u.split(':', 2)[0])}.first end def cmd_repo_mode(path) diff --git a/git-mirror-gitlab-ce b/git-mirror-gitlab-ce index feb8602..1c628e8 100755 --- a/git-mirror-gitlab-ce +++ b/git-mirror-gitlab-ce @@ -121,7 +121,7 @@ class GitLabCE < GitMirrorBackend end info = JSON::parse(res.body) end - return self.get_meta + return nil end def repo_mode @@ -133,6 +133,7 @@ class GitLabCE < GitMirrorBackend def cmd_config(*args) args.each do |arg| key, val = arg.split('=', 2) + key = key.gsub('-', '_') case key when "apiurl" val = URI(val) @@ -142,19 +143,28 @@ class GitLabCE < GitMirrorBackend end @config[key] = val end + return nil end def cmd_get_meta(project_id) - return self.project(project_id).get_meta() + map = self.project(project_id).get_meta() + ret = {} + map.each do |key,val| + key = key.gsub('_', '-') + map[key] = val + end + return ret end def cmd_set_meta(project_id, *pairs) map = {} pairs.each do |pair| key, val = arg.split('=', 2) + key = key.gsub('-', '_') map[key] = val end - return self.project(project_id).set_meta(map) + self.project(project_id).set_meta(map) + return nil end def cmd_push_url(project_id) diff --git a/git-mirror-gitlab-ee b/git-mirror-gitlab-ee index 5467e16..394a5f7 100755 --- a/git-mirror-gitlab-ee +++ b/git-mirror-gitlab-ee @@ -14,7 +14,7 @@ # pretty clean, except that screen-scraping the reads (via nokogiri) # is gross, and that the error messages are unhelpful. -load 'gitlab-ce' +load 'git-mirror-gitlab-ce' require 'net/http' require 'uri' require 'nokogiri' @@ -73,7 +73,7 @@ class GitLabEE < GitLabCE @cache.delete(:mirror_cookie) return URI(url) end - + def get_meta map = super map["mirror"] = self.mirrorURL.to_s @@ -89,10 +89,11 @@ class GitLabEE < GitLabCE else super(map) end + return nil end def repo_mode - return (self.mirrorURL.nil? ? "active" : "passive") + return (self.mirrorURL.nil? ? "passive" : "active") end end end -- cgit v1.2.3