diff options
| -rwxr-xr-x | backends/github | 1 | ||||
| -rwxr-xr-x | backends/gitlab-ce | 152 | ||||
| -rwxr-xr-x | backends/gitlab-ee | 99 | ||||
| -rw-r--r-- | docs.org | 16 | ||||
| -rwxr-xr-x | git-mirror | 149 | ||||
| -rw-r--r-- | git-mirror.conf.example | 27 |
6 files changed, 419 insertions, 25 deletions
diff --git a/backends/github b/backends/github new file mode 100755 index 0000000..49ffd26 --- /dev/null +++ b/backends/github @@ -0,0 +1 @@ +#!/usr/bin/env ruby diff --git a/backends/gitlab-ce b/backends/gitlab-ce new file mode 100755 index 0000000..3b6cc90 --- /dev/null +++ b/backends/gitlab-ce @@ -0,0 +1,152 @@ +#!/usr/bin/env ruby +# coding: utf-8 + +# http://docs.gitlab.com/ee/workflow/repository_mirroring.html +# https://gitlab.com/gitlab-org/gitlab-ee/issues/767 + +require 'net/http' +require 'uri' +require 'cgi' +require 'json' + +class GitLabCE + def initialize(api_uri, api_key, project_id) + @api_uri = URI(api_uri) + unless @api_uri.path.end_with?("/") + @api_uri.path += "/" + end + @api_key = api_key.to_s + @project_id = project_id.to_s + + @connections = {} + @cache = {} + + # API docs suck, just look at `lib/api/projects.rb` + @vars = [ + :builds_enabled, # create | create-user | edit + :container_registry_enabled, # create | | edit + :default_branch # | create-user | edit + :description, # create | create-user | edit + :import_url, # create | create-user | + :issues_enabled, # create | create-user | edit + :lfs_enabled, # create | create-user | edit + :merge_requests_enabled, # create | create-user | edit + :name, # create | create-user | edit + :namespace_id, # create | | + :only_allow_merge_if_build_succeeds, # create | create-user | edit + :path, # create | | edit + :public, # create | create-user | edit + :public_builds, # create | create-user | edit + :request_access_enabled, # create | create-user | edit + :shared_runners_enabled, # create | create-user | edit + :snippets_enabled, # create | create-user | edit + :visibility_level, # create | create-user | edit + :wiki_enabled, # create | create-user | edit + :only_allow_merge_if_all_discussions_are_resolved # create | create-user | edit + ] + + @vars = [ + "builds_enabled", + "container_registry_enabled", + "default_branch", + "description", + "issues_enabled", + "lfs_enabled", + "merge_requests_enabled", + "name", + "only_allow_merge_if_all_discussions_are_resolved", + "only_allow_merge_if_build_succeeds", + "path", + "public", + "public_builds", + "request_access_enabled", + "shared_runners_enabled", + "snippets_enabled", + "visibility_level", + "wiki_enabled", + ] + end + + def _connection(uri) + key=URI(uri.scheme+":") + key.host = uri.host + key.port = uri.port + + @connections[key] ||= Net::HTTP::start(uri.host, uri.port, :use_ssl => uri.scheme == 'https') + return @connections[key] + end + + def _info + unless @cache.has_key?(:info) + req = Net::HTTP::Get.new(@api_uri + "projects/" + CGI::escape(@project_id)) + req.add_field("PRIVATE-TOKEN", @api_key) + con = _connection(req.uri) + res = con.request(req) + if res.code != "200" + throw res + end + @cache[:info] = JSON::parse(res.body) + end + return @cache[:info] + end + + def get_meta + return _info.select{|k,v| @vars.include?(k)} + end + + def set_meta(map) + illegal = map.select{|k,v| not @vars.include?(k)} + if illegal.count > 0 + throw illegal + end + + req = Net::HTTP::Put.new(@api_uri + "projects/" + CGI::escape(_info["id"].to_s)) + req.add_field("PRIVATE-TOKEN", @api_key) + req.add_field("Content-Type", "application/json") + req.body = JSON::dump(map) + con = _connection(req.uri) + res = con.request(req) + if res.code != "200" + throw res + end + @cache[:info] = JSON::parse(res.body) + return get_meta + end + + def pushURL + return _info["ssh_url_to_repo"] + end + + def pullURL + return _info["http_url_to_repo"] + end + + def create(id, map) + if map.has_key?("mirror") + map["import_url"] = map["mirror"] + map.delete("mirror") + end + req = Net::HTTP::Post.new(@api_uri + "projects") + req.add_field("PRIVATE-TOKEN", @api_key) + req.add_field("Content-Type", "application/json") + req.body = JSON::dump(map) + con = _connection(req.uri) + res = con.request(req) + end + + def capabilities + return [ + [ "get-meta", @vars ].flatten, + [ "set-meta", @vars ].flatten, + [ "push-url" ], + [ "pull-url" ], + ] + end + + def finish + @connections.each do |k,v| + v.finish() + end + @connections = {} + end +end diff --git a/backends/gitlab-ee b/backends/gitlab-ee new file mode 100755 index 0000000..d0db8b9 --- /dev/null +++ b/backends/gitlab-ee @@ -0,0 +1,99 @@ +#!/usr/bin/env ruby +# coding: utf-8 + +# GitLab EE supports configuring a "project" (GitLab's term for a +# repository+metadata) to display as a mirror of another repository. +# +# http://docs.gitlab.com/ee/workflow/repository_mirroring.html +# +# Unfortunately, the JSON API doesn't support this +# +# https://gitlab.com/gitlab-org/gitlab-ee/issues/767 +# +# So, we must use the (undocumented!) HTTP API, which is actually +# pretty clean, except that screen-scraping the reads (via nokogiri) +# is gross, and that the error messages are unhelpful. + +load 'gitlab-ce' +require 'net/http' +require 'uri' +require 'nokogiri' + +class GitLabEE < GitLabCE + def _mirrorURL + unless @cache.has_key?(:mirror) + req = Net::HTTP::Get.new(URI(_info["web_url"]+"/mirror")) + req.add_field("PRIVATE-TOKEN", @api_key) + con = _connection(req.uri) + res = con.request(req) + if res.code != "200" + throw res + end + @cache[:mirror_res]=res + doc = Nokogiri::HTML(res.body) + + @cache[:mirror_cookie] = res["set-cookie"] + @cache[:mirror_token] = doc.css('input[name="authenticity_token"]').first["value"] + is_mirror = doc.css("#project_mirror").first["checked"] + if !is_mirror + @cache[:mirror] = nil + else + @cache[:mirror] = URI(doc.css("#project_import_url").first["value"]) + end + end + return @cache[:mirror] + end + + def _mirrorURL=(url) + _mirrorURL + + req = Net::HTTP::Patch.new(URI(_info["web_url"]+"/mirror")) + req.add_field("PRIVATE-TOKEN", @api_key) # authenticate + req.add_field("Cookie", @cache[:mirror_cookie]) # session id + req.form_data = { + "utf8" => "✓", + "authenticity_token" => @cache[:mirror_token], # session state + "project[mirror]" => (url.nil? ? "0" : "1"), + "project[import_url]" => url.to_s, + } + + con = _connection(req.uri) + res = con.request(req) + if res.code != "302" + throw res + end + + @cache.delete(:mirror) + @cache.delete(:mirror_token) + @cache.delete(:mirror_cookie) + return URI(url) + end + + def get_meta + map = super + map["mirror"] = _mirrorURL.to_s + return map + end + + def set_meta(map) + if map.has_key?("mirror") + self._mirrorURL=map["mirror"] + map.delete("mirror") + end + return super(map) + end + + def create(id, map) + super(id, map) + self._mirrorURL=map["mirror"] + end + + def capabilities + return super.map{|c| + if c[0] == "get-meta" or c[0] == "set-meta" + c << "mirror" + end + c + } + end +end diff --git a/docs.org b/docs.org new file mode 100644 index 0000000..3861bd9 --- /dev/null +++ b/docs.org @@ -0,0 +1,16 @@ +* actual interface +** config KEY=VAL... +** pull-url PATH => URL +** push-url PATH => URL +** get-meta PATH => KEY=VAL\n... +** set-meta PATH KEY=VAL... +** repo-mode => push|pull +* ruby interface +** capabilities: .capabilities() => array + Returns an array of arrays. Each item in the array is an array of + tokens that make a line. +** create: .create(id string, meta map) => TODO +** get-meta: .get_meta() => map +** set-meta: .set_meta(map) => map + Returns the same as `get-meta` +** <EOF>: .finish() @@ -1,5 +1,6 @@ #!/bin/bash -# Copyright © 2014 Luke Shumaker <lukeshu@sbcglobal.net> +# Copyright © 2014, 2016 Luke Shumaker <lukeshu@sbcglobal.net> +# # This work is free. You can redistribute it and/or modify it under the # terms of the Do What The Fuck You Want To Public License, Version 2, # as published by Sam Hocevar. See the COPYING file for more details. @@ -8,46 +9,144 @@ # GNU/Linux-libre, those are the 'gitget' and 'librelib' packages, # respectively. # -# For other systems, the both live at: -# https://projects.parabolagnulinux.org/packages/libretools.git/ +# For other systems, they both live at: +# https://git.parabola.nu/packages/libretools.git/ set -o pipefail set -e . libremessages -config-get() { - [[ $# == 1 ]] || panic - git config --file "$conf_file" --get "$1" +usage() { + print 'Usage %s CONFIG-FILE' "${0##*/}" } -list-repos() { - [[ $# == 0 ]] || panic - git config --file "$conf_file" --list | cut -s -d. -f2 | sort -u +main() { + if [[ $# != 1 ]]; then + usage + exit + fi + declare -g cfg_file="$1" + + local r=0 + while read -r repo; do + handle-repo "$repo" || r=$? + done < <(cfg-list-repos) + return $r } -mirror-repo() { +handle-repo() { [[ $# == 1 ]] || panic - repo=$1 - canonical_path="$(config-get "repo.${repo}.canonical")" - local_path="$(config-get "repo.${repo}.local")" - description="$(config-get "repo.${repo}.description" || printf 'mirror of %s' "${canonical_path}")" + local repo=$1 + local local url upstream downstreams downstream r=0 + + # read configuration + local="$(cfg-get "repo.$repo.local")" + url="$(cfg-get "repo.$repo.url")" || true + upstream="$(cfg-get "repo.$repo.upstream")" || true + downstreams=($(cfg-get-all "repo.$repo.downstream")) || true - gitget -f -n "$repo" bare "$canonical_path" "$local_path" - printf '%s\n' "$description" > "$local_path/description.tmp" - mv -- "$local_path/description.tmp" "$local_path/description" + # download + if [[ -n "$upstream" ]]; then + download "$upstream" "$local" + fi + + # ensure that $local exists + test -f "$local"/HEAD + + # upload + for downstream in "${downstreams[@]}"; do + upload "$url" "$local" "$downstream" || r=$? + done + + return $r } -usage() { - print 'Usage %s CONFIG-FILE' "${0##*/}" +download() { + [[ $# == 3 ]] || panic + local repo=$1 + local remote=$2 + local local=$3 + + # download the repository + local url + url="$(remote "$remote" pull-url)" + gitget -f -n "$repo" "$url" "$local" + # download the metadata + remote "$remote" get-meta > "$local/git-mirror.tmp" + git config --file "$local/config" --rename-section git-mirror git-mirror-bak + local IFS='=' + while read -r key val; do + git config --file "$local/config" --add git-mirror."$key" "$val" + done < "$local/git-mirror.tmp" + rm -f "$local/git-mirror.tmp" + git config --file "$local/config" --remove-section git-mirror-bak } -main() { - if [[ $# != 1 ]]; then - usage - exit +upload() { + [[ $# == 3 ]] || panic + local clonable_url=$1 + local local=$2 + local remote=$3 + + # push metadata + { + printf '%q ' set-meta "${remote#*:}" + [[ -z "$clonable_url" ]] || printf '%q ' "mirror=$clonable_url" + git config --file "$local/config" --get-regexp '^git-mirror[.]' -z|sed -z 's/ /=/'|xargs -0r printf '%q ' + } | account "${remote%%:*}" + # push repository + local repo_mode + repo_mode=$(remote "$remote" repo-mode) + if [[ $repo_mode == push ]]; then + local push_url + push_url="$(remote "$remote" push-url)" + cd "$local" && git push --mirror "$push_url" fi - declare -g conf_file="$1" - while read -r repo; do mirror-repo "$repo"; done < <(list-repos) +} + +# Spawn an 'account.type' helper. It will read commands from stdin. +account() { + [[ $# == 1 ]] || panic + local account=$1 + + local account_type + account_type="$(cfg-get "account.$account.type")" + + { + cfg --list -z|sed -zn "s=^account[.]$account[.]=config =p"|grep -z -v '^type='|xargs -r0 printf '%s\n' + cat + } | git mirror-"$type" "$account" +} + +# `account` is awkward to use; so let's wrap it. +remote() { + [[ $# > 1 ]] || panic + local remote=$1 + + [[ $remote = *:* ]] + local account="${remote%%:*}" + local path="${remote#*:}" + + printf '%q ' "$2" "$path" "${@:3}" | account "$account" +} + +cfg() { + git config --file "$cfg_file" "$@" +} + +cfg-get() { + [[ $# == 1 ]] || panic + cfg --get-all "$1" +} + +cfg-get-all() { + [[ $# == 1 ]] || panic + cfg --get-all "$1" +} + +cfg-list-repos() { + [[ $# == 0 ]] || panic + cfg --name-only --get-regexp '^repo[.].*[.].*$' -z|sed -z -e 's|^repo[.]||' -e 's|[.][^.]*$||'|sort -zu|xargs -r0 printf '%s\n' } main "$@" diff --git a/git-mirror.conf.example b/git-mirror.conf.example new file mode 100644 index 0000000..8f34dc1 --- /dev/null +++ b/git-mirror.conf.example @@ -0,0 +1,27 @@ +[account "parabola"] + type = cgit + url = https://git.parabola.nu + +[account "gitlab"] + type = gitlab-ee + apiurl = https://gitlab.com/api/v3 + apikey = ABCDEFGHIJKLMNOPQRST + +# [account "github"] +# type = github +# url = https://github.com + +# [account "notabug"] +# type = gogs +# url = https://notabug.org + +# [account "repo-or-cz"] +# type = gitweb +# url = https://repo.or.cz + +[repo "abslibre/abslibre"] + upstream = parabola:abslibre/abslibre.git + local = abslibre/abslibre.git + url = https://git.parabola.nu/abslibre/abslibre.git + downstream = gitlab:parabola/abslibre_abslibre + downstream = github:parabola/abslibre_abslibre |