summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLuke Shumaker <lukeshu@sbcglobal.net>2016-12-15 02:07:07 -0500
committerLuke Shumaker <lukeshu@sbcglobal.net>2016-12-15 02:07:07 -0500
commit4974d0297cff29aa3d3731782e0fc661be31530b (patch)
tree916dbcf7e1428b410549a6d8ebab4adeafc50917
parent81092fa4520971697d47e6a5d498d3cbd785adaa (diff)
wip
-rwxr-xr-xbackends/github1
-rwxr-xr-xbackends/gitlab-ce152
-rwxr-xr-xbackends/gitlab-ee99
-rw-r--r--docs.org16
-rwxr-xr-xgit-mirror149
-rw-r--r--git-mirror.conf.example27
6 files changed, 419 insertions, 25 deletions
diff --git a/backends/github b/backends/github
new file mode 100755
index 0000000..49ffd26
--- /dev/null
+++ b/backends/github
@@ -0,0 +1 @@
+#!/usr/bin/env ruby
diff --git a/backends/gitlab-ce b/backends/gitlab-ce
new file mode 100755
index 0000000..3b6cc90
--- /dev/null
+++ b/backends/gitlab-ce
@@ -0,0 +1,152 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+
+# http://docs.gitlab.com/ee/workflow/repository_mirroring.html
+# https://gitlab.com/gitlab-org/gitlab-ee/issues/767
+
+require 'net/http'
+require 'uri'
+require 'cgi'
+require 'json'
+
+class GitLabCE
+ def initialize(api_uri, api_key, project_id)
+ @api_uri = URI(api_uri)
+ unless @api_uri.path.end_with?("/")
+ @api_uri.path += "/"
+ end
+ @api_key = api_key.to_s
+ @project_id = project_id.to_s
+
+ @connections = {}
+ @cache = {}
+
+ # API docs suck, just look at `lib/api/projects.rb`
+ @vars = [
+ :builds_enabled, # create | create-user | edit
+ :container_registry_enabled, # create | | edit
+ :default_branch # | create-user | edit
+ :description, # create | create-user | edit
+ :import_url, # create | create-user |
+ :issues_enabled, # create | create-user | edit
+ :lfs_enabled, # create | create-user | edit
+ :merge_requests_enabled, # create | create-user | edit
+ :name, # create | create-user | edit
+ :namespace_id, # create | |
+ :only_allow_merge_if_build_succeeds, # create | create-user | edit
+ :path, # create | | edit
+ :public, # create | create-user | edit
+ :public_builds, # create | create-user | edit
+ :request_access_enabled, # create | create-user | edit
+ :shared_runners_enabled, # create | create-user | edit
+ :snippets_enabled, # create | create-user | edit
+ :visibility_level, # create | create-user | edit
+ :wiki_enabled, # create | create-user | edit
+ :only_allow_merge_if_all_discussions_are_resolved # create | create-user | edit
+ ]
+
+ @vars = [
+ "builds_enabled",
+ "container_registry_enabled",
+ "default_branch",
+ "description",
+ "issues_enabled",
+ "lfs_enabled",
+ "merge_requests_enabled",
+ "name",
+ "only_allow_merge_if_all_discussions_are_resolved",
+ "only_allow_merge_if_build_succeeds",
+ "path",
+ "public",
+ "public_builds",
+ "request_access_enabled",
+ "shared_runners_enabled",
+ "snippets_enabled",
+ "visibility_level",
+ "wiki_enabled",
+ ]
+ end
+
+ def _connection(uri)
+ key=URI(uri.scheme+":")
+ key.host = uri.host
+ key.port = uri.port
+
+ @connections[key] ||= Net::HTTP::start(uri.host, uri.port, :use_ssl => uri.scheme == 'https')
+ return @connections[key]
+ end
+
+ def _info
+ unless @cache.has_key?(:info)
+ req = Net::HTTP::Get.new(@api_uri + "projects/" + CGI::escape(@project_id))
+ req.add_field("PRIVATE-TOKEN", @api_key)
+ con = _connection(req.uri)
+ res = con.request(req)
+ if res.code != "200"
+ throw res
+ end
+ @cache[:info] = JSON::parse(res.body)
+ end
+ return @cache[:info]
+ end
+
+ def get_meta
+ return _info.select{|k,v| @vars.include?(k)}
+ end
+
+ def set_meta(map)
+ illegal = map.select{|k,v| not @vars.include?(k)}
+ if illegal.count > 0
+ throw illegal
+ end
+
+ req = Net::HTTP::Put.new(@api_uri + "projects/" + CGI::escape(_info["id"].to_s))
+ req.add_field("PRIVATE-TOKEN", @api_key)
+ req.add_field("Content-Type", "application/json")
+ req.body = JSON::dump(map)
+ con = _connection(req.uri)
+ res = con.request(req)
+ if res.code != "200"
+ throw res
+ end
+ @cache[:info] = JSON::parse(res.body)
+ return get_meta
+ end
+
+ def pushURL
+ return _info["ssh_url_to_repo"]
+ end
+
+ def pullURL
+ return _info["http_url_to_repo"]
+ end
+
+ def create(id, map)
+ if map.has_key?("mirror")
+ map["import_url"] = map["mirror"]
+ map.delete("mirror")
+ end
+ req = Net::HTTP::Post.new(@api_uri + "projects")
+ req.add_field("PRIVATE-TOKEN", @api_key)
+ req.add_field("Content-Type", "application/json")
+ req.body = JSON::dump(map)
+ con = _connection(req.uri)
+ res = con.request(req)
+ end
+
+ def capabilities
+ return [
+ [ "get-meta", @vars ].flatten,
+ [ "set-meta", @vars ].flatten,
+ [ "push-url" ],
+ [ "pull-url" ],
+ ]
+ end
+
+ def finish
+ @connections.each do |k,v|
+ v.finish()
+ end
+ @connections = {}
+ end
+end
diff --git a/backends/gitlab-ee b/backends/gitlab-ee
new file mode 100755
index 0000000..d0db8b9
--- /dev/null
+++ b/backends/gitlab-ee
@@ -0,0 +1,99 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+
+# GitLab EE supports configuring a "project" (GitLab's term for a
+# repository+metadata) to display as a mirror of another repository.
+#
+# http://docs.gitlab.com/ee/workflow/repository_mirroring.html
+#
+# Unfortunately, the JSON API doesn't support this
+#
+# https://gitlab.com/gitlab-org/gitlab-ee/issues/767
+#
+# So, we must use the (undocumented!) HTTP API, which is actually
+# pretty clean, except that screen-scraping the reads (via nokogiri)
+# is gross, and that the error messages are unhelpful.
+
+load 'gitlab-ce'
+require 'net/http'
+require 'uri'
+require 'nokogiri'
+
+class GitLabEE < GitLabCE
+ def _mirrorURL
+ unless @cache.has_key?(:mirror)
+ req = Net::HTTP::Get.new(URI(_info["web_url"]+"/mirror"))
+ req.add_field("PRIVATE-TOKEN", @api_key)
+ con = _connection(req.uri)
+ res = con.request(req)
+ if res.code != "200"
+ throw res
+ end
+ @cache[:mirror_res]=res
+ doc = Nokogiri::HTML(res.body)
+
+ @cache[:mirror_cookie] = res["set-cookie"]
+ @cache[:mirror_token] = doc.css('input[name="authenticity_token"]').first["value"]
+ is_mirror = doc.css("#project_mirror").first["checked"]
+ if !is_mirror
+ @cache[:mirror] = nil
+ else
+ @cache[:mirror] = URI(doc.css("#project_import_url").first["value"])
+ end
+ end
+ return @cache[:mirror]
+ end
+
+ def _mirrorURL=(url)
+ _mirrorURL
+
+ req = Net::HTTP::Patch.new(URI(_info["web_url"]+"/mirror"))
+ req.add_field("PRIVATE-TOKEN", @api_key) # authenticate
+ req.add_field("Cookie", @cache[:mirror_cookie]) # session id
+ req.form_data = {
+ "utf8" => "✓",
+ "authenticity_token" => @cache[:mirror_token], # session state
+ "project[mirror]" => (url.nil? ? "0" : "1"),
+ "project[import_url]" => url.to_s,
+ }
+
+ con = _connection(req.uri)
+ res = con.request(req)
+ if res.code != "302"
+ throw res
+ end
+
+ @cache.delete(:mirror)
+ @cache.delete(:mirror_token)
+ @cache.delete(:mirror_cookie)
+ return URI(url)
+ end
+
+ def get_meta
+ map = super
+ map["mirror"] = _mirrorURL.to_s
+ return map
+ end
+
+ def set_meta(map)
+ if map.has_key?("mirror")
+ self._mirrorURL=map["mirror"]
+ map.delete("mirror")
+ end
+ return super(map)
+ end
+
+ def create(id, map)
+ super(id, map)
+ self._mirrorURL=map["mirror"]
+ end
+
+ def capabilities
+ return super.map{|c|
+ if c[0] == "get-meta" or c[0] == "set-meta"
+ c << "mirror"
+ end
+ c
+ }
+ end
+end
diff --git a/docs.org b/docs.org
new file mode 100644
index 0000000..3861bd9
--- /dev/null
+++ b/docs.org
@@ -0,0 +1,16 @@
+* actual interface
+** config KEY=VAL...
+** pull-url PATH => URL
+** push-url PATH => URL
+** get-meta PATH => KEY=VAL\n...
+** set-meta PATH KEY=VAL...
+** repo-mode => push|pull
+* ruby interface
+** capabilities: .capabilities() => array
+ Returns an array of arrays. Each item in the array is an array of
+ tokens that make a line.
+** create: .create(id string, meta map) => TODO
+** get-meta: .get_meta() => map
+** set-meta: .set_meta(map) => map
+ Returns the same as `get-meta`
+** <EOF>: .finish()
diff --git a/git-mirror b/git-mirror
index 76763c1..1a33774 100755
--- a/git-mirror
+++ b/git-mirror
@@ -1,5 +1,6 @@
#!/bin/bash
-# Copyright © 2014 Luke Shumaker <lukeshu@sbcglobal.net>
+# Copyright © 2014, 2016 Luke Shumaker <lukeshu@sbcglobal.net>
+#
# This work is free. You can redistribute it and/or modify it under the
# terms of the Do What The Fuck You Want To Public License, Version 2,
# as published by Sam Hocevar. See the COPYING file for more details.
@@ -8,46 +9,144 @@
# GNU/Linux-libre, those are the 'gitget' and 'librelib' packages,
# respectively.
#
-# For other systems, the both live at:
-# https://projects.parabolagnulinux.org/packages/libretools.git/
+# For other systems, they both live at:
+# https://git.parabola.nu/packages/libretools.git/
set -o pipefail
set -e
. libremessages
-config-get() {
- [[ $# == 1 ]] || panic
- git config --file "$conf_file" --get "$1"
+usage() {
+ print 'Usage %s CONFIG-FILE' "${0##*/}"
}
-list-repos() {
- [[ $# == 0 ]] || panic
- git config --file "$conf_file" --list | cut -s -d. -f2 | sort -u
+main() {
+ if [[ $# != 1 ]]; then
+ usage
+ exit
+ fi
+ declare -g cfg_file="$1"
+
+ local r=0
+ while read -r repo; do
+ handle-repo "$repo" || r=$?
+ done < <(cfg-list-repos)
+ return $r
}
-mirror-repo() {
+handle-repo() {
[[ $# == 1 ]] || panic
- repo=$1
- canonical_path="$(config-get "repo.${repo}.canonical")"
- local_path="$(config-get "repo.${repo}.local")"
- description="$(config-get "repo.${repo}.description" || printf 'mirror of %s' "${canonical_path}")"
+ local repo=$1
+ local local url upstream downstreams downstream r=0
+
+ # read configuration
+ local="$(cfg-get "repo.$repo.local")"
+ url="$(cfg-get "repo.$repo.url")" || true
+ upstream="$(cfg-get "repo.$repo.upstream")" || true
+ downstreams=($(cfg-get-all "repo.$repo.downstream")) || true
- gitget -f -n "$repo" bare "$canonical_path" "$local_path"
- printf '%s\n' "$description" > "$local_path/description.tmp"
- mv -- "$local_path/description.tmp" "$local_path/description"
+ # download
+ if [[ -n "$upstream" ]]; then
+ download "$upstream" "$local"
+ fi
+
+ # ensure that $local exists
+ test -f "$local"/HEAD
+
+ # upload
+ for downstream in "${downstreams[@]}"; do
+ upload "$url" "$local" "$downstream" || r=$?
+ done
+
+ return $r
}
-usage() {
- print 'Usage %s CONFIG-FILE' "${0##*/}"
+download() {
+ [[ $# == 3 ]] || panic
+ local repo=$1
+ local remote=$2
+ local local=$3
+
+ # download the repository
+ local url
+ url="$(remote "$remote" pull-url)"
+ gitget -f -n "$repo" "$url" "$local"
+ # download the metadata
+ remote "$remote" get-meta > "$local/git-mirror.tmp"
+ git config --file "$local/config" --rename-section git-mirror git-mirror-bak
+ local IFS='='
+ while read -r key val; do
+ git config --file "$local/config" --add git-mirror."$key" "$val"
+ done < "$local/git-mirror.tmp"
+ rm -f "$local/git-mirror.tmp"
+ git config --file "$local/config" --remove-section git-mirror-bak
}
-main() {
- if [[ $# != 1 ]]; then
- usage
- exit
+upload() {
+ [[ $# == 3 ]] || panic
+ local clonable_url=$1
+ local local=$2
+ local remote=$3
+
+ # push metadata
+ {
+ printf '%q ' set-meta "${remote#*:}"
+ [[ -z "$clonable_url" ]] || printf '%q ' "mirror=$clonable_url"
+ git config --file "$local/config" --get-regexp '^git-mirror[.]' -z|sed -z 's/ /=/'|xargs -0r printf '%q '
+ } | account "${remote%%:*}"
+ # push repository
+ local repo_mode
+ repo_mode=$(remote "$remote" repo-mode)
+ if [[ $repo_mode == push ]]; then
+ local push_url
+ push_url="$(remote "$remote" push-url)"
+ cd "$local" && git push --mirror "$push_url"
fi
- declare -g conf_file="$1"
- while read -r repo; do mirror-repo "$repo"; done < <(list-repos)
+}
+
+# Spawn an 'account.type' helper. It will read commands from stdin.
+account() {
+ [[ $# == 1 ]] || panic
+ local account=$1
+
+ local account_type
+ account_type="$(cfg-get "account.$account.type")"
+
+ {
+ cfg --list -z|sed -zn "s=^account[.]$account[.]=config =p"|grep -z -v '^type='|xargs -r0 printf '%s\n'
+ cat
+ } | git mirror-"$type" "$account"
+}
+
+# `account` is awkward to use; so let's wrap it.
+remote() {
+ [[ $# > 1 ]] || panic
+ local remote=$1
+
+ [[ $remote = *:* ]]
+ local account="${remote%%:*}"
+ local path="${remote#*:}"
+
+ printf '%q ' "$2" "$path" "${@:3}" | account "$account"
+}
+
+cfg() {
+ git config --file "$cfg_file" "$@"
+}
+
+cfg-get() {
+ [[ $# == 1 ]] || panic
+ cfg --get-all "$1"
+}
+
+cfg-get-all() {
+ [[ $# == 1 ]] || panic
+ cfg --get-all "$1"
+}
+
+cfg-list-repos() {
+ [[ $# == 0 ]] || panic
+ cfg --name-only --get-regexp '^repo[.].*[.].*$' -z|sed -z -e 's|^repo[.]||' -e 's|[.][^.]*$||'|sort -zu|xargs -r0 printf '%s\n'
}
main "$@"
diff --git a/git-mirror.conf.example b/git-mirror.conf.example
new file mode 100644
index 0000000..8f34dc1
--- /dev/null
+++ b/git-mirror.conf.example
@@ -0,0 +1,27 @@
+[account "parabola"]
+ type = cgit
+ url = https://git.parabola.nu
+
+[account "gitlab"]
+ type = gitlab-ee
+ apiurl = https://gitlab.com/api/v3
+ apikey = ABCDEFGHIJKLMNOPQRST
+
+# [account "github"]
+# type = github
+# url = https://github.com
+
+# [account "notabug"]
+# type = gogs
+# url = https://notabug.org
+
+# [account "repo-or-cz"]
+# type = gitweb
+# url = https://repo.or.cz
+
+[repo "abslibre/abslibre"]
+ upstream = parabola:abslibre/abslibre.git
+ local = abslibre/abslibre.git
+ url = https://git.parabola.nu/abslibre/abslibre.git
+ downstream = gitlab:parabola/abslibre_abslibre
+ downstream = github:parabola/abslibre_abslibre