diff options
Diffstat (limited to 'src/import/pull-dkr.c')
| -rw-r--r-- | src/import/pull-dkr.c | 896 |
1 files changed, 896 insertions, 0 deletions
diff --git a/src/import/pull-dkr.c b/src/import/pull-dkr.c new file mode 100644 index 0000000000..ecbf8063ce --- /dev/null +++ b/src/import/pull-dkr.c @@ -0,0 +1,896 @@ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ + +/*** + This file is part of systemd. + + Copyright 2014 Lennart Poettering + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + systemd is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with systemd; If not, see <http://www.gnu.org/licenses/>. +***/ + +#include <curl/curl.h> +#include <sys/prctl.h> + +#include "sd-daemon.h" +#include "json.h" +#include "strv.h" +#include "btrfs-util.h" +#include "utf8.h" +#include "mkdir.h" +#include "path-util.h" +#include "import-util.h" +#include "curl-util.h" +#include "aufs-util.h" +#include "pull-job.h" +#include "pull-common.h" +#include "pull-dkr.h" + +typedef enum DkrProgress { + DKR_SEARCHING, + DKR_RESOLVING, + DKR_METADATA, + DKR_DOWNLOADING, + DKR_COPYING, +} DkrProgress; + +struct DkrPull { + sd_event *event; + CurlGlue *glue; + + char *index_url; + char *image_root; + + PullJob *images_job; + PullJob *tags_job; + PullJob *ancestry_job; + PullJob *json_job; + PullJob *layer_job; + + char *name; + char *tag; + char *id; + + char *response_token; + char **response_registries; + + char **ancestry; + unsigned n_ancestry; + unsigned current_ancestry; + + DkrPullFinished on_finished; + void *userdata; + + char *local; + bool force_local; + bool grow_machine_directory; + + char *temp_path; + char *final_path; + + pid_t tar_pid; +}; + +#define PROTOCOL_PREFIX "https://" + +#define HEADER_TOKEN "X-Do" /* the HTTP header for the auth token */ "cker-Token:" +#define HEADER_REGISTRY "X-Do" /*the HTTP header for the registry */ "cker-Endpoints:" + +#define LAYERS_MAX 2048 + +static void dkr_pull_job_on_finished(PullJob *j); + +DkrPull* dkr_pull_unref(DkrPull *i) { + if (!i) + return NULL; + + if (i->tar_pid > 1) { + (void) kill_and_sigcont(i->tar_pid, SIGKILL); + (void) wait_for_terminate(i->tar_pid, NULL); + } + + pull_job_unref(i->images_job); + pull_job_unref(i->tags_job); + pull_job_unref(i->ancestry_job); + pull_job_unref(i->json_job); + pull_job_unref(i->layer_job); + + curl_glue_unref(i->glue); + sd_event_unref(i->event); + + if (i->temp_path) { + (void) btrfs_subvol_remove(i->temp_path); + (void) rm_rf_dangerous(i->temp_path, false, true, false); + free(i->temp_path); + } + + free(i->name); + free(i->tag); + free(i->id); + free(i->response_token); + free(i->response_registries); + strv_free(i->ancestry); + free(i->final_path); + free(i->index_url); + free(i->image_root); + free(i->local); + free(i); + + return NULL; +} + +int dkr_pull_new( + DkrPull **ret, + sd_event *event, + const char *index_url, + const char *image_root, + DkrPullFinished on_finished, + void *userdata) { + + _cleanup_(dkr_pull_unrefp) DkrPull *i = NULL; + char *e; + int r; + + assert(ret); + assert(index_url); + + if (!http_url_is_valid(index_url)) + return -EINVAL; + + i = new0(DkrPull, 1); + if (!i) + return -ENOMEM; + + i->on_finished = on_finished; + i->userdata = userdata; + + i->image_root = strdup(image_root ?: "/var/lib/machines"); + if (!i->image_root) + return -ENOMEM; + + i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines"); + + i->index_url = strdup(index_url); + if (!i->index_url) + return -ENOMEM; + + e = endswith(i->index_url, "/"); + if (e) + *e = 0; + + if (event) + i->event = sd_event_ref(event); + else { + r = sd_event_default(&i->event); + if (r < 0) + return r; + } + + r = curl_glue_new(&i->glue, i->event); + if (r < 0) + return r; + + i->glue->on_finished = pull_job_curl_on_finished; + i->glue->userdata = i; + + *ret = i; + i = NULL; + + return 0; +} + +static void dkr_pull_report_progress(DkrPull *i, DkrProgress p) { + unsigned percent; + + assert(i); + + switch (p) { + + case DKR_SEARCHING: + percent = 0; + if (i->images_job) + percent += i->images_job->progress_percent * 5 / 100; + break; + + case DKR_RESOLVING: + percent = 5; + if (i->tags_job) + percent += i->tags_job->progress_percent * 5 / 100; + break; + + case DKR_METADATA: + percent = 10; + if (i->ancestry_job) + percent += i->ancestry_job->progress_percent * 5 / 100; + if (i->json_job) + percent += i->json_job->progress_percent * 5 / 100; + break; + + case DKR_DOWNLOADING: + percent = 20; + percent += 75 * i->current_ancestry / MAX(1U, i->n_ancestry); + if (i->layer_job) + percent += i->layer_job->progress_percent * 75 / MAX(1U, i->n_ancestry) / 100; + + break; + + case DKR_COPYING: + percent = 95; + break; + + default: + assert_not_reached("Unknown progress state"); + } + + sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent); + log_debug("Combined progress %u%%", percent); +} + +static int parse_id(const void *payload, size_t size, char **ret) { + _cleanup_free_ char *buf = NULL, *id = NULL, *other = NULL; + union json_value v = {}; + void *json_state = NULL; + const char *p; + int t; + + assert(payload); + assert(ret); + + if (size <= 0) + return -EBADMSG; + + if (memchr(payload, 0, size)) + return -EBADMSG; + + buf = strndup(payload, size); + if (!buf) + return -ENOMEM; + + p = buf; + t = json_tokenize(&p, &id, &v, &json_state, NULL); + if (t < 0) + return t; + if (t != JSON_STRING) + return -EBADMSG; + + t = json_tokenize(&p, &other, &v, &json_state, NULL); + if (t < 0) + return t; + if (t != JSON_END) + return -EBADMSG; + + if (!dkr_id_is_valid(id)) + return -EBADMSG; + + *ret = id; + id = NULL; + + return 0; +} + +static int parse_ancestry(const void *payload, size_t size, char ***ret) { + _cleanup_free_ char *buf = NULL; + void *json_state = NULL; + const char *p; + enum { + STATE_BEGIN, + STATE_ITEM, + STATE_COMMA, + STATE_END, + } state = STATE_BEGIN; + _cleanup_strv_free_ char **l = NULL; + size_t n = 0, allocated = 0; + + if (size <= 0) + return -EBADMSG; + + if (memchr(payload, 0, size)) + return -EBADMSG; + + buf = strndup(payload, size); + if (!buf) + return -ENOMEM; + + p = buf; + for (;;) { + _cleanup_free_ char *str; + union json_value v = {}; + int t; + + t = json_tokenize(&p, &str, &v, &json_state, NULL); + if (t < 0) + return t; + + switch (state) { + + case STATE_BEGIN: + if (t == JSON_ARRAY_OPEN) + state = STATE_ITEM; + else + return -EBADMSG; + + break; + + case STATE_ITEM: + if (t == JSON_STRING) { + if (!dkr_id_is_valid(str)) + return -EBADMSG; + + if (n+1 > LAYERS_MAX) + return -EFBIG; + + if (!GREEDY_REALLOC(l, allocated, n + 2)) + return -ENOMEM; + + l[n++] = str; + str = NULL; + l[n] = NULL; + + state = STATE_COMMA; + + } else if (t == JSON_ARRAY_CLOSE) + state = STATE_END; + else + return -EBADMSG; + + break; + + case STATE_COMMA: + if (t == JSON_COMMA) + state = STATE_ITEM; + else if (t == JSON_ARRAY_CLOSE) + state = STATE_END; + else + return -EBADMSG; + break; + + case STATE_END: + if (t == JSON_END) { + + if (strv_isempty(l)) + return -EBADMSG; + + if (!strv_is_uniq(l)) + return -EBADMSG; + + l = strv_reverse(l); + + *ret = l; + l = NULL; + return 0; + } else + return -EBADMSG; + } + + } +} + +static const char *dkr_pull_current_layer(DkrPull *i) { + assert(i); + + if (strv_isempty(i->ancestry)) + return NULL; + + return i->ancestry[i->current_ancestry]; +} + +static const char *dkr_pull_current_base_layer(DkrPull *i) { + assert(i); + + if (strv_isempty(i->ancestry)) + return NULL; + + if (i->current_ancestry <= 0) + return NULL; + + return i->ancestry[i->current_ancestry-1]; +} + +static int dkr_pull_add_token(DkrPull *i, PullJob *j) { + const char *t; + + assert(i); + assert(j); + + if (i->response_token) + t = strjoina("Authorization: Token ", i->response_token); + else + t = HEADER_TOKEN " true"; + + j->request_header = curl_slist_new("Accept: application/json", t, NULL); + if (!j->request_header) + return -ENOMEM; + + return 0; +} + +static bool dkr_pull_is_done(DkrPull *i) { + assert(i); + assert(i->images_job); + + if (i->images_job->state != PULL_JOB_DONE) + return false; + + if (!i->tags_job || i->tags_job->state != PULL_JOB_DONE) + return false; + + if (!i->ancestry_job || i->ancestry_job->state != PULL_JOB_DONE) + return false; + + if (!i->json_job || i->json_job->state != PULL_JOB_DONE) + return false; + + if (i->layer_job && i->layer_job->state != PULL_JOB_DONE) + return false; + + if (dkr_pull_current_layer(i)) + return false; + + return true; +} + +static int dkr_pull_make_local_copy(DkrPull *i) { + int r; + + assert(i); + + if (!i->local) + return 0; + + if (!i->final_path) { + i->final_path = strjoin(i->image_root, "/.dkr-", i->id, NULL); + if (!i->final_path) + return log_oom(); + } + + r = pull_make_local_copy(i->final_path, i->image_root, i->local, i->force_local); + if (r < 0) + return r; + + return 0; +} + +static int dkr_pull_job_on_open_disk(PullJob *j) { + const char *base; + DkrPull *i; + int r; + + assert(j); + assert(j->userdata); + + i = j->userdata; + assert(i->layer_job == j); + assert(i->final_path); + assert(!i->temp_path); + assert(i->tar_pid <= 0); + + r = tempfn_random(i->final_path, &i->temp_path); + if (r < 0) + return log_oom(); + + mkdir_parents_label(i->temp_path, 0700); + + base = dkr_pull_current_base_layer(i); + if (base) { + const char *base_path; + + base_path = strjoina(i->image_root, "/.dkr-", base); + r = btrfs_subvol_snapshot(base_path, i->temp_path, false, true); + } else + r = btrfs_subvol_make(i->temp_path); + if (r < 0) + return log_error_errno(r, "Failed to make btrfs subvolume %s: %m", i->temp_path); + + j->disk_fd = pull_fork_tar(i->temp_path, &i->tar_pid); + if (j->disk_fd < 0) + return j->disk_fd; + + return 0; +} + +static void dkr_pull_job_on_progress(PullJob *j) { + DkrPull *i; + + assert(j); + assert(j->userdata); + + i = j->userdata; + + dkr_pull_report_progress( + i, + j == i->images_job ? DKR_SEARCHING : + j == i->tags_job ? DKR_RESOLVING : + j == i->ancestry_job || j == i->json_job ? DKR_METADATA : + DKR_DOWNLOADING); +} + +static int dkr_pull_pull_layer(DkrPull *i) { + _cleanup_free_ char *path = NULL; + const char *url, *layer = NULL; + int r; + + assert(i); + assert(!i->layer_job); + assert(!i->temp_path); + assert(!i->final_path); + + for (;;) { + layer = dkr_pull_current_layer(i); + if (!layer) + return 0; /* no more layers */ + + path = strjoin(i->image_root, "/.dkr-", layer, NULL); + if (!path) + return log_oom(); + + if (laccess(path, F_OK) < 0) { + if (errno == ENOENT) + break; + + return log_error_errno(errno, "Failed to check for container: %m"); + } + + log_info("Layer %s already exists, skipping.", layer); + + i->current_ancestry++; + + free(path); + path = NULL; + } + + log_info("Pulling layer %s...", layer); + + i->final_path = path; + path = NULL; + + url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", layer, "/layer"); + r = pull_job_new(&i->layer_job, url, i->glue, i); + if (r < 0) + return log_error_errno(r, "Failed to allocate layer job: %m"); + + r = dkr_pull_add_token(i, i->layer_job); + if (r < 0) + return log_oom(); + + i->layer_job->on_finished = dkr_pull_job_on_finished; + i->layer_job->on_open_disk = dkr_pull_job_on_open_disk; + i->layer_job->on_progress = dkr_pull_job_on_progress; + i->layer_job->grow_machine_directory = i->grow_machine_directory; + + r = pull_job_begin(i->layer_job); + if (r < 0) + return log_error_errno(r, "Failed to start layer job: %m"); + + return 0; +} + +static void dkr_pull_job_on_finished(PullJob *j) { + DkrPull *i; + int r; + + assert(j); + assert(j->userdata); + + i = j->userdata; + if (j->error != 0) { + if (j == i->images_job) + log_error_errno(j->error, "Failed to retrieve images list. (Wrong index URL?)"); + else if (j == i->tags_job) + log_error_errno(j->error, "Failed to retrieve tags list."); + else if (j == i->ancestry_job) + log_error_errno(j->error, "Failed to retrieve ancestry list."); + else if (j == i->json_job) + log_error_errno(j->error, "Failed to retrieve json data."); + else + log_error_errno(j->error, "Failed to retrieve layer data."); + + r = j->error; + goto finish; + } + + if (i->images_job == j) { + const char *url; + + assert(!i->tags_job); + assert(!i->ancestry_job); + assert(!i->json_job); + assert(!i->layer_job); + + if (strv_isempty(i->response_registries)) { + r = -EBADMSG; + log_error("Didn't get registry information."); + goto finish; + } + + log_info("Index lookup succeeded, directed to registry %s.", i->response_registries[0]); + dkr_pull_report_progress(i, DKR_RESOLVING); + + url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/repositories/", i->name, "/tags/", i->tag); + r = pull_job_new(&i->tags_job, url, i->glue, i); + if (r < 0) { + log_error_errno(r, "Failed to allocate tags job: %m"); + goto finish; + } + + r = dkr_pull_add_token(i, i->tags_job); + if (r < 0) { + log_oom(); + goto finish; + } + + i->tags_job->on_finished = dkr_pull_job_on_finished; + i->tags_job->on_progress = dkr_pull_job_on_progress; + + r = pull_job_begin(i->tags_job); + if (r < 0) { + log_error_errno(r, "Failed to start tags job: %m"); + goto finish; + } + + } else if (i->tags_job == j) { + const char *url; + char *id = NULL; + + assert(!i->ancestry_job); + assert(!i->json_job); + assert(!i->layer_job); + + r = parse_id(j->payload, j->payload_size, &id); + if (r < 0) { + log_error_errno(r, "Failed to parse JSON id."); + goto finish; + } + + free(i->id); + i->id = id; + + log_info("Tag lookup succeeded, resolved to layer %s.", i->id); + dkr_pull_report_progress(i, DKR_METADATA); + + url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/ancestry"); + r = pull_job_new(&i->ancestry_job, url, i->glue, i); + if (r < 0) { + log_error_errno(r, "Failed to allocate ancestry job: %m"); + goto finish; + } + + r = dkr_pull_add_token(i, i->ancestry_job); + if (r < 0) { + log_oom(); + goto finish; + } + + i->ancestry_job->on_finished = dkr_pull_job_on_finished; + i->ancestry_job->on_progress = dkr_pull_job_on_progress; + + url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/json"); + r = pull_job_new(&i->json_job, url, i->glue, i); + if (r < 0) { + log_error_errno(r, "Failed to allocate json job: %m"); + goto finish; + } + + r = dkr_pull_add_token(i, i->json_job); + if (r < 0) { + log_oom(); + goto finish; + } + + i->json_job->on_finished = dkr_pull_job_on_finished; + i->json_job->on_progress = dkr_pull_job_on_progress; + + r = pull_job_begin(i->ancestry_job); + if (r < 0) { + log_error_errno(r, "Failed to start ancestry job: %m"); + goto finish; + } + + r = pull_job_begin(i->json_job); + if (r < 0) { + log_error_errno(r, "Failed to start json job: %m"); + goto finish; + } + + } else if (i->ancestry_job == j) { + char **ancestry = NULL, **k; + unsigned n; + + assert(!i->layer_job); + + r = parse_ancestry(j->payload, j->payload_size, &ancestry); + if (r < 0) { + log_error_errno(r, "Failed to parse JSON id."); + goto finish; + } + + n = strv_length(ancestry); + if (n <= 0 || !streq(ancestry[n-1], i->id)) { + log_error("Ancestry doesn't end in main layer."); + strv_free(ancestry); + r = -EBADMSG; + goto finish; + } + + log_info("Ancestor lookup succeeded, requires layers:\n"); + STRV_FOREACH(k, ancestry) + log_info("\t%s", *k); + + strv_free(i->ancestry); + i->ancestry = ancestry; + i->n_ancestry = n; + i->current_ancestry = 0; + + dkr_pull_report_progress(i, DKR_DOWNLOADING); + + r = dkr_pull_pull_layer(i); + if (r < 0) + goto finish; + + } else if (i->layer_job == j) { + assert(i->temp_path); + assert(i->final_path); + + j->disk_fd = safe_close(j->disk_fd); + + if (i->tar_pid > 0) { + r = wait_for_terminate_and_warn("tar", i->tar_pid, true); + i->tar_pid = 0; + if (r < 0) + goto finish; + } + + r = aufs_resolve(i->temp_path); + if (r < 0) { + log_error_errno(r, "Failed to resolve aufs whiteouts: %m"); + goto finish; + } + + r = btrfs_subvol_set_read_only(i->temp_path, true); + if (r < 0) { + log_error_errno(r, "Failed to mark snapshot read-only: %m"); + goto finish; + } + + if (rename(i->temp_path, i->final_path) < 0) { + log_error_errno(errno, "Failed to rename snaphsot: %m"); + goto finish; + } + + log_info("Completed writing to layer %s.", i->final_path); + + i->layer_job = pull_job_unref(i->layer_job); + free(i->temp_path); + i->temp_path = NULL; + free(i->final_path); + i->final_path = NULL; + + i->current_ancestry ++; + r = dkr_pull_pull_layer(i); + if (r < 0) + goto finish; + + } else if (i->json_job != j) + assert_not_reached("Got finished event for unknown curl object"); + + if (!dkr_pull_is_done(i)) + return; + + dkr_pull_report_progress(i, DKR_COPYING); + + r = dkr_pull_make_local_copy(i); + if (r < 0) + goto finish; + + r = 0; + +finish: + if (i->on_finished) + i->on_finished(i, r, i->userdata); + else + sd_event_exit(i->event, r); +} + +static int dkr_pull_job_on_header(PullJob *j, const char *header, size_t sz) { + _cleanup_free_ char *registry = NULL; + char *token; + DkrPull *i; + int r; + + assert(j); + assert(j->userdata); + + i = j->userdata; + + r = curl_header_strdup(header, sz, HEADER_TOKEN, &token); + if (r < 0) + return log_oom(); + if (r > 0) { + free(i->response_token); + i->response_token = token; + return 0; + } + + r = curl_header_strdup(header, sz, HEADER_REGISTRY, ®istry); + if (r < 0) + return log_oom(); + if (r > 0) { + char **l, **k; + + l = strv_split(registry, ","); + if (!l) + return log_oom(); + + STRV_FOREACH(k, l) { + if (!hostname_is_valid(*k)) { + log_error("Registry hostname is not valid."); + strv_free(l); + return -EBADMSG; + } + } + + strv_free(i->response_registries); + i->response_registries = l; + } + + return 0; +} + +int dkr_pull_start(DkrPull *i, const char *name, const char *tag, const char *local, bool force_local) { + const char *url; + int r; + + assert(i); + + if (!dkr_name_is_valid(name)) + return -EINVAL; + + if (tag && !dkr_tag_is_valid(tag)) + return -EINVAL; + + if (local && !machine_name_is_valid(local)) + return -EINVAL; + + if (i->images_job) + return -EBUSY; + + if (!tag) + tag = "latest"; + + r = free_and_strdup(&i->local, local); + if (r < 0) + return r; + i->force_local = force_local; + + r = free_and_strdup(&i->name, name); + if (r < 0) + return r; + r = free_and_strdup(&i->tag, tag); + if (r < 0) + return r; + + url = strjoina(i->index_url, "/v1/repositories/", name, "/images"); + + r = pull_job_new(&i->images_job, url, i->glue, i); + if (r < 0) + return r; + + r = dkr_pull_add_token(i, i->images_job); + if (r < 0) + return r; + + i->images_job->on_finished = dkr_pull_job_on_finished; + i->images_job->on_header = dkr_pull_job_on_header; + i->images_job->on_progress = dkr_pull_job_on_progress; + + return pull_job_begin(i->images_job); +} |