From 2c4fb0eab8fd05d6375fa3b78b5e2ee5eb93bf09 Mon Sep 17 00:00:00 2001 From: Pavel Odvody Date: Tue, 19 May 2015 16:30:33 +0200 Subject: import/pull-dkr: V2 image specification and manifest support The maximum number of layers changed to 127, as in Dkr. --- src/import/pull-dkr.c | 542 +++++++++++++++++++++++++++++++++++++++++++++----- src/import/pull-dkr.h | 3 +- 2 files changed, 491 insertions(+), 54 deletions(-) (limited to 'src/import') diff --git a/src/import/pull-dkr.c b/src/import/pull-dkr.c index f89eb88669..38c8db9eb2 100644 --- a/src/import/pull-dkr.c +++ b/src/import/pull-dkr.c @@ -52,6 +52,9 @@ struct DkrPull { sd_event *event; CurlGlue *glue; + char *index_protocol; + char *index_address; + char *index_url; char *image_root; @@ -62,9 +65,10 @@ struct DkrPull { PullJob *layer_job; char *name; - char *tag; + char *reference; char *id; + char *response_digest; char *response_token; char **response_registries; @@ -88,9 +92,9 @@ struct DkrPull { #define PROTOCOL_PREFIX "https://" #define HEADER_TOKEN "X-Do" /* the HTTP header for the auth token */ "cker-Token:" -#define HEADER_REGISTRY "X-Do" /*the HTTP header for the registry */ "cker-Endpoints:" - -#define LAYERS_MAX 2048 +#define HEADER_REGISTRY "X-Do" /* the HTTP header for the registry */ "cker-Endpoints:" +#define HEADER_DIGEST "Do" /* the HTTP header for the manifest digest */ "cker-Content-Digest:" +#define LAYERS_MAX 127 static void dkr_pull_job_on_finished(PullJob *j); @@ -118,12 +122,13 @@ DkrPull* dkr_pull_unref(DkrPull *i) { } free(i->name); - free(i->tag); + free(i->reference); free(i->id); free(i->response_token); - free(i->response_registries); strv_free(i->ancestry); free(i->final_path); + free(i->index_address); + free(i->index_protocol); free(i->index_url); free(i->image_root); free(i->local); @@ -417,10 +422,27 @@ static int dkr_pull_add_token(DkrPull *i, PullJob *j) { return 0; } +static int dkr_pull_add_bearer_token(DkrPull *i, PullJob *j) { + const char *t = NULL; + + assert(i); + assert(j); + + if (i->response_token) + t = strjoina("Authorization: Bearer ", i->response_token); + else + return -EINVAL; + + j->request_header = curl_slist_new("Accept: application/json", t, NULL); + if (!j->request_header) + return -ENOMEM; + + return 0; +} + static bool dkr_pull_is_done(DkrPull *i) { assert(i); assert(i->images_job); - if (i->images_job->state != PULL_JOB_DONE) return false; @@ -430,7 +452,7 @@ static bool dkr_pull_is_done(DkrPull *i) { if (!i->ancestry_job || i->ancestry_job->state != PULL_JOB_DONE) return false; - if (!i->json_job || i->json_job->state != PULL_JOB_DONE) + if (i->json_job && i->json_job->state != PULL_JOB_DONE) return false; if (i->layer_job && i->layer_job->state != PULL_JOB_DONE) @@ -442,8 +464,9 @@ static bool dkr_pull_is_done(DkrPull *i) { return true; } -static int dkr_pull_make_local_copy(DkrPull *i) { +static int dkr_pull_make_local_copy(DkrPull *i, DkrPullVersion version) { int r; + _cleanup_free_ char *p = NULL; assert(i); @@ -456,10 +479,30 @@ static int dkr_pull_make_local_copy(DkrPull *i) { return log_oom(); } - r = pull_make_local_copy(i->final_path, i->image_root, i->local, i->force_local); + if (version == DKR_PULL_V2) { + r = path_get_parent(i->image_root, &p); + if (r < 0) + return r; + } + + r = pull_make_local_copy(i->final_path, p ?: i->image_root, i->local, i->force_local); if (r < 0) return r; + if (version == DKR_PULL_V2) { + char **k = NULL; + STRV_FOREACH(k, i->ancestry) { + _cleanup_free_ char *d = strjoin(i->image_root, "/.dkr-", *k, NULL); + r = btrfs_subvol_remove(d, false); + if (r < 0) + return r; + } + + r = rmdir(i->image_root); + if (r < 0) + return r; + } + return 0; } @@ -517,6 +560,68 @@ static void dkr_pull_job_on_progress(PullJob *j) { DKR_DOWNLOADING); } +static void dkr_pull_job_on_finished_v2(PullJob *j); + +static int dkr_pull_pull_layer_v2(DkrPull *i) { + _cleanup_free_ char *path = NULL; + const char *url, *layer = NULL; + int r; + + assert(i); + assert(!i->layer_job); + assert(!i->temp_path); + assert(!i->final_path); + + for (;;) { + layer = dkr_pull_current_layer(i); + if (!layer) + return 0; /* no more layers */ + + path = strjoin(i->image_root, "/.dkr-", layer, NULL); + if (!path) + return log_oom(); + + if (laccess(path, F_OK) < 0) { + if (errno == ENOENT) + break; + + return log_error_errno(errno, "Failed to check for container: %m"); + } + + log_info("Layer %s already exists, skipping.", layer); + + i->current_ancestry++; + + free(path); + path = NULL; + } + + log_info("Pulling layer %s...", layer); + + i->final_path = path; + path = NULL; + + url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v2/", i->name, "/blobs/", layer); + r = pull_job_new(&i->layer_job, url, i->glue, i); + if (r < 0) + return log_error_errno(r, "Failed to allocate layer job: %m"); + + r = dkr_pull_add_bearer_token(i, i->layer_job); + if (r < 0) + return log_oom(); + + i->layer_job->on_finished = dkr_pull_job_on_finished_v2; + i->layer_job->on_open_disk = dkr_pull_job_on_open_disk; + i->layer_job->on_progress = dkr_pull_job_on_progress; + i->layer_job->grow_machine_directory = i->grow_machine_directory; + + r = pull_job_begin(i->layer_job); + if (r < 0) + return log_error_errno(r, "Failed to start layer job: %m"); + + return 0; +} + static int dkr_pull_pull_layer(DkrPull *i) { _cleanup_free_ char *path = NULL; const char *url, *layer = NULL; @@ -577,6 +682,343 @@ static int dkr_pull_pull_layer(DkrPull *i) { return 0; } +static int dkr_pull_job_on_header(PullJob *j, const char *header, size_t sz) { + _cleanup_free_ char *registry = NULL; + char *token, *digest; + DkrPull *i; + int r; + + assert(j); + assert(j->userdata); + + i = j->userdata; + r = curl_header_strdup(header, sz, HEADER_TOKEN, &token); + if (r < 0) + return log_oom(); + if (r > 0) { + free(i->response_token); + i->response_token = token; + return 0; + } + + r = curl_header_strdup(header, sz, HEADER_DIGEST, &digest); + if (r < 0) + return log_oom(); + if (r > 0) { + free(i->response_digest); + i->response_digest = digest; + return 0; + } + + r = curl_header_strdup(header, sz, HEADER_REGISTRY, ®istry); + if (r < 0) + return log_oom(); + if (r > 0) { + char **l, **k; + + l = strv_split(registry, ","); + if (!l) + return log_oom(); + + STRV_FOREACH(k, l) { + if (!hostname_is_valid(*k)) { + log_error("Registry hostname is not valid."); + strv_free(l); + return -EBADMSG; + } + } + + strv_free(i->response_registries); + i->response_registries = l; + } + + return 0; +} + +static void dkr_pull_job_on_finished_v2(PullJob *j) { + DkrPull *i; + int r; + + assert(j); + assert(j->userdata); + + i = j->userdata; + if (j->error != 0) { + if (j == i->images_job) + log_error_errno(j->error, "Failed to retrieve images list. (Wrong index URL?)"); + else if (j == i->ancestry_job) + log_error_errno(j->error, "Failed to retrieve manifest."); + else if (j == i->json_job) + log_error_errno(j->error, "Failed to retrieve json data."); + else + log_error_errno(j->error, "Failed to retrieve layer data."); + + r = j->error; + goto finish; + } + + if (i->images_job == j) { + const char *url; + + assert(!i->tags_job); + assert(!i->ancestry_job); + assert(!i->json_job); + assert(!i->layer_job); + + if (strv_isempty(i->response_registries)) { + r = -EBADMSG; + log_error("Didn't get registry information."); + goto finish; + } + + log_info("Index lookup succeeded, directed to registry %s.", i->response_registries[0]); + dkr_pull_report_progress(i, DKR_RESOLVING); + + url = strjoina(i->index_protocol, "auth.", i->index_address, "/v2/token/?scope=repository:", + i->name, ":pull&service=registry.", i->index_address); + r = pull_job_new(&i->tags_job, url, i->glue, i); + if (r < 0) { + log_error_errno(r, "Failed to allocate tags job: %m"); + goto finish; + } + + i->tags_job->on_finished = dkr_pull_job_on_finished_v2; + i->tags_job->on_progress = dkr_pull_job_on_progress; + + r = pull_job_begin(i->tags_job); + if (r < 0) { + log_error_errno(r, "Failed to start tags job: %m"); + goto finish; + } + + } else if (i->tags_job == j) { + const char *url; + _cleanup_free_ const char *buf; + _cleanup_jsonunref_ JsonVariant *doc = NULL; + JsonVariant *e = NULL; + + assert(!i->ancestry_job); + assert(!i->json_job); + assert(!i->layer_job); + + buf = strndup((const char *)j->payload, j->payload_size); + if (!buf) { + r = -ENOMEM; + log_oom(); + goto finish; + } + + r = json_parse(buf, &doc); + if (r < 0) { + log_error("Unable to parse bearer token\n%s", j->payload); + goto finish; + } + + e = json_variant_value(doc, "token"); + if (!e || e->type != JSON_VARIANT_STRING) { + r = -EBADMSG; + log_error("Invalid JSON format for Bearer token"); + goto finish; + } + + r = free_and_strdup(&i->response_token, json_variant_string(e)); + if (r < 0) { + log_oom(); + goto finish; + } + + url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v2/", i->name, "/manifests/", i->reference); + r = pull_job_new(&i->ancestry_job, url, i->glue, i); + if (r < 0) { + log_error_errno(r, "Failed to allocate ancestry job: %m"); + goto finish; + } + + r = dkr_pull_add_bearer_token(i, i->ancestry_job); + if (r < 0) + goto finish; + + i->ancestry_job->on_finished = dkr_pull_job_on_finished_v2; + i->ancestry_job->on_progress = dkr_pull_job_on_progress; + i->ancestry_job->on_header = dkr_pull_job_on_header; + + + r = pull_job_begin(i->ancestry_job); + if (r < 0) { + log_error_errno(r, "Failed to start ancestry job: %m"); + goto finish; + } + + } else if (i->ancestry_job == j) { + + _cleanup_jsonunref_ JsonVariant *doc = NULL; + _cleanup_jsonunref_ JsonVariant *compat = NULL; + JsonVariant *e = NULL; + _cleanup_strv_free_ char **ancestry = NULL; + size_t allocated = 0, size = 0; + char *path = NULL, **k = NULL; + + r = json_parse((const char *)j->payload, &doc); + if (r < 0) { + log_error("Invalid JSON Manifest"); + goto finish; + } + + e = json_variant_value(doc, "fsLayers"); + if (!e || e->type != JSON_VARIANT_ARRAY) { + r = -EBADMSG; + goto finish; + } + + log_info("JSON manifest with schema v%"PRIi64" for %s parsed!", + json_variant_integer(json_variant_value(doc, "schemaVersion")), + json_variant_string(json_variant_value(doc, "name"))); + + for (unsigned z = 0; z < e->size; z++) { + JsonVariant *f = json_variant_element(e, z), *g = NULL; + const char *layer; + if (f->type != JSON_VARIANT_OBJECT) { + r = -EBADMSG; + goto finish; + } + + g = json_variant_value(f, "blobSum"); + + layer = json_variant_string(g); + if (!dkr_digest_is_valid(layer)) { + r = -EBADMSG; + goto finish; + } + + if (!GREEDY_REALLOC(ancestry, allocated, size + 2)) { + r = -ENOMEM; + log_oom(); + goto finish; + } + + ancestry[size] = strdup(layer); + if (!ancestry[size]) { + r = -ENOMEM; + log_oom(); + goto finish; + } + + ancestry[size+1] = NULL; + size += 1; + } + + e = json_variant_value(doc, "history"); + if (!e || e->type != JSON_VARIANT_ARRAY) { + r = -EBADMSG; + goto finish; + } + + e = json_variant_element(e, 0); + e = json_variant_value(e, "v1Compatibility"); + r = json_parse(json_variant_string(e), &compat); + if (r < 0) { + log_error("Invalid v1Compatibility JSON"); + goto finish; + } + + e = json_variant_value(compat, "id"); + + strv_free(i->ancestry); + i->ancestry = strv_reverse(strv_uniq(ancestry)); + i->n_ancestry = strv_length(i->ancestry); + i->current_ancestry = 0; + i->id = strdup(i->ancestry[i->n_ancestry - 1]); + if (!i->id) { + r = -ENOMEM; + log_oom(); + goto finish; + } + path = strjoin(i->image_root, "/.dkr-", json_variant_string(e), NULL); + if (!path) { + r = -ENOMEM; + log_oom(); + goto finish; + } + free(i->image_root); + i->image_root = path; + ancestry = NULL; + + log_info("Required layers:\n"); + STRV_FOREACH(k, i->ancestry) + log_info("\t%s", *k); + log_info("\nProvenance:\n\tImageID: %s\n\tDigest: %s", json_variant_string(e), i->response_digest); + + dkr_pull_report_progress(i, DKR_DOWNLOADING); + + r = dkr_pull_pull_layer_v2(i); + if (r < 0) + goto finish; + + } else if (i->layer_job == j) { + assert(i->temp_path); + assert(i->final_path); + + j->disk_fd = safe_close(j->disk_fd); + + if (i->tar_pid > 0) { + r = wait_for_terminate_and_warn("tar", i->tar_pid, true); + i->tar_pid = 0; + if (r < 0) + goto finish; + } + + r = aufs_resolve(i->temp_path); + if (r < 0) { + log_error_errno(r, "Failed to resolve aufs whiteouts: %m"); + goto finish; + } + + r = btrfs_subvol_set_read_only(i->temp_path, true); + if (r < 0) { + log_error_errno(r, "Failed to mark snapshot read-only: %m"); + goto finish; + } + + if (rename(i->temp_path, i->final_path) < 0) { + log_error_errno(errno, "Failed to rename snaphsot: %m"); + goto finish; + } + + log_info("Completed writing to layer %s.", i->final_path); + + i->layer_job = pull_job_unref(i->layer_job); + free(i->temp_path); + i->temp_path = NULL; + free(i->final_path); + i->final_path = NULL; + + i->current_ancestry ++; + r = dkr_pull_pull_layer_v2(i); + if (r < 0) + goto finish; + + } else if (i->json_job != j) + assert_not_reached("Got finished event for unknown curl object"); + + if (!dkr_pull_is_done(i)) + return; + + dkr_pull_report_progress(i, DKR_COPYING); + + r = dkr_pull_make_local_copy(i, DKR_PULL_V2); + if (r < 0) + goto finish; + + r = 0; + +finish: + if (i->on_finished) + i->on_finished(i, r, i->userdata); + else + sd_event_exit(i->event, r); + +} + static void dkr_pull_job_on_finished(PullJob *j) { DkrPull *i; int r; @@ -618,7 +1060,7 @@ static void dkr_pull_job_on_finished(PullJob *j) { log_info("Index lookup succeeded, directed to registry %s.", i->response_registries[0]); dkr_pull_report_progress(i, DKR_RESOLVING); - url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/repositories/", i->name, "/tags/", i->tag); + url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/repositories/", i->name, "/tags/", i->reference); r = pull_job_new(&i->tags_job, url, i->glue, i); if (r < 0) { log_error_errno(r, "Failed to allocate tags job: %m"); @@ -790,12 +1232,11 @@ static void dkr_pull_job_on_finished(PullJob *j) { dkr_pull_report_progress(i, DKR_COPYING); - r = dkr_pull_make_local_copy(i); + r = dkr_pull_make_local_copy(i, DKR_PULL_V1); if (r < 0) goto finish; r = 0; - finish: if (i->on_finished) i->on_finished(i, r, i->userdata); @@ -803,52 +1244,37 @@ finish: sd_event_exit(i->event, r); } -static int dkr_pull_job_on_header(PullJob *j, const char *header, size_t sz) { - _cleanup_free_ char *registry = NULL; - char *token; - DkrPull *i; - int r; - - assert(j); - assert(j->userdata); +static int get_protocol_address(char **protocol, char **address, const char *url) { + const char *sep, *dot; + char *a, *p; - i = j->userdata; + sep = strstr(url, "://"); + if (!sep) + return -EINVAL; - r = curl_header_strdup(header, sz, HEADER_TOKEN, &token); - if (r < 0) - return log_oom(); - if (r > 0) { - free(i->response_token); - i->response_token = token; - return 0; - } + dot = strrchr(url, '.'); + if (!dot) + return -EINVAL; + dot--; - r = curl_header_strdup(header, sz, HEADER_REGISTRY, ®istry); - if (r < 0) + p = strndup(url, (sep - url) + 3); + if (!p) return log_oom(); - if (r > 0) { - char **l, **k; - l = strv_split(registry, ","); - if (!l) - return log_oom(); + while (dot > (sep + 3) && *dot != '.') + dot--; - STRV_FOREACH(k, l) { - if (!hostname_is_valid(*k)) { - log_error("Registry hostname is not valid."); - strv_free(l); - return -EBADMSG; - } - } + a = strdup(dot + 1); + if (!a) + return log_oom(); - strv_free(i->response_registries); - i->response_registries = l; - } + *address = a; + *protocol = p; return 0; } -int dkr_pull_start(DkrPull *i, const char *name, const char *tag, const char *local, bool force_local) { +int dkr_pull_start(DkrPull *i, const char *name, const char *reference, const char *local, bool force_local, DkrPullVersion version) { const char *url; int r; @@ -857,7 +1283,7 @@ int dkr_pull_start(DkrPull *i, const char *name, const char *tag, const char *lo if (!dkr_name_is_valid(name)) return -EINVAL; - if (tag && !dkr_tag_is_valid(tag)) + if (reference && !dkr_ref_is_valid(reference)) return -EINVAL; if (local && !machine_name_is_valid(local)) @@ -866,8 +1292,14 @@ int dkr_pull_start(DkrPull *i, const char *name, const char *tag, const char *lo if (i->images_job) return -EBUSY; - if (!tag) - tag = "latest"; + if (!reference) + reference = "latest"; + + free(i->index_protocol); + free(i->index_address); + r = get_protocol_address(&i->index_protocol, &i->index_address, i->index_url); + if (r < 0) + return r; r = free_and_strdup(&i->local, local); if (r < 0) @@ -877,7 +1309,7 @@ int dkr_pull_start(DkrPull *i, const char *name, const char *tag, const char *lo r = free_and_strdup(&i->name, name); if (r < 0) return r; - r = free_and_strdup(&i->tag, tag); + r = free_and_strdup(&i->reference, reference); if (r < 0) return r; @@ -891,7 +1323,11 @@ int dkr_pull_start(DkrPull *i, const char *name, const char *tag, const char *lo if (r < 0) return r; - i->images_job->on_finished = dkr_pull_job_on_finished; + if (version == DKR_PULL_V1) + i->images_job->on_finished = dkr_pull_job_on_finished; + else + i->images_job->on_finished = dkr_pull_job_on_finished_v2; + i->images_job->on_header = dkr_pull_job_on_header; i->images_job->on_progress = dkr_pull_job_on_progress; diff --git a/src/import/pull-dkr.h b/src/import/pull-dkr.h index 4c4b10c7ac..33d18cb394 100644 --- a/src/import/pull-dkr.h +++ b/src/import/pull-dkr.h @@ -24,6 +24,7 @@ #include "sd-event.h" #include "util.h" +typedef enum { DKR_PULL_V1, DKR_PULL_V2 } DkrPullVersion; typedef struct DkrPull DkrPull; typedef void (*DkrPullFinished)(DkrPull *pull, int error, void *userdata); @@ -33,4 +34,4 @@ DkrPull* dkr_pull_unref(DkrPull *pull); DEFINE_TRIVIAL_CLEANUP_FUNC(DkrPull*, dkr_pull_unref); -int dkr_pull_start(DkrPull *pull, const char *name, const char *tag, const char *local, bool force_local); +int dkr_pull_start(DkrPull *pull, const char *name, const char *tag, const char *local, bool force_local, DkrPullVersion version); -- cgit v1.2.3-54-g00ecf