import: add a simple scheme for validating the SHA256 sums of downloaded raw files

7 files changed, 270 insertions, 61 deletions

diff --git a/Makefile.am b/Makefile.am
index f165042cb0..b6a4e3e4e5 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -5238,6 +5238,9 @@ lib_LTLIBRARIES += \
 	libnss_mymachines.la
 
 if HAVE_LIBCURL
+if HAVE_XZ
+if HAVE_ZLIB
+if HAVE_GCRYPT
 
 bin_PROGRAMS += \
 	systemd-import
@@ -5265,7 +5268,8 @@ systemd_import_CFLAGS = \
 	$(AM_CFLAGS) \
 	$(LIBCURL_CFLAGS) \
 	$(XZ_CFLAGS) \
-	$(ZLIB_CFLAGS)
+	$(ZLIB_CFLAGS) \
+	$(GCRYPT_CFLAGS)
 
 systemd_import_LDADD = \
 	libsystemd-internal.la \
@@ -5273,11 +5277,9 @@ systemd_import_LDADD = \
 	libsystemd-shared.la \
 	$(LIBCURL_LIBS) \
 	$(XZ_LIBS) \
-	$(ZLIB_LIBS)
-
-endif
+	$(ZLIB_LIBS) \
+	$(GCRYPT_LIBS)
 
-if HAVE_ZLIB
 manual_tests += \
 	test-qcow2
 
@@ -5296,6 +5298,9 @@ test_qcow2_LDADD = \
 	libsystemd-shared.la \
 	$(ZLIB_LIBS)
 endif
+endif
+endif
+endif
 
 endif
 
diff --git a/src/import/import-job.c b/src/import/import-job.c
index 37f8ef76e4..6de32686c5 100644
--- a/src/import/import-job.c
+++ b/src/import/import-job.c
@@ -38,10 +38,14 @@ ImportJob* import_job_unref(ImportJob *j) {
         else if (j->compressed == IMPORT_JOB_GZIP)
                 inflateEnd(&j->gzip);
 
+        if (j->hash_context)
+                gcry_md_close(j->hash_context);
+
         free(j->url);
         free(j->etag);
         strv_free(j->old_etags);
         free(j->payload);
+        free(j->sha256);
 
         free(j);
 
@@ -94,6 +98,7 @@ void import_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
                 goto finish;
         } else if (status == 304) {
                 log_info("Image already downloaded. Skipping download.");
+                j->etag_exists = true;
                 r = 0;
                 goto finish;
         } else if (status >= 300) {
@@ -119,6 +124,25 @@ void import_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
                 goto finish;
         }
 
+        if (j->hash_context) {
+                uint8_t *k;
+
+                k = gcry_md_read(j->hash_context, GCRY_MD_SHA256);
+                if (!k) {
+                        log_error("Failed to get checksum.");
+                        r = -EIO;
+                        goto finish;
+                }
+
+                j->sha256 = hexmem(k, gcry_md_get_algo_dlen(GCRY_MD_SHA256));
+                if (!j->sha256) {
+                        r = log_oom();
+                        goto finish;
+                }
+
+                log_debug("SHA256 of %s is %s.", j->url, j->sha256);
+        }
+
         if (j->disk_fd >= 0 && j->allow_sparse) {
                 /* Make sure the file size is right, in case the file was
                  * sparse and we just seeked for the last part */
@@ -151,14 +175,12 @@ finish:
         import_job_finish(j, r);
 }
 
-
 static int import_job_write_uncompressed(ImportJob *j, void *p, size_t sz) {
         ssize_t n;
 
         assert(j);
         assert(p);
         assert(sz > 0);
-        assert(j->disk_fd >= 0);
 
         if (j->written_uncompressed + sz < j->written_uncompressed) {
                 log_error("File too large, overflow");
@@ -204,7 +226,6 @@ static int import_job_write_compressed(ImportJob *j, void *p, size_t sz) {
         assert(j);
         assert(p);
         assert(sz > 0);
-        assert(j->disk_fd >= 0);
 
         if (j->written_compressed + sz < j->written_compressed) {
                 log_error("File too large, overflow");
@@ -222,6 +243,9 @@ static int import_job_write_compressed(ImportJob *j, void *p, size_t sz) {
                 return -EFBIG;
         }
 
+        if (j->hash_context)
+                gcry_md_write(j->hash_context, p, sz);
+
         switch (j->compressed) {
 
         case IMPORT_JOB_UNCOMPRESSED:
@@ -311,6 +335,13 @@ static int import_job_open_disk(ImportJob *j) {
                 }
         }
 
+        if (j->calc_hash) {
+                if (gcry_md_open(&j->hash_context, GCRY_MD_SHA256, 0) != 0) {
+                        log_error("Failed to initialize hash context.");
+                        return -EIO;
+                }
+        }
+
         return 0;
 }
 
@@ -459,6 +490,7 @@ static size_t import_job_header_callback(void *contents, size_t size, size_t nme
 
                 if (strv_contains(j->old_etags, j->etag)) {
                         log_info("Image already downloaded. Skipping download.");
+                        j->etag_exists = true;
                         import_job_finish(j, 0);
                         return sz;
                 }
diff --git a/src/import/import-job.h b/src/import/import-job.h
index 843daa217c..9ccaaf2c04 100644
--- a/src/import/import-job.h
+++ b/src/import/import-job.h
@@ -23,6 +23,7 @@
 
 #include <lzma.h>
 #include <zlib.h>
+#include <gcrypt.h>
 
 #include "macro.h"
 #include "curl-util.h"
@@ -42,6 +43,8 @@ typedef enum ImportJobState {
         _IMPORT_JOB_STATE_INVALID = -1,
 } ImportJobState;
 
+#define IMPORT_JOB_STATE_IS_COMPLETE(j) (IN_SET((j)->state, IMPORT_JOB_DONE, IMPORT_JOB_FAILED))
+
 typedef enum ImportJobCompression {
         IMPORT_JOB_UNCOMPRESSED,
         IMPORT_JOB_XZ,
@@ -66,6 +69,7 @@ struct ImportJob {
 
         char *etag;
         char **old_etags;
+        bool etag_exists;
 
         uint64_t content_length;
         uint64_t written_compressed;
@@ -91,6 +95,11 @@ struct ImportJob {
         usec_t last_status_usec;
 
         bool allow_sparse;
+
+        bool calc_hash;
+        gcry_md_hd_t hash_context;
+
+        char *sha256;
 };
 
 int import_job_new(ImportJob **job, const char *url, CurlGlue *glue, void *userdata);
diff --git a/src/import/import-raw.c b/src/import/import-raw.c
index 635779c1b5..8ca10919af 100644
--- a/src/import/import-raw.c
+++ b/src/import/import-raw.c
@@ -22,6 +22,7 @@
 #include <sys/xattr.h>
 #include <linux/fs.h>
 #include <curl/curl.h>
+#include <gcrypt.h>
 
 #include "utf8.h"
 #include "strv.h"
@@ -45,6 +46,7 @@ struct RawImport {
         char *image_root;
 
         ImportJob *raw_job;
+        ImportJob *sha256sums_job;
 
         RawImportFinished on_finished;
         void *userdata;
@@ -176,10 +178,11 @@ static int raw_import_make_local_copy(RawImport *i) {
         if (!i->local)
                 return 0;
 
-        if (i->raw_job->disk_fd >= 0) {
-                if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
-                        return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
-        } else {
+        if (i->raw_job->etag_exists) {
+                /* We have downloaded this one previously, reopen it */
+
+                assert(i->raw_job->disk_fd < 0);
+
                 if (!i->final_path) {
                         r = import_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", ".raw", &i->final_path);
                         if (r < 0)
@@ -189,6 +192,13 @@ static int raw_import_make_local_copy(RawImport *i) {
                 i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
                 if (i->raw_job->disk_fd < 0)
                         return log_error_errno(errno, "Failed to open vendor image: %m");
+        } else {
+                /* We freshly downloaded the image, use it */
+
+                assert(i->raw_job->disk_fd >= 0);
+
+                if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
+                        return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
         }
 
         p = strappenda(i->image_root, "/", i->local, ".raw");
@@ -235,7 +245,91 @@ static int raw_import_make_local_copy(RawImport *i) {
         return 0;
 }
 
-static void raw_import_job_on_finished(ImportJob *j) {
+static int raw_import_verify_sha256sum(RawImport *i) {
+        _cleanup_free_ char *fn = NULL;
+        const char *p, *line;
+        int r;
+
+        assert(i);
+
+        assert(i->raw_job);
+        assert(i->raw_job->sha256);
+
+        assert(i->sha256sums_job);
+        assert(i->sha256sums_job->payload);
+        assert(i->sha256sums_job->payload_size > 0);
+
+        r = import_url_last_component(i->raw_job->url, &fn);
+        if (r < 0)
+                return log_oom();
+
+        if (!filename_is_valid(fn)) {
+                log_error("Cannot verify checksum, could not determine valid server-side file name.");
+                return -EBADMSG;
+        }
+
+        line = strappenda(i->raw_job->sha256, " *", fn, "\n");
+
+        p = memmem(i->sha256sums_job->payload,
+                   i->sha256sums_job->payload_size,
+                   line,
+                   strlen(line));
+
+        if (!p || (p != (char*) i->sha256sums_job->payload && p[-1] != '\n')) {
+                log_error("Checksum did not check out, payload has been tempered with.");
+                return -EBADMSG;
+        }
+
+        log_info("SHA256 checksum of %s is valid.", i->raw_job->url);
+
+        return 0;
+}
+
+static int raw_import_finalize(RawImport *i) {
+        int r;
+
+        assert(i);
+
+        if (!IMPORT_JOB_STATE_IS_COMPLETE(i->raw_job) ||
+            !IMPORT_JOB_STATE_IS_COMPLETE(i->sha256sums_job))
+                return 0;
+
+        if (!i->raw_job->etag_exists) {
+                assert(i->temp_path);
+                assert(i->final_path);
+                assert(i->raw_job->disk_fd >= 0);
+
+                r = raw_import_verify_sha256sum(i);
+                if (r < 0)
+                        return r;
+
+                r = rename(i->temp_path, i->final_path);
+                if (r < 0)
+                        return log_error_errno(errno, "Failed to move RAW file into place: %m");
+
+                free(i->temp_path);
+                i->temp_path = NULL;
+        }
+
+        r = raw_import_make_local_copy(i);
+        if (r < 0)
+                return r;
+
+        i->raw_job->disk_fd = safe_close(i->raw_job->disk_fd);
+
+        return 1;
+}
+
+static void raw_import_invoke_finished(RawImport *i, int r) {
+        assert(i);
+
+        if (i->on_finished)
+                i->on_finished(i, r, i->userdata);
+        else
+                sd_event_exit(i->event, r);
+}
+
+static void raw_import_raw_job_on_finished(ImportJob *j) {
         RawImport *i;
         int r;
 
@@ -250,9 +344,12 @@ static void raw_import_job_on_finished(ImportJob *j) {
 
         /* This is invoked if either the download completed
          * successfully, or the download was skipped because we
-         * already have the etag. */
+         * already have the etag. In this case ->etag_exists is
+         * true. */
+
+        if (!j->etag_exists) {
+                assert(j->disk_fd >= 0);
 
-        if (j->disk_fd >= 0) {
                 r = raw_import_maybe_convert_qcow2(i);
                 if (r < 0)
                         goto finish;
@@ -260,33 +357,45 @@ static void raw_import_job_on_finished(ImportJob *j) {
                 r = import_make_read_only_fd(j->disk_fd);
                 if (r < 0)
                         goto finish;
+        }
 
-                r = rename(i->temp_path, i->final_path);
-                if (r < 0) {
-                        r = log_error_errno(errno, "Failed to move RAW file into place: %m");
-                        goto finish;
-                }
+        r = raw_import_finalize(i);
+        if (r < 0)
+                goto finish;
+        if (r == 0)
+                return;
 
-                free(i->temp_path);
-                i->temp_path = NULL;
+        r = 0;
+
+finish:
+        raw_import_invoke_finished(i, r);
+}
+
+static void raw_import_sha256sums_job_on_finished(ImportJob *j) {
+        RawImport *i;
+        int r;
+
+        assert(j);
+        assert(j->userdata);
+
+        i = j->userdata;
+        if (j->error != 0) {
+                r = j->error;
+                goto finish;
         }
 
-        r = raw_import_make_local_copy(i);
+        r = raw_import_finalize(i);
         if (r < 0)
                 goto finish;
-
-        j->disk_fd = safe_close(j->disk_fd);
+        if (r == 0)
+                return;
 
         r = 0;
-
 finish:
-        if (i->on_finished)
-                i->on_finished(i, r, i->userdata);
-        else
-                sd_event_exit(i->event, r);
+        raw_import_invoke_finished(i, r);
 }
 
-static int raw_import_job_on_open_disk(ImportJob *j) {
+static int raw_import_raw_job_on_open_disk(ImportJob *j) {
         RawImport *i;
         int r;
 
@@ -317,6 +426,7 @@ static int raw_import_job_on_open_disk(ImportJob *j) {
 }
 
 int raw_import_pull(RawImport *i, const char *url, const char *local, bool force_local) {
+        _cleanup_free_ char *sha256sums_url = NULL;
         int r;
 
         assert(i);
@@ -333,19 +443,40 @@ int raw_import_pull(RawImport *i, const char *url, const char *local, bool force
         r = free_and_strdup(&i->local, local);
         if (r < 0)
                 return r;
-
         i->force_local = force_local;
 
+        /* Queue job for the image itself */
         r = import_job_new(&i->raw_job, url, i->glue, i);
         if (r < 0)
                 return r;
 
-        i->raw_job->on_finished = raw_import_job_on_finished;
-        i->raw_job->on_open_disk = raw_import_job_on_open_disk;
+        i->raw_job->on_finished = raw_import_raw_job_on_finished;
+        i->raw_job->on_open_disk = raw_import_raw_job_on_open_disk;
+        i->raw_job->calc_hash = true;
 
         r = import_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
         if (r < 0)
                 return r;
 
-        return import_job_begin(i->raw_job);
+        /* Queue job for the SHA256SUMS file for the image */
+        r = import_url_change_last_component(url, "SHA256SUMS", &sha256sums_url);
+        if (r < 0)
+                return r;
+
+        r = import_job_new(&i->sha256sums_job, sha256sums_url, i->glue, i);
+        if (r < 0)
+                return r;
+
+        i->sha256sums_job->on_finished = raw_import_sha256sums_job_on_finished;
+        i->sha256sums_job->uncompressed_max = i->sha256sums_job->compressed_max = 1ULL * 1024ULL * 1024ULL;
+
+        r = import_job_begin(i->raw_job);
+        if (r < 0)
+                return r;
+
+        r = import_job_begin(i->sha256sums_job);
+        if (r < 0)
+                return r;
+
+        return 0;
 }
diff --git a/src/import/import-util.c b/src/import/import-util.c
index 341a566e37..1212025d43 100644
--- a/src/import/import-util.c
+++ b/src/import/import-util.c
@@ -218,3 +218,55 @@ int import_make_path(const char *url, const char *etag, const char *image_root,
         *ret = path;
         return 0;
 }
+
+int import_url_last_component(const char *url, char **ret) {
+        const char *e, *p;
+        char *s;
+
+        e = strchrnul(url, '?');
+
+        while (e > url && e[-1] == '/')
+                e--;
+
+        p = e;
+        while (p > url && p[-1] != '/')
+                p--;
+
+        if (e <= p)
+                return -EINVAL;
+
+        s = strndup(p, e - p);
+        if (!s)
+                return -ENOMEM;
+
+        *ret = s;
+        return 0;
+}
+
+
+int import_url_change_last_component(const char *url, const char *suffix, char **ret) {
+        const char *e;
+        char *s;
+
+        assert(url);
+        assert(ret);
+
+        e = strchrnul(url, '?');
+
+        while (e > url && e[-1] == '/')
+                e--;
+
+        while (e > url && e[-1] != '/')
+                e--;
+
+        if (e <= url)
+                return -EINVAL;
+
+        s = new(char, (e - url) + strlen(suffix) + 1);
+        if (!s)
+                return -ENOMEM;
+
+        strcpy(mempcpy(s, url, e - url), suffix);
+        *ret = s;
+        return 0;
+}
diff --git a/src/import/import-util.h b/src/import/import-util.h
index a930ea48a0..a8a5ca5699 100644
--- a/src/import/import-util.h
+++ b/src/import/import-util.h
@@ -33,3 +33,6 @@ int import_make_read_only_fd(int fd);
 int import_make_read_only(const char *path);
 
 int import_make_path(const char *url, const char *etag, const char *image_root, const char *prefix, const char *suffix, char **ret);
+
+int import_url_last_component(const char *url, char **ret);
+int import_url_change_last_component(const char *url, const char *suffix, char **ret);
diff --git a/src/import/import.c b/src/import/import.c
index 861a448583..3362f4a9ef 100644
--- a/src/import/import.c
+++ b/src/import/import.c
@@ -29,6 +29,7 @@
 #include "import-tar.h"
 #include "import-raw.h"
 #include "import-dkr.h"
+#include "import-util.h"
 
 static bool arg_force = false;
 static const char *arg_image_root = "/var/lib/machines";
@@ -47,30 +48,6 @@ static void on_tar_finished(TarImport *import, int error, void *userdata) {
         sd_event_exit(event, error);
 }
 
-static int url_final_component(const char *url, char **ret) {
-        const char *e, *p;
-        char *s;
-
-        e = strchrnul(url, '?');
-
-        while (e > url && e[-1] == '/')
-                        e--;
-
-        p = e;
-        while (p > url && p[-1] != '/')
-                p--;
-
-        if (e <= p)
-                return -EINVAL;
-
-        s = strndup(p, e - p);
-        if (!s)
-                return -ENOMEM;
-
-        *ret = s;
-        return 0;
-}
-
 static int strip_tar_suffixes(const char *name, char **ret) {
         const char *e;
         char *s;
@@ -112,7 +89,7 @@ static int pull_tar(int argc, char *argv[], void *userdata) {
         if (argc >= 3)
                 local = argv[2];
         else {
-                r = url_final_component(url, &l);
+                r = import_url_last_component(url, &l);
                 if (r < 0)
                         return log_error_errno(r, "Failed get final component of URL: %m");
 
@@ -238,7 +215,7 @@ static int pull_raw(int argc, char *argv[], void *userdata) {
         if (argc >= 3)
                 local = argv[2];
         else {
-                r = url_final_component(url, &l);
+                r = import_url_last_component(url, &l);
                 if (r < 0)
                         return log_error_errno(r, "Failed get final component of URL: %m");