import: port pull-raw to helper tools implemented for pull-tar

This allows us to reuse a lot more code, and simplify pull-raw
drastically.

8 files changed, 272 insertions, 818 deletions

diff --git a/src/import/import-raw.c b/src/import/import-raw.c
index 1fe27b6932..635779c1b5 100644
--- a/src/import/import-raw.c
+++ b/src/import/import-raw.c
@@ -22,242 +22,116 @@
 #include <sys/xattr.h>
 #include <linux/fs.h>
 #include <curl/curl.h>
-#include <lzma.h>
 
-#include "hashmap.h"
 #include "utf8.h"
-#include "curl-util.h"
-#include "qcow2-util.h"
 #include "strv.h"
 #include "copy.h"
+#include "btrfs-util.h"
+#include "util.h"
+#include "macro.h"
+#include "mkdir.h"
+#include "curl-util.h"
+#include "qcow2-util.h"
+#include "import-job.h"
 #include "import-util.h"
 #include "import-raw.h"
 
 typedef struct RawImportFile RawImportFile;
 
-struct RawImportFile {
-        RawImport *import;
-
-        char *url;
-        char *local;
-
-        CURL *curl;
-        struct curl_slist *request_header;
-
-        char *temp_path;
-        char *final_path;
-        char *etag;
-        char **old_etags;
-
-        uint64_t content_length;
-        uint64_t written_compressed;
-        uint64_t written_uncompressed;
-
-        void *payload;
-        size_t payload_size;
-
-        usec_t mtime;
-
-        bool force_local;
-        bool done;
-
-        int disk_fd;
-
-        lzma_stream lzma;
-        bool compressed;
-
-        unsigned progress_percent;
-        usec_t start_usec;
-        usec_t last_status_usec;
-};
-
 struct RawImport {
         sd_event *event;
         CurlGlue *glue;
 
         char *image_root;
-        Hashmap *files;
 
-        raw_import_on_finished on_finished;
-        void *userdata;
+        ImportJob *raw_job;
 
-        bool finished;
-};
+        RawImportFinished on_finished;
+        void *userdata;
 
-#define FILENAME_ESCAPE "/.#\"\'"
+        char *local;
+        bool force_local;
 
-#define RAW_MAX_SIZE (1024LLU*1024LLU*1024LLU*8) /* 8 GB */
+        char *temp_path;
+        char *final_path;
+};
 
-static RawImportFile *raw_import_file_unref(RawImportFile *f) {
-        if (!f)
+RawImport* raw_import_unref(RawImport *i) {
+        if (!i)
                 return NULL;
 
-        if (f->import)
-                curl_glue_remove_and_free(f->import->glue, f->curl);
-        curl_slist_free_all(f->request_header);
+        import_job_unref(i->raw_job);
 
-        safe_close(f->disk_fd);
+        curl_glue_unref(i->glue);
+        sd_event_unref(i->event);
 
-        free(f->final_path);
-
-        if (f->temp_path) {
-                unlink(f->temp_path);
-                free(f->temp_path);
+        if (i->temp_path) {
+                (void) unlink(i->temp_path);
+                free(i->temp_path);
         }
 
-        lzma_end(&f->lzma);
-        free(f->url);
-        free(f->local);
-        free(f->etag);
-        strv_free(f->old_etags);
-        free(f->payload);
-        free(f);
+        free(i->final_path);
+        free(i->image_root);
+        free(i->local);
+        free(i);
 
         return NULL;
 }
 
-DEFINE_TRIVIAL_CLEANUP_FUNC(RawImportFile*, raw_import_file_unref);
-
-static void raw_import_finish(RawImport *import, int error) {
-        assert(import);
-
-        if (import->finished)
-                return;
-
-        import->finished = true;
-
-        if (import->on_finished)
-                import->on_finished(import, error, import->userdata);
-        else
-                sd_event_exit(import->event, error);
-}
-
-static int raw_import_file_make_final_path(RawImportFile *f) {
-        _cleanup_free_ char *escaped_url = NULL, *escaped_etag = NULL;
-
-        assert(f);
+int raw_import_new(RawImport **ret, sd_event *event, const char *image_root, RawImportFinished on_finished, void *userdata) {
+        _cleanup_(raw_import_unrefp) RawImport *i = NULL;
+        int r;
 
-        if (f->final_path)
-                return 0;
+        assert(ret);
 
-        escaped_url = xescape(f->url, FILENAME_ESCAPE);
-        if (!escaped_url)
+        i = new0(RawImport, 1);
+        if (!i)
                 return -ENOMEM;
 
-        if (f->etag) {
-                escaped_etag = xescape(f->etag, FILENAME_ESCAPE);
-                if (!escaped_etag)
-                        return -ENOMEM;
+        i->on_finished = on_finished;
+        i->userdata = userdata;
 
-                f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".", escaped_etag, ".raw", NULL);
-        } else
-                f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".raw", NULL);
-        if (!f->final_path)
+        i->image_root = strdup(image_root ?: "/var/lib/machines");
+        if (!i->image_root)
                 return -ENOMEM;
 
-        return 0;
-}
-
-static int raw_import_file_make_local_copy(RawImportFile *f) {
-        _cleanup_free_ char *tp = NULL;
-        _cleanup_close_ int dfd = -1;
-        const char *p;
-        int r;
-
-        assert(f);
-
-        if (!f->local)
-                return 0;
-
-        if (f->disk_fd >= 0) {
-                if (lseek(f->disk_fd, SEEK_SET, 0) == (off_t) -1)
-                        return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
-        } else {
-                r = raw_import_file_make_final_path(f);
+        if (event)
+                i->event = sd_event_ref(event);
+        else {
+                r = sd_event_default(&i->event);
                 if (r < 0)
-                        return log_oom();
-
-                f->disk_fd = open(f->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
-                if (f->disk_fd < 0)
-                        return log_error_errno(errno, "Failed to open vendor image: %m");
+                        return r;
         }
 
-        p = strappenda(f->import->image_root, "/", f->local, ".raw");
-        if (f->force_local)
-                (void) rm_rf_dangerous(p, false, true, false);
-
-        r = tempfn_random(p, &tp);
-        if (r < 0)
-                return log_oom();
-
-        dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
-        if (dfd < 0)
-                return log_error_errno(errno, "Failed to create writable copy of image: %m");
-
-        /* Turn off COW writing. This should greatly improve
-         * performance on COW file systems like btrfs, since it
-         * reduces fragmentation caused by not allowing in-place
-         * writes. */
-        r = chattr_fd(dfd, true, FS_NOCOW_FL);
+        r = curl_glue_new(&i->glue, i->event);
         if (r < 0)
-                log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
-
-        r = copy_bytes(f->disk_fd, dfd, (off_t) -1, true);
-        if (r < 0) {
-                unlink(tp);
-                return log_error_errno(r, "Failed to make writable copy of image: %m");
-        }
-
-        (void) copy_times(f->disk_fd, dfd);
-        (void) copy_xattr(f->disk_fd, dfd);
+                return r;
 
-        dfd = safe_close(dfd);
+        i->glue->on_finished = import_job_curl_on_finished;
+        i->glue->userdata = i;
 
-        r = rename(tp, p);
-        if (r < 0)  {
-                unlink(tp);
-                return log_error_errno(errno, "Failed to move writable image into place: %m");
-        }
+        *ret = i;
+        i = NULL;
 
-        log_info("Created new local image %s.", p);
         return 0;
 }
 
-static void raw_import_file_success(RawImportFile *f) {
-        int r;
-
-        assert(f);
-
-        f->done = true;
-
-        r = raw_import_file_make_local_copy(f);
-        if (r < 0)
-                goto finish;
-
-        f->disk_fd = safe_close(f->disk_fd);
-        r = 0;
-
-finish:
-        raw_import_finish(f->import, r);
-}
-
-static int raw_import_maybe_convert_qcow2(RawImportFile *f) {
+static int raw_import_maybe_convert_qcow2(RawImport *i) {
         _cleanup_close_ int converted_fd = -1;
         _cleanup_free_ char *t = NULL;
         int r;
 
-        assert(f);
-        assert(f->disk_fd);
-        assert(f->temp_path);
+        assert(i);
+        assert(i->raw_job);
 
-        r = qcow2_detect(f->disk_fd);
+        r = qcow2_detect(i->raw_job->disk_fd);
         if (r < 0)
                 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
         if (r == 0)
                 return 0;
 
         /* This is a QCOW2 image, let's convert it */
-        r = tempfn_random(f->final_path, &t);
+        r = tempfn_random(i->final_path, &t);
         if (r < 0)
                 return log_oom();
 
@@ -265,677 +139,213 @@ static int raw_import_maybe_convert_qcow2(RawImportFile *f) {
         if (converted_fd < 0)
                 return log_error_errno(errno, "Failed to create %s: %m", t);
 
+        r = chattr_fd(converted_fd, true, FS_NOCOW_FL);
+        if (r < 0)
+                log_warning_errno(errno, "Failed to set file attributes on %s: %m", t);
+
         log_info("Unpacking QCOW2 file.");
 
-        r = qcow2_convert(f->disk_fd, converted_fd);
+        r = qcow2_convert(i->raw_job->disk_fd, converted_fd);
         if (r < 0) {
                 unlink(t);
                 return log_error_errno(r, "Failed to convert qcow2 image: %m");
         }
 
-        unlink(f->temp_path);
-        free(f->temp_path);
+        unlink(i->temp_path);
+        free(i->temp_path);
 
-        f->temp_path = t;
+        i->temp_path = t;
         t = NULL;
 
-        safe_close(f->disk_fd);
-        f->disk_fd = converted_fd;
+        safe_close(i->raw_job->disk_fd);
+        i->raw_job->disk_fd = converted_fd;
         converted_fd = -1;
 
         return 1;
 }
 
-static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
-        RawImportFile *f = NULL;
-        struct stat st;
-        CURLcode code;
-        long status;
-        int r;
-
-        if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, &f) != CURLE_OK)
-                return;
-
-        if (!f || f->done)
-                return;
-
-        f->done = true;
-
-        if (result != CURLE_OK) {
-                log_error("Transfer failed: %s", curl_easy_strerror(result));
-                r = -EIO;
-                goto fail;
-        }
-
-        code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
-        if (code != CURLE_OK) {
-                log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
-                r = -EIO;
-                goto fail;
-        } else if (status == 304) {
-                log_info("Image already downloaded. Skipping download.");
-                raw_import_file_success(f);
-                return;
-        } else if (status >= 300) {
-                log_error("HTTP request to %s failed with code %li.", f->url, status);
-                r = -EIO;
-                goto fail;
-        } else if (status < 200) {
-                log_error("HTTP request to %s finished with unexpected code %li.", f->url, status);
-                r = -EIO;
-                goto fail;
-        }
-
-        if (f->disk_fd < 0) {
-                log_error("No data received.");
-                r = -EIO;
-                goto fail;
-        }
-
-        if (f->content_length != (uint64_t) -1 &&
-            f->content_length != f->written_compressed) {
-                log_error("Download truncated.");
-                r = -EIO;
-                goto fail;
-        }
-
-        /* Make sure the file size is right, in case the file was
-         * sparse and we just seeked for the last part */
-        if (ftruncate(f->disk_fd, f->written_uncompressed) < 0) {
-                log_error_errno(errno, "Failed to truncate file: %m");
-                r = -errno;
-                goto fail;
-        }
-
-        r = raw_import_maybe_convert_qcow2(f);
-        if (r < 0)
-                goto fail;
-
-        if (f->etag)
-                (void) fsetxattr(f->disk_fd, "user.source_etag", f->etag, strlen(f->etag), 0);
-        if (f->url)
-                (void) fsetxattr(f->disk_fd, "user.source_url", f->url, strlen(f->url), 0);
-
-        if (f->mtime != 0) {
-                struct timespec ut[2];
-
-                timespec_store(&ut[0], f->mtime);
-                ut[1] = ut[0];
-                (void) futimens(f->disk_fd, ut);
-
-                fd_setcrtime(f->disk_fd, f->mtime);
-        }
-
-        if (fstat(f->disk_fd, &st) < 0) {
-                r = log_error_errno(errno, "Failed to stat file: %m");
-                goto fail;
-        }
-
-        /* Mark read-only */
-        (void) fchmod(f->disk_fd, st.st_mode & 07444);
-
-        assert(f->temp_path);
-        assert(f->final_path);
-
-        r = rename(f->temp_path, f->final_path);
-        if (r < 0) {
-                r = log_error_errno(errno, "Failed to move RAW file into place: %m");
-                goto fail;
-        }
-
-        free(f->temp_path);
-        f->temp_path = NULL;
-
-        log_info("Completed writing vendor image %s.", f->final_path);
-
-        raw_import_file_success(f);
-        return;
-
-fail:
-        raw_import_finish(f->import, r);
-}
-
-static int raw_import_file_open_disk_for_write(RawImportFile *f) {
+static int raw_import_make_local_copy(RawImport *i) {
+        _cleanup_free_ char *tp = NULL;
+        _cleanup_close_ int dfd = -1;
+        const char *p;
         int r;
 
-        assert(f);
+        assert(i);
+        assert(i->raw_job);
 
-        if (f->disk_fd >= 0)
+        if (!i->local)
                 return 0;
 
-        r = raw_import_file_make_final_path(f);
-        if (r < 0)
-                return log_oom();
-
-        if (!f->temp_path) {
-                r = tempfn_random(f->final_path, &f->temp_path);
-                if (r < 0)
-                        return log_oom();
-        }
-
-        f->disk_fd = open(f->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
-        if (f->disk_fd < 0)
-                return log_error_errno(errno, "Failed to create %s: %m", f->temp_path);
-
-        r = chattr_fd(f->disk_fd, true, FS_NOCOW_FL);
-        if (r < 0)
-                log_warning_errno(errno, "Failed to set file attributes on %s: %m", f->temp_path);
-
-        return 0;
-}
-
-static int raw_import_file_write_uncompressed(RawImportFile *f, void *p, size_t sz) {
-        ssize_t n;
-
-        assert(f);
-        assert(p);
-        assert(sz > 0);
-        assert(f->disk_fd >= 0);
-
-        if (f->written_uncompressed + sz < f->written_uncompressed) {
-                log_error("File too large, overflow");
-                return -EOVERFLOW;
-        }
-
-        if (f->written_uncompressed + sz > RAW_MAX_SIZE) {
-                log_error("File overly large, refusing");
-                return -EFBIG;
-        }
-
-        n = sparse_write(f->disk_fd, p, sz, 64);
-        if (n < 0) {
-                log_error_errno(errno, "Failed to write file: %m");
-                return -errno;
-        }
-        if ((size_t) n < sz) {
-                log_error("Short write");
-                return -EIO;
-        }
-
-        f->written_uncompressed += sz;
-
-        return 0;
-}
-
-static int raw_import_file_write_compressed(RawImportFile *f, void *p, size_t sz) {
-        int r;
-
-        assert(f);
-        assert(p);
-        assert(sz > 0);
-        assert(f->disk_fd >= 0);
-
-        if (f->written_compressed + sz < f->written_compressed) {
-                log_error("File too large, overflow");
-                return -EOVERFLOW;
-        }
-
-        if (f->content_length != (uint64_t) -1 &&
-            f->written_compressed + sz > f->content_length) {
-                log_error("Content length incorrect.");
-                return -EFBIG;
-        }
-
-        if (!f->compressed) {
-                r = raw_import_file_write_uncompressed(f, p, sz);
-                if (r < 0)
-                        return r;
+        if (i->raw_job->disk_fd >= 0) {
+                if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
+                        return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
         } else {
-                f->lzma.next_in = p;
-                f->lzma.avail_in = sz;
-
-                while (f->lzma.avail_in > 0) {
-                        uint8_t buffer[16 * 1024];
-                        lzma_ret lzr;
-
-                        f->lzma.next_out = buffer;
-                        f->lzma.avail_out = sizeof(buffer);
-
-                        lzr = lzma_code(&f->lzma, LZMA_RUN);
-                        if (lzr != LZMA_OK && lzr != LZMA_STREAM_END) {
-                                log_error("Decompression error.");
-                                return -EIO;
-                        }
-
-                        r = raw_import_file_write_uncompressed(f, buffer, sizeof(buffer) - f->lzma.avail_out);
+                if (!i->final_path) {
+                        r = import_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", ".raw", &i->final_path);
                         if (r < 0)
-                                return r;
+                                return log_oom();
                 }
-        }
-
-        f->written_compressed += sz;
-
-        return 0;
-}
-
-static int raw_import_file_detect_xz(RawImportFile *f) {
-        static const uint8_t xz_signature[] = {
-                '\xfd', '7', 'z', 'X', 'Z', '\x00'
-        };
-        lzma_ret lzr;
-        int r;
-
-        assert(f);
-
-        if (f->payload_size < sizeof(xz_signature))
-                return 0;
-
-        f->compressed = memcmp(f->payload, xz_signature, sizeof(xz_signature)) == 0;
-        log_debug("Stream is XZ compressed: %s", yes_no(f->compressed));
-
-        if (f->compressed) {
-                lzr = lzma_stream_decoder(&f->lzma, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK);
-                if (lzr != LZMA_OK) {
-                        log_error("Failed to initialize LZMA decoder.");
-                        return -EIO;
-                }
-        }
-
-        r = raw_import_file_open_disk_for_write(f);
-        if (r < 0)
-                return r;
-
-        r = raw_import_file_write_compressed(f, f->payload, f->payload_size);
-        if (r < 0)
-                return r;
-
-        free(f->payload);
-        f->payload = NULL;
-        f->payload_size = 0;
-
-        return 0;
-}
 
-static size_t raw_import_file_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
-        RawImportFile *f = userdata;
-        size_t sz = size * nmemb;
-        int r;
-
-        assert(contents);
-        assert(f);
-
-        if (f->done) {
-                r = -ESTALE;
-                goto fail;
+                i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
+                if (i->raw_job->disk_fd < 0)
+                        return log_error_errno(errno, "Failed to open vendor image: %m");
         }
 
-        if (f->disk_fd < 0) {
-                uint8_t *p;
-
-                /* We haven't opened the file yet, let's first check what it actually is */
+        p = strappenda(i->image_root, "/", i->local, ".raw");
 
-                p = realloc(f->payload, f->payload_size + sz);
-                if (!p) {
-                        r = log_oom();
-                        goto fail;
-                }
-
-                memcpy(p + f->payload_size, contents, sz);
-                f->payload_size = sz;
-                f->payload = p;
-
-                r = raw_import_file_detect_xz(f);
-                if (r < 0)
-                        goto fail;
-
-                return sz;
+        if (i->force_local) {
+                (void) btrfs_subvol_remove(p);
+                (void) rm_rf_dangerous(p, false, true, false);
         }
 
-        r = raw_import_file_write_compressed(f, contents, sz);
+        r = tempfn_random(p, &tp);
         if (r < 0)
-                goto fail;
-
-        return sz;
-
-fail:
-        raw_import_finish(f->import, r);
-        return 0;
-}
-
-static size_t raw_import_file_header_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
-        RawImportFile *f = userdata;
-        size_t sz = size * nmemb;
-        _cleanup_free_ char *length = NULL, *last_modified = NULL;
-        char *etag;
-        int r;
-
-        assert(contents);
-        assert(f);
-
-        if (f->done) {
-                r = -ESTALE;
-                goto fail;
-        }
-
-        r = curl_header_strdup(contents, sz, "ETag:", &etag);
-        if (r < 0) {
-                log_oom();
-                goto fail;
-        }
-        if (r > 0) {
-                free(f->etag);
-                f->etag = etag;
-
-                if (strv_contains(f->old_etags, f->etag)) {
-                        log_info("Image already downloaded. Skipping download.");
-                        raw_import_file_success(f);
-                        return sz;
-                }
-
-                return sz;
-        }
-
-        r = curl_header_strdup(contents, sz, "Content-Length:", &length);
-        if (r < 0) {
-                log_oom();
-                goto fail;
-        }
-        if (r > 0) {
-                (void) safe_atou64(length, &f->content_length);
+                return log_oom();
 
-                if (f->content_length != (uint64_t) -1) {
-                        char bytes[FORMAT_BYTES_MAX];
-                        log_info("Downloading %s.", format_bytes(bytes, sizeof(bytes), f->content_length));
-                }
+        dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
+        if (dfd < 0)
+                return log_error_errno(errno, "Failed to create writable copy of image: %m");
 
-                return sz;
-        }
+        /* Turn off COW writing. This should greatly improve
+         * performance on COW file systems like btrfs, since it
+         * reduces fragmentation caused by not allowing in-place
+         * writes. */
+        r = chattr_fd(dfd, true, FS_NOCOW_FL);
+        if (r < 0)
+                log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
 
-        r = curl_header_strdup(contents, sz, "Last-Modified:", &last_modified);
+        r = copy_bytes(i->raw_job->disk_fd, dfd, (off_t) -1, true);
         if (r < 0) {
-                log_oom();
-                goto fail;
-        }
-        if (r > 0) {
-                (void) curl_parse_http_time(last_modified, &f->mtime);
-                return sz;
+                unlink(tp);
+                return log_error_errno(r, "Failed to make writable copy of image: %m");
         }
 
-        return sz;
-
-fail:
-        raw_import_finish(f->import, r);
-        return 0;
-}
-
-static int raw_import_file_progress_callback(void *userdata, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) {
-        RawImportFile *f = userdata;
-        unsigned percent;
-        usec_t n;
-
-        assert(f);
+        (void) copy_times(i->raw_job->disk_fd, dfd);
+        (void) copy_xattr(i->raw_job->disk_fd, dfd);
 
-        if (dltotal <= 0)
-                return 0;
-
-        percent = ((100 * dlnow) / dltotal);
-        n = now(CLOCK_MONOTONIC);
-
-        if (n > f->last_status_usec + USEC_PER_SEC &&
-            percent != f->progress_percent) {
-                char buf[FORMAT_TIMESPAN_MAX];
-
-                if (n - f->start_usec > USEC_PER_SEC && dlnow > 0) {
-                        usec_t left, done;
-
-                        done = n - f->start_usec;
-                        left = (usec_t) (((double) done * (double) dltotal) / dlnow) - done;
-
-                        log_info("Got %u%%. %s left.", percent, format_timespan(buf, sizeof(buf), left, USEC_PER_SEC));
-                } else
-                        log_info("Got %u%%.", percent);
+        dfd = safe_close(dfd);
 
-                f->progress_percent = percent;
-                f->last_status_usec = n;
+        r = rename(tp, p);
+        if (r < 0)  {
+                unlink(tp);
+                return log_error_errno(errno, "Failed to move writable image into place: %m");
         }
 
+        log_info("Created new local image '%s'.", i->local);
         return 0;
 }
 
-static int raw_import_file_find_old_etags(RawImportFile *f) {
-        _cleanup_free_ char *escaped_url = NULL;
-        _cleanup_closedir_ DIR *d = NULL;
-        struct dirent *de;
+static void raw_import_job_on_finished(ImportJob *j) {
+        RawImport *i;
         int r;
 
-        escaped_url = xescape(f->url, FILENAME_ESCAPE);
-        if (!escaped_url)
-                return -ENOMEM;
-
-        d = opendir(f->import->image_root);
-        if (!d) {
-                if (errno == ENOENT)
-                        return 0;
+        assert(j);
+        assert(j->userdata);
 
-                return -errno;
+        i = j->userdata;
+        if (j->error != 0) {
+                r = j->error;
+                goto finish;
         }
 
-        FOREACH_DIRENT_ALL(de, d, return -errno) {
-                const char *a, *b;
-                char *u;
-
-                if (de->d_type != DT_UNKNOWN &&
-                    de->d_type != DT_REG)
-                        continue;
-
-                a = startswith(de->d_name, ".raw-");
-                if (!a)
-                        continue;
-
-                a = startswith(a, escaped_url);
-                if (!a)
-                        continue;
+        /* This is invoked if either the download completed
+         * successfully, or the download was skipped because we
+         * already have the etag. */
 
-                a = startswith(a, ".");
-                if (!a)
-                        continue;
-
-                b = endswith(de->d_name, ".raw");
-                if (!b)
-                        continue;
-
-                if (a >= b)
-                        continue;
+        if (j->disk_fd >= 0) {
+                r = raw_import_maybe_convert_qcow2(i);
+                if (r < 0)
+                        goto finish;
 
-                u = cunescape_length(a, b - a);
-                if (!u)
-                        return -ENOMEM;
+                r = import_make_read_only_fd(j->disk_fd);
+                if (r < 0)
+                        goto finish;
 
-                if (!http_etag_is_valid(u)) {
-                        free(u);
-                        continue;
+                r = rename(i->temp_path, i->final_path);
+                if (r < 0) {
+                        r = log_error_errno(errno, "Failed to move RAW file into place: %m");
+                        goto finish;
                 }
 
-                r = strv_consume(&f->old_etags, u);
-                if (r < 0)
-                        return r;
+                free(i->temp_path);
+                i->temp_path = NULL;
         }
 
-        return 0;
-}
-
-static int raw_import_file_begin(RawImportFile *f) {
-        int r;
-
-        assert(f);
-        assert(!f->curl);
-
-        log_info("Getting %s.", f->url);
-
-        r = raw_import_file_find_old_etags(f);
+        r = raw_import_make_local_copy(i);
         if (r < 0)
-                return r;
-
-        r = curl_glue_make(&f->curl, f->url, f);
-        if (r < 0)
-                return r;
-
-        if (!strv_isempty(f->old_etags)) {
-                _cleanup_free_ char *cc = NULL, *hdr = NULL;
-
-                cc = strv_join(f->old_etags, ", ");
-                if (!cc)
-                        return -ENOMEM;
+                goto finish;
 
-                hdr = strappend("If-None-Match: ", cc);
-                if (!hdr)
-                        return -ENOMEM;
+        j->disk_fd = safe_close(j->disk_fd);
 
-                f->request_header = curl_slist_new(hdr, NULL);
-                if (!f->request_header)
-                        return -ENOMEM;
+        r = 0;
 
-                if (curl_easy_setopt(f->curl, CURLOPT_HTTPHEADER, f->request_header) != CURLE_OK)
-                        return -EIO;
-        }
+finish:
+        if (i->on_finished)
+                i->on_finished(i, r, i->userdata);
+        else
+                sd_event_exit(i->event, r);
+}
 
-        if (curl_easy_setopt(f->curl, CURLOPT_WRITEFUNCTION, raw_import_file_write_callback) != CURLE_OK)
-                return -EIO;
+static int raw_import_job_on_open_disk(ImportJob *j) {
+        RawImport *i;
+        int r;
 
-        if (curl_easy_setopt(f->curl, CURLOPT_WRITEDATA, f) != CURLE_OK)
-                return -EIO;
+        assert(j);
+        assert(j->userdata);
 
-        if (curl_easy_setopt(f->curl, CURLOPT_HEADERFUNCTION, raw_import_file_header_callback) != CURLE_OK)
-                return -EIO;
+        i = j->userdata;
 
-        if (curl_easy_setopt(f->curl, CURLOPT_HEADERDATA, f) != CURLE_OK)
-                return -EIO;
+        r = import_make_path(j->url, j->etag, i->image_root, ".raw-", ".raw", &i->final_path);
+        if (r < 0)
+                return log_oom();
 
-        if (curl_easy_setopt(f->curl, CURLOPT_XFERINFOFUNCTION, raw_import_file_progress_callback) != CURLE_OK)
-                return -EIO;
+        r = tempfn_random(i->final_path, &i->temp_path);
+        if (r <0)
+                return log_oom();
 
-        if (curl_easy_setopt(f->curl, CURLOPT_XFERINFODATA, f) != CURLE_OK)
-                return -EIO;
+        mkdir_parents_label(i->temp_path, 0700);
 
-        if (curl_easy_setopt(f->curl, CURLOPT_NOPROGRESS, 0) != CURLE_OK)
-                return -EIO;
+        j->disk_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
+        if (j->disk_fd < 0)
+                return log_error_errno(errno, "Failed to create %s: %m", i->temp_path);
 
-        r = curl_glue_add(f->import->glue, f->curl);
+        r = chattr_fd(j->disk_fd, true, FS_NOCOW_FL);
         if (r < 0)
-                return r;
+                log_warning_errno(errno, "Failed to set file attributes on %s: %m", i->temp_path);
 
         return 0;
 }
 
-int raw_import_new(RawImport **import, sd_event *event, const char *image_root, raw_import_on_finished on_finished, void *userdata) {
-        _cleanup_(raw_import_unrefp) RawImport *i = NULL;
+int raw_import_pull(RawImport *i, const char *url, const char *local, bool force_local) {
         int r;
 
-        assert(import);
-        assert(image_root);
+        assert(i);
 
-        i = new0(RawImport, 1);
-        if (!i)
-                return -ENOMEM;
+        if (i->raw_job)
+                return -EBUSY;
 
-        i->on_finished = on_finished;
-        i->userdata = userdata;
+        if (!http_url_is_valid(url))
+                return -EINVAL;
 
-        i->image_root = strdup(image_root);
-        if (!i->image_root)
-                return -ENOMEM;
+        if (local && !machine_name_is_valid(local))
+                return -EINVAL;
 
-        if (event)
-                i->event = sd_event_ref(event);
-        else {
-                r = sd_event_default(&i->event);
-                if (r < 0)
-                        return r;
-        }
-
-        r = curl_glue_new(&i->glue, i->event);
+        r = free_and_strdup(&i->local, local);
         if (r < 0)
                 return r;
 
-        i->glue->on_finished = raw_import_curl_on_finished;
-        i->glue->userdata = i;
-
-        *import = i;
-        i = NULL;
-
-        return 0;
-}
-
-RawImport* raw_import_unref(RawImport *import) {
-        RawImportFile *f;
-
-        if (!import)
-                return NULL;
-
-        while ((f = hashmap_steal_first(import->files)))
-                raw_import_file_unref(f);
-        hashmap_free(import->files);
-
-        curl_glue_unref(import->glue);
-        sd_event_unref(import->event);
-
-        free(import->image_root);
-        free(import);
-
-        return NULL;
-}
-
-int raw_import_cancel(RawImport *import, const char *url) {
-        RawImportFile *f;
+        i->force_local = force_local;
 
-        assert(import);
-        assert(url);
-
-        f = hashmap_remove(import->files, url);
-        if (!f)
-                return 0;
-
-        raw_import_file_unref(f);
-        return 1;
-}
-
-int raw_import_pull(RawImport *import, const char *url, const char *local, bool force_local) {
-        _cleanup_(raw_import_file_unrefp) RawImportFile *f = NULL;
-        int r;
-
-        assert(import);
-        assert(http_url_is_valid(url));
-        assert(!local || machine_name_is_valid(local));
-
-        if (hashmap_get(import->files, url))
-                return -EEXIST;
-
-        r = hashmap_ensure_allocated(&import->files, &string_hash_ops);
+        r = import_job_new(&i->raw_job, url, i->glue, i);
         if (r < 0)
                 return r;
 
-        f = new0(RawImportFile, 1);
-        if (!f)
-                return -ENOMEM;
-
-        f->import = import;
-        f->disk_fd = -1;
-        f->content_length = (uint64_t) -1;
-        f->start_usec = now(CLOCK_MONOTONIC);
-
-        f->url = strdup(url);
-        if (!f->url)
-                return -ENOMEM;
-
-        if (local) {
-                f->local = strdup(local);
-                if (!f->local)
-                        return -ENOMEM;
-
-                f->force_local = force_local;
-        }
+        i->raw_job->on_finished = raw_import_job_on_finished;
+        i->raw_job->on_open_disk = raw_import_job_on_open_disk;
 
-        r = hashmap_put(import->files, f->url, f);
+        r = import_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
         if (r < 0)
                 return r;
 
-        r = raw_import_file_begin(f);
-        if (r < 0) {
-                raw_import_cancel(import, f->url);
-                f = NULL;
-                return r;
-        }
-
-        f = NULL;
-        return 0;
+        return import_job_begin(i->raw_job);
 }
diff --git a/src/import/import-raw.h b/src/import/import-raw.h
index 17f7a1a8ee..9e23142fee 100644
--- a/src/import/import-raw.h
+++ b/src/import/import-raw.h
@@ -1,5 +1,7 @@
 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
 
+#pragma once
+
 /***
   This file is part of systemd.
 
@@ -24,12 +26,11 @@
 
 typedef struct RawImport RawImport;
 
-typedef void (*raw_import_on_finished)(RawImport *import, int error, void *userdata);
+typedef void (*RawImportFinished)(RawImport *import, int error, void *userdata);
 
-int raw_import_new(RawImport **import, sd_event *event, const char *image_root, raw_import_on_finished on_finished, void *userdata);
+int raw_import_new(RawImport **import, sd_event *event, const char *image_root, RawImportFinished on_finished, void *userdata);
 RawImport* raw_import_unref(RawImport *import);
 
 DEFINE_TRIVIAL_CLEANUP_FUNC(RawImport*, raw_import_unref);
 
 int raw_import_pull(RawImport *import, const char *url, const char *local, bool force_local);
-int raw_import_cancel(RawImport *import, const char *name);
diff --git a/src/import/import-tar.c b/src/import/import-tar.c
index 43227f61f3..08839caae7 100644
--- a/src/import/import-tar.c
+++ b/src/import/import-tar.c
@@ -22,7 +22,6 @@
 #include <sys/prctl.h>
 #include <curl/curl.h>
 
-#include "hashmap.h"
 #include "utf8.h"
 #include "strv.h"
 #include "copy.h"
@@ -46,8 +45,6 @@ struct TarImport {
         TarImportFinished on_finished;
         void *userdata;
 
-        bool finished;
-
         char *local;
         bool force_local;
 
@@ -74,6 +71,7 @@ TarImport* tar_import_unref(TarImport *i) {
         if (i->temp_path) {
                 (void) btrfs_subvol_remove(i->temp_path);
                 (void) rm_rf_dangerous(i->temp_path, false, true, false);
+                free(i->temp_path);
         }
 
         free(i->final_path);
@@ -124,6 +122,28 @@ int tar_import_new(TarImport **ret, sd_event *event, const char *image_root, Tar
         return 0;
 }
 
+static int tar_import_make_local_copy(TarImport *i) {
+        int r;
+
+        assert(i);
+        assert(i->tar_job);
+
+        if (!i->local)
+                return 0;
+
+        if (!i->final_path) {
+                r = import_make_path(i->tar_job->url, i->tar_job->etag, i->image_root, ".tar-", NULL, &i->final_path);
+                if (r < 0)
+                        return log_oom();
+
+                r = import_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
+                if (r < 0)
+                        return r;
+        }
+
+        return 0;
+}
+
 static void tar_import_job_on_finished(ImportJob *j) {
         TarImport *i;
         int r;
@@ -132,7 +152,6 @@ static void tar_import_job_on_finished(ImportJob *j) {
         assert(j->userdata);
 
         i = j->userdata;
-
         if (j->error != 0) {
                 r = j->error;
                 goto finish;
@@ -160,25 +179,18 @@ static void tar_import_job_on_finished(ImportJob *j) {
                         r = log_error_errno(errno, "Failed to rename to final image name: %m");
                         goto finish;
                 }
-        }
 
-        if (i->local) {
-                if (!i->final_path) {
-                        r = import_make_path(j->url, j->etag, i->image_root, ".tar-", NULL, &i->final_path);
-                        if (r < 0)
-                                goto finish;
-                }
-
-                r = import_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
-                if (r < 0)
-                        goto finish;
+                free(i->temp_path);
+                i->temp_path = NULL;
         }
 
+        r = tar_import_make_local_copy(i);
+        if (r < 0)
+                goto finish;
+
         r = 0;
 
 finish:
-        i->finished = true;
-
         if (i->on_finished)
                 i->on_finished(i, r, i->userdata);
         else
diff --git a/src/import/import-util.c b/src/import/import-util.c
index 24c8ac1d13..341a566e37 100644
--- a/src/import/import-util.c
+++ b/src/import/import-util.c
@@ -147,28 +147,47 @@ int import_make_local_copy(const char *final, const char *image_root, const char
         return 0;
 }
 
-int import_make_read_only(const char *path) {
+int import_make_read_only_fd(int fd) {
         int r;
 
-        r = btrfs_subvol_set_read_only(path, true);
-        if (r == -ENOTTY) {
+        assert(fd >= 0);
+
+        /* First, let's make this a read-only subvolume if it refers
+         * to a subvolume */
+        r = btrfs_subvol_set_read_only_fd(fd, true);
+        if (r == -ENOTTY || r == -ENOTDIR || r == -EINVAL) {
                 struct stat st;
 
-                r = stat(path, &st);
+                /* This doesn't refer to a subvolume, or the file
+                 * system isn't even btrfs. In that, case fall back to
+                 * chmod()ing */
+
+                r = fstat(fd, &st);
                 if (r < 0)
                         return log_error_errno(errno, "Failed to stat temporary image: %m");
 
-                if (chmod(path, st.st_mode & 0755) < 0)
+                /* Drop "w" flag */
+                if (fchmod(fd, st.st_mode & 07555) < 0)
                         return log_error_errno(errno, "Failed to chmod() final image: %m");
 
                 return 0;
-        }
-        if (r < 0)
-                return log_error_errno(r, "Failed to mark final image read-only: %m");
+
+        } else if (r < 0)
+                return log_error_errno(r, "Failed to make subvolume read-only: %m");
 
         return 0;
 }
 
+int import_make_read_only(const char *path) {
+        _cleanup_close_ int fd = 1;
+
+        fd = open(path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
+        if (fd < 0)
+                return log_error_errno(errno, "Failed to open %s: %m", path);
+
+        return import_make_read_only_fd(fd);
+}
+
 int import_make_path(const char *url, const char *etag, const char *image_root, const char *prefix, const char *suffix, char **ret) {
         _cleanup_free_ char *escaped_url = NULL;
         char *path;
diff --git a/src/import/import-util.h b/src/import/import-util.h
index ad5ab502ae..a930ea48a0 100644
--- a/src/import/import-util.h
+++ b/src/import/import-util.h
@@ -26,6 +26,10 @@
 bool http_etag_is_valid(const char *etag);
 
 int import_make_local_copy(const char *final, const char *root, const char *local, bool force_local);
+
 int import_find_old_etags(const char *url, const char *root, int dt, const char *prefix, const char *suffix, char ***etags);
+
+int import_make_read_only_fd(int fd);
 int import_make_read_only(const char *path);
+
 int import_make_path(const char *url, const char *etag, const char *image_root, const char *prefix, const char *suffix, char **ret);
diff --git a/src/import/import.c b/src/import/import.c
index c0fc22427e..861a448583 100644
--- a/src/import/import.c
+++ b/src/import/import.c
@@ -226,7 +226,7 @@ static int pull_raw(int argc, char *argv[], void *userdata) {
         _cleanup_(raw_import_unrefp) RawImport *import = NULL;
         _cleanup_event_unref_ sd_event *event = NULL;
         const char *url, *local;
-        _cleanup_free_ char *l = NULL;
+        _cleanup_free_ char *l = NULL, *ll = NULL;
         int r;
 
         url = argv[1];
@@ -238,44 +238,37 @@ static int pull_raw(int argc, char *argv[], void *userdata) {
         if (argc >= 3)
                 local = argv[2];
         else {
-                const char *e, *p;
-
-                e = url + strlen(url);
-                while (e > url && e[-1] == '/')
-                        e--;
-
-                p = e;
-                while (p > url && p[-1] != '/')
-                        p--;
+                r = url_final_component(url, &l);
+                if (r < 0)
+                        return log_error_errno(r, "Failed get final component of URL: %m");
 
-                local = strndupa(p, e - p);
+                local = l;
         }
 
         if (isempty(local) || streq(local, "-"))
                 local = NULL;
 
         if (local) {
-                const char *p;
-
-                r = strip_raw_suffixes(local, &l);
+                r = strip_raw_suffixes(local, &ll);
                 if (r < 0)
                         return log_oom();
 
-                local = l;
+                local = ll;
 
                 if (!machine_name_is_valid(local)) {
                         log_error("Local image name '%s' is not valid.", local);
                         return -EINVAL;
                 }
 
-                p = strappenda(arg_image_root, "/", local, ".raw");
-                if (laccess(p, F_OK) >= 0) {
-                        if (!arg_force) {
-                                log_info("Image '%s' already exists.", local);
-                                return 0;
+                if (!arg_force) {
+                        r = image_find(local, NULL);
+                        if (r < 0)
+                                return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
+                        else if (r > 0) {
+                                log_error_errno(EEXIST, "Image '%s' already exists.", local);
+                                return -EEXIST;
                         }
-                } else if (errno != ENOENT)
-                        return log_error_errno(errno, "Can't check if image '%s' already exists: %m", local);
+                }
 
                 log_info("Pulling '%s', saving as '%s'.", url, local);
         } else
@@ -417,7 +410,7 @@ static int help(int argc, char *argv[], void *userdata) {
                "     --image-root=            Image root directory\n"
                "     --dkr-index-url=URL      Specify index URL to use for downloads\n\n"
                "Commands:\n"
-               "  pull-tar URL                Download a TAR image\n"
+               "  pull-tar URL [NAME]         Download a TAR image\n"
                "  pull-raw URL [NAME]         Download a RAW image\n"
                "  pull-dkr REMOTE [NAME]      Download a DKR image\n",
                program_invocation_short_name);
diff --git a/src/shared/btrfs-util.c b/src/shared/btrfs-util.c
index 254483c31a..b34ac8b15a 100644
--- a/src/shared/btrfs-util.c
+++ b/src/shared/btrfs-util.c
@@ -228,14 +228,18 @@ int btrfs_subvol_remove(const char *path) {
         return 0;
 }
 
-int btrfs_subvol_set_read_only(const char *path, bool b) {
-        _cleanup_close_ int fd = -1;
+int btrfs_subvol_set_read_only_fd(int fd, bool b) {
         uint64_t flags, nflags;
+        struct stat st;
 
-        fd = open(path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
-        if (fd < 0)
+        assert(fd >= 0);
+
+        if (fstat(fd, &st) < 0)
                 return -errno;
 
+        if (!S_ISDIR(st.st_mode) || st.st_ino != 256)
+                return -EINVAL;
+
         if (ioctl(fd, BTRFS_IOC_SUBVOL_GETFLAGS, &flags) < 0)
                 return -errno;
 
@@ -253,6 +257,16 @@ int btrfs_subvol_set_read_only(const char *path, bool b) {
         return 0;
 }
 
+int btrfs_subvol_set_read_only(const char *path, bool b) {
+        _cleanup_close_ int fd = -1;
+
+        fd = open(path, O_RDONLY|O_NOCTTY|O_CLOEXEC|O_DIRECTORY);
+        if (fd < 0)
+                return -errno;
+
+        return btrfs_subvol_set_read_only_fd(fd, b);
+}
+
 int btrfs_subvol_get_read_only_fd(int fd) {
         uint64_t flags;
 
diff --git a/src/shared/btrfs-util.h b/src/shared/btrfs-util.h
index 28946c60c9..d5249d1bf1 100644
--- a/src/shared/btrfs-util.h
+++ b/src/shared/btrfs-util.h
@@ -48,6 +48,7 @@ int btrfs_subvol_make_label(const char *path);
 int btrfs_subvol_remove(const char *path);
 int btrfs_subvol_snapshot(const char *old_path, const char *new_path, bool read_only, bool fallback_copy);
 
+int btrfs_subvol_set_read_only_fd(int fd, bool b);
 int btrfs_subvol_set_read_only(const char *path, bool b);
 int btrfs_subvol_get_read_only_fd(int fd);
 int btrfs_subvol_get_id_fd(int fd, uint64_t *ret);