import: support importing qcow2 images

With this change the import tool will now unpack qcow2 images into
normal raw disk images, suitable for usage with nspawn.

This allows has the benefit of also allowing importing Ubuntu Cloud
images for usage with nspawn.

8 files changed, 560 insertions, 11 deletions

diff --git a/.gitignore b/.gitignore
index caef8310fa..d4704feedf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -227,6 +227,7 @@
 /test-pppoe
 /test-prioq
 /test-pty
+/test-qcow2
 /test-ratelimit
 /test-replace-var
 /test-resolve
diff --git a/Makefile.am b/Makefile.am
index ee73598cb1..f91c78658d 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -5233,19 +5233,43 @@ systemd_import_SOURCES = \
 	src/import/curl-util.c \
 	src/import/curl-util.h \
 	src/import/aufs-util.c \
-	src/import/aufs-util.h
+	src/import/aufs-util.h \
+	src/import/qcow2-util.c \
+	src/import/qcow2-util.h
 
 systemd_import_CFLAGS = \
 	$(AM_CFLAGS) \
 	$(LIBCURL_CFLAGS) \
-	$(XZ_CFLAGS)
+	$(XZ_CFLAGS) \
+	$(ZLIB_CFLAGS)
 
 systemd_import_LDADD = \
 	libsystemd-internal.la \
 	libsystemd-label.la \
 	libsystemd-shared.la \
 	$(LIBCURL_LIBS) \
-	$(XZ_LIBS)
+	$(XZ_LIBS) \
+	$(ZLIB_LIBS)
+
+endif
+
+if HAVE_ZLIB
+manual_tests += \
+	test-qcow2
+
+test_qcow2_SOURCES = \
+	src/import/test-qcow2.c \
+	src/import/qcow2-util.c \
+	src/import/qcow2-util.h
+
+test_qcow2_CFLAGS = \
+	$(AM_CFLAGS) \
+	$(ZLIB_CFLAGS)
+
+test_qcow2_LDADD = \
+	libsystemd-internal.la \
+	libsystemd-shared.la \
+	$(ZLIB_LIBS)
 endif
 
 endif
diff --git a/configure.ac b/configure.ac
index 5057f8e992..6d510df332 100644
--- a/configure.ac
+++ b/configure.ac
@@ -564,6 +564,18 @@ fi
 AM_CONDITIONAL(HAVE_XZ, [test "$have_xz" = "yes"])
 
 # ------------------------------------------------------------------------------
+have_zlib=no
+AC_ARG_ENABLE(zlib, AS_HELP_STRING([--disable-zlib], [Disable optional ZLIB support]))
+if test "x$enable_zlib" != "xno"; then
+        PKG_CHECK_MODULES(ZLIB, [ zlib ],
+                [AC_DEFINE(HAVE_ZLIB, 1, [Define if ZLIB is available]) have_zlib=yes])
+        if test "x$have_zlib" = xno -a "x$enable_zlib" = xyes; then
+                AC_MSG_ERROR([*** ZLIB support requested but libraries not found])
+        fi
+fi
+AM_CONDITIONAL(HAVE_ZLIB, [test "$have_zlib" = "yes"])
+
+# ------------------------------------------------------------------------------
 have_lz4=no
 AC_ARG_ENABLE(lz4, AS_HELP_STRING([--enable-lz4], [Enable optional LZ4 support]))
 AS_IF([test "x$enable_lz4" = "xyes"], [
@@ -1410,6 +1422,7 @@ AC_MSG_RESULT([
         SELinux:                 ${have_selinux}
         SECCOMP:                 ${have_seccomp}
         SMACK:                   ${have_smack}
+        ZLIB:                    ${have_zlib}
         XZ:                      ${have_xz}
         LZ4:                     ${have_lz4}
         ACL:                     ${have_acl}
diff --git a/src/import/import-raw.c b/src/import/import-raw.c
index f1b36cbfcc..c15765d51c 100644
--- a/src/import/import-raw.c
+++ b/src/import/import-raw.c
@@ -27,6 +27,7 @@
 #include "hashmap.h"
 #include "utf8.h"
 #include "curl-util.h"
+#include "qcow2-util.h"
 #include "import-raw.h"
 #include "strv.h"
 #include "copy.h"
@@ -235,6 +236,49 @@ finish:
         raw_import_finish(f->import, r);
 }
 
+static int raw_import_maybe_convert_qcow2(RawImportFile *f) {
+        _cleanup_close_ int converted_fd = -1;
+        _cleanup_free_ char *t = NULL;
+        int r;
+
+        assert(f);
+        assert(f->disk_fd);
+        assert(f->temp_path);
+
+        r = qcow2_detect(f->disk_fd);
+        if (r < 0)
+                return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
+        if (r == 0)
+                return 0;
+
+        /* This is a QCOW2 image, let's convert it */
+        r = tempfn_random(f->final_path, &t);
+        if (r < 0)
+                return log_oom();
+
+        converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
+        if (converted_fd < 0)
+                return log_error_errno(errno, "Failed to create %s: %m", t);
+
+        r = qcow2_convert(f->disk_fd, converted_fd);
+        if (r < 0) {
+                unlink(t);
+                return log_error_errno(r, "Failed to convert qcow2 image: %m");
+        }
+
+        unlink(f->temp_path);
+        free(f->temp_path);
+
+        f->temp_path = t;
+        t = NULL;
+
+        safe_close(f->disk_fd);
+        f->disk_fd = converted_fd;
+        converted_fd = -1;
+
+        return 1;
+}
+
 static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
         RawImportFile *f = NULL;
         struct stat st;
@@ -288,6 +332,10 @@ static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result
                 goto fail;
         }
 
+        r = raw_import_maybe_convert_qcow2(f);
+        if (r < 0)
+                goto fail;
+
         if (f->etag)
                 (void) fsetxattr(f->disk_fd, "user.source_etag", f->etag, strlen(f->etag), 0);
         if (f->url)
diff --git a/src/import/import.c b/src/import/import.c
index e457adf629..af8d0ec42b 100644
--- a/src/import/import.c
+++ b/src/import/import.c
@@ -45,10 +45,48 @@ static void on_raw_finished(RawImport *import, int error, void *userdata) {
         sd_event_exit(event, error);
 }
 
+static int strip_raw_suffixes(const char *p, char **ret) {
+        static const char suffixes[] =
+                ".xz\0"
+                ".raw\0"
+                ".qcow2\0"
+                ".img\0";
+
+        _cleanup_free_ char *q = NULL;
+
+        q = strdup(p);
+        if (!q)
+                return -ENOMEM;
+
+        for (;;) {
+                const char *sfx;
+                bool changed = false;
+
+                NULSTR_FOREACH(sfx, suffixes) {
+                        char *e;
+
+                        e = endswith(q, sfx);
+                        if (e) {
+                                *e = 0;
+                                changed = true;
+                        }
+                }
+
+                if (!changed)
+                        break;
+        }
+
+        *ret = q;
+        q = NULL;
+
+        return 0;
+}
+
 static int pull_raw(int argc, char *argv[], void *userdata) {
         _cleanup_(raw_import_unrefp) RawImport *import = NULL;
         _cleanup_event_unref_ sd_event *event = NULL;
-        const char *url, *local, *suffix;
+        const char *url, *local;
+        _cleanup_free_ char *l = NULL;
         int r;
 
         url = argv[1];
@@ -79,13 +117,11 @@ static int pull_raw(int argc, char *argv[], void *userdata) {
         if (local) {
                 const char *p;
 
-                suffix = endswith(local, ".raw.xz");
-                if (!suffix)
-                        suffix = endswith(local, ".raw");
-                if (!suffix)
-                        suffix = endswith(local, ".xz");
-                if (suffix)
-                        local = strndupa(local, suffix - local);
+                r = strip_raw_suffixes(local, &l);
+                if (r < 0)
+                        return log_oom();
+
+                local = l;
 
                 if (!machine_name_is_valid(local)) {
                         log_error("Local image name '%s' is not valid.", local);
diff --git a/src/import/qcow2-util.c b/src/import/qcow2-util.c
new file mode 100644
index 0000000000..c84c6aa0d7
--- /dev/null
+++ b/src/import/qcow2-util.c
@@ -0,0 +1,347 @@
+/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
+
+/***
+  This file is part of systemd.
+
+  Copyright 2015 Lennart Poettering
+
+  systemd is free software; you can redistribute it and/or modify it
+  under the terms of the GNU Lesser General Public License as published by
+  the Free Software Foundation; either version 2.1 of the License, or
+  (at your option) any later version.
+
+  systemd is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public License
+  along with systemd; If not, see <http://www.gnu.org/licenses/>.
+***/
+
+#include <zlib.h>
+
+#include "util.h"
+#include "sparse-endian.h"
+#include "qcow2-util.h"
+
+#define QCOW2_MAGIC 0x514649fb
+
+#define QCOW2_COPIED (1ULL << 63)
+#define QCOW2_COMPRESSED (1ULL << 62)
+#define QCOW2_ZERO (1ULL << 0)
+
+typedef struct _packed_ Header {
+      be32_t magic;
+      be32_t version;
+
+      be64_t backing_file_offset;
+      be32_t backing_file_size;
+
+      be32_t cluster_bits;
+      be64_t size;
+      be32_t crypt_method;
+
+      be32_t l1_size;
+      be64_t l1_table_offset;
+
+      be64_t refcount_table_offset;
+      be32_t refcount_table_clusters;
+
+      be32_t nb_snapshots;
+      be64_t snapshots_offset;
+
+      /* The remainder is only present on QCOW3 */
+      be64_t incompatible_features;
+      be64_t compatible_features;
+      be64_t autoclear_features;
+
+      be32_t refcount_order;
+      be32_t header_length;
+} Header;
+
+#define HEADER_MAGIC(header) be32toh((header)->magic)
+#define HEADER_VERSION(header) be32toh((header)->version)
+#define HEADER_CLUSTER_BITS(header) be32toh((header)->cluster_bits)
+#define HEADER_CLUSTER_SIZE(header) (1ULL << HEADER_CLUSTER_BITS(header))
+#define HEADER_L2_BITS(header) (HEADER_CLUSTER_BITS(header) - 3)
+#define HEADER_SIZE(header) be64toh((header)->size)
+#define HEADER_CRYPT_METHOD(header) be32toh((header)->crypt_method)
+#define HEADER_L1_SIZE(header) be32toh((header)->l1_size)
+#define HEADER_L2_SIZE(header) (HEADER_CLUSTER_SIZE(header)/sizeof(uint64_t))
+#define HEADER_L1_TABLE_OFFSET(header) be64toh((header)->l1_table_offset)
+
+static uint32_t HEADER_HEADER_LENGTH(const Header *h) {
+        if (HEADER_VERSION(h) < 3)
+                return offsetof(Header, incompatible_features);
+
+        return be32toh(h->header_length);
+}
+
+static int copy_cluster(
+                int sfd, uint64_t soffset,
+                int dfd, uint64_t doffset,
+                uint64_t cluster_size,
+                void *buffer) {
+
+        ssize_t l;
+
+        l = pread(sfd, buffer, cluster_size, soffset);
+        if (l < 0)
+                return -errno;
+        if ((uint64_t) l != cluster_size)
+                return -EIO;
+
+        l = pwrite(dfd, buffer, cluster_size, doffset);
+        if (l < 0)
+                return -errno;
+        if ((uint64_t) l != cluster_size)
+                return -EIO;
+
+        return 0;
+}
+
+static int decompress_cluster(
+                int sfd, uint64_t soffset,
+                int dfd, uint64_t doffset,
+                uint64_t compressed_size,
+                uint64_t cluster_size,
+                void *buffer1,
+                void *buffer2) {
+
+        _cleanup_free_ void *large_buffer = NULL;
+        z_stream s = {};
+        uint64_t sz;
+        ssize_t l;
+        int r;
+
+        if (compressed_size > cluster_size) {
+                /* The usual cluster buffer doesn't suffice, let's
+                 * allocate a larger one, temporarily */
+
+                large_buffer = malloc(compressed_size);
+                if (!large_buffer)
+                        return -ENOMEM;
+
+                buffer1 = large_buffer;
+        }
+
+        l = pread(sfd, buffer1, compressed_size, soffset);
+        if (l < 0)
+                return -errno;
+        if ((uint64_t) l != compressed_size)
+                return -EIO;
+
+        s.next_in = buffer1;
+        s.avail_in = compressed_size;
+        s.next_out = buffer2;
+        s.avail_out = cluster_size;
+
+        r = inflateInit2(&s, -12);
+        if (r != Z_OK)
+                return -EIO;
+
+        r = inflate(&s, Z_FINISH);
+        sz = (uint8_t*) s.next_out - (uint8_t*) buffer2;
+        inflateEnd(&s);
+        if (r != Z_STREAM_END || sz != cluster_size)
+                return -EIO;
+
+        l = pwrite(dfd, buffer2, cluster_size, doffset);
+        if (l < 0)
+                return -errno;
+        if ((uint64_t) l != cluster_size)
+                return -EIO;
+
+        return 0;
+}
+
+static int normalize_offset(
+                const Header *header,
+                uint64_t p,
+                uint64_t *ret,
+                bool *compressed,
+                uint64_t *compressed_size) {
+
+        uint64_t q;
+
+        q = be64toh(p);
+
+        if (q & QCOW2_COMPRESSED) {
+                uint64_t sz, csize_shift, csize_mask;
+
+                if (!compressed)
+                        return -ENOTSUP;
+
+                csize_shift = 64 - 2 - (HEADER_CLUSTER_BITS(header) - 8);
+                csize_mask = (1ULL << (HEADER_CLUSTER_BITS(header) - 8)) - 1;
+                sz = (((q >> csize_shift) & csize_mask) + 1) * 512 - (q & 511);
+                q &= ((1ULL << csize_shift) - 1);
+
+                if (compressed_size)
+                        *compressed_size = sz;
+
+                *compressed = true;
+
+        } else {
+                if (compressed)  {
+                        *compressed = false;
+                        *compressed_size = 0;
+                }
+
+                if (q & QCOW2_ZERO) {
+                        /* We make no distinction between zero blocks and holes */
+                        *ret = 0;
+                        return 0;
+                }
+
+                q &= ~QCOW2_COPIED;
+        }
+
+        *ret = q;
+        return q > 0;  /* returns positive if not a hole */
+}
+
+static int verify_header(const Header *header) {
+        assert(header);
+
+        if (HEADER_MAGIC(header) != QCOW2_MAGIC)
+                return -EBADMSG;
+
+        if (HEADER_VERSION(header) != 2 &&
+            HEADER_VERSION(header) != 3)
+                return -ENOTSUP;
+
+        if (HEADER_CRYPT_METHOD(header) != 0)
+                return -ENOTSUP;
+
+        if (HEADER_CLUSTER_BITS(header) < 9) /* 512K */
+                return -EBADMSG;
+
+        if (HEADER_CLUSTER_BITS(header) > 21) /* 2MB */
+                return -EBADMSG;
+
+        if (HEADER_SIZE(header) % HEADER_CLUSTER_SIZE(header) != 0)
+                return -EBADMSG;
+
+        if (HEADER_L1_SIZE(header) > 32*1024*1024) /* 32MB */
+                return -EBADMSG;
+
+        if (HEADER_VERSION(header) == 3) {
+
+                if (header->incompatible_features != 0)
+                        return -ENOTSUP;
+
+                if (HEADER_HEADER_LENGTH(header) < sizeof(Header))
+                        return -EBADMSG;
+        }
+
+        return 0;
+}
+
+int qcow2_convert(int qcow2_fd, int raw_fd) {
+        _cleanup_free_ void *buffer1 = NULL, *buffer2 = NULL;
+        _cleanup_free_ be64_t *l1_table = NULL, *l2_table = NULL;
+        uint64_t sz, i;
+        Header header;
+        ssize_t l;
+        int r;
+
+        l = pread(qcow2_fd, &header, sizeof(header), 0);
+        if (l < 0)
+                return -errno;
+        if (l != sizeof(header))
+                return -EIO;
+
+        r = verify_header(&header);
+        if (r < 0)
+                return r;
+
+        l1_table = new(be64_t, HEADER_L1_SIZE(&header));
+        if (!l1_table)
+                return -ENOMEM;
+
+        l2_table = malloc(HEADER_CLUSTER_SIZE(&header));
+        if (!l2_table)
+                return -ENOMEM;
+
+        buffer1 = malloc(HEADER_CLUSTER_SIZE(&header));
+        if (!buffer1)
+                return -ENOMEM;
+
+        buffer2 = malloc(HEADER_CLUSTER_SIZE(&header));
+        if (!buffer2)
+                return -ENOMEM;
+
+        /* Empty the file if it exists, we rely on zero bits */
+        if (ftruncate(raw_fd, 0) < 0)
+                return -errno;
+
+        if (ftruncate(raw_fd, HEADER_SIZE(&header)) < 0)
+                return -errno;
+
+        sz = sizeof(uint64_t) * HEADER_L1_SIZE(&header);
+        l = pread(qcow2_fd, l1_table, sz, HEADER_L1_TABLE_OFFSET(&header));
+        if (l < 0)
+                return -errno;
+        if ((uint64_t) l != sz)
+                return -EIO;
+
+        for (i = 0; i < HEADER_L1_SIZE(&header); i ++) {
+                uint64_t l2_begin, j;
+
+                r = normalize_offset(&header, l1_table[i], &l2_begin, NULL, NULL);
+                if (r < 0)
+                        return r;
+                if (r == 0)
+                        continue;
+
+                l = pread(qcow2_fd, l2_table, HEADER_CLUSTER_SIZE(&header), l2_begin);
+                if (l < 0)
+                        return -errno;
+                if ((uint64_t) l != HEADER_CLUSTER_SIZE(&header))
+                        return -EIO;
+
+                for (j = 0; j < HEADER_L2_SIZE(&header); j++) {
+                        uint64_t data_begin, p, compressed_size;
+                        bool compressed;
+
+                        p = ((i << HEADER_L2_BITS(&header)) + j) << HEADER_CLUSTER_BITS(&header);
+
+                        r = normalize_offset(&header, l2_table[j], &data_begin, &compressed, &compressed_size);
+                        if (r < 0)
+                                return r;
+                        if (r == 0)
+                                continue;
+
+                        if (compressed)
+                                r = decompress_cluster(
+                                                qcow2_fd, data_begin,
+                                                raw_fd, p,
+                                                compressed_size, HEADER_CLUSTER_SIZE(&header),
+                                                buffer1, buffer2);
+                        else
+                                r = copy_cluster(
+                                                qcow2_fd, data_begin,
+                                                raw_fd, p,
+                                                HEADER_CLUSTER_SIZE(&header), buffer1);
+                        if (r < 0)
+                                return r;
+                }
+        }
+
+        return 0;
+}
+
+int qcow2_detect(int fd) {
+        be32_t id;
+        ssize_t l;
+
+        l = pread(fd, &id, sizeof(id), 0);
+        if (l < 0)
+                return -errno;
+        if (l != sizeof(id))
+                return -EIO;
+
+        return htobe32(QCOW2_MAGIC) == id;
+}
diff --git a/src/import/qcow2-util.h b/src/import/qcow2-util.h
new file mode 100644
index 0000000000..be7fd1d0c9
--- /dev/null
+++ b/src/import/qcow2-util.h
@@ -0,0 +1,25 @@
+/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
+
+#pragma once
+
+/***
+  This file is part of systemd.
+
+  Copyright 2015 Lennart Poettering
+
+  systemd is free software; you can redistribute it and/or modify it
+  under the terms of the GNU Lesser General Public License as published by
+  the Free Software Foundation; either version 2.1 of the License, or
+  (at your option) any later version.
+
+  systemd is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public License
+  along with systemd; If not, see <http://www.gnu.org/licenses/>.
+***/
+
+int qcow2_detect(int fd);
+int qcow2_convert(int qcow2_fd, int raw_fd);
diff --git a/src/import/test-qcow2.c b/src/import/test-qcow2.c
new file mode 100644
index 0000000000..9a6c3e8b35
--- /dev/null
+++ b/src/import/test-qcow2.c
@@ -0,0 +1,55 @@
+/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
+
+/***
+  This file is part of systemd.
+
+  Copyright 2015 Lennart Poettering
+
+  systemd is free software; you can redistribute it and/or modify it
+  under the terms of the GNU Lesser General Public License as published by
+  the Free Software Foundation; either version 2.1 of the License, or
+  (at your option) any later version.
+
+  systemd is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public License
+  along with systemd; If not, see <http://www.gnu.org/licenses/>.
+***/
+
+#include "log.h"
+#include "util.h"
+
+#include "qcow2-util.h"
+
+int main(int argc, char *argv[]) {
+        _cleanup_close_ int sfd = -1, dfd = -1;
+        int r;
+
+        if (argc != 3) {
+                log_error("Needs two arguments.");
+                return EXIT_FAILURE;
+        }
+
+        sfd = open(argv[1], O_RDONLY|O_CLOEXEC|O_NOCTTY);
+        if (sfd < 0) {
+                log_error_errno(errno, "Can't open source file: %m");
+                return EXIT_FAILURE;
+        }
+
+        dfd = open(argv[2], O_WRONLY|O_CREAT|O_CLOEXEC|O_NOCTTY, 0666);
+        if (dfd < 0) {
+                log_error_errno(errno, "Can't open destination file: %m");
+                return EXIT_FAILURE;
+        }
+
+        r = qcow2_convert(sfd, dfd);
+        if (r < 0) {
+                log_error_errno(r, "Failed to unpack: %m");
+                return EXIT_FAILURE;
+        }
+
+        return EXIT_SUCCESS;
+}