journal: add LZ4 as optional compressor

Add liblz4 as an optional dependency when requested with --enable-lz4,
and use it in preference to liblzma for journal blob and coredump
compression. To retain backwards compatibility, XZ is used to
decompress old blobs.

Things will function correctly only with lz4-119.

Based on the benchmarks found on the web, lz4 seems to be the best
choice for "quick" compressors atm.

For pkg-config status, see http://code.google.com/p/lz4/issues/detail?id=135.

1 files changed, 62 insertions, 53 deletions

diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c
index b3b1ffc3c0..b6de502da8 100644
--- a/src/journal/journal-file.c
+++ b/src/journal/journal-file.c
@@ -138,7 +138,7 @@ void journal_file_close(JournalFile *f) {
 
         hashmap_free_free(f->chain_cache);
 
-#ifdef HAVE_XZ
+#if defined(HAVE_XZ) || defined(HAVE_LZ4)
         free(f->compress_buffer);
 #endif
 
@@ -158,21 +158,21 @@ void journal_file_close(JournalFile *f) {
 }
 
 static int journal_file_init_header(JournalFile *f, JournalFile *template) {
-        Header h;
+        Header h = {};
         ssize_t k;
         int r;
 
         assert(f);
 
-        zero(h);
         memcpy(h.signature, HEADER_SIGNATURE, 8);
         h.header_size = htole64(ALIGN64(sizeof(h)));
 
-        h.incompatible_flags =
-                htole32(f->compress ? HEADER_INCOMPATIBLE_COMPRESSED : 0);
+        h.incompatible_flags |= htole32(
+                f->compress_xz * HEADER_INCOMPATIBLE_COMPRESSED_XZ |
+                f->compress_lz4 * HEADER_INCOMPATIBLE_COMPRESSED_LZ4);
 
-        h.compatible_flags =
-                htole32(f->seal ? HEADER_COMPATIBLE_SEALED : 0);
+        h.compatible_flags = htole32(
+                f->seal * HEADER_COMPATIBLE_SEALED);
 
         r = sd_id128_randomize(&h.file_id);
         if (r < 0)
@@ -222,6 +222,8 @@ static int journal_file_refresh_header(JournalFile *f) {
 }
 
 static int journal_file_verify_header(JournalFile *f) {
+        uint32_t flags;
+
         assert(f);
 
         if (memcmp(f->header->signature, HEADER_SIGNATURE, 8))
@@ -229,24 +231,30 @@ static int journal_file_verify_header(JournalFile *f) {
 
         /* In both read and write mode we refuse to open files with
          * incompatible flags we don't know */
-#ifdef HAVE_XZ
-        if ((le32toh(f->header->incompatible_flags) & ~HEADER_INCOMPATIBLE_COMPRESSED) != 0)
-                return -EPROTONOSUPPORT;
-#else
-        if (f->header->incompatible_flags != 0)
+        flags = le32toh(f->header->incompatible_flags);
+        if (flags & ~HEADER_INCOMPATIBLE_SUPPORTED) {
+                if (flags & ~HEADER_INCOMPATIBLE_ANY)
+                        log_debug("Journal file %s has unknown incompatible flags %"PRIx32,
+                                  f->path, flags & ~HEADER_INCOMPATIBLE_ANY);
+                flags = (flags & HEADER_INCOMPATIBLE_ANY) & ~HEADER_INCOMPATIBLE_SUPPORTED;
+                if (flags)
+                        log_debug("Journal file %s uses incompatible flags %"PRIx32
+                                  " disabled at compilation time.", f->path, flags);
                 return -EPROTONOSUPPORT;
-#endif
+        }
 
         /* When open for writing we refuse to open files with
          * compatible flags, too */
-        if (f->writable) {
-#ifdef HAVE_GCRYPT
-                if ((le32toh(f->header->compatible_flags) & ~HEADER_COMPATIBLE_SEALED) != 0)
-                        return -EPROTONOSUPPORT;
-#else
-                if (f->header->compatible_flags != 0)
-                        return -EPROTONOSUPPORT;
-#endif
+        flags = le32toh(f->header->compatible_flags);
+        if (f->writable && (flags & ~HEADER_COMPATIBLE_SUPPORTED)) {
+                if (flags & ~HEADER_COMPATIBLE_ANY)
+                        log_debug("Journal file %s has unknown compatible flags %"PRIx32,
+                                  f->path, flags & ~HEADER_COMPATIBLE_ANY);
+                flags = (flags & HEADER_COMPATIBLE_ANY) & ~HEADER_COMPATIBLE_SUPPORTED;
+                if (flags)
+                        log_debug("Journal file %s uses compatible flags %"PRIx32
+                                  " disabled at compilation time.", f->path, flags);
+                return -EPROTONOSUPPORT;
         }
 
         if (f->header->state >= _STATE_MAX)
@@ -302,7 +310,8 @@ static int journal_file_verify_header(JournalFile *f) {
                 }
         }
 
-        f->compress = JOURNAL_HEADER_COMPRESSED(f->header);
+        f->compress_xz = JOURNAL_HEADER_COMPRESSED_XZ(f->header);
+        f->compress_lz4 = JOURNAL_HEADER_COMPRESSED_LZ4(f->header);
 
         f->seal = JOURNAL_HEADER_SEALED(f->header);
 
@@ -809,8 +818,7 @@ int journal_file_find_data_object_with_hash(
                 if (le64toh(o->data.hash) != hash)
                         goto next;
 
-                if (o->object.flags & OBJECT_COMPRESSED) {
-#ifdef HAVE_XZ
+                if (o->object.flags & OBJECT_COMPRESSION_MASK) {
                         uint64_t l, rsize;
 
                         l = le64toh(o->object.size);
@@ -819,8 +827,10 @@ int journal_file_find_data_object_with_hash(
 
                         l -= offsetof(Object, data.payload);
 
-                        if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, 0))
-                                return -EBADMSG;
+                        r = decompress_blob(o->object.flags & OBJECT_COMPRESSION_MASK,
+                                            o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, 0);
+                        if (r < 0)
+                                return r;
 
                         if (rsize == size &&
                             memcmp(f->compress_buffer, data, size) == 0) {
@@ -833,9 +843,6 @@ int journal_file_find_data_object_with_hash(
 
                                 return 1;
                         }
-#else
-                        return -EPROTONOSUPPORT;
-#endif
 
                 } else if (le64toh(o->object.size) == osize &&
                            memcmp(o->data.payload, data, size) == 0) {
@@ -943,8 +950,7 @@ static int journal_file_append_data(
         uint64_t hash, p;
         uint64_t osize;
         Object *o;
-        int r;
-        bool compressed = false;
+        int r, compression = 0;
         const void *eq;
 
         assert(f);
@@ -973,23 +979,24 @@ static int journal_file_append_data(
 
         o->data.hash = htole64(hash);
 
-#ifdef HAVE_XZ
-        if (f->compress &&
+#if defined(HAVE_XZ) || defined(HAVE_LZ4)
+        if (f->compress_xz &&
             size >= COMPRESSION_SIZE_THRESHOLD) {
                 uint64_t rsize;
 
-                compressed = compress_blob(data, size, o->data.payload, &rsize);
+                compression = compress_blob(data, size, o->data.payload, &rsize);
 
-                if (compressed) {
+                if (compression) {
                         o->object.size = htole64(offsetof(Object, data.payload) + rsize);
-                        o->object.flags |= OBJECT_COMPRESSED;
+                        o->object.flags |= compression;
 
-                        log_debug("Compressed data object %"PRIu64" -> %"PRIu64, size, rsize);
+                        log_debug("Compressed data object %"PRIu64" -> %"PRIu64" using %s",
+                                  size, rsize, object_compressed_to_string(compression));
                 }
         }
 #endif
 
-        if (!compressed && size > 0)
+        if (!compression && size > 0)
                 memcpy(o->data.payload, data, size);
 
         r = journal_file_link_data(f, o, p, hash);
@@ -2332,8 +2339,9 @@ void journal_file_dump(JournalFile *f) {
                         break;
                 }
 
-                if (o->object.flags & OBJECT_COMPRESSED)
-                        printf("Flags: COMPRESSED\n");
+                if (o->object.flags & OBJECT_COMPRESSION_MASK)
+                        printf("Flags: %s\n",
+                               object_compressed_to_string(o->object.flags & OBJECT_COMPRESSION_MASK));
 
                 if (p == le64toh(f->header->tail_object_offset))
                         p = 0;
@@ -2370,7 +2378,7 @@ void journal_file_print_header(JournalFile *f) {
                "Sequential Number ID: %s\n"
                "State: %s\n"
                "Compatible Flags:%s%s\n"
-               "Incompatible Flags:%s%s\n"
+               "Incompatible Flags:%s%s%s\n"
                "Header size: %"PRIu64"\n"
                "Arena size: %"PRIu64"\n"
                "Data Hash Table Size: %"PRIu64"\n"
@@ -2392,9 +2400,10 @@ void journal_file_print_header(JournalFile *f) {
                f->header->state == STATE_ONLINE ? "ONLINE" :
                f->header->state == STATE_ARCHIVED ? "ARCHIVED" : "UNKNOWN",
                JOURNAL_HEADER_SEALED(f->header) ? " SEALED" : "",
-               (le32toh(f->header->compatible_flags) & ~HEADER_COMPATIBLE_SEALED) ? " ???" : "",
-               JOURNAL_HEADER_COMPRESSED(f->header) ? " COMPRESSED" : "",
-               (le32toh(f->header->incompatible_flags) & ~HEADER_INCOMPATIBLE_COMPRESSED) ? " ???" : "",
+               (le32toh(f->header->compatible_flags) & ~HEADER_COMPATIBLE_ANY) ? " ???" : "",
+               JOURNAL_HEADER_COMPRESSED_XZ(f->header) ? " COMPRESSED-XZ" : "",
+               JOURNAL_HEADER_COMPRESSED_LZ4(f->header) ? " COMPRESSED-LZ4" : "",
+               (le32toh(f->header->incompatible_flags) & ~HEADER_INCOMPATIBLE_ANY) ? " ???" : "",
                le64toh(f->header->header_size),
                le64toh(f->header->arena_size),
                le64toh(f->header->data_hash_table_size) / sizeof(HashItem),
@@ -2467,8 +2476,10 @@ int journal_file_open(
         f->flags = flags;
         f->prot = prot_from_flags(flags);
         f->writable = (flags & O_ACCMODE) != O_RDONLY;
-#ifdef HAVE_XZ
-        f->compress = compress;
+#if defined(HAVE_LZ)
+        f->compress_lz4 = compress;
+#elif defined(HAVE_XZ)
+        f->compress_xz = compress;
 #endif
 #ifdef HAVE_GCRYPT
         f->seal = seal;
@@ -2760,18 +2771,16 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6
                 if ((uint64_t) t != l)
                         return -E2BIG;
 
-                if (o->object.flags & OBJECT_COMPRESSED) {
-#ifdef HAVE_XZ
+                if (o->object.flags & OBJECT_COMPRESSION_MASK) {
                         uint64_t rsize;
 
-                        if (!uncompress_blob(o->data.payload, l, &from->compress_buffer, &from->compress_buffer_size, &rsize, 0))
-                                return -EBADMSG;
+                        r = decompress_blob(o->object.flags & OBJECT_COMPRESSION_MASK,
+                                            o->data.payload, l, &from->compress_buffer, &from->compress_buffer_size, &rsize, 0);
+                        if (r < 0)
+                                return r;
 
                         data = from->compress_buffer;
                         l = rsize;
-#else
-                        return -EPROTONOSUPPORT;
-#endif
                 } else
                         data = o->data.payload;