From 93b73b064c663d6248bebfbbbd82989b5ca10fc5 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Wed, 21 Nov 2012 00:28:00 +0100 Subject: journal: by default do not decompress dat objects larger than 64K This introduces a new data threshold setting for sd_journal objects which controls the maximum size of objects to decompress. This is relieves the library from having to decompress full data objects even if a client program is only interested in the initial part of them. This speeds up "systemd-coredumpctl" drastically when invoked without parameters. --- src/journal/compress.c | 20 ++++++++++++++------ src/journal/compress.h | 2 +- src/journal/coredumpctl.c | 10 ++++++++++ src/journal/journal-file.c | 5 ++--- src/journal/journal-internal.h | 2 ++ src/journal/journal-verify.c | 2 +- src/journal/journald-server.c | 2 ++ src/journal/libsystemd-journal.sym | 2 ++ src/journal/sd-journal.c | 36 +++++++++++++++++++++++++++++------- 9 files changed, 63 insertions(+), 18 deletions(-) (limited to 'src/journal') diff --git a/src/journal/compress.c b/src/journal/compress.c index 75e70c5ffa..a4427be75a 100644 --- a/src/journal/compress.c +++ b/src/journal/compress.c @@ -24,6 +24,7 @@ #include #include +#include "macro.h" #include "compress.h" bool compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) { @@ -66,10 +67,11 @@ fail: } bool uncompress_blob(const void *src, uint64_t src_size, - void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size) { + void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) { lzma_stream s = LZMA_STREAM_INIT; lzma_ret ret; + uint64_t space; bool b = false; assert(src); @@ -98,7 +100,8 @@ bool uncompress_blob(const void *src, uint64_t src_size, s.avail_in = src_size; s.next_out = *dst; - s.avail_out = *dst_alloc_size; + space = dst_max > 0 ? MIN(*dst_alloc_size, dst_max) : *dst_alloc_size; + s.avail_out = space; for (;;) { void *p; @@ -111,18 +114,23 @@ bool uncompress_blob(const void *src, uint64_t src_size, if (ret != LZMA_OK) goto fail; - p = realloc(*dst, *dst_alloc_size*2); + if (dst_max > 0 && (space - s.avail_out) >= dst_max) + break; + + p = realloc(*dst, space*2); if (!p) goto fail; s.next_out = (uint8_t*) p + ((uint8_t*) s.next_out - (uint8_t*) *dst); - s.avail_out += *dst_alloc_size; + s.avail_out += space; + + space *= 2; *dst = p; - *dst_alloc_size *= 2; + *dst_alloc_size = space; } - *dst_size = *dst_alloc_size - s.avail_out; + *dst_size = space - s.avail_out; b = true; fail: diff --git a/src/journal/compress.h b/src/journal/compress.h index b6f1aa0ed7..2b87e73a26 100644 --- a/src/journal/compress.h +++ b/src/journal/compress.h @@ -27,7 +27,7 @@ bool compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size); bool uncompress_blob(const void *src, uint64_t src_size, - void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size); + void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max); bool uncompress_startswith(const void *src, uint64_t src_size, void **buffer, uint64_t *buffer_size, diff --git a/src/journal/coredumpctl.c b/src/journal/coredumpctl.c index 4adc9236f1..f9cebb7a42 100644 --- a/src/journal/coredumpctl.c +++ b/src/journal/coredumpctl.c @@ -342,6 +342,11 @@ static int dump_list(sd_journal *j) { assert(j); + /* The coredumps are likely to compressed, and for just + * listing them we don#t need to decompress them, so let's + * pick a fairly low data threshold here */ + sd_journal_set_data_threshold(j, 4096); + SD_JOURNAL_FOREACH(j) { if (field) print_field(stdout, j); @@ -381,6 +386,9 @@ static int dump_core(sd_journal* j) { assert(j); + /* We want full data, nothing truncated. */ + sd_journal_set_data_threshold(j, 0); + r = focus(j); if (r < 0) return r; @@ -428,6 +436,8 @@ static int run_gdb(sd_journal *j) { assert(j); + sd_journal_set_data_threshold(j, 0); + r = focus(j); if (r < 0) return r; diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c index 3df099dbd2..13fc8edea9 100644 --- a/src/journal/journal-file.c +++ b/src/journal/journal-file.c @@ -780,7 +780,7 @@ int journal_file_find_data_object_with_hash( l -= offsetof(Object, data.payload); - if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize)) + if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, 0)) return -EBADMSG; if (rsize == size && @@ -2591,7 +2591,6 @@ int journal_file_open_reliably( metrics, mmap_cache, template, ret); } - int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint64_t p, uint64_t *seqnum, Object **ret, uint64_t *offset) { uint64_t i, n; uint64_t q, xor_hash = 0; @@ -2645,7 +2644,7 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6 #ifdef HAVE_XZ uint64_t rsize; - if (!uncompress_blob(o->data.payload, l, &from->compress_buffer, &from->compress_buffer_size, &rsize)) + if (!uncompress_blob(o->data.payload, l, &from->compress_buffer, &from->compress_buffer_size, &rsize, 0)) return -EBADMSG; data = from->compress_buffer; diff --git a/src/journal/journal-internal.h b/src/journal/journal-internal.h index 75a4129e5b..97de0e75ff 100644 --- a/src/journal/journal-internal.h +++ b/src/journal/journal-internal.h @@ -121,6 +121,8 @@ struct sd_journal { uint64_t unique_offset; bool on_network; + + size_t data_threshold; }; char *journal_make_match_string(sd_journal *j); diff --git a/src/journal/journal-verify.c b/src/journal/journal-verify.c index 1a67d5a04b..ed28b45737 100644 --- a/src/journal/journal-verify.c +++ b/src/journal/journal-verify.c @@ -69,7 +69,7 @@ static int journal_file_object_verify(JournalFile *f, Object *o) { if (!uncompress_blob(o->data.payload, le64toh(o->object.size) - offsetof(Object, data.payload), - &b, &alloc, &b_size)) + &b, &alloc, &b_size, 0)) return -EBADMSG; h2 = hash64(b, b_size); diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index 3eaf0d3ee2..43ffe75560 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -934,6 +934,8 @@ int server_flush_to_var(Server *s) { return r; } + sd_journal_set_data_threshold(j, 0); + SD_JOURNAL_FOREACH(j) { Object *o = NULL; JournalFile *f; diff --git a/src/journal/libsystemd-journal.sym b/src/journal/libsystemd-journal.sym index 17b5bf80d9..7b602f59cb 100644 --- a/src/journal/libsystemd-journal.sym +++ b/src/journal/libsystemd-journal.sym @@ -86,4 +86,6 @@ global: sd_journal_fd_reliable; sd_journal_get_catalog; sd_journal_get_catalog_for_message_id; + sd_journal_set_data_threshold; + sd_journal_get_data_threshold; } LIBSYSTEMD_JOURNAL_195; diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c index fe0478f166..095fbb249c 100644 --- a/src/journal/sd-journal.c +++ b/src/journal/sd-journal.c @@ -47,6 +47,8 @@ #define REPLACE_VAR_MAX 256 +#define DEFAULT_DATA_THRESHOLD (64*1024) + static void detach_location(sd_journal *j) { Iterator i; JournalFile *f; @@ -1560,6 +1562,7 @@ static sd_journal *journal_new(int flags, const char *path) { j->inotify_fd = -1; j->flags = flags; + j->data_threshold = DEFAULT_DATA_THRESHOLD; if (path) { j->path = strdup(path); @@ -1838,7 +1841,8 @@ _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void ** uint64_t rsize; if (!uncompress_blob(o->data.payload, l, - &f->compress_buffer, &f->compress_buffer_size, &rsize)) + &f->compress_buffer, &f->compress_buffer_size, &rsize, + j->data_threshold)) return -EBADMSG; *data = f->compress_buffer; @@ -1862,7 +1866,7 @@ _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void ** *data = o->data.payload; *size = t; - return 0; + return 1; } r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o); @@ -1873,7 +1877,7 @@ _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void ** return -ENOENT; } -static int return_data(JournalFile *f, Object *o, const void **data, size_t *size) { +static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) { size_t t; uint64_t l; @@ -1888,7 +1892,7 @@ static int return_data(JournalFile *f, Object *o, const void **data, size_t *siz #ifdef HAVE_XZ uint64_t rsize; - if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize)) + if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold)) return -EBADMSG; *data = f->compress_buffer; @@ -1942,7 +1946,7 @@ _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t if (le_hash != o->data.hash) return -EBADMSG; - r = return_data(f, o, data, size); + r = return_data(j, f, o, data, size); if (r < 0) return r; @@ -2339,7 +2343,7 @@ _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_ if (o->object.type != OBJECT_DATA) return -EBADMSG; - r = return_data(j->unique_file, o, &odata, &ol); + r = return_data(j, j->unique_file, o, &odata, &ol); if (r < 0) return r; @@ -2371,7 +2375,7 @@ _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_ if (found) continue; - r = return_data(j->unique_file, o, data, l); + r = return_data(j, j->unique_file, o, data, l); if (r < 0) return r; @@ -2456,3 +2460,21 @@ _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) { return catalog_get(id, ret); } + +_public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) { + if (!j) + return -EINVAL; + + j->data_threshold = sz; + return 0; +} + +_public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) { + if (!j) + return -EINVAL; + if (!sz) + return -EINVAL; + + *sz = j->data_threshold; + return 0; +} -- cgit v1.2.3-54-g00ecf