diff options
Diffstat (limited to 'src/journal')
55 files changed, 2070 insertions, 1207 deletions
diff --git a/src/journal/audit-type.c b/src/journal/audit-type.c index 4888c7d05d..086bf7e7e3 100644 --- a/src/journal/audit-type.c +++ b/src/journal/audit-type.c @@ -25,8 +25,7 @@ # include <libaudit.h> #endif -#include "audit-type.h" -#include "macro.h" #include "missing.h" - +#include "audit-type.h" #include "audit_type-to-name.h" +#include "macro.h" diff --git a/src/journal/cat.c b/src/journal/cat.c index be2c2e3354..7fd4198df8 100644 --- a/src/journal/cat.c +++ b/src/journal/cat.c @@ -19,17 +19,20 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <stdio.h> -#include <getopt.h> -#include <unistd.h> -#include <stdlib.h> #include <errno.h> #include <fcntl.h> +#include <getopt.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> -#include "systemd/sd-journal.h" +#include "sd-journal.h" +#include "fd-util.h" +#include "parse-util.h" +#include "string-util.h" +#include "syslog-util.h" #include "util.h" -#include "build.h" static char *arg_identifier = NULL; static int arg_priority = LOG_INFO; @@ -76,9 +79,7 @@ static int parse_argv(int argc, char *argv[]) { return 0; case ARG_VERSION: - puts(PACKAGE_STRING); - puts(SYSTEMD_FEATURES); - return 0; + return version(); case 't': free(arg_identifier); @@ -95,7 +96,7 @@ static int parse_argv(int argc, char *argv[]) { arg_priority = log_level_from_string(optarg); if (arg_priority < 0) { log_error("Failed to parse priority value."); - return arg_priority; + return -EINVAL; } break; @@ -103,10 +104,9 @@ static int parse_argv(int argc, char *argv[]) { int k; k = parse_boolean(optarg); - if (k < 0) { - log_error("Failed to parse level prefix value."); - return k; - } + if (k < 0) + return log_error_errno(k, "Failed to parse level prefix value."); + arg_level_prefix = k; break; } @@ -122,7 +122,8 @@ static int parse_argv(int argc, char *argv[]) { } int main(int argc, char *argv[]) { - int r, fd = -1, saved_stderr = -1; + _cleanup_close_ int fd = -1, saved_stderr = -1; + int r; log_parse_environment(); log_open(); @@ -133,8 +134,7 @@ int main(int argc, char *argv[]) { fd = sd_journal_stream_fd(arg_identifier, arg_priority, arg_level_prefix); if (fd < 0) { - log_error_errno(fd, "Failed to create stream fd: %m"); - r = fd; + r = log_error_errno(fd, "Failed to create stream fd: %m"); goto finish; } @@ -148,25 +148,20 @@ int main(int argc, char *argv[]) { if (fd >= 3) safe_close(fd); - fd = -1; if (argc <= optind) - execl("/bin/cat", "/bin/cat", NULL); + (void) execl("/bin/cat", "/bin/cat", NULL); else - execvp(argv[optind], argv + optind); - + (void) execvp(argv[optind], argv + optind); r = -errno; /* Let's try to restore a working stderr, so we can print the error message */ if (saved_stderr >= 0) - dup3(saved_stderr, STDERR_FILENO, 0); + (void) dup3(saved_stderr, STDERR_FILENO, 0); log_error_errno(r, "Failed to execute process: %m"); finish: - safe_close(fd); - safe_close(saved_stderr); - return r < 0 ? EXIT_FAILURE : EXIT_SUCCESS; } diff --git a/src/journal/catalog.c b/src/journal/catalog.c index a3e51e2f52..fcaa54aa0c 100644 --- a/src/journal/catalog.c +++ b/src/journal/catalog.c @@ -19,25 +19,31 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ +#include <errno.h> #include <fcntl.h> +#include <locale.h> #include <stdio.h> -#include <unistd.h> -#include <errno.h> #include <string.h> #include <sys/mman.h> -#include <locale.h> +#include <unistd.h> -#include "util.h" -#include "log.h" -#include "sparse-endian.h" #include "sd-id128.h" -#include "hashmap.h" -#include "strv.h" -#include "strbuf.h" + +#include "alloc-util.h" +#include "catalog.h" #include "conf-files.h" +#include "fd-util.h" +#include "fileio.h" +#include "hashmap.h" +#include "log.h" #include "mkdir.h" -#include "catalog.h" +#include "path-util.h" #include "siphash24.h" +#include "sparse-endian.h" +#include "strbuf.h" +#include "string-util.h" +#include "strv.h" +#include "util.h" const char * const catalog_file_dirs[] = { "/usr/local/lib/systemd/catalog/", @@ -62,21 +68,11 @@ typedef struct CatalogItem { le64_t offset; } CatalogItem; -static unsigned long catalog_hash_func(const void *p, const uint8_t hash_key[HASH_KEY_SIZE]) { +static void catalog_hash_func(const void *p, struct siphash *state) { const CatalogItem *i = p; - uint64_t u; - size_t l, sz; - void *v; - l = strlen(i->language); - sz = sizeof(i->id) + l; - v = alloca(sz); - - memcpy(mempcpy(v, &i->id, sizeof(i->id)), i->language, l); - - siphash24((uint8_t*) &u, v, sz, hash_key); - - return (unsigned long) u; + siphash24_compress(&i->id, sizeof(i->id), state); + siphash24_compress(i->language, strlen(i->language), state); } static int catalog_compare_func(const void *a, const void *b) { @@ -212,7 +208,7 @@ int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) { r = catalog_file_lang(path, &deflang); if (r < 0) - log_error_errno(errno, "Failed to determine language for file %s: %m", path); + log_error_errno(r, "Failed to determine language for file %s: %m", path); if (r == 1) log_debug("File %s has language %s.", path, deflang); @@ -225,8 +221,7 @@ int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) { if (feof(f)) break; - log_error_errno(errno, "Failed to read file %s: %m", path); - return -errno; + return log_error_errno(errno, "Failed to read file %s: %m", path); } n++; @@ -323,8 +318,8 @@ int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) { return 0; } -static long write_catalog(const char *database, Hashmap *h, struct strbuf *sb, - CatalogItem *items, size_t n) { +static int64_t write_catalog(const char *database, struct strbuf *sb, + CatalogItem *items, size_t n) { CatalogHeader header; _cleanup_fclose_ FILE *w = NULL; int r; @@ -348,7 +343,7 @@ static long write_catalog(const char *database, Hashmap *h, struct strbuf *sb, memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature)); header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8)); header.catalog_item_size = htole64(sizeof(CatalogItem)); - header.n_items = htole64(hashmap_size(h)); + header.n_items = htole64(n); r = -EIO; @@ -383,7 +378,7 @@ static long write_catalog(const char *database, Hashmap *h, struct strbuf *sb, goto error; } - return ftell(w); + return ftello(w); error: (void) unlink(p); @@ -399,7 +394,8 @@ int catalog_update(const char* database, const char* root, const char* const* di CatalogItem *i; Iterator j; unsigned n; - long r; + int r; + int64_t sz; h = hashmap_new(&catalog_hash_ops); sb = strbuf_new(); @@ -419,8 +415,7 @@ int catalog_update(const char* database, const char* root, const char* const* di log_debug("Reading file '%s'", *f); r = catalog_import_file(h, sb, *f); if (r < 0) { - log_error("Failed to import file '%s': %s.", - *f, strerror(-r)); + log_error_errno(r, "Failed to import file '%s': %m", *f); goto finish; } } @@ -450,18 +445,19 @@ int catalog_update(const char* database, const char* root, const char* const* di assert(n == hashmap_size(h)); qsort_safe(items, n, sizeof(CatalogItem), catalog_compare_func); - r = write_catalog(database, h, sb, items, n); - if (r < 0) - log_error_errno(r, "Failed to write %s: %m", database); - else - log_debug("%s: wrote %u items, with %zu bytes of strings, %ld total size.", - database, n, sb->len, r); + sz = write_catalog(database, sb, items, n); + if (sz < 0) + r = log_error_errno(sz, "Failed to write %s: %m", database); + else { + r = 0; + log_debug("%s: wrote %u items, with %zu bytes of strings, %"PRIi64" total size.", + database, n, sb->len, sz); + } finish: - if (sb) - strbuf_cleanup(sb); + strbuf_cleanup(sb); - return r < 0 ? r : 0; + return r; } static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) { @@ -676,8 +672,7 @@ int catalog_list_items(FILE *f, const char *database, bool oneline, char **items k = sd_id128_from_string(*item, &id); if (k < 0) { - log_error_errno(k, "Failed to parse id128 '%s': %m", - *item); + log_error_errno(k, "Failed to parse id128 '%s': %m", *item); if (r == 0) r = k; continue; @@ -685,9 +680,8 @@ int catalog_list_items(FILE *f, const char *database, bool oneline, char **items k = catalog_get(database, id, &msg); if (k < 0) { - log_full(k == -ENOENT ? LOG_NOTICE : LOG_ERR, - "Failed to retrieve catalog entry for '%s': %s", - *item, strerror(-k)); + log_full_errno(k == -ENOENT ? LOG_NOTICE : LOG_ERR, k, + "Failed to retrieve catalog entry for '%s': %m", *item); if (r == 0) r = k; continue; diff --git a/src/journal/catalog.h b/src/journal/catalog.h index a72ecf6de7..bcc73c2631 100644 --- a/src/journal/catalog.h +++ b/src/journal/catalog.h @@ -24,6 +24,7 @@ #include <stdbool.h> #include "sd-id128.h" + #include "hashmap.h" #include "strbuf.h" diff --git a/src/journal/compress.c b/src/journal/compress.c index c66043e503..78935fee74 100644 --- a/src/journal/compress.c +++ b/src/journal/compress.c @@ -21,21 +21,33 @@ #include <stdlib.h> #include <string.h> +#include <sys/mman.h> #include <unistd.h> #ifdef HAVE_XZ -# include <lzma.h> +#include <lzma.h> #endif #ifdef HAVE_LZ4 -# include <lz4.h> +#include <lz4.h> +#include <lz4frame.h> #endif +#include "alloc-util.h" #include "compress.h" +#include "fd-util.h" +#include "io-util.h" +#include "journal-def.h" #include "macro.h" -#include "util.h" #include "sparse-endian.h" -#include "journal-def.h" +#include "string-table.h" +#include "string-util.h" +#include "util.h" + +#ifdef HAVE_LZ4 +DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_compressionContext_t, LZ4F_freeCompressionContext); +DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_decompressionContext_t, LZ4F_freeDecompressionContext); +#endif #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t)) @@ -46,14 +58,16 @@ static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = { DEFINE_STRING_TABLE_LOOKUP(object_compressed, int); -int compress_blob_xz(const void *src, uint64_t src_size, void *dst, size_t *dst_size) { +int compress_blob_xz(const void *src, uint64_t src_size, + void *dst, size_t dst_alloc_size, size_t *dst_size) { #ifdef HAVE_XZ static const lzma_options_lzma opt = { 1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT, - LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4}; - static const lzma_filter filters[2] = { - {LZMA_FILTER_LZMA2, (lzma_options_lzma*) &opt}, - {LZMA_VLI_UNKNOWN, NULL} + LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4 + }; + static const lzma_filter filters[] = { + { LZMA_FILTER_LZMA2, (lzma_options_lzma*) &opt }, + { LZMA_VLI_UNKNOWN, NULL } }; lzma_ret ret; size_t out_pos = 0; @@ -61,6 +75,7 @@ int compress_blob_xz(const void *src, uint64_t src_size, void *dst, size_t *dst_ assert(src); assert(src_size > 0); assert(dst); + assert(dst_alloc_size > 0); assert(dst_size); /* Returns < 0 if we couldn't compress the data or the @@ -70,7 +85,7 @@ int compress_blob_xz(const void *src, uint64_t src_size, void *dst, size_t *dst_ return -ENOBUFS; ret = lzma_stream_buffer_encode((lzma_filter*) filters, LZMA_CHECK_NONE, NULL, - src, src_size, dst, &out_pos, src_size - 1); + src, src_size, dst, &out_pos, dst_alloc_size); if (ret != LZMA_OK) return -ENOBUFS; @@ -81,13 +96,15 @@ int compress_blob_xz(const void *src, uint64_t src_size, void *dst, size_t *dst_ #endif } -int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, size_t *dst_size) { +int compress_blob_lz4(const void *src, uint64_t src_size, + void *dst, size_t dst_alloc_size, size_t *dst_size) { #ifdef HAVE_LZ4 int r; assert(src); assert(src_size > 0); assert(dst); + assert(dst_alloc_size > 0); assert(dst_size); /* Returns < 0 if we couldn't compress the data or the @@ -96,7 +113,7 @@ int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, size_t *dst if (src_size < 9) return -ENOBUFS; - r = LZ4_compress_limitedOutput(src, dst + 8, src_size, src_size - 8 - 1); + r = LZ4_compress_limitedOutput(src, dst + 8, src_size, (int) dst_alloc_size - 8); if (r <= 0) return -ENOBUFS; @@ -188,7 +205,7 @@ int decompress_blob_lz4(const void *src, uint64_t src_size, return -EBADMSG; size = le64toh( *(le64_t*)src ); - if (size < 0 || (le64_t) size != *(le64_t*)src) + if (size < 0 || (unsigned) size != le64toh(*(le64_t*)src)) return -EFBIG; if ((size_t) size > *dst_alloc_size) { out = realloc(*dst, size); @@ -293,6 +310,7 @@ int decompress_startswith_lz4(const void *src, uint64_t src_size, * prefix */ int r; + size_t size; assert(src); assert(src_size > 0); @@ -309,10 +327,18 @@ int decompress_startswith_lz4(const void *src, uint64_t src_size, r = LZ4_decompress_safe_partial(src + 8, *buffer, src_size - 8, prefix_len + 1, *buffer_size); + if (r >= 0) + size = (unsigned) r; + else { + /* lz4 always tries to decode full "sequence", so in + * pathological cases might need to decompress the + * full field. */ + r = decompress_blob_lz4(src, src_size, buffer, buffer_size, &size, 0); + if (r < 0) + return r; + } - if (r < 0) - return -EBADMSG; - if ((unsigned) r >= prefix_len + 1) + if (size >= prefix_len + 1) return memcmp(*buffer, prefix, prefix_len) == 0 && ((const uint8_t*) *buffer)[prefix_len] == extra; else @@ -416,81 +442,96 @@ int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes) { #endif } -#define LZ4_BUFSIZE (512*1024) +#define LZ4_BUFSIZE (512*1024u) int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) { #ifdef HAVE_LZ4 + LZ4F_errorCode_t c; + _cleanup_(LZ4F_freeCompressionContextp) LZ4F_compressionContext_t ctx = NULL; + _cleanup_free_ char *buf = NULL; + char *src = NULL; + size_t size, n, total_in = 0, total_out, offset = 0, frame_size; + struct stat st; + int r; + static const LZ4F_compressOptions_t options = { + .stableSrc = 1, + }; + static const LZ4F_preferences_t preferences = { + .frameInfo.blockSizeID = 5, + }; - _cleanup_free_ char *buf1 = NULL, *buf2 = NULL, *out = NULL; - char *buf; - LZ4_stream_t lz4_data = {}; - le32_t header; - size_t total_in = 0, total_out = sizeof(header); - ssize_t n; + c = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); + if (LZ4F_isError(c)) + return -ENOMEM; - assert(fdf >= 0); - assert(fdt >= 0); + if (fstat(fdf, &st) < 0) + return log_debug_errno(errno, "fstat() failed: %m"); - buf1 = malloc(LZ4_BUFSIZE); - buf2 = malloc(LZ4_BUFSIZE); - out = malloc(LZ4_COMPRESSBOUND(LZ4_BUFSIZE)); - if (!buf1 || !buf2 || !out) - return log_oom(); + frame_size = LZ4F_compressBound(LZ4_BUFSIZE, &preferences); + size = frame_size + 64*1024; /* add some space for header and trailer */ + buf = malloc(size); + if (!buf) + return -ENOMEM; - buf = buf1; - for (;;) { - size_t m; - int r; + n = offset = total_out = LZ4F_compressBegin(ctx, buf, size, &preferences); + if (LZ4F_isError(n)) + return -EINVAL; - m = LZ4_BUFSIZE; - if (max_bytes != (uint64_t) -1 && (uint64_t) m > (max_bytes - total_in)) - m = (size_t) (max_bytes - total_in); + src = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fdf, 0); + if (src == MAP_FAILED) + return -errno; - n = read(fdf, buf, m); - if (n < 0) - return -errno; - if (n == 0) - break; + log_debug("Buffer size is %zu bytes, header size %zu bytes.", size, n); - total_in += n; + while (total_in < (size_t) st.st_size) { + ssize_t k; - r = LZ4_compress_continue(&lz4_data, buf, out, n); - if (r == 0) { - log_error("LZ4 compression failed."); - return -EBADMSG; + k = MIN(LZ4_BUFSIZE, st.st_size - total_in); + n = LZ4F_compressUpdate(ctx, buf + offset, size - offset, + src + total_in, k, &options); + if (LZ4F_isError(n)) { + r = -ENOTRECOVERABLE; + goto cleanup; } - header = htole32(r); - errno = 0; - - n = write(fdt, &header, sizeof(header)); - if (n < 0) - return -errno; - if (n != sizeof(header)) - return errno ? -errno : -EIO; + total_in += k; + offset += n; + total_out += n; - n = loop_write(fdt, out, r, false); - if (n < 0) - return n; + if (max_bytes != (uint64_t) -1 && total_out > (size_t) max_bytes) { + log_debug("Compressed stream longer than %zd bytes", max_bytes); + return -EFBIG; + } - total_out += sizeof(header) + r; + if (size - offset < frame_size + 4) { + k = loop_write(fdt, buf, offset, false); + if (k < 0) { + r = k; + goto cleanup; + } + offset = 0; + } + } - buf = buf == buf1 ? buf2 : buf1; + n = LZ4F_compressEnd(ctx, buf + offset, size - offset, &options); + if (LZ4F_isError(n)) { + r = -ENOTRECOVERABLE; + goto cleanup; } - header = htole32(0); - n = write(fdt, &header, sizeof(header)); - if (n < 0) - return -errno; - if (n != sizeof(header)) - return errno ? -errno : -EIO; + offset += n; + total_out += n; + r = loop_write(fdt, buf, offset, false); + if (r < 0) + goto cleanup; log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)", total_in, total_out, (double) total_out / total_in * 100); - - return 0; + cleanup: + munmap(src, st.st_size); + return r; #else return -EPROTONOSUPPORT; #endif @@ -510,7 +551,7 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) { ret = lzma_stream_decoder(&s, UINT64_MAX, 0); if (ret != LZMA_OK) { - log_error("Failed to initialize XZ decoder: code %u", ret); + log_debug("Failed to initialize XZ decoder: code %u", ret); return -ENOMEM; } @@ -536,7 +577,7 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) { ret = lzma_code(&s, action); if (ret != LZMA_OK && ret != LZMA_STREAM_END) { - log_error("Decompression failed: code %u", ret); + log_debug("Decompression failed: code %u", ret); return -EBADMSG; } @@ -566,82 +607,68 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) { } } #else - log_error("Cannot decompress file. Compiled without XZ support."); + log_debug("Cannot decompress file. Compiled without XZ support."); return -EPROTONOSUPPORT; #endif } -int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) { - +int decompress_stream_lz4(int in, int out, uint64_t max_bytes) { #ifdef HAVE_LZ4 - _cleanup_free_ char *buf = NULL, *out = NULL; - size_t buf_size = 0; - LZ4_streamDecode_t lz4_data = {}; - le32_t header; - size_t total_in = sizeof(header), total_out = 0; - - assert(fdf >= 0); - assert(fdt >= 0); + size_t c; + _cleanup_(LZ4F_freeDecompressionContextp) LZ4F_decompressionContext_t ctx = NULL; + _cleanup_free_ char *buf = NULL; + char *src; + struct stat st; + int r = 0; + size_t total_in = 0, total_out = 0; + + c = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); + if (LZ4F_isError(c)) + return -ENOMEM; - out = malloc(4*LZ4_BUFSIZE); - if (!out) - return log_oom(); + if (fstat(in, &st) < 0) + return log_debug_errno(errno, "fstat() failed: %m"); - for (;;) { - ssize_t m; - int r; + buf = malloc(LZ4_BUFSIZE); + if (!buf) + return -ENOMEM; - r = loop_read_exact(fdf, &header, sizeof(header), false); - if (r < 0) - return r; + src = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, in, 0); + if (src == MAP_FAILED) + return -errno; - m = le32toh(header); - if (m == 0) - break; + while (total_in < (size_t) st.st_size) { + size_t produced = LZ4_BUFSIZE; + size_t used = st.st_size - total_in; - /* We refuse to use a bigger decompression buffer than - * the one used for compression by 4 times. This means - * that compression buffer size can be enlarged 4 - * times. This can be changed, but old binaries might - * not accept buffers compressed by newer binaries then. - */ - if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) { - log_error("Compressed stream block too big: %zd bytes", m); - return -EBADMSG; + c = LZ4F_decompress(ctx, buf, &produced, src + total_in, &used, NULL); + if (LZ4F_isError(c)) { + r = -EBADMSG; + goto cleanup; } - total_in += sizeof(header) + m; - - if (!GREEDY_REALLOC(buf, buf_size, m)) - return log_oom(); - - r = loop_read_exact(fdf, buf, m, false); - if (r < 0) - return r; - - r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE); - if (r <= 0) - log_error("LZ4 decompression failed."); + total_in += used; + total_out += produced; - total_out += r; - - if (max_bytes != (uint64_t) -1 && (uint64_t) total_out > max_bytes) { - log_debug("Decompressed stream longer than %" PRIu64 " bytes", max_bytes); - return -EFBIG; + if (max_bytes != (uint64_t) -1 && total_out > (size_t) max_bytes) { + log_debug("Decompressed stream longer than %zd bytes", max_bytes); + r = -EFBIG; + goto cleanup; } - r = loop_write(fdt, out, r, false); + r = loop_write(out, buf, produced, false); if (r < 0) - return r; + goto cleanup; } log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)", total_in, total_out, (double) total_out / total_in * 100); - - return 0; + cleanup: + munmap(src, st.st_size); + return r; #else - log_error("Cannot decompress file. Compiled without LZ4 support."); + log_debug("Cannot decompress file. Compiled without LZ4 support."); return -EPROTONOSUPPORT; #endif } diff --git a/src/journal/compress.h b/src/journal/compress.h index 9a065eb763..758598730a 100644 --- a/src/journal/compress.h +++ b/src/journal/compress.h @@ -28,17 +28,20 @@ const char* object_compressed_to_string(int compression); int object_compressed_from_string(const char *compression); -int compress_blob_xz(const void *src, uint64_t src_size, void *dst, size_t *dst_size); -int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, size_t *dst_size); +int compress_blob_xz(const void *src, uint64_t src_size, + void *dst, size_t dst_alloc_size, size_t *dst_size); +int compress_blob_lz4(const void *src, uint64_t src_size, + void *dst, size_t dst_alloc_size, size_t *dst_size); -static inline int compress_blob(const void *src, uint64_t src_size, void *dst, size_t *dst_size) { +static inline int compress_blob(const void *src, uint64_t src_size, + void *dst, size_t dst_alloc_size, size_t *dst_size) { int r; #ifdef HAVE_LZ4 - r = compress_blob_lz4(src, src_size, dst, dst_size); + r = compress_blob_lz4(src, src_size, dst, dst_alloc_size, dst_size); if (r == 0) return OBJECT_COMPRESSED_LZ4; #else - r = compress_blob_xz(src, src_size, dst, dst_size); + r = compress_blob_xz(src, src_size, dst, dst_alloc_size, dst_size); if (r == 0) return OBJECT_COMPRESSED_XZ; #endif diff --git a/src/journal/coredump-vacuum.c b/src/journal/coredump-vacuum.c index efe418615a..09ab60c6c4 100644 --- a/src/journal/coredump-vacuum.c +++ b/src/journal/coredump-vacuum.c @@ -21,12 +21,16 @@ #include <sys/statvfs.h> -#include "util.h" -#include "time-util.h" +#include "alloc-util.h" +#include "coredump-vacuum.h" +#include "dirent-util.h" +#include "fd-util.h" #include "hashmap.h" #include "macro.h" - -#include "coredump-vacuum.h" +#include "string-util.h" +#include "time-util.h" +#include "user-util.h" +#include "util.h" #define DEFAULT_MAX_USE_LOWER (uint64_t) (1ULL*1024ULL*1024ULL) /* 1 MiB */ #define DEFAULT_MAX_USE_UPPER (uint64_t) (4ULL*1024ULL*1024ULL*1024ULL) /* 4 GiB */ @@ -153,8 +157,7 @@ int coredump_vacuum(int exclude_fd, uint64_t keep_free, uint64_t max_use) { if (errno == ENOENT) return 0; - log_error_errno(errno, "Can't open coredump directory: %m"); - return -errno; + return log_error_errno(errno, "Can't open coredump directory: %m"); } for (;;) { @@ -179,7 +182,7 @@ int coredump_vacuum(int exclude_fd, uint64_t keep_free, uint64_t max_use) { if (errno == ENOENT) continue; - log_warning("Failed to stat /var/lib/systemd/coredump/%s", de->d_name); + log_warning_errno(errno, "Failed to stat /var/lib/systemd/coredump/%s: %m", de->d_name); continue; } @@ -197,7 +200,7 @@ int coredump_vacuum(int exclude_fd, uint64_t keep_free, uint64_t max_use) { t = timespec_load(&st.st_mtim); - c = hashmap_get(h, UINT32_TO_PTR(uid)); + c = hashmap_get(h, UID_TO_PTR(uid)); if (c) { if (t < c->oldest_mtime) { @@ -225,7 +228,7 @@ int coredump_vacuum(int exclude_fd, uint64_t keep_free, uint64_t max_use) { n->oldest_mtime = t; - r = hashmap_put(h, UINT32_TO_PTR(uid), n); + r = hashmap_put(h, UID_TO_PTR(uid), n); if (r < 0) return log_oom(); @@ -255,8 +258,7 @@ int coredump_vacuum(int exclude_fd, uint64_t keep_free, uint64_t max_use) { if (errno == ENOENT) continue; - log_error_errno(errno, "Failed to remove file %s: %m", worst->oldest_file); - return -errno; + return log_error_errno(errno, "Failed to remove file %s: %m", worst->oldest_file); } else log_info("Removed old coredump %s.", worst->oldest_file); } @@ -264,6 +266,5 @@ int coredump_vacuum(int exclude_fd, uint64_t keep_free, uint64_t max_use) { return 0; fail: - log_error_errno(errno, "Failed to read directory: %m"); - return -errno; + return log_error_errno(errno, "Failed to read directory: %m"); } diff --git a/src/journal/coredump.c b/src/journal/coredump.c index e1e66b9826..869c8fea03 100644 --- a/src/journal/coredump.c +++ b/src/journal/coredump.c @@ -20,10 +20,10 @@ ***/ #include <errno.h> -#include <unistd.h> #include <stdio.h> #include <sys/prctl.h> #include <sys/xattr.h> +#include <unistd.h> #ifdef HAVE_ELFUTILS # include <dwarf.h> @@ -32,23 +32,34 @@ #include "sd-journal.h" #include "sd-login.h" -#include "log.h" -#include "util.h" -#include "fileio.h" -#include "strv.h" -#include "macro.h" -#include "mkdir.h" -#include "special.h" + +#include "acl-util.h" +#include "alloc-util.h" +#include "capability-util.h" #include "cgroup-util.h" +#include "compress.h" #include "conf-parser.h" #include "copy.h" -#include "stacktrace.h" -#include "compress.h" -#include "acl-util.h" -#include "capability.h" -#include "journald-native.h" #include "coredump-vacuum.h" +#include "dirent-util.h" +#include "escape.h" +#include "fd-util.h" +#include "fileio.h" +#include "fs-util.h" +#include "io-util.h" +#include "journald-native.h" +#include "log.h" +#include "macro.h" +#include "mkdir.h" +#include "parse-util.h" #include "process-util.h" +#include "special.h" +#include "stacktrace.h" +#include "string-table.h" +#include "string-util.h" +#include "strv.h" +#include "user-util.h" +#include "util.h" /* The maximum size up to which we process coredumps */ #define PROCESS_SIZE_MAX ((uint64_t) (2LLU*1024LLU*1024LLU*1024LLU)) @@ -115,8 +126,8 @@ static int parse_config(void) { {} }; - return config_parse_many("/etc/systemd/coredump.conf", - CONF_DIRS_NULSTR("systemd/coredump.conf"), + return config_parse_many(PKGSYSCONFDIR "/coredump.conf", + CONF_PATHS_NULSTR("systemd/coredump.conf.d"), "Coredump\0", config_item_table_lookup, items, false, NULL); @@ -128,6 +139,7 @@ static int fix_acl(int fd, uid_t uid) { _cleanup_(acl_freep) acl_t acl = NULL; acl_entry_t entry; acl_permset_t permset; + int r; assert(fd >= 0); @@ -149,11 +161,12 @@ static int fix_acl(int fd, uid_t uid) { } if (acl_get_permset(entry, &permset) < 0 || - acl_add_perm(permset, ACL_READ) < 0 || - calc_acl_mask_if_needed(&acl) < 0) { - log_warning_errno(errno, "Failed to patch ACL: %m"); - return -errno; - } + acl_add_perm(permset, ACL_READ) < 0) + return log_warning_errno(errno, "Failed to patch ACL: %m"); + + r = calc_acl_mask_if_needed(&acl); + if (r < 0) + return log_warning_errno(r, "Failed to patch ACL: %m"); if (acl_set_fd(fd, acl) < 0) return log_error_errno(errno, "Failed to apply ACL: %m"); @@ -514,7 +527,7 @@ static int compose_open_fds(pid_t pid, char **open_fds) { errno = 0; stream = safe_fclose(stream); - if (errno != 0) + if (errno > 0) return -errno; *open_fds = buffer; diff --git a/src/journal/coredumpctl.c b/src/journal/coredumpctl.c index 6628e82421..40ffa6afbe 100644 --- a/src/journal/coredumpctl.c +++ b/src/journal/coredumpctl.c @@ -19,27 +19,33 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ +#include <fcntl.h> +#include <getopt.h> #include <locale.h> #include <stdio.h> #include <string.h> -#include <getopt.h> -#include <fcntl.h> #include <unistd.h> #include "sd-journal.h" -#include "build.h" -#include "set.h" -#include "util.h" + +#include "alloc-util.h" +#include "compress.h" +#include "fd-util.h" +#include "fileio.h" +#include "journal-internal.h" #include "log.h" -#include "path-util.h" -#include "pager.h" #include "macro.h" -#include "journal-internal.h" -#include "compress.h" -#include "sigbus.h" +#include "pager.h" +#include "parse-util.h" +#include "path-util.h" #include "process-util.h" -#include "terminal-util.h" +#include "set.h" +#include "sigbus.h" #include "signal-util.h" +#include "string-util.h" +#include "terminal-util.h" +#include "user-util.h" +#include "util.h" static enum { ACTION_NONE, @@ -84,37 +90,35 @@ static Set *new_matches(void) { } static int add_match(Set *set, const char *match) { - int r = -ENOMEM; - unsigned pid; - const char* prefix; - char *pattern = NULL; _cleanup_free_ char *p = NULL; + char *pattern = NULL; + const char* prefix; + pid_t pid; + int r; if (strchr(match, '=')) prefix = ""; else if (strchr(match, '/')) { - p = path_make_absolute_cwd(match); - if (!p) + r = path_make_absolute_cwd(match, &p); + if (r < 0) goto fail; - match = p; prefix = "COREDUMP_EXE="; - } - else if (safe_atou(match, &pid) == 0) + } else if (parse_pid(match, &pid) >= 0) prefix = "COREDUMP_PID="; else prefix = "COREDUMP_COMM="; pattern = strjoin(prefix, match, NULL); - if (!pattern) + if (!pattern) { + r = -ENOMEM; goto fail; + } log_debug("Adding pattern: %s", pattern); r = set_consume(set, pattern); - if (r < 0) { - log_error_errno(r, "Failed to add pattern: %m"); + if (r < 0) goto fail; - } return 0; fail: @@ -175,9 +179,7 @@ static int parse_argv(int argc, char *argv[], Set *matches) { case ARG_VERSION: arg_action = ACTION_NONE; - puts(PACKAGE_STRING); - puts(SYSTEMD_FEATURES); - return 0; + return version(); case ARG_NO_PAGER: arg_no_pager = true; @@ -615,7 +617,7 @@ static int save_core(sd_journal *j, int fd, char **path, bool *unlink_temp) { fdt = mkostemp_safe(temp, O_WRONLY|O_CLOEXEC); if (fdt < 0) - return log_error_errno(errno, "Failed to create temporary file: %m"); + return log_error_errno(fdt, "Failed to create temporary file: %m"); log_debug("Created temporary file %s", temp); fd = fdt; @@ -774,7 +776,7 @@ static int run_gdb(sd_journal *j) { r = wait_for_terminate(pid, &st); if (r < 0) { - log_error_errno(errno, "Failed to wait for gdb: %m"); + log_error_errno(r, "Failed to wait for gdb: %m"); goto finish; } @@ -790,7 +792,7 @@ finish: } int main(int argc, char *argv[]) { - _cleanup_journal_close_ sd_journal*j = NULL; + _cleanup_(sd_journal_closep) sd_journal*j = NULL; const char* match; Iterator it; int r = 0; diff --git a/src/journal/fsprg.h b/src/journal/fsprg.h index 150d034828..b79221fc2e 100644 --- a/src/journal/fsprg.h +++ b/src/journal/fsprg.h @@ -25,10 +25,11 @@ * */ -#include <sys/types.h> #include <inttypes.h> +#include <sys/types.h> #include "macro.h" +#include "util.h" #ifdef __cplusplus extern "C" { diff --git a/src/journal/journal-authenticate.c b/src/journal/journal-authenticate.c index cdc80e2d26..aeec83da1e 100644 --- a/src/journal/journal-authenticate.c +++ b/src/journal/journal-authenticate.c @@ -22,10 +22,12 @@ #include <fcntl.h> #include <sys/mman.h> +#include "fd-util.h" +#include "fsprg.h" +#include "hexdecoct.h" +#include "journal-authenticate.h" #include "journal-def.h" #include "journal-file.h" -#include "journal-authenticate.h" -#include "fsprg.h" static uint64_t journal_file_tag_seqnum(JournalFile *f) { uint64_t r; diff --git a/src/journal/journal-def.h b/src/journal/journal-def.h index 39c9dd0dbf..c003ac05dd 100644 --- a/src/journal/journal-def.h +++ b/src/journal/journal-def.h @@ -21,11 +21,10 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include "sparse-endian.h" - -#include "systemd/sd-id128.h" +#include "sd-id128.h" #include "macro.h" +#include "sparse-endian.h" /* * If you change this file you probably should also change its documentation: diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c index 73d3a4bb9d..9e362bacae 100644 --- a/src/journal/journal-file.c +++ b/src/journal/journal-file.c @@ -19,22 +19,28 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <sys/mman.h> #include <errno.h> -#include <sys/uio.h> -#include <unistd.h> -#include <sys/statvfs.h> #include <fcntl.h> -#include <stddef.h> #include <linux/fs.h> +#include <stddef.h> +#include <sys/mman.h> +#include <sys/statvfs.h> +#include <sys/uio.h> +#include <unistd.h> +#include "alloc-util.h" #include "btrfs-util.h" +#include "chattr-util.h" +#include "compress.h" +#include "fd-util.h" +#include "journal-authenticate.h" #include "journal-def.h" #include "journal-file.h" -#include "journal-authenticate.h" #include "lookup3.h" -#include "compress.h" +#include "parse-util.h" #include "random-util.h" +#include "string-util.h" +#include "xattr-util.h" #define DEFAULT_DATA_HASH_TABLE_SIZE (2047ULL*sizeof(HashItem)) #define DEFAULT_FIELD_HASH_TABLE_SIZE (333ULL*sizeof(HashItem)) @@ -42,13 +48,16 @@ #define COMPRESSION_SIZE_THRESHOLD (512ULL) /* This is the minimum journal file size */ -#define JOURNAL_FILE_SIZE_MIN (4ULL*1024ULL*1024ULL) /* 4 MiB */ +#define JOURNAL_FILE_SIZE_MIN (512ULL*1024ULL) /* 512 KiB */ /* These are the lower and upper bounds if we deduce the max_use value * from the file system size */ #define DEFAULT_MAX_USE_LOWER (1ULL*1024ULL*1024ULL) /* 1 MiB */ #define DEFAULT_MAX_USE_UPPER (4ULL*1024ULL*1024ULL*1024ULL) /* 4 GiB */ +/* This is the default minimal use limit, how much we'll use even if keep_free suggests otherwise. */ +#define DEFAULT_MIN_USE (1ULL*1024ULL*1024ULL) /* 1 MiB */ + /* This is the upper bound if we deduce max_size from max_use */ #define DEFAULT_MAX_SIZE_UPPER (128ULL*1024ULL*1024ULL) /* 128 MiB */ @@ -60,6 +69,9 @@ * size */ #define DEFAULT_KEEP_FREE (1024ULL*1024ULL) /* 1 MB */ +/* This is the default maximum number of journal files to keep around. */ +#define DEFAULT_N_MAX_FILES (100) + /* n_data was the first entry we added after the initial file format design */ #define HEADER_SIZE_MIN ALIGN64(offsetof(Header, n_data)) @@ -128,7 +140,7 @@ int journal_file_set_offline(JournalFile *f) { return 0; } -void journal_file_close(JournalFile *f) { +JournalFile* journal_file_close(JournalFile *f) { assert(f); #ifdef HAVE_GCRYPT @@ -157,8 +169,7 @@ void journal_file_close(JournalFile *f) { safe_close(f->fd); free(f->path); - if (f->mmap) - mmap_cache_unref(f->mmap); + mmap_cache_unref(f->mmap); ordered_hashmap_free_free(f->chain_cache); @@ -179,6 +190,7 @@ void journal_file_close(JournalFile *f) { #endif free(f); + return NULL; } static int journal_file_init_header(JournalFile *f, JournalFile *template) { @@ -398,12 +410,7 @@ static int journal_file_allocate(JournalFile *f, uint64_t offset, uint64_t size) if (fstatvfs(f->fd, &svfs) >= 0) { uint64_t available; - available = svfs.f_bfree * svfs.f_bsize; - - if (available >= f->metrics.keep_free) - available -= f->metrics.keep_free; - else - available = 0; + available = LESS_BY((uint64_t) svfs.f_bfree * (uint64_t) svfs.f_bsize, f->metrics.keep_free); if (new_size - old_size > available) return -E2BIG; @@ -604,10 +611,10 @@ static int journal_file_setup_data_hash_table(JournalFile *f) { assert(f); - /* We estimate that we need 1 hash table entry per 768 of - journal file and we want to make sure we never get beyond - 75% fill level. Calculate the hash table size for the - maximum file size based on these metrics. */ + /* We estimate that we need 1 hash table entry per 768 bytes + of journal file and we want to make sure we never get + beyond 75% fill level. Calculate the hash table size for + the maximum file size based on these metrics. */ s = (f->metrics.max_size * 4 / 768 / 3) * sizeof(HashItem); if (s < DEFAULT_DATA_HASH_TABLE_SIZE) @@ -1055,7 +1062,7 @@ static int journal_file_append_data( r = journal_file_find_data_object_with_hash(f, data, size, hash, &o, &p); if (r < 0) return r; - else if (r > 0) { + if (r > 0) { if (ret) *ret = o; @@ -1074,23 +1081,24 @@ static int journal_file_append_data( o->data.hash = htole64(hash); #if defined(HAVE_XZ) || defined(HAVE_LZ4) - if (f->compress_xz && - size >= COMPRESSION_SIZE_THRESHOLD) { + if (JOURNAL_FILE_COMPRESS(f) && size >= COMPRESSION_SIZE_THRESHOLD) { size_t rsize = 0; - compression = compress_blob(data, size, o->data.payload, &rsize); + compression = compress_blob(data, size, o->data.payload, size - 1, &rsize); - if (compression) { + if (compression >= 0) { o->object.size = htole64(offsetof(Object, data.payload) + rsize); o->object.flags |= compression; log_debug("Compressed data object %"PRIu64" -> %zu using %s", size, rsize, object_compressed_to_string(compression)); - } + } else + /* Compression didn't work, we don't really care why, let's continue without compression */ + compression = 0; } #endif - if (!compression && size > 0) + if (compression == 0 && size > 0) memcpy(o->data.payload, data, size); r = journal_file_link_data(f, o, p, hash); @@ -2696,7 +2704,7 @@ int journal_file_open( } if (f->last_stat.st_size < (off_t) HEADER_SIZE_MIN) { - r = -EIO; + r = -ENODATA; goto fail; } @@ -2833,8 +2841,7 @@ int journal_file_open_reliably( size_t l; _cleanup_free_ char *p = NULL; - r = journal_file_open(fname, flags, mode, compress, seal, - metrics, mmap_cache, template, ret); + r = journal_file_open(fname, flags, mode, compress, seal, metrics, mmap_cache, template, ret); if (!IN_SET(r, -EBADMSG, /* corrupted */ -ENODATA, /* truncated */ @@ -2864,8 +2871,7 @@ int journal_file_open_reliably( random_u64()) < 0) return -ENOMEM; - r = rename(fname, p); - if (r < 0) + if (rename(fname, p) < 0) return -errno; /* btrfs doesn't cope well with our write pattern and @@ -2874,10 +2880,9 @@ int journal_file_open_reliably( (void) chattr_path(p, false, FS_NOCOW_FL); (void) btrfs_defrag(p); - log_warning("File %s corrupted or uncleanly shut down, renaming and replacing.", fname); + log_warning_errno(r, "File %s corrupted or uncleanly shut down, renaming and replacing.", fname); - return journal_file_open(fname, flags, mode, compress, seal, - metrics, mmap_cache, template, ret); + return journal_file_open(fname, flags, mode, compress, seal, metrics, mmap_cache, template, ret); } int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint64_t p, uint64_t *seqnum, Object **ret, uint64_t *offset) { @@ -2964,16 +2969,35 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6 return r; } +void journal_reset_metrics(JournalMetrics *m) { + assert(m); + + /* Set everything to "pick automatic values". */ + + *m = (JournalMetrics) { + .min_use = (uint64_t) -1, + .max_use = (uint64_t) -1, + .min_size = (uint64_t) -1, + .max_size = (uint64_t) -1, + .keep_free = (uint64_t) -1, + .n_max_files = (uint64_t) -1, + }; +} + void journal_default_metrics(JournalMetrics *m, int fd) { - uint64_t fs_size = 0; + char a[FORMAT_BYTES_MAX], b[FORMAT_BYTES_MAX], c[FORMAT_BYTES_MAX], d[FORMAT_BYTES_MAX], e[FORMAT_BYTES_MAX]; struct statvfs ss; - char a[FORMAT_BYTES_MAX], b[FORMAT_BYTES_MAX], c[FORMAT_BYTES_MAX], d[FORMAT_BYTES_MAX]; + uint64_t fs_size; assert(m); assert(fd >= 0); if (fstatvfs(fd, &ss) >= 0) fs_size = ss.f_frsize * ss.f_blocks; + else { + log_debug_errno(errno, "Failed to detremine disk size: %m"); + fs_size = 0; + } if (m->max_use == (uint64_t) -1) { @@ -2990,10 +3014,16 @@ void journal_default_metrics(JournalMetrics *m, int fd) { } else { m->max_use = PAGE_ALIGN(m->max_use); - if (m->max_use < JOURNAL_FILE_SIZE_MIN*2) + if (m->max_use != 0 && m->max_use < JOURNAL_FILE_SIZE_MIN*2) m->max_use = JOURNAL_FILE_SIZE_MIN*2; } + if (m->min_use == (uint64_t) -1) + m->min_use = DEFAULT_MIN_USE; + + if (m->min_use > m->max_use) + m->min_use = m->max_use; + if (m->max_size == (uint64_t) -1) { m->max_size = PAGE_ALIGN(m->max_use / 8); /* 8 chunks */ @@ -3002,11 +3032,13 @@ void journal_default_metrics(JournalMetrics *m, int fd) { } else m->max_size = PAGE_ALIGN(m->max_size); - if (m->max_size < JOURNAL_FILE_SIZE_MIN) - m->max_size = JOURNAL_FILE_SIZE_MIN; + if (m->max_size != 0) { + if (m->max_size < JOURNAL_FILE_SIZE_MIN) + m->max_size = JOURNAL_FILE_SIZE_MIN; - if (m->max_size*2 > m->max_use) - m->max_use = m->max_size*2; + if (m->max_use != 0 && m->max_size*2 > m->max_use) + m->max_use = m->max_size*2; + } if (m->min_size == (uint64_t) -1) m->min_size = JOURNAL_FILE_SIZE_MIN; @@ -3016,7 +3048,7 @@ void journal_default_metrics(JournalMetrics *m, int fd) { if (m->min_size < JOURNAL_FILE_SIZE_MIN) m->min_size = JOURNAL_FILE_SIZE_MIN; - if (m->min_size > m->max_size) + if (m->max_size != 0 && m->min_size > m->max_size) m->max_size = m->min_size; } @@ -3032,11 +3064,16 @@ void journal_default_metrics(JournalMetrics *m, int fd) { m->keep_free = DEFAULT_KEEP_FREE; } - log_debug("Fixed max_use=%s max_size=%s min_size=%s keep_free=%s", - format_bytes(a, sizeof(a), m->max_use), - format_bytes(b, sizeof(b), m->max_size), - format_bytes(c, sizeof(c), m->min_size), - format_bytes(d, sizeof(d), m->keep_free)); + if (m->n_max_files == (uint64_t) -1) + m->n_max_files = DEFAULT_N_MAX_FILES; + + log_debug("Fixed min_use=%s max_use=%s max_size=%s min_size=%s keep_free=%s n_max_files=%" PRIu64, + format_bytes(a, sizeof(a), m->min_use), + format_bytes(b, sizeof(b), m->max_use), + format_bytes(c, sizeof(c), m->max_size), + format_bytes(d, sizeof(d), m->min_size), + format_bytes(e, sizeof(e), m->keep_free), + m->n_max_files); } int journal_file_get_cutoff_realtime_usec(JournalFile *f, usec_t *from, usec_t *to) { diff --git a/src/journal/journal-file.h b/src/journal/journal-file.h index e92b75eabe..46c1f3278e 100644 --- a/src/journal/journal-file.h +++ b/src/journal/journal-file.h @@ -29,18 +29,20 @@ #include "sd-id128.h" -#include "sparse-endian.h" +#include "hashmap.h" #include "journal-def.h" #include "macro.h" #include "mmap-cache.h" -#include "hashmap.h" +#include "sparse-endian.h" typedef struct JournalMetrics { - uint64_t max_use; - uint64_t use; - uint64_t max_size; - uint64_t min_size; - uint64_t keep_free; + /* For all these: -1 means "pick automatically", and 0 means "no limit enforced" */ + uint64_t max_size; /* how large journal files grow at max */ + uint64_t min_size; /* how large journal files grow at least */ + uint64_t max_use; /* how much disk space to use in total at max, keep_free permitting */ + uint64_t min_use; /* how much disk space to use in total at least, even if keep_free says not to */ + uint64_t keep_free; /* how much to keep free on disk */ + uint64_t n_max_files; /* how many files to keep around at max */ } JournalMetrics; typedef enum direction { @@ -136,7 +138,7 @@ int journal_file_open( JournalFile **ret); int journal_file_set_offline(JournalFile *f); -void journal_file_close(JournalFile *j); +JournalFile* journal_file_close(JournalFile *j); int journal_file_open_reliably( const char *fname, @@ -223,6 +225,7 @@ int journal_file_rotate(JournalFile **f, bool compress, bool seal); void journal_file_post_change(JournalFile *f); +void journal_reset_metrics(JournalMetrics *m); void journal_default_metrics(JournalMetrics *m, int fd); int journal_file_get_cutoff_realtime_usec(JournalFile *f, usec_t *from, usec_t *to); @@ -232,3 +235,8 @@ bool journal_file_rotate_suggested(JournalFile *f, usec_t max_file_usec); int journal_file_map_data_hash_table(JournalFile *f); int journal_file_map_field_hash_table(JournalFile *f); + +static inline bool JOURNAL_FILE_COMPRESS(JournalFile *f) { + assert(f); + return f->compress_xz || f->compress_lz4; +} diff --git a/src/journal/journal-internal.h b/src/journal/journal-internal.h index b51ecdb600..9ff4fea7fb 100644 --- a/src/journal/journal-internal.h +++ b/src/journal/journal-internal.h @@ -21,18 +21,18 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <sys/types.h> #include <inttypes.h> #include <stdbool.h> +#include <sys/types.h> -#include "systemd/sd-id128.h" +#include "sd-id128.h" +#include "sd-journal.h" +#include "hashmap.h" #include "journal-def.h" +#include "journal-file.h" #include "list.h" -#include "hashmap.h" #include "set.h" -#include "journal-file.h" -#include "sd-journal.h" typedef struct Match Match; typedef struct Location Location; @@ -121,14 +121,11 @@ struct sd_journal { Hashmap *directories_by_path; Hashmap *directories_by_wd; - Set *errors; + Hashmap *errors; }; char *journal_make_match_string(sd_journal *j); void journal_print_header(sd_journal *j); -DEFINE_TRIVIAL_CLEANUP_FUNC(sd_journal*, sd_journal_close); -#define _cleanup_journal_close_ _cleanup_(sd_journal_closep) - #define JOURNAL_FOREACH_DATA_RETVAL(j, data, l, retval) \ for (sd_journal_restart_data(j); ((retval) = sd_journal_enumerate_data((j), &(data), &(l))) > 0; ) diff --git a/src/journal/journal-qrcode.c b/src/journal/journal-qrcode.c index 1db66e89c6..257ddb302b 100644 --- a/src/journal/journal-qrcode.c +++ b/src/journal/journal-qrcode.c @@ -20,12 +20,11 @@ ***/ #include <assert.h> -#include <stdio.h> #include <errno.h> -#include <stdlib.h> -#include <stdbool.h> - #include <qrencode.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> #include "journal-qrcode.h" diff --git a/src/journal/journal-qrcode.h b/src/journal/journal-qrcode.h index 3ff6a3ad4a..7d14e8754b 100644 --- a/src/journal/journal-qrcode.h +++ b/src/journal/journal-qrcode.h @@ -21,8 +21,9 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ +#include <inttypes.h> #include <stdio.h> -#include "systemd/sd-id128.h" +#include "sd-id128.h" int print_qr_code(FILE *f, const void *seed, size_t seed_size, uint64_t start, uint64_t interval, const char *hn, sd_id128_t machine); diff --git a/src/journal/journal-send.c b/src/journal/journal-send.c index 1e3a463504..44fa11a00e 100644 --- a/src/journal/journal-send.c +++ b/src/journal/journal-send.c @@ -19,20 +19,27 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <sys/socket.h> -#include <sys/un.h> #include <errno.h> -#include <stddef.h> -#include <unistd.h> #include <fcntl.h> #include <printf.h> +#include <stddef.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <unistd.h> #define SD_JOURNAL_SUPPRESS_LOCATION #include "sd-journal.h" -#include "util.h" -#include "socket-util.h" + +#include "alloc-util.h" +#include "fd-util.h" +#include "fileio.h" +#include "io-util.h" #include "memfd-util.h" +#include "socket-util.h" +#include "stdio-util.h" +#include "string-util.h" +#include "util.h" #define SNDBUF_SIZE (8*1024*1024) @@ -212,19 +219,14 @@ _public_ int sd_journal_sendv(const struct iovec *iov, int n) { .msg_namelen = offsetof(struct sockaddr_un, sun_path) + strlen(sa.sun_path), }; ssize_t k; - union { - struct cmsghdr cmsghdr; - uint8_t buf[CMSG_SPACE(sizeof(int))]; - } control; - struct cmsghdr *cmsg; bool have_syslog_identifier = false; bool seal = true; assert_return(iov, -EINVAL); assert_return(n > 0, -EINVAL); - w = alloca(sizeof(struct iovec) * n * 5 + 3); - l = alloca(sizeof(uint64_t) * n); + w = newa(struct iovec, n * 5 + 3); + l = newa(uint64_t, n); for (i = 0; i < n; i++) { char *c, *nl; @@ -335,26 +337,11 @@ _public_ int sd_journal_sendv(const struct iovec *iov, int n) { return r; } - mh.msg_iov = NULL; - mh.msg_iovlen = 0; - - zero(control); - mh.msg_control = &control; - mh.msg_controllen = sizeof(control); - - cmsg = CMSG_FIRSTHDR(&mh); - cmsg->cmsg_level = SOL_SOCKET; - cmsg->cmsg_type = SCM_RIGHTS; - cmsg->cmsg_len = CMSG_LEN(sizeof(int)); - memcpy(CMSG_DATA(cmsg), &buffer_fd, sizeof(int)); - - mh.msg_controllen = cmsg->cmsg_len; - - k = sendmsg(fd, &mh, MSG_NOSIGNAL); - if (k < 0) - return -errno; - - return 0; + r = send_one_fd_sa(fd, buffer_fd, mh.msg_name, mh.msg_namelen, 0); + if (r == -ENOENT) + /* Fail silently if the journal is not available */ + return 0; + return r; } static int fill_iovec_perror_and_send(const char *message, int skip, struct iovec iov[]) { diff --git a/src/journal/journal-vacuum.c b/src/journal/journal-vacuum.c index 17499bbc30..4b5fc76eb1 100644 --- a/src/journal/journal-vacuum.c +++ b/src/journal/journal-vacuum.c @@ -23,20 +23,27 @@ #include <sys/stat.h> #include <unistd.h> +#include "sd-id128.h" + +#include "alloc-util.h" +#include "dirent-util.h" +#include "fd-util.h" #include "journal-def.h" #include "journal-file.h" #include "journal-vacuum.h" -#include "sd-id128.h" +#include "parse-util.h" +#include "string-util.h" #include "util.h" +#include "xattr-util.h" struct vacuum_info { uint64_t usage; char *filename; uint64_t realtime; + sd_id128_t seqnum_id; uint64_t seqnum; - bool have_seqnum; }; @@ -67,19 +74,18 @@ static int vacuum_compare(const void *_a, const void *_b) { } static void patch_realtime( - const char *dir, + int fd, const char *fn, const struct stat *st, unsigned long long *realtime) { - _cleanup_free_ char *path = NULL; usec_t x, crtime = 0; /* The timestamp was determined by the file name, but let's * see if the file might actually be older than the file name * suggested... */ - assert(dir); + assert(fd >= 0); assert(fn); assert(st); assert(realtime); @@ -101,14 +107,7 @@ static void patch_realtime( * unfortunately there's currently no sane API to query * it. Hence let's implement this manually... */ - /* Unfortunately there is is not fgetxattrat(), so we need to - * go via path here. :-( */ - - path = strjoin(dir, "/", fn, NULL); - if (!path) - return; - - if (path_getcrtime(path, &crtime) >= 0) { + if (fd_getcrtime_at(fd, fn, &crtime, 0) >= 0) { if (crtime < *realtime) *realtime = crtime; } @@ -120,9 +119,13 @@ static int journal_file_empty(int dir_fd, const char *name) { le64_t n_entries; ssize_t n; - fd = openat(dir_fd, name, O_RDONLY|O_CLOEXEC|O_NOFOLLOW|O_NONBLOCK); - if (fd < 0) - return -errno; + fd = openat(dir_fd, name, O_RDONLY|O_CLOEXEC|O_NOFOLLOW|O_NONBLOCK|O_NOATIME); + if (fd < 0) { + /* Maybe failed due to O_NOATIME and lack of privileges? */ + fd = openat(dir_fd, name, O_RDONLY|O_CLOEXEC|O_NOFOLLOW|O_NONBLOCK); + if (fd < 0) + return -errno; + } if (fstat(fd, &st) < 0) return -errno; @@ -144,22 +147,24 @@ static int journal_file_empty(int dir_fd, const char *name) { int journal_directory_vacuum( const char *directory, uint64_t max_use, + uint64_t n_max_files, usec_t max_retention_usec, usec_t *oldest_usec, bool verbose) { _cleanup_closedir_ DIR *d = NULL; - int r = 0; struct vacuum_info *list = NULL; - unsigned n_list = 0, i; + unsigned n_list = 0, i, n_active_files = 0; size_t n_allocated = 0; uint64_t sum = 0, freed = 0; usec_t retention_limit = 0; char sbytes[FORMAT_BYTES_MAX]; + struct dirent *de; + int r; assert(directory); - if (max_use <= 0 && max_retention_usec <= 0) + if (max_use <= 0 && max_retention_usec <= 0 && n_max_files <= 0) return 0; if (max_retention_usec > 0) { @@ -174,27 +179,20 @@ int journal_directory_vacuum( if (!d) return -errno; - for (;;) { - struct dirent *de; - size_t q; - struct stat st; - char *p; + FOREACH_DIRENT_ALL(de, d, r = -errno; goto finish) { + unsigned long long seqnum = 0, realtime; + _cleanup_free_ char *p = NULL; sd_id128_t seqnum_id; bool have_seqnum; + uint64_t size; + struct stat st; + size_t q; - errno = 0; - de = readdir(d); - if (!de && errno != 0) { - r = -errno; - goto finish; - } - - if (!de) - break; - - if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) + if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) { + log_debug_errno(errno, "Failed to stat file %s while vacuuming, ignoring: %m", de->d_name); continue; + } if (!S_ISREG(st.st_mode)) continue; @@ -203,15 +201,20 @@ int journal_directory_vacuum( if (endswith(de->d_name, ".journal")) { - /* Vacuum archived files */ + /* Vacuum archived files. Active files are + * left around */ - if (q < 1 + 32 + 1 + 16 + 1 + 16 + 8) + if (q < 1 + 32 + 1 + 16 + 1 + 16 + 8) { + n_active_files++; continue; + } if (de->d_name[q-8-16-1] != '-' || de->d_name[q-8-16-1-16-1] != '-' || - de->d_name[q-8-16-1-16-1-32-1] != '@') + de->d_name[q-8-16-1-16-1-32-1] != '@') { + n_active_files++; continue; + } p = strdup(de->d_name); if (!p) { @@ -221,12 +224,12 @@ int journal_directory_vacuum( de->d_name[q-8-16-1-16-1] = 0; if (sd_id128_from_string(de->d_name + q-8-16-1-16-1-32, &seqnum_id) < 0) { - free(p); + n_active_files++; continue; } if (sscanf(de->d_name + q-8-16-1-16, "%16llx-%16llx.journal", &seqnum, &realtime) != 2) { - free(p); + n_active_files++; continue; } @@ -237,12 +240,16 @@ int journal_directory_vacuum( /* Vacuum corrupted files */ - if (q < 1 + 16 + 1 + 16 + 8 + 1) + if (q < 1 + 16 + 1 + 16 + 8 + 1) { + n_active_files ++; continue; + } if (de->d_name[q-1-8-16-1] != '-' || - de->d_name[q-1-8-16-1-16-1] != '@') + de->d_name[q-1-8-16-1-16-1] != '@') { + n_active_files ++; continue; + } p = strdup(de->d_name); if (!p) { @@ -251,55 +258,68 @@ int journal_directory_vacuum( } if (sscanf(de->d_name + q-1-8-16-1-16, "%16llx-%16llx.journal~", &realtime, &tmp) != 2) { - free(p); + n_active_files ++; continue; } have_seqnum = false; - } else - /* We do not vacuum active files or unknown files! */ + } else { + /* We do not vacuum unknown files! */ + log_debug("Not vacuuming unknown file %s.", de->d_name); continue; + } - if (journal_file_empty(dirfd(d), p)) { - /* Always vacuum empty non-online files. */ + size = 512UL * (uint64_t) st.st_blocks; - uint64_t size = 512UL * (uint64_t) st.st_blocks; + r = journal_file_empty(dirfd(d), p); + if (r < 0) { + log_debug_errno(r, "Failed check if %s is empty, ignoring: %m", p); + continue; + } + if (r > 0) { + /* Always vacuum empty non-online files. */ if (unlinkat(dirfd(d), p, 0) >= 0) { - log_full(verbose ? LOG_INFO : LOG_DEBUG, "Deleted empty archived journal %s/%s (%s).", directory, p, format_bytes(sbytes, sizeof(sbytes), size)); + + log_full(verbose ? LOG_INFO : LOG_DEBUG, + "Deleted empty archived journal %s/%s (%s).", directory, p, format_bytes(sbytes, sizeof(sbytes), size)); + freed += size; } else if (errno != ENOENT) log_warning_errno(errno, "Failed to delete empty archived journal %s/%s: %m", directory, p); - free(p); continue; } - patch_realtime(directory, p, &st, &realtime); + patch_realtime(dirfd(d), p, &st, &realtime); if (!GREEDY_REALLOC(list, n_allocated, n_list + 1)) { - free(p); r = -ENOMEM; goto finish; } list[n_list].filename = p; - list[n_list].usage = 512UL * (uint64_t) st.st_blocks; + list[n_list].usage = size; list[n_list].seqnum = seqnum; list[n_list].realtime = realtime; list[n_list].seqnum_id = seqnum_id; list[n_list].have_seqnum = have_seqnum; - - sum += list[n_list].usage; - n_list ++; + + p = NULL; + sum += size; } qsort_safe(list, n_list, sizeof(struct vacuum_info), vacuum_compare); for (i = 0; i < n_list; i++) { + unsigned left; + + left = n_active_files + n_list - i; + if ((max_retention_usec <= 0 || list[i].realtime >= retention_limit) && - (max_use <= 0 || sum <= max_use)) + (max_use <= 0 || sum <= max_use) && + (n_max_files <= 0 || left <= n_max_files)) break; if (unlinkat(dirfd(d), list[i].filename, 0) >= 0) { @@ -318,6 +338,8 @@ int journal_directory_vacuum( if (oldest_usec && i < n_list && (*oldest_usec == 0 || list[i].realtime < *oldest_usec)) *oldest_usec = list[i].realtime; + r = 0; + finish: for (i = 0; i < n_list; i++) free(list[i].filename); diff --git a/src/journal/journal-vacuum.h b/src/journal/journal-vacuum.h index c45cc31d0e..49ab90af91 100644 --- a/src/journal/journal-vacuum.h +++ b/src/journal/journal-vacuum.h @@ -21,5 +21,9 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ +#include <inttypes.h> +#include <stdbool.h> -int journal_directory_vacuum(const char *directory, uint64_t max_use, usec_t max_retention_usec, usec_t *oldest_usec, bool vacuum); +#include "time-util.h" + +int journal_directory_vacuum(const char *directory, uint64_t max_use, uint64_t n_max_files, usec_t max_retention_usec, usec_t *oldest_usec, bool verbose); diff --git a/src/journal/journal-verify.c b/src/journal/journal-verify.c index 32d59c716f..715847e018 100644 --- a/src/journal/journal-verify.c +++ b/src/journal/journal-verify.c @@ -19,20 +19,23 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <unistd.h> -#include <sys/mman.h> #include <fcntl.h> #include <stddef.h> +#include <sys/mman.h> +#include <unistd.h> -#include "util.h" -#include "macro.h" +#include "alloc-util.h" +#include "compress.h" +#include "fd-util.h" +#include "fileio.h" +#include "journal-authenticate.h" #include "journal-def.h" #include "journal-file.h" -#include "journal-authenticate.h" #include "journal-verify.h" #include "lookup3.h" -#include "compress.h" +#include "macro.h" #include "terminal-util.h" +#include "util.h" static void draw_progress(uint64_t p, usec_t *last_usec) { unsigned n, i, j, k; @@ -839,19 +842,19 @@ int journal_file_verify( data_fd = open_tmpfile("/var/tmp", O_RDWR | O_CLOEXEC); if (data_fd < 0) { - r = log_error_errno(errno, "Failed to create data file: %m"); + r = log_error_errno(data_fd, "Failed to create data file: %m"); goto fail; } entry_fd = open_tmpfile("/var/tmp", O_RDWR | O_CLOEXEC); if (entry_fd < 0) { - r = log_error_errno(errno, "Failed to create entry file: %m"); + r = log_error_errno(entry_fd, "Failed to create entry file: %m"); goto fail; } entry_array_fd = open_tmpfile("/var/tmp", O_RDWR | O_CLOEXEC); if (entry_array_fd < 0) { - r = log_error_errno(errno, + r = log_error_errno(entry_array_fd, "Failed to create entry array file: %m"); goto fail; } @@ -897,7 +900,7 @@ int journal_file_verify( r = journal_file_object_verify(f, p, o); if (r < 0) { - error(p, "Envalid object contents: %s", strerror(-r)); + error(p, "Invalid object contents: %s", strerror(-r)); goto fail; } diff --git a/src/journal/journalctl.c b/src/journal/journalctl.c index b38b151485..db11421e7a 100644 --- a/src/journal/journalctl.c +++ b/src/journal/journalctl.c @@ -19,48 +19,58 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <locale.h> +#include <errno.h> #include <fcntl.h> #include <fnmatch.h> -#include <errno.h> +#include <getopt.h> +#include <linux/fs.h> +#include <locale.h> +#include <poll.h> +#include <signal.h> #include <stddef.h> -#include <string.h> #include <stdio.h> -#include <unistd.h> #include <stdlib.h> -#include <getopt.h> -#include <signal.h> -#include <poll.h> -#include <sys/stat.h> +#include <string.h> #include <sys/inotify.h> -#include <linux/fs.h> +#include <sys/stat.h> +#include <unistd.h> -#include "sd-journal.h" #include "sd-bus.h" -#include "log.h" -#include "logs-show.h" -#include "util.h" +#include "sd-journal.h" + #include "acl-util.h" -#include "path-util.h" +#include "alloc-util.h" +#include "bus-error.h" +#include "bus-util.h" +#include "catalog.h" +#include "chattr-util.h" +#include "fd-util.h" #include "fileio.h" -#include "build.h" -#include "pager.h" -#include "strv.h" -#include "set.h" -#include "sigbus.h" -#include "journal-internal.h" +#include "fs-util.h" +#include "fsprg.h" +#include "glob-util.h" +#include "hostname-util.h" +#include "io-util.h" #include "journal-def.h" -#include "journal-verify.h" +#include "journal-internal.h" #include "journal-qrcode.h" #include "journal-vacuum.h" -#include "fsprg.h" -#include "unit-name.h" -#include "catalog.h" +#include "journal-verify.h" +#include "locale-util.h" +#include "log.h" +#include "logs-show.h" #include "mkdir.h" -#include "bus-util.h" -#include "bus-error.h" +#include "pager.h" +#include "parse-util.h" +#include "path-util.h" +#include "rlimit-util.h" +#include "set.h" +#include "sigbus.h" +#include "strv.h" +#include "syslog-util.h" #include "terminal-util.h" -#include "hostname-util.h" +#include "unit-name.h" +#include "user-util.h" #define DEFAULT_FSS_INTERVAL_USEC (15*USEC_PER_MINUTE) @@ -105,10 +115,11 @@ static const char *arg_field = NULL; static bool arg_catalog = false; static bool arg_reverse = false; static int arg_journal_type = 0; -static const char *arg_root = NULL; +static char *arg_root = NULL; static const char *arg_machine = NULL; -static uint64_t arg_vacuum_size = (uint64_t) -1; -static usec_t arg_vacuum_time = USEC_INFINITY; +static uint64_t arg_vacuum_size = 0; +static uint64_t arg_vacuum_n_files = 0; +static usec_t arg_vacuum_time = 0; static enum { ACTION_SHOW, @@ -122,6 +133,8 @@ static enum { ACTION_UPDATE_CATALOG, ACTION_LIST_BOOTS, ACTION_FLUSH, + ACTION_SYNC, + ACTION_ROTATE, ACTION_VACUUM, } arg_action = ACTION_SHOW; @@ -189,12 +202,12 @@ static void help(void) { printf("%s [OPTIONS...] [MATCHES...]\n\n" "Query the journal.\n\n" - "Flags:\n" + "Options:\n" " --system Show the system journal\n" " --user Show the user journal for the current user\n" " -M --machine=CONTAINER Operate on local container\n" - " --since=DATE Show entries not older than the specified date\n" - " --until=DATE Show entries not newer than the specified date\n" + " -S --since=DATE Show entries not older than the specified date\n" + " -U --until=DATE Show entries not newer than the specified date\n" " -c --cursor=CURSOR Show entries starting at the specified cursor\n" " --after-cursor=CURSOR Show entries after the specified cursor\n" " --show-cursor Print the cursor after all the entries\n" @@ -217,12 +230,12 @@ static void help(void) { " -x --catalog Add message explanations where available\n" " --no-full Ellipsize fields\n" " -a --all Show all fields, including long and unprintable\n" - " -q --quiet Do not show privilege warning\n" + " -q --quiet Do not show info messages and privilege warning\n" " --no-pager Do not pipe output into a pager\n" " -m --merge Show entries from all available journals\n" " -D --directory=PATH Show journal files from directory\n" " --file=PATH Show journal file\n" - " --root=ROOT Operate on catalog files underneath the root ROOT\n" + " --root=ROOT Operate on catalog files below a root directory\n" #ifdef HAVE_GCRYPT " --interval=TIME Time interval for changing the FSS sealing key\n" " --verify-key=KEY Specify FSS verification key\n" @@ -232,18 +245,21 @@ static void help(void) { " -h --help Show this help text\n" " --version Show package version\n" " -F --field=FIELD List all values that a specified field takes\n" - " --new-id128 Generate a new 128-bit ID\n" " --disk-usage Show total disk usage of all journal files\n" " --vacuum-size=BYTES Reduce disk usage below specified size\n" - " --vacuum-time=TIME Remove journal files older than specified date\n" + " --vacuum-files=INT Leave only the specified number of journal files\n" + " --vacuum-time=TIME Remove journal files older than specified time\n" + " --verify Verify journal file consistency\n" + " --sync Synchronize unwritten journal messages to disk\n" " --flush Flush all journal data from /run into /var\n" + " --rotate Request immediate rotation of the journal files\n" " --header Show journal header information\n" " --list-catalog Show all message IDs in the catalog\n" " --dump-catalog Show entries in the message catalog\n" " --update-catalog Update the message catalog database\n" + " --new-id128 Generate a new 128-bit ID\n" #ifdef HAVE_GCRYPT " --setup-keys Generate a new FSS key pair\n" - " --verify Verify journal file consistency\n" #endif , program_invocation_short_name); } @@ -267,8 +283,6 @@ static int parse_argv(int argc, char *argv[]) { ARG_VERIFY, ARG_VERIFY_KEY, ARG_DISK_USAGE, - ARG_SINCE, - ARG_UNTIL, ARG_AFTER_CURSOR, ARG_SHOW_CURSOR, ARG_USER_UNIT, @@ -277,8 +291,11 @@ static int parse_argv(int argc, char *argv[]) { ARG_UPDATE_CATALOG, ARG_FORCE, ARG_UTC, + ARG_SYNC, ARG_FLUSH, + ARG_ROTATE, ARG_VACUUM_SIZE, + ARG_VACUUM_FILES, ARG_VACUUM_TIME, }; @@ -318,8 +335,8 @@ static int parse_argv(int argc, char *argv[]) { { "cursor", required_argument, NULL, 'c' }, { "after-cursor", required_argument, NULL, ARG_AFTER_CURSOR }, { "show-cursor", no_argument, NULL, ARG_SHOW_CURSOR }, - { "since", required_argument, NULL, ARG_SINCE }, - { "until", required_argument, NULL, ARG_UNTIL }, + { "since", required_argument, NULL, 'S' }, + { "until", required_argument, NULL, 'U' }, { "unit", required_argument, NULL, 'u' }, { "user-unit", required_argument, NULL, ARG_USER_UNIT }, { "field", required_argument, NULL, 'F' }, @@ -331,7 +348,10 @@ static int parse_argv(int argc, char *argv[]) { { "machine", required_argument, NULL, 'M' }, { "utc", no_argument, NULL, ARG_UTC }, { "flush", no_argument, NULL, ARG_FLUSH }, + { "sync", no_argument, NULL, ARG_SYNC }, + { "rotate", no_argument, NULL, ARG_ROTATE }, { "vacuum-size", required_argument, NULL, ARG_VACUUM_SIZE }, + { "vacuum-files", required_argument, NULL, ARG_VACUUM_FILES }, { "vacuum-time", required_argument, NULL, ARG_VACUUM_TIME }, {} }; @@ -341,7 +361,7 @@ static int parse_argv(int argc, char *argv[]) { assert(argc >= 0); assert(argv); - while ((c = getopt_long(argc, argv, "hefo:aln::qmb::kD:p:c:t:u:F:xrM:", options, NULL)) >= 0) + while ((c = getopt_long(argc, argv, "hefo:aln::qmb::kD:p:c:S:U:t:u:F:xrM:", options, NULL)) >= 0) switch (c) { @@ -350,9 +370,7 @@ static int parse_argv(int argc, char *argv[]) { return 0; case ARG_VERSION: - puts(PACKAGE_STRING); - puts(SYSTEMD_FEATURES); - return 0; + return version(); case ARG_NO_PAGER: arg_no_pager = true; @@ -502,7 +520,9 @@ static int parse_argv(int argc, char *argv[]) { break; case ARG_ROOT: - arg_root = optarg; + r = parse_path_argument_and_warn(optarg, true, &arg_root); + if (r < 0) + return r; break; case 'c': @@ -539,6 +559,16 @@ static int parse_argv(int argc, char *argv[]) { arg_action = ACTION_VACUUM; break; + case ARG_VACUUM_FILES: + r = safe_atou64(optarg, &arg_vacuum_n_files); + if (r < 0) { + log_error("Failed to parse vacuum files: %s", optarg); + return r; + } + + arg_action = ACTION_VACUUM; + break; + case ARG_VACUUM_TIME: r = parse_sec(optarg, &arg_vacuum_time); if (r < 0) { @@ -631,7 +661,7 @@ static int parse_argv(int argc, char *argv[]) { break; } - case ARG_SINCE: + case 'S': r = parse_timestamp(optarg, &arg_since); if (r < 0) { log_error("Failed to parse timestamp: %s", optarg); @@ -640,7 +670,7 @@ static int parse_argv(int argc, char *argv[]) { arg_since_set = true; break; - case ARG_UNTIL: + case 'U': r = parse_timestamp(optarg, &arg_until); if (r < 0) { log_error("Failed to parse timestamp: %s", optarg); @@ -699,6 +729,14 @@ static int parse_argv(int argc, char *argv[]) { arg_action = ACTION_FLUSH; break; + case ARG_ROTATE: + arg_action = ACTION_ROTATE; + break; + + case ARG_SYNC: + arg_action = ACTION_SYNC; + break; + case '?': return -EINVAL; @@ -729,7 +767,7 @@ static int parse_argv(int argc, char *argv[]) { return -EINVAL; } - if (arg_action != ACTION_SHOW && optind < argc) { + if (!IN_SET(arg_action, ACTION_SHOW, ACTION_DUMP_CATALOG, ACTION_LIST_CATALOG) && optind < argc) { log_error("Extraneous arguments starting with '%s'", argv[optind]); return -EINVAL; } @@ -1453,7 +1491,7 @@ static int setup_keys(void) { safe_close(fd); fd = mkostemp_safe(k, O_WRONLY|O_CLOEXEC); if (fd < 0) { - r = log_error_errno(errno, "Failed to open %s: %m", k); + r = log_error_errno(fd, "Failed to open %s: %m", k); goto finish; } @@ -1461,7 +1499,7 @@ static int setup_keys(void) { * writing and in-place updating */ r = chattr_fd(fd, FS_SECRM_FL|FS_NODUMP_FL|FS_SYNC_FL|FS_NOCOW_FL, FS_SECRM_FL|FS_NODUMP_FL|FS_SYNC_FL|FS_NOCOW_FL); if (r < 0) - log_warning_errno(errno, "Failed to set file attributes: %m"); + log_warning_errno(r, "Failed to set file attributes: %m"); zero(h); memcpy(h.signature, "KSHHRHLP", 8); @@ -1580,7 +1618,7 @@ static int verify(sd_journal *j) { /* If the key was invalid give up right-away. */ return k; } else if (k < 0) { - log_warning("FAIL: %s (%s)", f->path, strerror(-k)); + log_warning_errno(k, "FAIL: %s (%m)", f->path); r = k; } else { char a[FORMAT_TIMESTAMP_MAX], b[FORMAT_TIMESTAMP_MAX], c[FORMAT_TIMESPAN_MAX]; @@ -1678,54 +1716,73 @@ static int access_check_var_log_journal(sd_journal *j) { static int access_check(sd_journal *j) { Iterator it; void *code; + char *path; int r = 0; assert(j); - if (set_isempty(j->errors)) { + if (hashmap_isempty(j->errors)) { if (ordered_hashmap_isempty(j->files)) log_notice("No journal files were found."); return 0; } - if (set_contains(j->errors, INT_TO_PTR(-EACCES))) { + if (hashmap_contains(j->errors, INT_TO_PTR(-EACCES))) { (void) access_check_var_log_journal(j); if (ordered_hashmap_isempty(j->files)) r = log_error_errno(EACCES, "No journal files were opened due to insufficient permissions."); } - SET_FOREACH(code, j->errors, it) { + HASHMAP_FOREACH_KEY(path, code, j->errors, it) { int err; - err = -PTR_TO_INT(code); - assert(err > 0); + err = abs(PTR_TO_INT(code)); - if (err == EACCES) + switch (err) { + case EACCES: continue; - log_warning_errno(err, "Error was encountered while opening journal files: %m"); - if (r == 0) - r = -err; + case ENODATA: + log_warning_errno(err, "Journal file %s is truncated, ignoring file.", path); + break; + + case EPROTONOSUPPORT: + log_warning_errno(err, "Journal file %s uses an unsupported feature, ignoring file.", path); + break; + + case EBADMSG: + log_warning_errno(err, "Journal file %s corrupted, ignoring file.", path); + break; + + default: + log_warning_errno(err, "An error was encountered while opening journal file %s, ignoring file.", path); + break; + } } return r; } static int flush_to_var(void) { - _cleanup_bus_error_free_ sd_bus_error error = SD_BUS_ERROR_NULL; - _cleanup_bus_flush_close_unref_ sd_bus *bus = NULL; + _cleanup_(sd_bus_error_free) sd_bus_error error = SD_BUS_ERROR_NULL; + _cleanup_(sd_bus_flush_close_unrefp) sd_bus *bus = NULL; _cleanup_close_ int watch_fd = -1; int r; + if (arg_machine) { + log_error("--flush is not supported in conjunction with --machine=."); + return -EOPNOTSUPP; + } + /* Quick exit */ if (access("/run/systemd/journal/flushed", F_OK) >= 0) return 0; /* OK, let's actually do the full logic, send SIGUSR1 to the * daemon and set up inotify to wait for the flushed file to appear */ - r = bus_open_system_systemd(&bus); + r = bus_connect_system_systemd(&bus); if (r < 0) return log_error_errno(r, "Failed to get D-Bus connection: %m"); @@ -1738,10 +1795,8 @@ static int flush_to_var(void) { &error, NULL, "ssi", "systemd-journald.service", "main", SIGUSR1); - if (r < 0) { - log_error("Failed to kill journal service: %s", bus_error_message(&error, r)); - return r; - } + if (r < 0) + return log_error_errno(r, "Failed to kill journal service: %s", bus_error_message(&error, r)); mkdir_p("/run/systemd/journal", 0755); @@ -1772,9 +1827,100 @@ static int flush_to_var(void) { return 0; } +static int send_signal_and_wait(int sig, const char *watch_path) { + _cleanup_(sd_bus_flush_close_unrefp) sd_bus *bus = NULL; + _cleanup_close_ int watch_fd = -1; + usec_t start; + int r; + + if (arg_machine) { + log_error("--sync and --rotate are not supported in conjunction with --machine=."); + return -EOPNOTSUPP; + } + + start = now(CLOCK_MONOTONIC); + + /* This call sends the specified signal to journald, and waits + * for acknowledgment by watching the mtime of the specified + * flag file. This is used to trigger syncing or rotation and + * then wait for the operation to complete. */ + + for (;;) { + usec_t tstamp; + + /* See if a sync happened by now. */ + r = read_timestamp_file(watch_path, &tstamp); + if (r < 0 && r != -ENOENT) + return log_error_errno(errno, "Failed to read %s: %m", watch_path); + if (r >= 0 && tstamp >= start) + return 0; + + /* Let's ask for a sync, but only once. */ + if (!bus) { + _cleanup_(sd_bus_error_free) sd_bus_error error = SD_BUS_ERROR_NULL; + + r = bus_connect_system_systemd(&bus); + if (r < 0) + return log_error_errno(r, "Failed to get D-Bus connection: %m"); + + r = sd_bus_call_method( + bus, + "org.freedesktop.systemd1", + "/org/freedesktop/systemd1", + "org.freedesktop.systemd1.Manager", + "KillUnit", + &error, + NULL, + "ssi", "systemd-journald.service", "main", sig); + if (r < 0) + return log_error_errno(r, "Failed to kill journal service: %s", bus_error_message(&error, r)); + + continue; + } + + /* Let's install the inotify watch, if we didn't do that yet. */ + if (watch_fd < 0) { + + mkdir_p("/run/systemd/journal", 0755); + + watch_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC); + if (watch_fd < 0) + return log_error_errno(errno, "Failed to create inotify watch: %m"); + + r = inotify_add_watch(watch_fd, "/run/systemd/journal", IN_MOVED_TO|IN_DONT_FOLLOW|IN_ONLYDIR); + if (r < 0) + return log_error_errno(errno, "Failed to watch journal directory: %m"); + + /* Recheck the flag file immediately, so that we don't miss any event since the last check. */ + continue; + } + + /* OK, all preparatory steps done, let's wait until + * inotify reports an event. */ + + r = fd_wait_for_event(watch_fd, POLLIN, USEC_INFINITY); + if (r < 0) + return log_error_errno(r, "Failed to wait for event: %m"); + + r = flush_fd(watch_fd); + if (r < 0) + return log_error_errno(r, "Failed to flush inotify events: %m"); + } + + return 0; +} + +static int rotate(void) { + return send_signal_and_wait(SIGUSR2, "/run/systemd/journal/rotated"); +} + +static int sync_journal(void) { + return send_signal_and_wait(SIGRTMIN+1, "/run/systemd/journal/synced"); +} + int main(int argc, char *argv[]) { int r; - _cleanup_journal_close_ sd_journal *j = NULL; + _cleanup_(sd_journal_closep) sd_journal *j = NULL; bool need_seek = false; sd_id128_t previous_boot_id; bool previous_boot_id_valid = false, first_line = true; @@ -1797,25 +1943,19 @@ int main(int argc, char *argv[]) { * be split up into many files. */ setrlimit_closest(RLIMIT_NOFILE, &RLIMIT_MAKE_CONST(16384)); - if (arg_action == ACTION_NEW_ID128) { - r = generate_new_id128(); - goto finish; - } + switch (arg_action) { - if (arg_action == ACTION_FLUSH) { - r = flush_to_var(); + case ACTION_NEW_ID128: + r = generate_new_id128(); goto finish; - } - if (arg_action == ACTION_SETUP_KEYS) { + case ACTION_SETUP_KEYS: r = setup_keys(); goto finish; - } - - if (arg_action == ACTION_UPDATE_CATALOG || - arg_action == ACTION_LIST_CATALOG || - arg_action == ACTION_DUMP_CATALOG) { + case ACTION_LIST_CATALOG: + case ACTION_DUMP_CATALOG: + case ACTION_UPDATE_CATALOG: { _cleanup_free_ char *database; database = path_join(arg_root, CATALOG_DATABASE, NULL); @@ -1831,9 +1971,10 @@ int main(int argc, char *argv[]) { } else { bool oneline = arg_action == ACTION_LIST_CATALOG; + pager_open_if_enabled(); + if (optind < argc) - r = catalog_list_items(stdout, database, - oneline, argv + optind); + r = catalog_list_items(stdout, database, oneline, argv + optind); else r = catalog_list(stdout, database, oneline); if (r < 0) @@ -1843,6 +1984,31 @@ int main(int argc, char *argv[]) { goto finish; } + case ACTION_FLUSH: + r = flush_to_var(); + goto finish; + + case ACTION_SYNC: + r = sync_journal(); + goto finish; + + case ACTION_ROTATE: + r = rotate(); + goto finish; + + case ACTION_SHOW: + case ACTION_PRINT_HEADER: + case ACTION_VERIFY: + case ACTION_DISK_USAGE: + case ACTION_LIST_BOOTS: + case ACTION_VACUUM: + /* These ones require access to the journal files, continue below. */ + break; + + default: + assert_not_reached("Unknown action"); + } + if (arg_directory) r = sd_journal_open_directory(&j, arg_directory, arg_journal_type); else if (arg_file) @@ -1852,8 +2018,7 @@ int main(int argc, char *argv[]) { else r = sd_journal_open(&j, !arg_merge*SD_JOURNAL_LOCAL_ONLY + arg_journal_type); if (r < 0) { - log_error_errno(r, "Failed to open %s: %m", - arg_directory ? arg_directory : arg_file ? "files" : "journal"); + log_error_errno(r, "Failed to open %s: %m", arg_directory ?: arg_file ? "files" : "journal"); goto finish; } @@ -1861,18 +2026,28 @@ int main(int argc, char *argv[]) { if (r < 0) goto finish; - if (arg_action == ACTION_VERIFY) { - r = verify(j); - goto finish; - } + switch (arg_action) { - if (arg_action == ACTION_PRINT_HEADER) { + case ACTION_NEW_ID128: + case ACTION_SETUP_KEYS: + case ACTION_LIST_CATALOG: + case ACTION_DUMP_CATALOG: + case ACTION_UPDATE_CATALOG: + case ACTION_FLUSH: + case ACTION_SYNC: + case ACTION_ROTATE: + assert_not_reached("Unexpected action."); + + case ACTION_PRINT_HEADER: journal_print_header(j); r = 0; goto finish; - } - if (arg_action == ACTION_DISK_USAGE) { + case ACTION_VERIFY: + r = verify(j); + goto finish; + + case ACTION_DISK_USAGE: { uint64_t bytes = 0; char sbytes[FORMAT_BYTES_MAX]; @@ -1885,7 +2060,11 @@ int main(int argc, char *argv[]) { goto finish; } - if (arg_action == ACTION_VACUUM) { + case ACTION_LIST_BOOTS: + r = list_boots(j); + goto finish; + + case ACTION_VACUUM: { Directory *d; Iterator i; @@ -1895,9 +2074,9 @@ int main(int argc, char *argv[]) { if (d->is_root) continue; - q = journal_directory_vacuum(d->path, arg_vacuum_size, arg_vacuum_time, NULL, true); + q = journal_directory_vacuum(d->path, arg_vacuum_size, arg_vacuum_n_files, arg_vacuum_time, NULL, true); if (q < 0) { - log_error_errno(q, "Failed to vacuum: %m"); + log_error_errno(q, "Failed to vacuum %s: %m", d->path); r = q; } } @@ -1905,9 +2084,11 @@ int main(int argc, char *argv[]) { goto finish; } - if (arg_action == ACTION_LIST_BOOTS) { - r = list_boots(j); - goto finish; + case ACTION_SHOW: + break; + + default: + assert_not_reached("Unknown action"); } /* add_boot() must be called first! @@ -2066,7 +2247,8 @@ int main(int argc, char *argv[]) { if (arg_follow) need_seek = true; else { - printf("-- No entries --\n"); + if (!arg_quiet) + printf("-- No entries --\n"); goto finish; } } @@ -2154,7 +2336,7 @@ int main(int argc, char *argv[]) { flags = arg_all * OUTPUT_SHOW_ALL | arg_full * OUTPUT_FULL_WIDTH | - on_tty() * OUTPUT_COLOR | + colors_enabled() * OUTPUT_COLOR | arg_catalog * OUTPUT_CATALOG | arg_utc * OUTPUT_UTC; @@ -2200,5 +2382,7 @@ finish: strv_free(arg_system_units); strv_free(arg_user_units); + free(arg_root); + return r < 0 ? EXIT_FAILURE : EXIT_SUCCESS; } diff --git a/src/journal/journald-audit.c b/src/journal/journald-audit.c index fe8ae194c9..3c13fe0d67 100644 --- a/src/journal/journald-audit.c +++ b/src/journal/journald-audit.c @@ -19,9 +19,14 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include "missing.h" -#include "journald-audit.h" +#include "alloc-util.h" #include "audit-type.h" +#include "fd-util.h" +#include "hexdecoct.h" +#include "io-util.h" +#include "journald-audit.h" +#include "missing.h" +#include "string-util.h" typedef struct MapField { const char *audit_field; diff --git a/src/journal/journald-audit.h b/src/journal/journald-audit.h index 68cdfb3410..5c88bb6383 100644 --- a/src/journal/journald-audit.h +++ b/src/journal/journald-audit.h @@ -21,8 +21,8 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include "socket-util.h" #include "journald-server.h" +#include "socket-util.h" void server_process_audit_message(Server *s, const void *buffer, size_t buffer_size, const struct ucred *ucred, const union sockaddr_union *sa, socklen_t salen); diff --git a/src/journal/journald-console.c b/src/journal/journald-console.c index 307bdc3949..04487c29b5 100644 --- a/src/journal/journald-console.c +++ b/src/journal/journald-console.c @@ -19,15 +19,20 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <time.h> #include <fcntl.h> #include <sys/socket.h> +#include <time.h> +#include "alloc-util.h" +#include "fd-util.h" #include "fileio.h" -#include "journald-server.h" -#include "journald-console.h" #include "formats-util.h" +#include "io-util.h" +#include "journald-console.h" +#include "journald-server.h" +#include "parse-util.h" #include "process-util.h" +#include "stdio-util.h" #include "terminal-util.h" static bool prefix_timestamp(void) { @@ -101,7 +106,7 @@ void server_forward_console( fd = open_terminal(tty, O_WRONLY|O_NOCTTY|O_CLOEXEC); if (fd < 0) { - log_debug_errno(errno, "Failed to open %s for logging: %m", tty); + log_debug_errno(fd, "Failed to open %s for logging: %m", tty); return; } diff --git a/src/journal/journald-gperf.gperf b/src/journal/journald-gperf.gperf index bf7c773009..c154610c54 100644 --- a/src/journal/journald-gperf.gperf +++ b/src/journal/journald-gperf.gperf @@ -24,9 +24,11 @@ Journal.RateLimitBurst, config_parse_unsigned, 0, offsetof(Server, rate_li Journal.SystemMaxUse, config_parse_iec_uint64, 0, offsetof(Server, system_metrics.max_use) Journal.SystemMaxFileSize, config_parse_iec_uint64, 0, offsetof(Server, system_metrics.max_size) Journal.SystemKeepFree, config_parse_iec_uint64, 0, offsetof(Server, system_metrics.keep_free) +Journal.SystemMaxFiles, config_parse_uint64, 0, offsetof(Server, system_metrics.n_max_files) Journal.RuntimeMaxUse, config_parse_iec_uint64, 0, offsetof(Server, runtime_metrics.max_use) Journal.RuntimeMaxFileSize, config_parse_iec_uint64, 0, offsetof(Server, runtime_metrics.max_size) Journal.RuntimeKeepFree, config_parse_iec_uint64, 0, offsetof(Server, runtime_metrics.keep_free) +Journal.RuntimeMaxFiles, config_parse_uint64, 0, offsetof(Server, runtime_metrics.n_max_files) Journal.MaxRetentionSec, config_parse_sec, 0, offsetof(Server, max_retention_usec) Journal.MaxFileSec, config_parse_sec, 0, offsetof(Server, max_file_usec) Journal.ForwardToSyslog, config_parse_bool, 0, offsetof(Server, forward_to_syslog) diff --git a/src/journal/journald-kmsg.c b/src/journal/journald-kmsg.c index 51fe3aa50a..e048e04716 100644 --- a/src/journal/journald-kmsg.c +++ b/src/journal/journald-kmsg.c @@ -19,20 +19,26 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <unistd.h> -#include <sys/epoll.h> #include <fcntl.h> +#include <sys/epoll.h> #include <sys/mman.h> #include <sys/socket.h> +#include <unistd.h> -#include "systemd/sd-messages.h" -#include <libudev.h> +#include "libudev.h" +#include "sd-messages.h" -#include "journald-server.h" +#include "escape.h" +#include "fd-util.h" +#include "formats-util.h" +#include "io-util.h" #include "journald-kmsg.h" +#include "journald-server.h" #include "journald-syslog.h" -#include "formats-util.h" +#include "parse-util.h" #include "process-util.h" +#include "stdio-util.h" +#include "string-util.h" void server_forward_kmsg( Server *s, @@ -341,8 +347,7 @@ static int server_read_dev_kmsg(Server *s) { if (errno == EAGAIN || errno == EINTR || errno == EPIPE) return 0; - log_error_errno(errno, "Failed to read from kernel: %m"); - return -errno; + return log_error_errno(errno, "Failed to read from kernel: %m"); } dev_kmsg_record(s, buffer, l); @@ -436,6 +441,7 @@ fail: int server_open_kernel_seqnum(Server *s) { _cleanup_close_ int fd; uint64_t *p; + int r; assert(s); @@ -449,8 +455,9 @@ int server_open_kernel_seqnum(Server *s) { return 0; } - if (posix_fallocate(fd, 0, sizeof(uint64_t)) < 0) { - log_error_errno(errno, "Failed to allocate sequential number file, ignoring: %m"); + r = posix_fallocate(fd, 0, sizeof(uint64_t)); + if (r != 0) { + log_error_errno(r, "Failed to allocate sequential number file, ignoring: %m"); return 0; } diff --git a/src/journal/journald-native.c b/src/journal/journald-native.c index 3e8a7a05f6..f80a6ebfe5 100644 --- a/src/journal/journald-native.c +++ b/src/journal/journald-native.c @@ -19,21 +19,28 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <unistd.h> #include <stddef.h> #include <sys/epoll.h> #include <sys/mman.h> +#include <sys/statvfs.h> +#include <unistd.h> -#include "socket-util.h" -#include "path-util.h" -#include "selinux-util.h" -#include "journald-server.h" -#include "journald-native.h" -#include "journald-kmsg.h" +#include "alloc-util.h" +#include "fd-util.h" +#include "fs-util.h" +#include "io-util.h" #include "journald-console.h" +#include "journald-kmsg.h" +#include "journald-native.h" +#include "journald-server.h" #include "journald-syslog.h" #include "journald-wall.h" #include "memfd-util.h" +#include "parse-util.h" +#include "path-util.h" +#include "selinux-util.h" +#include "socket-util.h" +#include "string-util.h" bool valid_user_field(const char *p, size_t l, bool allow_protected) { const char *a; @@ -338,7 +345,7 @@ void server_process_native_file( r = readlink_malloc(sl, &k); if (r < 0) { - log_error_errno(errno, "readlink(%s) failed: %m", sl); + log_error_errno(r, "readlink(%s) failed: %m", sl); return; } @@ -393,8 +400,37 @@ void server_process_native_file( assert_se(munmap(p, ps) >= 0); } else { _cleanup_free_ void *p = NULL; + struct statvfs vfs; ssize_t n; + if (fstatvfs(fd, &vfs) < 0) { + log_error_errno(errno, "Failed to stat file system of passed file, ignoring: %m"); + return; + } + + /* Refuse operating on file systems that have + * mandatory locking enabled, see: + * + * https://github.com/systemd/systemd/issues/1822 + */ + if (vfs.f_flag & ST_MANDLOCK) { + log_error("Received file descriptor from file system with mandatory locking enable, refusing."); + return; + } + + /* Make the fd non-blocking. On regular files this has + * the effect of bypassing mandatory locking. Of + * course, this should normally not be necessary given + * the check above, but let's better be safe than + * sorry, after all NFS is pretty confusing regarding + * file system flags, and we better don't trust it, + * and so is SMB. */ + r = fd_nonblock(fd, true); + if (r < 0) { + log_error_errno(r, "Failed to make fd non-blocking, ignoring: %m"); + return; + } + /* The file is not sealed, we can't map the file here, since * clients might then truncate it and trigger a SIGBUS for * us. So let's stupidly read it */ @@ -407,7 +443,7 @@ void server_process_native_file( n = pread(fd, p, st.st_size, 0); if (n < 0) - log_error_errno(n, "Failed to read file, ignoring: %m"); + log_error_errno(errno, "Failed to read file, ignoring: %m"); else if (n > 0) server_process_native_message(s, p, n, ucred, tv, label, label_len); } @@ -444,7 +480,7 @@ int server_open_native_socket(Server*s) { return log_error_errno(errno, "SO_PASSCRED failed: %m"); #ifdef HAVE_SELINUX - if (mac_selinux_use()) { + if (mac_selinux_have()) { r = setsockopt(s->native_fd, SOL_SOCKET, SO_PASSSEC, &one, sizeof(one)); if (r < 0) log_warning_errno(errno, "SO_PASSSEC failed: %m"); @@ -459,5 +495,9 @@ int server_open_native_socket(Server*s) { if (r < 0) return log_error_errno(r, "Failed to add native server fd to event loop: %m"); + r = sd_event_source_set_priority(s->native_event_source, SD_EVENT_PRIORITY_NORMAL+5); + if (r < 0) + return log_error_errno(r, "Failed to adjust native event source priority: %m"); + return 0; } diff --git a/src/journal/journald-rate-limit.c b/src/journal/journald-rate-limit.c index 6f83035a4e..1c406aef8e 100644 --- a/src/journal/journald-rate-limit.c +++ b/src/journal/journald-rate-limit.c @@ -19,14 +19,16 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <string.h> #include <errno.h> +#include <string.h> +#include "alloc-util.h" +#include "hashmap.h" #include "journald-rate-limit.h" #include "list.h" -#include "util.h" -#include "hashmap.h" #include "random-util.h" +#include "string-util.h" +#include "util.h" #define POOLS_MAX 5 #define BUCKETS_MAX 127 @@ -57,7 +59,7 @@ struct JournalRateLimitGroup { char *id; JournalRateLimitPool pools[POOLS_MAX]; - unsigned long hash; + uint64_t hash; LIST_FIELDS(JournalRateLimitGroup, bucket); LIST_FIELDS(JournalRateLimitGroup, lru); @@ -145,6 +147,7 @@ static void journal_rate_limit_vacuum(JournalRateLimit *r, usec_t ts) { static JournalRateLimitGroup* journal_rate_limit_group_new(JournalRateLimit *r, const char *id, usec_t ts) { JournalRateLimitGroup *g; + struct siphash state; assert(r); assert(id); @@ -157,7 +160,9 @@ static JournalRateLimitGroup* journal_rate_limit_group_new(JournalRateLimit *r, if (!g->id) goto fail; - g->hash = string_hash_func(g->id, r->hash_key); + siphash24_init(&state, r->hash_key); + string_hash_func(g->id, &state); + g->hash = siphash24_finalize(&state); journal_rate_limit_vacuum(r, ts); @@ -204,9 +209,10 @@ static unsigned burst_modulate(unsigned burst, uint64_t available) { } int journal_rate_limit_test(JournalRateLimit *r, const char *id, int priority, uint64_t available) { - unsigned long h; + uint64_t h; JournalRateLimitGroup *g; JournalRateLimitPool *p; + struct siphash state; unsigned burst; usec_t ts; @@ -222,7 +228,9 @@ int journal_rate_limit_test(JournalRateLimit *r, const char *id, int priority, u ts = now(CLOCK_MONOTONIC); - h = string_hash_func(id, r->hash_key); + siphash24_init(&state, r->hash_key); + string_hash_func(id, &state); + h = siphash24_finalize(&state); g = r->buckets[h % BUCKETS_MAX]; LIST_FOREACH(bucket, g, g) diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index fa2e9b9825..cfcc2c4302 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -19,45 +19,57 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <sys/signalfd.h> -#include <sys/ioctl.h> -#include <linux/sockios.h> -#include <sys/statvfs.h> -#include <sys/mman.h> - #ifdef HAVE_SELINUX #include <selinux/selinux.h> #endif +#include <sys/ioctl.h> +#include <sys/mman.h> +#include <sys/signalfd.h> +#include <sys/statvfs.h> +#include <linux/sockios.h> -#include <libudev.h> - +#include "libudev.h" +#include "sd-daemon.h" #include "sd-journal.h" #include "sd-messages.h" -#include "sd-daemon.h" -#include "mkdir.h" -#include "rm-rf.h" -#include "hashmap.h" -#include "journal-file.h" -#include "socket-util.h" + +#include "acl-util.h" +#include "alloc-util.h" +#include "audit-util.h" #include "cgroup-util.h" -#include "missing.h" #include "conf-parser.h" -#include "selinux-util.h" -#include "acl-util.h" +#include "dirent-util.h" +#include "extract-word.h" +#include "fd-util.h" +#include "fileio.h" #include "formats-util.h" -#include "process-util.h" +#include "fs-util.h" +#include "hashmap.h" #include "hostname-util.h" -#include "signal-util.h" +#include "io-util.h" +#include "journal-authenticate.h" +#include "journal-file.h" #include "journal-internal.h" #include "journal-vacuum.h" -#include "journal-authenticate.h" -#include "journald-rate-limit.h" +#include "journald-audit.h" #include "journald-kmsg.h" -#include "journald-syslog.h" -#include "journald-stream.h" #include "journald-native.h" -#include "journald-audit.h" +#include "journald-rate-limit.h" #include "journald-server.h" +#include "journald-stream.h" +#include "journald-syslog.h" +#include "missing.h" +#include "mkdir.h" +#include "parse-util.h" +#include "proc-cmdline.h" +#include "process-util.h" +#include "rm-rf.h" +#include "selinux-util.h" +#include "signal-util.h" +#include "socket-util.h" +#include "string-table.h" +#include "string-util.h" +#include "user-util.h" #define USER_JOURNALS_MAX 1024 @@ -66,88 +78,63 @@ #define DEFAULT_RATE_LIMIT_BURST 1000 #define DEFAULT_MAX_FILE_USEC USEC_PER_MONTH -#define RECHECK_AVAILABLE_SPACE_USEC (30*USEC_PER_SEC) - -static const char* const storage_table[_STORAGE_MAX] = { - [STORAGE_AUTO] = "auto", - [STORAGE_VOLATILE] = "volatile", - [STORAGE_PERSISTENT] = "persistent", - [STORAGE_NONE] = "none" -}; - -DEFINE_STRING_TABLE_LOOKUP(storage, Storage); -DEFINE_CONFIG_PARSE_ENUM(config_parse_storage, storage, Storage, "Failed to parse storage setting"); - -static const char* const split_mode_table[_SPLIT_MAX] = { - [SPLIT_LOGIN] = "login", - [SPLIT_UID] = "uid", - [SPLIT_NONE] = "none", -}; +#define RECHECK_SPACE_USEC (30*USEC_PER_SEC) -DEFINE_STRING_TABLE_LOOKUP(split_mode, SplitMode); -DEFINE_CONFIG_PARSE_ENUM(config_parse_split_mode, split_mode, SplitMode, "Failed to parse split mode setting"); +#define NOTIFY_SNDBUF_SIZE (8*1024*1024) -static uint64_t available_space(Server *s, bool verbose) { - char ids[33]; - _cleanup_free_ char *p = NULL; - sd_id128_t machine; - struct statvfs ss; - uint64_t sum = 0, ss_avail = 0, avail = 0; - int r; +static int determine_space_for( + Server *s, + JournalMetrics *metrics, + const char *path, + const char *name, + bool verbose, + bool patch_min_use, + uint64_t *available, + uint64_t *limit) { + + uint64_t sum = 0, ss_avail, avail; _cleanup_closedir_ DIR *d = NULL; + struct dirent *de; + struct statvfs ss; + const char *p; usec_t ts; - const char *f; - JournalMetrics *m; - - ts = now(CLOCK_MONOTONIC); - if (s->cached_available_space_timestamp + RECHECK_AVAILABLE_SPACE_USEC > ts - && !verbose) - return s->cached_available_space; + assert(s); + assert(metrics); + assert(path); + assert(name); - r = sd_id128_get_machine(&machine); - if (r < 0) - return 0; + ts = now(CLOCK_MONOTONIC); - if (s->system_journal) { - f = "/var/log/journal/"; - m = &s->system_metrics; - } else { - f = "/run/log/journal/"; - m = &s->runtime_metrics; - } + if (!verbose && s->cached_space_timestamp + RECHECK_SPACE_USEC > ts) { - assert(m); + if (available) + *available = s->cached_space_available; + if (limit) + *limit = s->cached_space_limit; - p = strappend(f, sd_id128_to_string(machine, ids)); - if (!p) return 0; + } + p = strjoina(path, SERVER_MACHINE_ID(s)); d = opendir(p); if (!d) - return 0; + return log_full_errno(errno == ENOENT ? LOG_DEBUG : LOG_ERR, errno, "Failed to open %s: %m", p); if (fstatvfs(dirfd(d), &ss) < 0) - return 0; + return log_error_errno(errno, "Failed to fstatvfs(%s): %m", p); - for (;;) { + FOREACH_DIRENT_ALL(de, d, break) { struct stat st; - struct dirent *de; - - errno = 0; - de = readdir(d); - if (!de && errno != 0) - return 0; - - if (!de) - break; if (!endswith(de->d_name, ".journal") && !endswith(de->d_name, ".journal~")) continue; - if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) + if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) { + log_debug_errno(errno, "Failed to stat %s/%s, ignoring: %m", p, de->d_name); continue; + } if (!S_ISREG(st.st_mode)) continue; @@ -155,90 +142,81 @@ static uint64_t available_space(Server *s, bool verbose) { sum += (uint64_t) st.st_blocks * 512UL; } - ss_avail = ss.f_bsize * ss.f_bavail; - - /* If we reached a high mark, we will always allow this much - * again, unless usage goes above max_use. This watermark - * value is cached so that we don't give up space on pressure, - * but hover below the maximum usage. */ + /* If request, then let's bump the min_use limit to the + * current usage on disk. We do this when starting up and + * first opening the journal files. This way sudden spikes in + * disk usage will not cause journald to vacuum files without + * bounds. Note that this means that only a restart of + * journald will make it reset this value. */ - if (m->use < sum) - m->use = sum; + if (patch_min_use) + metrics->min_use = MAX(metrics->min_use, sum); - avail = LESS_BY(ss_avail, m->keep_free); + ss_avail = ss.f_bsize * ss.f_bavail; + avail = LESS_BY(ss_avail, metrics->keep_free); - s->cached_available_space = LESS_BY(MIN(m->max_use, avail), sum); - s->cached_available_space_timestamp = ts; + s->cached_space_limit = MIN(MAX(sum + avail, metrics->min_use), metrics->max_use); + s->cached_space_available = LESS_BY(s->cached_space_limit, sum); + s->cached_space_timestamp = ts; if (verbose) { char fb1[FORMAT_BYTES_MAX], fb2[FORMAT_BYTES_MAX], fb3[FORMAT_BYTES_MAX], - fb4[FORMAT_BYTES_MAX], fb5[FORMAT_BYTES_MAX]; + fb4[FORMAT_BYTES_MAX], fb5[FORMAT_BYTES_MAX], fb6[FORMAT_BYTES_MAX]; server_driver_message(s, SD_MESSAGE_JOURNAL_USAGE, - "%s is currently using %s.\n" + "%s (%s) is currently using %s.\n" "Maximum allowed usage is set to %s.\n" "Leaving at least %s free (of currently available %s of space).\n" - "Enforced usage limit is thus %s.", - s->system_journal ? "Permanent journal (/var/log/journal/)" : "Runtime journal (/run/log/journal/)", + "Enforced usage limit is thus %s, of which %s are still available.", + name, path, format_bytes(fb1, sizeof(fb1), sum), - format_bytes(fb2, sizeof(fb2), m->max_use), - format_bytes(fb3, sizeof(fb3), m->keep_free), + format_bytes(fb2, sizeof(fb2), metrics->max_use), + format_bytes(fb3, sizeof(fb3), metrics->keep_free), format_bytes(fb4, sizeof(fb4), ss_avail), - format_bytes(fb5, sizeof(fb5), s->cached_available_space + sum)); + format_bytes(fb5, sizeof(fb5), s->cached_space_limit), + format_bytes(fb6, sizeof(fb6), s->cached_space_available)); } - return s->cached_available_space; + if (available) + *available = s->cached_space_available; + if (limit) + *limit = s->cached_space_limit; + + return 1; } -void server_fix_perms(Server *s, JournalFile *f, uid_t uid) { - int r; +static int determine_space(Server *s, bool verbose, bool patch_min_use, uint64_t *available, uint64_t *limit) { + JournalMetrics *metrics; + const char *path, *name; + + assert(s); + + if (s->system_journal) { + path = "/var/log/journal/"; + metrics = &s->system_metrics; + name = "System journal"; + } else { + path = "/run/log/journal/"; + metrics = &s->runtime_metrics; + name = "Runtime journal"; + } + + return determine_space_for(s, metrics, path, name, verbose, patch_min_use, available, limit); +} + +static void server_add_acls(JournalFile *f, uid_t uid) { #ifdef HAVE_ACL - acl_t acl; - acl_entry_t entry; - acl_permset_t permset; + int r; #endif - assert(f); - r = fchmod(f->fd, 0640); - if (r < 0) - log_warning_errno(r, "Failed to fix access mode on %s, ignoring: %m", f->path); - #ifdef HAVE_ACL if (uid <= SYSTEM_UID_MAX) return; - acl = acl_get_fd(f->fd); - if (!acl) { - log_warning_errno(errno, "Failed to read ACL on %s, ignoring: %m", f->path); - return; - } - - r = acl_find_uid(acl, uid, &entry); - if (r <= 0) { - - if (acl_create_entry(&acl, &entry) < 0 || - acl_set_tag_type(entry, ACL_USER) < 0 || - acl_set_qualifier(entry, &uid) < 0) { - log_warning_errno(errno, "Failed to patch ACL on %s, ignoring: %m", f->path); - goto finish; - } - } - - /* We do not recalculate the mask unconditionally here, - * so that the fchmod() mask above stays intact. */ - if (acl_get_permset(entry, &permset) < 0 || - acl_add_perm(permset, ACL_READ) < 0 || - calc_acl_mask_if_needed(&acl) < 0) { - log_warning_errno(errno, "Failed to patch ACL on %s, ignoring: %m", f->path); - goto finish; - } - - if (acl_set_fd(f->fd, acl) < 0) - log_warning_errno(errno, "Failed to set ACL on %s, ignoring: %m", f->path); - -finish: - acl_free(acl); + r = add_acls_for_user(f->fd, uid); + if (r < 0) + log_warning_errno(r, "Failed to set ACL on %s, ignoring: %m", f->path); #endif } @@ -265,7 +243,7 @@ static JournalFile* find_journal(Server *s, uid_t uid) { if (r < 0) return s->system_journal; - f = ordered_hashmap_get(s->user_journals, UINT32_TO_PTR(uid)); + f = ordered_hashmap_get(s->user_journals, UID_TO_PTR(uid)); if (f) return f; @@ -284,9 +262,9 @@ static JournalFile* find_journal(Server *s, uid_t uid) { if (r < 0) return s->system_journal; - server_fix_perms(s, f, uid); + server_add_acls(f, uid); - r = ordered_hashmap_put(s->user_journals, UINT32_TO_PTR(uid), f); + r = ordered_hashmap_put(s->user_journals, UID_TO_PTR(uid), f); if (r < 0) { journal_file_close(f); return s->system_journal; @@ -315,7 +293,7 @@ static int do_rotate( else log_error_errno(r, "Failed to create new %s journal: %m", name); else - server_fix_perms(s, *f, uid); + server_add_acls(*f, uid); return r; } @@ -328,11 +306,11 @@ void server_rotate(Server *s) { log_debug("Rotating..."); - do_rotate(s, &s->runtime_journal, "runtime", false, 0); - do_rotate(s, &s->system_journal, "system", s->seal, 0); + (void) do_rotate(s, &s->runtime_journal, "runtime", false, 0); + (void) do_rotate(s, &s->system_journal, "system", s->seal, 0); ORDERED_HASHMAP_FOREACH_KEY(f, k, s->user_journals, i) { - r = do_rotate(s, &f, "user", s->seal, PTR_TO_UINT32(k)); + r = do_rotate(s, &f, "user", s->seal, PTR_TO_UID(k)); if (r >= 0) ordered_hashmap_replace(s->user_journals, k, f); else if (!f) @@ -343,20 +321,19 @@ void server_rotate(Server *s) { void server_sync(Server *s) { JournalFile *f; - void *k; Iterator i; int r; if (s->system_journal) { r = journal_file_set_offline(s->system_journal); if (r < 0) - log_error_errno(r, "Failed to sync system journal: %m"); + log_warning_errno(r, "Failed to sync system journal, ignoring: %m"); } - ORDERED_HASHMAP_FOREACH_KEY(f, k, s->user_journals, i) { + ORDERED_HASHMAP_FOREACH(f, s->user_journals, i) { r = journal_file_set_offline(f); if (r < 0) - log_error_errno(r, "Failed to sync user journal: %m"); + log_warning_errno(r, "Failed to sync user journal, ignoring: %m"); } if (s->sync_event_source) { @@ -370,43 +347,50 @@ void server_sync(Server *s) { static void do_vacuum( Server *s, - const char *id, JournalFile *f, - const char* path, - JournalMetrics *metrics) { + JournalMetrics *metrics, + const char *path, + const char *name, + bool verbose, + bool patch_min_use) { const char *p; + uint64_t limit; int r; + assert(s); + assert(metrics); + assert(path); + assert(name); + if (!f) return; - p = strjoina(path, id); - r = journal_directory_vacuum(p, metrics->max_use, s->max_retention_usec, &s->oldest_file_usec, false); + p = strjoina(path, SERVER_MACHINE_ID(s)); + + limit = metrics->max_use; + (void) determine_space_for(s, metrics, path, name, verbose, patch_min_use, NULL, &limit); + + r = journal_directory_vacuum(p, limit, metrics->n_max_files, s->max_retention_usec, &s->oldest_file_usec, verbose); if (r < 0 && r != -ENOENT) - log_error_errno(r, "Failed to vacuum %s: %m", p); + log_warning_errno(r, "Failed to vacuum %s, ignoring: %m", p); } -void server_vacuum(Server *s) { - char ids[33]; - sd_id128_t machine; - int r; +int server_vacuum(Server *s, bool verbose, bool patch_min_use) { + assert(s); log_debug("Vacuuming..."); s->oldest_file_usec = 0; - r = sd_id128_get_machine(&machine); - if (r < 0) { - log_error_errno(r, "Failed to get machine ID: %m"); - return; - } - sd_id128_to_string(machine, ids); + do_vacuum(s, s->system_journal, &s->system_metrics, "/var/log/journal/", "System journal", verbose, patch_min_use); + do_vacuum(s, s->runtime_journal, &s->runtime_metrics, "/run/log/journal/", "Runtime journal", verbose, patch_min_use); - do_vacuum(s, ids, s->system_journal, "/var/log/journal/", &s->system_metrics); - do_vacuum(s, ids, s->runtime_journal, "/run/log/journal/", &s->runtime_metrics); + s->cached_space_limit = 0; + s->cached_space_available = 0; + s->cached_space_timestamp = 0; - s->cached_available_space_timestamp = 0; + return 0; } static void server_cache_machine_id(Server *s) { @@ -504,7 +488,7 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, unsigned if (journal_file_rotate_suggested(f, s->max_file_usec)) { log_debug("%s: Journal header limits reached or header out-of-date, rotating.", f->path); server_rotate(s); - server_vacuum(s); + server_vacuum(s, false, false); vacuumed = true; f = find_journal(s, uid); @@ -524,7 +508,7 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, unsigned } server_rotate(s); - server_vacuum(s); + server_vacuum(s, false, false); f = find_journal(s, uid); if (!f) @@ -684,7 +668,7 @@ static void dispatch_message_real( } #ifdef HAVE_SELINUX - if (mac_selinux_use()) { + if (mac_selinux_have()) { if (label) { x = alloca(strlen("_SELINUX_CONTEXT=") + label_len + 1); @@ -825,7 +809,7 @@ static void dispatch_message_real( void server_driver_message(Server *s, sd_id128_t message_id, const char *format, ...) { char mid[11 + 32 + 1]; char buffer[16 + LINE_MAX + 1]; - struct iovec iovec[N_IOVEC_META_FIELDS + 4]; + struct iovec iovec[N_IOVEC_META_FIELDS + 6]; int n = 0; va_list ap; struct ucred ucred = {}; @@ -833,6 +817,9 @@ void server_driver_message(Server *s, sd_id128_t message_id, const char *format, assert(s); assert(format); + IOVEC_SET_STRING(iovec[n++], "SYSLOG_FACILITY=3"); + IOVEC_SET_STRING(iovec[n++], "SYSLOG_IDENTIFIER=systemd-journald"); + IOVEC_SET_STRING(iovec[n++], "PRIORITY=6"); IOVEC_SET_STRING(iovec[n++], "_TRANSPORT=driver"); @@ -866,6 +853,7 @@ void server_dispatch_message( int rl, r; _cleanup_free_ char *path = NULL; + uint64_t available = 0; char *c; assert(s); @@ -905,9 +893,8 @@ void server_dispatch_message( } } - rl = journal_rate_limit_test(s->rate_limit, path, - priority & LOG_PRIMASK, available_space(s, false)); - + (void) determine_space(s, false, false, &available, NULL); + rl = journal_rate_limit_test(s->rate_limit, path, priority & LOG_PRIMASK, available); if (rl == 0) return; @@ -922,16 +909,8 @@ finish: static int system_journal_open(Server *s, bool flush_requested) { - int r; - char *fn; - sd_id128_t machine; - char ids[33]; - - r = sd_id128_get_machine(&machine); - if (r < 0) - return log_error_errno(r, "Failed to get machine id: %m"); - - sd_id128_to_string(machine, ids); + const char *fn; + int r = 0; if (!s->system_journal && (s->storage == STORAGE_PERSISTENT || s->storage == STORAGE_AUTO) && @@ -947,15 +926,15 @@ static int system_journal_open(Server *s, bool flush_requested) { if (s->storage == STORAGE_PERSISTENT) (void) mkdir_p("/var/log/journal/", 0755); - fn = strjoina("/var/log/journal/", ids); + fn = strjoina("/var/log/journal/", SERVER_MACHINE_ID(s)); (void) mkdir(fn, 0755); fn = strjoina(fn, "/system.journal"); r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, s->compress, s->seal, &s->system_metrics, s->mmap, NULL, &s->system_journal); - - if (r >= 0) - server_fix_perms(s, s->system_journal, 0); - else if (r < 0) { + if (r >= 0) { + server_add_acls(s->system_journal, 0); + (void) determine_space_for(s, &s->system_metrics, "/var/log/journal/", "System journal", true, true, NULL, NULL); + } else if (r < 0) { if (r != -ENOENT && r != -EROFS) log_warning_errno(r, "Failed to open system journal: %m"); @@ -966,9 +945,7 @@ static int system_journal_open(Server *s, bool flush_requested) { if (!s->runtime_journal && (s->storage != STORAGE_NONE)) { - fn = strjoin("/run/log/journal/", ids, "/system.journal", NULL); - if (!fn) - return -ENOMEM; + fn = strjoina("/run/log/journal/", SERVER_MACHINE_ID(s), "/system.journal"); if (s->system_journal) { @@ -977,8 +954,6 @@ static int system_journal_open(Server *s, bool flush_requested) { * it into the system journal */ r = journal_file_open(fn, O_RDWR, 0640, s->compress, false, &s->runtime_metrics, s->mmap, NULL, &s->runtime_journal); - free(fn); - if (r < 0) { if (r != -ENOENT) log_warning_errno(r, "Failed to open runtime journal: %m"); @@ -996,18 +971,16 @@ static int system_journal_open(Server *s, bool flush_requested) { (void) mkdir_parents(fn, 0750); r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, s->compress, false, &s->runtime_metrics, s->mmap, NULL, &s->runtime_journal); - free(fn); - if (r < 0) return log_error_errno(r, "Failed to open runtime journal: %m"); } - if (s->runtime_journal) - server_fix_perms(s, s->runtime_journal, 0); + if (s->runtime_journal) { + server_add_acls(s->runtime_journal, 0); + (void) determine_space_for(s, &s->runtime_metrics, "/run/log/journal/", "Runtime journal", true, true, NULL, NULL); + } } - available_space(s, true); - return r; } @@ -1028,7 +1001,7 @@ int server_flush_to_var(Server *s) { if (!s->runtime_journal) return 0; - system_journal_open(s, true); + (void) system_journal_open(s, true); if (!s->system_journal) return 0; @@ -1072,7 +1045,7 @@ int server_flush_to_var(Server *s) { } server_rotate(s); - server_vacuum(s); + server_vacuum(s, false, false); if (!s->system_journal) { log_notice("Didn't flush runtime journal since rotation of system journal wasn't successful."); @@ -1088,11 +1061,12 @@ int server_flush_to_var(Server *s) { } } + r = 0; + finish: journal_file_post_change(s->system_journal); - journal_file_close(s->runtime_journal); - s->runtime_journal = NULL; + s->runtime_journal = journal_file_close(s->runtime_journal); if (r >= 0) (void) rm_rf("/run/log/journal", REMOVE_ROOT); @@ -1228,28 +1202,37 @@ int server_process_datagram(sd_event_source *es, int fd, uint32_t revents, void static int dispatch_sigusr1(sd_event_source *es, const struct signalfd_siginfo *si, void *userdata) { Server *s = userdata; + int r; assert(s); - log_info("Received request to flush runtime journal from PID %"PRIu32, si->ssi_pid); + log_info("Received request to flush runtime journal from PID " PID_FMT, si->ssi_pid); server_flush_to_var(s); server_sync(s); - server_vacuum(s); + server_vacuum(s, false, false); - touch("/run/systemd/journal/flushed"); + r = touch("/run/systemd/journal/flushed"); + if (r < 0) + log_warning_errno(r, "Failed to touch /run/systemd/journal/flushed, ignoring: %m"); return 0; } static int dispatch_sigusr2(sd_event_source *es, const struct signalfd_siginfo *si, void *userdata) { Server *s = userdata; + int r; assert(s); - log_info("Received request to rotate journal from PID %"PRIu32, si->ssi_pid); + log_info("Received request to rotate journal from PID " PID_FMT, si->ssi_pid); server_rotate(s); - server_vacuum(s); + server_vacuum(s, true, true); + + /* Let clients know when the most recent rotation happened. */ + r = write_timestamp_file_atomic("/run/systemd/journal/rotated", now(CLOCK_MONOTONIC)); + if (r < 0) + log_warning_errno(r, "Failed to write /run/systemd/journal/rotated, ignoring: %m"); return 0; } @@ -1265,12 +1248,30 @@ static int dispatch_sigterm(sd_event_source *es, const struct signalfd_siginfo * return 0; } +static int dispatch_sigrtmin1(sd_event_source *es, const struct signalfd_siginfo *si, void *userdata) { + Server *s = userdata; + int r; + + assert(s); + + log_debug("Received request to sync from PID " PID_FMT, si->ssi_pid); + + server_sync(s); + + /* Let clients know when the most recent sync happened. */ + r = write_timestamp_file_atomic("/run/systemd/journal/synced", now(CLOCK_MONOTONIC)); + if (r < 0) + log_warning_errno(r, "Failed to write /run/systemd/journal/synced, ignoring: %m"); + + return 0; +} + static int setup_signals(Server *s) { int r; assert(s); - assert(sigprocmask_many(SIG_SETMASK, NULL, SIGINT, SIGTERM, SIGUSR1, SIGUSR2, -1) >= 0); + assert(sigprocmask_many(SIG_SETMASK, NULL, SIGINT, SIGTERM, SIGUSR1, SIGUSR2, SIGRTMIN+1, -1) >= 0); r = sd_event_add_signal(s->event, &s->sigusr1_event_source, SIGUSR1, dispatch_sigusr1, s); if (r < 0) @@ -1284,17 +1285,41 @@ static int setup_signals(Server *s) { if (r < 0) return r; + /* Let's process SIGTERM late, so that we flush all queued + * messages to disk before we exit */ + r = sd_event_source_set_priority(s->sigterm_event_source, SD_EVENT_PRIORITY_NORMAL+20); + if (r < 0) + return r; + + /* When journald is invoked on the terminal (when debugging), + * it's useful if C-c is handled equivalent to SIGTERM. */ r = sd_event_add_signal(s->event, &s->sigint_event_source, SIGINT, dispatch_sigterm, s); if (r < 0) return r; + r = sd_event_source_set_priority(s->sigint_event_source, SD_EVENT_PRIORITY_NORMAL+20); + if (r < 0) + return r; + + /* SIGRTMIN+1 causes an immediate sync. We process this very + * late, so that everything else queued at this point is + * really written to disk. Clients can watch + * /run/systemd/journal/synced with inotify until its mtime + * changes to see when a sync happened. */ + r = sd_event_add_signal(s->event, &s->sigrtmin1_event_source, SIGRTMIN+1, dispatch_sigrtmin1, s); + if (r < 0) + return r; + + r = sd_event_source_set_priority(s->sigrtmin1_event_source, SD_EVENT_PRIORITY_NORMAL+15); + if (r < 0) + return r; + return 0; } static int server_parse_proc_cmdline(Server *s) { _cleanup_free_ char *line = NULL; - const char *w, *state; - size_t l; + const char *p; int r; r = proc_cmdline(&line); @@ -1303,12 +1328,16 @@ static int server_parse_proc_cmdline(Server *s) { return 0; } - FOREACH_WORD_QUOTED(w, l, line, state) { - _cleanup_free_ char *word; + p = line; + for(;;) { + _cleanup_free_ char *word = NULL; - word = strndup(w, l); - if (!word) - return -ENOMEM; + r = extract_first_word(&p, &word, NULL, 0); + if (r < 0) + return log_error_errno(r, "Failed to parse journald syntax \"%s\": %m", line); + + if (r == 0) + break; if (startswith(word, "systemd.journald.forward_to_syslog=")) { r = parse_boolean(word + 35); @@ -1337,16 +1366,16 @@ static int server_parse_proc_cmdline(Server *s) { } else if (startswith(word, "systemd.journald")) log_warning("Invalid systemd.journald parameter. Ignoring."); } - /* do not warn about state here, since probably systemd already did */ + /* do not warn about state here, since probably systemd already did */ return 0; } static int server_parse_config_file(Server *s) { assert(s); - return config_parse_many("/etc/systemd/journald.conf", - CONF_DIRS_NULSTR("systemd/journald.conf"), + return config_parse_many(PKGSYSCONFDIR "/journald.conf", + CONF_PATHS_NULSTR("systemd/journald.conf.d"), "Journal\0", config_item_perf_lookup, journald_gperf_lookup, false, s); @@ -1434,8 +1463,7 @@ static int server_open_hostname(Server *s) { /* kernels prior to 3.2 don't support polling this file. Ignore * the failure. */ if (r == -EPERM) { - log_warning("Failed to register hostname fd in event loop: %s. Ignoring.", - strerror(-r)); + log_warning_errno(r, "Failed to register hostname fd in event loop, ignoring: %m"); s->hostname_fd = safe_close(s->hostname_fd); return 0; } @@ -1450,17 +1478,184 @@ static int server_open_hostname(Server *s) { return 0; } +static int dispatch_notify_event(sd_event_source *es, int fd, uint32_t revents, void *userdata) { + Server *s = userdata; + int r; + + assert(s); + assert(s->notify_event_source == es); + assert(s->notify_fd == fd); + + /* The $NOTIFY_SOCKET is writable again, now send exactly one + * message on it. Either it's the wtachdog event, the initial + * READY=1 event or an stdout stream event. If there's nothing + * to write anymore, turn our event source off. The next time + * there's something to send it will be turned on again. */ + + if (!s->sent_notify_ready) { + static const char p[] = + "READY=1\n" + "STATUS=Processing requests..."; + ssize_t l; + + l = send(s->notify_fd, p, strlen(p), MSG_DONTWAIT); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + return log_error_errno(errno, "Failed to send READY=1 notification message: %m"); + } + + s->sent_notify_ready = true; + log_debug("Sent READY=1 notification."); + + } else if (s->send_watchdog) { + + static const char p[] = + "WATCHDOG=1"; + + ssize_t l; + + l = send(s->notify_fd, p, strlen(p), MSG_DONTWAIT); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + return log_error_errno(errno, "Failed to send WATCHDOG=1 notification message: %m"); + } + + s->send_watchdog = false; + log_debug("Sent WATCHDOG=1 notification."); + + } else if (s->stdout_streams_notify_queue) + /* Dispatch one stream notification event */ + stdout_stream_send_notify(s->stdout_streams_notify_queue); + + /* Leave us enabled if there's still more to to do. */ + if (s->send_watchdog || s->stdout_streams_notify_queue) + return 0; + + /* There was nothing to do anymore, let's turn ourselves off. */ + r = sd_event_source_set_enabled(es, SD_EVENT_OFF); + if (r < 0) + return log_error_errno(r, "Failed to turn off notify event source: %m"); + + return 0; +} + +static int dispatch_watchdog(sd_event_source *es, uint64_t usec, void *userdata) { + Server *s = userdata; + int r; + + assert(s); + + s->send_watchdog = true; + + r = sd_event_source_set_enabled(s->notify_event_source, SD_EVENT_ON); + if (r < 0) + log_warning_errno(r, "Failed to turn on notify event source: %m"); + + r = sd_event_source_set_time(s->watchdog_event_source, usec + s->watchdog_usec / 2); + if (r < 0) + return log_error_errno(r, "Failed to restart watchdog event source: %m"); + + r = sd_event_source_set_enabled(s->watchdog_event_source, SD_EVENT_ON); + if (r < 0) + return log_error_errno(r, "Failed to enable watchdog event source: %m"); + + return 0; +} + +static int server_connect_notify(Server *s) { + union sockaddr_union sa = { + .un.sun_family = AF_UNIX, + }; + const char *e; + int r; + + assert(s); + assert(s->notify_fd < 0); + assert(!s->notify_event_source); + + /* + So here's the problem: we'd like to send notification + messages to PID 1, but we cannot do that via sd_notify(), + since that's synchronous, and we might end up blocking on + it. Specifically: given that PID 1 might block on + dbus-daemon during IPC, and dbus-daemon is logging to us, + and might hence block on us, we might end up in a deadlock + if we block on sending PID 1 notification messages -- by + generating a full blocking circle. To avoid this, let's + create a non-blocking socket, and connect it to the + notification socket, and then wait for POLLOUT before we + send anything. This should efficiently avoid any deadlocks, + as we'll never block on PID 1, hence PID 1 can safely block + on dbus-daemon which can safely block on us again. + + Don't think that this issue is real? It is, see: + https://github.com/systemd/systemd/issues/1505 + */ + + e = getenv("NOTIFY_SOCKET"); + if (!e) + return 0; + + if ((e[0] != '@' && e[0] != '/') || e[1] == 0) { + log_error("NOTIFY_SOCKET set to an invalid value: %s", e); + return -EINVAL; + } + + if (strlen(e) > sizeof(sa.un.sun_path)) { + log_error("NOTIFY_SOCKET path too long: %s", e); + return -EINVAL; + } + + s->notify_fd = socket(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0); + if (s->notify_fd < 0) + return log_error_errno(errno, "Failed to create notify socket: %m"); + + (void) fd_inc_sndbuf(s->notify_fd, NOTIFY_SNDBUF_SIZE); + + strncpy(sa.un.sun_path, e, sizeof(sa.un.sun_path)); + if (sa.un.sun_path[0] == '@') + sa.un.sun_path[0] = 0; + + r = connect(s->notify_fd, &sa.sa, offsetof(struct sockaddr_un, sun_path) + strlen(e)); + if (r < 0) + return log_error_errno(errno, "Failed to connect to notify socket: %m"); + + r = sd_event_add_io(s->event, &s->notify_event_source, s->notify_fd, EPOLLOUT, dispatch_notify_event, s); + if (r < 0) + return log_error_errno(r, "Failed to watch notification socket: %m"); + + if (sd_watchdog_enabled(false, &s->watchdog_usec) > 0) { + s->send_watchdog = true; + + r = sd_event_add_time(s->event, &s->watchdog_event_source, CLOCK_MONOTONIC, now(CLOCK_MONOTONIC) + s->watchdog_usec/2, s->watchdog_usec/4, dispatch_watchdog, s); + if (r < 0) + return log_error_errno(r, "Failed to add watchdog time event: %m"); + } + + /* This should fire pretty soon, which we'll use to send the + * READY=1 event. */ + + return 0; +} + int server_init(Server *s) { _cleanup_fdset_free_ FDSet *fds = NULL; int n, r, fd; + bool no_sockets; assert(s); zero(*s); - s->syslog_fd = s->native_fd = s->stdout_fd = s->dev_kmsg_fd = s->audit_fd = s->hostname_fd = -1; + s->syslog_fd = s->native_fd = s->stdout_fd = s->dev_kmsg_fd = s->audit_fd = s->hostname_fd = s->notify_fd = -1; s->compress = true; s->seal = true; + s->watchdog_usec = USEC_INFINITY; + s->sync_interval_usec = DEFAULT_SYNC_INTERVAL_USEC; s->sync_scheduled = false; @@ -1477,18 +1672,19 @@ int server_init(Server *s) { s->max_level_console = LOG_INFO; s->max_level_wall = LOG_EMERG; - memset(&s->system_metrics, 0xFF, sizeof(s->system_metrics)); - memset(&s->runtime_metrics, 0xFF, sizeof(s->runtime_metrics)); + journal_reset_metrics(&s->system_metrics); + journal_reset_metrics(&s->runtime_metrics); server_parse_config_file(s); server_parse_proc_cmdline(s); + if (!!s->rate_limit_interval ^ !!s->rate_limit_burst) { log_debug("Setting both rate limit interval and burst from "USEC_FMT",%u to 0,0", s->rate_limit_interval, s->rate_limit_burst); s->rate_limit_interval = s->rate_limit_burst = 0; } - mkdir_p("/run/systemd/journal", 0755); + (void) mkdir_p("/run/systemd/journal", 0755); s->user_journals = ordered_hashmap_new(NULL); if (!s->user_journals) @@ -1502,8 +1698,6 @@ int server_init(Server *s) { if (r < 0) return log_error_errno(r, "Failed to create event loop: %m"); - sd_event_set_watchdog(s->event, true); - n = sd_listen_fds(true); if (n < 0) return log_error_errno(n, "Failed to read listening file descriptors from environment: %m"); @@ -1561,30 +1755,44 @@ int server_init(Server *s) { } } - r = server_open_stdout_socket(s, fds); - if (r < 0) - return r; + /* Try to restore streams, but don't bother if this fails */ + (void) server_restore_streams(s, fds); if (fdset_size(fds) > 0) { log_warning("%u unknown file descriptors passed, closing.", fdset_size(fds)); fds = fdset_free(fds); } + no_sockets = s->native_fd < 0 && s->stdout_fd < 0 && s->syslog_fd < 0 && s->audit_fd < 0; + + /* always open stdout, syslog, native, and kmsg sockets */ + + /* systemd-journald.socket: /run/systemd/journal/stdout */ + r = server_open_stdout_socket(s); + if (r < 0) + return r; + + /* systemd-journald-dev-log.socket: /run/systemd/journal/dev-log */ r = server_open_syslog_socket(s); if (r < 0) return r; + /* systemd-journald.socket: /run/systemd/journal/socket */ r = server_open_native_socket(s); if (r < 0) return r; + /* /dev/ksmg */ r = server_open_dev_kmsg(s); if (r < 0) return r; - r = server_open_audit(s); - if (r < 0) - return r; + /* Unless we got *some* sockets and not audit, open audit socket */ + if (s->audit_fd >= 0 || no_sockets) { + r = server_open_audit(s); + if (r < 0) + return r; + } r = server_open_kernel_seqnum(s); if (r < 0) @@ -1614,11 +1822,9 @@ int server_init(Server *s) { server_cache_boot_id(s); server_cache_machine_id(s); - r = system_journal_open(s, false); - if (r < 0) - return r; + (void) server_connect_notify(s); - return 0; + return system_journal_open(s, false); } void server_maybe_append_tags(Server *s) { @@ -1665,7 +1871,10 @@ void server_done(Server *s) { sd_event_source_unref(s->sigusr2_event_source); sd_event_source_unref(s->sigterm_event_source); sd_event_source_unref(s->sigint_event_source); + sd_event_source_unref(s->sigrtmin1_event_source); sd_event_source_unref(s->hostname_event_source); + sd_event_source_unref(s->notify_event_source); + sd_event_source_unref(s->watchdog_event_source); sd_event_unref(s->event); safe_close(s->syslog_fd); @@ -1674,6 +1883,7 @@ void server_done(Server *s) { safe_close(s->dev_kmsg_fd); safe_close(s->audit_fd); safe_close(s->hostname_fd); + safe_close(s->notify_fd); if (s->rate_limit) journal_rate_limit_free(s->rate_limit); @@ -1691,3 +1901,22 @@ void server_done(Server *s) { udev_unref(s->udev); } + +static const char* const storage_table[_STORAGE_MAX] = { + [STORAGE_AUTO] = "auto", + [STORAGE_VOLATILE] = "volatile", + [STORAGE_PERSISTENT] = "persistent", + [STORAGE_NONE] = "none" +}; + +DEFINE_STRING_TABLE_LOOKUP(storage, Storage); +DEFINE_CONFIG_PARSE_ENUM(config_parse_storage, storage, Storage, "Failed to parse storage setting"); + +static const char* const split_mode_table[_SPLIT_MAX] = { + [SPLIT_LOGIN] = "login", + [SPLIT_UID] = "uid", + [SPLIT_NONE] = "none", +}; + +DEFINE_STRING_TABLE_LOOKUP(split_mode, SplitMode); +DEFINE_CONFIG_PARSE_ENUM(config_parse_split_mode, split_mode, SplitMode, "Failed to parse split mode setting"); diff --git a/src/journal/journald-server.h b/src/journal/journald-server.h index d954c5190d..1822765228 100644 --- a/src/journal/journald-server.h +++ b/src/journal/journald-server.h @@ -25,10 +25,13 @@ #include <sys/types.h> #include "sd-event.h" -#include "journal-file.h" + +typedef struct Server Server; + #include "hashmap.h" -#include "audit.h" +#include "journal-file.h" #include "journald-rate-limit.h" +#include "journald-stream.h" #include "list.h" typedef enum Storage { @@ -48,15 +51,14 @@ typedef enum SplitMode { _SPLIT_INVALID = -1 } SplitMode; -typedef struct StdoutStream StdoutStream; - -typedef struct Server { +struct Server { int syslog_fd; int native_fd; int stdout_fd; int dev_kmsg_fd; int audit_fd; int hostname_fd; + int notify_fd; sd_event *event; @@ -70,7 +72,10 @@ typedef struct Server { sd_event_source *sigusr2_event_source; sd_event_source *sigterm_event_source; sd_event_source *sigint_event_source; + sd_event_source *sigrtmin1_event_source; sd_event_source *hostname_event_source; + sd_event_source *notify_event_source; + sd_event_source *watchdog_event_source; JournalFile *runtime_journal; JournalFile *system_journal; @@ -100,8 +105,9 @@ typedef struct Server { unsigned n_forward_syslog_missed; usec_t last_warn_forward_syslog_missed; - uint64_t cached_available_space; - usec_t cached_available_space_timestamp; + uint64_t cached_space_available; + uint64_t cached_space_limit; + usec_t cached_space_timestamp; uint64_t var_available_timestamp; @@ -110,6 +116,7 @@ typedef struct Server { usec_t oldest_file_usec; LIST_HEAD(StdoutStream, stdout_streams); + LIST_HEAD(StdoutStream, stdout_streams_notify_queue); unsigned n_stdout_streams; char *tty_path; @@ -125,13 +132,14 @@ typedef struct Server { MMapCache *mmap; - bool dev_kmsg_readable; + struct udev *udev; uint64_t *kernel_seqnum; + bool dev_kmsg_readable:1; - struct udev *udev; - - bool sync_scheduled; + bool send_watchdog:1; + bool sent_notify_ready:1; + bool sync_scheduled:1; char machine_id_field[sizeof("_MACHINE_ID=") + 32]; char boot_id_field[sizeof("_BOOT_ID=") + 32]; @@ -139,7 +147,11 @@ typedef struct Server { /* Cached cgroup root, so that we don't have to query that all the time */ char *cgroup_root; -} Server; + + usec_t watchdog_usec; +}; + +#define SERVER_MACHINE_ID(s) ((s)->machine_id_field + strlen("_MACHINE_ID=")) #define N_IOVEC_META_FIELDS 20 #define N_IOVEC_KERNEL_FIELDS 64 @@ -162,11 +174,10 @@ int config_parse_split_mode(const char *unit, const char *filename, unsigned lin const char *split_mode_to_string(SplitMode s) _const_; SplitMode split_mode_from_string(const char *s) _pure_; -void server_fix_perms(Server *s, JournalFile *f, uid_t uid); int server_init(Server *s); void server_done(Server *s); void server_sync(Server *s); -void server_vacuum(Server *s); +int server_vacuum(Server *s, bool verbose, bool patch_min_use); void server_rotate(Server *s); int server_schedule_sync(Server *s, int priority); int server_flush_to_var(Server *s); diff --git a/src/journal/journald-stream.c b/src/journal/journald-stream.c index 69e2d41863..90884b6929 100644 --- a/src/journal/journald-stream.c +++ b/src/journal/journald-stream.c @@ -19,25 +19,35 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <unistd.h> #include <stddef.h> +#include <unistd.h> #ifdef HAVE_SELINUX #include <selinux/selinux.h> #endif -#include "sd-event.h" #include "sd-daemon.h" -#include "socket-util.h" -#include "selinux-util.h" -#include "mkdir.h" +#include "sd-event.h" + +#include "alloc-util.h" +#include "dirent-util.h" +#include "escape.h" +#include "fd-util.h" #include "fileio.h" +#include "io-util.h" +#include "journald-console.h" +#include "journald-kmsg.h" #include "journald-server.h" #include "journald-stream.h" #include "journald-syslog.h" -#include "journald-kmsg.h" -#include "journald-console.h" #include "journald-wall.h" +#include "mkdir.h" +#include "parse-util.h" +#include "selinux-util.h" +#include "socket-util.h" +#include "stdio-util.h" +#include "string-util.h" +#include "syslog-util.h" #define STDOUT_STREAMS_MAX 4096 @@ -69,6 +79,7 @@ struct StdoutStream { bool forward_to_console:1; bool fdstore:1; + bool in_notify_queue:1; char buffer[LINE_MAX+1]; size_t length; @@ -78,6 +89,7 @@ struct StdoutStream { char *state_file; LIST_FIELDS(StdoutStream, stdout_stream); + LIST_FIELDS(StdoutStream, stdout_stream_notify_queue); }; void stdout_stream_free(StdoutStream *s) { @@ -88,6 +100,9 @@ void stdout_stream_free(StdoutStream *s) { assert(s->server->n_stdout_streams > 0); s->server->n_stdout_streams --; LIST_REMOVE(stdout_stream, s->server->stdout_streams, s); + + if (s->in_notify_queue) + LIST_REMOVE(stdout_stream_notify_queue, s->server->stdout_streams_notify_queue, s); } if (s->event_source) { @@ -111,7 +126,7 @@ static void stdout_stream_destroy(StdoutStream *s) { return; if (s->state_file) - unlink(s->state_file); + (void) unlink(s->state_file); stdout_stream_free(s); } @@ -190,11 +205,15 @@ static int stdout_stream_save(StdoutStream *s) { goto fail; } - /* Store the connection fd in PID 1, so that we get it passed - * in again on next start */ - if (!s->fdstore) { - sd_pid_notify_with_fds(0, false, "FDSTORE=1", &s->fd, 1); - s->fdstore = true; + if (!s->fdstore && !s->in_notify_queue) { + LIST_PREPEND(stdout_stream_notify_queue, s->server->stdout_streams_notify_queue, s); + s->in_notify_queue = true; + + if (s->server->notify_event_source) { + r = sd_event_source_set_enabled(s->server->notify_event_source, SD_EVENT_ON); + if (r < 0) + log_warning_errno(r, "Failed to enable notify event source: %m"); + } } return 0; @@ -220,14 +239,14 @@ static int stdout_stream_log(StdoutStream *s, const char *p) { assert(s); assert(p); - if (isempty(p)) - return 0; - priority = s->priority; if (s->level_prefix) syslog_parse_priority(&p, &priority, false); + if (isempty(p)) + return 0; + if (s->forward_to_syslog || s->server->forward_to_syslog) server_forward_syslog(s->server, syslog_fixup_facility(priority), s->identifier, p, &s->ucred, NULL); @@ -267,10 +286,12 @@ static int stdout_stream_log(StdoutStream *s, const char *p) { static int stdout_stream_line(StdoutStream *s, char *p) { int r; + char *orig; assert(s); assert(p); + orig = p; p = strstrip(p); switch (s->state) { @@ -359,7 +380,7 @@ static int stdout_stream_line(StdoutStream *s, char *p) { return 0; case STDOUT_STREAM_RUNNING: - return stdout_stream_log(s, p); + return stdout_stream_log(s, orig); } assert_not_reached("Unknown stream state"); @@ -472,7 +493,7 @@ static int stdout_stream_install(Server *s, int fd, StdoutStream **ret) { if (r < 0) return log_error_errno(r, "Failed to determine peer credentials: %m"); - if (mac_selinux_use()) { + if (mac_selinux_have()) { r = getpeersec(fd, &stream->label); if (r < 0 && r != -EOPNOTSUPP) (void) log_warning_errno(r, "Failed to determine peer security context: %m"); @@ -519,8 +540,7 @@ static int stdout_stream_new(sd_event_source *es, int listen_fd, uint32_t revent if (errno == EAGAIN) return 0; - log_error_errno(errno, "Failed to accept stdout connection: %m"); - return -errno; + return log_error_errno(errno, "Failed to accept stdout connection: %m"); } if (s->n_stdout_streams >= STDOUT_STREAMS_MAX) { @@ -627,7 +647,7 @@ static int stdout_stream_restore(Server *s, const char *fname, int fd) { return 0; } -static int server_restore_streams(Server *s, FDSet *fds) { +int server_restore_streams(Server *s, FDSet *fds) { _cleanup_closedir_ DIR *d = NULL; struct dirent *de; int r; @@ -681,7 +701,7 @@ fail: return log_error_errno(errno, "Failed to read streams directory: %m"); } -int server_open_stdout_socket(Server *s, FDSet *fds) { +int server_open_stdout_socket(Server *s) { int r; assert(s); @@ -713,12 +733,56 @@ int server_open_stdout_socket(Server *s, FDSet *fds) { if (r < 0) return log_error_errno(r, "Failed to add stdout server fd to event source: %m"); - r = sd_event_source_set_priority(s->stdout_event_source, SD_EVENT_PRIORITY_NORMAL+10); + r = sd_event_source_set_priority(s->stdout_event_source, SD_EVENT_PRIORITY_NORMAL+5); if (r < 0) return log_error_errno(r, "Failed to adjust priority of stdout server event source: %m"); - /* Try to restore streams, but don't bother if this fails */ - (void) server_restore_streams(s, fds); - return 0; } + +void stdout_stream_send_notify(StdoutStream *s) { + struct iovec iovec = { + .iov_base = (char*) "FDSTORE=1", + .iov_len = strlen("FDSTORE=1"), + }; + struct msghdr msghdr = { + .msg_iov = &iovec, + .msg_iovlen = 1, + }; + struct cmsghdr *cmsg; + ssize_t l; + + assert(s); + assert(!s->fdstore); + assert(s->in_notify_queue); + assert(s->server); + assert(s->server->notify_fd >= 0); + + /* Store the connection fd in PID 1, so that we get it passed + * in again on next start */ + + msghdr.msg_controllen = CMSG_SPACE(sizeof(int)); + msghdr.msg_control = alloca0(msghdr.msg_controllen); + + cmsg = CMSG_FIRSTHDR(&msghdr); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(int)); + + memcpy(CMSG_DATA(cmsg), &s->fd, sizeof(int)); + + l = sendmsg(s->server->notify_fd, &msghdr, MSG_DONTWAIT|MSG_NOSIGNAL); + if (l < 0) { + if (errno == EAGAIN) + return; + + log_error_errno(errno, "Failed to send stream file descriptor to service manager: %m"); + } else { + log_debug("Successfully sent stream file descriptor to service manager."); + s->fdstore = 1; + } + + LIST_REMOVE(stdout_stream_notify_queue, s->server->stdout_streams_notify_queue, s); + s->in_notify_queue = false; + +} diff --git a/src/journal/journald-stream.h b/src/journal/journald-stream.h index 94bf955d78..e3497f0ded 100644 --- a/src/journal/journald-stream.h +++ b/src/journal/journald-stream.h @@ -21,9 +21,13 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ +typedef struct StdoutStream StdoutStream; + #include "fdset.h" #include "journald-server.h" -int server_open_stdout_socket(Server *s, FDSet *fds); +int server_open_stdout_socket(Server *s); +int server_restore_streams(Server *s, FDSet *fds); void stdout_stream_free(StdoutStream *s); +void stdout_stream_send_notify(StdoutStream *s); diff --git a/src/journal/journald-syslog.c b/src/journal/journald-syslog.c index ffba451955..0be73088e2 100644 --- a/src/journal/journald-syslog.c +++ b/src/journal/journald-syslog.c @@ -19,20 +19,27 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <unistd.h> #include <stddef.h> #include <sys/epoll.h> +#include <unistd.h> -#include "systemd/sd-messages.h" -#include "socket-util.h" -#include "selinux-util.h" +#include "sd-messages.h" + +#include "alloc-util.h" +#include "fd-util.h" +#include "formats-util.h" +#include "io-util.h" +#include "journald-console.h" +#include "journald-kmsg.h" #include "journald-server.h" #include "journald-syslog.h" -#include "journald-kmsg.h" -#include "journald-console.h" #include "journald-wall.h" -#include "formats-util.h" #include "process-util.h" +#include "selinux-util.h" +#include "socket-util.h" +#include "stdio-util.h" +#include "string-util.h" +#include "syslog-util.h" /* Warn once every 30s if we missed syslog message */ #define WARN_FORWARD_SYSLOG_MISSED_USEC (30 * USEC_PER_SEC) @@ -319,7 +326,7 @@ void server_process_syslog_message( size_t label_len) { char syslog_priority[sizeof("PRIORITY=") + DECIMAL_STR_MAX(int)], - syslog_facility[sizeof("SYSLOG_FACILITY") + DECIMAL_STR_MAX(int)]; + syslog_facility[sizeof("SYSLOG_FACILITY=") + DECIMAL_STR_MAX(int)]; const char *message = NULL, *syslog_identifier = NULL, *syslog_pid = NULL; struct iovec iovec[N_IOVEC_META_FIELDS + 6]; unsigned n = 0; @@ -350,11 +357,11 @@ void server_process_syslog_message( IOVEC_SET_STRING(iovec[n++], "_TRANSPORT=syslog"); - sprintf(syslog_priority, "PRIORITY=%i", priority & LOG_PRIMASK); + xsprintf(syslog_priority, "PRIORITY=%i", priority & LOG_PRIMASK); IOVEC_SET_STRING(iovec[n++], syslog_priority); if (priority & LOG_FACMASK) { - sprintf(syslog_facility, "SYSLOG_FACILITY=%i", LOG_FAC(priority)); + xsprintf(syslog_facility, "SYSLOG_FACILITY=%i", LOG_FAC(priority)); IOVEC_SET_STRING(iovec[n++], syslog_facility); } @@ -408,7 +415,7 @@ int server_open_syslog_socket(Server *s) { return log_error_errno(errno, "SO_PASSCRED failed: %m"); #ifdef HAVE_SELINUX - if (mac_selinux_use()) { + if (mac_selinux_have()) { r = setsockopt(s->syslog_fd, SOL_SOCKET, SO_PASSSEC, &one, sizeof(one)); if (r < 0) log_warning_errno(errno, "SO_PASSSEC failed: %m"); @@ -423,6 +430,10 @@ int server_open_syslog_socket(Server *s) { if (r < 0) return log_error_errno(r, "Failed to add syslog server fd to event loop: %m"); + r = sd_event_source_set_priority(s->syslog_event_source, SD_EVENT_PRIORITY_NORMAL+5); + if (r < 0) + return log_error_errno(r, "Failed to adjust syslog event source priority: %m"); + return 0; } diff --git a/src/journal/journald-wall.c b/src/journal/journald-wall.c index 7863766ae7..88bea3b86e 100644 --- a/src/journal/journald-wall.c +++ b/src/journal/journald-wall.c @@ -19,11 +19,13 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include "utmp-wtmp.h" +#include "alloc-util.h" +#include "formats-util.h" #include "journald-server.h" #include "journald-wall.h" -#include "formats-util.h" #include "process-util.h" +#include "string-util.h" +#include "utmp-wtmp.h" void server_forward_wall( Server *s, diff --git a/src/journal/journald.c b/src/journal/journald.c index b2624c6d28..b9f5c099e1 100644 --- a/src/journal/journald.c +++ b/src/journal/journald.c @@ -21,16 +21,15 @@ #include <unistd.h> -#include "systemd/sd-messages.h" -#include "systemd/sd-daemon.h" +#include "sd-daemon.h" +#include "sd-messages.h" +#include "formats-util.h" #include "journal-authenticate.h" -#include "journald-server.h" #include "journald-kmsg.h" +#include "journald-server.h" #include "journald-syslog.h" - #include "sigbus.h" -#include "formats-util.h" int main(int argc, char *argv[]) { Server server; @@ -54,17 +53,13 @@ int main(int argc, char *argv[]) { if (r < 0) goto finish; - server_vacuum(&server); + server_vacuum(&server, false, false); server_flush_to_var(&server); server_flush_dev_kmsg(&server); log_debug("systemd-journald running as pid "PID_FMT, getpid()); server_driver_message(&server, SD_MESSAGE_JOURNAL_START, "Journal started"); - sd_notify(false, - "READY=1\n" - "STATUS=Processing requests..."); - for (;;) { usec_t t = USEC_INFINITY, n; @@ -82,7 +77,7 @@ int main(int argc, char *argv[]) { if (server.oldest_file_usec + server.max_retention_usec < n) { log_info("Retention time reached."); server_rotate(&server); - server_vacuum(&server); + server_vacuum(&server, false, false); continue; } @@ -117,10 +112,6 @@ int main(int argc, char *argv[]) { server_driver_message(&server, SD_MESSAGE_JOURNAL_STOP, "Journal stopped"); finish: - sd_notify(false, - "STOPPING=1\n" - "STATUS=Shutting down..."); - server_done(&server); return r < 0 ? EXIT_FAILURE : EXIT_SUCCESS; diff --git a/src/journal/journald.conf b/src/journal/journald.conf index 47eefe91c1..7beb96c671 100644 --- a/src/journal/journald.conf +++ b/src/journal/journald.conf @@ -22,9 +22,11 @@ #SystemMaxUse= #SystemKeepFree= #SystemMaxFileSize= +#SystemMaxFiles=100 #RuntimeMaxUse= #RuntimeKeepFree= #RuntimeMaxFileSize= +#RuntimeMaxFiles=100 #MaxRetentionSec= #MaxFileSec=1month #ForwardToSyslog=no diff --git a/src/journal/lookup3.c b/src/journal/lookup3.c index 52ffdf7b1d..3d791234f4 100644 --- a/src/journal/lookup3.c +++ b/src/journal/lookup3.c @@ -40,10 +40,10 @@ on 1 byte), but shoehorning those bytes into integers efficiently is messy. */ /* #define SELF_TEST 1 */ -#include <stdio.h> /* defines printf for tests */ -#include <time.h> /* defines time_t for timings in the test */ #include <stdint.h> /* defines uint32_t etc */ +#include <stdio.h> /* defines printf for tests */ #include <sys/param.h> /* attempt to define endianness */ +#include <time.h> /* defines time_t for timings in the test */ #ifdef linux # include <endian.h> /* attempt to define endianness */ #endif diff --git a/src/journal/mmap-cache.c b/src/journal/mmap-cache.c index 22f75540b8..eb4b092e80 100644 --- a/src/journal/mmap-cache.c +++ b/src/journal/mmap-cache.c @@ -23,13 +23,15 @@ #include <stdlib.h> #include <sys/mman.h> +#include "alloc-util.h" +#include "fd-util.h" #include "hashmap.h" #include "list.h" #include "log.h" -#include "util.h" #include "macro.h" -#include "sigbus.h" #include "mmap-cache.h" +#include "sigbus.h" +#include "util.h" typedef struct Window Window; typedef struct Context Context; @@ -288,7 +290,7 @@ static void fd_free(FileDescriptor *f) { window_free(f->windows); if (f->cache) - assert_se(hashmap_remove(f->cache->fds, INT_TO_PTR(f->fd + 1))); + assert_se(hashmap_remove(f->cache->fds, FD_TO_PTR(f->fd))); free(f); } @@ -300,7 +302,7 @@ static FileDescriptor* fd_add(MMapCache *m, int fd) { assert(m); assert(fd >= 0); - f = hashmap_get(m->fds, INT_TO_PTR(fd + 1)); + f = hashmap_get(m->fds, FD_TO_PTR(fd)); if (f) return f; @@ -315,7 +317,7 @@ static FileDescriptor* fd_add(MMapCache *m, int fd) { f->cache = m; f->fd = fd; - r = hashmap_put(m->fds, UINT_TO_PTR(fd + 1), f); + r = hashmap_put(m->fds, FD_TO_PTR(fd), f); if (r < 0) { free(f); return NULL; @@ -346,7 +348,10 @@ static void mmap_cache_free(MMapCache *m) { } MMapCache* mmap_cache_unref(MMapCache *m) { - assert(m); + + if (!m) + return NULL; + assert(m->n_ref > 0); m->n_ref --; @@ -428,7 +433,7 @@ static int find_mmap( assert(fd >= 0); assert(size > 0); - f = hashmap_get(m->fds, INT_TO_PTR(fd + 1)); + f = hashmap_get(m->fds, FD_TO_PTR(fd)); if (!f) return 0; @@ -678,7 +683,7 @@ bool mmap_cache_got_sigbus(MMapCache *m, int fd) { mmap_cache_process_sigbus(m); - f = hashmap_get(m->fds, INT_TO_PTR(fd + 1)); + f = hashmap_get(m->fds, FD_TO_PTR(fd)); if (!f) return false; @@ -697,7 +702,7 @@ void mmap_cache_close_fd(MMapCache *m, int fd) { mmap_cache_process_sigbus(m); - f = hashmap_get(m->fds, INT_TO_PTR(fd + 1)); + f = hashmap_get(m->fds, FD_TO_PTR(fd)); if (!f) return; diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c index 13fa9b52fc..cd5160154a 100644 --- a/src/journal/sd-journal.c +++ b/src/journal/sd-journal.c @@ -21,29 +21,38 @@ #include <errno.h> #include <fcntl.h> +#include <linux/magic.h> +#include <poll.h> #include <stddef.h> -#include <unistd.h> #include <sys/inotify.h> -#include <poll.h> #include <sys/vfs.h> -#include <linux/magic.h> +#include <unistd.h> #include "sd-journal.h" + +#include "alloc-util.h" +#include "catalog.h" +#include "compress.h" +#include "dirent-util.h" +#include "fd-util.h" +#include "fileio.h" +#include "formats-util.h" +#include "fs-util.h" +#include "hashmap.h" +#include "hostname-util.h" +#include "io-util.h" #include "journal-def.h" #include "journal-file.h" -#include "hashmap.h" +#include "journal-internal.h" #include "list.h" -#include "strv.h" -#include "path-util.h" #include "lookup3.h" -#include "compress.h" -#include "journal-internal.h" #include "missing.h" -#include "catalog.h" +#include "path-util.h" #include "replace-var.h" -#include "fileio.h" -#include "formats-util.h" -#include "hostname-util.h" +#include "stat-util.h" +#include "stdio-util.h" +#include "string-util.h" +#include "strv.h" #define JOURNAL_FILES_MAX 7168 @@ -64,19 +73,46 @@ static bool journal_pid_changed(sd_journal *j) { return j->original_pid != getpid(); } -/* We return an error here only if we didn't manage to - memorize the real error. */ -static int set_put_error(sd_journal *j, int r) { +static int journal_put_error(sd_journal *j, int r, const char *path) { + char *copy; int k; + /* Memorize an error we encountered, and store which + * file/directory it was generated from. Note that we store + * only *one* path per error code, as the error code is the + * key into the hashmap, and the path is the value. This means + * we keep track only of all error kinds, but not of all error + * locations. This has the benefit that the hashmap cannot + * grow beyond bounds. + * + * We return an error here only if we didn't manage to + * memorize the real error. */ + if (r >= 0) return r; - k = set_ensure_allocated(&j->errors, NULL); + k = hashmap_ensure_allocated(&j->errors, NULL); if (k < 0) return k; - return set_put(j->errors, INT_TO_PTR(r)); + if (path) { + copy = strdup(path); + if (!copy) + return -ENOMEM; + } else + copy = NULL; + + k = hashmap_put(j->errors, INT_TO_PTR(r), copy); + if (k < 0) { + free(copy); + + if (k == -EEXIST) + return 0; + + return k; + } + + return 0; } static void detach_location(sd_journal *j) { @@ -1016,8 +1052,6 @@ _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) { _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) { int r; - const char *word, *state; - size_t l; Object *o; assert_return(j, -EINVAL); @@ -1031,20 +1065,23 @@ _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) { if (r < 0) return r; - FOREACH_WORD_SEPARATOR(word, l, cursor, ";", state) { + for(;;) { _cleanup_free_ char *item = NULL; - sd_id128_t id; unsigned long long ll; + sd_id128_t id; int k = 0; - if (l < 2 || word[1] != '=') - return -EINVAL; + r = extract_first_word(&cursor, &item, ";", EXTRACT_DONT_COALESCE_SEPARATORS); + if (r < 0) + return r; - item = strndup(word, l); - if (!item) - return -ENOMEM; + if (r == 0) + break; - switch (word[0]) { + if (strlen(item) < 2 || item[1] != '=') + return -EINVAL; + + switch (item[0]) { case 's': k = sd_id128_from_string(item+2, &id); @@ -1173,6 +1210,8 @@ static bool file_has_type_prefix(const char *prefix, const char *filename) { } static bool file_type_wanted(int flags, const char *filename) { + assert(filename); + if (!endswith(filename, ".journal") && !endswith(filename, ".journal~")) return false; @@ -1197,7 +1236,7 @@ static bool file_type_wanted(int flags, const char *filename) { static int add_any_file(sd_journal *j, const char *path) { JournalFile *f = NULL; - int r; + int r, k; assert(j); assert(path); @@ -1206,20 +1245,23 @@ static int add_any_file(sd_journal *j, const char *path) { return 0; if (ordered_hashmap_size(j->files) >= JOURNAL_FILES_MAX) { - log_warning("Too many open journal files, not adding %s.", path); - return set_put_error(j, -ETOOMANYREFS); + log_debug("Too many open journal files, not adding %s.", path); + r = -ETOOMANYREFS; + goto fail; } r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f); - if (r < 0) - return r; + if (r < 0) { + log_debug_errno(r, "Failed to open journal file %s: %m", path); + goto fail; + } /* journal_file_dump(f); */ r = ordered_hashmap_put(j->files, f->path, f); if (r < 0) { journal_file_close(f); - return r; + goto fail; } log_debug("File %s added.", f->path); @@ -1229,11 +1271,17 @@ static int add_any_file(sd_journal *j, const char *path) { j->current_invalidate_counter ++; return 0; + +fail: + k = journal_put_error(j, r, path); + if (k < 0) + return k; + + return r; } static int add_file(sd_journal *j, const char *prefix, const char *filename) { - _cleanup_free_ char *path = NULL; - int r; + const char *path; assert(j); assert(prefix); @@ -1243,34 +1291,24 @@ static int add_file(sd_journal *j, const char *prefix, const char *filename) { !file_type_wanted(j->flags, filename)) return 0; - path = strjoin(prefix, "/", filename, NULL); - if (!path) - return -ENOMEM; - - r = add_any_file(j, path); - if (r == -ENOENT) - return 0; - return r; + path = strjoina(prefix, "/", filename); + return add_any_file(j, path); } -static int remove_file(sd_journal *j, const char *prefix, const char *filename) { - _cleanup_free_ char *path; +static void remove_file(sd_journal *j, const char *prefix, const char *filename) { + const char *path; JournalFile *f; assert(j); assert(prefix); assert(filename); - path = strjoin(prefix, "/", filename, NULL); - if (!path) - return -ENOMEM; - + path = strjoina(prefix, "/", filename); f = ordered_hashmap_get(j->files, path); if (!f) - return 0; + return; remove_file_real(j, f); - return 0; } static void remove_file_real(sd_journal *j, JournalFile *f) { @@ -1299,12 +1337,27 @@ static void remove_file_real(sd_journal *j, JournalFile *f) { j->current_invalidate_counter ++; } +static int dirname_is_machine_id(const char *fn) { + sd_id128_t id, machine; + int r; + + r = sd_id128_get_machine(&machine); + if (r < 0) + return r; + + r = sd_id128_from_string(fn, &id); + if (r < 0) + return r; + + return sd_id128_equal(id, machine); +} + static int add_directory(sd_journal *j, const char *prefix, const char *dirname) { _cleanup_free_ char *path = NULL; - int r; _cleanup_closedir_ DIR *d = NULL; - sd_id128_t id, mid; + struct dirent *de = NULL; Directory *m; + int r, k; assert(j); assert(prefix); @@ -1313,35 +1366,36 @@ static int add_directory(sd_journal *j, const char *prefix, const char *dirname) log_debug("Considering %s/%s.", prefix, dirname); if ((j->flags & SD_JOURNAL_LOCAL_ONLY) && - (sd_id128_from_string(dirname, &id) < 0 || - sd_id128_get_machine(&mid) < 0 || - !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run")))) + !(dirname_is_machine_id(dirname) > 0 || path_startswith(prefix, "/run"))) return 0; path = strjoin(prefix, "/", dirname, NULL); - if (!path) - return -ENOMEM; + if (!path) { + r = -ENOMEM; + goto fail; + } d = opendir(path); if (!d) { - log_debug_errno(errno, "Failed to open %s: %m", path); - if (errno == ENOENT) - return 0; - return -errno; + r = log_debug_errno(errno, "Failed to open directory %s: %m", path); + goto fail; } m = hashmap_get(j->directories_by_path, path); if (!m) { m = new0(Directory, 1); - if (!m) - return -ENOMEM; + if (!m) { + r = -ENOMEM; + goto fail; + } m->is_root = false; m->path = path; if (hashmap_put(j->directories_by_path, m->path, m) < 0) { free(m); - return -ENOMEM; + r = -ENOMEM; + goto fail; } path = NULL; /* avoid freeing in cleanup */ @@ -1363,41 +1417,30 @@ static int add_directory(sd_journal *j, const char *prefix, const char *dirname) inotify_rm_watch(j->inotify_fd, m->wd); } - for (;;) { - struct dirent *de; - - errno = 0; - de = readdir(d); - if (!de && errno != 0) { - r = -errno; - log_debug_errno(errno, "Failed to read directory %s: %m", m->path); - return r; - } - if (!de) - break; + FOREACH_DIRENT_ALL(de, d, return log_debug_errno(errno, "Failed to read directory %s: %m", m->path)) { if (dirent_is_file_with_suffix(de, ".journal") || - dirent_is_file_with_suffix(de, ".journal~")) { - r = add_file(j, m->path, de->d_name); - if (r < 0) { - log_debug_errno(r, "Failed to add file %s/%s: %m", - m->path, de->d_name); - r = set_put_error(j, r); - if (r < 0) - return r; - } - } + dirent_is_file_with_suffix(de, ".journal~")) + (void) add_file(j, m->path, de->d_name); } check_network(j, dirfd(d)); return 0; + +fail: + k = journal_put_error(j, r, path ?: dirname); + if (k < 0) + return k; + + return r; } -static int add_root_directory(sd_journal *j, const char *p) { +static int add_root_directory(sd_journal *j, const char *p, bool missing_ok) { _cleanup_closedir_ DIR *d = NULL; + struct dirent *de; Directory *m; - int r; + int r, k; assert(j); assert(p); @@ -1410,26 +1453,35 @@ static int add_root_directory(sd_journal *j, const char *p) { p = strjoina(j->prefix, p); d = opendir(p); - if (!d) - return -errno; + if (!d) { + if (errno == ENOENT && missing_ok) + return 0; + + r = log_debug_errno(errno, "Failed to open root directory %s: %m", p); + goto fail; + } m = hashmap_get(j->directories_by_path, p); if (!m) { m = new0(Directory, 1); - if (!m) - return -ENOMEM; + if (!m) { + r = -ENOMEM; + goto fail; + } m->is_root = true; m->path = strdup(p); if (!m->path) { free(m); - return -ENOMEM; + r = -ENOMEM; + goto fail; } if (hashmap_put(j->directories_by_path, m->path, m) < 0) { free(m->path); free(m); - return -ENOMEM; + r = -ENOMEM; + goto fail; } j->current_invalidate_counter ++; @@ -1452,42 +1504,27 @@ static int add_root_directory(sd_journal *j, const char *p) { if (j->no_new_files) return 0; - for (;;) { - struct dirent *de; + FOREACH_DIRENT_ALL(de, d, return log_debug_errno(errno, "Failed to read directory %s: %m", m->path)) { sd_id128_t id; - errno = 0; - de = readdir(d); - if (!de && errno != 0) { - r = -errno; - log_debug_errno(errno, "Failed to read directory %s: %m", m->path); - return r; - } - if (!de) - break; - if (dirent_is_file_with_suffix(de, ".journal") || - dirent_is_file_with_suffix(de, ".journal~")) { - r = add_file(j, m->path, de->d_name); - if (r < 0) { - log_debug_errno(r, "Failed to add file %s/%s: %m", - m->path, de->d_name); - r = set_put_error(j, r); - if (r < 0) - return r; - } - } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) && - sd_id128_from_string(de->d_name, &id) >= 0) { - - r = add_directory(j, m->path, de->d_name); - if (r < 0) - log_debug_errno(r, "Failed to add directory %s/%s: %m", m->path, de->d_name); - } + dirent_is_file_with_suffix(de, ".journal~")) + (void) add_file(j, m->path, de->d_name); + else if (IN_SET(de->d_type, DT_DIR, DT_LNK, DT_UNKNOWN) && + sd_id128_from_string(de->d_name, &id) >= 0) + (void) add_directory(j, m->path, de->d_name); } check_network(j, dirfd(d)); return 0; + +fail: + k = journal_put_error(j, r, p); + if (k < 0) + return k; + + return r; } static void remove_directory(sd_journal *j, Directory *d) { @@ -1512,8 +1549,8 @@ static void remove_directory(sd_journal *j, Directory *d) { } static int add_search_paths(sd_journal *j) { - int r; - const char search_paths[] = + + static const char search_paths[] = "/run/log/journal\0" "/var/log/journal\0"; const char *p; @@ -1523,14 +1560,8 @@ static int add_search_paths(sd_journal *j) { /* We ignore most errors here, since the idea is to only open * what's actually accessible, and ignore the rest. */ - NULSTR_FOREACH(p, search_paths) { - r = add_root_directory(j, p); - if (r < 0 && r != -ENOENT) { - r = set_put_error(j, r); - if (r < 0) - return r; - } - } + NULSTR_FOREACH(p, search_paths) + (void) add_root_directory(j, p, true); return 0; } @@ -1554,17 +1585,14 @@ static int add_current_paths(sd_journal *j) { if (!dir) return -ENOMEM; - r = add_root_directory(j, dir); - if (r < 0) { - set_put_error(j, r); + r = add_root_directory(j, dir, true); + if (r < 0) return r; - } } return 0; } - static int allocate_inotify(sd_journal *j) { assert(j); @@ -1692,11 +1720,9 @@ _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int f if (!j) return -ENOMEM; - r = add_root_directory(j, path); - if (r < 0) { - set_put_error(j, r); + r = add_root_directory(j, path, false); + if (r < 0) goto fail; - } *ret = j; return 0; @@ -1721,10 +1747,8 @@ _public_ int sd_journal_open_files(sd_journal **ret, const char **paths, int fla STRV_FOREACH(path, paths) { r = add_any_file(j, *path); - if (r < 0) { - log_error_errno(r, "Failed to open %s: %m", *path); + if (r < 0) goto fail; - } } j->no_new_files = true; @@ -1741,6 +1765,7 @@ fail: _public_ void sd_journal_close(sd_journal *j) { Directory *d; JournalFile *f; + char *p; if (!j) return; @@ -1768,10 +1793,13 @@ _public_ void sd_journal_close(sd_journal *j) { mmap_cache_unref(j->mmap); } + while ((p = hashmap_steal_first(j->errors))) + free(p); + hashmap_free(j->errors); + free(j->path); free(j->prefix); free(j->unique_field); - set_free(j->errors); free(j); } @@ -1912,10 +1940,14 @@ _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void ** compression = o->object.flags & OBJECT_COMPRESSION_MASK; if (compression) { #if defined(HAVE_XZ) || defined(HAVE_LZ4) - if (decompress_startswith(compression, + r = decompress_startswith(compression, o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, - field, field_length, '=')) { + field, field_length, '='); + if (r < 0) + log_debug_errno(r, "Cannot decompress %s object of length %zu at offset "OFSfmt": %m", + object_compressed_to_string(compression), l, p); + else if (r > 0) { size_t rsize; @@ -2064,7 +2096,7 @@ _public_ int sd_journal_get_fd(sd_journal *j) { if (j->no_new_files) r = add_current_paths(j); else if (j->path) - r = add_root_directory(j, j->path); + r = add_root_directory(j, j->path, true); else r = add_search_paths(j); if (r < 0) @@ -2111,7 +2143,6 @@ _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) { static void process_inotify_event(sd_journal *j, struct inotify_event *e) { Directory *d; - int r; assert(j); assert(e); @@ -2127,20 +2158,10 @@ static void process_inotify_event(sd_journal *j, struct inotify_event *e) { /* Event for a journal file */ - if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) { - r = add_file(j, d->path, e->name); - if (r < 0) { - log_debug_errno(r, "Failed to add file %s/%s: %m", - d->path, e->name); - set_put_error(j, r); - } - - } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) { - - r = remove_file(j, d->path, e->name); - if (r < 0) - log_debug_errno(r, "Failed to remove file %s/%s: %m", d->path, e->name); - } + if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) + (void) add_file(j, d->path, e->name); + else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) + remove_file(j, d->path, e->name); } else if (!d->is_root && e->len == 0) { @@ -2153,11 +2174,8 @@ static void process_inotify_event(sd_journal *j, struct inotify_event *e) { /* Event for root directory */ - if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) { - r = add_directory(j, d->path, e->name); - if (r < 0) - log_debug_errno(r, "Failed to add directory %s/%s: %m", d->path, e->name); - } + if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) + (void) add_directory(j, d->path, e->name); } return; @@ -2166,7 +2184,7 @@ static void process_inotify_event(sd_journal *j, struct inotify_event *e) { if (e->mask & IN_IGNORED) return; - log_warning("Unknown inotify event."); + log_debug("Unknown inotify event."); } static int determine_change(sd_journal *j) { diff --git a/src/journal/stacktrace.c b/src/journal/stacktrace.c index 98a54ff269..4305462f80 100644 --- a/src/journal/stacktrace.c +++ b/src/journal/stacktrace.c @@ -22,10 +22,13 @@ #include <dwarf.h> #include <elfutils/libdwfl.h> -#include "util.h" +#include "alloc-util.h" +#include "fd-util.h" +#include "formats-util.h" #include "macro.h" #include "stacktrace.h" -#include "formats-util.h" +#include "string-util.h" +#include "util.h" #define FRAMES_MAX 64 #define THREADS_MAX 64 diff --git a/src/journal/test-catalog.c b/src/journal/test-catalog.c index dbfdea609d..25980b7744 100644 --- a/src/journal/test-catalog.c +++ b/src/journal/test-catalog.c @@ -20,16 +20,21 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <locale.h> -#include <unistd.h> #include <errno.h> #include <fcntl.h> +#include <locale.h> +#include <unistd.h> -#include "util.h" -#include "log.h" -#include "macro.h" #include "sd-messages.h" + +#include "alloc-util.h" #include "catalog.h" +#include "fd-util.h" +#include "fileio.h" +#include "log.h" +#include "macro.h" +#include "string-util.h" +#include "util.h" static const char *catalog_dirs[] = { CATALOG_DIR, diff --git a/src/journal/test-compress-benchmark.c b/src/journal/test-compress-benchmark.c index c8e5b76c6c..baed0d82a4 100644 --- a/src/journal/test-compress-benchmark.c +++ b/src/journal/test-compress-benchmark.c @@ -1,3 +1,5 @@ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ + /*** This file is part of systemd @@ -17,30 +19,74 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ +#include "alloc-util.h" #include "compress.h" -#include "util.h" #include "macro.h" +#include "parse-util.h" +#include "random-util.h" +#include "string-util.h" +#include "util.h" -typedef int (compress_t)(const void *src, uint64_t src_size, void *dst, size_t *dst_size); +typedef int (compress_t)(const void *src, uint64_t src_size, void *dst, + size_t dst_alloc_size, size_t *dst_size); typedef int (decompress_t)(const void *src, uint64_t src_size, void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max); +static usec_t arg_duration = 2 * USEC_PER_SEC; +static size_t arg_start; + #define MAX_SIZE (1024*1024LU) +#define PRIME 1048571 /* A prime close enough to one megabyte that mod 4 == 3 */ + +static size_t _permute(size_t x) { + size_t residue; + + if (x >= PRIME) + return x; + + residue = x*x % PRIME; + if (x <= PRIME / 2) + return residue; + else + return PRIME - residue; +} + +static size_t permute(size_t x) { + return _permute((_permute(x) + arg_start) % MAX_SIZE ^ 0xFF345); +} -static char* make_buf(size_t count) { +static char* make_buf(size_t count, const char *type) { char *buf; size_t i; buf = malloc(count); assert_se(buf); - for (i = 0; i < count; i++) - buf[i] = 'a' + i % ('z' - 'a' + 1); + if (streq(type, "zeros")) + memzero(buf, count); + else if (streq(type, "simple")) + for (i = 0; i < count; i++) + buf[i] = 'a' + i % ('z' - 'a' + 1); + else if (streq(type, "random")) { + size_t step = count / 10; + + random_bytes(buf, step); + memzero(buf + 1*step, step); + random_bytes(buf + 2*step, step); + memzero(buf + 3*step, step); + random_bytes(buf + 4*step, step); + memzero(buf + 5*step, step); + random_bytes(buf + 6*step, step); + memzero(buf + 7*step, step); + random_bytes(buf + 8*step, step); + memzero(buf + 9*step, step); + } else + assert_not_reached("here"); return buf; } -static void test_compress_decompress(const char* label, +static void test_compress_decompress(const char* label, const char* type, compress_t compress, decompress_t decompress) { usec_t n, n2 = 0; float dt; @@ -50,64 +96,85 @@ static void test_compress_decompress(const char* label, size_t buf2_allocated = 0; size_t skipped = 0, compressed = 0, total = 0; - text = make_buf(MAX_SIZE); + text = make_buf(MAX_SIZE, type); buf = calloc(MAX_SIZE + 1, 1); assert_se(text && buf); n = now(CLOCK_MONOTONIC); - for (size_t i = 1; i <= MAX_SIZE; i += (i < 2048 ? 1 : 217)) { - size_t j = 0, k = 0; + for (size_t i = 0; i <= MAX_SIZE; i++) { + size_t j = 0, k = 0, size; int r; - r = compress(text, i, buf, &j); - /* assume compression must be successful except for small inputs */ - assert_se(r == 0 || (i < 2048 && r == -ENOBUFS)); + size = permute(i); + + log_debug("%s %zu %zu", type, i, size); + + memzero(buf, MIN(size + 1000, MAX_SIZE)); + + r = compress(text, size, buf, size, &j); + /* assume compression must be successful except for small or random inputs */ + assert_se(r == 0 || (size < 2048 && r == -ENOBUFS) || streq(type, "random")); + /* check for overwrites */ - assert_se(buf[i] == 0); + assert_se(buf[size] == 0); if (r != 0) { - skipped += i; + skipped += size; continue; } assert_se(j > 0); - if (j >= i) - log_error("%s \"compressed\" %zu -> %zu", label, i, j); + if (j >= size) + log_error("%s \"compressed\" %zu -> %zu", label, size, j); r = decompress(buf, j, &buf2, &buf2_allocated, &k, 0); assert_se(r == 0); assert_se(buf2_allocated >= k); - assert_se(k == i); + assert_se(k == size); - assert_se(memcmp(text, buf2, i) == 0); + assert_se(memcmp(text, buf2, size) == 0); - total += i; + total += size; compressed += j; n2 = now(CLOCK_MONOTONIC); - if (n2 - n > 60 * USEC_PER_SEC) + if (n2 - n > arg_duration) break; } dt = (n2-n) / 1e6; - log_info("%s: compressed & decompressed %zu bytes in %.2fs (%.2fMiB/s), " + log_info("%s/%s: compressed & decompressed %zu bytes in %.2fs (%.2fMiB/s), " "mean compresion %.2f%%, skipped %zu bytes", - label, total, dt, + label, type, total, dt, total / 1024. / 1024 / dt, 100 - compressed * 100. / total, skipped); } int main(int argc, char *argv[]) { + const char *i; - log_set_max_level(LOG_DEBUG); + log_set_max_level(LOG_INFO); + if (argc >= 2) { + unsigned x; + + assert_se(safe_atou(argv[1], &x) >= 0); + arg_duration = x * USEC_PER_SEC; + } + if (argc == 3) + (void) safe_atolu(argv[2], &arg_start); + else + arg_start = getpid(); + + NULSTR_FOREACH(i, "zeros\0simple\0random\0") { #ifdef HAVE_XZ - test_compress_decompress("XZ", compress_blob_xz, decompress_blob_xz); + test_compress_decompress("XZ", i, compress_blob_xz, decompress_blob_xz); #endif #ifdef HAVE_LZ4 - test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4); + test_compress_decompress("LZ4", i, compress_blob_lz4, decompress_blob_lz4); #endif + } return 0; } diff --git a/src/journal/test-compress.c b/src/journal/test-compress.c index f17c00e60d..68c9a4d76c 100644 --- a/src/journal/test-compress.c +++ b/src/journal/test-compress.c @@ -17,10 +17,17 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ +#ifdef HAVE_LZ4 +#include <lz4.h> +#endif + +#include "alloc-util.h" #include "compress.h" -#include "util.h" +#include "fd-util.h" +#include "fileio.h" #include "macro.h" #include "random-util.h" +#include "util.h" #ifdef HAVE_XZ # define XZ_OK 0 @@ -35,7 +42,7 @@ #endif typedef int (compress_blob_t)(const void *src, uint64_t src_size, - void *dst, size_t *dst_size); + void *dst, size_t dst_alloc_size, size_t *dst_size); typedef int (decompress_blob_t)(const void *src, uint64_t src_size, void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max); @@ -54,15 +61,14 @@ static void test_compress_decompress(int compression, size_t data_len, bool may_fail) { char compressed[512]; - size_t csize = 512; - size_t usize = 0; + size_t csize, usize = 0; _cleanup_free_ char *decompressed = NULL; int r; log_info("/* testing %s %s blob compression/decompression */", object_compressed_to_string(compression), data); - r = compress(data, data_len, compressed, &csize); + r = compress(data, data_len, compressed, sizeof(compressed), &csize); if (r == -ENOBUFS) { log_info_errno(r, "compression failed: %m"); assert_se(may_fail); @@ -98,43 +104,45 @@ static void test_decompress_startswith(int compression, size_t data_len, bool may_fail) { - char compressed[512]; - size_t csize = 512; - size_t usize = 0; - _cleanup_free_ char *decompressed = NULL; + char *compressed; + _cleanup_free_ char *compressed1 = NULL, *compressed2 = NULL, *decompressed = NULL; + size_t csize, usize = 0, len; int r; - log_info("/* testing decompress_startswith with %s on %s text*/", + log_info("/* testing decompress_startswith with %s on %.20s text*/", object_compressed_to_string(compression), data); - r = compress(data, data_len, compressed, &csize); +#define BUFSIZE_1 512 +#define BUFSIZE_2 20000 + + compressed = compressed1 = malloc(BUFSIZE_1); + assert_se(compressed1); + r = compress(data, data_len, compressed, BUFSIZE_1, &csize); if (r == -ENOBUFS) { log_info_errno(r, "compression failed: %m"); assert_se(may_fail); - return; + + compressed = compressed2 = malloc(BUFSIZE_2); + assert_se(compressed2); + r = compress(data, data_len, compressed, BUFSIZE_2, &csize); + assert(r == 0); } assert_se(r == 0); - assert_se(decompress_sw(compressed, - csize, - (void **) &decompressed, - &usize, - data, strlen(data), '\0') > 0); - assert_se(decompress_sw(compressed, - csize, - (void **) &decompressed, - &usize, - data, strlen(data), 'w') == 0); - assert_se(decompress_sw(compressed, - csize, - (void **) &decompressed, - &usize, - "barbarbar", 9, ' ') == 0); - assert_se(decompress_sw(compressed, - csize, - (void **) &decompressed, - &usize, - data, strlen(data), '\0') > 0); + len = strlen(data); + + r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, '\0'); + assert_se(r > 0); + r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, 'w'); + assert_se(r == 0); + r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, "barbarbar", 9, ' '); + assert_se(r == 0); + r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len - 1, data[len-1]); + assert_se(r > 0); + r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len - 1, 'w'); + assert_se(r == 0); + r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, '\0'); + assert_se(r > 0); } static void test_compress_stream(int compression, @@ -144,8 +152,8 @@ static void test_compress_stream(int compression, const char *srcfile) { _cleanup_close_ int src = -1, dst = -1, dst2 = -1; - char pattern[] = "/tmp/systemd-test.xz.XXXXXX", - pattern2[] = "/tmp/systemd-test.xz.XXXXXX"; + char pattern[] = "/tmp/systemd-test.compressed.XXXXXX", + pattern2[] = "/tmp/systemd-test.compressed.XXXXXX"; int r; _cleanup_free_ char *cmd = NULL, *cmd2; struct stat st = {}; @@ -185,7 +193,7 @@ static void test_compress_stream(int compression, assert_se(lseek(dst, 1, SEEK_SET) == 1); r = decompress(dst, dst2, st.st_size); - assert_se(r == -EBADMSG); + assert_se(r == -EBADMSG || r == 0); assert_se(lseek(dst, 0, SEEK_SET) == 0); assert_se(lseek(dst2, 0, SEEK_SET) == 0); @@ -196,6 +204,44 @@ static void test_compress_stream(int compression, assert_se(unlink(pattern2) == 0); } +#ifdef HAVE_LZ4 +static void test_lz4_decompress_partial(void) { + char buf[20000]; + size_t buf_size = sizeof(buf), compressed; + int r; + _cleanup_free_ char *huge = NULL; + +#define HUGE_SIZE (4096*1024) + huge = malloc(HUGE_SIZE); + memset(huge, 'x', HUGE_SIZE); + memcpy(huge, "HUGE=", 5); + + r = LZ4_compress_limitedOutput(huge, buf, HUGE_SIZE, buf_size); + assert_se(r >= 0); + compressed = r; + log_info("Compressed %i → %zu", HUGE_SIZE, compressed); + + r = LZ4_decompress_safe(buf, huge, r, HUGE_SIZE); + assert_se(r >= 0); + log_info("Decompressed → %i", r); + + r = LZ4_decompress_safe_partial(buf, huge, + compressed, + 12, HUGE_SIZE); + assert_se(r >= 0); + log_info("Decompressed partial %i/%i → %i", 12, HUGE_SIZE, r); + + /* We expect this to fail, because that's how current lz4 works. If this + * call succeeds, then lz4 has been fixed, and we need to change our code. + */ + r = LZ4_decompress_safe_partial(buf, huge, + compressed, + 12, HUGE_SIZE-1); + assert_se(r < 0); + log_info("Decompressed partial %i/%i → %i", 12, HUGE_SIZE-1, r); +} +#endif + int main(int argc, char *argv[]) { const char text[] = "text\0foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF" @@ -203,6 +249,11 @@ int main(int argc, char *argv[]) { char data[512] = "random\0"; + char huge[4096*1024]; + memset(huge, 'x', sizeof(huge)); + memcpy(huge, "HUGE=", 5); + char_array_0(huge); + log_set_max_level(LOG_DEBUG); random_bytes(data + 7, sizeof(data) - 7); @@ -212,12 +263,17 @@ int main(int argc, char *argv[]) { text, sizeof(text), false); test_compress_decompress(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_blob_xz, data, sizeof(data), true); + test_decompress_startswith(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_startswith_xz, text, sizeof(text), false); test_decompress_startswith(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_startswith_xz, data, sizeof(data), true); + test_decompress_startswith(OBJECT_COMPRESSED_XZ, + compress_blob_xz, decompress_startswith_xz, + huge, sizeof(huge), true); + test_compress_stream(OBJECT_COMPRESSED_XZ, "xzcat", compress_stream_xz, decompress_stream_xz, argv[0]); #else @@ -229,16 +285,21 @@ int main(int argc, char *argv[]) { text, sizeof(text), false); test_compress_decompress(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_blob_lz4, data, sizeof(data), true); + test_decompress_startswith(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_startswith_lz4, text, sizeof(text), false); test_decompress_startswith(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_startswith_lz4, data, sizeof(data), true); + test_decompress_startswith(OBJECT_COMPRESSED_LZ4, + compress_blob_lz4, decompress_startswith_lz4, + huge, sizeof(huge), true); - /* Produced stream is not compatible with lz4 binary, skip lz4cat check. */ - test_compress_stream(OBJECT_COMPRESSED_LZ4, NULL, + test_compress_stream(OBJECT_COMPRESSED_LZ4, "lz4cat", compress_stream_lz4, decompress_stream_lz4, argv[0]); + + test_lz4_decompress_partial(); #else log_info("/* LZ4 test skipped */"); #endif diff --git a/src/journal/test-journal-enum.c b/src/journal/test-journal-enum.c index cde2025ae9..d396fabdab 100644 --- a/src/journal/test-journal-enum.c +++ b/src/journal/test-journal-enum.c @@ -21,14 +21,15 @@ #include <stdio.h> -#include "log.h" #include "sd-journal.h" -#include "macro.h" + #include "journal-internal.h" +#include "log.h" +#include "macro.h" int main(int argc, char *argv[]) { unsigned n = 0; - _cleanup_journal_close_ sd_journal*j = NULL; + _cleanup_(sd_journal_closep) sd_journal*j = NULL; log_set_max_level(LOG_DEBUG); diff --git a/src/journal/test-journal-flush.c b/src/journal/test-journal-flush.c index 2d4f531e9b..03d1522e23 100644 --- a/src/journal/test-journal-flush.c +++ b/src/journal/test-journal-flush.c @@ -22,9 +22,12 @@ #include <fcntl.h> #include "sd-journal.h" -#include "macro.h" + +#include "alloc-util.h" #include "journal-file.h" #include "journal-internal.h" +#include "macro.h" +#include "string-util.h" int main(int argc, char *argv[]) { _cleanup_free_ char *fn = NULL; diff --git a/src/journal/test-journal-init.c b/src/journal/test-journal-init.c index e6599f366d..142da85041 100644 --- a/src/journal/test-journal-init.c +++ b/src/journal/test-journal-init.c @@ -19,11 +19,12 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include "systemd/sd-journal.h" +#include "sd-journal.h" #include "log.h" -#include "util.h" +#include "parse-util.h" #include "rm-rf.h" +#include "util.h" int main(int argc, char *argv[]) { sd_journal *j; diff --git a/src/journal/test-journal-interleaving.c b/src/journal/test-journal-interleaving.c index adefa1b026..5c055ef748 100644 --- a/src/journal/test-journal-interleaving.c +++ b/src/journal/test-journal-interleaving.c @@ -20,15 +20,18 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <unistd.h> #include <fcntl.h> +#include <unistd.h> #include "sd-journal.h" + +#include "alloc-util.h" #include "journal-file.h" #include "journal-vacuum.h" -#include "util.h" #include "log.h" +#include "parse-util.h" #include "rm-rf.h" +#include "util.h" /* This program tests skipping around in a multi-file journal. */ @@ -197,7 +200,7 @@ static void test_skip(void (*setup)(void)) { if (arg_keep) log_info("Not removing %s", t); else { - journal_directory_vacuum(".", 3000000, 0, NULL, true); + journal_directory_vacuum(".", 3000000, 0, 0, NULL, true); assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0); } @@ -282,7 +285,7 @@ static void test_sequence_numbers(void) { if (arg_keep) log_info("Not removing %s", t); else { - journal_directory_vacuum(".", 3000000, 0, NULL, true); + journal_directory_vacuum(".", 3000000, 0, 0, NULL, true); assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0); } diff --git a/src/journal/test-journal-match.c b/src/journal/test-journal-match.c index a3187053c9..4ebaa8b31a 100644 --- a/src/journal/test-journal-match.c +++ b/src/journal/test-journal-match.c @@ -21,14 +21,16 @@ #include <stdio.h> -#include "systemd/sd-journal.h" +#include "sd-journal.h" +#include "alloc-util.h" #include "journal-internal.h" -#include "util.h" #include "log.h" +#include "string-util.h" +#include "util.h" int main(int argc, char *argv[]) { - _cleanup_journal_close_ sd_journal*j; + _cleanup_(sd_journal_closep) sd_journal*j = NULL; _cleanup_free_ char *t; log_set_max_level(LOG_DEBUG); diff --git a/src/journal/test-journal-send.c b/src/journal/test-journal-send.c index 81ca47ed8d..e537c1fe5f 100644 --- a/src/journal/test-journal-send.c +++ b/src/journal/test-journal-send.c @@ -19,58 +19,84 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include "systemd/sd-journal.h" +#include <errno.h> #include <stdlib.h> #include <unistd.h> -#include "log.h" +#include "sd-journal.h" + +#include "macro.h" int main(int argc, char *argv[]) { char huge[4096*1024]; - log_set_max_level(LOG_DEBUG); - - sd_journal_print(LOG_INFO, "piepapo"); - - sd_journal_send("MESSAGE=foobar", - "VALUE=%i", 7, - NULL); + /* utf-8 and non-utf-8, message-less and message-ful iovecs */ + struct iovec graph1[] = { + {(char*) "GRAPH=graph", strlen("GRAPH=graph")} + }; + struct iovec graph2[] = { + {(char*) "GRAPH=graph\n", strlen("GRAPH=graph\n")} + }; + struct iovec message1[] = { + {(char*) "MESSAGE=graph", strlen("MESSAGE=graph")} + }; + struct iovec message2[] = { + {(char*) "MESSAGE=graph\n", strlen("MESSAGE=graph\n")} + }; + + assert_se(sd_journal_print(LOG_INFO, "piepapo") == 0); + + assert_se(sd_journal_send("MESSAGE=foobar", + "VALUE=%i", 7, + NULL) == 0); errno = ENOENT; - sd_journal_perror("Foobar"); + assert_se(sd_journal_perror("Foobar") == 0); - sd_journal_perror(""); + assert_se(sd_journal_perror("") == 0); memset(huge, 'x', sizeof(huge)); memcpy(huge, "HUGE=", 5); char_array_0(huge); - sd_journal_send("MESSAGE=Huge field attached", - huge, - NULL); + assert_se(sd_journal_send("MESSAGE=Huge field attached", + huge, + NULL) == 0); - sd_journal_send("MESSAGE=uiui", - "VALUE=A", - "VALUE=B", - "VALUE=C", - "SINGLETON=1", - "OTHERVALUE=X", - "OTHERVALUE=Y", - "WITH_BINARY=this is a binary value \a", - NULL); + assert_se(sd_journal_send("MESSAGE=uiui", + "VALUE=A", + "VALUE=B", + "VALUE=C", + "SINGLETON=1", + "OTHERVALUE=X", + "OTHERVALUE=Y", + "WITH_BINARY=this is a binary value \a", + NULL) == 0); syslog(LOG_NOTICE, "Hello World!"); - sd_journal_print(LOG_NOTICE, "Hello World"); - - sd_journal_send("MESSAGE=Hello World!", - "MESSAGE_ID=52fb62f99e2c49d89cfbf9d6de5e3555", - "PRIORITY=5", - "HOME=%s", getenv("HOME"), - "TERM=%s", getenv("TERM"), - "PAGE_SIZE=%li", sysconf(_SC_PAGESIZE), - "N_CPUS=%li", sysconf(_SC_NPROCESSORS_ONLN), - NULL); + assert_se(sd_journal_print(LOG_NOTICE, "Hello World") == 0); + + assert_se(sd_journal_send("MESSAGE=Hello World!", + "MESSAGE_ID=52fb62f99e2c49d89cfbf9d6de5e3555", + "PRIORITY=5", + "HOME=%s", getenv("HOME"), + "TERM=%s", getenv("TERM"), + "PAGE_SIZE=%li", sysconf(_SC_PAGESIZE), + "N_CPUS=%li", sysconf(_SC_NPROCESSORS_ONLN), + NULL) == 0); + + assert_se(sd_journal_sendv(graph1, 1) == 0); + assert_se(sd_journal_sendv(graph2, 1) == 0); + assert_se(sd_journal_sendv(message1, 1) == 0); + assert_se(sd_journal_sendv(message2, 1) == 0); + + /* test without location fields */ +#undef sd_journal_sendv + assert_se(sd_journal_sendv(graph1, 1) == 0); + assert_se(sd_journal_sendv(graph2, 1) == 0); + assert_se(sd_journal_sendv(message1, 1) == 0); + assert_se(sd_journal_sendv(message2, 1) == 0); sleep(1); diff --git a/src/journal/test-journal-stream.c b/src/journal/test-journal-stream.c index b5ecf2f375..2c257e43b6 100644 --- a/src/journal/test-journal-stream.c +++ b/src/journal/test-journal-stream.c @@ -19,16 +19,19 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <unistd.h> #include <fcntl.h> +#include <unistd.h> #include "sd-journal.h" -#include "util.h" + +#include "alloc-util.h" +#include "journal-file.h" +#include "journal-internal.h" #include "log.h" #include "macro.h" +#include "parse-util.h" #include "rm-rf.h" -#include "journal-file.h" -#include "journal-internal.h" +#include "util.h" #define N_ENTRIES 200 @@ -76,7 +79,7 @@ int main(int argc, char *argv[]) { JournalFile *one, *two, *three; char t[] = "/tmp/journal-stream-XXXXXX"; unsigned i; - _cleanup_journal_close_ sd_journal *j = NULL; + _cleanup_(sd_journal_closep) sd_journal *j = NULL; char *z; const void *data; size_t l; diff --git a/src/journal/test-journal-syslog.c b/src/journal/test-journal-syslog.c index c99ca0654b..1784187fe9 100644 --- a/src/journal/test-journal-syslog.c +++ b/src/journal/test-journal-syslog.c @@ -19,8 +19,10 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ +#include "alloc-util.h" #include "journald-syslog.h" #include "macro.h" +#include "string-util.h" static void test_syslog_parse_identifier(const char* str, const char *ident, const char*pid, int ret) { diff --git a/src/journal/test-journal-verify.c b/src/journal/test-journal-verify.c index d89123dc64..a7abb11fba 100644 --- a/src/journal/test-journal-verify.c +++ b/src/journal/test-journal-verify.c @@ -19,16 +19,17 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ +#include <fcntl.h> #include <stdio.h> #include <unistd.h> -#include <fcntl.h> -#include "util.h" -#include "log.h" -#include "rm-rf.h" +#include "fd-util.h" #include "journal-file.h" #include "journal-verify.h" +#include "log.h" +#include "rm-rf.h" #include "terminal-util.h" +#include "util.h" #define N_ENTRIES 6000 #define RANDOM_RANGE 77 diff --git a/src/journal/test-journal.c b/src/journal/test-journal.c index caaab258c9..266e0d5473 100644 --- a/src/journal/test-journal.c +++ b/src/journal/test-journal.c @@ -22,11 +22,11 @@ #include <fcntl.h> #include <unistd.h> -#include "log.h" -#include "rm-rf.h" -#include "journal-file.h" #include "journal-authenticate.h" +#include "journal-file.h" #include "journal-vacuum.h" +#include "log.h" +#include "rm-rf.h" static bool arg_keep = false; @@ -116,7 +116,7 @@ static void test_non_empty(void) { if (arg_keep) log_info("Not removing %s", t); else { - journal_directory_vacuum(".", 3000000, 0, NULL, true); + journal_directory_vacuum(".", 3000000, 0, 0, NULL, true); assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0); } @@ -155,7 +155,7 @@ static void test_empty(void) { if (arg_keep) log_info("Not removing %s", t); else { - journal_directory_vacuum(".", 3000000, 0, NULL, true); + journal_directory_vacuum(".", 3000000, 0, 0, NULL, true); assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0); } diff --git a/src/journal/test-mmap-cache.c b/src/journal/test-mmap-cache.c index 3258b22702..fdd48e531c 100644 --- a/src/journal/test-mmap-cache.c +++ b/src/journal/test-mmap-cache.c @@ -19,14 +19,16 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ +#include <fcntl.h> #include <stdlib.h> #include <sys/mman.h> #include <unistd.h> -#include <fcntl.h> +#include "fd-util.h" +#include "fileio.h" #include "macro.h" -#include "util.h" #include "mmap-cache.h" +#include "util.h" int main(int argc, char *argv[]) { int x, y, z, r; |