diff options
Diffstat (limited to 'src/journal')
49 files changed, 1607 insertions, 923 deletions
diff --git a/src/journal/cat.c b/src/journal/cat.c index be2c2e3354..7fd4198df8 100644 --- a/src/journal/cat.c +++ b/src/journal/cat.c @@ -19,17 +19,20 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <stdio.h> -#include <getopt.h> -#include <unistd.h> -#include <stdlib.h> #include <errno.h> #include <fcntl.h> +#include <getopt.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> -#include "systemd/sd-journal.h" +#include "sd-journal.h" +#include "fd-util.h" +#include "parse-util.h" +#include "string-util.h" +#include "syslog-util.h" #include "util.h" -#include "build.h" static char *arg_identifier = NULL; static int arg_priority = LOG_INFO; @@ -76,9 +79,7 @@ static int parse_argv(int argc, char *argv[]) { return 0; case ARG_VERSION: - puts(PACKAGE_STRING); - puts(SYSTEMD_FEATURES); - return 0; + return version(); case 't': free(arg_identifier); @@ -95,7 +96,7 @@ static int parse_argv(int argc, char *argv[]) { arg_priority = log_level_from_string(optarg); if (arg_priority < 0) { log_error("Failed to parse priority value."); - return arg_priority; + return -EINVAL; } break; @@ -103,10 +104,9 @@ static int parse_argv(int argc, char *argv[]) { int k; k = parse_boolean(optarg); - if (k < 0) { - log_error("Failed to parse level prefix value."); - return k; - } + if (k < 0) + return log_error_errno(k, "Failed to parse level prefix value."); + arg_level_prefix = k; break; } @@ -122,7 +122,8 @@ static int parse_argv(int argc, char *argv[]) { } int main(int argc, char *argv[]) { - int r, fd = -1, saved_stderr = -1; + _cleanup_close_ int fd = -1, saved_stderr = -1; + int r; log_parse_environment(); log_open(); @@ -133,8 +134,7 @@ int main(int argc, char *argv[]) { fd = sd_journal_stream_fd(arg_identifier, arg_priority, arg_level_prefix); if (fd < 0) { - log_error_errno(fd, "Failed to create stream fd: %m"); - r = fd; + r = log_error_errno(fd, "Failed to create stream fd: %m"); goto finish; } @@ -148,25 +148,20 @@ int main(int argc, char *argv[]) { if (fd >= 3) safe_close(fd); - fd = -1; if (argc <= optind) - execl("/bin/cat", "/bin/cat", NULL); + (void) execl("/bin/cat", "/bin/cat", NULL); else - execvp(argv[optind], argv + optind); - + (void) execvp(argv[optind], argv + optind); r = -errno; /* Let's try to restore a working stderr, so we can print the error message */ if (saved_stderr >= 0) - dup3(saved_stderr, STDERR_FILENO, 0); + (void) dup3(saved_stderr, STDERR_FILENO, 0); log_error_errno(r, "Failed to execute process: %m"); finish: - safe_close(fd); - safe_close(saved_stderr); - return r < 0 ? EXIT_FAILURE : EXIT_SUCCESS; } diff --git a/src/journal/catalog.c b/src/journal/catalog.c index a3e51e2f52..fcaa54aa0c 100644 --- a/src/journal/catalog.c +++ b/src/journal/catalog.c @@ -19,25 +19,31 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ +#include <errno.h> #include <fcntl.h> +#include <locale.h> #include <stdio.h> -#include <unistd.h> -#include <errno.h> #include <string.h> #include <sys/mman.h> -#include <locale.h> +#include <unistd.h> -#include "util.h" -#include "log.h" -#include "sparse-endian.h" #include "sd-id128.h" -#include "hashmap.h" -#include "strv.h" -#include "strbuf.h" + +#include "alloc-util.h" +#include "catalog.h" #include "conf-files.h" +#include "fd-util.h" +#include "fileio.h" +#include "hashmap.h" +#include "log.h" #include "mkdir.h" -#include "catalog.h" +#include "path-util.h" #include "siphash24.h" +#include "sparse-endian.h" +#include "strbuf.h" +#include "string-util.h" +#include "strv.h" +#include "util.h" const char * const catalog_file_dirs[] = { "/usr/local/lib/systemd/catalog/", @@ -62,21 +68,11 @@ typedef struct CatalogItem { le64_t offset; } CatalogItem; -static unsigned long catalog_hash_func(const void *p, const uint8_t hash_key[HASH_KEY_SIZE]) { +static void catalog_hash_func(const void *p, struct siphash *state) { const CatalogItem *i = p; - uint64_t u; - size_t l, sz; - void *v; - l = strlen(i->language); - sz = sizeof(i->id) + l; - v = alloca(sz); - - memcpy(mempcpy(v, &i->id, sizeof(i->id)), i->language, l); - - siphash24((uint8_t*) &u, v, sz, hash_key); - - return (unsigned long) u; + siphash24_compress(&i->id, sizeof(i->id), state); + siphash24_compress(i->language, strlen(i->language), state); } static int catalog_compare_func(const void *a, const void *b) { @@ -212,7 +208,7 @@ int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) { r = catalog_file_lang(path, &deflang); if (r < 0) - log_error_errno(errno, "Failed to determine language for file %s: %m", path); + log_error_errno(r, "Failed to determine language for file %s: %m", path); if (r == 1) log_debug("File %s has language %s.", path, deflang); @@ -225,8 +221,7 @@ int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) { if (feof(f)) break; - log_error_errno(errno, "Failed to read file %s: %m", path); - return -errno; + return log_error_errno(errno, "Failed to read file %s: %m", path); } n++; @@ -323,8 +318,8 @@ int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) { return 0; } -static long write_catalog(const char *database, Hashmap *h, struct strbuf *sb, - CatalogItem *items, size_t n) { +static int64_t write_catalog(const char *database, struct strbuf *sb, + CatalogItem *items, size_t n) { CatalogHeader header; _cleanup_fclose_ FILE *w = NULL; int r; @@ -348,7 +343,7 @@ static long write_catalog(const char *database, Hashmap *h, struct strbuf *sb, memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature)); header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8)); header.catalog_item_size = htole64(sizeof(CatalogItem)); - header.n_items = htole64(hashmap_size(h)); + header.n_items = htole64(n); r = -EIO; @@ -383,7 +378,7 @@ static long write_catalog(const char *database, Hashmap *h, struct strbuf *sb, goto error; } - return ftell(w); + return ftello(w); error: (void) unlink(p); @@ -399,7 +394,8 @@ int catalog_update(const char* database, const char* root, const char* const* di CatalogItem *i; Iterator j; unsigned n; - long r; + int r; + int64_t sz; h = hashmap_new(&catalog_hash_ops); sb = strbuf_new(); @@ -419,8 +415,7 @@ int catalog_update(const char* database, const char* root, const char* const* di log_debug("Reading file '%s'", *f); r = catalog_import_file(h, sb, *f); if (r < 0) { - log_error("Failed to import file '%s': %s.", - *f, strerror(-r)); + log_error_errno(r, "Failed to import file '%s': %m", *f); goto finish; } } @@ -450,18 +445,19 @@ int catalog_update(const char* database, const char* root, const char* const* di assert(n == hashmap_size(h)); qsort_safe(items, n, sizeof(CatalogItem), catalog_compare_func); - r = write_catalog(database, h, sb, items, n); - if (r < 0) - log_error_errno(r, "Failed to write %s: %m", database); - else - log_debug("%s: wrote %u items, with %zu bytes of strings, %ld total size.", - database, n, sb->len, r); + sz = write_catalog(database, sb, items, n); + if (sz < 0) + r = log_error_errno(sz, "Failed to write %s: %m", database); + else { + r = 0; + log_debug("%s: wrote %u items, with %zu bytes of strings, %"PRIi64" total size.", + database, n, sb->len, sz); + } finish: - if (sb) - strbuf_cleanup(sb); + strbuf_cleanup(sb); - return r < 0 ? r : 0; + return r; } static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) { @@ -676,8 +672,7 @@ int catalog_list_items(FILE *f, const char *database, bool oneline, char **items k = sd_id128_from_string(*item, &id); if (k < 0) { - log_error_errno(k, "Failed to parse id128 '%s': %m", - *item); + log_error_errno(k, "Failed to parse id128 '%s': %m", *item); if (r == 0) r = k; continue; @@ -685,9 +680,8 @@ int catalog_list_items(FILE *f, const char *database, bool oneline, char **items k = catalog_get(database, id, &msg); if (k < 0) { - log_full(k == -ENOENT ? LOG_NOTICE : LOG_ERR, - "Failed to retrieve catalog entry for '%s': %s", - *item, strerror(-k)); + log_full_errno(k == -ENOENT ? LOG_NOTICE : LOG_ERR, k, + "Failed to retrieve catalog entry for '%s': %m", *item); if (r == 0) r = k; continue; diff --git a/src/journal/compress.c b/src/journal/compress.c index c66043e503..e1ca0a8818 100644 --- a/src/journal/compress.c +++ b/src/journal/compress.c @@ -21,21 +21,33 @@ #include <stdlib.h> #include <string.h> +#include <sys/mman.h> #include <unistd.h> #ifdef HAVE_XZ -# include <lzma.h> +#include <lzma.h> #endif #ifdef HAVE_LZ4 -# include <lz4.h> +#include <lz4.h> +#include <lz4frame.h> #endif +#include "alloc-util.h" #include "compress.h" +#include "fd-util.h" +#include "io-util.h" +#include "journal-def.h" #include "macro.h" -#include "util.h" #include "sparse-endian.h" -#include "journal-def.h" +#include "string-table.h" +#include "string-util.h" +#include "util.h" + +#ifdef HAVE_LZ4 +DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_compressionContext_t, LZ4F_freeCompressionContext); +DEFINE_TRIVIAL_CLEANUP_FUNC(LZ4F_decompressionContext_t, LZ4F_freeDecompressionContext); +#endif #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t)) @@ -50,10 +62,11 @@ int compress_blob_xz(const void *src, uint64_t src_size, void *dst, size_t *dst_ #ifdef HAVE_XZ static const lzma_options_lzma opt = { 1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT, - LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4}; - static const lzma_filter filters[2] = { - {LZMA_FILTER_LZMA2, (lzma_options_lzma*) &opt}, - {LZMA_VLI_UNKNOWN, NULL} + LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4 + }; + static const lzma_filter filters[] = { + { LZMA_FILTER_LZMA2, (lzma_options_lzma*) &opt }, + { LZMA_VLI_UNKNOWN, NULL } }; lzma_ret ret; size_t out_pos = 0; @@ -416,81 +429,96 @@ int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes) { #endif } -#define LZ4_BUFSIZE (512*1024) +#define LZ4_BUFSIZE (512*1024u) int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) { #ifdef HAVE_LZ4 + LZ4F_errorCode_t c; + _cleanup_(LZ4F_freeCompressionContextp) LZ4F_compressionContext_t ctx = NULL; + _cleanup_free_ char *buf = NULL; + char *src = NULL; + size_t size, n, total_in = 0, total_out = 0, offset = 0, frame_size; + struct stat st; + int r; + static const LZ4F_compressOptions_t options = { + .stableSrc = 1, + }; + static const LZ4F_preferences_t preferences = { + .frameInfo.blockSizeID = 5, + }; - _cleanup_free_ char *buf1 = NULL, *buf2 = NULL, *out = NULL; - char *buf; - LZ4_stream_t lz4_data = {}; - le32_t header; - size_t total_in = 0, total_out = sizeof(header); - ssize_t n; + c = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); + if (LZ4F_isError(c)) + return -ENOMEM; - assert(fdf >= 0); - assert(fdt >= 0); + if (fstat(fdf, &st) < 0) + return log_debug_errno(errno, "fstat() failed: %m"); - buf1 = malloc(LZ4_BUFSIZE); - buf2 = malloc(LZ4_BUFSIZE); - out = malloc(LZ4_COMPRESSBOUND(LZ4_BUFSIZE)); - if (!buf1 || !buf2 || !out) - return log_oom(); + frame_size = LZ4F_compressBound(LZ4_BUFSIZE, &preferences); + size = frame_size + 64*1024; /* add some space for header and trailer */ + buf = malloc(size); + if (!buf) + return -ENOMEM; - buf = buf1; - for (;;) { - size_t m; - int r; + n = offset = LZ4F_compressBegin(ctx, buf, size, &preferences); + if (LZ4F_isError(n)) + return -EINVAL; - m = LZ4_BUFSIZE; - if (max_bytes != (uint64_t) -1 && (uint64_t) m > (max_bytes - total_in)) - m = (size_t) (max_bytes - total_in); + src = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fdf, 0); + if (src == MAP_FAILED) + return -errno; - n = read(fdf, buf, m); - if (n < 0) - return -errno; - if (n == 0) - break; + log_debug("Buffer size is %zu bytes, header size %zu bytes.", size, n); - total_in += n; + while (total_in < (size_t) st.st_size) { + ssize_t k; - r = LZ4_compress_continue(&lz4_data, buf, out, n); - if (r == 0) { - log_error("LZ4 compression failed."); - return -EBADMSG; + k = MIN(LZ4_BUFSIZE, st.st_size - total_in); + n = LZ4F_compressUpdate(ctx, buf + offset, size - offset, + src + total_in, k, &options); + if (LZ4F_isError(n)) { + r = -ENOTRECOVERABLE; + goto cleanup; } - header = htole32(r); - errno = 0; - - n = write(fdt, &header, sizeof(header)); - if (n < 0) - return -errno; - if (n != sizeof(header)) - return errno ? -errno : -EIO; + total_in += k; + offset += n; + total_out += n; - n = loop_write(fdt, out, r, false); - if (n < 0) - return n; + if (max_bytes != (uint64_t) -1 && total_out > (size_t) max_bytes) { + log_debug("Compressed stream longer than %zd bytes", max_bytes); + return -EFBIG; + } - total_out += sizeof(header) + r; + if (size - offset < frame_size + 4) { + k = loop_write(fdt, buf, offset, false); + if (k < 0) { + r = k; + goto cleanup; + } + offset = 0; + } + } - buf = buf == buf1 ? buf2 : buf1; + n = LZ4F_compressEnd(ctx, buf + offset, size - offset, &options); + if (LZ4F_isError(n)) { + r = -ENOTRECOVERABLE; + goto cleanup; } - header = htole32(0); - n = write(fdt, &header, sizeof(header)); - if (n < 0) - return -errno; - if (n != sizeof(header)) - return errno ? -errno : -EIO; + offset += n; + total_out += n; + r = loop_write(fdt, buf, offset, false); + if (r < 0) + goto cleanup; log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)", total_in, total_out, (double) total_out / total_in * 100); - - return 0; + cleanup: + munmap(src, st.st_size); + return r; #else return -EPROTONOSUPPORT; #endif @@ -510,7 +538,7 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) { ret = lzma_stream_decoder(&s, UINT64_MAX, 0); if (ret != LZMA_OK) { - log_error("Failed to initialize XZ decoder: code %u", ret); + log_debug("Failed to initialize XZ decoder: code %u", ret); return -ENOMEM; } @@ -536,7 +564,7 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) { ret = lzma_code(&s, action); if (ret != LZMA_OK && ret != LZMA_STREAM_END) { - log_error("Decompression failed: code %u", ret); + log_debug("Decompression failed: code %u", ret); return -EBADMSG; } @@ -566,14 +594,14 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) { } } #else - log_error("Cannot decompress file. Compiled without XZ support."); + log_debug("Cannot decompress file. Compiled without XZ support."); return -EPROTONOSUPPORT; #endif } -int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) { - #ifdef HAVE_LZ4 +static int decompress_stream_lz4_v1(int fdf, int fdt, uint64_t max_bytes) { + _cleanup_free_ char *buf = NULL, *out = NULL; size_t buf_size = 0; LZ4_streamDecode_t lz4_data = {}; @@ -585,7 +613,7 @@ int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) { out = malloc(4*LZ4_BUFSIZE); if (!out) - return log_oom(); + return -ENOMEM; for (;;) { ssize_t m; @@ -606,22 +634,24 @@ int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) { * not accept buffers compressed by newer binaries then. */ if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) { - log_error("Compressed stream block too big: %zd bytes", m); - return -EBADMSG; + log_debug("Compressed stream block too big: %zd bytes", m); + return -ENOBUFS; } total_in += sizeof(header) + m; if (!GREEDY_REALLOC(buf, buf_size, m)) - return log_oom(); + return -ENOMEM; r = loop_read_exact(fdf, buf, m, false); if (r < 0) return r; r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE); - if (r <= 0) - log_error("LZ4 decompression failed."); + if (r <= 0) { + log_debug("LZ4 decompression failed (legacy format)."); + return -EBADMSG; + } total_out += r; @@ -635,13 +665,80 @@ int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) { return r; } - log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)", + log_debug("LZ4 decompression finished (legacy format, %zu -> %zu bytes, %.1f%%)", total_in, total_out, (double) total_out / total_in * 100); return 0; +} + +static int decompress_stream_lz4_v2(int in, int out, uint64_t max_bytes) { + size_t c; + _cleanup_(LZ4F_freeDecompressionContextp) LZ4F_decompressionContext_t ctx = NULL; + _cleanup_free_ char *buf = NULL; + char *src; + struct stat st; + int r = 0; + size_t total_in = 0, total_out = 0; + + c = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); + if (LZ4F_isError(c)) + return -ENOMEM; + + if (fstat(in, &st) < 0) + return log_debug_errno(errno, "fstat() failed: %m"); + + buf = malloc(LZ4_BUFSIZE); + if (!buf) + return -ENOMEM; + + src = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, in, 0); + if (src == MAP_FAILED) + return -errno; + + while (total_in < (size_t) st.st_size) { + size_t produced = LZ4_BUFSIZE; + size_t used = st.st_size - total_in; + + c = LZ4F_decompress(ctx, buf, &produced, src + total_in, &used, NULL); + if (LZ4F_isError(c)) { + r = -EBADMSG; + goto cleanup; + } + + total_in += used; + total_out += produced; + + if (max_bytes != (uint64_t) -1 && total_out > (size_t) max_bytes) { + log_debug("Decompressed stream longer than %zd bytes", max_bytes); + r = -EFBIG; + goto cleanup; + } + + r = loop_write(out, buf, produced, false); + if (r < 0) + goto cleanup; + } + + log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)", + total_in, total_out, + (double) total_out / total_in * 100); + cleanup: + munmap(src, st.st_size); + return r; +} +#endif + +int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) { +#ifdef HAVE_LZ4 + int r; + + r = decompress_stream_lz4_v2(fdf, fdt, max_bytes); + if (r == -EBADMSG) + r = decompress_stream_lz4_v1(fdf, fdt, max_bytes); + return r; #else - log_error("Cannot decompress file. Compiled without LZ4 support."); + log_debug("Cannot decompress file. Compiled without LZ4 support."); return -EPROTONOSUPPORT; #endif } diff --git a/src/journal/coredump-vacuum.c b/src/journal/coredump-vacuum.c index efe418615a..39bc2e4270 100644 --- a/src/journal/coredump-vacuum.c +++ b/src/journal/coredump-vacuum.c @@ -21,12 +21,16 @@ #include <sys/statvfs.h> -#include "util.h" -#include "time-util.h" +#include "alloc-util.h" +#include "coredump-vacuum.h" +#include "dirent-util.h" +#include "fd-util.h" #include "hashmap.h" #include "macro.h" - -#include "coredump-vacuum.h" +#include "string-util.h" +#include "time-util.h" +#include "user-util.h" +#include "util.h" #define DEFAULT_MAX_USE_LOWER (uint64_t) (1ULL*1024ULL*1024ULL) /* 1 MiB */ #define DEFAULT_MAX_USE_UPPER (uint64_t) (4ULL*1024ULL*1024ULL*1024ULL) /* 4 GiB */ diff --git a/src/journal/coredump.c b/src/journal/coredump.c index e1e66b9826..4c83e311db 100644 --- a/src/journal/coredump.c +++ b/src/journal/coredump.c @@ -20,10 +20,10 @@ ***/ #include <errno.h> -#include <unistd.h> #include <stdio.h> #include <sys/prctl.h> #include <sys/xattr.h> +#include <unistd.h> #ifdef HAVE_ELFUTILS # include <dwarf.h> @@ -32,23 +32,34 @@ #include "sd-journal.h" #include "sd-login.h" -#include "log.h" -#include "util.h" -#include "fileio.h" -#include "strv.h" -#include "macro.h" -#include "mkdir.h" -#include "special.h" + +#include "acl-util.h" +#include "alloc-util.h" +#include "capability-util.h" #include "cgroup-util.h" +#include "compress.h" #include "conf-parser.h" #include "copy.h" -#include "stacktrace.h" -#include "compress.h" -#include "acl-util.h" -#include "capability.h" -#include "journald-native.h" #include "coredump-vacuum.h" +#include "dirent-util.h" +#include "escape.h" +#include "fd-util.h" +#include "fileio.h" +#include "fs-util.h" +#include "io-util.h" +#include "journald-native.h" +#include "log.h" +#include "macro.h" +#include "mkdir.h" +#include "parse-util.h" #include "process-util.h" +#include "special.h" +#include "stacktrace.h" +#include "string-table.h" +#include "string-util.h" +#include "strv.h" +#include "user-util.h" +#include "util.h" /* The maximum size up to which we process coredumps */ #define PROCESS_SIZE_MAX ((uint64_t) (2LLU*1024LLU*1024LLU*1024LLU)) @@ -128,6 +139,7 @@ static int fix_acl(int fd, uid_t uid) { _cleanup_(acl_freep) acl_t acl = NULL; acl_entry_t entry; acl_permset_t permset; + int r; assert(fd >= 0); @@ -149,11 +161,12 @@ static int fix_acl(int fd, uid_t uid) { } if (acl_get_permset(entry, &permset) < 0 || - acl_add_perm(permset, ACL_READ) < 0 || - calc_acl_mask_if_needed(&acl) < 0) { - log_warning_errno(errno, "Failed to patch ACL: %m"); - return -errno; - } + acl_add_perm(permset, ACL_READ) < 0) + return log_warning_errno(errno, "Failed to patch ACL: %m"); + + r = calc_acl_mask_if_needed(&acl); + if (r < 0) + return log_warning_errno(r, "Failed to patch ACL: %m"); if (acl_set_fd(fd, acl) < 0) return log_error_errno(errno, "Failed to apply ACL: %m"); diff --git a/src/journal/coredumpctl.c b/src/journal/coredumpctl.c index 6628e82421..1df28d774a 100644 --- a/src/journal/coredumpctl.c +++ b/src/journal/coredumpctl.c @@ -19,27 +19,33 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ +#include <fcntl.h> +#include <getopt.h> #include <locale.h> #include <stdio.h> #include <string.h> -#include <getopt.h> -#include <fcntl.h> #include <unistd.h> #include "sd-journal.h" -#include "build.h" -#include "set.h" -#include "util.h" + +#include "alloc-util.h" +#include "compress.h" +#include "fd-util.h" +#include "fileio.h" +#include "journal-internal.h" #include "log.h" -#include "path-util.h" -#include "pager.h" #include "macro.h" -#include "journal-internal.h" -#include "compress.h" -#include "sigbus.h" +#include "pager.h" +#include "parse-util.h" +#include "path-util.h" #include "process-util.h" -#include "terminal-util.h" +#include "set.h" +#include "sigbus.h" #include "signal-util.h" +#include "string-util.h" +#include "terminal-util.h" +#include "user-util.h" +#include "util.h" static enum { ACTION_NONE, @@ -84,37 +90,35 @@ static Set *new_matches(void) { } static int add_match(Set *set, const char *match) { - int r = -ENOMEM; - unsigned pid; - const char* prefix; - char *pattern = NULL; _cleanup_free_ char *p = NULL; + char *pattern = NULL; + const char* prefix; + pid_t pid; + int r; if (strchr(match, '=')) prefix = ""; else if (strchr(match, '/')) { - p = path_make_absolute_cwd(match); - if (!p) + r = path_make_absolute_cwd(match, &p); + if (r < 0) goto fail; - match = p; prefix = "COREDUMP_EXE="; - } - else if (safe_atou(match, &pid) == 0) + } else if (parse_pid(match, &pid) >= 0) prefix = "COREDUMP_PID="; else prefix = "COREDUMP_COMM="; pattern = strjoin(prefix, match, NULL); - if (!pattern) + if (!pattern) { + r = -ENOMEM; goto fail; + } log_debug("Adding pattern: %s", pattern); r = set_consume(set, pattern); - if (r < 0) { - log_error_errno(r, "Failed to add pattern: %m"); + if (r < 0) goto fail; - } return 0; fail: @@ -175,9 +179,7 @@ static int parse_argv(int argc, char *argv[], Set *matches) { case ARG_VERSION: arg_action = ACTION_NONE; - puts(PACKAGE_STRING); - puts(SYSTEMD_FEATURES); - return 0; + return version(); case ARG_NO_PAGER: arg_no_pager = true; @@ -615,7 +617,7 @@ static int save_core(sd_journal *j, int fd, char **path, bool *unlink_temp) { fdt = mkostemp_safe(temp, O_WRONLY|O_CLOEXEC); if (fdt < 0) - return log_error_errno(errno, "Failed to create temporary file: %m"); + return log_error_errno(fdt, "Failed to create temporary file: %m"); log_debug("Created temporary file %s", temp); fd = fdt; @@ -774,7 +776,7 @@ static int run_gdb(sd_journal *j) { r = wait_for_terminate(pid, &st); if (r < 0) { - log_error_errno(errno, "Failed to wait for gdb: %m"); + log_error_errno(r, "Failed to wait for gdb: %m"); goto finish; } diff --git a/src/journal/fsprg.h b/src/journal/fsprg.h index 150d034828..5959b1fed2 100644 --- a/src/journal/fsprg.h +++ b/src/journal/fsprg.h @@ -29,6 +29,7 @@ #include <inttypes.h> #include "macro.h" +#include "util.h" #ifdef __cplusplus extern "C" { diff --git a/src/journal/journal-authenticate.c b/src/journal/journal-authenticate.c index cdc80e2d26..0c4ac5cdc3 100644 --- a/src/journal/journal-authenticate.c +++ b/src/journal/journal-authenticate.c @@ -22,10 +22,12 @@ #include <fcntl.h> #include <sys/mman.h> +#include "fd-util.h" +#include "fsprg.h" +#include "journal-authenticate.h" #include "journal-def.h" #include "journal-file.h" -#include "journal-authenticate.h" -#include "fsprg.h" +#include "hexdecoct.h" static uint64_t journal_file_tag_seqnum(JournalFile *f) { uint64_t r; diff --git a/src/journal/journal-def.h b/src/journal/journal-def.h index 39c9dd0dbf..c003ac05dd 100644 --- a/src/journal/journal-def.h +++ b/src/journal/journal-def.h @@ -21,11 +21,10 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include "sparse-endian.h" - -#include "systemd/sd-id128.h" +#include "sd-id128.h" #include "macro.h" +#include "sparse-endian.h" /* * If you change this file you probably should also change its documentation: diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c index 73d3a4bb9d..f9ff9545dd 100644 --- a/src/journal/journal-file.c +++ b/src/journal/journal-file.c @@ -19,22 +19,28 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <sys/mman.h> #include <errno.h> -#include <sys/uio.h> -#include <unistd.h> -#include <sys/statvfs.h> #include <fcntl.h> -#include <stddef.h> #include <linux/fs.h> +#include <stddef.h> +#include <sys/mman.h> +#include <sys/statvfs.h> +#include <sys/uio.h> +#include <unistd.h> +#include "alloc-util.h" #include "btrfs-util.h" +#include "chattr-util.h" +#include "compress.h" +#include "fd-util.h" +#include "journal-authenticate.h" #include "journal-def.h" #include "journal-file.h" -#include "journal-authenticate.h" #include "lookup3.h" -#include "compress.h" +#include "parse-util.h" #include "random-util.h" +#include "string-util.h" +#include "xattr-util.h" #define DEFAULT_DATA_HASH_TABLE_SIZE (2047ULL*sizeof(HashItem)) #define DEFAULT_FIELD_HASH_TABLE_SIZE (333ULL*sizeof(HashItem)) @@ -42,13 +48,16 @@ #define COMPRESSION_SIZE_THRESHOLD (512ULL) /* This is the minimum journal file size */ -#define JOURNAL_FILE_SIZE_MIN (4ULL*1024ULL*1024ULL) /* 4 MiB */ +#define JOURNAL_FILE_SIZE_MIN (512ULL*1024ULL) /* 512 KiB */ /* These are the lower and upper bounds if we deduce the max_use value * from the file system size */ #define DEFAULT_MAX_USE_LOWER (1ULL*1024ULL*1024ULL) /* 1 MiB */ #define DEFAULT_MAX_USE_UPPER (4ULL*1024ULL*1024ULL*1024ULL) /* 4 GiB */ +/* This is the default minimal use limit, how much we'll use even if keep_free suggests otherwise. */ +#define DEFAULT_MIN_USE (1ULL*1024ULL*1024ULL) /* 1 MiB */ + /* This is the upper bound if we deduce max_size from max_use */ #define DEFAULT_MAX_SIZE_UPPER (128ULL*1024ULL*1024ULL) /* 128 MiB */ @@ -60,6 +69,9 @@ * size */ #define DEFAULT_KEEP_FREE (1024ULL*1024ULL) /* 1 MB */ +/* This is the default maximum number of journal files to keep around. */ +#define DEFAULT_N_MAX_FILES (100) + /* n_data was the first entry we added after the initial file format design */ #define HEADER_SIZE_MIN ALIGN64(offsetof(Header, n_data)) @@ -128,7 +140,7 @@ int journal_file_set_offline(JournalFile *f) { return 0; } -void journal_file_close(JournalFile *f) { +JournalFile* journal_file_close(JournalFile *f) { assert(f); #ifdef HAVE_GCRYPT @@ -179,6 +191,7 @@ void journal_file_close(JournalFile *f) { #endif free(f); + return NULL; } static int journal_file_init_header(JournalFile *f, JournalFile *template) { @@ -398,12 +411,7 @@ static int journal_file_allocate(JournalFile *f, uint64_t offset, uint64_t size) if (fstatvfs(f->fd, &svfs) >= 0) { uint64_t available; - available = svfs.f_bfree * svfs.f_bsize; - - if (available >= f->metrics.keep_free) - available -= f->metrics.keep_free; - else - available = 0; + available = LESS_BY((uint64_t) svfs.f_bfree * (uint64_t) svfs.f_bsize, f->metrics.keep_free); if (new_size - old_size > available) return -E2BIG; @@ -604,10 +612,10 @@ static int journal_file_setup_data_hash_table(JournalFile *f) { assert(f); - /* We estimate that we need 1 hash table entry per 768 of - journal file and we want to make sure we never get beyond - 75% fill level. Calculate the hash table size for the - maximum file size based on these metrics. */ + /* We estimate that we need 1 hash table entry per 768 bytes + of journal file and we want to make sure we never get + beyond 75% fill level. Calculate the hash table size for + the maximum file size based on these metrics. */ s = (f->metrics.max_size * 4 / 768 / 3) * sizeof(HashItem); if (s < DEFAULT_DATA_HASH_TABLE_SIZE) @@ -1055,7 +1063,7 @@ static int journal_file_append_data( r = journal_file_find_data_object_with_hash(f, data, size, hash, &o, &p); if (r < 0) return r; - else if (r > 0) { + if (r > 0) { if (ret) *ret = o; @@ -1074,23 +1082,24 @@ static int journal_file_append_data( o->data.hash = htole64(hash); #if defined(HAVE_XZ) || defined(HAVE_LZ4) - if (f->compress_xz && - size >= COMPRESSION_SIZE_THRESHOLD) { + if (JOURNAL_FILE_COMPRESS(f) && size >= COMPRESSION_SIZE_THRESHOLD) { size_t rsize = 0; compression = compress_blob(data, size, o->data.payload, &rsize); - if (compression) { + if (compression >= 0) { o->object.size = htole64(offsetof(Object, data.payload) + rsize); o->object.flags |= compression; log_debug("Compressed data object %"PRIu64" -> %zu using %s", size, rsize, object_compressed_to_string(compression)); - } + } else + /* Compression didn't work, we don't really care why, let's continue without compression */ + compression = 0; } #endif - if (!compression && size > 0) + if (compression == 0 && size > 0) memcpy(o->data.payload, data, size); r = journal_file_link_data(f, o, p, hash); @@ -2696,7 +2705,7 @@ int journal_file_open( } if (f->last_stat.st_size < (off_t) HEADER_SIZE_MIN) { - r = -EIO; + r = -ENODATA; goto fail; } @@ -2833,8 +2842,7 @@ int journal_file_open_reliably( size_t l; _cleanup_free_ char *p = NULL; - r = journal_file_open(fname, flags, mode, compress, seal, - metrics, mmap_cache, template, ret); + r = journal_file_open(fname, flags, mode, compress, seal, metrics, mmap_cache, template, ret); if (!IN_SET(r, -EBADMSG, /* corrupted */ -ENODATA, /* truncated */ @@ -2864,8 +2872,7 @@ int journal_file_open_reliably( random_u64()) < 0) return -ENOMEM; - r = rename(fname, p); - if (r < 0) + if (rename(fname, p) < 0) return -errno; /* btrfs doesn't cope well with our write pattern and @@ -2874,10 +2881,9 @@ int journal_file_open_reliably( (void) chattr_path(p, false, FS_NOCOW_FL); (void) btrfs_defrag(p); - log_warning("File %s corrupted or uncleanly shut down, renaming and replacing.", fname); + log_warning_errno(r, "File %s corrupted or uncleanly shut down, renaming and replacing.", fname); - return journal_file_open(fname, flags, mode, compress, seal, - metrics, mmap_cache, template, ret); + return journal_file_open(fname, flags, mode, compress, seal, metrics, mmap_cache, template, ret); } int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint64_t p, uint64_t *seqnum, Object **ret, uint64_t *offset) { @@ -2964,16 +2970,35 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6 return r; } +void journal_reset_metrics(JournalMetrics *m) { + assert(m); + + /* Set everything to "pick automatic values". */ + + *m = (JournalMetrics) { + .min_use = (uint64_t) -1, + .max_use = (uint64_t) -1, + .min_size = (uint64_t) -1, + .max_size = (uint64_t) -1, + .keep_free = (uint64_t) -1, + .n_max_files = (uint64_t) -1, + }; +} + void journal_default_metrics(JournalMetrics *m, int fd) { - uint64_t fs_size = 0; + char a[FORMAT_BYTES_MAX], b[FORMAT_BYTES_MAX], c[FORMAT_BYTES_MAX], d[FORMAT_BYTES_MAX], e[FORMAT_BYTES_MAX]; struct statvfs ss; - char a[FORMAT_BYTES_MAX], b[FORMAT_BYTES_MAX], c[FORMAT_BYTES_MAX], d[FORMAT_BYTES_MAX]; + uint64_t fs_size; assert(m); assert(fd >= 0); if (fstatvfs(fd, &ss) >= 0) fs_size = ss.f_frsize * ss.f_blocks; + else { + log_debug_errno(errno, "Failed to detremine disk size: %m"); + fs_size = 0; + } if (m->max_use == (uint64_t) -1) { @@ -2990,10 +3015,16 @@ void journal_default_metrics(JournalMetrics *m, int fd) { } else { m->max_use = PAGE_ALIGN(m->max_use); - if (m->max_use < JOURNAL_FILE_SIZE_MIN*2) + if (m->max_use != 0 && m->max_use < JOURNAL_FILE_SIZE_MIN*2) m->max_use = JOURNAL_FILE_SIZE_MIN*2; } + if (m->min_use == (uint64_t) -1) + m->min_use = DEFAULT_MIN_USE; + + if (m->min_use > m->max_use) + m->min_use = m->max_use; + if (m->max_size == (uint64_t) -1) { m->max_size = PAGE_ALIGN(m->max_use / 8); /* 8 chunks */ @@ -3002,11 +3033,13 @@ void journal_default_metrics(JournalMetrics *m, int fd) { } else m->max_size = PAGE_ALIGN(m->max_size); - if (m->max_size < JOURNAL_FILE_SIZE_MIN) - m->max_size = JOURNAL_FILE_SIZE_MIN; + if (m->max_size != 0) { + if (m->max_size < JOURNAL_FILE_SIZE_MIN) + m->max_size = JOURNAL_FILE_SIZE_MIN; - if (m->max_size*2 > m->max_use) - m->max_use = m->max_size*2; + if (m->max_use != 0 && m->max_size*2 > m->max_use) + m->max_use = m->max_size*2; + } if (m->min_size == (uint64_t) -1) m->min_size = JOURNAL_FILE_SIZE_MIN; @@ -3016,7 +3049,7 @@ void journal_default_metrics(JournalMetrics *m, int fd) { if (m->min_size < JOURNAL_FILE_SIZE_MIN) m->min_size = JOURNAL_FILE_SIZE_MIN; - if (m->min_size > m->max_size) + if (m->max_size != 0 && m->min_size > m->max_size) m->max_size = m->min_size; } @@ -3032,11 +3065,16 @@ void journal_default_metrics(JournalMetrics *m, int fd) { m->keep_free = DEFAULT_KEEP_FREE; } - log_debug("Fixed max_use=%s max_size=%s min_size=%s keep_free=%s", - format_bytes(a, sizeof(a), m->max_use), - format_bytes(b, sizeof(b), m->max_size), - format_bytes(c, sizeof(c), m->min_size), - format_bytes(d, sizeof(d), m->keep_free)); + if (m->n_max_files == (uint64_t) -1) + m->n_max_files = DEFAULT_N_MAX_FILES; + + log_debug("Fixed min_use=%s max_use=%s max_size=%s min_size=%s keep_free=%s n_max_files=%" PRIu64, + format_bytes(a, sizeof(a), m->min_use), + format_bytes(b, sizeof(b), m->max_use), + format_bytes(c, sizeof(c), m->max_size), + format_bytes(d, sizeof(d), m->min_size), + format_bytes(e, sizeof(e), m->keep_free), + m->n_max_files); } int journal_file_get_cutoff_realtime_usec(JournalFile *f, usec_t *from, usec_t *to) { diff --git a/src/journal/journal-file.h b/src/journal/journal-file.h index e92b75eabe..898d12d992 100644 --- a/src/journal/journal-file.h +++ b/src/journal/journal-file.h @@ -36,11 +36,13 @@ #include "hashmap.h" typedef struct JournalMetrics { - uint64_t max_use; - uint64_t use; - uint64_t max_size; - uint64_t min_size; - uint64_t keep_free; + /* For all these: -1 means "pick automatically", and 0 means "no limit enforced" */ + uint64_t max_size; /* how large journal files grow at max */ + uint64_t min_size; /* how large journal files grow at least */ + uint64_t max_use; /* how much disk space to use in total at max, keep_free permitting */ + uint64_t min_use; /* how much disk space to use in total at least, even if keep_free says not to */ + uint64_t keep_free; /* how much to keep free on disk */ + uint64_t n_max_files; /* how many files to keep around at max */ } JournalMetrics; typedef enum direction { @@ -136,7 +138,7 @@ int journal_file_open( JournalFile **ret); int journal_file_set_offline(JournalFile *f); -void journal_file_close(JournalFile *j); +JournalFile* journal_file_close(JournalFile *j); int journal_file_open_reliably( const char *fname, @@ -223,6 +225,7 @@ int journal_file_rotate(JournalFile **f, bool compress, bool seal); void journal_file_post_change(JournalFile *f); +void journal_reset_metrics(JournalMetrics *m); void journal_default_metrics(JournalMetrics *m, int fd); int journal_file_get_cutoff_realtime_usec(JournalFile *f, usec_t *from, usec_t *to); @@ -232,3 +235,8 @@ bool journal_file_rotate_suggested(JournalFile *f, usec_t max_file_usec); int journal_file_map_data_hash_table(JournalFile *f); int journal_file_map_field_hash_table(JournalFile *f); + +static inline bool JOURNAL_FILE_COMPRESS(JournalFile *f) { + assert(f); + return f->compress_xz || f->compress_lz4; +} diff --git a/src/journal/journal-internal.h b/src/journal/journal-internal.h index b51ecdb600..06847402e0 100644 --- a/src/journal/journal-internal.h +++ b/src/journal/journal-internal.h @@ -25,14 +25,14 @@ #include <inttypes.h> #include <stdbool.h> -#include "systemd/sd-id128.h" +#include "sd-id128.h" +#include "sd-journal.h" +#include "hashmap.h" #include "journal-def.h" +#include "journal-file.h" #include "list.h" -#include "hashmap.h" #include "set.h" -#include "journal-file.h" -#include "sd-journal.h" typedef struct Match Match; typedef struct Location Location; @@ -121,7 +121,7 @@ struct sd_journal { Hashmap *directories_by_path; Hashmap *directories_by_wd; - Set *errors; + Hashmap *errors; }; char *journal_make_match_string(sd_journal *j); diff --git a/src/journal/journal-qrcode.h b/src/journal/journal-qrcode.h index 3ff6a3ad4a..7d14e8754b 100644 --- a/src/journal/journal-qrcode.h +++ b/src/journal/journal-qrcode.h @@ -21,8 +21,9 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ +#include <inttypes.h> #include <stdio.h> -#include "systemd/sd-id128.h" +#include "sd-id128.h" int print_qr_code(FILE *f, const void *seed, size_t seed_size, uint64_t start, uint64_t interval, const char *hn, sd_id128_t machine); diff --git a/src/journal/journal-send.c b/src/journal/journal-send.c index 1e3a463504..fa5dee73c3 100644 --- a/src/journal/journal-send.c +++ b/src/journal/journal-send.c @@ -19,20 +19,27 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <sys/socket.h> -#include <sys/un.h> #include <errno.h> -#include <stddef.h> -#include <unistd.h> #include <fcntl.h> #include <printf.h> +#include <stddef.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <unistd.h> #define SD_JOURNAL_SUPPRESS_LOCATION #include "sd-journal.h" -#include "util.h" -#include "socket-util.h" + +#include "alloc-util.h" +#include "fd-util.h" +#include "fileio.h" +#include "io-util.h" #include "memfd-util.h" +#include "socket-util.h" +#include "stdio-util.h" +#include "string-util.h" +#include "util.h" #define SNDBUF_SIZE (8*1024*1024) @@ -212,11 +219,6 @@ _public_ int sd_journal_sendv(const struct iovec *iov, int n) { .msg_namelen = offsetof(struct sockaddr_un, sun_path) + strlen(sa.sun_path), }; ssize_t k; - union { - struct cmsghdr cmsghdr; - uint8_t buf[CMSG_SPACE(sizeof(int))]; - } control; - struct cmsghdr *cmsg; bool have_syslog_identifier = false; bool seal = true; @@ -335,26 +337,7 @@ _public_ int sd_journal_sendv(const struct iovec *iov, int n) { return r; } - mh.msg_iov = NULL; - mh.msg_iovlen = 0; - - zero(control); - mh.msg_control = &control; - mh.msg_controllen = sizeof(control); - - cmsg = CMSG_FIRSTHDR(&mh); - cmsg->cmsg_level = SOL_SOCKET; - cmsg->cmsg_type = SCM_RIGHTS; - cmsg->cmsg_len = CMSG_LEN(sizeof(int)); - memcpy(CMSG_DATA(cmsg), &buffer_fd, sizeof(int)); - - mh.msg_controllen = cmsg->cmsg_len; - - k = sendmsg(fd, &mh, MSG_NOSIGNAL); - if (k < 0) - return -errno; - - return 0; + return send_one_fd(fd, buffer_fd, 0); } static int fill_iovec_perror_and_send(const char *message, int skip, struct iovec iov[]) { diff --git a/src/journal/journal-vacuum.c b/src/journal/journal-vacuum.c index 17499bbc30..4b5fc76eb1 100644 --- a/src/journal/journal-vacuum.c +++ b/src/journal/journal-vacuum.c @@ -23,20 +23,27 @@ #include <sys/stat.h> #include <unistd.h> +#include "sd-id128.h" + +#include "alloc-util.h" +#include "dirent-util.h" +#include "fd-util.h" #include "journal-def.h" #include "journal-file.h" #include "journal-vacuum.h" -#include "sd-id128.h" +#include "parse-util.h" +#include "string-util.h" #include "util.h" +#include "xattr-util.h" struct vacuum_info { uint64_t usage; char *filename; uint64_t realtime; + sd_id128_t seqnum_id; uint64_t seqnum; - bool have_seqnum; }; @@ -67,19 +74,18 @@ static int vacuum_compare(const void *_a, const void *_b) { } static void patch_realtime( - const char *dir, + int fd, const char *fn, const struct stat *st, unsigned long long *realtime) { - _cleanup_free_ char *path = NULL; usec_t x, crtime = 0; /* The timestamp was determined by the file name, but let's * see if the file might actually be older than the file name * suggested... */ - assert(dir); + assert(fd >= 0); assert(fn); assert(st); assert(realtime); @@ -101,14 +107,7 @@ static void patch_realtime( * unfortunately there's currently no sane API to query * it. Hence let's implement this manually... */ - /* Unfortunately there is is not fgetxattrat(), so we need to - * go via path here. :-( */ - - path = strjoin(dir, "/", fn, NULL); - if (!path) - return; - - if (path_getcrtime(path, &crtime) >= 0) { + if (fd_getcrtime_at(fd, fn, &crtime, 0) >= 0) { if (crtime < *realtime) *realtime = crtime; } @@ -120,9 +119,13 @@ static int journal_file_empty(int dir_fd, const char *name) { le64_t n_entries; ssize_t n; - fd = openat(dir_fd, name, O_RDONLY|O_CLOEXEC|O_NOFOLLOW|O_NONBLOCK); - if (fd < 0) - return -errno; + fd = openat(dir_fd, name, O_RDONLY|O_CLOEXEC|O_NOFOLLOW|O_NONBLOCK|O_NOATIME); + if (fd < 0) { + /* Maybe failed due to O_NOATIME and lack of privileges? */ + fd = openat(dir_fd, name, O_RDONLY|O_CLOEXEC|O_NOFOLLOW|O_NONBLOCK); + if (fd < 0) + return -errno; + } if (fstat(fd, &st) < 0) return -errno; @@ -144,22 +147,24 @@ static int journal_file_empty(int dir_fd, const char *name) { int journal_directory_vacuum( const char *directory, uint64_t max_use, + uint64_t n_max_files, usec_t max_retention_usec, usec_t *oldest_usec, bool verbose) { _cleanup_closedir_ DIR *d = NULL; - int r = 0; struct vacuum_info *list = NULL; - unsigned n_list = 0, i; + unsigned n_list = 0, i, n_active_files = 0; size_t n_allocated = 0; uint64_t sum = 0, freed = 0; usec_t retention_limit = 0; char sbytes[FORMAT_BYTES_MAX]; + struct dirent *de; + int r; assert(directory); - if (max_use <= 0 && max_retention_usec <= 0) + if (max_use <= 0 && max_retention_usec <= 0 && n_max_files <= 0) return 0; if (max_retention_usec > 0) { @@ -174,27 +179,20 @@ int journal_directory_vacuum( if (!d) return -errno; - for (;;) { - struct dirent *de; - size_t q; - struct stat st; - char *p; + FOREACH_DIRENT_ALL(de, d, r = -errno; goto finish) { + unsigned long long seqnum = 0, realtime; + _cleanup_free_ char *p = NULL; sd_id128_t seqnum_id; bool have_seqnum; + uint64_t size; + struct stat st; + size_t q; - errno = 0; - de = readdir(d); - if (!de && errno != 0) { - r = -errno; - goto finish; - } - - if (!de) - break; - - if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) + if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) { + log_debug_errno(errno, "Failed to stat file %s while vacuuming, ignoring: %m", de->d_name); continue; + } if (!S_ISREG(st.st_mode)) continue; @@ -203,15 +201,20 @@ int journal_directory_vacuum( if (endswith(de->d_name, ".journal")) { - /* Vacuum archived files */ + /* Vacuum archived files. Active files are + * left around */ - if (q < 1 + 32 + 1 + 16 + 1 + 16 + 8) + if (q < 1 + 32 + 1 + 16 + 1 + 16 + 8) { + n_active_files++; continue; + } if (de->d_name[q-8-16-1] != '-' || de->d_name[q-8-16-1-16-1] != '-' || - de->d_name[q-8-16-1-16-1-32-1] != '@') + de->d_name[q-8-16-1-16-1-32-1] != '@') { + n_active_files++; continue; + } p = strdup(de->d_name); if (!p) { @@ -221,12 +224,12 @@ int journal_directory_vacuum( de->d_name[q-8-16-1-16-1] = 0; if (sd_id128_from_string(de->d_name + q-8-16-1-16-1-32, &seqnum_id) < 0) { - free(p); + n_active_files++; continue; } if (sscanf(de->d_name + q-8-16-1-16, "%16llx-%16llx.journal", &seqnum, &realtime) != 2) { - free(p); + n_active_files++; continue; } @@ -237,12 +240,16 @@ int journal_directory_vacuum( /* Vacuum corrupted files */ - if (q < 1 + 16 + 1 + 16 + 8 + 1) + if (q < 1 + 16 + 1 + 16 + 8 + 1) { + n_active_files ++; continue; + } if (de->d_name[q-1-8-16-1] != '-' || - de->d_name[q-1-8-16-1-16-1] != '@') + de->d_name[q-1-8-16-1-16-1] != '@') { + n_active_files ++; continue; + } p = strdup(de->d_name); if (!p) { @@ -251,55 +258,68 @@ int journal_directory_vacuum( } if (sscanf(de->d_name + q-1-8-16-1-16, "%16llx-%16llx.journal~", &realtime, &tmp) != 2) { - free(p); + n_active_files ++; continue; } have_seqnum = false; - } else - /* We do not vacuum active files or unknown files! */ + } else { + /* We do not vacuum unknown files! */ + log_debug("Not vacuuming unknown file %s.", de->d_name); continue; + } - if (journal_file_empty(dirfd(d), p)) { - /* Always vacuum empty non-online files. */ + size = 512UL * (uint64_t) st.st_blocks; - uint64_t size = 512UL * (uint64_t) st.st_blocks; + r = journal_file_empty(dirfd(d), p); + if (r < 0) { + log_debug_errno(r, "Failed check if %s is empty, ignoring: %m", p); + continue; + } + if (r > 0) { + /* Always vacuum empty non-online files. */ if (unlinkat(dirfd(d), p, 0) >= 0) { - log_full(verbose ? LOG_INFO : LOG_DEBUG, "Deleted empty archived journal %s/%s (%s).", directory, p, format_bytes(sbytes, sizeof(sbytes), size)); + + log_full(verbose ? LOG_INFO : LOG_DEBUG, + "Deleted empty archived journal %s/%s (%s).", directory, p, format_bytes(sbytes, sizeof(sbytes), size)); + freed += size; } else if (errno != ENOENT) log_warning_errno(errno, "Failed to delete empty archived journal %s/%s: %m", directory, p); - free(p); continue; } - patch_realtime(directory, p, &st, &realtime); + patch_realtime(dirfd(d), p, &st, &realtime); if (!GREEDY_REALLOC(list, n_allocated, n_list + 1)) { - free(p); r = -ENOMEM; goto finish; } list[n_list].filename = p; - list[n_list].usage = 512UL * (uint64_t) st.st_blocks; + list[n_list].usage = size; list[n_list].seqnum = seqnum; list[n_list].realtime = realtime; list[n_list].seqnum_id = seqnum_id; list[n_list].have_seqnum = have_seqnum; - - sum += list[n_list].usage; - n_list ++; + + p = NULL; + sum += size; } qsort_safe(list, n_list, sizeof(struct vacuum_info), vacuum_compare); for (i = 0; i < n_list; i++) { + unsigned left; + + left = n_active_files + n_list - i; + if ((max_retention_usec <= 0 || list[i].realtime >= retention_limit) && - (max_use <= 0 || sum <= max_use)) + (max_use <= 0 || sum <= max_use) && + (n_max_files <= 0 || left <= n_max_files)) break; if (unlinkat(dirfd(d), list[i].filename, 0) >= 0) { @@ -318,6 +338,8 @@ int journal_directory_vacuum( if (oldest_usec && i < n_list && (*oldest_usec == 0 || list[i].realtime < *oldest_usec)) *oldest_usec = list[i].realtime; + r = 0; + finish: for (i = 0; i < n_list; i++) free(list[i].filename); diff --git a/src/journal/journal-vacuum.h b/src/journal/journal-vacuum.h index c45cc31d0e..49ab90af91 100644 --- a/src/journal/journal-vacuum.h +++ b/src/journal/journal-vacuum.h @@ -21,5 +21,9 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ +#include <inttypes.h> +#include <stdbool.h> -int journal_directory_vacuum(const char *directory, uint64_t max_use, usec_t max_retention_usec, usec_t *oldest_usec, bool vacuum); +#include "time-util.h" + +int journal_directory_vacuum(const char *directory, uint64_t max_use, uint64_t n_max_files, usec_t max_retention_usec, usec_t *oldest_usec, bool verbose); diff --git a/src/journal/journal-verify.c b/src/journal/journal-verify.c index 32d59c716f..3676cb8788 100644 --- a/src/journal/journal-verify.c +++ b/src/journal/journal-verify.c @@ -24,15 +24,18 @@ #include <fcntl.h> #include <stddef.h> -#include "util.h" -#include "macro.h" +#include "alloc-util.h" +#include "compress.h" +#include "fd-util.h" +#include "fileio.h" +#include "journal-authenticate.h" #include "journal-def.h" #include "journal-file.h" -#include "journal-authenticate.h" #include "journal-verify.h" #include "lookup3.h" -#include "compress.h" +#include "macro.h" #include "terminal-util.h" +#include "util.h" static void draw_progress(uint64_t p, usec_t *last_usec) { unsigned n, i, j, k; @@ -839,19 +842,19 @@ int journal_file_verify( data_fd = open_tmpfile("/var/tmp", O_RDWR | O_CLOEXEC); if (data_fd < 0) { - r = log_error_errno(errno, "Failed to create data file: %m"); + r = log_error_errno(data_fd, "Failed to create data file: %m"); goto fail; } entry_fd = open_tmpfile("/var/tmp", O_RDWR | O_CLOEXEC); if (entry_fd < 0) { - r = log_error_errno(errno, "Failed to create entry file: %m"); + r = log_error_errno(entry_fd, "Failed to create entry file: %m"); goto fail; } entry_array_fd = open_tmpfile("/var/tmp", O_RDWR | O_CLOEXEC); if (entry_array_fd < 0) { - r = log_error_errno(errno, + r = log_error_errno(entry_array_fd, "Failed to create entry array file: %m"); goto fail; } @@ -897,7 +900,7 @@ int journal_file_verify( r = journal_file_object_verify(f, p, o); if (r < 0) { - error(p, "Envalid object contents: %s", strerror(-r)); + error(p, "Invalid object contents: %s", strerror(-r)); goto fail; } diff --git a/src/journal/journalctl.c b/src/journal/journalctl.c index b38b151485..277adba904 100644 --- a/src/journal/journalctl.c +++ b/src/journal/journalctl.c @@ -19,48 +19,58 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <locale.h> +#include <errno.h> #include <fcntl.h> #include <fnmatch.h> -#include <errno.h> +#include <getopt.h> +#include <linux/fs.h> +#include <locale.h> +#include <poll.h> +#include <signal.h> #include <stddef.h> -#include <string.h> #include <stdio.h> -#include <unistd.h> #include <stdlib.h> -#include <getopt.h> -#include <signal.h> -#include <poll.h> -#include <sys/stat.h> +#include <string.h> #include <sys/inotify.h> -#include <linux/fs.h> +#include <sys/stat.h> +#include <unistd.h> -#include "sd-journal.h" #include "sd-bus.h" -#include "log.h" -#include "logs-show.h" -#include "util.h" +#include "sd-journal.h" + #include "acl-util.h" -#include "path-util.h" +#include "alloc-util.h" +#include "bus-error.h" +#include "bus-util.h" +#include "catalog.h" +#include "chattr-util.h" +#include "fd-util.h" #include "fileio.h" -#include "build.h" -#include "pager.h" -#include "strv.h" -#include "set.h" -#include "sigbus.h" -#include "journal-internal.h" +#include "fs-util.h" +#include "fsprg.h" +#include "glob-util.h" +#include "hostname-util.h" +#include "io-util.h" #include "journal-def.h" -#include "journal-verify.h" +#include "journal-internal.h" #include "journal-qrcode.h" #include "journal-vacuum.h" -#include "fsprg.h" -#include "unit-name.h" -#include "catalog.h" +#include "journal-verify.h" +#include "locale-util.h" +#include "log.h" +#include "logs-show.h" #include "mkdir.h" -#include "bus-util.h" -#include "bus-error.h" +#include "pager.h" +#include "parse-util.h" +#include "path-util.h" +#include "rlimit-util.h" +#include "set.h" +#include "sigbus.h" +#include "strv.h" +#include "syslog-util.h" #include "terminal-util.h" -#include "hostname-util.h" +#include "unit-name.h" +#include "user-util.h" #define DEFAULT_FSS_INTERVAL_USEC (15*USEC_PER_MINUTE) @@ -105,10 +115,11 @@ static const char *arg_field = NULL; static bool arg_catalog = false; static bool arg_reverse = false; static int arg_journal_type = 0; -static const char *arg_root = NULL; +static char *arg_root = NULL; static const char *arg_machine = NULL; -static uint64_t arg_vacuum_size = (uint64_t) -1; -static usec_t arg_vacuum_time = USEC_INFINITY; +static uint64_t arg_vacuum_size = 0; +static uint64_t arg_vacuum_n_files = 0; +static usec_t arg_vacuum_time = 0; static enum { ACTION_SHOW, @@ -122,6 +133,7 @@ static enum { ACTION_UPDATE_CATALOG, ACTION_LIST_BOOTS, ACTION_FLUSH, + ACTION_ROTATE, ACTION_VACUUM, } arg_action = ACTION_SHOW; @@ -193,8 +205,8 @@ static void help(void) { " --system Show the system journal\n" " --user Show the user journal for the current user\n" " -M --machine=CONTAINER Operate on local container\n" - " --since=DATE Show entries not older than the specified date\n" - " --until=DATE Show entries not newer than the specified date\n" + " -S --since=DATE Show entries not older than the specified date\n" + " -U --until=DATE Show entries not newer than the specified date\n" " -c --cursor=CURSOR Show entries starting at the specified cursor\n" " --after-cursor=CURSOR Show entries after the specified cursor\n" " --show-cursor Print the cursor after all the entries\n" @@ -217,7 +229,7 @@ static void help(void) { " -x --catalog Add message explanations where available\n" " --no-full Ellipsize fields\n" " -a --all Show all fields, including long and unprintable\n" - " -q --quiet Do not show privilege warning\n" + " -q --quiet Do not show info messages and privilege warning\n" " --no-pager Do not pipe output into a pager\n" " -m --merge Show entries from all available journals\n" " -D --directory=PATH Show journal files from directory\n" @@ -235,8 +247,10 @@ static void help(void) { " --new-id128 Generate a new 128-bit ID\n" " --disk-usage Show total disk usage of all journal files\n" " --vacuum-size=BYTES Reduce disk usage below specified size\n" - " --vacuum-time=TIME Remove journal files older than specified date\n" + " --vacuum-files=INT Leave only the specified number of journal files\n" + " --vacuum-time=TIME Remove journal files older than specified time\n" " --flush Flush all journal data from /run into /var\n" + " --rotate Request immediate rotation of the journal files\n" " --header Show journal header information\n" " --list-catalog Show all message IDs in the catalog\n" " --dump-catalog Show entries in the message catalog\n" @@ -267,8 +281,6 @@ static int parse_argv(int argc, char *argv[]) { ARG_VERIFY, ARG_VERIFY_KEY, ARG_DISK_USAGE, - ARG_SINCE, - ARG_UNTIL, ARG_AFTER_CURSOR, ARG_SHOW_CURSOR, ARG_USER_UNIT, @@ -278,7 +290,9 @@ static int parse_argv(int argc, char *argv[]) { ARG_FORCE, ARG_UTC, ARG_FLUSH, + ARG_ROTATE, ARG_VACUUM_SIZE, + ARG_VACUUM_FILES, ARG_VACUUM_TIME, }; @@ -318,8 +332,8 @@ static int parse_argv(int argc, char *argv[]) { { "cursor", required_argument, NULL, 'c' }, { "after-cursor", required_argument, NULL, ARG_AFTER_CURSOR }, { "show-cursor", no_argument, NULL, ARG_SHOW_CURSOR }, - { "since", required_argument, NULL, ARG_SINCE }, - { "until", required_argument, NULL, ARG_UNTIL }, + { "since", required_argument, NULL, 'S' }, + { "until", required_argument, NULL, 'U' }, { "unit", required_argument, NULL, 'u' }, { "user-unit", required_argument, NULL, ARG_USER_UNIT }, { "field", required_argument, NULL, 'F' }, @@ -331,7 +345,9 @@ static int parse_argv(int argc, char *argv[]) { { "machine", required_argument, NULL, 'M' }, { "utc", no_argument, NULL, ARG_UTC }, { "flush", no_argument, NULL, ARG_FLUSH }, + { "rotate", no_argument, NULL, ARG_ROTATE }, { "vacuum-size", required_argument, NULL, ARG_VACUUM_SIZE }, + { "vacuum-files", required_argument, NULL, ARG_VACUUM_FILES }, { "vacuum-time", required_argument, NULL, ARG_VACUUM_TIME }, {} }; @@ -341,7 +357,7 @@ static int parse_argv(int argc, char *argv[]) { assert(argc >= 0); assert(argv); - while ((c = getopt_long(argc, argv, "hefo:aln::qmb::kD:p:c:t:u:F:xrM:", options, NULL)) >= 0) + while ((c = getopt_long(argc, argv, "hefo:aln::qmb::kD:p:c:S:U:t:u:F:xrM:", options, NULL)) >= 0) switch (c) { @@ -350,9 +366,7 @@ static int parse_argv(int argc, char *argv[]) { return 0; case ARG_VERSION: - puts(PACKAGE_STRING); - puts(SYSTEMD_FEATURES); - return 0; + return version(); case ARG_NO_PAGER: arg_no_pager = true; @@ -502,7 +516,9 @@ static int parse_argv(int argc, char *argv[]) { break; case ARG_ROOT: - arg_root = optarg; + r = parse_path_argument_and_warn(optarg, true, &arg_root); + if (r < 0) + return r; break; case 'c': @@ -539,6 +555,16 @@ static int parse_argv(int argc, char *argv[]) { arg_action = ACTION_VACUUM; break; + case ARG_VACUUM_FILES: + r = safe_atou64(optarg, &arg_vacuum_n_files); + if (r < 0) { + log_error("Failed to parse vacuum files: %s", optarg); + return r; + } + + arg_action = ACTION_VACUUM; + break; + case ARG_VACUUM_TIME: r = parse_sec(optarg, &arg_vacuum_time); if (r < 0) { @@ -631,7 +657,7 @@ static int parse_argv(int argc, char *argv[]) { break; } - case ARG_SINCE: + case 'S': r = parse_timestamp(optarg, &arg_since); if (r < 0) { log_error("Failed to parse timestamp: %s", optarg); @@ -640,7 +666,7 @@ static int parse_argv(int argc, char *argv[]) { arg_since_set = true; break; - case ARG_UNTIL: + case 'U': r = parse_timestamp(optarg, &arg_until); if (r < 0) { log_error("Failed to parse timestamp: %s", optarg); @@ -699,6 +725,10 @@ static int parse_argv(int argc, char *argv[]) { arg_action = ACTION_FLUSH; break; + case ARG_ROTATE: + arg_action = ACTION_ROTATE; + break; + case '?': return -EINVAL; @@ -729,7 +759,7 @@ static int parse_argv(int argc, char *argv[]) { return -EINVAL; } - if (arg_action != ACTION_SHOW && optind < argc) { + if (!IN_SET(arg_action, ACTION_SHOW, ACTION_DUMP_CATALOG, ACTION_LIST_CATALOG) && optind < argc) { log_error("Extraneous arguments starting with '%s'", argv[optind]); return -EINVAL; } @@ -1453,7 +1483,7 @@ static int setup_keys(void) { safe_close(fd); fd = mkostemp_safe(k, O_WRONLY|O_CLOEXEC); if (fd < 0) { - r = log_error_errno(errno, "Failed to open %s: %m", k); + r = log_error_errno(fd, "Failed to open %s: %m", k); goto finish; } @@ -1461,7 +1491,7 @@ static int setup_keys(void) { * writing and in-place updating */ r = chattr_fd(fd, FS_SECRM_FL|FS_NODUMP_FL|FS_SYNC_FL|FS_NOCOW_FL, FS_SECRM_FL|FS_NODUMP_FL|FS_SYNC_FL|FS_NOCOW_FL); if (r < 0) - log_warning_errno(errno, "Failed to set file attributes: %m"); + log_warning_errno(r, "Failed to set file attributes: %m"); zero(h); memcpy(h.signature, "KSHHRHLP", 8); @@ -1580,7 +1610,7 @@ static int verify(sd_journal *j) { /* If the key was invalid give up right-away. */ return k; } else if (k < 0) { - log_warning("FAIL: %s (%s)", f->path, strerror(-k)); + log_warning_errno(k, "FAIL: %s (%m)", f->path); r = k; } else { char a[FORMAT_TIMESTAMP_MAX], b[FORMAT_TIMESTAMP_MAX], c[FORMAT_TIMESPAN_MAX]; @@ -1678,36 +1708,50 @@ static int access_check_var_log_journal(sd_journal *j) { static int access_check(sd_journal *j) { Iterator it; void *code; + char *path; int r = 0; assert(j); - if (set_isempty(j->errors)) { + if (hashmap_isempty(j->errors)) { if (ordered_hashmap_isempty(j->files)) log_notice("No journal files were found."); return 0; } - if (set_contains(j->errors, INT_TO_PTR(-EACCES))) { + if (hashmap_contains(j->errors, INT_TO_PTR(-EACCES))) { (void) access_check_var_log_journal(j); if (ordered_hashmap_isempty(j->files)) r = log_error_errno(EACCES, "No journal files were opened due to insufficient permissions."); } - SET_FOREACH(code, j->errors, it) { + HASHMAP_FOREACH_KEY(path, code, j->errors, it) { int err; - err = -PTR_TO_INT(code); - assert(err > 0); + err = abs(PTR_TO_INT(code)); - if (err == EACCES) + switch (err) { + case EACCES: continue; - log_warning_errno(err, "Error was encountered while opening journal files: %m"); - if (r == 0) - r = -err; + case ENODATA: + log_warning_errno(err, "Journal file %s is truncated, ignoring file.", path); + break; + + case EPROTONOSUPPORT: + log_warning_errno(err, "Journal file %s uses an unsupported feature, ignoring file.", path); + break; + + case EBADMSG: + log_warning_errno(err, "Journal file %s corrupted, ignoring file.", path); + break; + + default: + log_warning_errno(err, "An error was encountered while opening journal file %s, ignoring file.", path); + break; + } } return r; @@ -1725,7 +1769,7 @@ static int flush_to_var(void) { /* OK, let's actually do the full logic, send SIGUSR1 to the * daemon and set up inotify to wait for the flushed file to appear */ - r = bus_open_system_systemd(&bus); + r = bus_connect_system_systemd(&bus); if (r < 0) return log_error_errno(r, "Failed to get D-Bus connection: %m"); @@ -1772,6 +1816,30 @@ static int flush_to_var(void) { return 0; } +static int rotate(void) { + _cleanup_bus_error_free_ sd_bus_error error = SD_BUS_ERROR_NULL; + _cleanup_bus_flush_close_unref_ sd_bus *bus = NULL; + int r; + + r = bus_connect_system_systemd(&bus); + if (r < 0) + return log_error_errno(r, "Failed to get D-Bus connection: %m"); + + r = sd_bus_call_method( + bus, + "org.freedesktop.systemd1", + "/org/freedesktop/systemd1", + "org.freedesktop.systemd1.Manager", + "KillUnit", + &error, + NULL, + "ssi", "systemd-journald.service", "main", SIGUSR2); + if (r < 0) + return log_error_errno(r, "Failed to kill journal service: %s", bus_error_message(&error, r)); + + return 0; +} + int main(int argc, char *argv[]) { int r; _cleanup_journal_close_ sd_journal *j = NULL; @@ -1807,6 +1875,11 @@ int main(int argc, char *argv[]) { goto finish; } + if (arg_action == ACTION_ROTATE) { + r = rotate(); + goto finish; + } + if (arg_action == ACTION_SETUP_KEYS) { r = setup_keys(); goto finish; @@ -1831,6 +1904,7 @@ int main(int argc, char *argv[]) { } else { bool oneline = arg_action == ACTION_LIST_CATALOG; + pager_open_if_enabled(); if (optind < argc) r = catalog_list_items(stdout, database, oneline, argv + optind); @@ -1895,9 +1969,9 @@ int main(int argc, char *argv[]) { if (d->is_root) continue; - q = journal_directory_vacuum(d->path, arg_vacuum_size, arg_vacuum_time, NULL, true); + q = journal_directory_vacuum(d->path, arg_vacuum_size, arg_vacuum_n_files, arg_vacuum_time, NULL, true); if (q < 0) { - log_error_errno(q, "Failed to vacuum: %m"); + log_error_errno(q, "Failed to vacuum %s: %m", d->path); r = q; } } @@ -2200,5 +2274,7 @@ finish: strv_free(arg_system_units); strv_free(arg_user_units); + free(arg_root); + return r < 0 ? EXIT_FAILURE : EXIT_SUCCESS; } diff --git a/src/journal/journald-audit.c b/src/journal/journald-audit.c index fe8ae194c9..3c13fe0d67 100644 --- a/src/journal/journald-audit.c +++ b/src/journal/journald-audit.c @@ -19,9 +19,14 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include "missing.h" -#include "journald-audit.h" +#include "alloc-util.h" #include "audit-type.h" +#include "fd-util.h" +#include "hexdecoct.h" +#include "io-util.h" +#include "journald-audit.h" +#include "missing.h" +#include "string-util.h" typedef struct MapField { const char *audit_field; diff --git a/src/journal/journald-console.c b/src/journal/journald-console.c index 307bdc3949..89f3d4b42f 100644 --- a/src/journal/journald-console.c +++ b/src/journal/journald-console.c @@ -23,11 +23,16 @@ #include <fcntl.h> #include <sys/socket.h> +#include "alloc-util.h" +#include "fd-util.h" #include "fileio.h" -#include "journald-server.h" -#include "journald-console.h" #include "formats-util.h" +#include "io-util.h" +#include "journald-console.h" +#include "journald-server.h" +#include "parse-util.h" #include "process-util.h" +#include "stdio-util.h" #include "terminal-util.h" static bool prefix_timestamp(void) { @@ -101,7 +106,7 @@ void server_forward_console( fd = open_terminal(tty, O_WRONLY|O_NOCTTY|O_CLOEXEC); if (fd < 0) { - log_debug_errno(errno, "Failed to open %s for logging: %m", tty); + log_debug_errno(fd, "Failed to open %s for logging: %m", tty); return; } diff --git a/src/journal/journald-gperf.gperf b/src/journal/journald-gperf.gperf index bf7c773009..c154610c54 100644 --- a/src/journal/journald-gperf.gperf +++ b/src/journal/journald-gperf.gperf @@ -24,9 +24,11 @@ Journal.RateLimitBurst, config_parse_unsigned, 0, offsetof(Server, rate_li Journal.SystemMaxUse, config_parse_iec_uint64, 0, offsetof(Server, system_metrics.max_use) Journal.SystemMaxFileSize, config_parse_iec_uint64, 0, offsetof(Server, system_metrics.max_size) Journal.SystemKeepFree, config_parse_iec_uint64, 0, offsetof(Server, system_metrics.keep_free) +Journal.SystemMaxFiles, config_parse_uint64, 0, offsetof(Server, system_metrics.n_max_files) Journal.RuntimeMaxUse, config_parse_iec_uint64, 0, offsetof(Server, runtime_metrics.max_use) Journal.RuntimeMaxFileSize, config_parse_iec_uint64, 0, offsetof(Server, runtime_metrics.max_size) Journal.RuntimeKeepFree, config_parse_iec_uint64, 0, offsetof(Server, runtime_metrics.keep_free) +Journal.RuntimeMaxFiles, config_parse_uint64, 0, offsetof(Server, runtime_metrics.n_max_files) Journal.MaxRetentionSec, config_parse_sec, 0, offsetof(Server, max_retention_usec) Journal.MaxFileSec, config_parse_sec, 0, offsetof(Server, max_file_usec) Journal.ForwardToSyslog, config_parse_bool, 0, offsetof(Server, forward_to_syslog) diff --git a/src/journal/journald-kmsg.c b/src/journal/journald-kmsg.c index 51fe3aa50a..e048e04716 100644 --- a/src/journal/journald-kmsg.c +++ b/src/journal/journald-kmsg.c @@ -19,20 +19,26 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <unistd.h> -#include <sys/epoll.h> #include <fcntl.h> +#include <sys/epoll.h> #include <sys/mman.h> #include <sys/socket.h> +#include <unistd.h> -#include "systemd/sd-messages.h" -#include <libudev.h> +#include "libudev.h" +#include "sd-messages.h" -#include "journald-server.h" +#include "escape.h" +#include "fd-util.h" +#include "formats-util.h" +#include "io-util.h" #include "journald-kmsg.h" +#include "journald-server.h" #include "journald-syslog.h" -#include "formats-util.h" +#include "parse-util.h" #include "process-util.h" +#include "stdio-util.h" +#include "string-util.h" void server_forward_kmsg( Server *s, @@ -341,8 +347,7 @@ static int server_read_dev_kmsg(Server *s) { if (errno == EAGAIN || errno == EINTR || errno == EPIPE) return 0; - log_error_errno(errno, "Failed to read from kernel: %m"); - return -errno; + return log_error_errno(errno, "Failed to read from kernel: %m"); } dev_kmsg_record(s, buffer, l); @@ -436,6 +441,7 @@ fail: int server_open_kernel_seqnum(Server *s) { _cleanup_close_ int fd; uint64_t *p; + int r; assert(s); @@ -449,8 +455,9 @@ int server_open_kernel_seqnum(Server *s) { return 0; } - if (posix_fallocate(fd, 0, sizeof(uint64_t)) < 0) { - log_error_errno(errno, "Failed to allocate sequential number file, ignoring: %m"); + r = posix_fallocate(fd, 0, sizeof(uint64_t)); + if (r != 0) { + log_error_errno(r, "Failed to allocate sequential number file, ignoring: %m"); return 0; } diff --git a/src/journal/journald-native.c b/src/journal/journald-native.c index 3e8a7a05f6..1e3774dafb 100644 --- a/src/journal/journald-native.c +++ b/src/journal/journald-native.c @@ -19,21 +19,27 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <unistd.h> #include <stddef.h> #include <sys/epoll.h> #include <sys/mman.h> +#include <unistd.h> -#include "socket-util.h" -#include "path-util.h" -#include "selinux-util.h" -#include "journald-server.h" -#include "journald-native.h" -#include "journald-kmsg.h" +#include "alloc-util.h" +#include "fd-util.h" +#include "fs-util.h" +#include "io-util.h" #include "journald-console.h" +#include "journald-kmsg.h" +#include "journald-native.h" +#include "journald-server.h" #include "journald-syslog.h" #include "journald-wall.h" #include "memfd-util.h" +#include "parse-util.h" +#include "path-util.h" +#include "selinux-util.h" +#include "socket-util.h" +#include "string-util.h" bool valid_user_field(const char *p, size_t l, bool allow_protected) { const char *a; @@ -338,7 +344,7 @@ void server_process_native_file( r = readlink_malloc(sl, &k); if (r < 0) { - log_error_errno(errno, "readlink(%s) failed: %m", sl); + log_error_errno(r, "readlink(%s) failed: %m", sl); return; } @@ -407,7 +413,7 @@ void server_process_native_file( n = pread(fd, p, st.st_size, 0); if (n < 0) - log_error_errno(n, "Failed to read file, ignoring: %m"); + log_error_errno(errno, "Failed to read file, ignoring: %m"); else if (n > 0) server_process_native_message(s, p, n, ucred, tv, label, label_len); } diff --git a/src/journal/journald-rate-limit.c b/src/journal/journald-rate-limit.c index 6f83035a4e..434ddc8ac9 100644 --- a/src/journal/journald-rate-limit.c +++ b/src/journal/journald-rate-limit.c @@ -19,14 +19,16 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <string.h> #include <errno.h> +#include <string.h> -#include "journald-rate-limit.h" -#include "list.h" -#include "util.h" +#include "alloc-util.h" #include "hashmap.h" +#include "list.h" #include "random-util.h" +#include "string-util.h" +#include "util.h" +#include "journald-rate-limit.h" #define POOLS_MAX 5 #define BUCKETS_MAX 127 @@ -57,7 +59,7 @@ struct JournalRateLimitGroup { char *id; JournalRateLimitPool pools[POOLS_MAX]; - unsigned long hash; + uint64_t hash; LIST_FIELDS(JournalRateLimitGroup, bucket); LIST_FIELDS(JournalRateLimitGroup, lru); @@ -145,6 +147,7 @@ static void journal_rate_limit_vacuum(JournalRateLimit *r, usec_t ts) { static JournalRateLimitGroup* journal_rate_limit_group_new(JournalRateLimit *r, const char *id, usec_t ts) { JournalRateLimitGroup *g; + struct siphash state; assert(r); assert(id); @@ -157,7 +160,9 @@ static JournalRateLimitGroup* journal_rate_limit_group_new(JournalRateLimit *r, if (!g->id) goto fail; - g->hash = string_hash_func(g->id, r->hash_key); + siphash24_init(&state, r->hash_key); + string_hash_func(g->id, &state); + siphash24_finalize((uint8_t*)&g->hash, &state); journal_rate_limit_vacuum(r, ts); @@ -204,9 +209,10 @@ static unsigned burst_modulate(unsigned burst, uint64_t available) { } int journal_rate_limit_test(JournalRateLimit *r, const char *id, int priority, uint64_t available) { - unsigned long h; + uint64_t h; JournalRateLimitGroup *g; JournalRateLimitPool *p; + struct siphash state; unsigned burst; usec_t ts; @@ -222,7 +228,9 @@ int journal_rate_limit_test(JournalRateLimit *r, const char *id, int priority, u ts = now(CLOCK_MONOTONIC); - h = string_hash_func(id, r->hash_key); + siphash24_init(&state, r->hash_key); + string_hash_func(id, &state); + siphash24_finalize((uint8_t*)&h, &state); g = r->buckets[h % BUCKETS_MAX]; LIST_FOREACH(bucket, g, g) diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index fa2e9b9825..7a70dcbc57 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -19,45 +19,55 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <sys/signalfd.h> -#include <sys/ioctl.h> -#include <linux/sockios.h> -#include <sys/statvfs.h> -#include <sys/mman.h> - #ifdef HAVE_SELINUX #include <selinux/selinux.h> #endif +#include <sys/ioctl.h> +#include <sys/mman.h> +#include <sys/signalfd.h> +#include <sys/statvfs.h> +#include <linux/sockios.h> -#include <libudev.h> - +#include "libudev.h" +#include "sd-daemon.h" #include "sd-journal.h" #include "sd-messages.h" -#include "sd-daemon.h" -#include "mkdir.h" -#include "rm-rf.h" -#include "hashmap.h" -#include "journal-file.h" -#include "socket-util.h" + +#include "acl-util.h" +#include "alloc-util.h" +#include "audit-util.h" #include "cgroup-util.h" -#include "missing.h" #include "conf-parser.h" -#include "selinux-util.h" -#include "acl-util.h" +#include "dirent-util.h" +#include "extract-word.h" +#include "fd-util.h" #include "formats-util.h" -#include "process-util.h" +#include "fs-util.h" +#include "hashmap.h" #include "hostname-util.h" -#include "signal-util.h" +#include "io-util.h" +#include "journal-authenticate.h" +#include "journal-file.h" #include "journal-internal.h" #include "journal-vacuum.h" -#include "journal-authenticate.h" -#include "journald-rate-limit.h" +#include "journald-audit.h" #include "journald-kmsg.h" -#include "journald-syslog.h" -#include "journald-stream.h" #include "journald-native.h" -#include "journald-audit.h" +#include "journald-rate-limit.h" #include "journald-server.h" +#include "journald-stream.h" +#include "journald-syslog.h" +#include "missing.h" +#include "mkdir.h" +#include "parse-util.h" +#include "proc-cmdline.h" +#include "process-util.h" +#include "rm-rf.h" +#include "selinux-util.h" +#include "signal-util.h" +#include "socket-util.h" +#include "string-table.h" +#include "string-util.h" #define USER_JOURNALS_MAX 1024 @@ -66,88 +76,63 @@ #define DEFAULT_RATE_LIMIT_BURST 1000 #define DEFAULT_MAX_FILE_USEC USEC_PER_MONTH -#define RECHECK_AVAILABLE_SPACE_USEC (30*USEC_PER_SEC) +#define RECHECK_SPACE_USEC (30*USEC_PER_SEC) -static const char* const storage_table[_STORAGE_MAX] = { - [STORAGE_AUTO] = "auto", - [STORAGE_VOLATILE] = "volatile", - [STORAGE_PERSISTENT] = "persistent", - [STORAGE_NONE] = "none" -}; - -DEFINE_STRING_TABLE_LOOKUP(storage, Storage); -DEFINE_CONFIG_PARSE_ENUM(config_parse_storage, storage, Storage, "Failed to parse storage setting"); - -static const char* const split_mode_table[_SPLIT_MAX] = { - [SPLIT_LOGIN] = "login", - [SPLIT_UID] = "uid", - [SPLIT_NONE] = "none", -}; - -DEFINE_STRING_TABLE_LOOKUP(split_mode, SplitMode); -DEFINE_CONFIG_PARSE_ENUM(config_parse_split_mode, split_mode, SplitMode, "Failed to parse split mode setting"); +#define NOTIFY_SNDBUF_SIZE (8*1024*1024) -static uint64_t available_space(Server *s, bool verbose) { - char ids[33]; - _cleanup_free_ char *p = NULL; - sd_id128_t machine; - struct statvfs ss; - uint64_t sum = 0, ss_avail = 0, avail = 0; - int r; +static int determine_space_for( + Server *s, + JournalMetrics *metrics, + const char *path, + const char *name, + bool verbose, + bool patch_min_use, + uint64_t *available, + uint64_t *limit) { + + uint64_t sum = 0, ss_avail, avail; _cleanup_closedir_ DIR *d = NULL; + struct dirent *de; + struct statvfs ss; + const char *p; usec_t ts; - const char *f; - JournalMetrics *m; - - ts = now(CLOCK_MONOTONIC); - if (s->cached_available_space_timestamp + RECHECK_AVAILABLE_SPACE_USEC > ts - && !verbose) - return s->cached_available_space; + assert(s); + assert(metrics); + assert(path); + assert(name); - r = sd_id128_get_machine(&machine); - if (r < 0) - return 0; + ts = now(CLOCK_MONOTONIC); - if (s->system_journal) { - f = "/var/log/journal/"; - m = &s->system_metrics; - } else { - f = "/run/log/journal/"; - m = &s->runtime_metrics; - } + if (!verbose && s->cached_space_timestamp + RECHECK_SPACE_USEC > ts) { - assert(m); + if (available) + *available = s->cached_space_available; + if (limit) + *limit = s->cached_space_limit; - p = strappend(f, sd_id128_to_string(machine, ids)); - if (!p) return 0; + } + p = strjoina(path, SERVER_MACHINE_ID(s)); d = opendir(p); if (!d) - return 0; + return log_full_errno(errno == ENOENT ? LOG_DEBUG : LOG_ERR, errno, "Failed to open %s: %m", p); if (fstatvfs(dirfd(d), &ss) < 0) - return 0; + return log_error_errno(errno, "Failed to fstatvfs(%s): %m", p); - for (;;) { + FOREACH_DIRENT_ALL(de, d, break) { struct stat st; - struct dirent *de; - - errno = 0; - de = readdir(d); - if (!de && errno != 0) - return 0; - - if (!de) - break; if (!endswith(de->d_name, ".journal") && !endswith(de->d_name, ".journal~")) continue; - if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) + if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) { + log_debug_errno(errno, "Failed to stat %s/%s, ignoring: %m", p, de->d_name); continue; + } if (!S_ISREG(st.st_mode)) continue; @@ -155,45 +140,72 @@ static uint64_t available_space(Server *s, bool verbose) { sum += (uint64_t) st.st_blocks * 512UL; } - ss_avail = ss.f_bsize * ss.f_bavail; - - /* If we reached a high mark, we will always allow this much - * again, unless usage goes above max_use. This watermark - * value is cached so that we don't give up space on pressure, - * but hover below the maximum usage. */ + /* If request, then let's bump the min_use limit to the + * current usage on disk. We do this when starting up and + * first opening the journal files. This way sudden spikes in + * disk usage will not cause journald to vacuum files without + * bounds. Note that this means that only a restart of + * journald will make it reset this value. */ - if (m->use < sum) - m->use = sum; + if (patch_min_use) + metrics->min_use = MAX(metrics->min_use, sum); - avail = LESS_BY(ss_avail, m->keep_free); + ss_avail = ss.f_bsize * ss.f_bavail; + avail = LESS_BY(ss_avail, metrics->keep_free); - s->cached_available_space = LESS_BY(MIN(m->max_use, avail), sum); - s->cached_available_space_timestamp = ts; + s->cached_space_limit = MIN(MAX(sum + avail, metrics->min_use), metrics->max_use); + s->cached_space_available = LESS_BY(s->cached_space_limit, sum); + s->cached_space_timestamp = ts; if (verbose) { char fb1[FORMAT_BYTES_MAX], fb2[FORMAT_BYTES_MAX], fb3[FORMAT_BYTES_MAX], - fb4[FORMAT_BYTES_MAX], fb5[FORMAT_BYTES_MAX]; + fb4[FORMAT_BYTES_MAX], fb5[FORMAT_BYTES_MAX], fb6[FORMAT_BYTES_MAX]; server_driver_message(s, SD_MESSAGE_JOURNAL_USAGE, - "%s is currently using %s.\n" + "%s (%s) is currently using %s.\n" "Maximum allowed usage is set to %s.\n" "Leaving at least %s free (of currently available %s of space).\n" - "Enforced usage limit is thus %s.", - s->system_journal ? "Permanent journal (/var/log/journal/)" : "Runtime journal (/run/log/journal/)", + "Enforced usage limit is thus %s, of which %s are still available.", + name, path, format_bytes(fb1, sizeof(fb1), sum), - format_bytes(fb2, sizeof(fb2), m->max_use), - format_bytes(fb3, sizeof(fb3), m->keep_free), + format_bytes(fb2, sizeof(fb2), metrics->max_use), + format_bytes(fb3, sizeof(fb3), metrics->keep_free), format_bytes(fb4, sizeof(fb4), ss_avail), - format_bytes(fb5, sizeof(fb5), s->cached_available_space + sum)); + format_bytes(fb5, sizeof(fb5), s->cached_space_limit), + format_bytes(fb6, sizeof(fb6), s->cached_space_available)); } - return s->cached_available_space; + if (available) + *available = s->cached_space_available; + if (limit) + *limit = s->cached_space_limit; + + return 1; +} + +static int determine_space(Server *s, bool verbose, bool patch_min_use, uint64_t *available, uint64_t *limit) { + JournalMetrics *metrics; + const char *path, *name; + + assert(s); + + if (s->system_journal) { + path = "/var/log/journal/"; + metrics = &s->system_metrics; + name = "System journal"; + } else { + path = "/run/log/journal/"; + metrics = &s->runtime_metrics; + name = "Runtime journal"; + } + + return determine_space_for(s, metrics, path, name, verbose, patch_min_use, available, limit); } void server_fix_perms(Server *s, JournalFile *f, uid_t uid) { int r; #ifdef HAVE_ACL - acl_t acl; + _cleanup_(acl_freep) acl_t acl = NULL; acl_entry_t entry; acl_permset_t permset; #endif @@ -202,7 +214,7 @@ void server_fix_perms(Server *s, JournalFile *f, uid_t uid) { r = fchmod(f->fd, 0640); if (r < 0) - log_warning_errno(r, "Failed to fix access mode on %s, ignoring: %m", f->path); + log_warning_errno(errno, "Failed to fix access mode on %s, ignoring: %m", f->path); #ifdef HAVE_ACL if (uid <= SYSTEM_UID_MAX) @@ -221,24 +233,27 @@ void server_fix_perms(Server *s, JournalFile *f, uid_t uid) { acl_set_tag_type(entry, ACL_USER) < 0 || acl_set_qualifier(entry, &uid) < 0) { log_warning_errno(errno, "Failed to patch ACL on %s, ignoring: %m", f->path); - goto finish; + return; } } /* We do not recalculate the mask unconditionally here, * so that the fchmod() mask above stays intact. */ if (acl_get_permset(entry, &permset) < 0 || - acl_add_perm(permset, ACL_READ) < 0 || - calc_acl_mask_if_needed(&acl) < 0) { + acl_add_perm(permset, ACL_READ) < 0) { log_warning_errno(errno, "Failed to patch ACL on %s, ignoring: %m", f->path); - goto finish; + return; + } + + r = calc_acl_mask_if_needed(&acl); + if (r < 0) { + log_warning_errno(r, "Failed to patch ACL on %s, ignoring: %m", f->path); + return; } if (acl_set_fd(f->fd, acl) < 0) log_warning_errno(errno, "Failed to set ACL on %s, ignoring: %m", f->path); -finish: - acl_free(acl); #endif } @@ -328,8 +343,8 @@ void server_rotate(Server *s) { log_debug("Rotating..."); - do_rotate(s, &s->runtime_journal, "runtime", false, 0); - do_rotate(s, &s->system_journal, "system", s->seal, 0); + (void) do_rotate(s, &s->runtime_journal, "runtime", false, 0); + (void) do_rotate(s, &s->system_journal, "system", s->seal, 0); ORDERED_HASHMAP_FOREACH_KEY(f, k, s->user_journals, i) { r = do_rotate(s, &f, "user", s->seal, PTR_TO_UINT32(k)); @@ -350,13 +365,13 @@ void server_sync(Server *s) { if (s->system_journal) { r = journal_file_set_offline(s->system_journal); if (r < 0) - log_error_errno(r, "Failed to sync system journal: %m"); + log_warning_errno(r, "Failed to sync system journal, ignoring: %m"); } ORDERED_HASHMAP_FOREACH_KEY(f, k, s->user_journals, i) { r = journal_file_set_offline(f); if (r < 0) - log_error_errno(r, "Failed to sync user journal: %m"); + log_warning_errno(r, "Failed to sync user journal, ignoring: %m"); } if (s->sync_event_source) { @@ -370,43 +385,50 @@ void server_sync(Server *s) { static void do_vacuum( Server *s, - const char *id, JournalFile *f, - const char* path, - JournalMetrics *metrics) { + JournalMetrics *metrics, + const char *path, + const char *name, + bool verbose, + bool patch_min_use) { const char *p; + uint64_t limit; int r; + assert(s); + assert(metrics); + assert(path); + assert(name); + if (!f) return; - p = strjoina(path, id); - r = journal_directory_vacuum(p, metrics->max_use, s->max_retention_usec, &s->oldest_file_usec, false); + p = strjoina(path, SERVER_MACHINE_ID(s)); + + limit = metrics->max_use; + (void) determine_space_for(s, metrics, path, name, verbose, patch_min_use, NULL, &limit); + + r = journal_directory_vacuum(p, limit, metrics->n_max_files, s->max_retention_usec, &s->oldest_file_usec, verbose); if (r < 0 && r != -ENOENT) - log_error_errno(r, "Failed to vacuum %s: %m", p); + log_warning_errno(r, "Failed to vacuum %s, ignoring: %m", p); } -void server_vacuum(Server *s) { - char ids[33]; - sd_id128_t machine; - int r; +int server_vacuum(Server *s, bool verbose, bool patch_min_use) { + assert(s); log_debug("Vacuuming..."); s->oldest_file_usec = 0; - r = sd_id128_get_machine(&machine); - if (r < 0) { - log_error_errno(r, "Failed to get machine ID: %m"); - return; - } - sd_id128_to_string(machine, ids); + do_vacuum(s, s->system_journal, &s->system_metrics, "/var/log/journal/", "System journal", verbose, patch_min_use); + do_vacuum(s, s->runtime_journal, &s->runtime_metrics, "/run/log/journal/", "Runtime journal", verbose, patch_min_use); - do_vacuum(s, ids, s->system_journal, "/var/log/journal/", &s->system_metrics); - do_vacuum(s, ids, s->runtime_journal, "/run/log/journal/", &s->runtime_metrics); + s->cached_space_limit = 0; + s->cached_space_available = 0; + s->cached_space_timestamp = 0; - s->cached_available_space_timestamp = 0; + return 0; } static void server_cache_machine_id(Server *s) { @@ -504,7 +526,7 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, unsigned if (journal_file_rotate_suggested(f, s->max_file_usec)) { log_debug("%s: Journal header limits reached or header out-of-date, rotating.", f->path); server_rotate(s); - server_vacuum(s); + server_vacuum(s, false, false); vacuumed = true; f = find_journal(s, uid); @@ -524,7 +546,7 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, unsigned } server_rotate(s); - server_vacuum(s); + server_vacuum(s, false, false); f = find_journal(s, uid); if (!f) @@ -825,7 +847,7 @@ static void dispatch_message_real( void server_driver_message(Server *s, sd_id128_t message_id, const char *format, ...) { char mid[11 + 32 + 1]; char buffer[16 + LINE_MAX + 1]; - struct iovec iovec[N_IOVEC_META_FIELDS + 4]; + struct iovec iovec[N_IOVEC_META_FIELDS + 6]; int n = 0; va_list ap; struct ucred ucred = {}; @@ -833,6 +855,9 @@ void server_driver_message(Server *s, sd_id128_t message_id, const char *format, assert(s); assert(format); + IOVEC_SET_STRING(iovec[n++], "SYSLOG_FACILITY=3"); + IOVEC_SET_STRING(iovec[n++], "SYSLOG_IDENTIFIER=systemd-journald"); + IOVEC_SET_STRING(iovec[n++], "PRIORITY=6"); IOVEC_SET_STRING(iovec[n++], "_TRANSPORT=driver"); @@ -866,6 +891,7 @@ void server_dispatch_message( int rl, r; _cleanup_free_ char *path = NULL; + uint64_t available = 0; char *c; assert(s); @@ -905,9 +931,8 @@ void server_dispatch_message( } } - rl = journal_rate_limit_test(s->rate_limit, path, - priority & LOG_PRIMASK, available_space(s, false)); - + (void) determine_space(s, false, false, &available, NULL); + rl = journal_rate_limit_test(s->rate_limit, path, priority & LOG_PRIMASK, available); if (rl == 0) return; @@ -922,16 +947,8 @@ finish: static int system_journal_open(Server *s, bool flush_requested) { - int r; - char *fn; - sd_id128_t machine; - char ids[33]; - - r = sd_id128_get_machine(&machine); - if (r < 0) - return log_error_errno(r, "Failed to get machine id: %m"); - - sd_id128_to_string(machine, ids); + const char *fn; + int r = 0; if (!s->system_journal && (s->storage == STORAGE_PERSISTENT || s->storage == STORAGE_AUTO) && @@ -947,15 +964,15 @@ static int system_journal_open(Server *s, bool flush_requested) { if (s->storage == STORAGE_PERSISTENT) (void) mkdir_p("/var/log/journal/", 0755); - fn = strjoina("/var/log/journal/", ids); + fn = strjoina("/var/log/journal/", SERVER_MACHINE_ID(s)); (void) mkdir(fn, 0755); fn = strjoina(fn, "/system.journal"); r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, s->compress, s->seal, &s->system_metrics, s->mmap, NULL, &s->system_journal); - - if (r >= 0) + if (r >= 0) { server_fix_perms(s, s->system_journal, 0); - else if (r < 0) { + (void) determine_space_for(s, &s->system_metrics, "/var/log/journal/", "System journal", true, true, NULL, NULL); + } else if (r < 0) { if (r != -ENOENT && r != -EROFS) log_warning_errno(r, "Failed to open system journal: %m"); @@ -966,9 +983,7 @@ static int system_journal_open(Server *s, bool flush_requested) { if (!s->runtime_journal && (s->storage != STORAGE_NONE)) { - fn = strjoin("/run/log/journal/", ids, "/system.journal", NULL); - if (!fn) - return -ENOMEM; + fn = strjoina("/run/log/journal/", SERVER_MACHINE_ID(s), "/system.journal"); if (s->system_journal) { @@ -977,8 +992,6 @@ static int system_journal_open(Server *s, bool flush_requested) { * it into the system journal */ r = journal_file_open(fn, O_RDWR, 0640, s->compress, false, &s->runtime_metrics, s->mmap, NULL, &s->runtime_journal); - free(fn); - if (r < 0) { if (r != -ENOENT) log_warning_errno(r, "Failed to open runtime journal: %m"); @@ -996,18 +1009,16 @@ static int system_journal_open(Server *s, bool flush_requested) { (void) mkdir_parents(fn, 0750); r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, s->compress, false, &s->runtime_metrics, s->mmap, NULL, &s->runtime_journal); - free(fn); - if (r < 0) return log_error_errno(r, "Failed to open runtime journal: %m"); } - if (s->runtime_journal) + if (s->runtime_journal) { server_fix_perms(s, s->runtime_journal, 0); + (void) determine_space_for(s, &s->runtime_metrics, "/run/log/journal/", "Runtime journal", true, true, NULL, NULL); + } } - available_space(s, true); - return r; } @@ -1028,7 +1039,7 @@ int server_flush_to_var(Server *s) { if (!s->runtime_journal) return 0; - system_journal_open(s, true); + (void) system_journal_open(s, true); if (!s->system_journal) return 0; @@ -1072,7 +1083,7 @@ int server_flush_to_var(Server *s) { } server_rotate(s); - server_vacuum(s); + server_vacuum(s, false, false); if (!s->system_journal) { log_notice("Didn't flush runtime journal since rotation of system journal wasn't successful."); @@ -1088,11 +1099,12 @@ int server_flush_to_var(Server *s) { } } + r = 0; + finish: journal_file_post_change(s->system_journal); - journal_file_close(s->runtime_journal); - s->runtime_journal = NULL; + s->runtime_journal = journal_file_close(s->runtime_journal); if (r >= 0) (void) rm_rf("/run/log/journal", REMOVE_ROOT); @@ -1235,9 +1247,9 @@ static int dispatch_sigusr1(sd_event_source *es, const struct signalfd_siginfo * server_flush_to_var(s); server_sync(s); - server_vacuum(s); + server_vacuum(s, false, false); - touch("/run/systemd/journal/flushed"); + (void) touch("/run/systemd/journal/flushed"); return 0; } @@ -1249,7 +1261,7 @@ static int dispatch_sigusr2(sd_event_source *es, const struct signalfd_siginfo * log_info("Received request to rotate journal from PID %"PRIu32, si->ssi_pid); server_rotate(s); - server_vacuum(s); + server_vacuum(s, true, true); return 0; } @@ -1293,8 +1305,7 @@ static int setup_signals(Server *s) { static int server_parse_proc_cmdline(Server *s) { _cleanup_free_ char *line = NULL; - const char *w, *state; - size_t l; + const char *p; int r; r = proc_cmdline(&line); @@ -1303,12 +1314,16 @@ static int server_parse_proc_cmdline(Server *s) { return 0; } - FOREACH_WORD_QUOTED(w, l, line, state) { + p = line; + for(;;) { _cleanup_free_ char *word; - word = strndup(w, l); - if (!word) - return -ENOMEM; + r = extract_first_word(&p, &word, NULL, 0); + if (r < 0) + return log_error_errno(r, "Failed to parse journald syntax \"%s\": %m", line); + + if (r == 0) + break; if (startswith(word, "systemd.journald.forward_to_syslog=")) { r = parse_boolean(word + 35); @@ -1337,8 +1352,8 @@ static int server_parse_proc_cmdline(Server *s) { } else if (startswith(word, "systemd.journald")) log_warning("Invalid systemd.journald parameter. Ignoring."); } - /* do not warn about state here, since probably systemd already did */ + /* do not warn about state here, since probably systemd already did */ return 0; } @@ -1434,8 +1449,7 @@ static int server_open_hostname(Server *s) { /* kernels prior to 3.2 don't support polling this file. Ignore * the failure. */ if (r == -EPERM) { - log_warning("Failed to register hostname fd in event loop: %s. Ignoring.", - strerror(-r)); + log_warning_errno(r, "Failed to register hostname fd in event loop, ignoring: %m"); s->hostname_fd = safe_close(s->hostname_fd); return 0; } @@ -1450,17 +1464,189 @@ static int server_open_hostname(Server *s) { return 0; } +static int dispatch_notify_event(sd_event_source *es, int fd, uint32_t revents, void *userdata) { + Server *s = userdata; + int r; + + assert(s); + assert(s->notify_event_source == es); + assert(s->notify_fd == fd); + + if (revents != EPOLLOUT) { + log_error("Invalid events on notify file descriptor."); + return -EINVAL; + } + + /* The $NOTIFY_SOCKET is writable again, now send exactly one + * message on it. Either it's the wtachdog event, the initial + * READY=1 event or an stdout stream event. If there's nothing + * to write anymore, turn our event source off. The next time + * there's something to send it will be turned on again. */ + + if (!s->sent_notify_ready) { + static const char p[] = + "READY=1\n" + "STATUS=Processing requests..."; + ssize_t l; + + l = send(s->notify_fd, p, strlen(p), MSG_DONTWAIT); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + return log_error_errno(errno, "Failed to send READY=1 notification message: %m"); + } + + s->sent_notify_ready = true; + log_debug("Sent READY=1 notification."); + + } else if (s->send_watchdog) { + + static const char p[] = + "WATCHDOG=1"; + + ssize_t l; + + l = send(s->notify_fd, p, strlen(p), MSG_DONTWAIT); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + return log_error_errno(errno, "Failed to send WATCHDOG=1 notification message: %m"); + } + + s->send_watchdog = false; + log_debug("Sent WATCHDOG=1 notification."); + + } else if (s->stdout_streams_notify_queue) + /* Dispatch one stream notification event */ + stdout_stream_send_notify(s->stdout_streams_notify_queue); + + /* Leave us enabled if there's still more to to do. */ + if (s->send_watchdog || s->stdout_streams_notify_queue) + return 0; + + /* There was nothing to do anymore, let's turn ourselves off. */ + r = sd_event_source_set_enabled(es, SD_EVENT_OFF); + if (r < 0) + return log_error_errno(r, "Failed to turn off notify event source: %m"); + + return 0; +} + +static int dispatch_watchdog(sd_event_source *es, uint64_t usec, void *userdata) { + Server *s = userdata; + int r; + + assert(s); + + s->send_watchdog = true; + + r = sd_event_source_set_enabled(s->notify_event_source, SD_EVENT_ON); + if (r < 0) + log_warning_errno(r, "Failed to turn on notify event source: %m"); + + r = sd_event_source_set_time(s->watchdog_event_source, usec + s->watchdog_usec / 2); + if (r < 0) + return log_error_errno(r, "Failed to restart watchdog event source: %m"); + + r = sd_event_source_set_enabled(s->watchdog_event_source, SD_EVENT_ON); + if (r < 0) + return log_error_errno(r, "Failed to enable watchdog event source: %m"); + + return 0; +} + +static int server_connect_notify(Server *s) { + union sockaddr_union sa = { + .un.sun_family = AF_UNIX, + }; + const char *e; + int r; + + assert(s); + assert(s->notify_fd < 0); + assert(!s->notify_event_source); + + /* + So here's the problem: we'd like to send notification + messages to PID 1, but we cannot do that via sd_notify(), + since that's synchronous, and we might end up blocking on + it. Specifically: given that PID 1 might block on + dbus-daemon during IPC, and dbus-daemon is logging to us, + and might hence block on us, we might end up in a deadlock + if we block on sending PID 1 notification messages -- by + generating a full blocking circle. To avoid this, let's + create a non-blocking socket, and connect it to the + notification socket, and then wait for POLLOUT before we + send anything. This should efficiently avoid any deadlocks, + as we'll never block on PID 1, hence PID 1 can safely block + on dbus-daemon which can safely block on us again. + + Don't think that this issue is real? It is, see: + https://github.com/systemd/systemd/issues/1505 + */ + + e = getenv("NOTIFY_SOCKET"); + if (!e) + return 0; + + if ((e[0] != '@' && e[0] != '/') || e[1] == 0) { + log_error("NOTIFY_SOCKET set to an invalid value: %s", e); + return -EINVAL; + } + + if (strlen(e) > sizeof(sa.un.sun_path)) { + log_error("NOTIFY_SOCKET path too long: %s", e); + return -EINVAL; + } + + s->notify_fd = socket(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0); + if (s->notify_fd < 0) + return log_error_errno(errno, "Failed to create notify socket: %m"); + + (void) fd_inc_sndbuf(s->notify_fd, NOTIFY_SNDBUF_SIZE); + + strncpy(sa.un.sun_path, e, sizeof(sa.un.sun_path)); + if (sa.un.sun_path[0] == '@') + sa.un.sun_path[0] = 0; + + r = connect(s->notify_fd, &sa.sa, offsetof(struct sockaddr_un, sun_path) + strlen(e)); + if (r < 0) + return log_error_errno(errno, "Failed to connect to notify socket: %m"); + + r = sd_event_add_io(s->event, &s->notify_event_source, s->notify_fd, EPOLLOUT, dispatch_notify_event, s); + if (r < 0) + return log_error_errno(r, "Failed to watch notification socket: %m"); + + if (sd_watchdog_enabled(false, &s->watchdog_usec) > 0) { + s->send_watchdog = true; + + r = sd_event_add_time(s->event, &s->watchdog_event_source, CLOCK_MONOTONIC, now(CLOCK_MONOTONIC) + s->watchdog_usec/2, s->watchdog_usec*3/4, dispatch_watchdog, s); + if (r < 0) + return log_error_errno(r, "Failed to add watchdog time event: %m"); + } + + /* This should fire pretty soon, which we'll use to send the + * READY=1 event. */ + + return 0; +} + int server_init(Server *s) { _cleanup_fdset_free_ FDSet *fds = NULL; int n, r, fd; + bool no_sockets; assert(s); zero(*s); - s->syslog_fd = s->native_fd = s->stdout_fd = s->dev_kmsg_fd = s->audit_fd = s->hostname_fd = -1; + s->syslog_fd = s->native_fd = s->stdout_fd = s->dev_kmsg_fd = s->audit_fd = s->hostname_fd = s->notify_fd = -1; s->compress = true; s->seal = true; + s->watchdog_usec = USEC_INFINITY; + s->sync_interval_usec = DEFAULT_SYNC_INTERVAL_USEC; s->sync_scheduled = false; @@ -1477,18 +1663,19 @@ int server_init(Server *s) { s->max_level_console = LOG_INFO; s->max_level_wall = LOG_EMERG; - memset(&s->system_metrics, 0xFF, sizeof(s->system_metrics)); - memset(&s->runtime_metrics, 0xFF, sizeof(s->runtime_metrics)); + journal_reset_metrics(&s->system_metrics); + journal_reset_metrics(&s->runtime_metrics); server_parse_config_file(s); server_parse_proc_cmdline(s); + if (!!s->rate_limit_interval ^ !!s->rate_limit_burst) { log_debug("Setting both rate limit interval and burst from "USEC_FMT",%u to 0,0", s->rate_limit_interval, s->rate_limit_burst); s->rate_limit_interval = s->rate_limit_burst = 0; } - mkdir_p("/run/systemd/journal", 0755); + (void) mkdir_p("/run/systemd/journal", 0755); s->user_journals = ordered_hashmap_new(NULL); if (!s->user_journals) @@ -1502,8 +1689,6 @@ int server_init(Server *s) { if (r < 0) return log_error_errno(r, "Failed to create event loop: %m"); - sd_event_set_watchdog(s->event, true); - n = sd_listen_fds(true); if (n < 0) return log_error_errno(n, "Failed to read listening file descriptors from environment: %m"); @@ -1561,30 +1746,44 @@ int server_init(Server *s) { } } - r = server_open_stdout_socket(s, fds); - if (r < 0) - return r; + /* Try to restore streams, but don't bother if this fails */ + (void) server_restore_streams(s, fds); if (fdset_size(fds) > 0) { log_warning("%u unknown file descriptors passed, closing.", fdset_size(fds)); fds = fdset_free(fds); } + no_sockets = s->native_fd < 0 && s->stdout_fd < 0 && s->syslog_fd < 0 && s->audit_fd < 0; + + /* always open stdout, syslog, native, and kmsg sockets */ + + /* systemd-journald.socket: /run/systemd/journal/stdout */ + r = server_open_stdout_socket(s); + if (r < 0) + return r; + + /* systemd-journald-dev-log.socket: /run/systemd/journal/dev-log */ r = server_open_syslog_socket(s); if (r < 0) return r; + /* systemd-journald.socket: /run/systemd/journal/socket */ r = server_open_native_socket(s); if (r < 0) return r; + /* /dev/ksmg */ r = server_open_dev_kmsg(s); if (r < 0) return r; - r = server_open_audit(s); - if (r < 0) - return r; + /* Unless we got *some* sockets and not audit, open audit socket */ + if (s->audit_fd >= 0 || no_sockets) { + r = server_open_audit(s); + if (r < 0) + return r; + } r = server_open_kernel_seqnum(s); if (r < 0) @@ -1614,11 +1813,9 @@ int server_init(Server *s) { server_cache_boot_id(s); server_cache_machine_id(s); - r = system_journal_open(s, false); - if (r < 0) - return r; + (void) server_connect_notify(s); - return 0; + return system_journal_open(s, false); } void server_maybe_append_tags(Server *s) { @@ -1666,6 +1863,8 @@ void server_done(Server *s) { sd_event_source_unref(s->sigterm_event_source); sd_event_source_unref(s->sigint_event_source); sd_event_source_unref(s->hostname_event_source); + sd_event_source_unref(s->notify_event_source); + sd_event_source_unref(s->watchdog_event_source); sd_event_unref(s->event); safe_close(s->syslog_fd); @@ -1674,6 +1873,7 @@ void server_done(Server *s) { safe_close(s->dev_kmsg_fd); safe_close(s->audit_fd); safe_close(s->hostname_fd); + safe_close(s->notify_fd); if (s->rate_limit) journal_rate_limit_free(s->rate_limit); @@ -1691,3 +1891,22 @@ void server_done(Server *s) { udev_unref(s->udev); } + +static const char* const storage_table[_STORAGE_MAX] = { + [STORAGE_AUTO] = "auto", + [STORAGE_VOLATILE] = "volatile", + [STORAGE_PERSISTENT] = "persistent", + [STORAGE_NONE] = "none" +}; + +DEFINE_STRING_TABLE_LOOKUP(storage, Storage); +DEFINE_CONFIG_PARSE_ENUM(config_parse_storage, storage, Storage, "Failed to parse storage setting"); + +static const char* const split_mode_table[_SPLIT_MAX] = { + [SPLIT_LOGIN] = "login", + [SPLIT_UID] = "uid", + [SPLIT_NONE] = "none", +}; + +DEFINE_STRING_TABLE_LOOKUP(split_mode, SplitMode); +DEFINE_CONFIG_PARSE_ENUM(config_parse_split_mode, split_mode, SplitMode, "Failed to parse split mode setting"); diff --git a/src/journal/journald-server.h b/src/journal/journald-server.h index d954c5190d..03a61bd2ed 100644 --- a/src/journal/journald-server.h +++ b/src/journal/journald-server.h @@ -25,10 +25,13 @@ #include <sys/types.h> #include "sd-event.h" -#include "journal-file.h" + +typedef struct Server Server; + #include "hashmap.h" -#include "audit.h" +#include "journal-file.h" #include "journald-rate-limit.h" +#include "journald-stream.h" #include "list.h" typedef enum Storage { @@ -48,15 +51,14 @@ typedef enum SplitMode { _SPLIT_INVALID = -1 } SplitMode; -typedef struct StdoutStream StdoutStream; - -typedef struct Server { +struct Server { int syslog_fd; int native_fd; int stdout_fd; int dev_kmsg_fd; int audit_fd; int hostname_fd; + int notify_fd; sd_event *event; @@ -71,6 +73,8 @@ typedef struct Server { sd_event_source *sigterm_event_source; sd_event_source *sigint_event_source; sd_event_source *hostname_event_source; + sd_event_source *notify_event_source; + sd_event_source *watchdog_event_source; JournalFile *runtime_journal; JournalFile *system_journal; @@ -100,8 +104,9 @@ typedef struct Server { unsigned n_forward_syslog_missed; usec_t last_warn_forward_syslog_missed; - uint64_t cached_available_space; - usec_t cached_available_space_timestamp; + uint64_t cached_space_available; + uint64_t cached_space_limit; + usec_t cached_space_timestamp; uint64_t var_available_timestamp; @@ -110,6 +115,7 @@ typedef struct Server { usec_t oldest_file_usec; LIST_HEAD(StdoutStream, stdout_streams); + LIST_HEAD(StdoutStream, stdout_streams_notify_queue); unsigned n_stdout_streams; char *tty_path; @@ -125,13 +131,14 @@ typedef struct Server { MMapCache *mmap; - bool dev_kmsg_readable; + struct udev *udev; uint64_t *kernel_seqnum; + bool dev_kmsg_readable:1; - struct udev *udev; - - bool sync_scheduled; + bool send_watchdog:1; + bool sent_notify_ready:1; + bool sync_scheduled:1; char machine_id_field[sizeof("_MACHINE_ID=") + 32]; char boot_id_field[sizeof("_BOOT_ID=") + 32]; @@ -139,7 +146,11 @@ typedef struct Server { /* Cached cgroup root, so that we don't have to query that all the time */ char *cgroup_root; -} Server; + + usec_t watchdog_usec; +}; + +#define SERVER_MACHINE_ID(s) ((s)->machine_id_field + strlen("_MACHINE_ID=")) #define N_IOVEC_META_FIELDS 20 #define N_IOVEC_KERNEL_FIELDS 64 @@ -166,7 +177,7 @@ void server_fix_perms(Server *s, JournalFile *f, uid_t uid); int server_init(Server *s); void server_done(Server *s); void server_sync(Server *s); -void server_vacuum(Server *s); +int server_vacuum(Server *s, bool verbose, bool patch_min_use); void server_rotate(Server *s); int server_schedule_sync(Server *s, int priority); int server_flush_to_var(Server *s); diff --git a/src/journal/journald-stream.c b/src/journal/journald-stream.c index 69e2d41863..fb800782fb 100644 --- a/src/journal/journald-stream.c +++ b/src/journal/journald-stream.c @@ -19,25 +19,35 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <unistd.h> #include <stddef.h> +#include <unistd.h> #ifdef HAVE_SELINUX #include <selinux/selinux.h> #endif -#include "sd-event.h" #include "sd-daemon.h" -#include "socket-util.h" -#include "selinux-util.h" -#include "mkdir.h" +#include "sd-event.h" + +#include "alloc-util.h" +#include "dirent-util.h" +#include "escape.h" +#include "fd-util.h" #include "fileio.h" +#include "io-util.h" +#include "journald-console.h" +#include "journald-kmsg.h" #include "journald-server.h" #include "journald-stream.h" #include "journald-syslog.h" -#include "journald-kmsg.h" -#include "journald-console.h" #include "journald-wall.h" +#include "mkdir.h" +#include "parse-util.h" +#include "selinux-util.h" +#include "socket-util.h" +#include "stdio-util.h" +#include "string-util.h" +#include "syslog-util.h" #define STDOUT_STREAMS_MAX 4096 @@ -69,6 +79,7 @@ struct StdoutStream { bool forward_to_console:1; bool fdstore:1; + bool in_notify_queue:1; char buffer[LINE_MAX+1]; size_t length; @@ -78,6 +89,7 @@ struct StdoutStream { char *state_file; LIST_FIELDS(StdoutStream, stdout_stream); + LIST_FIELDS(StdoutStream, stdout_stream_notify_queue); }; void stdout_stream_free(StdoutStream *s) { @@ -88,6 +100,9 @@ void stdout_stream_free(StdoutStream *s) { assert(s->server->n_stdout_streams > 0); s->server->n_stdout_streams --; LIST_REMOVE(stdout_stream, s->server->stdout_streams, s); + + if (s->in_notify_queue) + LIST_REMOVE(stdout_stream_notify_queue, s->server->stdout_streams_notify_queue, s); } if (s->event_source) { @@ -111,7 +126,7 @@ static void stdout_stream_destroy(StdoutStream *s) { return; if (s->state_file) - unlink(s->state_file); + (void) unlink(s->state_file); stdout_stream_free(s); } @@ -190,11 +205,15 @@ static int stdout_stream_save(StdoutStream *s) { goto fail; } - /* Store the connection fd in PID 1, so that we get it passed - * in again on next start */ - if (!s->fdstore) { - sd_pid_notify_with_fds(0, false, "FDSTORE=1", &s->fd, 1); - s->fdstore = true; + if (!s->fdstore && !s->in_notify_queue) { + LIST_PREPEND(stdout_stream_notify_queue, s->server->stdout_streams_notify_queue, s); + s->in_notify_queue = true; + + if (s->server->notify_event_source) { + r = sd_event_source_set_enabled(s->server->notify_event_source, SD_EVENT_ON); + if (r < 0) + log_warning_errno(r, "Failed to enable notify event source: %m"); + } } return 0; @@ -519,8 +538,7 @@ static int stdout_stream_new(sd_event_source *es, int listen_fd, uint32_t revent if (errno == EAGAIN) return 0; - log_error_errno(errno, "Failed to accept stdout connection: %m"); - return -errno; + return log_error_errno(errno, "Failed to accept stdout connection: %m"); } if (s->n_stdout_streams >= STDOUT_STREAMS_MAX) { @@ -627,7 +645,7 @@ static int stdout_stream_restore(Server *s, const char *fname, int fd) { return 0; } -static int server_restore_streams(Server *s, FDSet *fds) { +int server_restore_streams(Server *s, FDSet *fds) { _cleanup_closedir_ DIR *d = NULL; struct dirent *de; int r; @@ -681,7 +699,7 @@ fail: return log_error_errno(errno, "Failed to read streams directory: %m"); } -int server_open_stdout_socket(Server *s, FDSet *fds) { +int server_open_stdout_socket(Server *s) { int r; assert(s); @@ -717,8 +735,52 @@ int server_open_stdout_socket(Server *s, FDSet *fds) { if (r < 0) return log_error_errno(r, "Failed to adjust priority of stdout server event source: %m"); - /* Try to restore streams, but don't bother if this fails */ - (void) server_restore_streams(s, fds); - return 0; } + +void stdout_stream_send_notify(StdoutStream *s) { + struct iovec iovec = { + .iov_base = (char*) "FDSTORE=1", + .iov_len = strlen("FDSTORE=1"), + }; + struct msghdr msghdr = { + .msg_iov = &iovec, + .msg_iovlen = 1, + }; + struct cmsghdr *cmsg; + ssize_t l; + + assert(s); + assert(!s->fdstore); + assert(s->in_notify_queue); + assert(s->server); + assert(s->server->notify_fd >= 0); + + /* Store the connection fd in PID 1, so that we get it passed + * in again on next start */ + + msghdr.msg_controllen = CMSG_SPACE(sizeof(int)); + msghdr.msg_control = alloca0(msghdr.msg_controllen); + + cmsg = CMSG_FIRSTHDR(&msghdr); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(int)); + + memcpy(CMSG_DATA(cmsg), &s->fd, sizeof(int)); + + l = sendmsg(s->server->notify_fd, &msghdr, MSG_DONTWAIT|MSG_NOSIGNAL); + if (l < 0) { + if (errno == EAGAIN) + return; + + log_error_errno(errno, "Failed to send stream file descriptor to service manager: %m"); + } else { + log_debug("Successfully sent stream file descriptor to service manager."); + s->fdstore = 1; + } + + LIST_REMOVE(stdout_stream_notify_queue, s->server->stdout_streams_notify_queue, s); + s->in_notify_queue = false; + +} diff --git a/src/journal/journald-stream.h b/src/journal/journald-stream.h index 94bf955d78..e3497f0ded 100644 --- a/src/journal/journald-stream.h +++ b/src/journal/journald-stream.h @@ -21,9 +21,13 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ +typedef struct StdoutStream StdoutStream; + #include "fdset.h" #include "journald-server.h" -int server_open_stdout_socket(Server *s, FDSet *fds); +int server_open_stdout_socket(Server *s); +int server_restore_streams(Server *s, FDSet *fds); void stdout_stream_free(StdoutStream *s); +void stdout_stream_send_notify(StdoutStream *s); diff --git a/src/journal/journald-syslog.c b/src/journal/journald-syslog.c index ffba451955..f3ac1a7ae0 100644 --- a/src/journal/journald-syslog.c +++ b/src/journal/journald-syslog.c @@ -19,20 +19,27 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <unistd.h> #include <stddef.h> #include <sys/epoll.h> +#include <unistd.h> -#include "systemd/sd-messages.h" -#include "socket-util.h" -#include "selinux-util.h" +#include "sd-messages.h" + +#include "alloc-util.h" +#include "fd-util.h" +#include "formats-util.h" +#include "io-util.h" +#include "journald-console.h" +#include "journald-kmsg.h" #include "journald-server.h" #include "journald-syslog.h" -#include "journald-kmsg.h" -#include "journald-console.h" #include "journald-wall.h" -#include "formats-util.h" #include "process-util.h" +#include "selinux-util.h" +#include "socket-util.h" +#include "stdio-util.h" +#include "string-util.h" +#include "syslog-util.h" /* Warn once every 30s if we missed syslog message */ #define WARN_FORWARD_SYSLOG_MISSED_USEC (30 * USEC_PER_SEC) diff --git a/src/journal/journald-wall.c b/src/journal/journald-wall.c index 7863766ae7..69540f1141 100644 --- a/src/journal/journald-wall.c +++ b/src/journal/journald-wall.c @@ -19,11 +19,13 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include "utmp-wtmp.h" -#include "journald-server.h" -#include "journald-wall.h" +#include "alloc-util.h" #include "formats-util.h" +#include "journald-server.h" #include "process-util.h" +#include "string-util.h" +#include "utmp-wtmp.h" +#include "journald-wall.h" void server_forward_wall( Server *s, diff --git a/src/journal/journald.c b/src/journal/journald.c index b2624c6d28..b137e3c7be 100644 --- a/src/journal/journald.c +++ b/src/journal/journald.c @@ -21,8 +21,8 @@ #include <unistd.h> -#include "systemd/sd-messages.h" -#include "systemd/sd-daemon.h" +#include "sd-messages.h" +#include "sd-daemon.h" #include "journal-authenticate.h" #include "journald-server.h" @@ -54,17 +54,13 @@ int main(int argc, char *argv[]) { if (r < 0) goto finish; - server_vacuum(&server); + server_vacuum(&server, false, false); server_flush_to_var(&server); server_flush_dev_kmsg(&server); log_debug("systemd-journald running as pid "PID_FMT, getpid()); server_driver_message(&server, SD_MESSAGE_JOURNAL_START, "Journal started"); - sd_notify(false, - "READY=1\n" - "STATUS=Processing requests..."); - for (;;) { usec_t t = USEC_INFINITY, n; @@ -82,7 +78,7 @@ int main(int argc, char *argv[]) { if (server.oldest_file_usec + server.max_retention_usec < n) { log_info("Retention time reached."); server_rotate(&server); - server_vacuum(&server); + server_vacuum(&server, false, false); continue; } @@ -117,10 +113,6 @@ int main(int argc, char *argv[]) { server_driver_message(&server, SD_MESSAGE_JOURNAL_STOP, "Journal stopped"); finish: - sd_notify(false, - "STOPPING=1\n" - "STATUS=Shutting down..."); - server_done(&server); return r < 0 ? EXIT_FAILURE : EXIT_SUCCESS; diff --git a/src/journal/journald.conf b/src/journal/journald.conf index 47eefe91c1..7beb96c671 100644 --- a/src/journal/journald.conf +++ b/src/journal/journald.conf @@ -22,9 +22,11 @@ #SystemMaxUse= #SystemKeepFree= #SystemMaxFileSize= +#SystemMaxFiles=100 #RuntimeMaxUse= #RuntimeKeepFree= #RuntimeMaxFileSize= +#RuntimeMaxFiles=100 #MaxRetentionSec= #MaxFileSec=1month #ForwardToSyslog=no diff --git a/src/journal/mmap-cache.c b/src/journal/mmap-cache.c index 22f75540b8..3cb1dfa986 100644 --- a/src/journal/mmap-cache.c +++ b/src/journal/mmap-cache.c @@ -23,6 +23,7 @@ #include <stdlib.h> #include <sys/mman.h> +#include "alloc-util.h" #include "hashmap.h" #include "list.h" #include "log.h" diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c index 13fa9b52fc..5cde7f17f7 100644 --- a/src/journal/sd-journal.c +++ b/src/journal/sd-journal.c @@ -21,29 +21,38 @@ #include <errno.h> #include <fcntl.h> +#include <linux/magic.h> +#include <poll.h> #include <stddef.h> -#include <unistd.h> #include <sys/inotify.h> -#include <poll.h> #include <sys/vfs.h> -#include <linux/magic.h> +#include <unistd.h> #include "sd-journal.h" + +#include "alloc-util.h" +#include "catalog.h" +#include "compress.h" +#include "dirent-util.h" +#include "fd-util.h" +#include "fileio.h" +#include "formats-util.h" +#include "fs-util.h" +#include "hashmap.h" +#include "hostname-util.h" +#include "io-util.h" #include "journal-def.h" #include "journal-file.h" -#include "hashmap.h" +#include "journal-internal.h" #include "list.h" -#include "strv.h" -#include "path-util.h" #include "lookup3.h" -#include "compress.h" -#include "journal-internal.h" #include "missing.h" -#include "catalog.h" +#include "path-util.h" #include "replace-var.h" -#include "fileio.h" -#include "formats-util.h" -#include "hostname-util.h" +#include "stat-util.h" +#include "stdio-util.h" +#include "string-util.h" +#include "strv.h" #define JOURNAL_FILES_MAX 7168 @@ -64,19 +73,46 @@ static bool journal_pid_changed(sd_journal *j) { return j->original_pid != getpid(); } -/* We return an error here only if we didn't manage to - memorize the real error. */ -static int set_put_error(sd_journal *j, int r) { +static int journal_put_error(sd_journal *j, int r, const char *path) { + char *copy; int k; + /* Memorize an error we encountered, and store which + * file/directory it was generated from. Note that we store + * only *one* path per error code, as the error code is the + * key into the hashmap, and the path is the value. This means + * we keep track only of all error kinds, but not of all error + * locations. This has the benefit that the hashmap cannot + * grow beyond bounds. + * + * We return an error here only if we didn't manage to + * memorize the real error. */ + if (r >= 0) return r; - k = set_ensure_allocated(&j->errors, NULL); + k = hashmap_ensure_allocated(&j->errors, NULL); if (k < 0) return k; - return set_put(j->errors, INT_TO_PTR(r)); + if (path) { + copy = strdup(path); + if (!copy) + return -ENOMEM; + } else + copy = NULL; + + k = hashmap_put(j->errors, INT_TO_PTR(r), copy); + if (k < 0) { + free(copy); + + if (k == -EEXIST) + return 0; + + return k; + } + + return 0; } static void detach_location(sd_journal *j) { @@ -1016,8 +1052,6 @@ _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) { _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) { int r; - const char *word, *state; - size_t l; Object *o; assert_return(j, -EINVAL); @@ -1031,20 +1065,23 @@ _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) { if (r < 0) return r; - FOREACH_WORD_SEPARATOR(word, l, cursor, ";", state) { + for(;;) { _cleanup_free_ char *item = NULL; - sd_id128_t id; unsigned long long ll; + sd_id128_t id; int k = 0; - if (l < 2 || word[1] != '=') - return -EINVAL; + r = extract_first_word(&cursor, &item, ";", EXTRACT_DONT_COALESCE_SEPARATORS); + if (r < 0) + return r; - item = strndup(word, l); - if (!item) - return -ENOMEM; + if (r == 0) + break; - switch (word[0]) { + if (strlen(item) < 2 || item[1] != '=') + return -EINVAL; + + switch (item[0]) { case 's': k = sd_id128_from_string(item+2, &id); @@ -1173,6 +1210,8 @@ static bool file_has_type_prefix(const char *prefix, const char *filename) { } static bool file_type_wanted(int flags, const char *filename) { + assert(filename); + if (!endswith(filename, ".journal") && !endswith(filename, ".journal~")) return false; @@ -1197,7 +1236,7 @@ static bool file_type_wanted(int flags, const char *filename) { static int add_any_file(sd_journal *j, const char *path) { JournalFile *f = NULL; - int r; + int r, k; assert(j); assert(path); @@ -1206,20 +1245,23 @@ static int add_any_file(sd_journal *j, const char *path) { return 0; if (ordered_hashmap_size(j->files) >= JOURNAL_FILES_MAX) { - log_warning("Too many open journal files, not adding %s.", path); - return set_put_error(j, -ETOOMANYREFS); + log_debug("Too many open journal files, not adding %s.", path); + r = -ETOOMANYREFS; + goto fail; } r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f); - if (r < 0) - return r; + if (r < 0) { + log_debug_errno(r, "Failed to open journal file %s: %m", path); + goto fail; + } /* journal_file_dump(f); */ r = ordered_hashmap_put(j->files, f->path, f); if (r < 0) { journal_file_close(f); - return r; + goto fail; } log_debug("File %s added.", f->path); @@ -1229,11 +1271,17 @@ static int add_any_file(sd_journal *j, const char *path) { j->current_invalidate_counter ++; return 0; + +fail: + k = journal_put_error(j, r, path); + if (k < 0) + return k; + + return r; } static int add_file(sd_journal *j, const char *prefix, const char *filename) { - _cleanup_free_ char *path = NULL; - int r; + const char *path; assert(j); assert(prefix); @@ -1243,34 +1291,24 @@ static int add_file(sd_journal *j, const char *prefix, const char *filename) { !file_type_wanted(j->flags, filename)) return 0; - path = strjoin(prefix, "/", filename, NULL); - if (!path) - return -ENOMEM; - - r = add_any_file(j, path); - if (r == -ENOENT) - return 0; - return r; + path = strjoina(prefix, "/", filename); + return add_any_file(j, path); } -static int remove_file(sd_journal *j, const char *prefix, const char *filename) { - _cleanup_free_ char *path; +static void remove_file(sd_journal *j, const char *prefix, const char *filename) { + const char *path; JournalFile *f; assert(j); assert(prefix); assert(filename); - path = strjoin(prefix, "/", filename, NULL); - if (!path) - return -ENOMEM; - + path = strjoina(prefix, "/", filename); f = ordered_hashmap_get(j->files, path); if (!f) - return 0; + return; remove_file_real(j, f); - return 0; } static void remove_file_real(sd_journal *j, JournalFile *f) { @@ -1299,12 +1337,27 @@ static void remove_file_real(sd_journal *j, JournalFile *f) { j->current_invalidate_counter ++; } +static int dirname_is_machine_id(const char *fn) { + sd_id128_t id, machine; + int r; + + r = sd_id128_get_machine(&machine); + if (r < 0) + return r; + + r = sd_id128_from_string(fn, &id); + if (r < 0) + return r; + + return sd_id128_equal(id, machine); +} + static int add_directory(sd_journal *j, const char *prefix, const char *dirname) { _cleanup_free_ char *path = NULL; - int r; _cleanup_closedir_ DIR *d = NULL; - sd_id128_t id, mid; + struct dirent *de = NULL; Directory *m; + int r, k; assert(j); assert(prefix); @@ -1313,35 +1366,36 @@ static int add_directory(sd_journal *j, const char *prefix, const char *dirname) log_debug("Considering %s/%s.", prefix, dirname); if ((j->flags & SD_JOURNAL_LOCAL_ONLY) && - (sd_id128_from_string(dirname, &id) < 0 || - sd_id128_get_machine(&mid) < 0 || - !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run")))) + !(dirname_is_machine_id(dirname) > 0 || path_startswith(prefix, "/run"))) return 0; path = strjoin(prefix, "/", dirname, NULL); - if (!path) - return -ENOMEM; + if (!path) { + r = -ENOMEM; + goto fail; + } d = opendir(path); if (!d) { - log_debug_errno(errno, "Failed to open %s: %m", path); - if (errno == ENOENT) - return 0; - return -errno; + r = log_debug_errno(errno, "Failed to open directory %s: %m", path); + goto fail; } m = hashmap_get(j->directories_by_path, path); if (!m) { m = new0(Directory, 1); - if (!m) - return -ENOMEM; + if (!m) { + r = -ENOMEM; + goto fail; + } m->is_root = false; m->path = path; if (hashmap_put(j->directories_by_path, m->path, m) < 0) { free(m); - return -ENOMEM; + r = -ENOMEM; + goto fail; } path = NULL; /* avoid freeing in cleanup */ @@ -1363,41 +1417,30 @@ static int add_directory(sd_journal *j, const char *prefix, const char *dirname) inotify_rm_watch(j->inotify_fd, m->wd); } - for (;;) { - struct dirent *de; - - errno = 0; - de = readdir(d); - if (!de && errno != 0) { - r = -errno; - log_debug_errno(errno, "Failed to read directory %s: %m", m->path); - return r; - } - if (!de) - break; + FOREACH_DIRENT_ALL(de, d, return log_debug_errno(errno, "Failed to read directory %s: %m", m->path)) { if (dirent_is_file_with_suffix(de, ".journal") || - dirent_is_file_with_suffix(de, ".journal~")) { - r = add_file(j, m->path, de->d_name); - if (r < 0) { - log_debug_errno(r, "Failed to add file %s/%s: %m", - m->path, de->d_name); - r = set_put_error(j, r); - if (r < 0) - return r; - } - } + dirent_is_file_with_suffix(de, ".journal~")) + (void) add_file(j, m->path, de->d_name); } check_network(j, dirfd(d)); return 0; + +fail: + k = journal_put_error(j, r, path ?: dirname); + if (k < 0) + return k; + + return r; } -static int add_root_directory(sd_journal *j, const char *p) { +static int add_root_directory(sd_journal *j, const char *p, bool missing_ok) { _cleanup_closedir_ DIR *d = NULL; + struct dirent *de; Directory *m; - int r; + int r, k; assert(j); assert(p); @@ -1410,26 +1453,35 @@ static int add_root_directory(sd_journal *j, const char *p) { p = strjoina(j->prefix, p); d = opendir(p); - if (!d) - return -errno; + if (!d) { + if (errno == ENOENT && missing_ok) + return 0; + + r = log_debug_errno(errno, "Failed to open root directory %s: %m", p); + goto fail; + } m = hashmap_get(j->directories_by_path, p); if (!m) { m = new0(Directory, 1); - if (!m) - return -ENOMEM; + if (!m) { + r = -ENOMEM; + goto fail; + } m->is_root = true; m->path = strdup(p); if (!m->path) { free(m); - return -ENOMEM; + r = -ENOMEM; + goto fail; } if (hashmap_put(j->directories_by_path, m->path, m) < 0) { free(m->path); free(m); - return -ENOMEM; + r = -ENOMEM; + goto fail; } j->current_invalidate_counter ++; @@ -1452,42 +1504,27 @@ static int add_root_directory(sd_journal *j, const char *p) { if (j->no_new_files) return 0; - for (;;) { - struct dirent *de; + FOREACH_DIRENT_ALL(de, d, return log_debug_errno(errno, "Failed to read directory %s: %m", m->path)) { sd_id128_t id; - errno = 0; - de = readdir(d); - if (!de && errno != 0) { - r = -errno; - log_debug_errno(errno, "Failed to read directory %s: %m", m->path); - return r; - } - if (!de) - break; - if (dirent_is_file_with_suffix(de, ".journal") || - dirent_is_file_with_suffix(de, ".journal~")) { - r = add_file(j, m->path, de->d_name); - if (r < 0) { - log_debug_errno(r, "Failed to add file %s/%s: %m", - m->path, de->d_name); - r = set_put_error(j, r); - if (r < 0) - return r; - } - } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) && - sd_id128_from_string(de->d_name, &id) >= 0) { - - r = add_directory(j, m->path, de->d_name); - if (r < 0) - log_debug_errno(r, "Failed to add directory %s/%s: %m", m->path, de->d_name); - } + dirent_is_file_with_suffix(de, ".journal~")) + (void) add_file(j, m->path, de->d_name); + else if (IN_SET(de->d_type, DT_DIR, DT_LNK, DT_UNKNOWN) && + sd_id128_from_string(de->d_name, &id) >= 0) + (void) add_directory(j, m->path, de->d_name); } check_network(j, dirfd(d)); return 0; + +fail: + k = journal_put_error(j, r, p); + if (k < 0) + return k; + + return r; } static void remove_directory(sd_journal *j, Directory *d) { @@ -1512,8 +1549,8 @@ static void remove_directory(sd_journal *j, Directory *d) { } static int add_search_paths(sd_journal *j) { - int r; - const char search_paths[] = + + static const char search_paths[] = "/run/log/journal\0" "/var/log/journal\0"; const char *p; @@ -1523,14 +1560,8 @@ static int add_search_paths(sd_journal *j) { /* We ignore most errors here, since the idea is to only open * what's actually accessible, and ignore the rest. */ - NULSTR_FOREACH(p, search_paths) { - r = add_root_directory(j, p); - if (r < 0 && r != -ENOENT) { - r = set_put_error(j, r); - if (r < 0) - return r; - } - } + NULSTR_FOREACH(p, search_paths) + (void) add_root_directory(j, p, true); return 0; } @@ -1554,17 +1585,14 @@ static int add_current_paths(sd_journal *j) { if (!dir) return -ENOMEM; - r = add_root_directory(j, dir); - if (r < 0) { - set_put_error(j, r); + r = add_root_directory(j, dir, true); + if (r < 0) return r; - } } return 0; } - static int allocate_inotify(sd_journal *j) { assert(j); @@ -1692,11 +1720,9 @@ _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int f if (!j) return -ENOMEM; - r = add_root_directory(j, path); - if (r < 0) { - set_put_error(j, r); + r = add_root_directory(j, path, false); + if (r < 0) goto fail; - } *ret = j; return 0; @@ -1721,10 +1747,8 @@ _public_ int sd_journal_open_files(sd_journal **ret, const char **paths, int fla STRV_FOREACH(path, paths) { r = add_any_file(j, *path); - if (r < 0) { - log_error_errno(r, "Failed to open %s: %m", *path); + if (r < 0) goto fail; - } } j->no_new_files = true; @@ -1741,6 +1765,7 @@ fail: _public_ void sd_journal_close(sd_journal *j) { Directory *d; JournalFile *f; + char *p; if (!j) return; @@ -1768,10 +1793,13 @@ _public_ void sd_journal_close(sd_journal *j) { mmap_cache_unref(j->mmap); } + while ((p = hashmap_steal_first(j->errors))) + free(p); + hashmap_free(j->errors); + free(j->path); free(j->prefix); free(j->unique_field); - set_free(j->errors); free(j); } @@ -2064,7 +2092,7 @@ _public_ int sd_journal_get_fd(sd_journal *j) { if (j->no_new_files) r = add_current_paths(j); else if (j->path) - r = add_root_directory(j, j->path); + r = add_root_directory(j, j->path, true); else r = add_search_paths(j); if (r < 0) @@ -2111,7 +2139,6 @@ _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) { static void process_inotify_event(sd_journal *j, struct inotify_event *e) { Directory *d; - int r; assert(j); assert(e); @@ -2127,20 +2154,10 @@ static void process_inotify_event(sd_journal *j, struct inotify_event *e) { /* Event for a journal file */ - if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) { - r = add_file(j, d->path, e->name); - if (r < 0) { - log_debug_errno(r, "Failed to add file %s/%s: %m", - d->path, e->name); - set_put_error(j, r); - } - - } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) { - - r = remove_file(j, d->path, e->name); - if (r < 0) - log_debug_errno(r, "Failed to remove file %s/%s: %m", d->path, e->name); - } + if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) + (void) add_file(j, d->path, e->name); + else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) + remove_file(j, d->path, e->name); } else if (!d->is_root && e->len == 0) { @@ -2153,11 +2170,8 @@ static void process_inotify_event(sd_journal *j, struct inotify_event *e) { /* Event for root directory */ - if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) { - r = add_directory(j, d->path, e->name); - if (r < 0) - log_debug_errno(r, "Failed to add directory %s/%s: %m", d->path, e->name); - } + if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) + (void) add_directory(j, d->path, e->name); } return; @@ -2166,7 +2180,7 @@ static void process_inotify_event(sd_journal *j, struct inotify_event *e) { if (e->mask & IN_IGNORED) return; - log_warning("Unknown inotify event."); + log_debug("Unknown inotify event."); } static int determine_change(sd_journal *j) { diff --git a/src/journal/stacktrace.c b/src/journal/stacktrace.c index 98a54ff269..4305462f80 100644 --- a/src/journal/stacktrace.c +++ b/src/journal/stacktrace.c @@ -22,10 +22,13 @@ #include <dwarf.h> #include <elfutils/libdwfl.h> -#include "util.h" +#include "alloc-util.h" +#include "fd-util.h" +#include "formats-util.h" #include "macro.h" #include "stacktrace.h" -#include "formats-util.h" +#include "string-util.h" +#include "util.h" #define FRAMES_MAX 64 #define THREADS_MAX 64 diff --git a/src/journal/test-catalog.c b/src/journal/test-catalog.c index dbfdea609d..aea8fd15e6 100644 --- a/src/journal/test-catalog.c +++ b/src/journal/test-catalog.c @@ -25,11 +25,16 @@ #include <errno.h> #include <fcntl.h> -#include "util.h" -#include "log.h" -#include "macro.h" #include "sd-messages.h" + +#include "alloc-util.h" #include "catalog.h" +#include "fd-util.h" +#include "fileio.h" +#include "log.h" +#include "macro.h" +#include "string-util.h" +#include "util.h" static const char *catalog_dirs[] = { CATALOG_DIR, diff --git a/src/journal/test-compress-benchmark.c b/src/journal/test-compress-benchmark.c index c8e5b76c6c..93ea9c6318 100644 --- a/src/journal/test-compress-benchmark.c +++ b/src/journal/test-compress-benchmark.c @@ -1,3 +1,5 @@ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ + /*** This file is part of systemd @@ -17,30 +19,73 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ +#include "alloc-util.h" #include "compress.h" -#include "util.h" #include "macro.h" +#include "parse-util.h" +#include "random-util.h" +#include "string-util.h" +#include "util.h" typedef int (compress_t)(const void *src, uint64_t src_size, void *dst, size_t *dst_size); typedef int (decompress_t)(const void *src, uint64_t src_size, void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max); +static usec_t arg_duration = 2 * USEC_PER_SEC; +static size_t arg_start; + #define MAX_SIZE (1024*1024LU) +#define PRIME 1048571 /* A prime close enough to one megabyte that mod 4 == 3 */ + +static size_t _permute(size_t x) { + size_t residue; + + if (x >= PRIME) + return x; + + residue = x*x % PRIME; + if (x <= PRIME / 2) + return residue; + else + return PRIME - residue; +} + +static size_t permute(size_t x) { + return _permute((_permute(x) + arg_start) % MAX_SIZE ^ 0xFF345); +} -static char* make_buf(size_t count) { +static char* make_buf(size_t count, const char *type) { char *buf; size_t i; buf = malloc(count); assert_se(buf); - for (i = 0; i < count; i++) - buf[i] = 'a' + i % ('z' - 'a' + 1); + if (streq(type, "zeros")) + memzero(buf, count); + else if (streq(type, "simple")) + for (i = 0; i < count; i++) + buf[i] = 'a' + i % ('z' - 'a' + 1); + else if (streq(type, "random")) { + size_t step = count / 10; + + random_bytes(buf, step); + memzero(buf + 1*step, step); + random_bytes(buf + 2*step, step); + memzero(buf + 3*step, step); + random_bytes(buf + 4*step, step); + memzero(buf + 5*step, step); + random_bytes(buf + 6*step, step); + memzero(buf + 7*step, step); + random_bytes(buf + 8*step, step); + memzero(buf + 9*step, step); + } else + assert_not_reached("here"); return buf; } -static void test_compress_decompress(const char* label, +static void test_compress_decompress(const char* label, const char* type, compress_t compress, decompress_t decompress) { usec_t n, n2 = 0; float dt; @@ -50,64 +95,85 @@ static void test_compress_decompress(const char* label, size_t buf2_allocated = 0; size_t skipped = 0, compressed = 0, total = 0; - text = make_buf(MAX_SIZE); + text = make_buf(MAX_SIZE, type); buf = calloc(MAX_SIZE + 1, 1); assert_se(text && buf); n = now(CLOCK_MONOTONIC); - for (size_t i = 1; i <= MAX_SIZE; i += (i < 2048 ? 1 : 217)) { - size_t j = 0, k = 0; + for (size_t i = 0; i <= MAX_SIZE; i++) { + size_t j = 0, k = 0, size; int r; - r = compress(text, i, buf, &j); + size = permute(i); + + log_debug("%s %zu %zu", type, i, size); + + memzero(buf, MIN(size + 1000, MAX_SIZE)); + + r = compress(text, size, buf, &j); /* assume compression must be successful except for small inputs */ - assert_se(r == 0 || (i < 2048 && r == -ENOBUFS)); + assert_se(r == 0 || (size < 2048 && r == -ENOBUFS) || streq(type, "random")); + /* check for overwrites */ - assert_se(buf[i] == 0); + assert_se(buf[size] == 0); if (r != 0) { - skipped += i; + skipped += size; continue; } assert_se(j > 0); - if (j >= i) - log_error("%s \"compressed\" %zu -> %zu", label, i, j); + if (j >= size) + log_error("%s \"compressed\" %zu -> %zu", label, size, j); r = decompress(buf, j, &buf2, &buf2_allocated, &k, 0); assert_se(r == 0); assert_se(buf2_allocated >= k); - assert_se(k == i); + assert_se(k == size); - assert_se(memcmp(text, buf2, i) == 0); + assert_se(memcmp(text, buf2, size) == 0); - total += i; + total += size; compressed += j; n2 = now(CLOCK_MONOTONIC); - if (n2 - n > 60 * USEC_PER_SEC) + if (n2 - n > arg_duration) break; } dt = (n2-n) / 1e6; - log_info("%s: compressed & decompressed %zu bytes in %.2fs (%.2fMiB/s), " + log_info("%s/%s: compressed & decompressed %zu bytes in %.2fs (%.2fMiB/s), " "mean compresion %.2f%%, skipped %zu bytes", - label, total, dt, + label, type, total, dt, total / 1024. / 1024 / dt, 100 - compressed * 100. / total, skipped); } int main(int argc, char *argv[]) { + const char *i; - log_set_max_level(LOG_DEBUG); + log_set_max_level(LOG_INFO); + if (argc >= 2) { + unsigned x; + + assert_se(safe_atou(argv[1], &x) >= 0); + arg_duration = x * USEC_PER_SEC; + } + if (argc == 3) + (void) safe_atolu(argv[2], &arg_start); + else + arg_start = getpid(); + + NULSTR_FOREACH(i, "zeros\0simple\0random\0") { #ifdef HAVE_XZ - test_compress_decompress("XZ", compress_blob_xz, decompress_blob_xz); + test_compress_decompress("XZ", i, compress_blob_xz, decompress_blob_xz); #endif #ifdef HAVE_LZ4 - test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4); + test_compress_decompress("LZ4", i, compress_blob_lz4, decompress_blob_lz4); #endif + } return 0; } diff --git a/src/journal/test-compress.c b/src/journal/test-compress.c index f17c00e60d..b9d90a8988 100644 --- a/src/journal/test-compress.c +++ b/src/journal/test-compress.c @@ -17,10 +17,13 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ +#include "alloc-util.h" #include "compress.h" -#include "util.h" +#include "fd-util.h" +#include "fileio.h" #include "macro.h" #include "random-util.h" +#include "util.h" #ifdef HAVE_XZ # define XZ_OK 0 @@ -144,8 +147,8 @@ static void test_compress_stream(int compression, const char *srcfile) { _cleanup_close_ int src = -1, dst = -1, dst2 = -1; - char pattern[] = "/tmp/systemd-test.xz.XXXXXX", - pattern2[] = "/tmp/systemd-test.xz.XXXXXX"; + char pattern[] = "/tmp/systemd-test.compressed.XXXXXX", + pattern2[] = "/tmp/systemd-test.compressed.XXXXXX"; int r; _cleanup_free_ char *cmd = NULL, *cmd2; struct stat st = {}; @@ -185,7 +188,7 @@ static void test_compress_stream(int compression, assert_se(lseek(dst, 1, SEEK_SET) == 1); r = decompress(dst, dst2, st.st_size); - assert_se(r == -EBADMSG); + assert_se(r == -EBADMSG || r == 0); assert_se(lseek(dst, 0, SEEK_SET) == 0); assert_se(lseek(dst2, 0, SEEK_SET) == 0); @@ -236,8 +239,7 @@ int main(int argc, char *argv[]) { compress_blob_lz4, decompress_startswith_lz4, data, sizeof(data), true); - /* Produced stream is not compatible with lz4 binary, skip lz4cat check. */ - test_compress_stream(OBJECT_COMPRESSED_LZ4, NULL, + test_compress_stream(OBJECT_COMPRESSED_LZ4, "lz4cat", compress_stream_lz4, decompress_stream_lz4, argv[0]); #else log_info("/* LZ4 test skipped */"); diff --git a/src/journal/test-journal-enum.c b/src/journal/test-journal-enum.c index cde2025ae9..040c7d58fb 100644 --- a/src/journal/test-journal-enum.c +++ b/src/journal/test-journal-enum.c @@ -21,8 +21,9 @@ #include <stdio.h> -#include "log.h" #include "sd-journal.h" + +#include "log.h" #include "macro.h" #include "journal-internal.h" diff --git a/src/journal/test-journal-flush.c b/src/journal/test-journal-flush.c index 2d4f531e9b..03d1522e23 100644 --- a/src/journal/test-journal-flush.c +++ b/src/journal/test-journal-flush.c @@ -22,9 +22,12 @@ #include <fcntl.h> #include "sd-journal.h" -#include "macro.h" + +#include "alloc-util.h" #include "journal-file.h" #include "journal-internal.h" +#include "macro.h" +#include "string-util.h" int main(int argc, char *argv[]) { _cleanup_free_ char *fn = NULL; diff --git a/src/journal/test-journal-init.c b/src/journal/test-journal-init.c index e6599f366d..142da85041 100644 --- a/src/journal/test-journal-init.c +++ b/src/journal/test-journal-init.c @@ -19,11 +19,12 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include "systemd/sd-journal.h" +#include "sd-journal.h" #include "log.h" -#include "util.h" +#include "parse-util.h" #include "rm-rf.h" +#include "util.h" int main(int argc, char *argv[]) { sd_journal *j; diff --git a/src/journal/test-journal-interleaving.c b/src/journal/test-journal-interleaving.c index adefa1b026..4ad89fe4b6 100644 --- a/src/journal/test-journal-interleaving.c +++ b/src/journal/test-journal-interleaving.c @@ -24,11 +24,14 @@ #include <fcntl.h> #include "sd-journal.h" + +#include "alloc-util.h" #include "journal-file.h" #include "journal-vacuum.h" -#include "util.h" #include "log.h" +#include "parse-util.h" #include "rm-rf.h" +#include "util.h" /* This program tests skipping around in a multi-file journal. */ @@ -197,7 +200,7 @@ static void test_skip(void (*setup)(void)) { if (arg_keep) log_info("Not removing %s", t); else { - journal_directory_vacuum(".", 3000000, 0, NULL, true); + journal_directory_vacuum(".", 3000000, 0, 0, NULL, true); assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0); } @@ -282,7 +285,7 @@ static void test_sequence_numbers(void) { if (arg_keep) log_info("Not removing %s", t); else { - journal_directory_vacuum(".", 3000000, 0, NULL, true); + journal_directory_vacuum(".", 3000000, 0, 0, NULL, true); assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0); } diff --git a/src/journal/test-journal-match.c b/src/journal/test-journal-match.c index a3187053c9..abefedb992 100644 --- a/src/journal/test-journal-match.c +++ b/src/journal/test-journal-match.c @@ -21,11 +21,13 @@ #include <stdio.h> -#include "systemd/sd-journal.h" +#include "sd-journal.h" +#include "alloc-util.h" #include "journal-internal.h" -#include "util.h" #include "log.h" +#include "string-util.h" +#include "util.h" int main(int argc, char *argv[]) { _cleanup_journal_close_ sd_journal*j; diff --git a/src/journal/test-journal-send.c b/src/journal/test-journal-send.c index 81ca47ed8d..694376670d 100644 --- a/src/journal/test-journal-send.c +++ b/src/journal/test-journal-send.c @@ -19,10 +19,11 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include "systemd/sd-journal.h" #include <stdlib.h> #include <unistd.h> +#include "sd-journal.h" + #include "log.h" int main(int argc, char *argv[]) { diff --git a/src/journal/test-journal-stream.c b/src/journal/test-journal-stream.c index b5ecf2f375..0cbef4b8c5 100644 --- a/src/journal/test-journal-stream.c +++ b/src/journal/test-journal-stream.c @@ -19,16 +19,19 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ -#include <unistd.h> #include <fcntl.h> +#include <unistd.h> #include "sd-journal.h" -#include "util.h" + +#include "alloc-util.h" +#include "journal-file.h" +#include "journal-internal.h" #include "log.h" #include "macro.h" +#include "parse-util.h" #include "rm-rf.h" -#include "journal-file.h" -#include "journal-internal.h" +#include "util.h" #define N_ENTRIES 200 diff --git a/src/journal/test-journal-syslog.c b/src/journal/test-journal-syslog.c index c99ca0654b..1784187fe9 100644 --- a/src/journal/test-journal-syslog.c +++ b/src/journal/test-journal-syslog.c @@ -19,8 +19,10 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ +#include "alloc-util.h" #include "journald-syslog.h" #include "macro.h" +#include "string-util.h" static void test_syslog_parse_identifier(const char* str, const char *ident, const char*pid, int ret) { diff --git a/src/journal/test-journal-verify.c b/src/journal/test-journal-verify.c index d89123dc64..887a83efe1 100644 --- a/src/journal/test-journal-verify.c +++ b/src/journal/test-journal-verify.c @@ -23,12 +23,13 @@ #include <unistd.h> #include <fcntl.h> -#include "util.h" -#include "log.h" -#include "rm-rf.h" +#include "fd-util.h" #include "journal-file.h" #include "journal-verify.h" +#include "log.h" +#include "rm-rf.h" #include "terminal-util.h" +#include "util.h" #define N_ENTRIES 6000 #define RANDOM_RANGE 77 diff --git a/src/journal/test-journal.c b/src/journal/test-journal.c index caaab258c9..01d4bc968a 100644 --- a/src/journal/test-journal.c +++ b/src/journal/test-journal.c @@ -116,7 +116,7 @@ static void test_non_empty(void) { if (arg_keep) log_info("Not removing %s", t); else { - journal_directory_vacuum(".", 3000000, 0, NULL, true); + journal_directory_vacuum(".", 3000000, 0, 0, NULL, true); assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0); } @@ -155,7 +155,7 @@ static void test_empty(void) { if (arg_keep) log_info("Not removing %s", t); else { - journal_directory_vacuum(".", 3000000, 0, NULL, true); + journal_directory_vacuum(".", 3000000, 0, 0, NULL, true); assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0); } diff --git a/src/journal/test-mmap-cache.c b/src/journal/test-mmap-cache.c index 3258b22702..fdd48e531c 100644 --- a/src/journal/test-mmap-cache.c +++ b/src/journal/test-mmap-cache.c @@ -19,14 +19,16 @@ along with systemd; If not, see <http://www.gnu.org/licenses/>. ***/ +#include <fcntl.h> #include <stdlib.h> #include <sys/mman.h> #include <unistd.h> -#include <fcntl.h> +#include "fd-util.h" +#include "fileio.h" #include "macro.h" -#include "util.h" #include "mmap-cache.h" +#include "util.h" int main(int argc, char *argv[]) { int x, y, z, r; |