From 2e14c544a9c13c516f6918618138b38e0f6c7a17 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 2 Oct 2015 22:31:32 +0200 Subject: journal: port over to fd_getcrtime_at() Let's use fd_getcrtime_at(), since that *at() family of calls is how we read the rest of the file metadata, too. --- src/journal/journal-vacuum.c | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) (limited to 'src/journal') diff --git a/src/journal/journal-vacuum.c b/src/journal/journal-vacuum.c index 17499bbc30..8c642d16e4 100644 --- a/src/journal/journal-vacuum.c +++ b/src/journal/journal-vacuum.c @@ -34,9 +34,9 @@ struct vacuum_info { char *filename; uint64_t realtime; + sd_id128_t seqnum_id; uint64_t seqnum; - bool have_seqnum; }; @@ -67,19 +67,18 @@ static int vacuum_compare(const void *_a, const void *_b) { } static void patch_realtime( - const char *dir, + int fd, const char *fn, const struct stat *st, unsigned long long *realtime) { - _cleanup_free_ char *path = NULL; usec_t x, crtime = 0; /* The timestamp was determined by the file name, but let's * see if the file might actually be older than the file name * suggested... */ - assert(dir); + assert(fd >= 0); assert(fn); assert(st); assert(realtime); @@ -101,14 +100,7 @@ static void patch_realtime( * unfortunately there's currently no sane API to query * it. Hence let's implement this manually... */ - /* Unfortunately there is is not fgetxattrat(), so we need to - * go via path here. :-( */ - - path = strjoin(dir, "/", fn, NULL); - if (!path) - return; - - if (path_getcrtime(path, &crtime) >= 0) { + if (fd_getcrtime_at(fd, fn, &crtime, 0) >= 0) { if (crtime < *realtime) *realtime = crtime; } -- cgit v1.2.3-54-g00ecf From 804ae586d475d77946debb22c1bc9ee049d4750c Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 2 Oct 2015 22:36:33 +0200 Subject: journal: make journal_file_close() return NULL The way it is customary everywhere else in our sources. --- src/journal/journal-file.c | 3 ++- src/journal/journal-file.h | 2 +- src/journal/journald-server.c | 13 +++++-------- 3 files changed, 8 insertions(+), 10 deletions(-) (limited to 'src/journal') diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c index 73d3a4bb9d..960d6bfd8c 100644 --- a/src/journal/journal-file.c +++ b/src/journal/journal-file.c @@ -128,7 +128,7 @@ int journal_file_set_offline(JournalFile *f) { return 0; } -void journal_file_close(JournalFile *f) { +JournalFile* journal_file_close(JournalFile *f) { assert(f); #ifdef HAVE_GCRYPT @@ -179,6 +179,7 @@ void journal_file_close(JournalFile *f) { #endif free(f); + return NULL; } static int journal_file_init_header(JournalFile *f, JournalFile *template) { diff --git a/src/journal/journal-file.h b/src/journal/journal-file.h index e92b75eabe..52e6b7a261 100644 --- a/src/journal/journal-file.h +++ b/src/journal/journal-file.h @@ -136,7 +136,7 @@ int journal_file_open( JournalFile **ret); int journal_file_set_offline(JournalFile *f); -void journal_file_close(JournalFile *j); +JournalFile* journal_file_close(JournalFile *j); int journal_file_open_reliably( const char *fname, diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index 4566612949..50e9b08da9 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -1091,11 +1091,12 @@ int server_flush_to_var(Server *s) { } } + r = 0; + finish: journal_file_post_change(s->system_journal); - journal_file_close(s->runtime_journal); - s->runtime_journal = NULL; + s->runtime_journal = journal_file_close(s->runtime_journal); if (r >= 0) (void) rm_rf("/run/log/journal", REMOVE_ROOT); @@ -1340,8 +1341,8 @@ static int server_parse_proc_cmdline(Server *s) { } else if (startswith(word, "systemd.journald")) log_warning("Invalid systemd.journald parameter. Ignoring."); } - /* do not warn about state here, since probably systemd already did */ + /* do not warn about state here, since probably systemd already did */ return 0; } @@ -1616,11 +1617,7 @@ int server_init(Server *s) { server_cache_boot_id(s); server_cache_machine_id(s); - r = system_journal_open(s, false); - if (r < 0) - return r; - - return 0; + return system_journal_open(s, false); } void server_maybe_append_tags(Server *s) { -- cgit v1.2.3-54-g00ecf From 84267e4043cf88bf540b5bf9cd65e194670a4ffa Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 2 Oct 2015 22:37:53 +0200 Subject: journal: prefer stack allocation --- src/journal/journald-server.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) (limited to 'src/journal') diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index 50e9b08da9..3538ddc13f 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -926,7 +926,7 @@ finish: static int system_journal_open(Server *s, bool flush_requested) { int r; - char *fn; + const char *fn; sd_id128_t machine; char ids[33]; @@ -969,9 +969,7 @@ static int system_journal_open(Server *s, bool flush_requested) { if (!s->runtime_journal && (s->storage != STORAGE_NONE)) { - fn = strjoin("/run/log/journal/", ids, "/system.journal", NULL); - if (!fn) - return -ENOMEM; + fn = strjoina("/run/log/journal/", ids, "/system.journal", NULL); if (s->system_journal) { @@ -980,8 +978,6 @@ static int system_journal_open(Server *s, bool flush_requested) { * it into the system journal */ r = journal_file_open(fn, O_RDWR, 0640, s->compress, false, &s->runtime_metrics, s->mmap, NULL, &s->runtime_journal); - free(fn); - if (r < 0) { if (r != -ENOENT) log_warning_errno(r, "Failed to open runtime journal: %m"); @@ -999,8 +995,6 @@ static int system_journal_open(Server *s, bool flush_requested) { (void) mkdir_parents(fn, 0750); r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, s->compress, false, &s->runtime_metrics, s->mmap, NULL, &s->runtime_journal); - free(fn); - if (r < 0) return log_error_errno(r, "Failed to open runtime journal: %m"); } -- cgit v1.2.3-54-g00ecf From 7b5195e2740799a5e94cfafdcf8f6d6699c1b8af Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 2 Oct 2015 22:39:24 +0200 Subject: journal: don't affect atime of journal files when vacuuming Let's try to use O_NOATIME if we can when vacuuming old journal files, if we have the permissions for it, so that vacuuming doesn't count as proper journal read access. --- src/journal/journal-vacuum.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'src/journal') diff --git a/src/journal/journal-vacuum.c b/src/journal/journal-vacuum.c index 8c642d16e4..394f7be46b 100644 --- a/src/journal/journal-vacuum.c +++ b/src/journal/journal-vacuum.c @@ -112,9 +112,13 @@ static int journal_file_empty(int dir_fd, const char *name) { le64_t n_entries; ssize_t n; - fd = openat(dir_fd, name, O_RDONLY|O_CLOEXEC|O_NOFOLLOW|O_NONBLOCK); - if (fd < 0) - return -errno; + fd = openat(dir_fd, name, O_RDONLY|O_CLOEXEC|O_NOFOLLOW|O_NONBLOCK|O_NOATIME); + if (fd < 0) { + /* Maybe failed due to O_NOATIME and lack of privileges? */ + fd = openat(dir_fd, name, O_RDONLY|O_CLOEXEC|O_NOFOLLOW|O_NONBLOCK); + if (fd < 0) + return -errno; + } if (fstat(fd, &st) < 0) return -errno; -- cgit v1.2.3-54-g00ecf From 070052aba36e5e90d67141110fdcaf5014e36362 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 2 Oct 2015 22:42:13 +0200 Subject: journal: simplify things by using the LESS_BY() macro --- src/journal/journal-file.c | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) (limited to 'src/journal') diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c index 960d6bfd8c..e21e4a88f1 100644 --- a/src/journal/journal-file.c +++ b/src/journal/journal-file.c @@ -399,12 +399,7 @@ static int journal_file_allocate(JournalFile *f, uint64_t offset, uint64_t size) if (fstatvfs(f->fd, &svfs) >= 0) { uint64_t available; - available = svfs.f_bfree * svfs.f_bsize; - - if (available >= f->metrics.keep_free) - available -= f->metrics.keep_free; - else - available = 0; + available = LESS_BY((uint64_t) svfs.f_bfree * (uint64_t) svfs.f_bsize, f->metrics.keep_free); if (new_size - old_size > available) return -E2BIG; @@ -605,10 +600,10 @@ static int journal_file_setup_data_hash_table(JournalFile *f) { assert(f); - /* We estimate that we need 1 hash table entry per 768 of - journal file and we want to make sure we never get beyond - 75% fill level. Calculate the hash table size for the - maximum file size based on these metrics. */ + /* We estimate that we need 1 hash table entry per 768 bytes + of journal file and we want to make sure we never get + beyond 75% fill level. Calculate the hash table size for + the maximum file size based on these metrics. */ s = (f->metrics.max_size * 4 / 768 / 3) * sizeof(HashItem); if (s < DEFAULT_DATA_HASH_TABLE_SIZE) @@ -2834,8 +2829,7 @@ int journal_file_open_reliably( size_t l; _cleanup_free_ char *p = NULL; - r = journal_file_open(fname, flags, mode, compress, seal, - metrics, mmap_cache, template, ret); + r = journal_file_open(fname, flags, mode, compress, seal, metrics, mmap_cache, template, ret); if (!IN_SET(r, -EBADMSG, /* corrupted */ -ENODATA, /* truncated */ @@ -2877,8 +2871,7 @@ int journal_file_open_reliably( log_warning("File %s corrupted or uncleanly shut down, renaming and replacing.", fname); - return journal_file_open(fname, flags, mode, compress, seal, - metrics, mmap_cache, template, ret); + return journal_file_open(fname, flags, mode, compress, seal, metrics, mmap_cache, template, ret); } int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint64_t p, uint64_t *seqnum, Object **ret, uint64_t *offset) { -- cgit v1.2.3-54-g00ecf From 65089b82401cd395786a7987c470056ff3f01151 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 2 Oct 2015 23:19:00 +0200 Subject: journal: improve some messages Indicate that we are ignoring errors, when we ignore them, and log that at LOG_WARNING level. Use the right error code for the log message. --- src/journal/journal-file.c | 5 ++--- src/journal/journald-server.c | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) (limited to 'src/journal') diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c index e21e4a88f1..085d5f346d 100644 --- a/src/journal/journal-file.c +++ b/src/journal/journal-file.c @@ -2859,8 +2859,7 @@ int journal_file_open_reliably( random_u64()) < 0) return -ENOMEM; - r = rename(fname, p); - if (r < 0) + if (rename(fname, p) < 0) return -errno; /* btrfs doesn't cope well with our write pattern and @@ -2869,7 +2868,7 @@ int journal_file_open_reliably( (void) chattr_path(p, false, FS_NOCOW_FL); (void) btrfs_defrag(p); - log_warning("File %s corrupted or uncleanly shut down, renaming and replacing.", fname); + log_warning_errno(r, "File %s corrupted or uncleanly shut down, renaming and replacing.", fname); return journal_file_open(fname, flags, mode, compress, seal, metrics, mmap_cache, template, ret); } diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index 3538ddc13f..5c896d22c1 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -202,7 +202,7 @@ void server_fix_perms(Server *s, JournalFile *f, uid_t uid) { r = fchmod(f->fd, 0640); if (r < 0) - log_warning_errno(r, "Failed to fix access mode on %s, ignoring: %m", f->path); + log_warning_errno(errno, "Failed to fix access mode on %s, ignoring: %m", f->path); #ifdef HAVE_ACL if (uid <= SYSTEM_UID_MAX) @@ -350,13 +350,13 @@ void server_sync(Server *s) { if (s->system_journal) { r = journal_file_set_offline(s->system_journal); if (r < 0) - log_error_errno(r, "Failed to sync system journal: %m"); + log_warning_errno(r, "Failed to sync system journal, ignoring: %m"); } ORDERED_HASHMAP_FOREACH_KEY(f, k, s->user_journals, i) { r = journal_file_set_offline(f); if (r < 0) - log_error_errno(r, "Failed to sync user journal: %m"); + log_warning_errno(r, "Failed to sync user journal, ignoring: %m"); } if (s->sync_event_source) { -- cgit v1.2.3-54-g00ecf From 0fb398316c6705dfc168733361650fdb6824896d Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 2 Oct 2015 23:20:51 +0200 Subject: journal: use automatic clenup for ACL types --- src/journal/journald-server.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'src/journal') diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index 5c896d22c1..296e28e6e3 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -193,7 +193,7 @@ static uint64_t available_space(Server *s, bool verbose) { void server_fix_perms(Server *s, JournalFile *f, uid_t uid) { int r; #ifdef HAVE_ACL - acl_t acl; + _cleanup_(acl_freep) acl_t acl = NULL; acl_entry_t entry; acl_permset_t permset; #endif @@ -221,7 +221,7 @@ void server_fix_perms(Server *s, JournalFile *f, uid_t uid) { acl_set_tag_type(entry, ACL_USER) < 0 || acl_set_qualifier(entry, &uid) < 0) { log_warning_errno(errno, "Failed to patch ACL on %s, ignoring: %m", f->path); - goto finish; + return; } } @@ -231,14 +231,12 @@ void server_fix_perms(Server *s, JournalFile *f, uid_t uid) { acl_add_perm(permset, ACL_READ) < 0 || calc_acl_mask_if_needed(&acl) < 0) { log_warning_errno(errno, "Failed to patch ACL on %s, ignoring: %m", f->path); - goto finish; + return; } if (acl_set_fd(f->fd, acl) < 0) log_warning_errno(errno, "Failed to set ACL on %s, ignoring: %m", f->path); -finish: - acl_free(acl); #endif } -- cgit v1.2.3-54-g00ecf From 8580d1f73db36e9383e674e388b4fb55828c0c66 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 2 Oct 2015 23:21:59 +0200 Subject: journal: rework vacuuming logic Implement a maximum limit on number of journal files to keep around. Enforcing a limit is useful on this since our performance when viewing pays a heavy penalty for each journal file to interleve. This setting is turned on now by default, and set to 100. Also, actully implement what 348ced909724a1331b85d57aede80a102a00e428 promised: use whatever we find on disk at startup as lower bound on how much disk space we can use. That commit introduced some provisions to implement this, but actually never did. This also adds "journalctl --vacuum-files=" to vacuum files on disk by their number explicitly. --- man/journalctl.xml | 26 ++- man/journald.conf.xml | 23 ++- src/journal/journal-file.c | 64 +++++-- src/journal/journal-file.h | 13 +- src/journal/journal-vacuum.c | 98 ++++++---- src/journal/journal-vacuum.h | 6 +- src/journal/journalctl.c | 24 ++- src/journal/journald-gperf.gperf | 2 + src/journal/journald-server.c | 320 +++++++++++++++++--------------- src/journal/journald-server.h | 9 +- src/journal/journald.c | 8 +- src/journal/journald.conf | 2 + src/journal/test-journal-interleaving.c | 4 +- src/journal/test-journal.c | 4 +- 14 files changed, 365 insertions(+), 238 deletions(-) (limited to 'src/journal') diff --git a/man/journalctl.xml b/man/journalctl.xml index 305f1b9412..3efcfbd6bf 100644 --- a/man/journalctl.xml +++ b/man/journalctl.xml @@ -649,6 +649,7 @@ + Removes archived journal files until the disk space they use falls below the specified size (specified with @@ -658,15 +659,24 @@ timespan (specified with the usual s, min, h, days, months, - weeks, years - suffixes). Note that running - has only indirect effect on the output shown by + weeks, years suffixes), + or no more than the specified number of separate journal files + remain. Note that running has + only indirect effect on the output shown by as the latter includes active - journal files, while the former only operates on archived - journal files. and - may be combined in a single - invocation to enforce both a size and time limit on the - archived journal files. + journal files, while the the vacuuming operation only operates + on archived journal files. Similar, + might not actually reduce the + number of journal files to below the specified number, as it + will not remove active journal + files. , + and + may be combined in a single + invocation to enforce any combination of a size, a time and a + number of files limit on the archived journal + files. Specifying any of these three parameters as zero is + equivalent to not enforcing the specific limit, and is thus + redundant. diff --git a/man/journald.conf.xml b/man/journald.conf.xml index d6fe45d40c..109ca960fb 100644 --- a/man/journald.conf.xml +++ b/man/journald.conf.xml @@ -173,9 +173,11 @@ SystemMaxUse= SystemKeepFree= SystemMaxFileSize= + SystemMaxFiles= RuntimeMaxUse= RuntimeKeepFree= RuntimeMaxFileSize= + RuntimeMaxFiles= Enforce size limits on the journal files stored. The options prefixed with System @@ -197,8 +199,7 @@ names not ending with .journal or .journal~, so only such files, located in the appropriate directories, are taken into account when - calculating current disk usage. - + calculating current disk usage. SystemMaxUse= and RuntimeMaxUse= control how much disk space @@ -212,13 +213,14 @@ The first pair defaults to 10% and the second to 15% of the size of the respective file system. If the file system is nearly full and either SystemKeepFree= or - RuntimeKeepFree= is violated when - systemd-journald is started, the value will be raised to + RuntimeKeepFree= are violated when + systemd-journald is started, the limit will be raised to the percentage that is actually free. This means that if there was enough free space before and journal files were created, and subsequently something else causes the file system to fill up, journald will stop using more space, but it will not be - removing existing files to go reduce footprint either. + removing existing files to reduce footprint again + either. SystemMaxFileSize= and RuntimeMaxFileSize= control how large @@ -228,13 +230,22 @@ eighth of the values configured with SystemMaxUse= and RuntimeMaxUse=, so that usually seven - rotated journal files are kept as history. + rotated journal files are kept as history. Specify values in bytes or use K, M, G, T, P, E as units for the specified sizes (equal to 1024, 1024²,... bytes). Note that size limits are enforced synchronously when journal files are extended, and no explicit rotation step triggered by time is needed. + + SystemMaxFiles= and + RuntimeMaxFiles= control how many + individual journal files to keep at maximum. Note that only + archived files are deleted to reduce the number of files until + this limit is reached; active files will stay around. This + means that in effect there might still be more journal files + around in total than this limit after a vacuuming operation is + complete. This setting defaults to 100. diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c index 085d5f346d..1071c6d6d7 100644 --- a/src/journal/journal-file.c +++ b/src/journal/journal-file.c @@ -49,6 +49,9 @@ #define DEFAULT_MAX_USE_LOWER (1ULL*1024ULL*1024ULL) /* 1 MiB */ #define DEFAULT_MAX_USE_UPPER (4ULL*1024ULL*1024ULL*1024ULL) /* 4 GiB */ +/* This is the default minimal use limit, how much we'll use even if keep_free suggests otherwise. */ +#define DEFAULT_MIN_USE (1ULL*1024ULL*1024ULL) /* 1 MiB */ + /* This is the upper bound if we deduce max_size from max_use */ #define DEFAULT_MAX_SIZE_UPPER (128ULL*1024ULL*1024ULL) /* 128 MiB */ @@ -60,6 +63,9 @@ * size */ #define DEFAULT_KEEP_FREE (1024ULL*1024ULL) /* 1 MB */ +/* This is the default maximum number of journal files to keep around. */ +#define DEFAULT_N_MAX_FILES (100) + /* n_data was the first entry we added after the initial file format design */ #define HEADER_SIZE_MIN ALIGN64(offsetof(Header, n_data)) @@ -2957,16 +2963,35 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6 return r; } +void journal_reset_metrics(JournalMetrics *m) { + assert(m); + + /* Set everything to "pick automatic values". */ + + *m = (JournalMetrics) { + .min_use = (uint64_t) -1, + .max_use = (uint64_t) -1, + .min_size = (uint64_t) -1, + .max_size = (uint64_t) -1, + .keep_free = (uint64_t) -1, + .n_max_files = (uint64_t) -1, + }; +} + void journal_default_metrics(JournalMetrics *m, int fd) { - uint64_t fs_size = 0; + char a[FORMAT_BYTES_MAX], b[FORMAT_BYTES_MAX], c[FORMAT_BYTES_MAX], d[FORMAT_BYTES_MAX], e[FORMAT_BYTES_MAX]; struct statvfs ss; - char a[FORMAT_BYTES_MAX], b[FORMAT_BYTES_MAX], c[FORMAT_BYTES_MAX], d[FORMAT_BYTES_MAX]; + uint64_t fs_size; assert(m); assert(fd >= 0); if (fstatvfs(fd, &ss) >= 0) fs_size = ss.f_frsize * ss.f_blocks; + else { + log_debug_errno(errno, "Failed to detremine disk size: %m"); + fs_size = 0; + } if (m->max_use == (uint64_t) -1) { @@ -2983,10 +3008,16 @@ void journal_default_metrics(JournalMetrics *m, int fd) { } else { m->max_use = PAGE_ALIGN(m->max_use); - if (m->max_use < JOURNAL_FILE_SIZE_MIN*2) + if (m->max_use != 0 && m->max_use < JOURNAL_FILE_SIZE_MIN*2) m->max_use = JOURNAL_FILE_SIZE_MIN*2; } + if (m->min_use == (uint64_t) -1) + m->min_use = DEFAULT_MIN_USE; + + if (m->min_use > m->max_use) + m->min_use = m->max_use; + if (m->max_size == (uint64_t) -1) { m->max_size = PAGE_ALIGN(m->max_use / 8); /* 8 chunks */ @@ -2995,11 +3026,13 @@ void journal_default_metrics(JournalMetrics *m, int fd) { } else m->max_size = PAGE_ALIGN(m->max_size); - if (m->max_size < JOURNAL_FILE_SIZE_MIN) - m->max_size = JOURNAL_FILE_SIZE_MIN; + if (m->max_size != 0) { + if (m->max_size < JOURNAL_FILE_SIZE_MIN) + m->max_size = JOURNAL_FILE_SIZE_MIN; - if (m->max_size*2 > m->max_use) - m->max_use = m->max_size*2; + if (m->max_use != 0 && m->max_size*2 > m->max_use) + m->max_use = m->max_size*2; + } if (m->min_size == (uint64_t) -1) m->min_size = JOURNAL_FILE_SIZE_MIN; @@ -3009,7 +3042,7 @@ void journal_default_metrics(JournalMetrics *m, int fd) { if (m->min_size < JOURNAL_FILE_SIZE_MIN) m->min_size = JOURNAL_FILE_SIZE_MIN; - if (m->min_size > m->max_size) + if (m->max_size != 0 && m->min_size > m->max_size) m->max_size = m->min_size; } @@ -3025,11 +3058,16 @@ void journal_default_metrics(JournalMetrics *m, int fd) { m->keep_free = DEFAULT_KEEP_FREE; } - log_debug("Fixed max_use=%s max_size=%s min_size=%s keep_free=%s", - format_bytes(a, sizeof(a), m->max_use), - format_bytes(b, sizeof(b), m->max_size), - format_bytes(c, sizeof(c), m->min_size), - format_bytes(d, sizeof(d), m->keep_free)); + if (m->n_max_files == (uint64_t) -1) + m->n_max_files = DEFAULT_N_MAX_FILES; + + log_debug("Fixed min_use=%s max_use=%s max_size=%s min_size=%s keep_free=%s n_max_files=%" PRIu64, + format_bytes(a, sizeof(a), m->min_use), + format_bytes(b, sizeof(b), m->max_use), + format_bytes(c, sizeof(c), m->max_size), + format_bytes(d, sizeof(d), m->min_size), + format_bytes(e, sizeof(e), m->keep_free), + m->n_max_files); } int journal_file_get_cutoff_realtime_usec(JournalFile *f, usec_t *from, usec_t *to) { diff --git a/src/journal/journal-file.h b/src/journal/journal-file.h index 52e6b7a261..f2c07356c8 100644 --- a/src/journal/journal-file.h +++ b/src/journal/journal-file.h @@ -36,11 +36,13 @@ #include "hashmap.h" typedef struct JournalMetrics { - uint64_t max_use; - uint64_t use; - uint64_t max_size; - uint64_t min_size; - uint64_t keep_free; + /* For all these: -1 means "pick automatically", and 0 means "no limit enforced" */ + uint64_t max_size; /* how large journal files grow at max */ + uint64_t min_size; /* how large journal files grow at least */ + uint64_t max_use; /* how much disk space to use in total at max, keep_free permitting */ + uint64_t min_use; /* how much disk space to use in total at least, even if keep_free says not to */ + uint64_t keep_free; /* how much to keep free on disk */ + uint64_t n_max_files; /* how many files to keep around at max */ } JournalMetrics; typedef enum direction { @@ -223,6 +225,7 @@ int journal_file_rotate(JournalFile **f, bool compress, bool seal); void journal_file_post_change(JournalFile *f); +void journal_reset_metrics(JournalMetrics *m); void journal_default_metrics(JournalMetrics *m, int fd); int journal_file_get_cutoff_realtime_usec(JournalFile *f, usec_t *from, usec_t *to); diff --git a/src/journal/journal-vacuum.c b/src/journal/journal-vacuum.c index 394f7be46b..a394066cb4 100644 --- a/src/journal/journal-vacuum.c +++ b/src/journal/journal-vacuum.c @@ -140,22 +140,24 @@ static int journal_file_empty(int dir_fd, const char *name) { int journal_directory_vacuum( const char *directory, uint64_t max_use, + uint64_t n_max_files, usec_t max_retention_usec, usec_t *oldest_usec, bool verbose) { _cleanup_closedir_ DIR *d = NULL; - int r = 0; struct vacuum_info *list = NULL; - unsigned n_list = 0, i; + unsigned n_list = 0, i, n_active_files = 0; size_t n_allocated = 0; uint64_t sum = 0, freed = 0; usec_t retention_limit = 0; char sbytes[FORMAT_BYTES_MAX]; + struct dirent *de; + int r; assert(directory); - if (max_use <= 0 && max_retention_usec <= 0) + if (max_use <= 0 && max_retention_usec <= 0 && n_max_files <= 0) return 0; if (max_retention_usec > 0) { @@ -170,27 +172,20 @@ int journal_directory_vacuum( if (!d) return -errno; - for (;;) { - struct dirent *de; - size_t q; - struct stat st; - char *p; + FOREACH_DIRENT_ALL(de, d, r = -errno; goto finish) { + unsigned long long seqnum = 0, realtime; + _cleanup_free_ char *p = NULL; sd_id128_t seqnum_id; bool have_seqnum; + uint64_t size; + struct stat st; + size_t q; - errno = 0; - de = readdir(d); - if (!de && errno != 0) { - r = -errno; - goto finish; - } - - if (!de) - break; - - if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) + if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) { + log_debug_errno(errno, "Failed to stat file %s while vacuuming, ignoring: %m", de->d_name); continue; + } if (!S_ISREG(st.st_mode)) continue; @@ -199,15 +194,20 @@ int journal_directory_vacuum( if (endswith(de->d_name, ".journal")) { - /* Vacuum archived files */ + /* Vacuum archived files. Active files are + * left around */ - if (q < 1 + 32 + 1 + 16 + 1 + 16 + 8) + if (q < 1 + 32 + 1 + 16 + 1 + 16 + 8) { + n_active_files++; continue; + } if (de->d_name[q-8-16-1] != '-' || de->d_name[q-8-16-1-16-1] != '-' || - de->d_name[q-8-16-1-16-1-32-1] != '@') + de->d_name[q-8-16-1-16-1-32-1] != '@') { + n_active_files++; continue; + } p = strdup(de->d_name); if (!p) { @@ -218,11 +218,13 @@ int journal_directory_vacuum( de->d_name[q-8-16-1-16-1] = 0; if (sd_id128_from_string(de->d_name + q-8-16-1-16-1-32, &seqnum_id) < 0) { free(p); + n_active_files++; continue; } if (sscanf(de->d_name + q-8-16-1-16, "%16llx-%16llx.journal", &seqnum, &realtime) != 2) { free(p); + n_active_files++; continue; } @@ -233,12 +235,16 @@ int journal_directory_vacuum( /* Vacuum corrupted files */ - if (q < 1 + 16 + 1 + 16 + 8 + 1) + if (q < 1 + 16 + 1 + 16 + 8 + 1) { + n_active_files ++; continue; + } if (de->d_name[q-1-8-16-1] != '-' || - de->d_name[q-1-8-16-1-16-1] != '@') + de->d_name[q-1-8-16-1-16-1] != '@') { + n_active_files ++; continue; + } p = strdup(de->d_name); if (!p) { @@ -248,54 +254,68 @@ int journal_directory_vacuum( if (sscanf(de->d_name + q-1-8-16-1-16, "%16llx-%16llx.journal~", &realtime, &tmp) != 2) { free(p); + n_active_files ++; continue; } have_seqnum = false; - } else - /* We do not vacuum active files or unknown files! */ + } else { + /* We do not vacuum unknown files! */ + log_debug("Not vacuuming unknown file %s.", de->d_name); continue; + } - if (journal_file_empty(dirfd(d), p)) { - /* Always vacuum empty non-online files. */ + size = 512UL * (uint64_t) st.st_blocks; - uint64_t size = 512UL * (uint64_t) st.st_blocks; + r = journal_file_empty(dirfd(d), p); + if (r < 0) { + log_debug_errno(r, "Failed check if %s is empty, ignoring: %m", p); + continue; + } + if (r > 0) { + /* Always vacuum empty non-online files. */ if (unlinkat(dirfd(d), p, 0) >= 0) { - log_full(verbose ? LOG_INFO : LOG_DEBUG, "Deleted empty archived journal %s/%s (%s).", directory, p, format_bytes(sbytes, sizeof(sbytes), size)); + + log_full(verbose ? LOG_INFO : LOG_DEBUG, + "Deleted empty archived journal %s/%s (%s).", directory, p, format_bytes(sbytes, sizeof(sbytes), size)); + freed += size; } else if (errno != ENOENT) log_warning_errno(errno, "Failed to delete empty archived journal %s/%s: %m", directory, p); - free(p); continue; } - patch_realtime(directory, p, &st, &realtime); + patch_realtime(dirfd(d), p, &st, &realtime); if (!GREEDY_REALLOC(list, n_allocated, n_list + 1)) { - free(p); r = -ENOMEM; goto finish; } list[n_list].filename = p; - list[n_list].usage = 512UL * (uint64_t) st.st_blocks; + list[n_list].usage = size; list[n_list].seqnum = seqnum; list[n_list].realtime = realtime; list[n_list].seqnum_id = seqnum_id; list[n_list].have_seqnum = have_seqnum; - - sum += list[n_list].usage; - n_list ++; + + p = NULL; + sum += size; } qsort_safe(list, n_list, sizeof(struct vacuum_info), vacuum_compare); for (i = 0; i < n_list; i++) { + unsigned left; + + left = n_active_files + n_list - i; + if ((max_retention_usec <= 0 || list[i].realtime >= retention_limit) && - (max_use <= 0 || sum <= max_use)) + (max_use <= 0 || sum <= max_use) && + (n_max_files <= 0 || left <= n_max_files)) break; if (unlinkat(dirfd(d), list[i].filename, 0) >= 0) { @@ -314,6 +334,8 @@ int journal_directory_vacuum( if (oldest_usec && i < n_list && (*oldest_usec == 0 || list[i].realtime < *oldest_usec)) *oldest_usec = list[i].realtime; + r = 0; + finish: for (i = 0; i < n_list; i++) free(list[i].filename); diff --git a/src/journal/journal-vacuum.h b/src/journal/journal-vacuum.h index c45cc31d0e..49ab90af91 100644 --- a/src/journal/journal-vacuum.h +++ b/src/journal/journal-vacuum.h @@ -21,5 +21,9 @@ along with systemd; If not, see . ***/ +#include +#include -int journal_directory_vacuum(const char *directory, uint64_t max_use, usec_t max_retention_usec, usec_t *oldest_usec, bool vacuum); +#include "time-util.h" + +int journal_directory_vacuum(const char *directory, uint64_t max_use, uint64_t n_max_files, usec_t max_retention_usec, usec_t *oldest_usec, bool verbose); diff --git a/src/journal/journalctl.c b/src/journal/journalctl.c index 28ccb80661..d9851db36c 100644 --- a/src/journal/journalctl.c +++ b/src/journal/journalctl.c @@ -106,8 +106,9 @@ static bool arg_reverse = false; static int arg_journal_type = 0; static const char *arg_root = NULL; static const char *arg_machine = NULL; -static uint64_t arg_vacuum_size = (uint64_t) -1; -static usec_t arg_vacuum_time = USEC_INFINITY; +static uint64_t arg_vacuum_size = 0; +static uint64_t arg_vacuum_n_files = 0; +static usec_t arg_vacuum_time = 0; static enum { ACTION_SHOW, @@ -235,7 +236,8 @@ static void help(void) { " --new-id128 Generate a new 128-bit ID\n" " --disk-usage Show total disk usage of all journal files\n" " --vacuum-size=BYTES Reduce disk usage below specified size\n" - " --vacuum-time=TIME Remove journal files older than specified date\n" + " --vacuum-files=INT Leave only the specified number of journal files\n" + " --vacuum-time=TIME Remove journal files older than specified time\n" " --flush Flush all journal data from /run into /var\n" " --rotate Request immediate rotation of the journal files\n" " --header Show journal header information\n" @@ -281,6 +283,7 @@ static int parse_argv(int argc, char *argv[]) { ARG_FLUSH, ARG_ROTATE, ARG_VACUUM_SIZE, + ARG_VACUUM_FILES, ARG_VACUUM_TIME, }; @@ -335,6 +338,7 @@ static int parse_argv(int argc, char *argv[]) { { "flush", no_argument, NULL, ARG_FLUSH }, { "rotate", no_argument, NULL, ARG_ROTATE }, { "vacuum-size", required_argument, NULL, ARG_VACUUM_SIZE }, + { "vacuum-files", required_argument, NULL, ARG_VACUUM_FILES }, { "vacuum-time", required_argument, NULL, ARG_VACUUM_TIME }, {} }; @@ -540,6 +544,16 @@ static int parse_argv(int argc, char *argv[]) { arg_action = ACTION_VACUUM; break; + case ARG_VACUUM_FILES: + r = safe_atou64(optarg, &arg_vacuum_n_files); + if (r < 0) { + log_error("Failed to parse vacuum files: %s", optarg); + return r; + } + + arg_action = ACTION_VACUUM; + break; + case ARG_VACUUM_TIME: r = parse_sec(optarg, &arg_vacuum_time); if (r < 0) { @@ -1929,9 +1943,9 @@ int main(int argc, char *argv[]) { if (d->is_root) continue; - q = journal_directory_vacuum(d->path, arg_vacuum_size, arg_vacuum_time, NULL, true); + q = journal_directory_vacuum(d->path, arg_vacuum_size, arg_vacuum_n_files, arg_vacuum_time, NULL, true); if (q < 0) { - log_error_errno(q, "Failed to vacuum: %m"); + log_error_errno(q, "Failed to vacuum %s: %m", d->path); r = q; } } diff --git a/src/journal/journald-gperf.gperf b/src/journal/journald-gperf.gperf index bf7c773009..c154610c54 100644 --- a/src/journal/journald-gperf.gperf +++ b/src/journal/journald-gperf.gperf @@ -24,9 +24,11 @@ Journal.RateLimitBurst, config_parse_unsigned, 0, offsetof(Server, rate_li Journal.SystemMaxUse, config_parse_iec_uint64, 0, offsetof(Server, system_metrics.max_use) Journal.SystemMaxFileSize, config_parse_iec_uint64, 0, offsetof(Server, system_metrics.max_size) Journal.SystemKeepFree, config_parse_iec_uint64, 0, offsetof(Server, system_metrics.keep_free) +Journal.SystemMaxFiles, config_parse_uint64, 0, offsetof(Server, system_metrics.n_max_files) Journal.RuntimeMaxUse, config_parse_iec_uint64, 0, offsetof(Server, runtime_metrics.max_use) Journal.RuntimeMaxFileSize, config_parse_iec_uint64, 0, offsetof(Server, runtime_metrics.max_size) Journal.RuntimeKeepFree, config_parse_iec_uint64, 0, offsetof(Server, runtime_metrics.keep_free) +Journal.RuntimeMaxFiles, config_parse_uint64, 0, offsetof(Server, runtime_metrics.n_max_files) Journal.MaxRetentionSec, config_parse_sec, 0, offsetof(Server, max_retention_usec) Journal.MaxFileSec, config_parse_sec, 0, offsetof(Server, max_file_usec) Journal.ForwardToSyslog, config_parse_bool, 0, offsetof(Server, forward_to_syslog) diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index 296e28e6e3..fb172b7f5d 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -19,45 +19,44 @@ along with systemd; If not, see . ***/ -#include -#include #include -#include -#include - #ifdef HAVE_SELINUX #include #endif +#include +#include +#include +#include -#include - +#include "libudev.h" +#include "sd-daemon.h" #include "sd-journal.h" #include "sd-messages.h" -#include "sd-daemon.h" -#include "mkdir.h" -#include "rm-rf.h" -#include "hashmap.h" -#include "journal-file.h" -#include "socket-util.h" + +#include "acl-util.h" #include "cgroup-util.h" -#include "missing.h" #include "conf-parser.h" -#include "selinux-util.h" -#include "acl-util.h" #include "formats-util.h" -#include "process-util.h" +#include "hashmap.h" #include "hostname-util.h" +#include "missing.h" +#include "mkdir.h" +#include "process-util.h" +#include "rm-rf.h" +#include "selinux-util.h" #include "signal-util.h" +#include "socket-util.h" +#include "journal-authenticate.h" +#include "journal-file.h" #include "journal-internal.h" #include "journal-vacuum.h" -#include "journal-authenticate.h" -#include "journald-rate-limit.h" +#include "journald-audit.h" #include "journald-kmsg.h" -#include "journald-syslog.h" -#include "journald-stream.h" #include "journald-native.h" -#include "journald-audit.h" +#include "journald-rate-limit.h" #include "journald-server.h" +#include "journald-stream.h" +#include "journald-syslog.h" #define USER_JOURNALS_MAX 1024 @@ -66,88 +65,61 @@ #define DEFAULT_RATE_LIMIT_BURST 1000 #define DEFAULT_MAX_FILE_USEC USEC_PER_MONTH -#define RECHECK_AVAILABLE_SPACE_USEC (30*USEC_PER_SEC) - -static const char* const storage_table[_STORAGE_MAX] = { - [STORAGE_AUTO] = "auto", - [STORAGE_VOLATILE] = "volatile", - [STORAGE_PERSISTENT] = "persistent", - [STORAGE_NONE] = "none" -}; - -DEFINE_STRING_TABLE_LOOKUP(storage, Storage); -DEFINE_CONFIG_PARSE_ENUM(config_parse_storage, storage, Storage, "Failed to parse storage setting"); +#define RECHECK_SPACE_USEC (30*USEC_PER_SEC) -static const char* const split_mode_table[_SPLIT_MAX] = { - [SPLIT_LOGIN] = "login", - [SPLIT_UID] = "uid", - [SPLIT_NONE] = "none", -}; - -DEFINE_STRING_TABLE_LOOKUP(split_mode, SplitMode); -DEFINE_CONFIG_PARSE_ENUM(config_parse_split_mode, split_mode, SplitMode, "Failed to parse split mode setting"); - -static uint64_t available_space(Server *s, bool verbose) { - char ids[33]; - _cleanup_free_ char *p = NULL; - sd_id128_t machine; - struct statvfs ss; - uint64_t sum = 0, ss_avail = 0, avail = 0; - int r; +static int determine_space_for( + Server *s, + JournalMetrics *metrics, + const char *path, + const char *name, + bool verbose, + bool patch_min_use, + uint64_t *available, + uint64_t *limit) { + + uint64_t sum = 0, ss_avail, avail; _cleanup_closedir_ DIR *d = NULL; + struct dirent *de; + struct statvfs ss; + const char *p; usec_t ts; - const char *f; - JournalMetrics *m; - ts = now(CLOCK_MONOTONIC); - - if (s->cached_available_space_timestamp + RECHECK_AVAILABLE_SPACE_USEC > ts - && !verbose) - return s->cached_available_space; + assert(s); + assert(metrics); + assert(path); + assert(name); - r = sd_id128_get_machine(&machine); - if (r < 0) - return 0; + ts = now(CLOCK_MONOTONIC); - if (s->system_journal) { - f = "/var/log/journal/"; - m = &s->system_metrics; - } else { - f = "/run/log/journal/"; - m = &s->runtime_metrics; - } + if (!verbose && s->cached_space_timestamp + RECHECK_SPACE_USEC > ts) { - assert(m); + if (available) + *available = s->cached_space_available; + if (limit) + *limit = s->cached_space_limit; - p = strappend(f, sd_id128_to_string(machine, ids)); - if (!p) return 0; + } + p = strjoina(path, SERVER_MACHINE_ID(s)); d = opendir(p); if (!d) - return 0; + return log_full_errno(errno == ENOENT ? LOG_DEBUG : LOG_ERR, errno, "Failed to open %s: %m", p); if (fstatvfs(dirfd(d), &ss) < 0) - return 0; + return log_error_errno(errno, "Failed to fstatvfs(%s): %m", p); - for (;;) { + FOREACH_DIRENT_ALL(de, d, break) { struct stat st; - struct dirent *de; - - errno = 0; - de = readdir(d); - if (!de && errno != 0) - return 0; - - if (!de) - break; if (!endswith(de->d_name, ".journal") && !endswith(de->d_name, ".journal~")) continue; - if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) + if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) { + log_debug_errno(errno, "Failed to stat %s/%s, ignoring: %m", p, de->d_name); continue; + } if (!S_ISREG(st.st_mode)) continue; @@ -155,39 +127,66 @@ static uint64_t available_space(Server *s, bool verbose) { sum += (uint64_t) st.st_blocks * 512UL; } - ss_avail = ss.f_bsize * ss.f_bavail; + /* If request, then let's bump the min_use limit to the + * current usage on disk. We do this when starting up and + * first opening the journal files. This way sudden spikes in + * disk usage will not cause journald to vacuum files without + * bounds. Note that this means that only a restart of + * journald will make it reset this value. */ - /* If we reached a high mark, we will always allow this much - * again, unless usage goes above max_use. This watermark - * value is cached so that we don't give up space on pressure, - * but hover below the maximum usage. */ + if (patch_min_use) + metrics->min_use = MAX(metrics->min_use, sum); - if (m->use < sum) - m->use = sum; - - avail = LESS_BY(ss_avail, m->keep_free); + ss_avail = ss.f_bsize * ss.f_bavail; + avail = LESS_BY(ss_avail, metrics->keep_free); - s->cached_available_space = LESS_BY(MIN(m->max_use, avail), sum); - s->cached_available_space_timestamp = ts; + s->cached_space_limit = MIN(MAX(sum + avail, metrics->min_use), metrics->max_use); + s->cached_space_available = LESS_BY(s->cached_space_limit, sum); + s->cached_space_timestamp = ts; if (verbose) { char fb1[FORMAT_BYTES_MAX], fb2[FORMAT_BYTES_MAX], fb3[FORMAT_BYTES_MAX], - fb4[FORMAT_BYTES_MAX], fb5[FORMAT_BYTES_MAX]; + fb4[FORMAT_BYTES_MAX], fb5[FORMAT_BYTES_MAX], fb6[FORMAT_BYTES_MAX]; server_driver_message(s, SD_MESSAGE_JOURNAL_USAGE, - "%s is currently using %s.\n" + "%s (%s) is currently using %s.\n" "Maximum allowed usage is set to %s.\n" "Leaving at least %s free (of currently available %s of space).\n" - "Enforced usage limit is thus %s.", - s->system_journal ? "Permanent journal (/var/log/journal/)" : "Runtime journal (/run/log/journal/)", + "Enforced usage limit is thus %s, of which %s are still available.", + name, path, format_bytes(fb1, sizeof(fb1), sum), - format_bytes(fb2, sizeof(fb2), m->max_use), - format_bytes(fb3, sizeof(fb3), m->keep_free), + format_bytes(fb2, sizeof(fb2), metrics->max_use), + format_bytes(fb3, sizeof(fb3), metrics->keep_free), format_bytes(fb4, sizeof(fb4), ss_avail), - format_bytes(fb5, sizeof(fb5), s->cached_available_space + sum)); + format_bytes(fb5, sizeof(fb5), s->cached_space_limit), + format_bytes(fb6, sizeof(fb6), s->cached_space_available)); + } + + if (available) + *available = s->cached_space_available; + if (limit) + *limit = s->cached_space_limit; + + return 1; +} + +static int determine_space(Server *s, bool verbose, bool patch_min_use, uint64_t *available, uint64_t *limit) { + JournalMetrics *metrics; + const char *path, *name; + + assert(s); + + if (s->system_journal) { + path = "/var/log/journal/"; + metrics = &s->system_metrics; + name = "System journal"; + } else { + path = "/run/log/journal/"; + metrics = &s->runtime_metrics; + name = "Runtime journal"; } - return s->cached_available_space; + return determine_space_for(s, metrics, path, name, verbose, patch_min_use, available, limit); } void server_fix_perms(Server *s, JournalFile *f, uid_t uid) { @@ -326,8 +325,8 @@ void server_rotate(Server *s) { log_debug("Rotating..."); - do_rotate(s, &s->runtime_journal, "runtime", false, 0); - do_rotate(s, &s->system_journal, "system", s->seal, 0); + (void) do_rotate(s, &s->runtime_journal, "runtime", false, 0); + (void) do_rotate(s, &s->system_journal, "system", s->seal, 0); ORDERED_HASHMAP_FOREACH_KEY(f, k, s->user_journals, i) { r = do_rotate(s, &f, "user", s->seal, PTR_TO_UINT32(k)); @@ -368,43 +367,50 @@ void server_sync(Server *s) { static void do_vacuum( Server *s, - const char *id, JournalFile *f, - const char* path, - JournalMetrics *metrics) { + JournalMetrics *metrics, + const char *path, + const char *name, + bool verbose, + bool patch_min_use) { const char *p; + uint64_t limit; int r; + assert(s); + assert(metrics); + assert(path); + assert(name); + if (!f) return; - p = strjoina(path, id); - r = journal_directory_vacuum(p, metrics->max_use, s->max_retention_usec, &s->oldest_file_usec, false); + p = strjoina(path, SERVER_MACHINE_ID(s)); + + limit = metrics->max_use; + (void) determine_space_for(s, metrics, path, name, verbose, patch_min_use, NULL, &limit); + + r = journal_directory_vacuum(p, limit, metrics->n_max_files, s->max_retention_usec, &s->oldest_file_usec, verbose); if (r < 0 && r != -ENOENT) - log_error_errno(r, "Failed to vacuum %s: %m", p); + log_warning_errno(r, "Failed to vacuum %s, ignoring: %m", p); } -void server_vacuum(Server *s) { - char ids[33]; - sd_id128_t machine; - int r; +int server_vacuum(Server *s, bool verbose, bool patch_min_use) { + assert(s); log_debug("Vacuuming..."); s->oldest_file_usec = 0; - r = sd_id128_get_machine(&machine); - if (r < 0) { - log_error_errno(r, "Failed to get machine ID: %m"); - return; - } - sd_id128_to_string(machine, ids); + do_vacuum(s, s->system_journal, &s->system_metrics, "/var/log/journal/", "System journal", verbose, patch_min_use); + do_vacuum(s, s->runtime_journal, &s->runtime_metrics, "/run/log/journal/", "Runtime journal", verbose, patch_min_use); - do_vacuum(s, ids, s->system_journal, "/var/log/journal/", &s->system_metrics); - do_vacuum(s, ids, s->runtime_journal, "/run/log/journal/", &s->runtime_metrics); + s->cached_space_limit = 0; + s->cached_space_available = 0; + s->cached_space_timestamp = 0; - s->cached_available_space_timestamp = 0; + return 0; } static void server_cache_machine_id(Server *s) { @@ -502,7 +508,7 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, unsigned if (journal_file_rotate_suggested(f, s->max_file_usec)) { log_debug("%s: Journal header limits reached or header out-of-date, rotating.", f->path); server_rotate(s); - server_vacuum(s); + server_vacuum(s, false, false); vacuumed = true; f = find_journal(s, uid); @@ -522,7 +528,7 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, unsigned } server_rotate(s); - server_vacuum(s); + server_vacuum(s, false, false); f = find_journal(s, uid); if (!f) @@ -867,6 +873,7 @@ void server_dispatch_message( int rl, r; _cleanup_free_ char *path = NULL; + uint64_t available = 0; char *c; assert(s); @@ -906,9 +913,8 @@ void server_dispatch_message( } } - rl = journal_rate_limit_test(s->rate_limit, path, - priority & LOG_PRIMASK, available_space(s, false)); - + (void) determine_space(s, false, false, &available, NULL); + rl = journal_rate_limit_test(s->rate_limit, path, priority & LOG_PRIMASK, available); if (rl == 0) return; @@ -923,16 +929,8 @@ finish: static int system_journal_open(Server *s, bool flush_requested) { - int r; const char *fn; - sd_id128_t machine; - char ids[33]; - - r = sd_id128_get_machine(&machine); - if (r < 0) - return log_error_errno(r, "Failed to get machine id: %m"); - - sd_id128_to_string(machine, ids); + int r; if (!s->system_journal && (s->storage == STORAGE_PERSISTENT || s->storage == STORAGE_AUTO) && @@ -948,15 +946,15 @@ static int system_journal_open(Server *s, bool flush_requested) { if (s->storage == STORAGE_PERSISTENT) (void) mkdir_p("/var/log/journal/", 0755); - fn = strjoina("/var/log/journal/", ids); + fn = strjoina("/var/log/journal/", SERVER_MACHINE_ID(s)); (void) mkdir(fn, 0755); fn = strjoina(fn, "/system.journal"); r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, s->compress, s->seal, &s->system_metrics, s->mmap, NULL, &s->system_journal); - - if (r >= 0) + if (r >= 0) { server_fix_perms(s, s->system_journal, 0); - else if (r < 0) { + (void) determine_space_for(s, &s->system_metrics, "/var/log/journal/", "System journal", true, true, NULL, NULL); + } else if (r < 0) { if (r != -ENOENT && r != -EROFS) log_warning_errno(r, "Failed to open system journal: %m"); @@ -967,7 +965,7 @@ static int system_journal_open(Server *s, bool flush_requested) { if (!s->runtime_journal && (s->storage != STORAGE_NONE)) { - fn = strjoina("/run/log/journal/", ids, "/system.journal", NULL); + fn = strjoina("/run/log/journal/", SERVER_MACHINE_ID(s), "/system.journal"); if (s->system_journal) { @@ -997,12 +995,12 @@ static int system_journal_open(Server *s, bool flush_requested) { return log_error_errno(r, "Failed to open runtime journal: %m"); } - if (s->runtime_journal) + if (s->runtime_journal) { server_fix_perms(s, s->runtime_journal, 0); + (void) determine_space_for(s, &s->runtime_metrics, "/run/log/journal/", "Runtime journal", true, true, NULL, NULL); + } } - available_space(s, true); - return r; } @@ -1023,7 +1021,7 @@ int server_flush_to_var(Server *s) { if (!s->runtime_journal) return 0; - system_journal_open(s, true); + (void) system_journal_open(s, true); if (!s->system_journal) return 0; @@ -1067,7 +1065,7 @@ int server_flush_to_var(Server *s) { } server_rotate(s); - server_vacuum(s); + server_vacuum(s, false, false); if (!s->system_journal) { log_notice("Didn't flush runtime journal since rotation of system journal wasn't successful."); @@ -1231,7 +1229,7 @@ static int dispatch_sigusr1(sd_event_source *es, const struct signalfd_siginfo * server_flush_to_var(s); server_sync(s); - server_vacuum(s); + server_vacuum(s, false, false); touch("/run/systemd/journal/flushed"); @@ -1245,7 +1243,7 @@ static int dispatch_sigusr2(sd_event_source *es, const struct signalfd_siginfo * log_info("Received request to rotate journal from PID %"PRIu32, si->ssi_pid); server_rotate(s); - server_vacuum(s); + server_vacuum(s, true, true); return 0; } @@ -1472,18 +1470,19 @@ int server_init(Server *s) { s->max_level_console = LOG_INFO; s->max_level_wall = LOG_EMERG; - memset(&s->system_metrics, 0xFF, sizeof(s->system_metrics)); - memset(&s->runtime_metrics, 0xFF, sizeof(s->runtime_metrics)); + journal_reset_metrics(&s->system_metrics); + journal_reset_metrics(&s->runtime_metrics); server_parse_config_file(s); server_parse_proc_cmdline(s); + if (!!s->rate_limit_interval ^ !!s->rate_limit_burst) { log_debug("Setting both rate limit interval and burst from "USEC_FMT",%u to 0,0", s->rate_limit_interval, s->rate_limit_burst); s->rate_limit_interval = s->rate_limit_burst = 0; } - mkdir_p("/run/systemd/journal", 0755); + (void) mkdir_p("/run/systemd/journal", 0755); s->user_journals = ordered_hashmap_new(NULL); if (!s->user_journals) @@ -1682,3 +1681,22 @@ void server_done(Server *s) { udev_unref(s->udev); } + +static const char* const storage_table[_STORAGE_MAX] = { + [STORAGE_AUTO] = "auto", + [STORAGE_VOLATILE] = "volatile", + [STORAGE_PERSISTENT] = "persistent", + [STORAGE_NONE] = "none" +}; + +DEFINE_STRING_TABLE_LOOKUP(storage, Storage); +DEFINE_CONFIG_PARSE_ENUM(config_parse_storage, storage, Storage, "Failed to parse storage setting"); + +static const char* const split_mode_table[_SPLIT_MAX] = { + [SPLIT_LOGIN] = "login", + [SPLIT_UID] = "uid", + [SPLIT_NONE] = "none", +}; + +DEFINE_STRING_TABLE_LOOKUP(split_mode, SplitMode); +DEFINE_CONFIG_PARSE_ENUM(config_parse_split_mode, split_mode, SplitMode, "Failed to parse split mode setting"); diff --git a/src/journal/journald-server.h b/src/journal/journald-server.h index d954c5190d..535c0ab9ab 100644 --- a/src/journal/journald-server.h +++ b/src/journal/journald-server.h @@ -100,8 +100,9 @@ typedef struct Server { unsigned n_forward_syslog_missed; usec_t last_warn_forward_syslog_missed; - uint64_t cached_available_space; - usec_t cached_available_space_timestamp; + uint64_t cached_space_available; + uint64_t cached_space_limit; + usec_t cached_space_timestamp; uint64_t var_available_timestamp; @@ -141,6 +142,8 @@ typedef struct Server { char *cgroup_root; } Server; +#define SERVER_MACHINE_ID(s) ((s)->machine_id_field + strlen("_MACHINE_ID=")) + #define N_IOVEC_META_FIELDS 20 #define N_IOVEC_KERNEL_FIELDS 64 #define N_IOVEC_UDEV_FIELDS 32 @@ -166,7 +169,7 @@ void server_fix_perms(Server *s, JournalFile *f, uid_t uid); int server_init(Server *s); void server_done(Server *s); void server_sync(Server *s); -void server_vacuum(Server *s); +int server_vacuum(Server *s, bool verbose, bool patch_min_use); void server_rotate(Server *s); int server_schedule_sync(Server *s, int priority); int server_flush_to_var(Server *s); diff --git a/src/journal/journald.c b/src/journal/journald.c index b2624c6d28..83236ceba9 100644 --- a/src/journal/journald.c +++ b/src/journal/journald.c @@ -21,8 +21,8 @@ #include -#include "systemd/sd-messages.h" -#include "systemd/sd-daemon.h" +#include "sd-messages.h" +#include "sd-daemon.h" #include "journal-authenticate.h" #include "journald-server.h" @@ -54,7 +54,7 @@ int main(int argc, char *argv[]) { if (r < 0) goto finish; - server_vacuum(&server); + server_vacuum(&server, false, false); server_flush_to_var(&server); server_flush_dev_kmsg(&server); @@ -82,7 +82,7 @@ int main(int argc, char *argv[]) { if (server.oldest_file_usec + server.max_retention_usec < n) { log_info("Retention time reached."); server_rotate(&server); - server_vacuum(&server); + server_vacuum(&server, false, false); continue; } diff --git a/src/journal/journald.conf b/src/journal/journald.conf index 47eefe91c1..7beb96c671 100644 --- a/src/journal/journald.conf +++ b/src/journal/journald.conf @@ -22,9 +22,11 @@ #SystemMaxUse= #SystemKeepFree= #SystemMaxFileSize= +#SystemMaxFiles=100 #RuntimeMaxUse= #RuntimeKeepFree= #RuntimeMaxFileSize= +#RuntimeMaxFiles=100 #MaxRetentionSec= #MaxFileSec=1month #ForwardToSyslog=no diff --git a/src/journal/test-journal-interleaving.c b/src/journal/test-journal-interleaving.c index adefa1b026..8069339c1f 100644 --- a/src/journal/test-journal-interleaving.c +++ b/src/journal/test-journal-interleaving.c @@ -197,7 +197,7 @@ static void test_skip(void (*setup)(void)) { if (arg_keep) log_info("Not removing %s", t); else { - journal_directory_vacuum(".", 3000000, 0, NULL, true); + journal_directory_vacuum(".", 3000000, 0, 0, NULL, true); assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0); } @@ -282,7 +282,7 @@ static void test_sequence_numbers(void) { if (arg_keep) log_info("Not removing %s", t); else { - journal_directory_vacuum(".", 3000000, 0, NULL, true); + journal_directory_vacuum(".", 3000000, 0, 0, NULL, true); assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0); } diff --git a/src/journal/test-journal.c b/src/journal/test-journal.c index caaab258c9..01d4bc968a 100644 --- a/src/journal/test-journal.c +++ b/src/journal/test-journal.c @@ -116,7 +116,7 @@ static void test_non_empty(void) { if (arg_keep) log_info("Not removing %s", t); else { - journal_directory_vacuum(".", 3000000, 0, NULL, true); + journal_directory_vacuum(".", 3000000, 0, 0, NULL, true); assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0); } @@ -155,7 +155,7 @@ static void test_empty(void) { if (arg_keep) log_info("Not removing %s", t); else { - journal_directory_vacuum(".", 3000000, 0, NULL, true); + journal_directory_vacuum(".", 3000000, 0, 0, NULL, true); assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0); } -- cgit v1.2.3-54-g00ecf