journal: rework vacuuming logic

Implement a maximum limit on number of journal files to keep around.
Enforcing a limit is useful on this since our performance when viewing
pays a heavy penalty for each journal file to interleve. This setting is
turned on now by default, and set to 100.

Also, actully implement what 348ced909724a1331b85d57aede80a102a00e428
promised: use whatever we find on disk at startup as lower bound on how
much disk space we can use. That commit introduced some provisions to
implement this, but actually never did.

This also adds "journalctl --vacuum-files=" to vacuum files on disk by
their number explicitly.

12 files changed, 330 insertions, 224 deletions

diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c
index 085d5f346d..1071c6d6d7 100644
--- a/src/journal/journal-file.c
+++ b/src/journal/journal-file.c
@@ -49,6 +49,9 @@
 #define DEFAULT_MAX_USE_LOWER (1ULL*1024ULL*1024ULL)           /* 1 MiB */
 #define DEFAULT_MAX_USE_UPPER (4ULL*1024ULL*1024ULL*1024ULL)   /* 4 GiB */
 
+/* This is the default minimal use limit, how much we'll use even if keep_free suggests otherwise. */
+#define DEFAULT_MIN_USE (1ULL*1024ULL*1024ULL)                 /* 1 MiB */
+
 /* This is the upper bound if we deduce max_size from max_use */
 #define DEFAULT_MAX_SIZE_UPPER (128ULL*1024ULL*1024ULL)        /* 128 MiB */
 
@@ -60,6 +63,9 @@
  * size */
 #define DEFAULT_KEEP_FREE (1024ULL*1024ULL)                    /* 1 MB */
 
+/* This is the default maximum number of journal files to keep around. */
+#define DEFAULT_N_MAX_FILES (100)
+
 /* n_data was the first entry we added after the initial file format design */
 #define HEADER_SIZE_MIN ALIGN64(offsetof(Header, n_data))
 
@@ -2957,16 +2963,35 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6
         return r;
 }
 
+void journal_reset_metrics(JournalMetrics *m) {
+        assert(m);
+
+        /* Set everything to "pick automatic values". */
+
+        *m = (JournalMetrics) {
+                .min_use = (uint64_t) -1,
+                .max_use = (uint64_t) -1,
+                .min_size = (uint64_t) -1,
+                .max_size = (uint64_t) -1,
+                .keep_free = (uint64_t) -1,
+                .n_max_files = (uint64_t) -1,
+        };
+}
+
 void journal_default_metrics(JournalMetrics *m, int fd) {
-        uint64_t fs_size = 0;
+        char a[FORMAT_BYTES_MAX], b[FORMAT_BYTES_MAX], c[FORMAT_BYTES_MAX], d[FORMAT_BYTES_MAX], e[FORMAT_BYTES_MAX];
         struct statvfs ss;
-        char a[FORMAT_BYTES_MAX], b[FORMAT_BYTES_MAX], c[FORMAT_BYTES_MAX], d[FORMAT_BYTES_MAX];
+        uint64_t fs_size;
 
         assert(m);
         assert(fd >= 0);
 
         if (fstatvfs(fd, &ss) >= 0)
                 fs_size = ss.f_frsize * ss.f_blocks;
+        else {
+                log_debug_errno(errno, "Failed to detremine disk size: %m");
+                fs_size = 0;
+        }
 
         if (m->max_use == (uint64_t) -1) {
 
@@ -2983,10 +3008,16 @@ void journal_default_metrics(JournalMetrics *m, int fd) {
         } else {
                 m->max_use = PAGE_ALIGN(m->max_use);
 
-                if (m->max_use < JOURNAL_FILE_SIZE_MIN*2)
+                if (m->max_use != 0 && m->max_use < JOURNAL_FILE_SIZE_MIN*2)
                         m->max_use = JOURNAL_FILE_SIZE_MIN*2;
         }
 
+        if (m->min_use == (uint64_t) -1)
+                m->min_use = DEFAULT_MIN_USE;
+
+        if (m->min_use > m->max_use)
+                m->min_use = m->max_use;
+
         if (m->max_size == (uint64_t) -1) {
                 m->max_size = PAGE_ALIGN(m->max_use / 8); /* 8 chunks */
 
@@ -2995,11 +3026,13 @@ void journal_default_metrics(JournalMetrics *m, int fd) {
         } else
                 m->max_size = PAGE_ALIGN(m->max_size);
 
-        if (m->max_size < JOURNAL_FILE_SIZE_MIN)
-                m->max_size = JOURNAL_FILE_SIZE_MIN;
+        if (m->max_size != 0) {
+                if (m->max_size < JOURNAL_FILE_SIZE_MIN)
+                        m->max_size = JOURNAL_FILE_SIZE_MIN;
 
-        if (m->max_size*2 > m->max_use)
-                m->max_use = m->max_size*2;
+                if (m->max_use != 0 && m->max_size*2 > m->max_use)
+                        m->max_use = m->max_size*2;
+        }
 
         if (m->min_size == (uint64_t) -1)
                 m->min_size = JOURNAL_FILE_SIZE_MIN;
@@ -3009,7 +3042,7 @@ void journal_default_metrics(JournalMetrics *m, int fd) {
                 if (m->min_size < JOURNAL_FILE_SIZE_MIN)
                         m->min_size = JOURNAL_FILE_SIZE_MIN;
 
-                if (m->min_size > m->max_size)
+                if (m->max_size != 0 && m->min_size > m->max_size)
                         m->max_size = m->min_size;
         }
 
@@ -3025,11 +3058,16 @@ void journal_default_metrics(JournalMetrics *m, int fd) {
                         m->keep_free = DEFAULT_KEEP_FREE;
         }
 
-        log_debug("Fixed max_use=%s max_size=%s min_size=%s keep_free=%s",
-                  format_bytes(a, sizeof(a), m->max_use),
-                  format_bytes(b, sizeof(b), m->max_size),
-                  format_bytes(c, sizeof(c), m->min_size),
-                  format_bytes(d, sizeof(d), m->keep_free));
+        if (m->n_max_files == (uint64_t) -1)
+                m->n_max_files = DEFAULT_N_MAX_FILES;
+
+        log_debug("Fixed min_use=%s max_use=%s max_size=%s min_size=%s keep_free=%s n_max_files=%" PRIu64,
+                  format_bytes(a, sizeof(a), m->min_use),
+                  format_bytes(b, sizeof(b), m->max_use),
+                  format_bytes(c, sizeof(c), m->max_size),
+                  format_bytes(d, sizeof(d), m->min_size),
+                  format_bytes(e, sizeof(e), m->keep_free),
+                  m->n_max_files);
 }
 
 int journal_file_get_cutoff_realtime_usec(JournalFile *f, usec_t *from, usec_t *to) {
diff --git a/src/journal/journal-file.h b/src/journal/journal-file.h
index 52e6b7a261..f2c07356c8 100644
--- a/src/journal/journal-file.h
+++ b/src/journal/journal-file.h
@@ -36,11 +36,13 @@
 #include "hashmap.h"
 
 typedef struct JournalMetrics {
-        uint64_t max_use;
-        uint64_t use;
-        uint64_t max_size;
-        uint64_t min_size;
-        uint64_t keep_free;
+        /* For all these: -1 means "pick automatically", and 0 means "no limit enforced" */
+        uint64_t max_size;     /* how large journal files grow at max */
+        uint64_t min_size;     /* how large journal files grow at least */
+        uint64_t max_use;      /* how much disk space to use in total at max, keep_free permitting */
+        uint64_t min_use;      /* how much disk space to use in total at least, even if keep_free says not to */
+        uint64_t keep_free;    /* how much to keep free on disk */
+        uint64_t n_max_files;  /* how many files to keep around at max */
 } JournalMetrics;
 
 typedef enum direction {
@@ -223,6 +225,7 @@ int journal_file_rotate(JournalFile **f, bool compress, bool seal);
 
 void journal_file_post_change(JournalFile *f);
 
+void journal_reset_metrics(JournalMetrics *m);
 void journal_default_metrics(JournalMetrics *m, int fd);
 
 int journal_file_get_cutoff_realtime_usec(JournalFile *f, usec_t *from, usec_t *to);
diff --git a/src/journal/journal-vacuum.c b/src/journal/journal-vacuum.c
index 394f7be46b..a394066cb4 100644
--- a/src/journal/journal-vacuum.c
+++ b/src/journal/journal-vacuum.c
@@ -140,22 +140,24 @@ static int journal_file_empty(int dir_fd, const char *name) {
 int journal_directory_vacuum(
                 const char *directory,
                 uint64_t max_use,
+                uint64_t n_max_files,
                 usec_t max_retention_usec,
                 usec_t *oldest_usec,
                 bool verbose) {
 
         _cleanup_closedir_ DIR *d = NULL;
-        int r = 0;
         struct vacuum_info *list = NULL;
-        unsigned n_list = 0, i;
+        unsigned n_list = 0, i, n_active_files = 0;
         size_t n_allocated = 0;
         uint64_t sum = 0, freed = 0;
         usec_t retention_limit = 0;
         char sbytes[FORMAT_BYTES_MAX];
+        struct dirent *de;
+        int r;
 
         assert(directory);
 
-        if (max_use <= 0 && max_retention_usec <= 0)
+        if (max_use <= 0 && max_retention_usec <= 0 && n_max_files <= 0)
                 return 0;
 
         if (max_retention_usec > 0) {
@@ -170,27 +172,20 @@ int journal_directory_vacuum(
         if (!d)
                 return -errno;
 
-        for (;;) {
-                struct dirent *de;
-                size_t q;
-                struct stat st;
-                char *p;
+        FOREACH_DIRENT_ALL(de, d, r = -errno; goto finish) {
+
                 unsigned long long seqnum = 0, realtime;
+                _cleanup_free_ char *p = NULL;
                 sd_id128_t seqnum_id;
                 bool have_seqnum;
+                uint64_t size;
+                struct stat st;
+                size_t q;
 
-                errno = 0;
-                de = readdir(d);
-                if (!de && errno != 0) {
-                        r = -errno;
-                        goto finish;
-                }
-
-                if (!de)
-                        break;
-
-                if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0)
+                if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) {
+                        log_debug_errno(errno, "Failed to stat file %s while vacuuming, ignoring: %m", de->d_name);
                         continue;
+                }
 
                 if (!S_ISREG(st.st_mode))
                         continue;
@@ -199,15 +194,20 @@ int journal_directory_vacuum(
 
                 if (endswith(de->d_name, ".journal")) {
 
-                        /* Vacuum archived files */
+                        /* Vacuum archived files. Active files are
+                         * left around */
 
-                        if (q < 1 + 32 + 1 + 16 + 1 + 16 + 8)
+                        if (q < 1 + 32 + 1 + 16 + 1 + 16 + 8) {
+                                n_active_files++;
                                 continue;
+                        }
 
                         if (de->d_name[q-8-16-1] != '-' ||
                             de->d_name[q-8-16-1-16-1] != '-' ||
-                            de->d_name[q-8-16-1-16-1-32-1] != '@')
+                            de->d_name[q-8-16-1-16-1-32-1] != '@') {
+                                n_active_files++;
                                 continue;
+                        }
 
                         p = strdup(de->d_name);
                         if (!p) {
@@ -218,11 +218,13 @@ int journal_directory_vacuum(
                         de->d_name[q-8-16-1-16-1] = 0;
                         if (sd_id128_from_string(de->d_name + q-8-16-1-16-1-32, &seqnum_id) < 0) {
                                 free(p);
+                                n_active_files++;
                                 continue;
                         }
 
                         if (sscanf(de->d_name + q-8-16-1-16, "%16llx-%16llx.journal", &seqnum, &realtime) != 2) {
                                 free(p);
+                                n_active_files++;
                                 continue;
                         }
 
@@ -233,12 +235,16 @@ int journal_directory_vacuum(
 
                         /* Vacuum corrupted files */
 
-                        if (q < 1 + 16 + 1 + 16 + 8 + 1)
+                        if (q < 1 + 16 + 1 + 16 + 8 + 1) {
+                                n_active_files ++;
                                 continue;
+                        }
 
                         if (de->d_name[q-1-8-16-1] != '-' ||
-                            de->d_name[q-1-8-16-1-16-1] != '@')
+                            de->d_name[q-1-8-16-1-16-1] != '@') {
+                                n_active_files ++;
                                 continue;
+                        }
 
                         p = strdup(de->d_name);
                         if (!p) {
@@ -248,54 +254,68 @@ int journal_directory_vacuum(
 
                         if (sscanf(de->d_name + q-1-8-16-1-16, "%16llx-%16llx.journal~", &realtime, &tmp) != 2) {
                                 free(p);
+                                n_active_files ++;
                                 continue;
                         }
 
                         have_seqnum = false;
-                } else
-                        /* We do not vacuum active files or unknown files! */
+                } else {
+                        /* We do not vacuum unknown files! */
+                        log_debug("Not vacuuming unknown file %s.", de->d_name);
                         continue;
+                }
 
-                if (journal_file_empty(dirfd(d), p)) {
-                        /* Always vacuum empty non-online files. */
+                size = 512UL * (uint64_t) st.st_blocks;
 
-                        uint64_t size = 512UL * (uint64_t) st.st_blocks;
+                r = journal_file_empty(dirfd(d), p);
+                if (r < 0) {
+                        log_debug_errno(r, "Failed check if %s is empty, ignoring: %m", p);
+                        continue;
+                }
+                if (r > 0) {
+                        /* Always vacuum empty non-online files. */
 
                         if (unlinkat(dirfd(d), p, 0) >= 0) {
-                                log_full(verbose ? LOG_INFO : LOG_DEBUG, "Deleted empty archived journal %s/%s (%s).", directory, p, format_bytes(sbytes, sizeof(sbytes), size));
+
+                                log_full(verbose ? LOG_INFO : LOG_DEBUG,
+                                         "Deleted empty archived journal %s/%s (%s).", directory, p, format_bytes(sbytes, sizeof(sbytes), size));
+
                                 freed += size;
                         } else if (errno != ENOENT)
                                 log_warning_errno(errno, "Failed to delete empty archived journal %s/%s: %m", directory, p);
 
-                        free(p);
                         continue;
                 }
 
-                patch_realtime(directory, p, &st, &realtime);
+                patch_realtime(dirfd(d), p, &st, &realtime);
 
                 if (!GREEDY_REALLOC(list, n_allocated, n_list + 1)) {
-                        free(p);
                         r = -ENOMEM;
                         goto finish;
                 }
 
                 list[n_list].filename = p;
-                list[n_list].usage = 512UL * (uint64_t) st.st_blocks;
+                list[n_list].usage = size;
                 list[n_list].seqnum = seqnum;
                 list[n_list].realtime = realtime;
                 list[n_list].seqnum_id = seqnum_id;
                 list[n_list].have_seqnum = have_seqnum;
-
-                sum += list[n_list].usage;
-
                 n_list ++;
+
+                p = NULL;
+                sum += size;
         }
 
         qsort_safe(list, n_list, sizeof(struct vacuum_info), vacuum_compare);
 
         for (i = 0; i < n_list; i++) {
+                unsigned left;
+
+                left = n_active_files + n_list - i;
+
                 if ((max_retention_usec <= 0 || list[i].realtime >= retention_limit) &&
-                    (max_use <= 0 || sum <= max_use))
+                    (max_use <= 0 || sum <= max_use) &&
+                    (n_max_files <= 0 || left <= n_max_files))
                         break;
 
                 if (unlinkat(dirfd(d), list[i].filename, 0) >= 0) {
@@ -314,6 +334,8 @@ int journal_directory_vacuum(
         if (oldest_usec && i < n_list && (*oldest_usec == 0 || list[i].realtime < *oldest_usec))
                 *oldest_usec = list[i].realtime;
 
+        r = 0;
+
 finish:
         for (i = 0; i < n_list; i++)
                 free(list[i].filename);
diff --git a/src/journal/journal-vacuum.h b/src/journal/journal-vacuum.h
index c45cc31d0e..49ab90af91 100644
--- a/src/journal/journal-vacuum.h
+++ b/src/journal/journal-vacuum.h
@@ -21,5 +21,9 @@
   along with systemd; If not, see <http://www.gnu.org/licenses/>.
 ***/
 
+#include <inttypes.h>
+#include <stdbool.h>
 
-int journal_directory_vacuum(const char *directory, uint64_t max_use, usec_t max_retention_usec, usec_t *oldest_usec, bool vacuum);
+#include "time-util.h"
+
+int journal_directory_vacuum(const char *directory, uint64_t max_use, uint64_t n_max_files, usec_t max_retention_usec, usec_t *oldest_usec, bool verbose);
diff --git a/src/journal/journalctl.c b/src/journal/journalctl.c
index 28ccb80661..d9851db36c 100644
--- a/src/journal/journalctl.c
+++ b/src/journal/journalctl.c
@@ -106,8 +106,9 @@ static bool arg_reverse = false;
 static int arg_journal_type = 0;
 static const char *arg_root = NULL;
 static const char *arg_machine = NULL;
-static uint64_t arg_vacuum_size = (uint64_t) -1;
-static usec_t arg_vacuum_time = USEC_INFINITY;
+static uint64_t arg_vacuum_size = 0;
+static uint64_t arg_vacuum_n_files = 0;
+static usec_t arg_vacuum_time = 0;
 
 static enum {
         ACTION_SHOW,
@@ -235,7 +236,8 @@ static void help(void) {
                "     --new-id128           Generate a new 128-bit ID\n"
                "     --disk-usage          Show total disk usage of all journal files\n"
                "     --vacuum-size=BYTES   Reduce disk usage below specified size\n"
-               "     --vacuum-time=TIME    Remove journal files older than specified date\n"
+               "     --vacuum-files=INT    Leave only the specified number of journal files\n"
+               "     --vacuum-time=TIME    Remove journal files older than specified time\n"
                "     --flush               Flush all journal data from /run into /var\n"
                "     --rotate              Request immediate rotation of the journal files\n"
                "     --header              Show journal header information\n"
@@ -281,6 +283,7 @@ static int parse_argv(int argc, char *argv[]) {
                 ARG_FLUSH,
                 ARG_ROTATE,
                 ARG_VACUUM_SIZE,
+                ARG_VACUUM_FILES,
                 ARG_VACUUM_TIME,
         };
 
@@ -335,6 +338,7 @@ static int parse_argv(int argc, char *argv[]) {
                 { "flush",          no_argument,       NULL, ARG_FLUSH          },
                 { "rotate",         no_argument,       NULL, ARG_ROTATE         },
                 { "vacuum-size",    required_argument, NULL, ARG_VACUUM_SIZE    },
+                { "vacuum-files",   required_argument, NULL, ARG_VACUUM_FILES   },
                 { "vacuum-time",    required_argument, NULL, ARG_VACUUM_TIME    },
                 {}
         };
@@ -540,6 +544,16 @@ static int parse_argv(int argc, char *argv[]) {
                         arg_action = ACTION_VACUUM;
                         break;
 
+                case ARG_VACUUM_FILES:
+                        r = safe_atou64(optarg, &arg_vacuum_n_files);
+                        if (r < 0) {
+                                log_error("Failed to parse vacuum files: %s", optarg);
+                                return r;
+                        }
+
+                        arg_action = ACTION_VACUUM;
+                        break;
+
                 case ARG_VACUUM_TIME:
                         r = parse_sec(optarg, &arg_vacuum_time);
                         if (r < 0) {
@@ -1929,9 +1943,9 @@ int main(int argc, char *argv[]) {
                         if (d->is_root)
                                 continue;
 
-                        q = journal_directory_vacuum(d->path, arg_vacuum_size, arg_vacuum_time, NULL, true);
+                        q = journal_directory_vacuum(d->path, arg_vacuum_size, arg_vacuum_n_files, arg_vacuum_time, NULL, true);
                         if (q < 0) {
-                                log_error_errno(q, "Failed to vacuum: %m");
+                                log_error_errno(q, "Failed to vacuum %s: %m", d->path);
                                 r = q;
                         }
                 }
diff --git a/src/journal/journald-gperf.gperf b/src/journal/journald-gperf.gperf
index bf7c773009..c154610c54 100644
--- a/src/journal/journald-gperf.gperf
+++ b/src/journal/journald-gperf.gperf
@@ -24,9 +24,11 @@ Journal.RateLimitBurst,     config_parse_unsigned,   0, offsetof(Server, rate_li
 Journal.SystemMaxUse,       config_parse_iec_uint64, 0, offsetof(Server, system_metrics.max_use)
 Journal.SystemMaxFileSize,  config_parse_iec_uint64, 0, offsetof(Server, system_metrics.max_size)
 Journal.SystemKeepFree,     config_parse_iec_uint64, 0, offsetof(Server, system_metrics.keep_free)
+Journal.SystemMaxFiles,     config_parse_uint64,     0, offsetof(Server, system_metrics.n_max_files)
 Journal.RuntimeMaxUse,      config_parse_iec_uint64, 0, offsetof(Server, runtime_metrics.max_use)
 Journal.RuntimeMaxFileSize, config_parse_iec_uint64, 0, offsetof(Server, runtime_metrics.max_size)
 Journal.RuntimeKeepFree,    config_parse_iec_uint64, 0, offsetof(Server, runtime_metrics.keep_free)
+Journal.RuntimeMaxFiles,    config_parse_uint64,     0, offsetof(Server, runtime_metrics.n_max_files)
 Journal.MaxRetentionSec,    config_parse_sec,        0, offsetof(Server, max_retention_usec)
 Journal.MaxFileSec,         config_parse_sec,        0, offsetof(Server, max_file_usec)
 Journal.ForwardToSyslog,    config_parse_bool,       0, offsetof(Server, forward_to_syslog)
diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c
index 296e28e6e3..fb172b7f5d 100644
--- a/src/journal/journald-server.c
+++ b/src/journal/journald-server.c
@@ -19,45 +19,44 @@
   along with systemd; If not, see <http://www.gnu.org/licenses/>.
 ***/
 
-#include <sys/signalfd.h>
-#include <sys/ioctl.h>
 #include <linux/sockios.h>
-#include <sys/statvfs.h>
-#include <sys/mman.h>
-
 #ifdef HAVE_SELINUX
 #include <selinux/selinux.h>
 #endif
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/signalfd.h>
+#include <sys/statvfs.h>
 
-#include <libudev.h>
-
+#include "libudev.h"
+#include "sd-daemon.h"
 #include "sd-journal.h"
 #include "sd-messages.h"
-#include "sd-daemon.h"
-#include "mkdir.h"
-#include "rm-rf.h"
-#include "hashmap.h"
-#include "journal-file.h"
-#include "socket-util.h"
+
+#include "acl-util.h"
 #include "cgroup-util.h"
-#include "missing.h"
 #include "conf-parser.h"
-#include "selinux-util.h"
-#include "acl-util.h"
 #include "formats-util.h"
-#include "process-util.h"
+#include "hashmap.h"
 #include "hostname-util.h"
+#include "missing.h"
+#include "mkdir.h"
+#include "process-util.h"
+#include "rm-rf.h"
+#include "selinux-util.h"
 #include "signal-util.h"
+#include "socket-util.h"
+#include "journal-authenticate.h"
+#include "journal-file.h"
 #include "journal-internal.h"
 #include "journal-vacuum.h"
-#include "journal-authenticate.h"
-#include "journald-rate-limit.h"
+#include "journald-audit.h"
 #include "journald-kmsg.h"
-#include "journald-syslog.h"
-#include "journald-stream.h"
 #include "journald-native.h"
-#include "journald-audit.h"
+#include "journald-rate-limit.h"
 #include "journald-server.h"
+#include "journald-stream.h"
+#include "journald-syslog.h"
 
 #define USER_JOURNALS_MAX 1024
 
@@ -66,88 +65,61 @@
 #define DEFAULT_RATE_LIMIT_BURST 1000
 #define DEFAULT_MAX_FILE_USEC USEC_PER_MONTH
 
-#define RECHECK_AVAILABLE_SPACE_USEC (30*USEC_PER_SEC)
-
-static const char* const storage_table[_STORAGE_MAX] = {
-        [STORAGE_AUTO] = "auto",
-        [STORAGE_VOLATILE] = "volatile",
-        [STORAGE_PERSISTENT] = "persistent",
-        [STORAGE_NONE] = "none"
-};
-
-DEFINE_STRING_TABLE_LOOKUP(storage, Storage);
-DEFINE_CONFIG_PARSE_ENUM(config_parse_storage, storage, Storage, "Failed to parse storage setting");
+#define RECHECK_SPACE_USEC (30*USEC_PER_SEC)
 
-static const char* const split_mode_table[_SPLIT_MAX] = {
-        [SPLIT_LOGIN] = "login",
-        [SPLIT_UID] = "uid",
-        [SPLIT_NONE] = "none",
-};
-
-DEFINE_STRING_TABLE_LOOKUP(split_mode, SplitMode);
-DEFINE_CONFIG_PARSE_ENUM(config_parse_split_mode, split_mode, SplitMode, "Failed to parse split mode setting");
-
-static uint64_t available_space(Server *s, bool verbose) {
-        char ids[33];
-        _cleanup_free_ char *p = NULL;
-        sd_id128_t machine;
-        struct statvfs ss;
-        uint64_t sum = 0, ss_avail = 0, avail = 0;
-        int r;
+static int determine_space_for(
+                Server *s,
+                JournalMetrics *metrics,
+                const char *path,
+                const char *name,
+                bool verbose,
+                bool patch_min_use,
+                uint64_t *available,
+                uint64_t *limit) {
+
+        uint64_t sum = 0, ss_avail, avail;
         _cleanup_closedir_ DIR *d = NULL;
+        struct dirent *de;
+        struct statvfs ss;
+        const char *p;
         usec_t ts;
-        const char *f;
-        JournalMetrics *m;
 
-        ts = now(CLOCK_MONOTONIC);
-
-        if (s->cached_available_space_timestamp + RECHECK_AVAILABLE_SPACE_USEC > ts
-            && !verbose)
-                return s->cached_available_space;
+        assert(s);
+        assert(metrics);
+        assert(path);
+        assert(name);
 
-        r = sd_id128_get_machine(&machine);
-        if (r < 0)
-                return 0;
+        ts = now(CLOCK_MONOTONIC);
 
-        if (s->system_journal) {
-                f = "/var/log/journal/";
-                m = &s->system_metrics;
-        } else {
-                f = "/run/log/journal/";
-                m = &s->runtime_metrics;
-        }
+        if (!verbose && s->cached_space_timestamp + RECHECK_SPACE_USEC > ts) {
 
-        assert(m);
+                if (available)
+                        *available = s->cached_space_available;
+                if (limit)
+                        *limit = s->cached_space_limit;
 
-        p = strappend(f, sd_id128_to_string(machine, ids));
-        if (!p)
                 return 0;
+        }
 
+        p = strjoina(path, SERVER_MACHINE_ID(s));
         d = opendir(p);
         if (!d)
-                return 0;
+                return log_full_errno(errno == ENOENT ? LOG_DEBUG : LOG_ERR, errno, "Failed to open %s: %m", p);
 
         if (fstatvfs(dirfd(d), &ss) < 0)
-                return 0;
+                return log_error_errno(errno, "Failed to fstatvfs(%s): %m", p);
 
-        for (;;) {
+        FOREACH_DIRENT_ALL(de, d, break) {
                 struct stat st;
-                struct dirent *de;
-
-                errno = 0;
-                de = readdir(d);
-                if (!de && errno != 0)
-                        return 0;
-
-                if (!de)
-                        break;
 
                 if (!endswith(de->d_name, ".journal") &&
                     !endswith(de->d_name, ".journal~"))
                         continue;
 
-                if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0)
+                if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) {
+                        log_debug_errno(errno, "Failed to stat %s/%s, ignoring: %m", p, de->d_name);
                         continue;
+                }
 
                 if (!S_ISREG(st.st_mode))
                         continue;
@@ -155,39 +127,66 @@ static uint64_t available_space(Server *s, bool verbose) {
                 sum += (uint64_t) st.st_blocks * 512UL;
         }
 
-        ss_avail = ss.f_bsize * ss.f_bavail;
+        /* If request, then let's bump the min_use limit to the
+         * current usage on disk. We do this when starting up and
+         * first opening the journal files. This way sudden spikes in
+         * disk usage will not cause journald to vacuum files without
+         * bounds. Note that this means that only a restart of
+         * journald will make it reset this value. */
 
-        /* If we reached a high mark, we will always allow this much
-         * again, unless usage goes above max_use. This watermark
-         * value is cached so that we don't give up space on pressure,
-         * but hover below the maximum usage. */
+        if (patch_min_use)
+                metrics->min_use = MAX(metrics->min_use, sum);
 
-        if (m->use < sum)
-                m->use = sum;
-
-        avail = LESS_BY(ss_avail, m->keep_free);
+        ss_avail = ss.f_bsize * ss.f_bavail;
+        avail = LESS_BY(ss_avail, metrics->keep_free);
 
-        s->cached_available_space = LESS_BY(MIN(m->max_use, avail), sum);
-        s->cached_available_space_timestamp = ts;
+        s->cached_space_limit = MIN(MAX(sum + avail, metrics->min_use), metrics->max_use);
+        s->cached_space_available = LESS_BY(s->cached_space_limit, sum);
+        s->cached_space_timestamp = ts;
 
         if (verbose) {
                 char    fb1[FORMAT_BYTES_MAX], fb2[FORMAT_BYTES_MAX], fb3[FORMAT_BYTES_MAX],
-                        fb4[FORMAT_BYTES_MAX], fb5[FORMAT_BYTES_MAX];
+                        fb4[FORMAT_BYTES_MAX], fb5[FORMAT_BYTES_MAX], fb6[FORMAT_BYTES_MAX];
 
                 server_driver_message(s, SD_MESSAGE_JOURNAL_USAGE,
-                                      "%s is currently using %s.\n"
+                                      "%s (%s) is currently using %s.\n"
                                       "Maximum allowed usage is set to %s.\n"
                                       "Leaving at least %s free (of currently available %s of space).\n"
-                                      "Enforced usage limit is thus %s.",
-                                      s->system_journal ? "Permanent journal (/var/log/journal/)" : "Runtime journal (/run/log/journal/)",
+                                      "Enforced usage limit is thus %s, of which %s are still available.",
+                                      name, path,
                                       format_bytes(fb1, sizeof(fb1), sum),
-                                      format_bytes(fb2, sizeof(fb2), m->max_use),
-                                      format_bytes(fb3, sizeof(fb3), m->keep_free),
+                                      format_bytes(fb2, sizeof(fb2), metrics->max_use),
+                                      format_bytes(fb3, sizeof(fb3), metrics->keep_free),
                                       format_bytes(fb4, sizeof(fb4), ss_avail),
-                                      format_bytes(fb5, sizeof(fb5), s->cached_available_space + sum));
+                                      format_bytes(fb5, sizeof(fb5), s->cached_space_limit),
+                                      format_bytes(fb6, sizeof(fb6), s->cached_space_available));
+        }
+
+        if (available)
+                *available = s->cached_space_available;
+        if (limit)
+                *limit = s->cached_space_limit;
+
+        return 1;
+}
+
+static int determine_space(Server *s, bool verbose, bool patch_min_use, uint64_t *available, uint64_t *limit) {
+        JournalMetrics *metrics;
+        const char *path, *name;
+
+        assert(s);
+
+        if (s->system_journal) {
+                path = "/var/log/journal/";
+                metrics = &s->system_metrics;
+                name = "System journal";
+        } else {
+                path = "/run/log/journal/";
+                metrics = &s->runtime_metrics;
+                name = "Runtime journal";
         }
 
-        return s->cached_available_space;
+        return determine_space_for(s, metrics, path, name, verbose, patch_min_use, available, limit);
 }
 
 void server_fix_perms(Server *s, JournalFile *f, uid_t uid) {
@@ -326,8 +325,8 @@ void server_rotate(Server *s) {
 
         log_debug("Rotating...");
 
-        do_rotate(s, &s->runtime_journal, "runtime", false, 0);
-        do_rotate(s, &s->system_journal, "system", s->seal, 0);
+        (void) do_rotate(s, &s->runtime_journal, "runtime", false, 0);
+        (void) do_rotate(s, &s->system_journal, "system", s->seal, 0);
 
         ORDERED_HASHMAP_FOREACH_KEY(f, k, s->user_journals, i) {
                 r = do_rotate(s, &f, "user", s->seal, PTR_TO_UINT32(k));
@@ -368,43 +367,50 @@ void server_sync(Server *s) {
 
 static void do_vacuum(
                 Server *s,
-                const char *id,
                 JournalFile *f,
-                const char* path,
-                JournalMetrics *metrics) {
+                JournalMetrics *metrics,
+                const char *path,
+                const char *name,
+                bool verbose,
+                bool patch_min_use) {
 
         const char *p;
+        uint64_t limit;
         int r;
 
+        assert(s);
+        assert(metrics);
+        assert(path);
+        assert(name);
+
         if (!f)
                 return;
 
-        p = strjoina(path, id);
-        r = journal_directory_vacuum(p, metrics->max_use, s->max_retention_usec, &s->oldest_file_usec, false);
+        p = strjoina(path, SERVER_MACHINE_ID(s));
+
+        limit = metrics->max_use;
+        (void) determine_space_for(s, metrics, path, name, verbose, patch_min_use, NULL, &limit);
+
+        r = journal_directory_vacuum(p, limit, metrics->n_max_files, s->max_retention_usec, &s->oldest_file_usec,  verbose);
         if (r < 0 && r != -ENOENT)
-                log_error_errno(r, "Failed to vacuum %s: %m", p);
+                log_warning_errno(r, "Failed to vacuum %s, ignoring: %m", p);
 }
 
-void server_vacuum(Server *s) {
-        char ids[33];
-        sd_id128_t machine;
-        int r;
+int server_vacuum(Server *s, bool verbose, bool patch_min_use) {
+        assert(s);
 
         log_debug("Vacuuming...");
 
         s->oldest_file_usec = 0;
 
-        r = sd_id128_get_machine(&machine);
-        if (r < 0) {
-                log_error_errno(r, "Failed to get machine ID: %m");
-                return;
-        }
-        sd_id128_to_string(machine, ids);
+        do_vacuum(s, s->system_journal, &s->system_metrics, "/var/log/journal/", "System journal", verbose, patch_min_use);
+        do_vacuum(s, s->runtime_journal, &s->runtime_metrics, "/run/log/journal/", "Runtime journal", verbose, patch_min_use);
 
-        do_vacuum(s, ids, s->system_journal, "/var/log/journal/", &s->system_metrics);
-        do_vacuum(s, ids, s->runtime_journal, "/run/log/journal/", &s->runtime_metrics);
+        s->cached_space_limit = 0;
+        s->cached_space_available = 0;
+        s->cached_space_timestamp = 0;
 
-        s->cached_available_space_timestamp = 0;
+        return 0;
 }
 
 static void server_cache_machine_id(Server *s) {
@@ -502,7 +508,7 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, unsigned
         if (journal_file_rotate_suggested(f, s->max_file_usec)) {
                 log_debug("%s: Journal header limits reached or header out-of-date, rotating.", f->path);
                 server_rotate(s);
-                server_vacuum(s);
+                server_vacuum(s, false, false);
                 vacuumed = true;
 
                 f = find_journal(s, uid);
@@ -522,7 +528,7 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, unsigned
         }
 
         server_rotate(s);
-        server_vacuum(s);
+        server_vacuum(s, false, false);
 
         f = find_journal(s, uid);
         if (!f)
@@ -867,6 +873,7 @@ void server_dispatch_message(
 
         int rl, r;
         _cleanup_free_ char *path = NULL;
+        uint64_t available = 0;
         char *c;
 
         assert(s);
@@ -906,9 +913,8 @@ void server_dispatch_message(
                 }
         }
 
-        rl = journal_rate_limit_test(s->rate_limit, path,
-                                     priority & LOG_PRIMASK, available_space(s, false));
-
+        (void) determine_space(s, false, false, &available, NULL);
+        rl = journal_rate_limit_test(s->rate_limit, path, priority & LOG_PRIMASK, available);
         if (rl == 0)
                 return;
 
@@ -923,16 +929,8 @@ finish:
 
 
 static int system_journal_open(Server *s, bool flush_requested) {
-        int r;
         const char *fn;
-        sd_id128_t machine;
-        char ids[33];
-
-        r = sd_id128_get_machine(&machine);
-        if (r < 0)
-                return log_error_errno(r, "Failed to get machine id: %m");
-
-        sd_id128_to_string(machine, ids);
+        int r;
 
         if (!s->system_journal &&
             (s->storage == STORAGE_PERSISTENT || s->storage == STORAGE_AUTO) &&
@@ -948,15 +946,15 @@ static int system_journal_open(Server *s, bool flush_requested) {
                 if (s->storage == STORAGE_PERSISTENT)
                         (void) mkdir_p("/var/log/journal/", 0755);
 
-                fn = strjoina("/var/log/journal/", ids);
+                fn = strjoina("/var/log/journal/", SERVER_MACHINE_ID(s));
                 (void) mkdir(fn, 0755);
 
                 fn = strjoina(fn, "/system.journal");
                 r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, s->compress, s->seal, &s->system_metrics, s->mmap, NULL, &s->system_journal);
-
-                if (r >= 0)
+                if (r >= 0) {
                         server_fix_perms(s, s->system_journal, 0);
-                else if (r < 0) {
+                        (void) determine_space_for(s, &s->system_metrics, "/var/log/journal/", "System journal", true, true, NULL, NULL);
+                } else if (r < 0) {
                         if (r != -ENOENT && r != -EROFS)
                                 log_warning_errno(r, "Failed to open system journal: %m");
 
@@ -967,7 +965,7 @@ static int system_journal_open(Server *s, bool flush_requested) {
         if (!s->runtime_journal &&
             (s->storage != STORAGE_NONE)) {
 
-                fn = strjoina("/run/log/journal/", ids, "/system.journal", NULL);
+                fn = strjoina("/run/log/journal/", SERVER_MACHINE_ID(s), "/system.journal");
 
                 if (s->system_journal) {
 
@@ -997,12 +995,12 @@ static int system_journal_open(Server *s, bool flush_requested) {
                                 return log_error_errno(r, "Failed to open runtime journal: %m");
                 }
 
-                if (s->runtime_journal)
+                if (s->runtime_journal) {
                         server_fix_perms(s, s->runtime_journal, 0);
+                        (void) determine_space_for(s, &s->runtime_metrics, "/run/log/journal/", "Runtime journal", true, true, NULL, NULL);
+                }
         }
 
-        available_space(s, true);
-
         return r;
 }
 
@@ -1023,7 +1021,7 @@ int server_flush_to_var(Server *s) {
         if (!s->runtime_journal)
                 return 0;
 
-        system_journal_open(s, true);
+        (void) system_journal_open(s, true);
 
         if (!s->system_journal)
                 return 0;
@@ -1067,7 +1065,7 @@ int server_flush_to_var(Server *s) {
                 }
 
                 server_rotate(s);
-                server_vacuum(s);
+                server_vacuum(s, false, false);
 
                 if (!s->system_journal) {
                         log_notice("Didn't flush runtime journal since rotation of system journal wasn't successful.");
@@ -1231,7 +1229,7 @@ static int dispatch_sigusr1(sd_event_source *es, const struct signalfd_siginfo *
 
         server_flush_to_var(s);
         server_sync(s);
-        server_vacuum(s);
+        server_vacuum(s, false, false);
 
         touch("/run/systemd/journal/flushed");
 
@@ -1245,7 +1243,7 @@ static int dispatch_sigusr2(sd_event_source *es, const struct signalfd_siginfo *
 
         log_info("Received request to rotate journal from PID %"PRIu32, si->ssi_pid);
         server_rotate(s);
-        server_vacuum(s);
+        server_vacuum(s, true, true);
 
         return 0;
 }
@@ -1472,18 +1470,19 @@ int server_init(Server *s) {
         s->max_level_console = LOG_INFO;
         s->max_level_wall = LOG_EMERG;
 
-        memset(&s->system_metrics, 0xFF, sizeof(s->system_metrics));
-        memset(&s->runtime_metrics, 0xFF, sizeof(s->runtime_metrics));
+        journal_reset_metrics(&s->system_metrics);
+        journal_reset_metrics(&s->runtime_metrics);
 
         server_parse_config_file(s);
         server_parse_proc_cmdline(s);
+
         if (!!s->rate_limit_interval ^ !!s->rate_limit_burst) {
                 log_debug("Setting both rate limit interval and burst from "USEC_FMT",%u to 0,0",
                           s->rate_limit_interval, s->rate_limit_burst);
                 s->rate_limit_interval = s->rate_limit_burst = 0;
         }
 
-        mkdir_p("/run/systemd/journal", 0755);
+        (void) mkdir_p("/run/systemd/journal", 0755);
 
         s->user_journals = ordered_hashmap_new(NULL);
         if (!s->user_journals)
@@ -1682,3 +1681,22 @@ void server_done(Server *s) {
 
         udev_unref(s->udev);
 }
+
+static const char* const storage_table[_STORAGE_MAX] = {
+        [STORAGE_AUTO] = "auto",
+        [STORAGE_VOLATILE] = "volatile",
+        [STORAGE_PERSISTENT] = "persistent",
+        [STORAGE_NONE] = "none"
+};
+
+DEFINE_STRING_TABLE_LOOKUP(storage, Storage);
+DEFINE_CONFIG_PARSE_ENUM(config_parse_storage, storage, Storage, "Failed to parse storage setting");
+
+static const char* const split_mode_table[_SPLIT_MAX] = {
+        [SPLIT_LOGIN] = "login",
+        [SPLIT_UID] = "uid",
+        [SPLIT_NONE] = "none",
+};
+
+DEFINE_STRING_TABLE_LOOKUP(split_mode, SplitMode);
+DEFINE_CONFIG_PARSE_ENUM(config_parse_split_mode, split_mode, SplitMode, "Failed to parse split mode setting");
diff --git a/src/journal/journald-server.h b/src/journal/journald-server.h
index d954c5190d..535c0ab9ab 100644
--- a/src/journal/journald-server.h
+++ b/src/journal/journald-server.h
@@ -100,8 +100,9 @@ typedef struct Server {
         unsigned n_forward_syslog_missed;
         usec_t last_warn_forward_syslog_missed;
 
-        uint64_t cached_available_space;
-        usec_t cached_available_space_timestamp;
+        uint64_t cached_space_available;
+        uint64_t cached_space_limit;
+        usec_t cached_space_timestamp;
 
         uint64_t var_available_timestamp;
 
@@ -141,6 +142,8 @@ typedef struct Server {
         char *cgroup_root;
 } Server;
 
+#define SERVER_MACHINE_ID(s) ((s)->machine_id_field + strlen("_MACHINE_ID="))
+
 #define N_IOVEC_META_FIELDS 20
 #define N_IOVEC_KERNEL_FIELDS 64
 #define N_IOVEC_UDEV_FIELDS 32
@@ -166,7 +169,7 @@ void server_fix_perms(Server *s, JournalFile *f, uid_t uid);
 int server_init(Server *s);
 void server_done(Server *s);
 void server_sync(Server *s);
-void server_vacuum(Server *s);
+int server_vacuum(Server *s, bool verbose, bool patch_min_use);
 void server_rotate(Server *s);
 int server_schedule_sync(Server *s, int priority);
 int server_flush_to_var(Server *s);
diff --git a/src/journal/journald.c b/src/journal/journald.c
index b2624c6d28..83236ceba9 100644
--- a/src/journal/journald.c
+++ b/src/journal/journald.c
@@ -21,8 +21,8 @@
 
 #include <unistd.h>
 
-#include "systemd/sd-messages.h"
-#include "systemd/sd-daemon.h"
+#include "sd-messages.h"
+#include "sd-daemon.h"
 
 #include "journal-authenticate.h"
 #include "journald-server.h"
@@ -54,7 +54,7 @@ int main(int argc, char *argv[]) {
         if (r < 0)
                 goto finish;
 
-        server_vacuum(&server);
+        server_vacuum(&server, false, false);
         server_flush_to_var(&server);
         server_flush_dev_kmsg(&server);
 
@@ -82,7 +82,7 @@ int main(int argc, char *argv[]) {
                         if (server.oldest_file_usec + server.max_retention_usec < n) {
                                 log_info("Retention time reached.");
                                 server_rotate(&server);
-                                server_vacuum(&server);
+                                server_vacuum(&server, false, false);
                                 continue;
                         }
 
diff --git a/src/journal/journald.conf b/src/journal/journald.conf
index 47eefe91c1..7beb96c671 100644
--- a/src/journal/journald.conf
+++ b/src/journal/journald.conf
@@ -22,9 +22,11 @@
 #SystemMaxUse=
 #SystemKeepFree=
 #SystemMaxFileSize=
+#SystemMaxFiles=100
 #RuntimeMaxUse=
 #RuntimeKeepFree=
 #RuntimeMaxFileSize=
+#RuntimeMaxFiles=100
 #MaxRetentionSec=
 #MaxFileSec=1month
 #ForwardToSyslog=no
diff --git a/src/journal/test-journal-interleaving.c b/src/journal/test-journal-interleaving.c
index adefa1b026..8069339c1f 100644
--- a/src/journal/test-journal-interleaving.c
+++ b/src/journal/test-journal-interleaving.c
@@ -197,7 +197,7 @@ static void test_skip(void (*setup)(void)) {
         if (arg_keep)
                 log_info("Not removing %s", t);
         else {
-                journal_directory_vacuum(".", 3000000, 0, NULL, true);
+                journal_directory_vacuum(".", 3000000, 0, 0, NULL, true);
 
                 assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
         }
@@ -282,7 +282,7 @@ static void test_sequence_numbers(void) {
         if (arg_keep)
                 log_info("Not removing %s", t);
         else {
-                journal_directory_vacuum(".", 3000000, 0, NULL, true);
+                journal_directory_vacuum(".", 3000000, 0, 0, NULL, true);
 
                 assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
         }
diff --git a/src/journal/test-journal.c b/src/journal/test-journal.c
index caaab258c9..01d4bc968a 100644
--- a/src/journal/test-journal.c
+++ b/src/journal/test-journal.c
@@ -116,7 +116,7 @@ static void test_non_empty(void) {
         if (arg_keep)
                 log_info("Not removing %s", t);
         else {
-                journal_directory_vacuum(".", 3000000, 0, NULL, true);
+                journal_directory_vacuum(".", 3000000, 0, 0, NULL, true);
 
                 assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
         }
@@ -155,7 +155,7 @@ static void test_empty(void) {
         if (arg_keep)
                 log_info("Not removing %s", t);
         else {
-                journal_directory_vacuum(".", 3000000, 0, NULL, true);
+                journal_directory_vacuum(".", 3000000, 0, 0, NULL, true);
 
                 assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
         }