coredump: add simple coredump vacuuming

When disk space taken up by coredumps grows beyond a configured limit
start removing the oldest coredump of the user with the most coredumps,
until we get below the limit again.

8 files changed, 393 insertions, 15 deletions

diff --git a/.gitignore b/.gitignore
index d2e81a271c..fe8c32d42c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -142,6 +142,7 @@
 /test-cgroup-util
 /test-compress
 /test-conf-files
+/test-coredump-vacuum
 /test-daemon
 /test-date
 /test-device-nodes
diff --git a/Makefile.am b/Makefile.am
index e02dede7a1..5d86f3eaec 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -3771,7 +3771,9 @@ systemd_socket_proxyd_LDADD = \
 # ------------------------------------------------------------------------------
 if ENABLE_COREDUMP
 systemd_coredump_SOURCES = \
-	src/journal/coredump.c
+	src/journal/coredump.c \
+	src/journal/coredump-vacuum.c \
+	src/journal/coredump-vacuum.h
 
 systemd_coredump_LDADD = \
 	libsystemd-journal-internal.la \
@@ -3810,6 +3812,18 @@ coredumpctl_LDADD = \
 bin_PROGRAMS += \
 	coredumpctl
 
+manual_tests += \
+	test-coredump-vacuum
+
+test_coredump_vacuum_SOURCES = \
+	src/journal/test-coredump-vacuum.c  \
+	src/journal/coredump-vacuum.c \
+	src/journal/coredump-vacuum.h
+
+test_coredump_vacuum_LDADD = \
+	libsystemd-internal.la \
+	libsystemd-shared.la
+
 dist_bashcompletion_DATA += \
 	shell-completion/bash/coredumpctl
 
diff --git a/man/coredump.conf.xml b/man/coredump.conf.xml
index 9e4adff46a..e24d958ac4 100644
--- a/man/coredump.conf.xml
+++ b/man/coredump.conf.xml
@@ -121,6 +121,23 @@
         <listitem><para>The maximum (uncompressed) size in bytes of a
         core to be saved.</para></listitem>
       </varlistentry>
+
+      <varlistentry>
+        <term><varname>MaxUse=</varname></term>
+        <term><varname>KeepFree=</varname></term>
+
+        <listitem><para>Enforce limits on the disk space taken up by
+        externally stored coredumps. <option>MaxUse=</option> makes
+        sure that old coredumps are removed as soon as the total disk
+        space taken up by coredumps grows beyond this limit (defaults
+        to 10% of the total disk size). <option>KeepFree=</option>
+        controls how much disk space to keep free at least (defaults
+        to 15% of the total disk size). Note that the disk space used
+        by coredumps might temporarily exceed these limits while
+        coredumps are processed. Note that old coredumps are also
+        removed based on on time via
+        <citerefentry><refentrytitle>systemd-tmpfiles</refentrytitle><manvolnum>8</manvolnum></citerefentry>.</para></listitem>
+      </varlistentry>
     </variablelist>
 
   </refsect1>
@@ -129,7 +146,8 @@
     <title>See Also</title>
     <para>
       <citerefentry><refentrytitle>systemd-journald.service</refentrytitle><manvolnum>8</manvolnum></citerefentry>,
-      <citerefentry><refentrytitle>coredumpctl</refentrytitle><manvolnum>1</manvolnum></citerefentry>
+      <citerefentry><refentrytitle>coredumpctl</refentrytitle><manvolnum>1</manvolnum></citerefentry>,
+      <citerefentry><refentrytitle>systemd-tmpfiles</refentrytitle><manvolnum>8</manvolnum></citerefentry>
     </para>
   </refsect1>
 
diff --git a/src/journal/coredump-vacuum.c b/src/journal/coredump-vacuum.c
new file mode 100644
index 0000000000..ad2e2fa7f4
--- /dev/null
+++ b/src/journal/coredump-vacuum.c
@@ -0,0 +1,272 @@
+/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
+
+/***
+  This file is part of systemd.
+
+  Copyright 2014 Lennart Poettering
+
+  systemd is free software; you can redistribute it and/or modify it
+  under the terms of the GNU Lesser General Public License as published by
+  the Free Software Foundation; either version 2.1 of the License, or
+  (at your option) any later version.
+
+  systemd is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public License
+  along with systemd; If not, see <http://www.gnu.org/licenses/>.
+***/
+
+#include <sys/statvfs.h>
+
+#include "util.h"
+#include "time-util.h"
+#include "hashmap.h"
+#include "macro.h"
+
+#include "coredump-vacuum.h"
+
+#define DEFAULT_MAX_USE_LOWER (off_t) (1ULL*1024ULL*1024ULL)           /* 1 MiB */
+#define DEFAULT_MAX_USE_UPPER (off_t) (4ULL*1024ULL*1024ULL*1024ULL)   /* 4 GiB */
+#define DEFAULT_KEEP_FREE_UPPER (off_t) (4ULL*1024ULL*1024ULL*1024ULL) /* 4 GiB */
+#define DEFAULT_KEEP_FREE (off_t) (1024ULL*1024ULL)                    /* 1 MB */
+
+struct vacuum_candidate {
+        unsigned n_files;
+        char *oldest_file;
+        usec_t oldest_mtime;
+};
+
+static void vacuum_candidate_free(struct vacuum_candidate *c) {
+        if (!c)
+                return;
+
+        free(c->oldest_file);
+        free(c);
+}
+
+DEFINE_TRIVIAL_CLEANUP_FUNC(struct vacuum_candidate*, vacuum_candidate_free);
+
+static void vacuum_candidate_hasmap_free(Hashmap *h) {
+        struct vacuum_candidate *c;
+
+        while ((c = hashmap_steal_first(h)))
+                vacuum_candidate_free(c);
+
+        hashmap_free(h);
+}
+
+DEFINE_TRIVIAL_CLEANUP_FUNC(Hashmap*, vacuum_candidate_hasmap_free);
+
+static int uid_from_file_name(const char *filename, uid_t *uid) {
+        const char *p, *e, *u;
+
+        p = startswith(filename, "core.");
+        if (!p)
+                return -EINVAL;
+
+        /* Skip the comm field */
+        p = strchr(p, '.');
+        if (!p)
+                return -EINVAL;
+        p++;
+
+        /* Find end up UID */
+        e = strchr(p, '.');
+        if (!e)
+                return -EINVAL;
+
+        u = strndupa(p, e-p);
+        return parse_uid(u, uid);
+}
+
+static bool vacuum_necessary(int fd, off_t sum, off_t keep_free, off_t max_use) {
+        off_t fs_size = 0, fs_free = (off_t) -1;
+        struct statvfs sv;
+
+        assert(fd >= 0);
+
+        if (fstatvfs(fd, &sv) >= 0) {
+                fs_size = sv.f_frsize * sv.f_blocks;
+                fs_free = sv.f_frsize * sv.f_bfree;
+        }
+
+        if (max_use == (off_t) -1) {
+
+                if (fs_size > 0) {
+                        max_use = PAGE_ALIGN(fs_size / 10); /* 10% */
+
+                        if (max_use > DEFAULT_MAX_USE_UPPER)
+                                max_use = DEFAULT_MAX_USE_UPPER;
+
+                        if (max_use < DEFAULT_MAX_USE_LOWER)
+                                max_use = DEFAULT_MAX_USE_LOWER;
+                }
+
+                max_use = DEFAULT_MAX_USE_LOWER;
+        } else
+                max_use = PAGE_ALIGN(max_use);
+
+        if (max_use > 0 && sum > max_use)
+                return true;
+
+        if (keep_free == (off_t) -1) {
+
+                if (fs_size > 0) {
+                        keep_free = PAGE_ALIGN((fs_size * 3) / 20); /* 15% */
+
+                        if (keep_free > DEFAULT_KEEP_FREE_UPPER)
+                                keep_free = DEFAULT_KEEP_FREE_UPPER;
+                } else
+                        keep_free = DEFAULT_KEEP_FREE;
+        } else
+                keep_free = PAGE_ALIGN(keep_free);
+
+        if (keep_free > 0 && fs_free < keep_free)
+                return true;
+
+        return false;
+}
+
+int coredump_vacuum(int exclude_fd, off_t keep_free, off_t max_use) {
+        _cleanup_closedir_ DIR *d = NULL;
+        struct stat exclude_st;
+        int r;
+
+        if (keep_free <= 0 && max_use <= 0)
+                return 0;
+
+        if (exclude_fd >= 0) {
+                if (fstat(exclude_fd, &exclude_st) < 0) {
+                        log_error("Failed to fstat(): %m");
+                        return -errno;
+                }
+        }
+
+        /* This algorithm will keep deleting the oldest file of the
+         * user with the most coredumps until we are back in the size
+         * limits. Note that vacuuming for journal files is different,
+         * because we rely on rate-limiting of the messages there,
+         * to avoid being flooded. */
+
+        d = opendir("/var/lib/systemd/coredump");
+        if (!d) {
+                if (errno == ENOENT)
+                        return 0;
+
+                log_error("Can't open coredump directory: %m");
+                return -errno;
+        }
+
+        for (;;) {
+                _cleanup_(vacuum_candidate_hasmap_freep) Hashmap *h = NULL;
+                struct vacuum_candidate *worst = NULL;
+                struct dirent *de;
+                off_t sum = 0;
+
+                rewinddir(d);
+
+                FOREACH_DIRENT(de, d, goto fail) {
+                        struct vacuum_candidate *c;
+                        struct stat st;
+                        uid_t uid;
+                        usec_t t;
+
+                        r = uid_from_file_name(de->d_name, &uid);
+                        if (r < 0)
+                                continue;
+
+                        if (fstatat(dirfd(d), de->d_name, &st, AT_NO_AUTOMOUNT|AT_SYMLINK_NOFOLLOW) < 0) {
+                                if (errno == ENOENT)
+                                        continue;
+
+                                log_warning("Failed to stat /var/lib/systemd/coredump/%s", de->d_name);
+                                continue;
+                        }
+
+                        if (!S_ISREG(st.st_mode))
+                                continue;
+
+                        if (exclude_fd >= 0 &&
+                            exclude_st.st_dev == st.st_dev &&
+                            exclude_st.st_ino == st.st_ino)
+                                continue;
+
+                        r = hashmap_ensure_allocated(&h, NULL, NULL);
+                        if (r < 0)
+                                return log_oom();
+
+                        t = timespec_load(&st.st_mtim);
+
+                        c = hashmap_get(h, UINT32_TO_PTR(uid));
+                        if (c) {
+
+                                if (t < c->oldest_mtime) {
+                                        char *n;
+
+                                        n = strdup(de->d_name);
+                                        if (!n)
+                                                return log_oom();
+
+                                        free(c->oldest_file);
+                                        c->oldest_file = n;
+                                        c->oldest_mtime = t;
+                                }
+
+                        } else {
+                                _cleanup_(vacuum_candidate_freep) struct vacuum_candidate *n = NULL;
+
+                                n = new0(struct vacuum_candidate, 1);
+                                if (!n)
+                                        return log_oom();
+
+                                n->oldest_file = strdup(de->d_name);
+                                if (!n->oldest_file)
+                                        return log_oom();
+
+                                n->oldest_mtime = t;
+
+                                r = hashmap_put(h, UINT32_TO_PTR(uid), n);
+                                if (r < 0)
+                                        return log_oom();
+
+                                c = n;
+                                n = NULL;
+                        }
+
+                        c->n_files++;
+
+                        if (!worst ||
+                            worst->n_files < c->n_files ||
+                            (worst->n_files == c->n_files && c->oldest_mtime < worst->oldest_mtime))
+                                worst = c;
+
+                        sum += st.st_blocks * 512;
+                }
+
+                if (!worst)
+                        break;
+
+                r = vacuum_necessary(dirfd(d), sum, keep_free, max_use);
+                if (r <= 0)
+                        return r;
+
+                if (unlinkat(dirfd(d), worst->oldest_file, 0) < 0) {
+
+                        if (errno == ENOENT)
+                                continue;
+
+                        log_error("Failed to remove file %s: %m", worst->oldest_file);
+                        return -errno;
+                } else
+                        log_info("Removed old coredump %s.", worst->oldest_file);
+        }
+
+        return 0;
+
+fail:
+        log_error("Failed to read directory: %m");
+        return -errno;
+}
diff --git a/src/journal/coredump-vacuum.h b/src/journal/coredump-vacuum.h
new file mode 100644
index 0000000000..7ad4399305
--- /dev/null
+++ b/src/journal/coredump-vacuum.h
@@ -0,0 +1,26 @@
+/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
+
+#pragma once
+
+/***
+  This file is part of systemd.
+
+  Copyright 2014 Lennart Poettering
+
+  systemd is free software; you can redistribute it and/or modify it
+  under the terms of the GNU Lesser General Public License as published by
+  the Free Software Foundation; either version 2.1 of the License, or
+  (at your option) any later version.
+
+  systemd is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public License
+  along with systemd; If not, see <http://www.gnu.org/licenses/>.
+***/
+
+#include <sys/types.h>
+
+int coredump_vacuum(int exclude_fd, off_t keep_free, off_t max_use);
diff --git a/src/journal/coredump.c b/src/journal/coredump.c
index ab8fd2cf0e..22e1932e2b 100644
--- a/src/journal/coredump.c
+++ b/src/journal/coredump.c
@@ -42,6 +42,7 @@
 #include "stacktrace.h"
 #include "path-util.h"
 #include "compress.h"
+#include "coredump-vacuum.h"
 
 #ifdef HAVE_ACL
 #  include <sys/acl.h>
@@ -121,10 +122,11 @@ static DEFINE_CONFIG_PARSE_ENUM(config_parse_coredump_compression, coredump_comp
 static CoredumpStorage arg_storage = COREDUMP_STORAGE_EXTERNAL;
 static CoredumpCompression arg_compression = COREDUMP_COMPRESSION_XZ;
 static unsigned arg_compression_level = LZMA_PRESET_DEFAULT;
-
 static off_t arg_process_size_max = PROCESS_SIZE_MAX;
 static off_t arg_external_size_max = EXTERNAL_SIZE_MAX;
 static size_t arg_journal_size_max = JOURNAL_SIZE_MAX;
+static off_t arg_keep_free = (off_t) -1;
+static off_t arg_max_use = (off_t) -1;
 
 static int parse_config(void) {
 
@@ -136,6 +138,8 @@ static int parse_config(void) {
                 { "Coredump", "ProcessSizeMax",   config_parse_iec_off,              0, &arg_process_size_max  },
                 { "Coredump", "ExternalSizeMax",  config_parse_iec_off,              0, &arg_external_size_max },
                 { "Coredump", "JournalSizeMax",   config_parse_iec_size,             0, &arg_journal_size_max  },
+                { "Coredump", "KeepFree",         config_parse_iec_off,              0, &arg_keep_free         },
+                { "Coredump", "MaxUse",           config_parse_iec_off,              0, &arg_max_use           },
                 {}
         };
 
@@ -274,14 +278,14 @@ static int maybe_remove_external_coredump(const char *filename, off_t size) {
         return 1;
 }
 
+static int save_external_coredump(
+                const char *info[_INFO_LEN],
+                uid_t uid,
+                char **ret_filename,
+                int *ret_fd,
+                off_t *ret_size) {
 
-static int save_external_coredump(const char *info[_INFO_LEN],
-                                  uid_t uid,
-                                  char **ret_filename,
-                                  int *ret_fd,
-                                  off_t *ret_size) {
-
-        _cleanup_free_ char *p = NULL, *t = NULL, *c = NULL, *fn = NULL, *tmp = NULL;
+        _cleanup_free_ char *p = NULL, *t = NULL, *c = NULL, *fn = NULL, *tmp = NULL, *u = NULL;
         _cleanup_close_ int fd = -1;
         sd_id128_t boot;
         struct stat st;
@@ -300,6 +304,10 @@ static int save_external_coredump(const char *info[_INFO_LEN],
         if (!p)
                 return log_oom();
 
+        u = filename_escape(info[INFO_UID]);
+        if (!u)
+                return log_oom();
+
         t = filename_escape(info[INFO_TIMESTAMP]);
         if (!t)
                 return log_oom();
@@ -311,8 +319,9 @@ static int save_external_coredump(const char *info[_INFO_LEN],
         }
 
         r = asprintf(&fn,
-                     "/var/lib/systemd/coredump/core.%s." SD_ID128_FORMAT_STR ".%s.%s000000",
+                     "/var/lib/systemd/coredump/core.%s.%s." SD_ID128_FORMAT_STR ".%s.%s000000",
                      c,
+                     u,
                      SD_ID128_FORMAT_VAL(boot),
                      p,
                      t);
@@ -333,12 +342,10 @@ static int save_external_coredump(const char *info[_INFO_LEN],
 
         r = copy_bytes(STDIN_FILENO, fd, arg_process_size_max);
         if (r == -E2BIG) {
-                log_error("Coredump of %s (%s) is larger than configured processing limit, refusing.",
-                          info[INFO_PID], info[INFO_COMM]);
+                log_error("Coredump of %s (%s) is larger than configured processing limit, refusing.", info[INFO_PID], info[INFO_COMM]);
                 goto fail;
         } else if (IN_SET(r, -EDQUOT, -ENOSPC)) {
-                log_error("Not enough disk space for coredump of %s (%s), refusing.",
-                          info[INFO_PID], info[INFO_COMM]);
+                log_error("Not enough disk space for coredump of %s (%s), refusing.", info[INFO_PID], info[INFO_COMM]);
                 goto fail;
         } else if (r < 0) {
                 log_error("Failed to dump coredump to file: %s", strerror(-r));
@@ -646,6 +653,9 @@ int main(int argc, char* argv[]) {
         IOVEC_SET_STRING(iovec[j++], "MESSAGE_ID=fc2e22bc6ee647b6b90729ab34a250b1");
         IOVEC_SET_STRING(iovec[j++], "PRIORITY=2");
 
+        /* Vacuum before we write anything again */
+        coredump_vacuum(-1, arg_keep_free, arg_max_use);
+
         /* Always stream the coredump to disk, if that's possible */
         r = save_external_coredump(info, uid, &filename, &coredump_fd, &coredump_size);
         if (r < 0)
@@ -666,6 +676,9 @@ int main(int argc, char* argv[]) {
                 IOVEC_SET_STRING(iovec[j++], coredump_filename);
         }
 
+        /* Vacuum again, but exclude the coredump we just created */
+        coredump_vacuum(coredump_fd, arg_keep_free, arg_max_use);
+
         /* Now, let's drop privileges to become the user who owns the
          * segfaulted process and allocate the coredump memory under
          * his uid. This also ensures that the credentials journald
diff --git a/src/journal/coredump.conf b/src/journal/coredump.conf
index 9f4edd34d9..050bde6491 100644
--- a/src/journal/coredump.conf
+++ b/src/journal/coredump.conf
@@ -15,3 +15,5 @@
 #ProcessSizeMax=2G
 #ExternalSizeMax=2G
 #JournalSizeMax=767M
+#MaxUse=
+#KeepFree=
diff --git a/src/journal/test-coredump-vacuum.c b/src/journal/test-coredump-vacuum.c
new file mode 100644
index 0000000000..a4dd00125d
--- /dev/null
+++ b/src/journal/test-coredump-vacuum.c
@@ -0,0 +1,32 @@
+/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
+
+/***
+  This file is part of systemd.
+
+  Copyright 2012 Zbigniew Jędrzejewski-Szmek
+
+  systemd is free software; you can redistribute it and/or modify it
+  under the terms of the GNU Lesser General Public License as published by
+  the Free Software Foundation; either version 2.1 of the License, or
+  (at your option) any later version.
+
+  systemd is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public License
+  along with systemd; If not, see <http://www.gnu.org/licenses/>.
+***/
+
+#include <stdlib.h>
+
+#include "coredump-vacuum.h"
+
+int main(int argc, char *argv[]) {
+
+        if (coredump_vacuum(-1, (off_t) -1, 70 * 1024) < 0)
+                return EXIT_FAILURE;
+
+        return EXIT_SUCCESS;
+}