coredump: include stacktrace of coredumps in the log message

elfutils' libdw is maintained, can read DWARF debug data and appears to
be the library of choice for generating backtraces today.

6 files changed, 311 insertions, 13 deletions

diff --git a/Makefile.am b/Makefile.am
index 7c20e33845..bd3313815a 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -3686,6 +3686,15 @@ systemd_coredump_LDADD = \
 	libsystemd-internal.la \
 	libsystemd-shared.la
 
+if HAVE_ELFUTILS
+systemd_coredump_SOURCES += \
+	src/journal/stacktrace.c \
+	src/journal/stacktrace.h
+
+systemd_coredump_LDADD += \
+	$(ELFUTILS_LIBS)
+endif
+
 rootlibexec_PROGRAMS += \
 	systemd-coredump
 
diff --git a/configure.ac b/configure.ac
index e35d86408d..1391d033b4 100644
--- a/configure.ac
+++ b/configure.ac
@@ -627,6 +627,44 @@ fi
 AC_SUBST(AUDIT_LIBS)
 
 # ------------------------------------------------------------------------------
+AC_ARG_ENABLE([elfutils],
+        AS_HELP_STRING([--disable-elfutils],[Disable optional ELFUTILS support]),
+                [case "${enableval}" in
+                        yes) have_elfutils=yes ;;
+                        no) have_elfutils=no ;;
+                        *) AC_MSG_ERROR(bad value ${enableval} for --disable-elfutils) ;;
+                esac],
+                [have_elfutils=auto])
+
+if test "x${have_elfutils}" != xno ; then
+        AC_CHECK_HEADERS(
+                [elfutils/libdwfl.h],
+                [have_elfutils=yes],
+                [if test "x$have_elfutils" = xyes ; then
+                        AC_MSG_ERROR([*** ELFUTILS headers not found.])
+                fi])
+
+        AC_CHECK_LIB(
+                [dw],
+                [dwfl_begin],
+                [have_elfutils=yes],
+                [if test "x$have_elfutils" = xyes ; then
+                        AC_MSG_ERROR([*** ELFUTILS libs not found.])
+                fi])
+
+        if test "x$have_elfutils" = xyes ; then
+                ELFUTILS_LIBS="-lelf -ldw"
+                AC_DEFINE(HAVE_ELFUTILS, 1, [ELFUTILS available])
+        else
+                have_elfutils=no
+        fi
+else
+        ELFUTILS_LIBS=
+fi
+AC_SUBST(ELFUTILS_LIBS)
+AM_CONDITIONAL(HAVE_ELFUTILS, [test "$have_elfutils" = "yes"])
+
+# ------------------------------------------------------------------------------
 have_libcryptsetup=no
 AC_ARG_ENABLE(libcryptsetup, AS_HELP_STRING([--disable-libcryptsetup], [disable libcryptsetup tools]))
 if test "x$enable_libcryptsetup" != "xno"; then
@@ -1171,6 +1209,7 @@ AC_MSG_RESULT([
         MICROHTTPD:              ${have_microhttpd}
         CHKCONFIG:               ${have_chkconfig}
         GNUTLS:                  ${have_gnutls}
+        ELFUTILS:                ${have_elfutils}
         binfmt:                  ${have_binfmt}
         vconsole:                ${have_vconsole}
         readahead:               ${have_readahead}
diff --git a/src/journal/coredump.c b/src/journal/coredump.c
index 3365f9f146..f48f4e2c89 100644
--- a/src/journal/coredump.c
+++ b/src/journal/coredump.c
@@ -38,6 +38,7 @@
 #include "journald-native.h"
 #include "conf-parser.h"
 #include "copy.h"
+#include "stacktrace.h"
 
 #ifdef HAVE_ACL
 #include <sys/acl.h>
@@ -290,6 +291,7 @@ static int allocate_journal_field(int fd, size_t size, char **ret, size_t *ret_s
         _cleanup_free_ char *field = NULL;
         ssize_t n;
 
+        assert(fd >= 0);
         assert(ret);
         assert(ret_size);
 
@@ -346,7 +348,8 @@ int main(int argc, char* argv[]) {
         _cleanup_free_ char *core_pid = NULL, *core_uid = NULL, *core_gid = NULL, *core_signal = NULL,
                 *core_timestamp = NULL, *core_comm = NULL, *core_exe = NULL, *core_unit = NULL,
                 *core_session = NULL, *core_message = NULL, *core_cmdline = NULL, *coredump_data = NULL,
-                *coredump_filename = NULL, *core_slice = NULL, *core_cgroup = NULL, *core_owner_uid = NULL;
+                *coredump_filename = NULL, *core_slice = NULL, *core_cgroup = NULL, *core_owner_uid = NULL,
+                *exe = NULL;
 
         _cleanup_close_ int coredump_fd = -1;
 
@@ -365,7 +368,6 @@ int main(int argc, char* argv[]) {
          * crashed and it might be journald which we'd rather not log
          * to then. */
         log_set_target(LOG_TARGET_KMSG);
-        log_set_max_level(LOG_DEBUG);
         log_open();
 
         if (argc != _ARG_MAX) {
@@ -474,10 +476,8 @@ int main(int argc, char* argv[]) {
                         IOVEC_SET_STRING(iovec[j++], core_slice);
         }
 
-        if (get_process_exe(pid, &t) >= 0) {
-                core_exe = strappend("COREDUMP_EXE=", t);
-                free(t);
-
+        if (get_process_exe(pid, &exe) >= 0) {
+                core_exe = strappend("COREDUMP_EXE=", exe);
                 if (core_exe)
                         IOVEC_SET_STRING(iovec[j++], core_exe);
         }
@@ -505,17 +505,15 @@ int main(int argc, char* argv[]) {
         IOVEC_SET_STRING(iovec[j++], "MESSAGE_ID=fc2e22bc6ee647b6b90729ab34a250b1");
         IOVEC_SET_STRING(iovec[j++], "PRIORITY=2");
 
-        core_message = strjoin("MESSAGE=Process ", argv[ARG_PID], " (", argv[ARG_COMM], ") dumped core.", NULL);
-        if (core_message)
-                IOVEC_SET_STRING(iovec[j++], core_message);
-
         /* Always stream the coredump to disk, if that's possible */
         r = save_external_coredump(argv, uid, &coredump_filename, &coredump_fd, &coredump_size);
         if (r < 0)
                 goto finish;
 
         /* If we don't want to keep the coredump on disk, remove it
-         * now, as later on we will lack the privileges for it. */
+         * now, as later on we will lack the privileges for
+         * it. However, we keep the fd to it, so that we can still
+         * process it and log it. */
         r = maybe_remove_external_coredump(coredump_filename, coredump_size);
         if (r < 0)
                 goto finish;
@@ -532,6 +530,24 @@ int main(int argc, char* argv[]) {
                 goto finish;
         }
 
+#ifdef HAVE_ELFUTILS
+        /* Try to get a strack trace if we can */
+        if (coredump_size <= arg_process_size_max) {
+                _cleanup_free_ char *stacktrace = NULL;
+
+                r = coredump_make_stack_trace(coredump_fd, exe, &stacktrace);
+                if (r >= 0)
+                        core_message = strjoin("MESSAGE=Process ", argv[ARG_PID], " (", argv[ARG_COMM], ") of user ", argv[ARG_UID], " dumped core.\n\n", stacktrace, NULL);
+                else
+                        log_warning("Failed to generate stack trace: %s", strerror(-r));
+        }
+
+        if (!core_message)
+#endif
+        core_message = strjoin("MESSAGE=Process ", argv[ARG_PID], " (", argv[ARG_COMM], ") of user ", argv[ARG_UID], " dumped core.", NULL);
+        if (core_message)
+                IOVEC_SET_STRING(iovec[j++], core_message);
+
         /* Optionally store the entire coredump in the journal */
         if (IN_SET(arg_storage, COREDUMP_STORAGE_JOURNAL, COREDUMP_STORAGE_BOTH) &&
             coredump_size <= (off_t) arg_journal_size_max) {
diff --git a/src/journal/coredumpctl.c b/src/journal/coredumpctl.c
index ea459469ee..9eaa8979a0 100644
--- a/src/journal/coredumpctl.c
+++ b/src/journal/coredumpctl.c
@@ -403,7 +403,8 @@ static int print_info(FILE *file, sd_journal *j, bool need_space) {
                 *sgnl = NULL, *exe = NULL, *comm = NULL, *cmdline = NULL,
                 *unit = NULL, *user_unit = NULL, *session = NULL,
                 *boot_id = NULL, *machine_id = NULL, *hostname = NULL,
-                *coredump = NULL, *slice = NULL, *cgroup = NULL, *owner_uid = NULL;
+                *coredump = NULL, *slice = NULL, *cgroup = NULL,
+                *owner_uid = NULL, *message = NULL;
         const void *d;
         size_t l;
 
@@ -427,6 +428,7 @@ static int print_info(FILE *file, sd_journal *j, bool need_space) {
                 retrieve(d, l, "_BOOT_ID", &boot_id);
                 retrieve(d, l, "_MACHINE_ID", &machine_id);
                 retrieve(d, l, "_HOSTNAME", &hostname);
+                retrieve(d, l, "MESSAGE", &message);
         }
 
         if (need_space)
@@ -522,6 +524,14 @@ static int print_info(FILE *file, sd_journal *j, bool need_space) {
                 if (access(coredump, F_OK) >= 0)
                         fprintf(file, "      Coredump: %s\n", coredump);
 
+        if (message) {
+                _cleanup_free_ char *m = NULL;
+
+                m = strreplace(message, "\n", "\n                ");
+
+                fprintf(file, "       Message: %s\n", strstrip(m ?: message));
+        }
+
         return 0;
 }
 
@@ -696,7 +706,7 @@ static int run_gdb(sd_journal *j) {
                         if (errno == ENOENT)
                                 log_error("Coredump neither in journal file nor stored externally on disk.");
                         else
-                                log_error("Failed to access coredump fiile: %s", strerror(-r));
+                                log_error("Failed to access coredump file: %m");
 
                         return -errno;
                 }
diff --git a/src/journal/stacktrace.c b/src/journal/stacktrace.c
new file mode 100644
index 0000000000..6b9d2729f5
--- /dev/null
+++ b/src/journal/stacktrace.c
@@ -0,0 +1,200 @@
+/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
+
+/***
+  This file is part of systemd.
+
+  Copyright 2014 Lennart Poettering
+
+  systemd is free software; you can redistribute it and/or modify it
+  under the terms of the GNU Lesser General Public License as published by
+  the Free Software Foundation; either version 2.1 of the License, or
+  (at your option) any later version.
+
+  systemd is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public License
+  along with systemd; If not, see <http://www.gnu.org/licenses/>.
+***/
+
+#include <dwarf.h>
+#include <elfutils/libdwfl.h>
+
+#include "util.h"
+#include "macro.h"
+#include "stacktrace.h"
+
+#define FRAMES_MAX 64
+#define THREADS_MAX 64
+
+struct stack_context {
+        FILE *f;
+        Dwfl *dwfl;
+        Elf *elf;
+        unsigned n_thread;
+        unsigned n_frame;
+};
+
+static int frame_callback(Dwfl_Frame *frame, void *userdata) {
+        struct stack_context *c = userdata;
+        Dwarf_Addr pc, pc_adjusted, bias = 0;
+        _cleanup_free_ Dwarf_Die *scopes = NULL;
+        const char *fname = NULL, *symbol = NULL;
+        Dwfl_Module *module;
+        bool is_activation;
+
+        assert(frame);
+        assert(c);
+
+        if (c->n_frame >= FRAMES_MAX)
+                return DWARF_CB_ABORT;
+
+        if (!dwfl_frame_pc(frame, &pc, &is_activation))
+                return DWARF_CB_ABORT;
+
+        pc_adjusted = pc - (is_activation ? 0 : 1);
+
+        module = dwfl_addrmodule(c->dwfl, pc_adjusted);
+        if (module) {
+                Dwarf_Die *s, *cudie;
+                int n;
+
+                cudie = dwfl_module_addrdie(module, pc_adjusted, &bias);
+                if (cudie) {
+                        n = dwarf_getscopes(cudie, pc_adjusted - bias, &scopes);
+                        for (s = scopes; s < scopes + n; s++) {
+                                if (IN_SET(dwarf_tag(s), DW_TAG_subprogram, DW_TAG_inlined_subroutine, DW_TAG_entry_point)) {
+                                        Dwarf_Attribute *a, space;
+
+                                        a = dwarf_attr_integrate(s, DW_AT_MIPS_linkage_name, &space);
+                                        if (!a)
+                                                a = dwarf_attr_integrate(s, DW_AT_linkage_name, &space);
+                                        if (a)
+                                                symbol = dwarf_formstring(a);
+                                        if (!symbol)
+                                                symbol = dwarf_diename(s);
+
+                                        if (symbol)
+                                                break;
+                                }
+                        }
+                }
+
+                if (!symbol)
+                        symbol = dwfl_module_addrname(module, pc_adjusted);
+
+                fname = dwfl_module_info(module, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
+        }
+
+        fprintf(c->f, "#%-2u 0x%016" PRIx64 " %s (%s)\n", c->n_frame, (uint64_t) pc, strna(symbol), strna(fname));
+        c->n_frame ++;
+
+        return DWARF_CB_OK;
+}
+
+static int thread_callback(Dwfl_Thread *thread, void *userdata) {
+        struct stack_context *c = userdata;
+        pid_t tid;
+
+        assert(thread);
+        assert(c);
+
+        if (c->n_thread >= THREADS_MAX)
+                return DWARF_CB_ABORT;
+
+        if (c->n_thread != 0)
+                fputc('\n', c->f);
+
+        c->n_frame = 0;
+
+        tid = dwfl_thread_tid(thread);
+        fprintf(c->f, "Stack trace of thread " PID_FMT ":\n", tid);
+
+        if (dwfl_thread_getframes(thread, frame_callback, c) < 0)
+                return DWARF_CB_ABORT;
+
+        c->n_thread ++;
+
+        return DWARF_CB_OK;
+}
+
+int coredump_make_stack_trace(int fd, const char *executable, char **ret) {
+
+        static const Dwfl_Callbacks callbacks = {
+                .find_elf = dwfl_build_id_find_elf,
+                .find_debuginfo = dwfl_standard_find_debuginfo,
+        };
+
+        struct stack_context c = {};
+        char *buf = NULL;
+        size_t sz = 0;
+        int r;
+
+        assert(fd >= 0);
+        assert(ret);
+
+        if (lseek(fd, 0, SEEK_SET) == (off_t) -1)
+                return -errno;
+
+        c.f = open_memstream(&buf, &sz);
+        if (!c.f)
+                return -ENOMEM;
+
+        elf_version(EV_CURRENT);
+
+        c.elf = elf_begin(fd, ELF_C_READ_MMAP, NULL);
+        if (!c.elf) {
+                r = -EINVAL;
+                goto finish;
+        }
+
+        c.dwfl = dwfl_begin(&callbacks);
+        if (!c.dwfl) {
+                r = -EINVAL;
+                goto finish;
+        }
+
+        if (dwfl_core_file_report(c.dwfl, c.elf, executable) < 0) {
+                r = -EINVAL;
+                goto finish;
+        }
+
+        if (dwfl_report_end(c.dwfl, NULL, NULL) != 0) {
+                r = -EINVAL;
+                goto finish;
+        }
+
+        if (dwfl_core_file_attach(c.dwfl, c.elf) < 0) {
+                r = -EINVAL;
+                goto finish;
+        }
+
+        if (dwfl_getthreads(c.dwfl, thread_callback, &c) < 0) {
+                r = -EINVAL;
+                goto finish;
+        }
+
+        fclose(c.f);
+        c.f = NULL;
+
+        *ret = buf;
+        buf = NULL;
+
+        r = 0;
+
+finish:
+        if (c.dwfl)
+                dwfl_end(c.dwfl);
+
+        if (c.elf)
+                elf_end(c.elf);
+
+        if (c.f)
+                fclose(c.f);
+
+        free(buf);
+
+        return r;
+}
diff --git a/src/journal/stacktrace.h b/src/journal/stacktrace.h
new file mode 100644
index 0000000000..189e5c4597
--- /dev/null
+++ b/src/journal/stacktrace.h
@@ -0,0 +1,24 @@
+/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
+
+#pragma once
+
+/***
+  This file is part of systemd.
+
+  Copyright 2014 Lennart Poettering
+
+  systemd is free software; you can redistribute it and/or modify it
+  under the terms of the GNU Lesser General Public License as published by
+  the Free Software Foundation; either version 2.1 of the License, or
+  (at your option) any later version.
+
+  systemd is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public License
+  along with systemd; If not, see <http://www.gnu.org/licenses/>.
+***/
+
+int coredump_make_stack_trace(int fd, const char *executable, char **ret);