journald: log when we fail to forward messages to syslog

https://bugzilla.redhat.com/show_bug.cgi?id=847207

5 files changed, 35 insertions, 2 deletions

diff --git a/src/journal/journald-syslog.c b/src/journal/journald-syslog.c
index 9db9198a49..d3e0f6f6c1 100644
--- a/src/journal/journald-syslog.c
+++ b/src/journal/journald-syslog.c
@@ -23,12 +23,16 @@
 #include <stddef.h>
 #include <sys/epoll.h>
 
+#include "systemd/sd-messages.h"
 #include "socket-util.h"
 #include "journald.h"
 #include "journald-syslog.h"
 #include "journald-kmsg.h"
 #include "journald-console.h"
 
+/* Warn once every 30s if we missed syslog message */
+#define WARN_FORWARD_SYSLOG_MISSED_USEC (30 * USEC_PER_SEC)
+
 static void forward_syslog_iovec(Server *s, const struct iovec *iovec, unsigned n_iovec, struct ucred *ucred, struct timeval *tv) {
         struct msghdr msghdr;
         struct cmsghdr *cmsg;
@@ -74,8 +78,10 @@ static void forward_syslog_iovec(Server *s, const struct iovec *iovec, unsigned
 
         /* The socket is full? I guess the syslog implementation is
          * too slow, and we shouldn't wait for that... */
-        if (errno == EAGAIN)
+        if (errno == EAGAIN) {
+                s->n_forward_syslog_missed++;
                 return;
+        }
 
         if (ucred && errno == ESRCH) {
                 struct ucred u;
@@ -91,8 +97,10 @@ static void forward_syslog_iovec(Server *s, const struct iovec *iovec, unsigned
                 if (sendmsg(s->syslog_fd, &msghdr, MSG_NOSIGNAL) >= 0)
                         return;
 
-                if (errno == EAGAIN)
+                if (errno == EAGAIN) {
+                        s->n_forward_syslog_missed++;
                         return;
+                }
         }
 
         if (errno != ENOENT)
@@ -464,3 +472,20 @@ int server_open_syslog_socket(Server *s) {
 
         return 0;
 }
+
+void server_maybe_warn_forward_syslog_missed(Server *s) {
+        usec_t n;
+        assert(s);
+
+        if (s->n_forward_syslog_missed <= 0)
+                return;
+
+        n = now(CLOCK_MONOTONIC);
+        if (s->last_warn_forward_syslog_missed + WARN_FORWARD_SYSLOG_MISSED_USEC > n)
+                return;
+
+        server_driver_message(s, SD_MESSAGE_FORWARD_SYSLOG_MISSED, "Forwarding to syslog missed %u messages.", s->n_forward_syslog_missed);
+
+        s->n_forward_syslog_missed = 0;
+        s->last_warn_forward_syslog_missed = n;
+}
diff --git a/src/journal/journald-syslog.h b/src/journal/journald-syslog.h
index 8845d28df2..e009ab7e72 100644
--- a/src/journal/journald-syslog.h
+++ b/src/journal/journald-syslog.h
@@ -32,3 +32,5 @@ void server_forward_syslog(Server *s, int priority, const char *identifier, cons
 
 void server_process_syslog_message(Server *s, const char *buf, struct ucred *ucred, struct timeval *tv, const char *label, size_t label_len);
 int server_open_syslog_socket(Server *s);
+
+void server_maybe_warn_forward_syslog_missed(Server *s);
diff --git a/src/journal/journald.c b/src/journal/journald.c
index 871bbb636f..74c80f5a08 100644
--- a/src/journal/journald.c
+++ b/src/journal/journald.c
@@ -1553,6 +1553,7 @@ int main(int argc, char *argv[]) {
                 }
 
                 server_maybe_append_tags(&server);
+                server_maybe_warn_forward_syslog_missed(&server);
         }
 
         log_debug("systemd-journald stopped as pid %lu", (unsigned long) getpid());
diff --git a/src/journal/journald.h b/src/journal/journald.h
index c126d198b1..cd06b40c2f 100644
--- a/src/journal/journald.h
+++ b/src/journal/journald.h
@@ -82,6 +82,9 @@ typedef struct Server {
         bool forward_to_syslog;
         bool forward_to_console;
 
+        unsigned n_forward_syslog_missed;
+        usec_t last_warn_forward_syslog_missed;
+
         uint64_t cached_available_space;
         usec_t cached_available_space_timestamp;
 
diff --git a/src/systemd/sd-messages.h b/src/systemd/sd-messages.h
index 91f928c00b..698aa60eb7 100644
--- a/src/systemd/sd-messages.h
+++ b/src/systemd/sd-messages.h
@@ -58,6 +58,8 @@ extern "C" {
 #define SD_MESSAGE_UNIT_RELOADING   SD_ID128_MAKE(d3,4d,03,7f,ff,18,47,e6,ae,66,9a,37,0e,69,47,25)
 #define SD_MESSAGE_UNIT_RELOADED    SD_ID128_MAKE(7b,05,eb,c6,68,38,42,22,ba,a8,88,11,79,cf,da,54)
 
+#define SD_MESSAGE_FORWARD_SYSLOG_MISSED SD_ID128_MAKE(00,27,22,9c,a0,64,41,81,a7,6c,4e,92,45,8a,fa,2e)
+
 #ifdef __cplusplus
 }
 #endif