journald: automatically rotate journal files when the clock jumps backwards

As soon as we notice that the clock jumps backwards, rotate journal files. This
is beneficial, as this makes sure that the entries in journal files remain
strictly ordered internally, and thus the bisection algorithm applied on it is
not confused.

This should help avoiding borked wallclock-based bisection on journal files as
witnessed in #4278.

2 files changed, 25 insertions, 6 deletions

diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c
index 7227c80c86..28aea35d18 100644
--- a/src/journal/journald-server.c
+++ b/src/journal/journald-server.c
@@ -627,8 +627,8 @@ static bool shall_try_append_again(JournalFile *f, int r) {
 }
 
 static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, unsigned n, int priority) {
+        bool vacuumed = false, rotate = false;
         struct dual_timestamp ts;
-        bool vacuumed = false;
         JournalFile *f;
         int r;
 
@@ -642,12 +642,27 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, unsigned
         assert_se(sd_event_now(s->event, CLOCK_REALTIME, &ts.realtime) >= 0);
         assert_se(sd_event_now(s->event, CLOCK_MONOTONIC, &ts.monotonic) >= 0);
 
-        f = find_journal(s, uid);
-        if (!f)
-                return;
+        if (ts.realtime < s->last_realtime_clock) {
+                /* When the time jumps backwards, let's immediately rotate. Of course, this should not happen during
+                 * regular operation. However, when it does happen, then we should make sure that we start fresh files
+                 * to ensure that the entries in the journal files are strictly ordered by time, in order to ensure
+                 * bisection works correctly. */
 
-        if (journal_file_rotate_suggested(f, s->max_file_usec)) {
-                log_debug("%s: Journal header limits reached or header out-of-date, rotating.", f->path);
+                log_debug("Time jumped backwards, rotating.");
+                rotate = true;
+        } else {
+
+                f = find_journal(s, uid);
+                if (!f)
+                        return;
+
+                if (journal_file_rotate_suggested(f, s->max_file_usec)) {
+                        log_debug("%s: Journal header limits reached or header out-of-date, rotating.", f->path);
+                        rotate = true;
+                }
+        }
+
+        if (rotate) {
                 server_rotate(s);
                 server_vacuum(s, false, false);
                 vacuumed = true;
@@ -657,6 +672,8 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, unsigned
                         return;
         }
 
+        s->last_realtime_clock = ts.realtime;
+
         r = journal_file_append_entry(f, &ts, iovec, n, &s->seqnum, NULL, NULL);
         if (r >= 0) {
                 server_schedule_sync(s, priority);
diff --git a/src/journal/journald-server.h b/src/journal/journald-server.h
index dfb5724794..cc68a0a690 100644
--- a/src/journal/journald-server.h
+++ b/src/journal/journald-server.h
@@ -149,6 +149,8 @@ struct Server {
         char *cgroup_root;
 
         usec_t watchdog_usec;
+
+        usec_t last_realtime_clock;
 };
 
 #define SERVER_MACHINE_ID(s) ((s)->machine_id_field + strlen("_MACHINE_ID="))