journalctl: add new --sync switch for syncing the journal to disk

With this new "--sync" switch we add a synchronous way to sync
everything queued to disk, and return only after that's complete. This
command gives the guarantee that anything queued before has hit the disk
before the command returns.

While we are at it, also improve the man pages and help text for
journalctl a bit.

5 files changed, 171 insertions, 18 deletions

diff --git a/man/journalctl.xml b/man/journalctl.xml
index a3192539dc..2160f3cba2 100644
--- a/man/journalctl.xml
+++ b/man/journalctl.xml
@@ -773,13 +773,31 @@
       </varlistentry>
 
       <varlistentry>
+        <term><option>--sync</option></term>
+
+        <listitem><para>Ask the journal daemon to write all yet
+        unwritten journal data to the backing file system and
+        synchronize all journals. This call does not return until the
+        operation is complete. This command guarantees that any log
+        messages written before its invocation are safely stored on
+        disk at the time it returns.</para></listitem>
+      </varlistentry>
+
+      <varlistentry>
         <term><option>--flush</option></term>
 
         <listitem><para>Asks the journal daemon to flush any log data
         stored in <filename>/run/log/journal</filename> into
-        <filename>/var/log/journal</filename>, if persistent storage is
-        enabled. This call does not return until the operation is
-        complete.</para></listitem>
+        <filename>/var/log/journal</filename>, if persistent storage
+        is enabled. This call does not return until the operation is
+        complete. Note that this call is idempotent: the data is only
+        flushed from <filename>/run/log/journal</filename> into
+        <filename>/var/log/journal</filename> once during system
+        runtime, and this command exits cleanly without executing any
+        operation if this has already has happened. This command
+        effectively guarantees that all data is flushed to
+        <filename>/var/log/journal</filename> at the time it
+        returns.</para></listitem>
       </varlistentry>
 
       <varlistentry>
diff --git a/man/systemd-journald.service.xml b/man/systemd-journald.service.xml
index 21fd684b8b..f1054b03bb 100644
--- a/man/systemd-journald.service.xml
+++ b/man/systemd-journald.service.xml
@@ -131,15 +131,30 @@ systemd-tmpfiles --create --prefix /var/log/journal</programlisting>
         this is enabled). This must be used after
         <filename>/var/</filename> is mounted, as otherwise log data
         from <filename>/run</filename> is never flushed to
-        <filename>/var</filename> regardless of the
-        configuration.</para></listitem>
+        <filename>/var</filename> regardless of the configuration. The
+        <command>journalctl --flush</command> command uses this signal
+        to request flushing of the journal files, and then waits for
+        the operation to complete. See
+        <citerefentry><refentrytitle>journalctl</refentrytitle><manvolnum>1</manvolnum></citerefentry>
+        for details.</para></listitem>
       </varlistentry>
 
       <varlistentry>
         <term>SIGUSR2</term>
 
         <listitem><para>Request immediate rotation of the journal
-        files.</para></listitem>
+        files. The <command>journalctl --rotate</command> command uses
+        this signal to request journal file
+        rotation.</para></listitem>
+      </varlistentry>
+
+      <varlistentry>
+        <term>SIGRTMIN+1</term>
+
+        <listitem><para>Request that all unwritten log data is written
+        to disk. The <command>journalctl --sync</command> command uses
+        this signal to trigger journal synchronization, and then waits
+        for the operation to complete.</para></listitem>
       </varlistentry>
     </variablelist>
   </refsect1>
@@ -261,7 +276,7 @@ systemd-tmpfiles --create --prefix /var/log/journal</programlisting>
       <citerefentry><refentrytitle>systemd-coredump</refentrytitle><manvolnum>8</manvolnum></citerefentry>,
       <citerefentry project='die-net'><refentrytitle>setfacl</refentrytitle><manvolnum>1</manvolnum></citerefentry>,
       <citerefentry><refentrytitle>sd_journal_print</refentrytitle><manvolnum>4</manvolnum></citerefentry>,
-      <command>pydoc systemd.journal</command>.
+      <command>pydoc systemd.journal</command>
     </para>
   </refsect1>
 
diff --git a/src/journal/journalctl.c b/src/journal/journalctl.c
index 277adba904..071349666c 100644
--- a/src/journal/journalctl.c
+++ b/src/journal/journalctl.c
@@ -135,6 +135,7 @@ static enum {
         ACTION_FLUSH,
         ACTION_ROTATE,
         ACTION_VACUUM,
+        ACTION_SYNC,
 } arg_action = ACTION_SHOW;
 
 typedef struct BootId {
@@ -201,7 +202,7 @@ static void help(void) {
 
         printf("%s [OPTIONS...] [MATCHES...]\n\n"
                "Query the journal.\n\n"
-               "Flags:\n"
+               "Options:\n"
                "     --system              Show the system journal\n"
                "     --user                Show the user journal for the current user\n"
                "  -M --machine=CONTAINER   Operate on local container\n"
@@ -234,7 +235,7 @@ static void help(void) {
                "  -m --merge               Show entries from all available journals\n"
                "  -D --directory=PATH      Show journal files from directory\n"
                "     --file=PATH           Show journal file\n"
-               "     --root=ROOT           Operate on catalog files underneath the root ROOT\n"
+               "     --root=ROOT           Operate on catalog files below a root directory\n"
 #ifdef HAVE_GCRYPT
                "     --interval=TIME       Time interval for changing the FSS sealing key\n"
                "     --verify-key=KEY      Specify FSS verification key\n"
@@ -244,20 +245,21 @@ static void help(void) {
                "  -h --help                Show this help text\n"
                "     --version             Show package version\n"
                "  -F --field=FIELD         List all values that a specified field takes\n"
-               "     --new-id128           Generate a new 128-bit ID\n"
                "     --disk-usage          Show total disk usage of all journal files\n"
                "     --vacuum-size=BYTES   Reduce disk usage below specified size\n"
                "     --vacuum-files=INT    Leave only the specified number of journal files\n"
                "     --vacuum-time=TIME    Remove journal files older than specified time\n"
+               "     --verify              Verify journal file consistency\n"
+               "     --sync                Synchronize unwritten journal messages to disk\n"
                "     --flush               Flush all journal data from /run into /var\n"
                "     --rotate              Request immediate rotation of the journal files\n"
                "     --header              Show journal header information\n"
                "     --list-catalog        Show all message IDs in the catalog\n"
                "     --dump-catalog        Show entries in the message catalog\n"
                "     --update-catalog      Update the message catalog database\n"
+               "     --new-id128           Generate a new 128-bit ID\n"
 #ifdef HAVE_GCRYPT
                "     --setup-keys          Generate a new FSS key pair\n"
-               "     --verify              Verify journal file consistency\n"
 #endif
                , program_invocation_short_name);
 }
@@ -289,6 +291,7 @@ static int parse_argv(int argc, char *argv[]) {
                 ARG_UPDATE_CATALOG,
                 ARG_FORCE,
                 ARG_UTC,
+                ARG_SYNC,
                 ARG_FLUSH,
                 ARG_ROTATE,
                 ARG_VACUUM_SIZE,
@@ -345,6 +348,7 @@ static int parse_argv(int argc, char *argv[]) {
                 { "machine",        required_argument, NULL, 'M'                },
                 { "utc",            no_argument,       NULL, ARG_UTC            },
                 { "flush",          no_argument,       NULL, ARG_FLUSH          },
+                { "sync",           no_argument,       NULL, ARG_SYNC           },
                 { "rotate",         no_argument,       NULL, ARG_ROTATE         },
                 { "vacuum-size",    required_argument, NULL, ARG_VACUUM_SIZE    },
                 { "vacuum-files",   required_argument, NULL, ARG_VACUUM_FILES   },
@@ -729,6 +733,10 @@ static int parse_argv(int argc, char *argv[]) {
                         arg_action = ACTION_ROTATE;
                         break;
 
+                case ARG_SYNC:
+                        arg_action = ACTION_SYNC;
+                        break;
+
                 case '?':
                         return -EINVAL;
 
@@ -1782,10 +1790,8 @@ static int flush_to_var(void) {
                         &error,
                         NULL,
                         "ssi", "systemd-journald.service", "main", SIGUSR1);
-        if (r < 0) {
-                log_error("Failed to kill journal service: %s", bus_error_message(&error, r));
-                return r;
-        }
+        if (r < 0)
+                return log_error_errno(r, "Failed to kill journal service: %s", bus_error_message(&error, r));
 
         mkdir_p("/run/systemd/journal", 0755);
 
@@ -1840,6 +1846,85 @@ static int rotate(void) {
         return 0;
 }
 
+static int sync_journal(void) {
+        _cleanup_bus_flush_close_unref_ sd_bus *bus = NULL;
+        _cleanup_close_ int watch_fd = -1;
+        usec_t start;
+        int r;
+
+        start = now(CLOCK_REALTIME);
+
+        /* Let's watch /run/systemd/sync until it's mtime is above
+         * the time we started the sync. Let's enqueue SIGRTMIN+1 to
+         * start the sync. */
+
+        for (;;) {
+                struct stat st;
+
+                /* See if a sync happened by now. */
+                if (stat("/run/systemd/journal/synced", &st) < 0) {
+                        if (errno != ENOENT)
+                                return log_error_errno(errno, "Failed to stat /run/systemd/journal/synced: %m");
+                } else {
+                        if (timespec_load(&st.st_mtim) >= start)
+                                return 0;
+                }
+
+                /* Let's ask for a sync, but only once. */
+                if (!bus) {
+                        _cleanup_bus_error_free_ sd_bus_error error = SD_BUS_ERROR_NULL;
+
+                        r = bus_connect_system_systemd(&bus);
+                        if (r < 0)
+                                return log_error_errno(r, "Failed to get D-Bus connection: %m");
+
+                        r = sd_bus_call_method(
+                                        bus,
+                                        "org.freedesktop.systemd1",
+                                        "/org/freedesktop/systemd1",
+                                        "org.freedesktop.systemd1.Manager",
+                                        "KillUnit",
+                                        &error,
+                                        NULL,
+                                        "ssi", "systemd-journald.service", "main", SIGRTMIN+1);
+                        if (r < 0)
+                                return log_error_errno(r, "Failed to kill journal service: %s", bus_error_message(&error, r));
+
+                        continue;
+                }
+
+                /* Let's install the inotify watch, if we didn't do that yet. */
+                if (watch_fd < 0) {
+
+                        mkdir_p("/run/systemd/journal", 0755);
+
+                        watch_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
+                        if (watch_fd < 0)
+                                return log_error_errno(errno, "Failed to create inotify watch: %m");
+
+                        r = inotify_add_watch(watch_fd, "/run/systemd/journal", IN_CREATE|IN_ATTRIB|IN_DONT_FOLLOW|IN_ONLYDIR);
+                        if (r < 0)
+                                return log_error_errno(errno, "Failed to watch journal directory: %m");
+
+                        /* Recheck the flag file immediately, so that we don't miss any event since the last check. */
+                        continue;
+                }
+
+                /* OK, all preparatory steps done, let's wait until
+                 * inotify reports an event. */
+
+                r = fd_wait_for_event(watch_fd, POLLIN, USEC_INFINITY);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to wait for event: %m");
+
+                r = flush_fd(watch_fd);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to flush inotify events: %m");
+        }
+
+        return 0;
+}
+
 int main(int argc, char *argv[]) {
         int r;
         _cleanup_journal_close_ sd_journal *j = NULL;
@@ -1875,6 +1960,11 @@ int main(int argc, char *argv[]) {
                 goto finish;
         }
 
+        if (arg_action == ACTION_SYNC) {
+                r = sync_journal();
+                goto finish;
+        }
+
         if (arg_action == ACTION_ROTATE) {
                 r = rotate();
                 goto finish;
diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c
index a6e5e4a20f..f0a3c82d98 100644
--- a/src/journal/journald-server.c
+++ b/src/journal/journald-server.c
@@ -1243,7 +1243,7 @@ static int dispatch_sigusr1(sd_event_source *es, const struct signalfd_siginfo *
 
         assert(s);
 
-        log_info("Received request to flush runtime journal from PID %"PRIu32, si->ssi_pid);
+        log_info("Received request to flush runtime journal from PID " PID_FMT, si->ssi_pid);
 
         server_flush_to_var(s);
         server_sync(s);
@@ -1259,7 +1259,7 @@ static int dispatch_sigusr2(sd_event_source *es, const struct signalfd_siginfo *
 
         assert(s);
 
-        log_info("Received request to rotate journal from PID %"PRIu32, si->ssi_pid);
+        log_info("Received request to rotate journal from PID " PID_FMT, si->ssi_pid);
         server_rotate(s);
         server_vacuum(s, true, true);
 
@@ -1277,12 +1277,27 @@ static int dispatch_sigterm(sd_event_source *es, const struct signalfd_siginfo *
         return 0;
 }
 
+static int dispatch_sigrtmin1(sd_event_source *es, const struct signalfd_siginfo *si, void *userdata) {
+        Server *s = userdata;
+
+        assert(s);
+
+        log_debug("Received request to sync from PID " PID_FMT, si->ssi_pid);
+
+        server_sync(s);
+
+        /* Let clients know when the most recent sync happened. */
+        (void) touch("/run/systemd/journal/synced");
+
+        return 0;
+}
+
 static int setup_signals(Server *s) {
         int r;
 
         assert(s);
 
-        assert(sigprocmask_many(SIG_SETMASK, NULL, SIGINT, SIGTERM, SIGUSR1, SIGUSR2, -1) >= 0);
+        assert(sigprocmask_many(SIG_SETMASK, NULL, SIGINT, SIGTERM, SIGUSR1, SIGUSR2, SIGRTMIN+1, -1) >= 0);
 
         r = sd_event_add_signal(s->event, &s->sigusr1_event_source, SIGUSR1, dispatch_sigusr1, s);
         if (r < 0)
@@ -1312,6 +1327,19 @@ static int setup_signals(Server *s) {
         if (r < 0)
                 return r;
 
+        /* SIGRTMIN+1 causes an immediate sync. We process this very
+         * late, so that everything else queued at this point is
+         * really written to disk. Clients can watch
+         * /run/systemd/journal/synced with inotify until its mtime
+         * changes to see when a sync happened. */
+        r = sd_event_add_signal(s->event, &s->sigrtmin1_event_source, SIGRTMIN+1, dispatch_sigrtmin1, s);
+        if (r < 0)
+                return r;
+
+        r = sd_event_source_set_priority(s->sigrtmin1_event_source, SD_EVENT_PRIORITY_NORMAL+15);
+        if (r < 0)
+                return r;
+
         return 0;
 }
 
@@ -1869,6 +1897,7 @@ void server_done(Server *s) {
         sd_event_source_unref(s->sigusr2_event_source);
         sd_event_source_unref(s->sigterm_event_source);
         sd_event_source_unref(s->sigint_event_source);
+        sd_event_source_unref(s->sigrtmin1_event_source);
         sd_event_source_unref(s->hostname_event_source);
         sd_event_source_unref(s->notify_event_source);
         sd_event_source_unref(s->watchdog_event_source);
diff --git a/src/journal/journald-server.h b/src/journal/journald-server.h
index 03a61bd2ed..dcc21bb7c3 100644
--- a/src/journal/journald-server.h
+++ b/src/journal/journald-server.h
@@ -72,6 +72,7 @@ struct Server {
         sd_event_source *sigusr2_event_source;
         sd_event_source *sigterm_event_source;
         sd_event_source *sigint_event_source;
+        sd_event_source *sigrtmin1_event_source;
         sd_event_source *hostname_event_source;
         sd_event_source *notify_event_source;
         sd_event_source *watchdog_event_source;