journal: rework directory enumeration/watch logic

There's now sd_journal_new_directory() for watching specific journal
directories. This is exposed in journalctl -D.

sd_journal_wait() and sd_journal_process() now return whether changes in
the journal are invalidating or just appending.

We now create inotify kernel watches only when we actually need them

7 files changed, 464 insertions, 291 deletions

diff --git a/TODO b/TODO
index 3f5c3e098b..5df72cdc5e 100644
--- a/TODO
+++ b/TODO
@@ -34,6 +34,8 @@ Bugfixes:
 
 Features:
 
+* compile libsystemd-journal statically into journalctl so that we can share util.c and suchlike
+
 * replace BindTo= by BindsTo=, but keep old name for compat
 
 * switch-root: sockets need relabelling
@@ -50,8 +52,6 @@ Features:
 
 * .device aliases need to be implemented with the "following" logic, probably.
 
-* add sd_journal_wait() to make things easier for sync programs that just want to wait for changes
-
 * refuse taking lower-case variable names in sd_journal_send() and friends.
 
 * when running as user instance: implicitly default to WorkingDirectory=$HOME for all services.
@@ -206,9 +206,6 @@ Features:
 
 * support container_ttys=
 
-* journald: make configurable "store-on-var", "store-on-run", "dont-store", "auto"
-  (store-persistent, store-volatile?)
-
 * introduce mix of BindTo and Requisite
 
 * journalctl: show multiline log messages sanely, expand tabs, and show all valid utf8 messages
diff --git a/man/journalctl.xml b/man/journalctl.xml
index bbc9b34151..ffe988a619 100644
--- a/man/journalctl.xml
+++ b/man/journalctl.xml
@@ -218,6 +218,18 @@
                         </varlistentry>
 
                         <varlistentry>
+                                <term><option>--directory=</option></term>
+                                <term><option>-D</option></term>
+
+                                <listitem><para>Takes an absolute
+                                directory path as argument. If
+                                specified will opearte on the
+                                specified journal directory instead of
+                                the default runtime and system journal
+                                paths.</para></listitem>
+                        </varlistentry>
+
+                        <varlistentry>
                                 <term><option>--new-id128</option></term>
 
                                 <listitem><para>Instead of showing
diff --git a/src/journal/journal-internal.h b/src/journal/journal-internal.h
index bcffa35055..929dfcdcb8 100644
--- a/src/journal/journal-internal.h
+++ b/src/journal/journal-internal.h
@@ -31,6 +31,14 @@
 #include "list.h"
 
 typedef struct Match Match;
+typedef struct Location Location;
+typedef struct Directory Directory;
+
+typedef enum location_type {
+        LOCATION_HEAD,
+        LOCATION_TAIL,
+        LOCATION_DISCRETE
+} location_type_t;
 
 struct Match {
         char *data;
@@ -40,13 +48,7 @@ struct Match {
         LIST_FIELDS(Match, matches);
 };
 
-typedef enum location_type {
-        LOCATION_HEAD,
-        LOCATION_TAIL,
-        LOCATION_DISCRETE
-} location_type_t;
-
-typedef struct Location {
+struct Location {
         location_type_t type;
 
         uint64_t seqnum;
@@ -62,7 +64,13 @@ typedef struct Location {
 
         uint64_t xor_hash;
         bool xor_hash_set;
-} Location;
+};
+
+struct Directory {
+        char *path;
+        int wd;
+        bool is_root;
+};
 
 struct sd_journal {
         int flags;
@@ -73,12 +81,15 @@ struct sd_journal {
         JournalFile *current_file;
         uint64_t current_field;
 
+        Hashmap *directories_by_path;
+        Hashmap *directories_by_wd;
+
         int inotify_fd;
-        Hashmap *inotify_wd_dirs;
-        Hashmap *inotify_wd_roots;
 
         LIST_HEAD(Match, matches);
         unsigned n_matches;
+
+        unsigned current_invalidate_counter, last_invalidate_counter;
 };
 
 #endif
diff --git a/src/journal/journalctl.c b/src/journal/journalctl.c
index 7d8b8e51c8..4c975d3e7c 100644
--- a/src/journal/journalctl.c
+++ b/src/journal/journalctl.c
@@ -39,6 +39,7 @@
 #include "build.h"
 #include "pager.h"
 #include "logs-show.h"
+#include "strv.h"
 
 static OutputMode arg_output = OUTPUT_SHORT;
 static bool arg_follow = false;
@@ -50,6 +51,7 @@ static bool arg_new_id128 = false;
 static bool arg_quiet = false;
 static bool arg_local = false;
 static bool arg_this_boot = false;
+static const char *arg_directory = NULL;
 
 static int help(void) {
 
@@ -67,6 +69,7 @@ static int help(void) {
                "  -q --quiet          Don't show privilege warning\n"
                "  -l --local          Only local entries\n"
                "  -b --this-boot      Show data only from current boot\n"
+               "  -D --directory=PATH Show journal files from directory\n"
                "     --new-id128      Generate a new 128 Bit id\n",
                program_invocation_short_name);
 
@@ -95,6 +98,7 @@ static int parse_argv(int argc, char *argv[]) {
                 { "quiet",     no_argument,       NULL, 'q'           },
                 { "local",     no_argument,       NULL, 'l'           },
                 { "this-boot", no_argument,       NULL, 'b'           },
+                { "directory", required_argument, NULL, 'D'           },
                 { NULL,        0,                 NULL, 0             }
         };
 
@@ -103,7 +107,7 @@ static int parse_argv(int argc, char *argv[]) {
         assert(argc >= 0);
         assert(argv);
 
-        while ((c = getopt_long(argc, argv, "hfo:an:qlb", options, NULL)) >= 0) {
+        while ((c = getopt_long(argc, argv, "hfo:an:qlbD:", options, NULL)) >= 0) {
 
                 switch (c) {
 
@@ -166,6 +170,10 @@ static int parse_argv(int argc, char *argv[]) {
                         arg_this_boot = true;
                         break;
 
+                case 'D':
+                        arg_directory = optarg;
+                        break;
+
                 case '?':
                         return -EINVAL;
 
@@ -209,68 +217,26 @@ static int generate_new_id128(void) {
         return 0;
 }
 
-int main(int argc, char *argv[]) {
-        int r, i;
-        sd_journal *j = NULL;
-        unsigned line = 0;
-        bool need_seek = false;
-        struct stat st;
-
-        log_parse_environment();
-        log_open();
-
-        r = parse_argv(argc, argv);
-        if (r <= 0)
-                goto finish;
-
-        if (arg_new_id128) {
-                r = generate_new_id128();
-                goto finish;
-        }
-
-#ifdef HAVE_ACL
-        if (!arg_quiet && geteuid() != 0 && in_group("adm") <= 0)
-                log_warning("Showing user generated messages only. Users in the group 'adm' can see all messages. Pass -q to turn this message off.");
-#endif
-
-        r = sd_journal_open(&j, arg_local ? SD_JOURNAL_LOCAL_ONLY : 0);
-        if (r < 0) {
-                log_error("Failed to open journal: %s", strerror(-r));
-                goto finish;
-        }
-
-        if (arg_this_boot) {
-                char match[9+32+1] = "_BOOT_ID=";
-                sd_id128_t boot_id;
+static int add_matches(sd_journal *j, char **args) {
+        char **i;
+        int r;
 
-                r = sd_id128_get_boot(&boot_id);
-                if (r < 0) {
-                        log_error("Failed to get boot id: %s", strerror(-r));
-                        goto finish;
-                }
+        assert(j);
 
-                sd_id128_to_string(boot_id, match + 9);
+        STRV_FOREACH(i, args) {
 
-                r = sd_journal_add_match(j, match, strlen(match));
-                if (r < 0) {
-                        log_error("Failed to add match: %s", strerror(-r));
-                        goto finish;
-                }
-        }
-
-        for (i = optind; i < argc; i++) {
-                if (path_is_absolute(argv[i])) {
-                        char *p = NULL;
+                if (path_is_absolute(*i)) {
+                        char *p;
                         const char *path;
+                        struct stat st;
 
-                        p = canonicalize_file_name(argv[i]);
-                        path = p ? p : argv[i];
+                        p = canonicalize_file_name(*i);
+                        path = p ? p : *i;
 
                         if (stat(path, &st) < 0)  {
                                 free(p);
                                 log_error("Couldn't stat file: %m");
-                                r = -errno;
-                                goto finish;
+                                return -errno;
                         }
 
                         if (S_ISREG(st.st_mode) && (0111 & st.st_mode)) {
@@ -280,27 +246,95 @@ int main(int argc, char *argv[]) {
                                 if (!t) {
                                         free(p);
                                         log_error("Out of memory");
-                                        goto finish;
+                                        return -ENOMEM;
                                 }
 
                                 r = sd_journal_add_match(j, t, strlen(t));
                                 free(t);
                         } else {
                                 free(p);
-                                log_error("File is not a regular file or is not executable: %s", argv[i]);
-                                goto finish;
+                                log_error("File is not a regular file or is not executable: %s", *i);
+                                return -EINVAL;
                         }
 
                         free(p);
                 } else
-                        r = sd_journal_add_match(j, argv[i], strlen(argv[i]));
+                        r = sd_journal_add_match(j, *i, strlen(*i));
 
                 if (r < 0) {
                         log_error("Failed to add match: %s", strerror(-r));
-                        goto finish;
+                        return r;
                 }
         }
 
+        return 0;
+}
+
+static int add_this_boot(sd_journal *j) {
+        char match[9+32+1] = "_BOOT_ID=";
+        sd_id128_t boot_id;
+        int r;
+
+        if (!arg_this_boot)
+                return 0;
+
+        r = sd_id128_get_boot(&boot_id);
+        if (r < 0) {
+                log_error("Failed to get boot id: %s", strerror(-r));
+                return r;
+        }
+
+        sd_id128_to_string(boot_id, match + 9);
+        r = sd_journal_add_match(j, match, strlen(match));
+        if (r < 0) {
+                log_error("Failed to add match: %s", strerror(-r));
+                return r;
+        }
+
+        return 0;
+}
+
+int main(int argc, char *argv[]) {
+        int r;
+        sd_journal *j = NULL;
+        unsigned line = 0;
+        bool need_seek = false;
+
+        log_parse_environment();
+        log_open();
+
+        r = parse_argv(argc, argv);
+        if (r <= 0)
+                goto finish;
+
+        if (arg_new_id128) {
+                r = generate_new_id128();
+                goto finish;
+        }
+
+#ifdef HAVE_ACL
+        if (!arg_quiet && geteuid() != 0 && in_group("adm") <= 0)
+                log_warning("Showing user generated messages only. Users in the group 'adm' can see all messages. Pass -q to turn this message off.");
+#endif
+
+        if (arg_directory)
+                r = sd_journal_open_directory(&j, arg_directory, 0);
+        else
+                r = sd_journal_open(&j, arg_local ? SD_JOURNAL_LOCAL_ONLY : 0);
+
+        if (r < 0) {
+                log_error("Failed to open journal: %s", strerror(-r));
+                goto finish;
+        }
+
+        r = add_this_boot(j);
+        if (r < 0)
+                goto finish;
+
+        r = add_matches(j, argv + optind);
+        if (r < 0)
+                goto finish;
+
         if (!arg_quiet) {
                 usec_t start, end;
                 char start_buf[FORMAT_TIMESTAMP_MAX], end_buf[FORMAT_TIMESTAMP_MAX];
diff --git a/src/journal/libsystemd-journal.sym b/src/journal/libsystemd-journal.sym
index d1ba9e8e31..fa4519a682 100644
--- a/src/journal/libsystemd-journal.sym
+++ b/src/journal/libsystemd-journal.sym
@@ -61,4 +61,5 @@ global:
 LIBSYSTEMD_JOURNAL_187 {
 global:
         sd_journal_wait;
+        sd_journal_open_directory;
 } LIBSYSTEMD_JOURNAL_184;
diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c
index 149dc10bdf..6331f042ce 100644
--- a/src/journal/sd-journal.c
+++ b/src/journal/sd-journal.c
@@ -935,8 +935,8 @@ _public_ int sd_journal_seek_tail(sd_journal *j) {
         return 0;
 }
 
-static int add_file(sd_journal *j, const char *prefix, const char *dir, const char *filename) {
-        char *fn;
+static int add_file(sd_journal *j, const char *prefix, const char *filename) {
+        char *path;
         int r;
         JournalFile *f;
 
@@ -949,27 +949,23 @@ static int add_file(sd_journal *j, const char *prefix, const char *dir, const ch
              (startswith(filename, "system@") && endswith(filename, ".journal"))))
                 return 0;
 
-        if (dir)
-                fn = join(prefix, "/", dir, "/", filename, NULL);
-        else
-                fn = join(prefix, "/", filename, NULL);
-
-        if (!fn)
+        path = join(prefix, "/", filename, NULL);
+        if (!path)
                 return -ENOMEM;
 
-        if (hashmap_get(j->files, fn)) {
-                free(fn);
+        if (hashmap_get(j->files, path)) {
+                free(path);
                 return 0;
         }
 
         if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
-                log_debug("Too many open journal files, not adding %s, ignoring.", fn);
-                free(fn);
+                log_debug("Too many open journal files, not adding %s, ignoring.", path);
+                free(path);
                 return 0;
         }
 
-        r = journal_file_open(fn, O_RDONLY, 0, NULL, &f);
-        free(fn);
+        r = journal_file_open(path, O_RDONLY, 0, NULL, &f);
+        free(path);
 
         if (r < 0) {
                 if (errno == ENOENT)
@@ -986,166 +982,302 @@ static int add_file(sd_journal *j, const char *prefix, const char *dir, const ch
                 return r;
         }
 
+        j->current_invalidate_counter ++;
+
         log_debug("File %s got added.", f->path);
 
         return 0;
 }
 
-static int remove_file(sd_journal *j, const char *prefix, const char *dir, const char *filename) {
-        char *fn;
+static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
+        char *path;
         JournalFile *f;
 
         assert(j);
         assert(prefix);
         assert(filename);
 
-        if (dir)
-                fn = join(prefix, "/", dir, "/", filename, NULL);
-        else
-                fn = join(prefix, "/", filename, NULL);
-
-        if (!fn)
+        path = join(prefix, "/", filename, NULL);
+        if (!path)
                 return -ENOMEM;
 
-        f = hashmap_get(j->files, fn);
-        free(fn);
-
+        f = hashmap_get(j->files, path);
+        free(path);
         if (!f)
                 return 0;
 
         hashmap_remove(j->files, f->path);
         journal_file_close(f);
 
+        j->current_invalidate_counter ++;
+
         log_debug("File %s got removed.", f->path);
         return 0;
 }
 
-static int add_directory(sd_journal *j, const char *prefix, const char *dir) {
-        char *fn;
+static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
+        char *path;
         int r;
         DIR *d;
-        int wd;
         sd_id128_t id, mid;
+        Directory *m;
 
         assert(j);
         assert(prefix);
-        assert(dir);
+        assert(dirname);
 
         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
-            (sd_id128_from_string(dir, &id) < 0 ||
+            (sd_id128_from_string(dirname, &id) < 0 ||
              sd_id128_get_machine(&mid) < 0 ||
              !sd_id128_equal(id, mid)))
             return 0;
 
-        fn = join(prefix, "/", dir, NULL);
-        if (!fn)
+        path = join(prefix, "/", dirname, NULL);
+        if (!path)
                 return -ENOMEM;
 
-        d = opendir(fn);
-
+        d = opendir(path);
         if (!d) {
-                free(fn);
+                log_debug("Failed to open %s: %m", path);
+                free(path);
+
                 if (errno == ENOENT)
                         return 0;
-
                 return -errno;
         }
 
-        wd = inotify_add_watch(j->inotify_fd, fn,
-                               IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
-                               IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|
-                               IN_DONT_FOLLOW|IN_ONLYDIR);
-        if (wd > 0) {
-                if (hashmap_put(j->inotify_wd_dirs, INT_TO_PTR(wd), fn) < 0)
-                        inotify_rm_watch(j->inotify_fd, wd);
-                else
-                        fn = NULL;
+        m = hashmap_get(j->directories_by_path, path);
+        if (!m) {
+                m = new0(Directory, 1);
+                if (!m) {
+                        closedir(d);
+                        free(path);
+                        return -ENOMEM;
+                }
+
+                m->is_root = false;
+                m->path = path;
+
+                if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
+                        closedir(d);
+                        free(m->path);
+                        free(m);
+                        return -ENOMEM;
+                }
+
+                j->current_invalidate_counter ++;
+
+                log_debug("Directory %s got added.", m->path);
+
+        } else if (m->is_root) {
+                free (path);
+                closedir(d);
+                return 0;
+        }  else
+                free(path);
+
+        if (m->wd <= 0 && j->inotify_fd >= 0) {
+
+                m->wd = inotify_add_watch(j->inotify_fd, m->path,
+                                          IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
+                                          IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|
+                                          IN_DONT_FOLLOW|IN_ONLYDIR);
+
+                if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
+                        inotify_rm_watch(j->inotify_fd, m->wd);
+        }
+
+        for (;;) {
+                struct dirent buf, *de;
+
+                r = readdir_r(d, &buf, &de);
+                if (r != 0 || !de)
+                        break;
+
+                if (dirent_is_file_with_suffix(de, ".journal")) {
+                        r = add_file(j, m->path, de->d_name);
+                        if (r < 0)
+                                log_debug("Failed to add file %s/%s: %s", m->path, de->d_name, strerror(-r));
+                }
+        }
+
+        closedir(d);
+
+        return 0;
+}
+
+static int add_root_directory(sd_journal *j, const char *p) {
+        DIR *d;
+        Directory *m;
+        int r;
+
+        assert(j);
+        assert(p);
+
+        if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
+            !path_startswith(p, "/run"))
+                return -EINVAL;
+
+        d = opendir(p);
+        if (!d)
+                return -errno;
+
+        m = hashmap_get(j->directories_by_path, p);
+        if (!m) {
+                m = new0(Directory, 1);
+                if (!m) {
+                        closedir(d);
+                        return -ENOMEM;
+                }
+
+                m->is_root = true;
+                m->path = strdup(p);
+                if (!m->path) {
+                        closedir(d);
+                        free(m);
+                        return -ENOMEM;
+                }
+
+                if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
+                        closedir(d);
+                        free(m->path);
+                        free(m);
+                        return -ENOMEM;
+                }
+
+                j->current_invalidate_counter ++;
+
+                log_debug("Root directory %s got added.", m->path);
+
+        } else if (!m->is_root) {
+                closedir(d);
+                return 0;
         }
 
-        free(fn);
+        if (m->wd <= 0 && j->inotify_fd >= 0) {
+
+                m->wd = inotify_add_watch(j->inotify_fd, m->path,
+                                          IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
+                                          IN_DONT_FOLLOW|IN_ONLYDIR);
+
+                if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
+                        inotify_rm_watch(j->inotify_fd, m->wd);
+        }
 
         for (;;) {
                 struct dirent buf, *de;
+                sd_id128_t id;
 
                 r = readdir_r(d, &buf, &de);
                 if (r != 0 || !de)
                         break;
 
-                if (!dirent_is_file_with_suffix(de, ".journal"))
-                        continue;
+                if (dirent_is_file_with_suffix(de, ".journal")) {
+                        r = add_file(j, m->path, de->d_name);
+                        if (r < 0)
+                                log_debug("Failed to add file %s/%s: %s", m->path, de->d_name, strerror(-r));
 
-                r = add_file(j, prefix, dir, de->d_name);
-                if (r < 0)
-                        log_debug("Failed to add file %s/%s/%s: %s", prefix, dir, de->d_name, strerror(-r));
+                } else if ((de->d_type == DT_DIR || de->d_type == DT_UNKNOWN) &&
+                           sd_id128_from_string(de->d_name, &id) >= 0) {
+
+                        r = add_directory(j, m->path, de->d_name);
+                        if (r < 0)
+                                log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
+                }
         }
 
         closedir(d);
 
-        log_debug("Directory %s/%s got added.", prefix, dir);
+        return 0;
+}
+
+static int remove_directory(sd_journal *j, Directory *d) {
+        assert(j);
+
+        if (d->wd > 0) {
+                hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
+
+                if (j->inotify_fd >= 0)
+                        inotify_rm_watch(j->inotify_fd, d->wd);
+        }
+
+        hashmap_remove(j->directories_by_path, d->path);
+
+        if (d->is_root)
+                log_debug("Root directory %s got removed.", d->path);
+        else
+                log_debug("Directory %s got removed.", d->path);
+
+        free(d->path);
+        free(d);
 
         return 0;
 }
 
-static void remove_directory_wd(sd_journal *j, int wd) {
-        char *p;
+static int add_search_paths(sd_journal *j) {
+
+        const char search_paths[] =
+                "/run/log/journal\0"
+                "/var/log/journal\0";
+        const char *p;
 
         assert(j);
-        assert(wd > 0);
 
-        if (j->inotify_fd >= 0)
-                inotify_rm_watch(j->inotify_fd, wd);
+        /* We ignore most errors here, since the idea is to only open
+         * what's actually accessible, and ignore the rest. */
 
-        p = hashmap_remove(j->inotify_wd_dirs, INT_TO_PTR(wd));
+        NULSTR_FOREACH(p, search_paths)
+                add_root_directory(j, p);
 
-        if (p) {
-                log_debug("Directory %s got removed.", p);
-                free(p);
-        }
+        return 0;
 }
 
-static void add_root_wd(sd_journal *j, const char *p) {
-        int wd;
-        char *k;
-
+static int allocate_inotify(sd_journal *j) {
         assert(j);
-        assert(p);
 
-        wd = inotify_add_watch(j->inotify_fd, p,
-                               IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
-                               IN_DONT_FOLLOW|IN_ONLYDIR);
-        if (wd <= 0)
-                return;
+        if (j->inotify_fd < 0) {
+                j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
+                if (j->inotify_fd < 0)
+                        return -errno;
+        }
 
-        k = strdup(p);
-        if (!k || hashmap_put(j->inotify_wd_roots, INT_TO_PTR(wd), k) < 0) {
-                inotify_rm_watch(j->inotify_fd, wd);
-                free(k);
+        if (!j->directories_by_wd) {
+                j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
+                if (!j->directories_by_wd)
+                        return -ENOMEM;
         }
+
+        return 0;
 }
 
-static void remove_root_wd(sd_journal *j, int wd) {
-        char *p;
+static sd_journal *journal_new(int flags) {
+        sd_journal *j;
 
-        assert(j);
-        assert(wd > 0);
+        j = new0(sd_journal, 1);
+        if (!j)
+                return NULL;
 
-        if (j->inotify_fd >= 0)
-                inotify_rm_watch(j->inotify_fd, wd);
+        j->inotify_fd = -1;
+        j->flags = flags;
 
-        p = hashmap_remove(j->inotify_wd_roots, INT_TO_PTR(wd));
+        j->files = hashmap_new(string_hash_func, string_compare_func);
+        if (!j->files) {
+                free(j);
+                return NULL;
+        }
 
-        if (p) {
-                log_debug("Root %s got removed.", p);
-                free(p);
+        j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
+        if (!j->directories_by_path) {
+                hashmap_free(j->files);
+                free(j);
+                return NULL;
         }
+
+        return j;
 }
 
 _public_ int sd_journal_open(sd_journal **ret, int flags) {
         sd_journal *j;
-        const char *p;
-        const char search_paths[] =
-                "/run/log/journal\0"
-                "/var/log/journal\0";
         int r;
 
         if (!ret)
@@ -1156,75 +1288,43 @@ _public_ int sd_journal_open(sd_journal **ret, int flags) {
                       SD_JOURNAL_SYSTEM_ONLY))
                 return -EINVAL;
 
-        j = new0(sd_journal, 1);
+        j = journal_new(flags);
         if (!j)
                 return -ENOMEM;
 
-        j->flags = flags;
-
-        j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
-        if (j->inotify_fd < 0) {
-                r = -errno;
-                goto fail;
-        }
-
-        j->files = hashmap_new(string_hash_func, string_compare_func);
-        if (!j->files) {
-                r = -ENOMEM;
-                goto fail;
-        }
-
-        j->inotify_wd_dirs = hashmap_new(trivial_hash_func, trivial_compare_func);
-        j->inotify_wd_roots = hashmap_new(trivial_hash_func, trivial_compare_func);
-
-        if (!j->inotify_wd_dirs || !j->inotify_wd_roots) {
-                r = -ENOMEM;
+        r = add_search_paths(j);
+        if (r < 0)
                 goto fail;
-        }
 
-        /* We ignore most errors here, since the idea is to only open
-         * what's actually accessible, and ignore the rest. */
-
-        NULSTR_FOREACH(p, search_paths) {
-                DIR *d;
-
-                if ((flags & SD_JOURNAL_RUNTIME_ONLY) &&
-                    !path_startswith(p, "/run"))
-                        continue;
+        *ret = j;
+        return 0;
 
-                d = opendir(p);
-                if (!d) {
-                        if (errno != ENOENT)
-                                log_debug("Failed to open %s: %m", p);
-                        continue;
-                }
+fail:
+        sd_journal_close(j);
 
-                add_root_wd(j, p);
+        return r;
+}
 
-                for (;;) {
-                        struct dirent buf, *de;
-                        sd_id128_t id;
+_public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
+        sd_journal *j;
+        int r;
 
-                        r = readdir_r(d, &buf, &de);
-                        if (r != 0 || !de)
-                                break;
+        if (!ret)
+                return -EINVAL;
 
-                        if (dirent_is_file_with_suffix(de, ".journal")) {
-                                r = add_file(j, p, NULL, de->d_name);
-                                if (r < 0)
-                                        log_debug("Failed to add file %s/%s: %s", p, de->d_name, strerror(-r));
+        if (!path || !path_is_absolute(path))
+                return -EINVAL;
 
-                        } else if ((de->d_type == DT_DIR || de->d_type == DT_UNKNOWN) &&
-                                   sd_id128_from_string(de->d_name, &id) >= 0) {
+        if (flags != 0)
+                return -EINVAL;
 
-                                r = add_directory(j, p, de->d_name);
-                                if (r < 0)
-                                        log_debug("Failed to add directory %s/%s: %s", p, de->d_name, strerror(-r));
-                        }
-                }
+        j = journal_new(flags);
+        if (!j)
+                return -ENOMEM;
 
-                closedir(d);
-        }
+        r = add_root_directory(j, path);
+        if (r < 0)
+                goto fail;
 
         *ret = j;
         return 0;
@@ -1233,44 +1333,34 @@ fail:
         sd_journal_close(j);
 
         return r;
-};
+}
 
 _public_ void sd_journal_close(sd_journal *j) {
+        Directory *d;
+        JournalFile *f;
+
         if (!j)
                 return;
 
-        if (j->inotify_wd_dirs) {
-                void *k;
-
-                while ((k = hashmap_first_key(j->inotify_wd_dirs)))
-                        remove_directory_wd(j, PTR_TO_INT(k));
-
-                hashmap_free(j->inotify_wd_dirs);
-        }
-
-        if (j->inotify_wd_roots) {
-                void *k;
-
-                while ((k = hashmap_first_key(j->inotify_wd_roots)))
-                        remove_root_wd(j, PTR_TO_INT(k));
-
-                hashmap_free(j->inotify_wd_roots);
-        }
+        while ((f = hashmap_steal_first(j->files)))
+                journal_file_close(f);
 
-        if (j->files) {
-                JournalFile *f;
+        hashmap_free(j->files);
 
-                while ((f = hashmap_steal_first(j->files)))
-                        journal_file_close(f);
+        while ((d = hashmap_first(j->directories_by_path)))
+                remove_directory(j, d);
 
-                hashmap_free(j->files);
-        }
+        while ((d = hashmap_first(j->directories_by_wd)))
+                remove_directory(j, d);
 
-        sd_journal_flush_matches(j);
+        hashmap_free(j->directories_by_path);
+        hashmap_free(j->directories_by_wd);
 
         if (j->inotify_fd >= 0)
                 close_nointr_nofail(j->inotify_fd);
 
+        sd_journal_flush_matches(j);
+
         free(j);
 }
 
@@ -1506,78 +1596,74 @@ _public_ void sd_journal_restart_data(sd_journal *j) {
 }
 
 _public_ int sd_journal_get_fd(sd_journal *j) {
+        int r;
+
         if (!j)
                 return -EINVAL;
 
+        if (j->inotify_fd >= 0)
+                return j->inotify_fd;
+
+        r = allocate_inotify(j);
+        if (r < 0)
+                return r;
+
+        /* Iterate through all dirs again, to add them to the
+         * inotify */
+        r = add_search_paths(j);
+        if (r < 0)
+                return r;
+
         return j->inotify_fd;
 }
 
 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
-        char *p;
+        Directory *d;
         int r;
 
         assert(j);
         assert(e);
 
         /* Is this a subdirectory we watch? */
-        p = hashmap_get(j->inotify_wd_dirs, INT_TO_PTR(e->wd));
-        if (p) {
+        d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
+        if (d) {
+                sd_id128_t id;
 
                 if (!(e->mask & IN_ISDIR) && e->len > 0 && endswith(e->name, ".journal")) {
 
                         /* Event for a journal file */
 
                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
-                                r = add_file(j, p, NULL, e->name);
+                                r = add_file(j, d->path, e->name);
                                 if (r < 0)
-                                        log_debug("Failed to add file %s/%s: %s", p, e->name, strerror(-r));
+                                        log_debug("Failed to add file %s/%s: %s", d->path, e->name, strerror(-r));
+
                         } else if (e->mask & (IN_DELETE|IN_UNMOUNT)) {
 
-                                r = remove_file(j, p, NULL, e->name);
+                                r = remove_file(j, d->path, e->name);
                                 if (r < 0)
-                                        log_debug("Failed to remove file %s/%s: %s", p, e->name, strerror(-r));
+                                        log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
                         }
 
-                } else if (e->len == 0) {
+                } else if (!d->is_root && e->len == 0) {
 
-                        /* Event for the directory itself */
-
-                        if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT))
-                                remove_directory_wd(j, e->wd);
-                }
-
-                return;
-        }
-
-        /* Must be the root directory then? */
-        p = hashmap_get(j->inotify_wd_roots, INT_TO_PTR(e->wd));
-        if (p) {
-                sd_id128_t id;
+                        /* Event for a subdirectory */
 
-                if (!(e->mask & IN_ISDIR) && e->len > 0 && endswith(e->name, ".journal")) {
-
-                        /* Event for a journal file */
-
-                        if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
-                                r = add_file(j, p, NULL, e->name);
-                                if (r < 0)
-                                        log_debug("Failed to add file %s/%s: %s", p, e->name, strerror(-r));
-                        } else if (e->mask & (IN_DELETE|IN_UNMOUNT)) {
-
-                                r = remove_file(j, p, NULL, e->name);
+                        if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
+                                r = remove_directory(j, d);
                                 if (r < 0)
-                                        log_debug("Failed to remove file %s/%s: %s", p, e->name, strerror(-r));
+                                        log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
                         }
 
-                } else if ((e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
 
-                        /* Event for subdirectory */
+                } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
 
-                        if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
+                        /* Event for root directory */
 
-                                r = add_directory(j, p, e->name);
+                        if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
+                                r = add_directory(j, d->path, e->name);
                                 if (r < 0)
-                                        log_debug("Failed to add directory %s/%s: %s", p, e->name, strerror(-r));
+                                        log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
                         }
                 }
 
@@ -1590,8 +1676,20 @@ static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
         log_warning("Unknown inotify event.");
 }
 
+static int determine_change(sd_journal *j) {
+        bool b;
+
+        assert(j);
+
+        b = j->current_invalidate_counter != j->last_invalidate_counter;
+        j->last_invalidate_counter = j->current_invalidate_counter;
+
+        return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
+}
+
 _public_ int sd_journal_process(sd_journal *j) {
         uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX];
+        bool got_something = false;
 
         if (!j)
                 return -EINVAL;
@@ -1602,12 +1700,14 @@ _public_ int sd_journal_process(sd_journal *j) {
 
                 l = read(j->inotify_fd, buffer, sizeof(buffer));
                 if (l < 0) {
-                        if (errno == EINTR || errno == EAGAIN)
-                                return 0;
+                        if (errno == EAGAIN || errno == EINTR)
+                                return got_something ? determine_change(j) : SD_JOURNAL_NOP;
 
                         return -errno;
                 }
 
+                got_something = true;
+
                 e = (struct inotify_event*) buffer;
                 while (l > 0) {
                         size_t step;
@@ -1621,20 +1721,38 @@ _public_ int sd_journal_process(sd_journal *j) {
                         l -= step;
                 }
         }
+
+        return determine_change(j);
 }
 
 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
-        int r, k;
+        int r;
 
         assert(j);
 
-        r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
-        k = sd_journal_process(j);
+        if (j->inotify_fd < 0) {
+
+                /* This is the first invocation, hence create the
+                 * inotify watch */
+                r = sd_journal_get_fd(j);
+                if (r < 0)
+                        return r;
+
+                /* The journal might have changed since the context
+                 * object was created and we weren't watching before,
+                 * hence don't wait for anything, and return
+                 * immediately. */
+                return determine_change(j);
+        }
+
+        do {
+                r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
+        } while (r == -EINTR);
 
         if (r < 0)
                 return r;
 
-        return k;
+        return sd_journal_process(j);
 }
 
 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
diff --git a/src/systemd/sd-journal.h b/src/systemd/sd-journal.h
index ee4acffd65..ab968b9496 100644
--- a/src/systemd/sd-journal.h
+++ b/src/systemd/sd-journal.h
@@ -72,6 +72,7 @@ enum {
 };
 
 int sd_journal_open(sd_journal **ret, int flags);
+int sd_journal_open_directory(sd_journal **ret, const char *path, int flags);
 void sd_journal_close(sd_journal *j);
 
 int sd_journal_previous(sd_journal *j);
@@ -107,8 +108,7 @@ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, const sd_id128_t boot_id
 enum {
         SD_JOURNAL_NOP,
         SD_JOURNAL_APPEND,
-        SD_JOURNAL_INVALIDATE_ADD,
-        SD_JOURNAL_INVALIDATE_REMOVE
+        SD_JOURNAL_INVALIDATE
 };
 
 int sd_journal_get_fd(sd_journal *j);