Merge pull request #3109 from poettering/journal-by-fd

rework "journalctl -M"

11 files changed, 379 insertions, 95 deletions

diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c
index 901bd8fb68..ac6c30f9f2 100644
--- a/src/journal/journal-file.c
+++ b/src/journal/journal-file.c
@@ -37,6 +37,7 @@
 #include "journal-file.h"
 #include "lookup3.h"
 #include "parse-util.h"
+#include "path-util.h"
 #include "random-util.h"
 #include "sd-event.h"
 #include "set.h"
@@ -364,7 +365,8 @@ JournalFile* journal_file_close(JournalFile *f) {
                 (void) btrfs_defrag_fd(f->fd);
         }
 
-        safe_close(f->fd);
+        if (f->close_fd)
+                safe_close(f->fd);
         free(f->path);
 
         mmap_cache_unref(f->mmap);
@@ -2468,8 +2470,7 @@ int journal_file_next_entry(
 
         if (p > 0 &&
             (direction == DIRECTION_DOWN ? ofs <= p : ofs >= p)) {
-                log_debug("%s: entry array corrupted at entry %"PRIu64,
-                          f->path, i);
+                log_debug("%s: entry array corrupted at entry %"PRIu64, f->path, i);
                 return -EBADMSG;
         }
 
@@ -2900,6 +2901,7 @@ static int journal_file_warn_btrfs(JournalFile *f) {
 }
 
 int journal_file_open(
+                int fd,
                 const char *fname,
                 int flags,
                 mode_t mode,
@@ -2916,22 +2918,24 @@ int journal_file_open(
         void *h;
         int r;
 
-        assert(fname);
         assert(ret);
+        assert(fd >= 0 || fname);
 
         if ((flags & O_ACCMODE) != O_RDONLY &&
             (flags & O_ACCMODE) != O_RDWR)
                 return -EINVAL;
 
-        if (!endswith(fname, ".journal") &&
-            !endswith(fname, ".journal~"))
-                return -EINVAL;
+        if (fname) {
+                if (!endswith(fname, ".journal") &&
+                    !endswith(fname, ".journal~"))
+                        return -EINVAL;
+        }
 
         f = new0(JournalFile, 1);
         if (!f)
                 return -ENOMEM;
 
-        f->fd = -1;
+        f->fd = fd;
         f->mode = mode;
 
         f->flags = flags;
@@ -2956,7 +2960,10 @@ int journal_file_open(
                 }
         }
 
-        f->path = strdup(fname);
+        if (fname)
+                f->path = strdup(fname);
+        else /* If we don't know the path, fill in something explanatory and vaguely useful */
+                asprintf(&f->path, "/proc/self/%i", fd);
         if (!f->path) {
                 r = -ENOMEM;
                 goto fail;
@@ -2968,10 +2975,15 @@ int journal_file_open(
                 goto fail;
         }
 
-        f->fd = open(f->path, f->flags|O_CLOEXEC, f->mode);
         if (f->fd < 0) {
-                r = -errno;
-                goto fail;
+                f->fd = open(f->path, f->flags|O_CLOEXEC, f->mode);
+                if (f->fd < 0) {
+                        r = -errno;
+                        goto fail;
+                }
+
+                /* fds we opened here by us should also be closed by us. */
+                f->close_fd = true;
         }
 
         r = journal_file_fstat(f);
@@ -3092,6 +3104,9 @@ int journal_file_open(
                         goto fail;
         }
 
+        /* The file is opened now successfully, thus we take possesion of any passed in fd. */
+        f->close_fd = true;
+
         *ret = f;
         return 0;
 
@@ -3118,6 +3133,11 @@ int journal_file_rotate(JournalFile **f, bool compress, bool seal, Set *deferred
         if (!old_file->writable)
                 return -EINVAL;
 
+        /* Is this a journal file that was passed to us as fd? If so, we synthesized a path name for it, and we refuse
+         * rotation, since we don't know the actual path, and couldn't rename the file hence.*/
+        if (path_startswith(old_file->path, "/proc/self/fd"))
+                return -EINVAL;
+
         if (!endswith(old_file->path, ".journal"))
                 return -EINVAL;
 
@@ -3144,7 +3164,7 @@ int journal_file_rotate(JournalFile **f, bool compress, bool seal, Set *deferred
          * we archive them */
         old_file->defrag_on_close = true;
 
-        r = journal_file_open(old_file->path, old_file->flags, old_file->mode, compress, seal, NULL, old_file->mmap, deferred_closes, old_file, &new_file);
+        r = journal_file_open(-1, old_file->path, old_file->flags, old_file->mode, compress, seal, NULL, old_file->mmap, deferred_closes, old_file, &new_file);
 
         if (deferred_closes &&
             set_put(deferred_closes, old_file) >= 0)
@@ -3172,7 +3192,7 @@ int journal_file_open_reliably(
         size_t l;
         _cleanup_free_ char *p = NULL;
 
-        r = journal_file_open(fname, flags, mode, compress, seal, metrics, mmap_cache, deferred_closes, template, ret);
+        r = journal_file_open(-1, fname, flags, mode, compress, seal, metrics, mmap_cache, deferred_closes, template, ret);
         if (!IN_SET(r,
                     -EBADMSG,           /* corrupted */
                     -ENODATA,           /* truncated */
@@ -3213,7 +3233,7 @@ int journal_file_open_reliably(
 
         log_warning_errno(r, "File %s corrupted or uncleanly shut down, renaming and replacing.", fname);
 
-        return journal_file_open(fname, flags, mode, compress, seal, metrics, mmap_cache, deferred_closes, template, ret);
+        return journal_file_open(-1, fname, flags, mode, compress, seal, metrics, mmap_cache, deferred_closes, template, ret);
 }
 
 int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint64_t p, uint64_t *seqnum, Object **ret, uint64_t *offset) {
diff --git a/src/journal/journal-file.h b/src/journal/journal-file.h
index 9ad6013359..a0ec8c284b 100644
--- a/src/journal/journal-file.h
+++ b/src/journal/journal-file.h
@@ -85,6 +85,7 @@ typedef struct JournalFile {
         bool compress_lz4:1;
         bool seal:1;
         bool defrag_on_close:1;
+        bool close_fd:1;
 
         bool tail_entry_monotonic_valid:1;
 
@@ -142,6 +143,7 @@ typedef struct JournalFile {
 } JournalFile;
 
 int journal_file_open(
+                int fd,
                 const char *fname,
                 int flags,
                 mode_t mode,
diff --git a/src/journal/journal-internal.h b/src/journal/journal-internal.h
index 7639325acf..34a48141f5 100644
--- a/src/journal/journal-internal.h
+++ b/src/journal/journal-internal.h
@@ -82,6 +82,8 @@ struct Directory {
 };
 
 struct sd_journal {
+        int toplevel_fd;
+
         char *path;
         char *prefix;
 
@@ -117,6 +119,7 @@ struct sd_journal {
 
         bool on_network:1;
         bool no_new_files:1;
+        bool no_inotify:1;
         bool unique_file_lost:1; /* File we were iterating over got
                                     removed, and there were no more
                                     files, so sd_j_enumerate_unique
diff --git a/src/journal/journalctl.c b/src/journal/journalctl.c
index 4b2736c9eb..f67c556783 100644
--- a/src/journal/journalctl.c
+++ b/src/journal/journalctl.c
@@ -101,6 +101,7 @@ static const char *arg_after_cursor = NULL;
 static bool arg_show_cursor = false;
 static const char *arg_directory = NULL;
 static char **arg_file = NULL;
+static bool arg_file_stdin = false;
 static int arg_priorities = 0xFF;
 static const char *arg_verify_key = NULL;
 #ifdef HAVE_GCRYPT
@@ -592,9 +593,17 @@ static int parse_argv(int argc, char *argv[]) {
                         break;
 
                 case ARG_FILE:
-                        r = glob_extend(&arg_file, optarg);
-                        if (r < 0)
-                                return log_error_errno(r, "Failed to add paths: %m");
+                        if (streq(optarg, "-"))
+                                /* An undocumented feature: we can read journal files from STDIN. We don't document
+                                 * this though, since after all we only support this for mmap-able, seekable files, and
+                                 * not for example pipes which are probably the primary usecase for reading things from
+                                 * STDIN. To avoid confusion we hence don't document this feature. */
+                                arg_file_stdin = true;
+                        else {
+                                r = glob_extend(&arg_file, optarg);
+                                if (r < 0)
+                                        return log_error_errno(r, "Failed to add paths: %m");
+                        }
                         break;
 
                 case ARG_ROOT:
@@ -864,6 +873,18 @@ static int parse_argv(int argc, char *argv[]) {
                 return -EINVAL;
         }
 
+        if (!strv_isempty(arg_system_units) && (arg_journal_type == SD_JOURNAL_CURRENT_USER)) {
+
+                /* Specifying --user and --unit= at the same time makes no sense (as the former excludes the user
+                 * journal, but the latter excludes the system journal, thus resulting in empty output). Let's be nice
+                 * to users, and automatically turn --unit= into --user-unit= if combined with --user. */
+                r = strv_extend_strv(&arg_user_units, arg_system_units, true);
+                if (r < 0)
+                        return -ENOMEM;
+
+                arg_system_units = strv_free(arg_system_units);
+        }
+
         return 1;
 }
 
@@ -1869,7 +1890,7 @@ static int access_check(sd_journal *j) {
                         break;
 
                 default:
-                        log_warning_errno(err, "An error was encountered while opening journal file %s, ignoring file.", path);
+                        log_warning_errno(err, "An error was encountered while opening journal file or directory %s, ignoring file: %m", path);
                         break;
                 }
         }
@@ -2125,11 +2146,61 @@ int main(int argc, char *argv[]) {
 
         if (arg_directory)
                 r = sd_journal_open_directory(&j, arg_directory, arg_journal_type);
-        else if (arg_file)
+        else if (arg_file_stdin) {
+                int ifd = STDIN_FILENO;
+                r = sd_journal_open_files_fd(&j, &ifd, 1, 0);
+        } else if (arg_file)
                 r = sd_journal_open_files(&j, (const char**) arg_file, 0);
-        else if (arg_machine)
-                r = sd_journal_open_container(&j, arg_machine, 0);
-        else
+        else if (arg_machine) {
+                _cleanup_(sd_bus_error_free) sd_bus_error error = SD_BUS_ERROR_NULL;
+                _cleanup_(sd_bus_message_unrefp) sd_bus_message *reply = NULL;
+                _cleanup_(sd_bus_flush_close_unrefp) sd_bus *bus = NULL;
+                int fd;
+
+                if (geteuid() != 0) {
+                        /* The file descriptor returned by OpenMachineRootDirectory() will be owned by users/groups of
+                         * the container, thus we need root privileges to override them. */
+                        log_error("Using the --machine= switch requires root privileges.");
+                        r = -EPERM;
+                        goto finish;
+                }
+
+                r = sd_bus_open_system(&bus);
+                if (r < 0) {
+                        log_error_errno(r, "Failed to open system bus: %m");
+                        goto finish;
+                }
+
+                r = sd_bus_call_method(
+                                bus,
+                                "org.freedesktop.machine1",
+                                "/org/freedesktop/machine1",
+                                "org.freedesktop.machine1.Manager",
+                                "OpenMachineRootDirectory",
+                                &error,
+                                &reply,
+                                "s", arg_machine);
+                if (r < 0) {
+                        log_error_errno(r, "Failed to open root directory: %s", bus_error_message(&error, r));
+                        goto finish;
+                }
+
+                r = sd_bus_message_read(reply, "h", &fd);
+                if (r < 0) {
+                        bus_log_parse_error(r);
+                        goto finish;
+                }
+
+                fd = fcntl(fd, F_DUPFD_CLOEXEC, 3);
+                if (fd < 0) {
+                        r = log_error_errno(errno, "Failed to duplicate file descriptor: %m");
+                        goto finish;
+                }
+
+                r = sd_journal_open_directory_fd(&j, fd, SD_JOURNAL_OS_ROOT);
+                if (r < 0)
+                        safe_close(fd);
+        } else
                 r = sd_journal_open(&j, !arg_merge*SD_JOURNAL_LOCAL_ONLY + arg_journal_type);
         if (r < 0) {
                 log_error_errno(r, "Failed to open %s: %m", arg_directory ?: arg_file ? "files" : "journal");
@@ -2305,6 +2376,10 @@ int main(int argc, char *argv[]) {
         /* Opening the fd now means the first sd_journal_wait() will actually wait */
         if (arg_follow) {
                 r = sd_journal_get_fd(j);
+                if (r == -EMEDIUMTYPE) {
+                        log_error_errno(r, "The --follow switch is not supported in conjunction with reading from STDIN.");
+                        goto finish;
+                }
                 if (r < 0) {
                         log_error_errno(r, "Failed to get journal fd: %m");
                         goto finish;
diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c
index 8089bb5883..e14d0ad980 100644
--- a/src/journal/journald-server.c
+++ b/src/journal/journald-server.c
@@ -253,7 +253,7 @@ static int open_journal(
         if (reliably)
                 r = journal_file_open_reliably(fname, flags, 0640, s->compress, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
         else
-                r = journal_file_open(fname, flags, 0640, s->compress, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
+                r = journal_file_open(-1, fname, flags, 0640, s->compress, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
         if (r < 0)
                 return r;
 
diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c
index 3c21d4129e..27c1dd346f 100644
--- a/src/journal/sd-journal.c
+++ b/src/journal/sd-journal.c
@@ -1233,14 +1233,37 @@ static bool file_type_wanted(int flags, const char *filename) {
         return false;
 }
 
-static int add_any_file(sd_journal *j, const char *path) {
+static bool path_has_prefix(sd_journal *j, const char *path, const char *prefix) {
+        assert(j);
+        assert(path);
+        assert(prefix);
+
+        if (j->toplevel_fd >= 0)
+                return false;
+
+        return path_startswith(path, prefix);
+}
+
+static const char *skip_slash(const char *p) {
+
+        if (!p)
+                return NULL;
+
+        while (*p == '/')
+                p++;
+
+        return p;
+}
+
+static int add_any_file(sd_journal *j, int fd, const char *path) {
         JournalFile *f = NULL;
+        bool close_fd = false;
         int r, k;
 
         assert(j);
-        assert(path);
+        assert(fd >= 0 || path);
 
-        if (ordered_hashmap_get(j->files, path))
+        if (path && ordered_hashmap_get(j->files, path))
                 return 0;
 
         if (ordered_hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
@@ -1249,8 +1272,24 @@ static int add_any_file(sd_journal *j, const char *path) {
                 goto fail;
         }
 
-        r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, NULL, &f);
+        if (fd < 0 && j->toplevel_fd >= 0) {
+
+                /* If there's a top-level fd defined, open the file relative to this now. (Make the path relative,
+                 * explicitly, since otherwise openat() ignores the first argument.) */
+
+                fd = openat(j->toplevel_fd, skip_slash(path), O_RDONLY|O_CLOEXEC);
+                if (fd < 0) {
+                        r = log_debug_errno(errno, "Failed to open journal file %s: %m", path);
+                        goto fail;
+                }
+
+                close_fd = true;
+        }
+
+        r = journal_file_open(fd, path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, NULL, &f);
         if (r < 0) {
+                if (close_fd)
+                        safe_close(fd);
                 log_debug_errno(r, "Failed to open journal file %s: %m", path);
                 goto fail;
         }
@@ -1259,10 +1298,16 @@ static int add_any_file(sd_journal *j, const char *path) {
 
         r = ordered_hashmap_put(j->files, f->path, f);
         if (r < 0) {
+                f->close_fd = close_fd;
                 (void) journal_file_close(f);
                 goto fail;
         }
 
+        if (!j->has_runtime_files && path_has_prefix(j, f->path, "/run"))
+                j->has_runtime_files = true;
+        else if (!j->has_persistent_files && path_has_prefix(j, f->path, "/var"))
+                j->has_persistent_files = true;
+
         log_debug("File %s added.", f->path);
 
         check_network(j, f->fd);
@@ -1286,18 +1331,14 @@ static int add_file(sd_journal *j, const char *prefix, const char *filename) {
         assert(prefix);
         assert(filename);
 
-        if (j->no_new_files ||
-            !file_type_wanted(j->flags, filename))
+        if (j->no_new_files)
+                return 0;
+
+        if (!file_type_wanted(j->flags, filename))
                 return 0;
 
         path = strjoina(prefix, "/", filename);
-
-        if (!j->has_runtime_files && path_startswith(path, "/run/log/journal"))
-                j->has_runtime_files = true;
-        else if (!j->has_persistent_files && path_startswith(path, "/var/log/journal"))
-                j->has_persistent_files = true;
-
-        return add_any_file(j, path);
+        return add_any_file(j, -1, path);
 }
 
 static void remove_file(sd_journal *j, const char *prefix, const char *filename) {
@@ -1373,21 +1414,33 @@ static int add_directory(sd_journal *j, const char *prefix, const char *dirname)
 
         assert(j);
         assert(prefix);
-        assert(dirname);
-
-        log_debug("Considering %s/%s.", prefix, dirname);
 
-        if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
-            !(dirname_is_machine_id(dirname) > 0 || path_startswith(prefix, "/run")))
-            return 0;
+        /* Adds a journal file directory to watch. If the directory is already tracked this updates the inotify watch
+         * and reenumerates directory contents */
 
-        path = strjoin(prefix, "/", dirname, NULL);
+        if (dirname)
+                path = strjoin(prefix, "/", dirname, NULL);
+        else
+                path = strdup(prefix);
         if (!path) {
                 r = -ENOMEM;
                 goto fail;
         }
 
-        d = opendir(path);
+        log_debug("Considering directory %s.", path);
+
+        /* We consider everything local that is in a directory for the local machine ID, or that is stored in /run */
+        if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
+            !((dirname && dirname_is_machine_id(dirname) > 0) || path_has_prefix(j, path, "/run")))
+            return 0;
+
+
+        if (j->toplevel_fd < 0)
+                d = opendir(path);
+        else
+                /* Open the specified directory relative to the the toplevel fd. Enforce that the path specified is
+                 * relative, by dropping the initial slash */
+                d = xopendirat(j->toplevel_fd, skip_slash(path), 0);
         if (!d) {
                 r = log_debug_errno(errno, "Failed to open directory %s: %m", path);
                 goto fail;
@@ -1419,17 +1472,18 @@ static int add_directory(sd_journal *j, const char *prefix, const char *dirname)
                 return 0;
 
         if (m->wd <= 0 && j->inotify_fd >= 0) {
+                /* Watch this directory, if it not being watched yet. */
 
-                m->wd = inotify_add_watch(j->inotify_fd, m->path,
-                                          IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
-                                          IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
-                                          IN_ONLYDIR);
+                m->wd = inotify_add_watch_fd(j->inotify_fd, dirfd(d),
+                                             IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
+                                             IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
+                                             IN_ONLYDIR);
 
                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
                         inotify_rm_watch(j->inotify_fd, m->wd);
         }
 
-        FOREACH_DIRENT_ALL(de, d, return log_debug_errno(errno, "Failed to read directory %s: %m", m->path)) {
+        FOREACH_DIRENT_ALL(de, d, r = log_debug_errno(errno, "Failed to read directory %s: %m", m->path); goto fail) {
 
                 if (dirent_is_file_with_suffix(de, ".journal") ||
                     dirent_is_file_with_suffix(de, ".journal~"))
@@ -1441,7 +1495,7 @@ static int add_directory(sd_journal *j, const char *prefix, const char *dirname)
         return 0;
 
 fail:
-        k = journal_put_error(j, r, path ?: dirname);
+        k = journal_put_error(j, r, path ?: prefix);
         if (k < 0)
                 return k;
 
@@ -1449,28 +1503,62 @@ fail:
 }
 
 static int add_root_directory(sd_journal *j, const char *p, bool missing_ok) {
+
         _cleanup_closedir_ DIR *d = NULL;
         struct dirent *de;
         Directory *m;
         int r, k;
 
         assert(j);
-        assert(p);
 
-        if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
-            !path_startswith(p, "/run"))
-                return -EINVAL;
+        /* Adds a root directory to our set of directories to use. If the root directory is already in the set, we
+         * update the inotify logic, and renumerate the directory entries. This call may hence be called to initially
+         * populate the set, as well as to update it later. */
 
-        if (j->prefix)
-                p = strjoina(j->prefix, p);
+        if (p) {
+                /* If there's a path specified, use it. */
 
-        d = opendir(p);
-        if (!d) {
-                if (errno == ENOENT && missing_ok)
-                        return 0;
+                if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
+                    !path_has_prefix(j, p, "/run"))
+                        return -EINVAL;
 
-                r = log_debug_errno(errno, "Failed to open root directory %s: %m", p);
-                goto fail;
+                if (j->prefix)
+                        p = strjoina(j->prefix, p);
+
+                if (j->toplevel_fd < 0)
+                        d = opendir(p);
+                else
+                        d = xopendirat(j->toplevel_fd, skip_slash(p), 0);
+
+                if (!d) {
+                        if (errno == ENOENT && missing_ok)
+                                return 0;
+
+                        r = log_debug_errno(errno, "Failed to open root directory %s: %m", p);
+                        goto fail;
+                }
+        } else {
+                int dfd;
+
+                /* If there's no path specified, then we use the top-level fd itself. We duplicate the fd here, since
+                 * opendir() will take possession of the fd, and close it, which we don't want. */
+
+                p = "."; /* store this as "." in the directories hashmap */
+
+                dfd = fcntl(j->toplevel_fd, F_DUPFD_CLOEXEC, 3);
+                if (dfd < 0) {
+                        r = -errno;
+                        goto fail;
+                }
+
+                d = fdopendir(dfd);
+                if (!d) {
+                        r = -errno;
+                        safe_close(dfd);
+                        goto fail;
+                }
+
+                rewinddir(d);
         }
 
         m = hashmap_get(j->directories_by_path, p);
@@ -1482,6 +1570,7 @@ static int add_root_directory(sd_journal *j, const char *p, bool missing_ok) {
                 }
 
                 m->is_root = true;
+
                 m->path = strdup(p);
                 if (!m->path) {
                         free(m);
@@ -1505,7 +1594,7 @@ static int add_root_directory(sd_journal *j, const char *p, bool missing_ok) {
 
         if (m->wd <= 0 && j->inotify_fd >= 0) {
 
-                m->wd = inotify_add_watch(j->inotify_fd, m->path,
+                m->wd = inotify_add_watch_fd(j->inotify_fd, dirfd(d),
                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
                                           IN_ONLYDIR);
 
@@ -1516,7 +1605,7 @@ static int add_root_directory(sd_journal *j, const char *p, bool missing_ok) {
         if (j->no_new_files)
                 return 0;
 
-        FOREACH_DIRENT_ALL(de, d, return log_debug_errno(errno, "Failed to read directory %s: %m", m->path)) {
+        FOREACH_DIRENT_ALL(de, d, r = log_debug_errno(errno, "Failed to read directory %s: %m", m->path); goto fail) {
                 sd_id128_t id;
 
                 if (dirent_is_file_with_suffix(de, ".journal") ||
@@ -1585,8 +1674,7 @@ static int add_current_paths(sd_journal *j) {
         assert(j);
         assert(j->no_new_files);
 
-        /* Simply adds all directories for files we have open as
-         * "root" directories. We don't expect errors here, so we
+        /* Simply adds all directories for files we have open as directories. We don't expect errors here, so we
          * treat them as fatal. */
 
         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
@@ -1597,7 +1685,7 @@ static int add_current_paths(sd_journal *j) {
                 if (!dir)
                         return -ENOMEM;
 
-                r = add_root_directory(j, dir, true);
+                r = add_directory(j, dir, NULL);
                 if (r < 0)
                         return r;
         }
@@ -1614,13 +1702,7 @@ static int allocate_inotify(sd_journal *j) {
                         return -errno;
         }
 
-        if (!j->directories_by_wd) {
-                j->directories_by_wd = hashmap_new(NULL);
-                if (!j->directories_by_wd)
-                        return -ENOMEM;
-        }
-
-        return 0;
+        return hashmap_ensure_allocated(&j->directories_by_wd, NULL);
 }
 
 static sd_journal *journal_new(int flags, const char *path) {
@@ -1631,6 +1713,7 @@ static sd_journal *journal_new(int flags, const char *path) {
                 return NULL;
 
         j->original_pid = getpid();
+        j->toplevel_fd = -1;
         j->inotify_fd = -1;
         j->flags = flags;
         j->data_threshold = DEFAULT_DATA_THRESHOLD;
@@ -1684,6 +1767,9 @@ _public_ int sd_journal_open_container(sd_journal **ret, const char *machine, in
         char *p;
         int r;
 
+        /* This is pretty much deprecated, people should use machined's OpenMachineRootDirectory() call instead in
+         * combination with sd_journal_open_directory_fd(). */
+
         assert_return(machine, -EINVAL);
         assert_return(ret, -EINVAL);
         assert_return((flags & ~(SD_JOURNAL_LOCAL_ONLY|SD_JOURNAL_SYSTEM)) == 0, -EINVAL);
@@ -1726,13 +1812,16 @@ _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int f
 
         assert_return(ret, -EINVAL);
         assert_return(path, -EINVAL);
-        assert_return(flags == 0, -EINVAL);
+        assert_return((flags & ~SD_JOURNAL_OS_ROOT) == 0, -EINVAL);
 
         j = journal_new(flags, path);
         if (!j)
                 return -ENOMEM;
 
-        r = add_root_directory(j, path, false);
+        if (flags & SD_JOURNAL_OS_ROOT)
+                r = add_search_paths(j);
+        else
+                r = add_root_directory(j, path, false);
         if (r < 0)
                 goto fail;
 
@@ -1741,7 +1830,6 @@ _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int f
 
 fail:
         sd_journal_close(j);
-
         return r;
 }
 
@@ -1758,7 +1846,7 @@ _public_ int sd_journal_open_files(sd_journal **ret, const char **paths, int fla
                 return -ENOMEM;
 
         STRV_FOREACH(path, paths) {
-                r = add_any_file(j, *path);
+                r = add_any_file(j, -1, *path);
                 if (r < 0)
                         goto fail;
         }
@@ -1770,7 +1858,96 @@ _public_ int sd_journal_open_files(sd_journal **ret, const char **paths, int fla
 
 fail:
         sd_journal_close(j);
+        return r;
+}
+
+_public_ int sd_journal_open_directory_fd(sd_journal **ret, int fd, int flags) {
+        sd_journal *j;
+        struct stat st;
+        int r;
+
+        assert_return(ret, -EINVAL);
+        assert_return(fd >= 0, -EBADF);
+        assert_return((flags & ~SD_JOURNAL_OS_ROOT) == 0, -EINVAL);
+
+        if (fstat(fd, &st) < 0)
+                return -errno;
+
+        if (!S_ISDIR(st.st_mode))
+                return -EBADFD;
+
+        j = journal_new(flags, NULL);
+        if (!j)
+                return -ENOMEM;
+
+        j->toplevel_fd = fd;
+
+        if (flags & SD_JOURNAL_OS_ROOT)
+                r = add_search_paths(j);
+        else
+                r = add_root_directory(j, NULL, false);
+        if (r < 0)
+                goto fail;
 
+        *ret = j;
+        return 0;
+
+fail:
+        sd_journal_close(j);
+        return r;
+}
+
+_public_ int sd_journal_open_files_fd(sd_journal **ret, int fds[], unsigned n_fds, int flags) {
+        Iterator iterator;
+        JournalFile *f;
+        sd_journal *j;
+        unsigned i;
+        int r;
+
+        assert_return(ret, -EINVAL);
+        assert_return(n_fds > 0, -EBADF);
+        assert_return(flags == 0, -EINVAL);
+
+        j = journal_new(flags, NULL);
+        if (!j)
+                return -ENOMEM;
+
+        for (i = 0; i < n_fds; i++) {
+                struct stat st;
+
+                if (fds[i] < 0) {
+                        r = -EBADF;
+                        goto fail;
+                }
+
+                if (fstat(fds[i], &st) < 0) {
+                        r = -errno;
+                        goto fail;
+                }
+
+                if (!S_ISREG(st.st_mode)) {
+                        r = -EBADFD;
+                        goto fail;
+                }
+
+                r = add_any_file(j, fds[i], NULL);
+                if (r < 0)
+                        goto fail;
+        }
+
+        j->no_new_files = true;
+        j->no_inotify = true;
+
+        *ret = j;
+        return 0;
+
+fail:
+        /* If we fail, make sure we don't take possession of the files we managed to make use of successfuly, and they
+         * remain open */
+        ORDERED_HASHMAP_FOREACH(f, j->files, iterator)
+                f->close_fd = false;
+
+        sd_journal_close(j);
         return r;
 }
 
@@ -2097,6 +2274,9 @@ _public_ int sd_journal_get_fd(sd_journal *j) {
         assert_return(j, -EINVAL);
         assert_return(!journal_pid_changed(j), -ECHILD);
 
+        if (j->no_inotify)
+                return -EMEDIUMTYPE;
+
         if (j->inotify_fd >= 0)
                 return j->inotify_fd;
 
@@ -2104,10 +2284,14 @@ _public_ int sd_journal_get_fd(sd_journal *j) {
         if (r < 0)
                 return r;
 
+        log_debug("Reiterating files to get inotify watches established");
+
         /* Iterate through all dirs again, to add them to the
          * inotify */
         if (j->no_new_files)
                 r = add_current_paths(j);
+        else if (j->toplevel_fd >= 0)
+                r = add_root_directory(j, NULL, false);
         else if (j->path)
                 r = add_root_directory(j, j->path, true);
         else
diff --git a/src/journal/test-journal-flush.c b/src/journal/test-journal-flush.c
index 93dc0e0d81..ba8b20b228 100644
--- a/src/journal/test-journal-flush.c
+++ b/src/journal/test-journal-flush.c
@@ -38,7 +38,7 @@ int main(int argc, char *argv[]) {
         assert_se(mkdtemp(dn));
         fn = strappend(dn, "/test.journal");
 
-        r = journal_file_open(fn, O_CREAT|O_RDWR, 0644, false, false, NULL, NULL, NULL, NULL, &new_journal);
+        r = journal_file_open(-1, fn, O_CREAT|O_RDWR, 0644, false, false, NULL, NULL, NULL, NULL, &new_journal);
         assert_se(r >= 0);
 
         r = sd_journal_open(&j, 0);
diff --git a/src/journal/test-journal-interleaving.c b/src/journal/test-journal-interleaving.c
index f887f43f0d..5e063f4d04 100644
--- a/src/journal/test-journal-interleaving.c
+++ b/src/journal/test-journal-interleaving.c
@@ -52,7 +52,7 @@ noreturn static void log_assert_errno(const char *text, int eno, const char *fil
 
 static JournalFile *test_open(const char *name) {
         JournalFile *f;
-        assert_ret(journal_file_open(name, O_RDWR|O_CREAT, 0644, true, false, NULL, NULL, NULL, NULL, &f));
+        assert_ret(journal_file_open(-1, name, O_RDWR|O_CREAT, 0644, true, false, NULL, NULL, NULL, NULL, &f));
         return f;
 }
 
@@ -216,7 +216,7 @@ static void test_sequence_numbers(void) {
         assert_se(mkdtemp(t));
         assert_se(chdir(t) >= 0);
 
-        assert_se(journal_file_open("one.journal", O_RDWR|O_CREAT, 0644,
+        assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0644,
                                     true, false, NULL, NULL, NULL, NULL, &one) == 0);
 
         append_number(one, 1, &seqnum);
@@ -233,7 +233,7 @@ static void test_sequence_numbers(void) {
 
         memcpy(&seqnum_id, &one->header->seqnum_id, sizeof(sd_id128_t));
 
-        assert_se(journal_file_open("two.journal", O_RDWR|O_CREAT, 0644,
+        assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0644,
                                     true, false, NULL, NULL, NULL, one, &two) == 0);
 
         assert_se(two->header->state == STATE_ONLINE);
@@ -264,7 +264,7 @@ static void test_sequence_numbers(void) {
         /* restart server */
         seqnum = 0;
 
-        assert_se(journal_file_open("two.journal", O_RDWR, 0,
+        assert_se(journal_file_open(-1, "two.journal", O_RDWR, 0,
                                     true, false, NULL, NULL, NULL, NULL, &two) == 0);
 
         assert_se(sd_id128_equal(two->header->seqnum_id, seqnum_id));
diff --git a/src/journal/test-journal-stream.c b/src/journal/test-journal-stream.c
index 839ea5a9a5..7e5a980719 100644
--- a/src/journal/test-journal-stream.c
+++ b/src/journal/test-journal-stream.c
@@ -92,9 +92,9 @@ int main(int argc, char *argv[]) {
         assert_se(mkdtemp(t));
         assert_se(chdir(t) >= 0);
 
-        assert_se(journal_file_open("one.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &one) == 0);
-        assert_se(journal_file_open("two.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &two) == 0);
-        assert_se(journal_file_open("three.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &three) == 0);
+        assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &one) == 0);
+        assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &two) == 0);
+        assert_se(journal_file_open(-1, "three.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &three) == 0);
 
         for (i = 0; i < N_ENTRIES; i++) {
                 char *p, *q;
diff --git a/src/journal/test-journal-verify.c b/src/journal/test-journal-verify.c
index 6b4643cd25..3d2312fc55 100644
--- a/src/journal/test-journal-verify.c
+++ b/src/journal/test-journal-verify.c
@@ -55,7 +55,7 @@ static int raw_verify(const char *fn, const char *verification_key) {
         JournalFile *f;
         int r;
 
-        r = journal_file_open(fn, O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f);
+        r = journal_file_open(-1, fn, O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f);
         if (r < 0)
                 return r;
 
@@ -88,7 +88,7 @@ int main(int argc, char *argv[]) {
 
         log_info("Generating...");
 
-        assert_se(journal_file_open("test.journal", O_RDWR|O_CREAT, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
+        assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
 
         for (n = 0; n < N_ENTRIES; n++) {
                 struct iovec iovec;
@@ -111,7 +111,7 @@ int main(int argc, char *argv[]) {
 
         log_info("Verifying...");
 
-        assert_se(journal_file_open("test.journal", O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
+        assert_se(journal_file_open(-1, "test.journal", O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
         /* journal_file_print_header(f); */
         journal_file_dump(f);
 
diff --git a/src/journal/test-journal.c b/src/journal/test-journal.c
index ea685af782..2543d64b5b 100644
--- a/src/journal/test-journal.c
+++ b/src/journal/test-journal.c
@@ -42,7 +42,7 @@ static void test_non_empty(void) {
         assert_se(mkdtemp(t));
         assert_se(chdir(t) >= 0);
 
-        assert_se(journal_file_open("test.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, NULL, &f) == 0);
+        assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, NULL, &f) == 0);
 
         dual_timestamp_get(&ts);
 
@@ -131,13 +131,13 @@ static void test_empty(void) {
         assert_se(mkdtemp(t));
         assert_se(chdir(t) >= 0);
 
-        assert_se(journal_file_open("test.journal", O_RDWR|O_CREAT, 0666, false, false, NULL, NULL, NULL, NULL, &f1) == 0);
+        assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, false, false, NULL, NULL, NULL, NULL, &f1) == 0);
 
-        assert_se(journal_file_open("test-compress.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &f2) == 0);
+        assert_se(journal_file_open(-1, "test-compress.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &f2) == 0);
 
-        assert_se(journal_file_open("test-seal.journal", O_RDWR|O_CREAT, 0666, false, true, NULL, NULL, NULL, NULL, &f3) == 0);
+        assert_se(journal_file_open(-1, "test-seal.journal", O_RDWR|O_CREAT, 0666, false, true, NULL, NULL, NULL, NULL, &f3) == 0);
 
-        assert_se(journal_file_open("test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, NULL, &f4) == 0);
+        assert_se(journal_file_open(-1, "test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, NULL, &f4) == 0);
 
         journal_file_print_header(f1);
         puts("");