Merge pull request #5131 from keszybz/environment-generators

Environment generators

11 files changed, 713 insertions, 224 deletions

diff --git a/src/basic/conf-files.c b/src/basic/conf-files.c
index b5780194df..b8f0f5d03d 100644
--- a/src/basic/conf-files.c
+++ b/src/basic/conf-files.c
@@ -137,7 +137,6 @@ int conf_files_list(char ***strv, const char *suffix, const char *root, const ch
         va_list ap;
 
         assert(strv);
-        assert(suffix);
 
         va_start(ap, dir);
         dirs = strv_new_ap(dir, ap);
@@ -153,7 +152,6 @@ int conf_files_list_nulstr(char ***strv, const char *suffix, const char *root, c
         _cleanup_strv_free_ char **dirs = NULL;
 
         assert(strv);
-        assert(suffix);
 
         dirs = strv_split_nulstr(d);
         if (!dirs)
diff --git a/src/basic/def.h b/src/basic/def.h
index 2266eff650..10d776ec8e 100644
--- a/src/basic/def.h
+++ b/src/basic/def.h
@@ -73,18 +73,18 @@
 #define NOTIFY_BUFFER_MAX PIPE_BUF
 
 #ifdef HAVE_SPLIT_USR
-#define _CONF_PATHS_SPLIT_USR(n) "/lib/" n "\0"
+#  define _CONF_PATHS_SPLIT_USR(n) "/lib/" n "\0"
 #else
-#define _CONF_PATHS_SPLIT_USR(n)
+#  define _CONF_PATHS_SPLIT_USR(n)
 #endif
 
 /* Return a nulstr for a standard cascade of configuration paths,
  * suitable to pass to conf_files_list_nulstr() or config_parse_many_nulstr()
  * to implement drop-in directories for extending configuration
  * files. */
-#define CONF_PATHS_NULSTR(n) \
-        "/etc/" n "\0" \
-        "/run/" n "\0" \
-        "/usr/local/lib/" n "\0" \
-        "/usr/lib/" n "\0" \
+#define CONF_PATHS_NULSTR(n)                    \
+        "/etc/" n "\0"                          \
+        "/run/" n "\0"                          \
+        "/usr/local/lib/" n "\0"                \
+        "/usr/lib/" n "\0"                      \
         _CONF_PATHS_SPLIT_USR(n)
diff --git a/src/basic/env-util.c b/src/basic/env-util.c
index 96da38d45e..2ca64c3301 100644
--- a/src/basic/env-util.c
+++ b/src/basic/env-util.c
@@ -26,6 +26,7 @@
 
 #include "alloc-util.h"
 #include "env-util.h"
+#include "escape.h"
 #include "extract-word.h"
 #include "macro.h"
 #include "parse-util.h"
@@ -247,7 +248,7 @@ fail:
         return NULL;
 }
 
-_pure_ static bool env_match(const char *t, const char *pattern) {
+static bool env_match(const char *t, const char *pattern) {
         assert(t);
         assert(pattern);
 
@@ -273,6 +274,19 @@ _pure_ static bool env_match(const char *t, const char *pattern) {
         return false;
 }
 
+static bool env_entry_has_name(const char *entry, const char *name) {
+        const char *t;
+
+        assert(entry);
+        assert(name);
+
+        t = startswith(entry, name);
+        if (!t)
+                return false;
+
+        return *t == '=';
+}
+
 char **strv_env_delete(char **x, unsigned n_lists, ...) {
         size_t n, i = 0;
         char **k, **r;
@@ -386,18 +400,24 @@ char **strv_env_unset_many(char **l, ...) {
 
 int strv_env_replace(char ***l, char *p) {
         char **f;
+        const char *t, *name;
 
         assert(p);
 
         /* Replace first occurrence of the env var or add a new one in the
          * string list. Drop other occurences. Edits in-place. Does not copy p.
+         * p must be a valid key=value assignment.
          */
 
+        t = strchr(p, '=');
+        assert(t);
+
+        name = strndupa(p, t - p);
+
         for (f = *l; f && *f; f++)
-                if (env_match(*f, p)) {
-                        free(*f);
-                        *f = p;
-                        strv_env_unset(f + 1, p);
+                if (env_entry_has_name(*f, name)) {
+                        free_and_replace(*f, p);
+                        strv_env_unset(f + 1, *f);
                         return 0;
                 }
 
@@ -434,7 +454,7 @@ fail:
         return NULL;
 }
 
-char *strv_env_get_n(char **l, const char *name, size_t k) {
+char *strv_env_get_n(char **l, const char *name, size_t k, unsigned flags) {
         char **i;
 
         assert(name);
@@ -442,18 +462,25 @@ char *strv_env_get_n(char **l, const char *name, size_t k) {
         if (k <= 0)
                 return NULL;
 
-        STRV_FOREACH(i, l)
+        STRV_FOREACH_BACKWARDS(i, l)
                 if (strneq(*i, name, k) &&
                     (*i)[k] == '=')
                         return *i + k + 1;
 
+        if (flags & REPLACE_ENV_USE_ENVIRONMENT) {
+                const char *t;
+
+                t = strndupa(name, k);
+                return getenv(t);
+        };
+
         return NULL;
 }
 
 char *strv_env_get(char **l, const char *name) {
         assert(name);
 
-        return strv_env_get_n(l, name, strlen(name));
+        return strv_env_get_n(l, name, strlen(name), 0);
 }
 
 char **strv_env_clean_with_callback(char **e, void (*invalid_callback)(const char *p, void *userdata), void *userdata) {
@@ -492,19 +519,26 @@ char **strv_env_clean_with_callback(char **e, void (*invalid_callback)(const cha
         return e;
 }
 
-char *replace_env(const char *format, char **env) {
+char *replace_env_n(const char *format, size_t n, char **env, unsigned flags) {
         enum {
                 WORD,
                 CURLY,
-                VARIABLE
+                VARIABLE,
+                VARIABLE_RAW,
+                TEST,
+                DEFAULT_VALUE,
+                ALTERNATE_VALUE,
         } state = WORD;
 
-        const char *e, *word = format;
-        char *r = NULL, *k;
+        const char *e, *word = format, *test_value;
+        char *k;
+        _cleanup_free_ char *r = NULL;
+        size_t i, len;
+        int nest = 0;
 
         assert(format);
 
-        for (e = format; *e; e ++) {
+        for (e = format, i = 0; *e && i < n; e ++, i ++) {
 
                 switch (state) {
 
@@ -517,24 +551,36 @@ char *replace_env(const char *format, char **env) {
                         if (*e == '{') {
                                 k = strnappend(r, word, e-word-1);
                                 if (!k)
-                                        goto fail;
+                                        return NULL;
 
                                 free(r);
                                 r = k;
 
                                 word = e-1;
                                 state = VARIABLE;
-
+                                nest++;
                         } else if (*e == '$') {
                                 k = strnappend(r, word, e-word);
                                 if (!k)
-                                        goto fail;
+                                        return NULL;
 
                                 free(r);
                                 r = k;
 
                                 word = e+1;
                                 state = WORD;
+
+                        } else if (flags & REPLACE_ENV_ALLOW_BRACELESS && strchr(VALID_CHARS_ENV_NAME, *e)) {
+                                k = strnappend(r, word, e-word-1);
+                                if (!k)
+                                        return NULL;
+
+                                free(r);
+                                r = k;
+
+                                word = e-1;
+                                state = VARIABLE_RAW;
+
                         } else
                                 state = WORD;
                         break;
@@ -543,31 +589,109 @@ char *replace_env(const char *format, char **env) {
                         if (*e == '}') {
                                 const char *t;
 
-                                t = strempty(strv_env_get_n(env, word+2, e-word-2));
+                                t = strv_env_get_n(env, word+2, e-word-2, flags);
 
                                 k = strappend(r, t);
                                 if (!k)
-                                        goto fail;
+                                        return NULL;
 
                                 free(r);
                                 r = k;
 
                                 word = e+1;
                                 state = WORD;
+                        } else if (*e == ':') {
+                                if (!(flags & REPLACE_ENV_ALLOW_EXTENDED))
+                                        /* Treat this as unsupported syntax, i.e. do no replacement */
+                                        state = WORD;
+                                else {
+                                        len = e-word-2;
+                                        state = TEST;
+                                }
+                        }
+                        break;
+
+                case TEST:
+                        if (*e == '-')
+                                state = DEFAULT_VALUE;
+                        else if (*e == '+')
+                                state = ALTERNATE_VALUE;
+                        else {
+                                state = WORD;
+                                break;
+                        }
+
+                        test_value = e+1;
+                        break;
+
+                case DEFAULT_VALUE: /* fall through */
+                case ALTERNATE_VALUE:
+                        assert(flags & REPLACE_ENV_ALLOW_EXTENDED);
+
+                        if (*e == '{') {
+                                nest++;
+                                break;
+                        }
+
+                        if (*e != '}')
+                                break;
+
+                        nest--;
+                        if (nest == 0) { // || !strchr(e+1, '}')) {
+                                const char *t;
+                                _cleanup_free_ char *v = NULL;
+
+                                t = strv_env_get_n(env, word+2, len, flags);
+
+                                if (t && state == ALTERNATE_VALUE)
+                                        t = v = replace_env_n(test_value, e-test_value, env, flags);
+                                else if (!t && state == DEFAULT_VALUE)
+                                        t = v = replace_env_n(test_value, e-test_value, env, flags);
+
+                                k = strappend(r, t);
+                                if (!k)
+                                        return NULL;
+
+                                free(r);
+                                r = k;
+
+                                word = e+1;
+                                state = WORD;
+                        }
+                        break;
+
+                case VARIABLE_RAW:
+                        assert(flags & REPLACE_ENV_ALLOW_BRACELESS);
+
+                        if (!strchr(VALID_CHARS_ENV_NAME, *e)) {
+                                const char *t;
+
+                                t = strv_env_get_n(env, word+1, e-word-1, flags);
+
+                                k = strappend(r, t);
+                                if (!k)
+                                        return NULL;
+
+                                free(r);
+                                r = k;
+
+                                word = e--;
+                                i--;
+                                state = WORD;
                         }
                         break;
                 }
         }
 
-        k = strnappend(r, word, e-word);
-        if (!k)
-                goto fail;
+        if (state == VARIABLE_RAW) {
+                const char *t;
 
-        free(r);
-        return k;
+                assert(flags & REPLACE_ENV_ALLOW_BRACELESS);
 
-fail:
-        return mfree(r);
+                t = strv_env_get_n(env, word+1, e-word-1, flags);
+                return strappend(r, t);
+        } else
+                return strnappend(r, word, e-word);
 }
 
 char **replace_env_argv(char **argv, char **env) {
@@ -623,7 +747,7 @@ char **replace_env_argv(char **argv, char **env) {
                 }
 
                 /* If ${FOO} appears as part of a word, replace it by the variable as-is */
-                ret[k] = replace_env(*i, env);
+                ret[k] = replace_env(*i, env, 0);
                 if (!ret[k]) {
                         strv_free(ret);
                         return NULL;
@@ -644,3 +768,39 @@ int getenv_bool(const char *p) {
 
         return parse_boolean(e);
 }
+
+int serialize_environment(FILE *f, char **environment) {
+        char **e;
+
+        STRV_FOREACH(e, environment) {
+                _cleanup_free_ char *ce;
+
+                ce = cescape(*e);
+                if (!ce)
+                        return -ENOMEM;
+
+                fprintf(f, "env=%s\n", *e);
+        }
+
+        /* caller should call ferror() */
+
+        return 0;
+}
+
+int deserialize_environment(char ***environment, const char *line) {
+        char *uce = NULL;
+        int r;
+
+        assert(line);
+        assert(environment);
+
+        assert(startswith(line, "env="));
+        r = cunescape(line + 4, UNESCAPE_RELAX, &uce);
+        if (r < 0)
+                return r;
+
+        if (!env_assignment_is_valid(uce))
+                return -EINVAL;
+
+        return strv_env_replace(environment, uce);
+}
diff --git a/src/basic/env-util.h b/src/basic/env-util.h
index 8cb0fc2131..e88fa6aac0 100644
--- a/src/basic/env-util.h
+++ b/src/basic/env-util.h
@@ -21,6 +21,7 @@
 
 #include <stdbool.h>
 #include <stddef.h>
+#include <stdio.h>
 
 #include "macro.h"
 
@@ -28,9 +29,19 @@ bool env_name_is_valid(const char *e);
 bool env_value_is_valid(const char *e);
 bool env_assignment_is_valid(const char *e);
 
-char *replace_env(const char *format, char **env);
+enum {
+        REPLACE_ENV_USE_ENVIRONMENT = 1u,
+        REPLACE_ENV_ALLOW_BRACELESS = 2u,
+        REPLACE_ENV_ALLOW_EXTENDED  = 4u,
+};
+
+char *replace_env_n(const char *format, size_t n, char **env, unsigned flags);
 char **replace_env_argv(char **argv, char **env);
 
+static inline char *replace_env(const char *format, char **env, unsigned flags) {
+        return replace_env_n(format, strlen(format), env, flags);
+}
+
 bool strv_env_is_valid(char **e);
 #define strv_env_clean(l) strv_env_clean_with_callback(l, NULL, NULL)
 char **strv_env_clean_with_callback(char **l, void (*invalid_callback)(const char *p, void *userdata), void *userdata);
@@ -46,7 +57,10 @@ char **strv_env_unset(char **l, const char *p); /* In place ... */
 char **strv_env_unset_many(char **l, ...) _sentinel_;
 int strv_env_replace(char ***l, char *p); /* In place ... */
 
-char *strv_env_get_n(char **l, const char *name, size_t k) _pure_;
+char *strv_env_get_n(char **l, const char *name, size_t k, unsigned flags) _pure_;
 char *strv_env_get(char **x, const char *n) _pure_;
 
 int getenv_bool(const char *p);
+
+int serialize_environment(FILE *f, char **environment);
+int deserialize_environment(char ***environment, const char *line);
diff --git a/src/basic/exec-util.c b/src/basic/exec-util.c
new file mode 100644
index 0000000000..aced9e8e3d
--- /dev/null
+++ b/src/basic/exec-util.c
@@ -0,0 +1,360 @@
+/***
+  This file is part of systemd.
+
+  Copyright 2010 Lennart Poettering
+
+  systemd is free software; you can redistribute it and/or modify it
+  under the terms of the GNU Lesser General Public License as published by
+  the Free Software Foundation; either version 2.1 of the License, or
+  (at your option) any later version.
+
+  systemd is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public License
+  along with systemd; If not, see <http://www.gnu.org/licenses/>.
+***/
+
+#include <dirent.h>
+#include <errno.h>
+#include <sys/prctl.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <stdio.h>
+
+#include "alloc-util.h"
+#include "conf-files.h"
+#include "env-util.h"
+#include "exec-util.h"
+#include "fd-util.h"
+#include "fileio.h"
+#include "hashmap.h"
+#include "macro.h"
+#include "process-util.h"
+#include "set.h"
+#include "signal-util.h"
+#include "stat-util.h"
+#include "string-util.h"
+#include "strv.h"
+#include "terminal-util.h"
+#include "util.h"
+
+/* Put this test here for a lack of better place */
+assert_cc(EAGAIN == EWOULDBLOCK);
+
+static int do_spawn(const char *path, char *argv[], int stdout_fd, pid_t *pid) {
+
+        pid_t _pid;
+
+        if (null_or_empty_path(path)) {
+                log_debug("%s is empty (a mask).", path);
+                return 0;
+        }
+
+        _pid = fork();
+        if (_pid < 0)
+                return log_error_errno(errno, "Failed to fork: %m");
+        if (_pid == 0) {
+                char *_argv[2];
+
+                assert_se(prctl(PR_SET_PDEATHSIG, SIGTERM) == 0);
+
+                if (stdout_fd >= 0) {
+                        /* If the fd happens to be in the right place, go along with that */
+                        if (stdout_fd != STDOUT_FILENO &&
+                            dup2(stdout_fd, STDOUT_FILENO) < 0)
+                                return -errno;
+
+                        fd_cloexec(STDOUT_FILENO, false);
+                }
+
+                if (!argv) {
+                        _argv[0] = (char*) path;
+                        _argv[1] = NULL;
+                        argv = _argv;
+                } else
+                        argv[0] = (char*) path;
+
+                execv(path, argv);
+                log_error_errno(errno, "Failed to execute %s: %m", path);
+                _exit(EXIT_FAILURE);
+        }
+
+        log_debug("Spawned %s as " PID_FMT ".", path, _pid);
+        *pid = _pid;
+        return 1;
+}
+
+static int do_execute(
+                char **directories,
+                usec_t timeout,
+                gather_stdout_callback_t const callbacks[_STDOUT_CONSUME_MAX],
+                void* const callback_args[_STDOUT_CONSUME_MAX],
+                int output_fd,
+                char *argv[]) {
+
+        _cleanup_hashmap_free_free_ Hashmap *pids = NULL;
+        _cleanup_strv_free_ char **paths = NULL;
+        char **path;
+        int r;
+
+        /* We fork this all off from a child process so that we can somewhat cleanly make
+         * use of SIGALRM to set a time limit.
+         *
+         * If callbacks is nonnull, execution is serial. Otherwise, we default to parallel.
+         */
+
+        (void) reset_all_signal_handlers();
+        (void) reset_signal_mask();
+
+        assert_se(prctl(PR_SET_PDEATHSIG, SIGTERM) == 0);
+
+        r = conf_files_list_strv(&paths, NULL, NULL, (const char* const*) directories);
+        if (r < 0)
+                return r;
+
+        if (!callbacks) {
+                pids = hashmap_new(NULL);
+                if (!pids)
+                        return log_oom();
+        }
+
+        /* Abort execution of this process after the timout. We simply rely on SIGALRM as
+         * default action terminating the process, and turn on alarm(). */
+
+        if (timeout != USEC_INFINITY)
+                alarm((timeout + USEC_PER_SEC - 1) / USEC_PER_SEC);
+
+        STRV_FOREACH(path, paths) {
+                _cleanup_free_ char *t = NULL;
+                _cleanup_close_ int fd = -1;
+                pid_t pid;
+
+                t = strdup(*path);
+                if (!t)
+                        return log_oom();
+
+                if (callbacks) {
+                        fd = open_serialization_fd(basename(*path));
+                        if (fd < 0)
+                                return log_error_errno(fd, "Failed to open serialization file: %m");
+                }
+
+                r = do_spawn(t, argv, fd, &pid);
+                if (r <= 0)
+                        continue;
+
+                if (pids) {
+                        r = hashmap_put(pids, PID_TO_PTR(pid), t);
+                        if (r < 0)
+                                return log_oom();
+                        t = NULL;
+                } else {
+                        r = wait_for_terminate_and_warn(t, pid, true);
+                        if (r < 0)
+                                continue;
+
+                        if (lseek(fd, 0, SEEK_SET) < 0)
+                                return log_error_errno(errno, "Failed to seek on serialization fd: %m");
+
+                        r = callbacks[STDOUT_GENERATE](fd, callback_args[STDOUT_GENERATE]);
+                        fd = -1;
+                        if (r < 0)
+                                return log_error_errno(r, "Failed to process output from %s: %m", *path);
+                }
+        }
+
+        if (callbacks) {
+                r = callbacks[STDOUT_COLLECT](output_fd, callback_args[STDOUT_COLLECT]);
+                if (r < 0)
+                        return log_error_errno(r, "Callback two failed: %m");
+        }
+
+        while (!hashmap_isempty(pids)) {
+                _cleanup_free_ char *t = NULL;
+                pid_t pid;
+
+                pid = PTR_TO_PID(hashmap_first_key(pids));
+                assert(pid > 0);
+
+                t = hashmap_remove(pids, PID_TO_PTR(pid));
+                assert(t);
+
+                wait_for_terminate_and_warn(t, pid, true);
+        }
+
+        return 0;
+}
+
+int execute_directories(
+                const char* const* directories,
+                usec_t timeout,
+                gather_stdout_callback_t const callbacks[_STDOUT_CONSUME_MAX],
+                void* const callback_args[_STDOUT_CONSUME_MAX],
+                char *argv[]) {
+
+        pid_t executor_pid;
+        char *name;
+        char **dirs = (char**) directories;
+        _cleanup_close_ int fd = -1;
+        int r;
+
+        assert(!strv_isempty(dirs));
+
+        name = basename(dirs[0]);
+        assert(!isempty(name));
+
+        if (callbacks) {
+                assert(callback_args);
+                assert(callbacks[STDOUT_GENERATE]);
+                assert(callbacks[STDOUT_COLLECT]);
+                assert(callbacks[STDOUT_CONSUME]);
+
+                fd = open_serialization_fd(name);
+                if (fd < 0)
+                        return log_error_errno(fd, "Failed to open serialization file: %m");
+        }
+
+        /* Executes all binaries in the directories serially or in parallel and waits for
+         * them to finish. Optionally a timeout is applied. If a file with the same name
+         * exists in more than one directory, the earliest one wins. */
+
+        executor_pid = fork();
+        if (executor_pid < 0)
+                return log_error_errno(errno, "Failed to fork: %m");
+
+        if (executor_pid == 0) {
+                r = do_execute(dirs, timeout, callbacks, callback_args, fd, argv);
+                _exit(r < 0 ? EXIT_FAILURE : EXIT_SUCCESS);
+        }
+
+        r = wait_for_terminate_and_warn(name, executor_pid, true);
+        if (r < 0)
+                return log_error_errno(r, "Execution failed: %m");
+        if (r > 0) {
+                /* non-zero return code from child */
+                log_error("Forker process failed.");
+                return -EREMOTEIO;
+        }
+
+        if (!callbacks)
+                return 0;
+
+        if (lseek(fd, 0, SEEK_SET) < 0)
+                return log_error_errno(errno, "Failed to rewind serialization fd: %m");
+
+        r = callbacks[STDOUT_CONSUME](fd, callback_args[STDOUT_CONSUME]);
+        fd = -1;
+        if (r < 0)
+                return log_error_errno(r, "Failed to parse returned data: %m");
+        return 0;
+}
+
+static int gather_environment_generate(int fd, void *arg) {
+        char ***env = arg, **x, **y;
+        _cleanup_fclose_ FILE *f = NULL;
+        _cleanup_strv_free_ char **new;
+        int r;
+
+        /* Read a series of VAR=value assignments from fd, use them to update the list of
+         * variables in env. Also update the exported environment.
+         *
+         * fd is always consumed, even on error.
+         */
+
+        assert(env);
+
+        f = fdopen(fd, "r");
+        if (!f) {
+                safe_close(fd);
+                return -errno;
+        }
+
+        r = load_env_file_pairs(f, NULL, NULL, &new);
+        if (r < 0)
+                return r;
+
+        STRV_FOREACH_PAIR(x, y, new) {
+                char *p;
+
+                if (!env_name_is_valid(*x)) {
+                        log_warning("Invalid variable assignment \"%s=...\", ignoring.", *x);
+                        continue;
+                }
+
+                p = strjoin(*x, "=", *y);
+                if (!p)
+                        return -ENOMEM;
+
+                r = strv_env_replace(env, p);
+                if (r < 0)
+                        return r;
+
+                if (setenv(*x, *y, true) < 0)
+                        return -errno;
+        }
+
+        return r;
+}
+
+static int gather_environment_collect(int fd, void *arg) {
+        char ***env = arg;
+        _cleanup_fclose_ FILE *f = NULL;
+        int r;
+
+        /* Write out a series of env=cescape(VAR=value) assignments to fd. */
+
+        assert(env);
+
+        f = fdopen(fd, "w");
+        if (!f) {
+                safe_close(fd);
+                return -errno;
+        }
+
+        r = serialize_environment(f, *env);
+        if (r < 0)
+                return r;
+
+        if (ferror(f))
+                return errno > 0 ? -errno : -EIO;
+
+        return 0;
+}
+
+static int gather_environment_consume(int fd, void *arg) {
+        char ***env = arg;
+        _cleanup_fclose_ FILE *f = NULL;
+        char line[LINE_MAX];
+        int r = 0, k;
+
+        /* Read a series of env=cescape(VAR=value) assignments from fd into env. */
+
+        assert(env);
+
+        f = fdopen(fd, "r");
+        if (!f) {
+                safe_close(fd);
+                return -errno;
+        }
+
+        FOREACH_LINE(line, f, return -EIO) {
+                truncate_nl(line);
+
+                k = deserialize_environment(env, line);
+                if (k < 0)
+                        log_error_errno(k, "Invalid line \"%s\": %m", line);
+                if (k < 0 && r == 0)
+                        r = k;
+        }
+
+        return r;
+}
+
+const gather_stdout_callback_t gather_environment[] = {
+        gather_environment_generate,
+        gather_environment_collect,
+        gather_environment_consume,
+};
diff --git a/src/basic/exec-util.h b/src/basic/exec-util.h
new file mode 100644
index 0000000000..72009799b2
--- /dev/null
+++ b/src/basic/exec-util.h
@@ -0,0 +1,40 @@
+/***
+  This file is part of systemd.
+
+  Copyright 2017 Zbigniew Jędrzejewski-Szmek
+
+  systemd is free software; you can redistribute it and/or modify it
+  under the terms of the GNU Lesser General Public License as published by
+  the Free Software Foundation; either version 2.1 of the License, or
+  (at your option) any later version.
+
+  systemd is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public License
+  along with systemd; If not, see <http://www.gnu.org/licenses/>.
+***/
+
+#include <stdbool.h>
+
+#include "time-util.h"
+
+typedef int (*gather_stdout_callback_t) (int fd, void *arg);
+
+enum {
+        STDOUT_GENERATE,   /* from generators to helper process */
+        STDOUT_COLLECT,    /* from helper process to main process */
+        STDOUT_CONSUME,    /* process data in main process */
+        _STDOUT_CONSUME_MAX,
+};
+
+int execute_directories(
+                const char* const* directories,
+                usec_t timeout,
+                gather_stdout_callback_t const callbacks[_STDOUT_CONSUME_MAX],
+                void* const callback_args[_STDOUT_CONSUME_MAX],
+                char *argv[]);
+
+extern const gather_stdout_callback_t gather_environment[_STDOUT_CONSUME_MAX];
diff --git a/src/basic/fileio.c b/src/basic/fileio.c
index c43b0583a4..b9a9f74892 100644
--- a/src/basic/fileio.c
+++ b/src/basic/fileio.c
@@ -30,6 +30,7 @@
 
 #include "alloc-util.h"
 #include "ctype.h"
+#include "env-util.h"
 #include "escape.h"
 #include "fd-util.h"
 #include "fileio.h"
@@ -586,14 +587,9 @@ fail:
         return r;
 }
 
-static int parse_env_file_push(
+static int check_utf8ness_and_warn(
                 const char *filename, unsigned line,
-                const char *key, char *value,
-                void *userdata,
-                int *n_pushed) {
-
-        const char *k;
-        va_list aq, *ap = userdata;
+                const char *key, char *value) {
 
         if (!utf8_is_valid(key)) {
                 _cleanup_free_ char *p = NULL;
@@ -611,6 +607,23 @@ static int parse_env_file_push(
                 return -EINVAL;
         }
 
+        return 0;
+}
+
+static int parse_env_file_push(
+                const char *filename, unsigned line,
+                const char *key, char *value,
+                void *userdata,
+                int *n_pushed) {
+
+        const char *k;
+        va_list aq, *ap = userdata;
+        int r;
+
+        r = check_utf8ness_and_warn(filename, line, key, value);
+        if (r < 0)
+                return r;
+
         va_copy(aq, *ap);
 
         while ((k = va_arg(aq, const char *))) {
@@ -662,27 +675,19 @@ static int load_env_file_push(
         char *p;
         int r;
 
-        if (!utf8_is_valid(key)) {
-                _cleanup_free_ char *t = utf8_escape_invalid(key);
-
-                log_error("%s:%u: invalid UTF-8 for key '%s', ignoring.", strna(filename), line, t);
-                return -EINVAL;
-        }
-
-        if (value && !utf8_is_valid(value)) {
-                _cleanup_free_ char *t = utf8_escape_invalid(value);
-
-                log_error("%s:%u: invalid UTF-8 value for key %s: '%s', ignoring.", strna(filename), line, key, t);
-                return -EINVAL;
-        }
+        r = check_utf8ness_and_warn(filename, line, key, value);
+        if (r < 0)
+                return r;
 
-        p = strjoin(key, "=", strempty(value));
+        p = strjoin(key, "=", value);
         if (!p)
                 return -ENOMEM;
 
-        r = strv_consume(m, p);
-        if (r < 0)
+        r = strv_env_replace(m, p);
+        if (r < 0) {
+                free(p);
                 return r;
+        }
 
         if (n_pushed)
                 (*n_pushed)++;
@@ -716,19 +721,9 @@ static int load_env_file_push_pairs(
         char ***m = userdata;
         int r;
 
-        if (!utf8_is_valid(key)) {
-                _cleanup_free_ char *t = utf8_escape_invalid(key);
-
-                log_error("%s:%u: invalid UTF-8 for key '%s', ignoring.", strna(filename), line, t);
-                return -EINVAL;
-        }
-
-        if (value && !utf8_is_valid(value)) {
-                _cleanup_free_ char *t = utf8_escape_invalid(value);
-
-                log_error("%s:%u: invalid UTF-8 value for key %s: '%s', ignoring.", strna(filename), line, key, t);
-                return -EINVAL;
-        }
+        r = check_utf8ness_and_warn(filename, line, key, value);
+        if (r < 0)
+                return r;
 
         r = strv_extend(m, key);
         if (r < 0)
@@ -767,6 +762,51 @@ int load_env_file_pairs(FILE *f, const char *fname, const char *newline, char **
         return 0;
 }
 
+static int merge_env_file_push(
+                const char *filename, unsigned line,
+                const char *key, char *value,
+                void *userdata,
+                int *n_pushed) {
+
+        char ***env = userdata;
+        char *expanded_value;
+
+        assert(env);
+
+        if (!value) {
+                log_error("%s:%u: invalid syntax (around \"%s\"), ignoring.", strna(filename), line, key);
+                return 0;
+        }
+
+        if (!env_name_is_valid(key)) {
+                log_error("%s:%u: invalid variable name \"%s\", ignoring.", strna(filename), line, key);
+                return 0;
+        }
+
+        expanded_value = replace_env(value, *env,
+                                     REPLACE_ENV_USE_ENVIRONMENT|
+                                     REPLACE_ENV_ALLOW_BRACELESS|
+                                     REPLACE_ENV_ALLOW_EXTENDED);
+        if (!expanded_value)
+                return -ENOMEM;
+
+        free_and_replace(value, expanded_value);
+
+        return load_env_file_push(filename, line, key, value, env, n_pushed);
+}
+
+int merge_env_file(
+                char ***env,
+                FILE *f,
+                const char *fname) {
+
+        /* NOTE: this function supports braceful and braceless variable expansions,
+         * plus "extended" substitutions, unlike other exported parsing functions.
+         */
+
+        return parse_env_file_internal(f, fname, NEWLINE, merge_env_file_push, env, NULL);
+}
+
 static void write_env_var(FILE *f, const char *v) {
         const char *p;
 
@@ -1342,6 +1382,25 @@ int open_tmpfile_linkable(const char *target, int flags, char **ret_path) {
         return fd;
 }
 
+int open_serialization_fd(const char *ident) {
+        int fd = -1;
+
+        fd = memfd_create(ident, MFD_CLOEXEC);
+        if (fd < 0) {
+                const char *path;
+
+                path = getpid() == 1 ? "/run/systemd" : "/tmp";
+                fd = open_tmpfile_unlinkable(path, O_RDWR|O_CLOEXEC);
+                if (fd < 0)
+                        return fd;
+
+                log_debug("Serializing %s to %s.", ident, path);
+        } else
+                log_debug("Serializing %s to memfd.", ident);
+
+        return fd;
+}
+
 int link_tmpfile(int fd, const char *path, const char *target) {
 
         assert(fd >= 0);
diff --git a/src/basic/fileio.h b/src/basic/fileio.h
index 17b38a5d60..e547614cc4 100644
--- a/src/basic/fileio.h
+++ b/src/basic/fileio.h
@@ -48,6 +48,8 @@ int parse_env_file(const char *fname, const char *separator, ...) _sentinel_;
 int load_env_file(FILE *f, const char *fname, const char *separator, char ***l);
 int load_env_file_pairs(FILE *f, const char *fname, const char *separator, char ***l);
 
+int merge_env_file(char ***env, FILE *f, const char *fname);
+
 int write_env_file(const char *fname, char **l);
 
 int executable_is_script(const char *path, char **interpreter);
@@ -84,6 +86,7 @@ int fputs_with_space(FILE *f, const char *s, const char *separator, bool *space)
 
 int open_tmpfile_unlinkable(const char *directory, int flags);
 int open_tmpfile_linkable(const char *target, int flags, char **ret_path);
+int open_serialization_fd(const char *ident);
 
 int link_tmpfile(int fd, const char *path, const char *target);
 
diff --git a/src/basic/strv.c b/src/basic/strv.c
index 0eec868eed..60f92e6373 100644
--- a/src/basic/strv.c
+++ b/src/basic/strv.c
@@ -564,9 +564,6 @@ int strv_extend_front(char ***l, const char *value) {
 
         /* Like strv_extend(), but prepends rather than appends the new entry */
 
-        if (!value)
-                return 0;
-
         n = strv_length(*l);
 
         /* Increase and overflow check. */
@@ -574,9 +571,12 @@ int strv_extend_front(char ***l, const char *value) {
         if (m < n)
                 return -ENOMEM;
 
-        v = strdup(value);
-        if (!v)
-                return -ENOMEM;
+        if (value) {
+                v = strdup(value);
+                if (!v)
+                        return -ENOMEM;
+        } else
+                v = NULL;
 
         c = realloc_multiply(*l, sizeof(char*), m);
         if (!c) {
diff --git a/src/basic/util.c b/src/basic/util.c
index 6204906f37..3dce0ea92e 100644
--- a/src/basic/util.c
+++ b/src/basic/util.c
@@ -59,9 +59,6 @@
 #include "user-util.h"
 #include "util.h"
 
-/* Put this test here for a lack of better place */
-assert_cc(EAGAIN == EWOULDBLOCK);
-
 int saved_argc = 0;
 char **saved_argv = NULL;
 static int saved_in_initrd = -1;
@@ -80,146 +77,6 @@ size_t page_size(void) {
         return pgsz;
 }
 
-static int do_execute(char **directories, usec_t timeout, char *argv[]) {
-        _cleanup_hashmap_free_free_ Hashmap *pids = NULL;
-        _cleanup_set_free_free_ Set *seen = NULL;
-        char **directory;
-
-        /* We fork this all off from a child process so that we can
-         * somewhat cleanly make use of SIGALRM to set a time limit */
-
-        (void) reset_all_signal_handlers();
-        (void) reset_signal_mask();
-
-        assert_se(prctl(PR_SET_PDEATHSIG, SIGTERM) == 0);
-
-        pids = hashmap_new(NULL);
-        if (!pids)
-                return log_oom();
-
-        seen = set_new(&string_hash_ops);
-        if (!seen)
-                return log_oom();
-
-        STRV_FOREACH(directory, directories) {
-                _cleanup_closedir_ DIR *d;
-                struct dirent *de;
-
-                d = opendir(*directory);
-                if (!d) {
-                        if (errno == ENOENT)
-                                continue;
-
-                        return log_error_errno(errno, "Failed to open directory %s: %m", *directory);
-                }
-
-                FOREACH_DIRENT(de, d, break) {
-                        _cleanup_free_ char *path = NULL;
-                        pid_t pid;
-                        int r;
-
-                        if (!dirent_is_file(de))
-                                continue;
-
-                        if (set_contains(seen, de->d_name)) {
-                                log_debug("%1$s/%2$s skipped (%2$s was already seen).", *directory, de->d_name);
-                                continue;
-                        }
-
-                        r = set_put_strdup(seen, de->d_name);
-                        if (r < 0)
-                                return log_oom();
-
-                        path = strjoin(*directory, "/", de->d_name);
-                        if (!path)
-                                return log_oom();
-
-                        if (null_or_empty_path(path)) {
-                                log_debug("%s is empty (a mask).", path);
-                                continue;
-                        }
-
-                        pid = fork();
-                        if (pid < 0) {
-                                log_error_errno(errno, "Failed to fork: %m");
-                                continue;
-                        } else if (pid == 0) {
-                                char *_argv[2];
-
-                                assert_se(prctl(PR_SET_PDEATHSIG, SIGTERM) == 0);
-
-                                if (!argv) {
-                                        _argv[0] = path;
-                                        _argv[1] = NULL;
-                                        argv = _argv;
-                                } else
-                                        argv[0] = path;
-
-                                execv(path, argv);
-                                return log_error_errno(errno, "Failed to execute %s: %m", path);
-                        }
-
-                        log_debug("Spawned %s as " PID_FMT ".", path, pid);
-
-                        r = hashmap_put(pids, PID_TO_PTR(pid), path);
-                        if (r < 0)
-                                return log_oom();
-                        path = NULL;
-                }
-        }
-
-        /* Abort execution of this process after the timout. We simply
-         * rely on SIGALRM as default action terminating the process,
-         * and turn on alarm(). */
-
-        if (timeout != USEC_INFINITY)
-                alarm((timeout + USEC_PER_SEC - 1) / USEC_PER_SEC);
-
-        while (!hashmap_isempty(pids)) {
-                _cleanup_free_ char *path = NULL;
-                pid_t pid;
-
-                pid = PTR_TO_PID(hashmap_first_key(pids));
-                assert(pid > 0);
-
-                path = hashmap_remove(pids, PID_TO_PTR(pid));
-                assert(path);
-
-                wait_for_terminate_and_warn(path, pid, true);
-        }
-
-        return 0;
-}
-
-void execute_directories(const char* const* directories, usec_t timeout, char *argv[]) {
-        pid_t executor_pid;
-        int r;
-        char *name;
-        char **dirs = (char**) directories;
-
-        assert(!strv_isempty(dirs));
-
-        name = basename(dirs[0]);
-        assert(!isempty(name));
-
-        /* Executes all binaries in the directories in parallel and waits
-         * for them to finish. Optionally a timeout is applied. If a file
-         * with the same name exists in more than one directory, the
-         * earliest one wins. */
-
-        executor_pid = fork();
-        if (executor_pid < 0) {
-                log_error_errno(errno, "Failed to fork: %m");
-                return;
-
-        } else if (executor_pid == 0) {
-                r = do_execute(dirs, timeout, argv);
-                _exit(r < 0 ? EXIT_FAILURE : EXIT_SUCCESS);
-        }
-
-        wait_for_terminate_and_warn(name, executor_pid, true);
-}
-
 bool plymouth_running(void) {
         return access("/run/plymouth/pid", F_OK) >= 0;
 }
diff --git a/src/basic/util.h b/src/basic/util.h
index c3802a811c..c7da6c39bf 100644
--- a/src/basic/util.h
+++ b/src/basic/util.h
@@ -65,8 +65,6 @@ static inline const char* enable_disable(bool b) {
         return b ? "enable" : "disable";
 }
 
-void execute_directories(const char* const* directories, usec_t timeout, char *argv[]);
-
 bool plymouth_running(void);
 
 bool display_is_local(const char *display) _pure_;