shared: rework env file reader

Implement this with a proper state machine, so that newlines and
escaped chars can appear in string assignments. This should bring the
parser much closer to shell.

7 files changed, 418 insertions, 138 deletions

diff --git a/src/core/execute.c b/src/core/execute.c
index 91815b838e..2c13d1f9f6 100644
--- a/src/core/execute.c
+++ b/src/core/execute.c
@@ -1741,7 +1741,7 @@ int exec_context_load_environment(const ExecContext *c, char ***l) {
                         return -EINVAL;
                 }
                 for (n = 0; n < count; n++) {
-                        k = load_env_file(pglob.gl_pathv[n], &p);
+                        k = load_env_file(pglob.gl_pathv[n], NULL, &p);
                         if (k < 0) {
                                 if (ignore)
                                         continue;
diff --git a/src/hostname/hostnamed.c b/src/hostname/hostnamed.c
index b4d6d51402..aaa2d6594a 100644
--- a/src/hostname/hostnamed.c
+++ b/src/hostname/hostnamed.c
@@ -303,7 +303,7 @@ static int write_data_other(void) {
         char **l = NULL;
         int r, p;
 
-        r = load_env_file("/etc/machine-info", &l);
+        r = load_env_file("/etc/machine-info", NULL, &l);
         if (r < 0 && r != -ENOENT)
                 return r;
 
diff --git a/src/locale/localed.c b/src/locale/localed.c
index 60083b7681..df812ee651 100644
--- a/src/locale/localed.c
+++ b/src/locale/localed.c
@@ -355,7 +355,7 @@ static int write_data_locale(void) {
         int r, p;
         char **l = NULL;
 
-        r = load_env_file("/etc/locale.conf", &l);
+        r = load_env_file("/etc/locale.conf", NULL, &l);
         if (r < 0 && r != -ENOENT)
                 return r;
 
@@ -494,7 +494,7 @@ static int write_data_vconsole(void) {
         int r;
         char **l = NULL;
 
-        r = load_env_file("/etc/vconsole.conf", &l);
+        r = load_env_file("/etc/vconsole.conf", NULL, &l);
         if (r < 0 && r != -ENOENT)
                 return r;
 
diff --git a/src/shared/fileio.c b/src/shared/fileio.c
index 5b8be5ce2d..96e23c5bbb 100644
--- a/src/shared/fileio.c
+++ b/src/shared/fileio.c
@@ -177,169 +177,348 @@ int read_full_file(const char *fn, char **contents, size_t *size) {
         return 0;
 }
 
-int parse_env_file(
+static int parse_env_file_internal(
                 const char *fname,
-                const char *separator, ...) {
+                const char *newline,
+                int (*push) (const char *key, char *value, void *userdata),
+                void *userdata) {
+
+        _cleanup_free_ char *contents = NULL, *key = NULL;
+        size_t key_alloc = 0, n_key = 0, value_alloc = 0, n_value = 0, last_whitespace = (size_t) -1;
+        char *p, *value = NULL;
+        int r;
 
-        int r = 0;
-        char *contents = NULL, *p;
+        enum {
+                PRE_KEY,
+                KEY,
+                PRE_EQUAL,
+                PRE_VALUE,
+                VALUE,
+                VALUE_ESCAPE,
+                SINGLE_QUOTE_VALUE,
+                SINGLE_QUOTE_VALUE_ESCAPE,
+                DOUBLE_QUOTE_VALUE,
+                DOUBLE_QUOTE_VALUE_ESCAPE,
+                COMMENT,
+                COMMENT_ESCAPE
+        } state = PRE_KEY;
 
         assert(fname);
-        assert(separator);
+        assert(newline);
 
         r = read_full_file(fname, &contents, NULL);
         if (r < 0)
                 return r;
 
-        p = contents;
-        for (;;) {
-                const char *key = NULL;
+        for (p = contents; *p; p++) {
+                char c = *p;
+
+                switch (state) {
+
+                case PRE_KEY:
+                        if (strchr(COMMENTS, c))
+                                state = COMMENT;
+                        else if (!strchr(WHITESPACE, c)) {
+                                state = KEY;
+                                if (!greedy_realloc((void**) &key, &key_alloc, n_key+2)) {
+                                        r = -ENOMEM;
+                                        goto fail;
+                                }
+
+                                key[n_key++] = c;
+                        }
+                        break;
+
+                case KEY:
+                        if (strchr(newline, c)) {
+                                state = PRE_KEY;
+                                n_key = 0;
+                        } else if (strchr(WHITESPACE, c))
+                                state = PRE_EQUAL;
+                        else if (c == '=')
+                                state = PRE_VALUE;
+                        else {
+                                if (!greedy_realloc((void**) &key, &key_alloc, n_key+2)) {
+                                        r = -ENOMEM;
+                                        goto fail;
+                                }
+
+                                key[n_key++] = c;
+                        }
 
-                p += strspn(p, separator);
-                p += strspn(p, WHITESPACE);
+                        break;
+
+                case PRE_EQUAL:
+                        if (strchr(newline, c)) {
+                                state = PRE_KEY;
+                                n_key = 0;
+                        } else if (c == '=')
+                                state = PRE_VALUE;
+                        else if (!strchr(WHITESPACE, c)) {
+                                n_key = 0;
+                                state = COMMENT;
+                        }
 
-                if (!*p)
                         break;
 
-                if (!strchr(COMMENTS, *p)) {
-                        va_list ap;
-                        char **value;
+                case PRE_VALUE:
+                        if (strchr(newline, c)) {
+                                state = PRE_KEY;
+                                key[n_key] = 0;
 
-                        va_start(ap, separator);
-                        while ((key = va_arg(ap, char *))) {
-                                size_t n;
-                                char *v;
+                                if (value)
+                                        value[n_value] = 0;
 
-                                value = va_arg(ap, char **);
+                                r = push(key, value, userdata);
+                                if (r < 0)
+                                        goto fail;
+
+                                n_key = 0;
+                                value = NULL;
+                                value_alloc = n_value = 0;
+                        } else if (c == '\'')
+                                state = SINGLE_QUOTE_VALUE;
+                        else if (c == '\"')
+                                state = DOUBLE_QUOTE_VALUE;
+                        else if (c == '\\')
+                                state = VALUE_ESCAPE;
+                        else if (!strchr(WHITESPACE, c)) {
+                                state = VALUE;
+
+                                if (!greedy_realloc((void**) &value, &value_alloc, n_value+2)) {
+                                        r = -ENOMEM;
+                                        goto fail;
+                                }
+
+                                value[n_value++] = c;
+                        }
+
+                        break;
+
+                case VALUE:
+                        if (strchr(newline, c)) {
+                                state = PRE_KEY;
+                                key[n_key] = 0;
+
+                                if (value)
+                                        value[n_value] = 0;
+
+                                /* Chomp off trailing whitespace */
+                                if (last_whitespace != (size_t) -1)
+                                        value[last_whitespace] = 0;
+
+                                r = push(key, value, userdata);
+                                if (r < 0)
+                                        goto fail;
+
+                                n_key = 0;
+                                value = NULL;
+                                value_alloc = n_value = 0;
+                        } else if (c == '\\') {
+                                state = VALUE_ESCAPE;
+                                last_whitespace = (size_t) -1;
+                        } else {
+                                if (!strchr(WHITESPACE, c))
+                                        last_whitespace = (size_t) -1;
+                                else if (last_whitespace == (size_t) -1)
+                                        last_whitespace = n_value;
+
+                                if (!greedy_realloc((void**) &value, &value_alloc, n_value+2)) {
+                                        r = -ENOMEM;
+                                        goto fail;
+                                }
+
+                                value[n_value++] = c;
+                        }
+
+                        break;
+
+                case VALUE_ESCAPE:
+                        state = VALUE;
+
+                        if (!strchr(newline, c)) {
+                                /* Escaped newlines we eat up entirely */
+                                if (!greedy_realloc((void**) &value, &value_alloc, n_value+2)) {
+                                        r = -ENOMEM;
+                                        goto fail;
+                                }
+
+                                value[n_value++] = c;
+                        }
+                        break;
+
+                case SINGLE_QUOTE_VALUE:
+                        if (c == '\'')
+                                state = PRE_VALUE;
+                        else if (c == '\\')
+                                state = SINGLE_QUOTE_VALUE_ESCAPE;
+                        else {
+                                if (!greedy_realloc((void**) &value, &value_alloc, n_value+2)) {
+                                        r = -ENOMEM;
+                                        goto fail;
+                                }
 
-                                n = strlen(key);
-                                if (!strneq(p, key, n) ||
-                                    p[n] != '=')
-                                        continue;
+                                value[n_value++] = c;
+                        }
 
-                                p += n + 1;
-                                n = strcspn(p, separator);
+                        break;
 
-                                if (n >= 2 &&
-                                    strchr(QUOTES, p[0]) &&
-                                    p[n-1] == p[0])
-                                        v = strndup(p+1, n-2);
-                                else
-                                        v = strndup(p, n);
+                case SINGLE_QUOTE_VALUE_ESCAPE:
+                        state = SINGLE_QUOTE_VALUE;
 
-                                if (!v) {
+                        if (!strchr(newline, c)) {
+                                if (!greedy_realloc((void**) &value, &value_alloc, n_value+2)) {
                                         r = -ENOMEM;
-                                        va_end(ap);
                                         goto fail;
                                 }
 
-                                if (v[0] == '\0') {
-                                        /* return empty value strings as NULL */
-                                        free(v);
-                                        v = NULL;
+                                value[n_value++] = c;
+                        }
+                        break;
+
+                case DOUBLE_QUOTE_VALUE:
+                        if (c == '\"')
+                                state = PRE_VALUE;
+                        else if (c == '\\')
+                                state = DOUBLE_QUOTE_VALUE_ESCAPE;
+                        else {
+                                if (!greedy_realloc((void**) &value, &value_alloc, n_value+2)) {
+                                        r = -ENOMEM;
+                                        goto fail;
                                 }
 
-                                free(*value);
-                                *value = v;
+                                value[n_value++] = c;
+                        }
+
+                        break;
+
+                case DOUBLE_QUOTE_VALUE_ESCAPE:
+                        state = DOUBLE_QUOTE_VALUE;
 
-                                p += n;
+                        if (!strchr(newline, c)) {
+                                if (!greedy_realloc((void**) &value, &value_alloc, n_value+2)) {
+                                        r = -ENOMEM;
+                                        goto fail;
+                                }
 
-                                r ++;
-                                break;
+                                value[n_value++] = c;
                         }
-                        va_end(ap);
+                        break;
+
+                case COMMENT:
+                        if (c == '\\')
+                                state = COMMENT_ESCAPE;
+                        else if (strchr(newline, c))
+                                state = PRE_KEY;
+                        break;
+
+                case COMMENT_ESCAPE:
+                        state = COMMENT;
+                        break;
                 }
+        }
+
+        if (state == PRE_VALUE ||
+            state == VALUE ||
+            state == VALUE_ESCAPE ||
+            state == SINGLE_QUOTE_VALUE ||
+            state == SINGLE_QUOTE_VALUE_ESCAPE ||
+            state == DOUBLE_QUOTE_VALUE ||
+            state == DOUBLE_QUOTE_VALUE_ESCAPE) {
 
-                if (!key)
-                        p += strcspn(p, separator);
+                key[n_key] = 0;
+
+                if (value)
+                        value[n_value] = 0;
+
+                r = push(key, value, userdata);
+                if (r < 0)
+                        goto fail;
         }
 
+        return 0;
+
 fail:
-        free(contents);
+        free(value);
         return r;
 }
 
-int load_env_file(const char *fname, char ***rl) {
+static int parse_env_file_push(const char *key, char *value, void *userdata) {
+        const char *k;
+        va_list* ap = (va_list*) userdata;
+        va_list aq;
 
-        _cleanup_fclose_ FILE *f;
-        _cleanup_strv_free_ char **m = NULL;
-        _cleanup_free_ char *c = NULL;
+        va_copy(aq, *ap);
 
-        assert(fname);
-        assert(rl);
+        while ((k = va_arg(aq, const char *))) {
+                char **v;
 
-        /* This reads an environment file, but will not complain about
-         * any invalid assignments, that needs to be done by the
-         * caller */
+                v = va_arg(aq, char **);
 
-        f = fopen(fname, "re");
-        if (!f)
-                return -errno;
+                if (streq(key, k)) {
+                        va_end(aq);
+                        free(*v);
+                        *v = value;
+                        return 1;
+                }
+        }
 
-        while (!feof(f)) {
-                char l[LINE_MAX], *p, *cs, *b;
+        va_end(aq);
 
-                if (!fgets(l, sizeof(l), f)) {
-                        if (ferror(f))
-                                return -errno;
+        free(value);
+        return 0;
+}
 
-                        /* The previous line was a continuation line?
-                         * Let's process it now, before we leave the
-                         * loop */
-                        if (c)
-                                goto process;
+int parse_env_file(
+                const char *fname,
+                const char *newline, ...) {
 
-                        break;
-                }
+        va_list ap;
+        int r;
 
-                /* Is this a continuation line? If so, just append
-                 * this to c, and go to next line right-away */
-                cs = endswith(l, "\\\n");
-                if (cs) {
-                        *cs = '\0';
-                        b = strappend(c, l);
-                        if (!b)
-                                return -ENOMEM;
-
-                        free(c);
-                        c = b;
-                        continue;
-                }
+        if (!newline)
+                newline = NEWLINE;
 
-                /* If the previous line was a continuation line,
-                 * append the current line to it */
-                if (c) {
-                        b = strappend(c, l);
-                        if (!b)
-                                return -ENOMEM;
+        va_start(ap, newline);
+        r = parse_env_file_internal(fname, newline, parse_env_file_push, &ap);
+        va_end(ap);
 
-                        free(c);
-                        c = b;
-                }
+        return r;
+}
 
-        process:
-                p = strstrip(c ? c : l);
+static int load_env_file_push(const char *key, char *value, void *userdata) {
+        char ***m = userdata;
+        char *p;
+        int r;
 
-                if (*p && !strchr(COMMENTS, *p)) {
-                        _cleanup_free_ char *u;
-                        int k;
+        p = strjoin(key, "=", strempty(value), NULL);
+        if (!p)
+                return -ENOMEM;
 
-                        u = normalize_env_assignment(p);
-                        if (!u)
-                                return -ENOMEM;
+        r = strv_push(m, p);
+        if (r < 0) {
+                free(p);
+                return r;
+        }
 
-                        k = strv_extend(&m, u);
-                        if (k < 0)
-                                return -ENOMEM;
-                }
+        free(value);
+        return 0;
+}
+
+int load_env_file(const char *fname, const char *newline, char ***rl) {
+        char **m = NULL;
+        int r;
 
-                free(c);
-                c = NULL;
+        if (!newline)
+                newline = NEWLINE;
+
+        r = parse_env_file_internal(fname, newline, load_env_file_push, &m);
+        if (r < 0) {
+                strv_free(m);
+                return r;
         }
 
         *rl = m;
-        m = NULL;
-
         return 0;
 }
 
diff --git a/src/shared/fileio.h b/src/shared/fileio.h
index 612968b93d..34ed3c1e97 100644
--- a/src/shared/fileio.h
+++ b/src/shared/fileio.h
@@ -29,5 +29,5 @@ int read_one_line_file(const char *fn, char **line);
 int read_full_file(const char *fn, char **contents, size_t *size);
 
 int parse_env_file(const char *fname, const char *separator, ...) _sentinel_;
-int load_env_file(const char *fname, char ***l);
+int load_env_file(const char *fname, const char *separator, char ***l);
 int write_env_file(const char *fname, char **l);
diff --git a/src/test/test-fileio.c b/src/test/test-fileio.c
new file mode 100644
index 0000000000..55eb7539fd
--- /dev/null
+++ b/src/test/test-fileio.c
@@ -0,0 +1,101 @@
+/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
+
+/***
+  This file is part of systemd.
+
+  Copyright 2013 Lennart Poettering
+
+  systemd is free software; you can redistribute it and/or modify it
+  under the terms of the GNU Lesser General Public License as published by
+  the Free Software Foundation; either version 2.1 of the License, or
+  (at your option) any later version.
+
+  systemd is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public License
+  along with systemd; If not, see <http://www.gnu.org/licenses/>.
+***/
+
+#include <stdio.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "util.h"
+#include "fileio.h"
+#include "strv.h"
+
+static void test_parse_env_file(void) {
+        char t[] = "/tmp/test-parse-env-file-XXXXXX";
+        int fd, r;
+        FILE *f;
+        _cleanup_free_ char *one = NULL, *two = NULL, *three = NULL, *four = NULL, *five = NULL, *six = NULL, *seven = NULL;
+        _cleanup_strv_free_ char **a = NULL;
+        char **i;
+
+        fd = mkostemp(t, O_CLOEXEC);
+        assert_se(fd >= 0);
+
+        f = fdopen(fd, "w");
+        assert_se(f);
+
+        fputs("one=BAR   \n"
+              "# comment\n"
+              " # comment \n"
+              "  two   =   bar    \n"
+              "invalid line\n"
+              "three = \"333\n"
+              "xxxx\"\n"
+              "four = \'44\\\"44\'\n"
+              "five = \'55\\\'55\' \"FIVE\" cinco   \n"
+              "six = seis sechs\\\n"
+              " sis\n"
+              "seven=", f);
+
+        fflush(f);
+        fclose(f);
+
+        r = parse_env_file(
+                        t, NULL,
+                       "one", &one,
+                       "two", &two,
+                       "three", &three,
+                       "four", &four,
+                       "five", &five,
+                       "six", &six,
+                       "seven", &seven,
+                       NULL);
+
+        assert_se(r >= 0);
+
+        log_info("one=[%s]", strna(one));
+        log_info("two=[%s]", strna(two));
+        log_info("three=[%s]", strna(three));
+        log_info("four=[%s]", strna(four));
+        log_info("five=[%s]", strna(five));
+        log_info("six=[%s]", strna(six));
+        log_info("seven=[%s]", strna(seven));
+
+        assert_se(streq(one, "BAR"));
+        assert_se(streq(two, "bar"));
+        assert_se(streq(three, "333\nxxxx"));
+        assert_se(streq(four, "44\"44"));
+        assert_se(streq(five, "55\'55FIVEcinco"));
+        assert_se(streq(six, "seis sechs sis"));
+        assert_se(seven == NULL);
+
+        r = load_env_file(t, NULL, &a);
+        assert_se(r >= 0);
+
+        STRV_FOREACH(i, a)
+                log_info("Got: %s", *i);
+
+        unlink(t);
+}
+
+int main(int argc, char *argv[]) {
+        test_parse_env_file();
+        return 0;
+}
diff --git a/src/test/test-unit-file.c b/src/test/test-unit-file.c
index c1a2d4a7f3..3cf84637e8 100644
--- a/src/test/test-unit-file.c
+++ b/src/test/test-unit-file.c
@@ -180,19 +180,19 @@ static void test_config_parse_exec(void) {
         exec_command_free_list(c);
 }
 
-#define env_file_1 \
-        "a\n"      \
-        "b\\\n"    \
-        "c\n"      \
-        "d\\\n"    \
-        "e\\\n"    \
-        "f\n"      \
-        "g\\ \n"   \
-        "h\n"      \
-        "i\\"
-
-#define env_file_2 \
-        "a\\\n"
+#define env_file_1                              \
+        "a=a\n"                                 \
+        "b=b\\\n"                               \
+        "c\n"                                   \
+        "d=d\\\n"                               \
+        "e\\\n"                                 \
+        "f\n"                                   \
+        "g=g\\ \n"                              \
+        "h=h\n"                                 \
+        "i=i\\"
+
+#define env_file_2                              \
+        "a=a\\\n"
 
 #define env_file_3 \
         "#SPAMD_ARGS=\"-d --socketpath=/var/lib/bulwark/spamd \\\n" \
@@ -208,14 +208,14 @@ static void test_load_env_file_1(void) {
         assert(fd >= 0);
         assert_se(write(fd, env_file_1, sizeof(env_file_1)) == sizeof(env_file_1));
 
-        r = load_env_file(name, &data);
+        r = load_env_file(name, NULL, &data);
         assert(r == 0);
-        assert(streq(data[0], "a"));
-        assert(streq(data[1], "bc"));
-        assert(streq(data[2], "def"));
-        assert(streq(data[3], "g\\"));
-        assert(streq(data[4], "h"));
-        assert(streq(data[5], "i\\"));
+        assert(streq(data[0], "a=a"));
+        assert(streq(data[1], "b=bc"));
+        assert(streq(data[2], "d=def"));
+        assert(streq(data[3], "g=g "));
+        assert(streq(data[4], "h=h"));
+        assert(streq(data[5], "i=i"));
         assert(data[6] == NULL);
         unlink(name);
 }
@@ -229,9 +229,9 @@ static void test_load_env_file_2(void) {
         assert(fd >= 0);
         assert_se(write(fd, env_file_2, sizeof(env_file_2)) == sizeof(env_file_2));
 
-        r = load_env_file(name, &data);
+        r = load_env_file(name, NULL, &data);
         assert(r == 0);
-        assert(streq(data[0], "a"));
+        assert(streq(data[0], "a=a"));
         assert(data[1] == NULL);
         unlink(name);
 }
@@ -245,7 +245,7 @@ static void test_load_env_file_3(void) {
         assert(fd >= 0);
         assert_se(write(fd, env_file_3, sizeof(env_file_3)) == sizeof(env_file_3));
 
-        r = load_env_file(name, &data);
+        r = load_env_file(name, NULL, &data);
         assert(r == 0);
         assert(data == NULL);
         unlink(name);
@@ -272,7 +272,7 @@ static void test_install_printf(void) {
         assert_se((host = gethostname_malloc()));
 
 #define expect(src, pattern, result)                                    \
-        {                                                               \
+        do {                                                            \
                 char _cleanup_free_ *t = install_full_printf(&src, pattern); \
                 char _cleanup_free_                                     \
                         *d1 = strdup(i.name),                           \
@@ -289,7 +289,7 @@ static void test_install_printf(void) {
                 strcpy(i.name, d1);                                     \
                 strcpy(i.path, d2);                                     \
                 strcpy(i.user, d3);                                     \
-        }
+        } while(false)
 
         assert_se(setenv("USER", "root", 1) == 0);