core: unified cgroup hierarchy support

This patch set adds full support the new unified cgroup hierarchy logic
of modern kernels.

A new kernel command line option "systemd.unified_cgroup_hierarchy=1" is
added. If specified the unified hierarchy is mounted to /sys/fs/cgroup
instead of a tmpfs. No further hierarchies are mounted. The kernel
command line option defaults to off. We can turn it on by default as
soon as the kernel's APIs regarding this are stabilized (but even then
downstream distros might want to turn this off, as this will break any
tools that access cgroupfs directly).

It is possibly to choose for each boot individually whether the unified
or the legacy hierarchy is used. nspawn will by default provide the
legacy hierarchy to containers if the host is using it, and the unified
otherwise. However it is possible to run containers with the unified
hierarchy on a legacy host and vice versa, by setting the
$UNIFIED_CGROUP_HIERARCHY environment variable for nspawn to 1 or 0,
respectively.

The unified hierarchy provides reliable cgroup empty notifications for
the first time, via inotify. To make use of this we maintain one
manager-wide inotify fd, and each cgroup to it.

This patch also removes cg_delete() which is unused now.

On kernel 4.2 only the "memory" controller is compatible with the
unified hierarchy, hence that's the only controller systemd exposes when
booted in unified heirarchy mode.

This introduces a new enum for enumerating supported controllers, plus a
related enum for the mask bits mapping to it. The core is changed to
make use of this everywhere.

This moves PID 1 into a new "init.scope" implicit scope unit in the root
slice. This is necessary since on the unified hierarchy cgroups may
either contain subgroups or processes but not both. PID 1 hence has to
move out of the root cgroup (strictly speaking the root cgroup is the
only one where processes and subgroups are still allowed, but in order
to support containers nicey, we move PID 1 into the new scope in all
cases.) This new unit is also used on legacy hierarchy setups. It's
actually pretty useful on all systems, as it can then be used to filter
journal messages coming from PID 1, and so on.

The root slice ("-.slice") is now implicitly created and started (and
does not require a unit file on disk anymore), since
that's where "init.scope" is located and the slice needs to be started
before the scope can.

To check whether we are in unified or legacy hierarchy mode we use
statfs() on /sys/fs/cgroup. If the .f_type field reports tmpfs we are in
legacy mode, if it reports cgroupfs we are in unified mode.

This patch set carefuly makes sure that cgls and cgtop continue to work
as desired.

When invoking nspawn as a service it will implicitly create two
subcgroups in the cgroup it is using, one to move the nspawn process
into, the other to move the actual container processes into. This is
done because of the requirement that cgroups may either contain
processes or other subgroups.

1 files changed, 479 insertions, 107 deletions

diff --git a/src/core/cgroup.c b/src/core/cgroup.c
index e92d2cc850..a70b4d33ae 100644
--- a/src/core/cgroup.c
+++ b/src/core/cgroup.c
@@ -283,7 +283,7 @@ fail:
         return -errno;
 }
 
-void cgroup_context_apply(CGroupContext *c, CGroupControllerMask mask, const char *path, ManagerState state) {
+void cgroup_context_apply(CGroupContext *c, CGroupMask mask, const char *path, ManagerState state) {
         bool is_root;
         int r;
 
@@ -304,7 +304,7 @@ void cgroup_context_apply(CGroupContext *c, CGroupControllerMask mask, const cha
          * cgroup trees (assuming we are running in a container then),
          * and missing cgroups, i.e. EROFS and ENOENT. */
 
-        if ((mask & CGROUP_CPU) && !is_root) {
+        if ((mask & CGROUP_MASK_CPU) && !is_root) {
                 char buf[MAX(DECIMAL_STR_MAX(unsigned long), DECIMAL_STR_MAX(usec_t)) + 1];
 
                 sprintf(buf, "%lu\n",
@@ -331,7 +331,7 @@ void cgroup_context_apply(CGroupContext *c, CGroupControllerMask mask, const cha
                                        "Failed to set cpu.cfs_quota_us on %s: %m", path);
         }
 
-        if (mask & CGROUP_BLKIO) {
+        if (mask & CGROUP_MASK_BLKIO) {
                 char buf[MAX3(DECIMAL_STR_MAX(unsigned long)+1,
                               DECIMAL_STR_MAX(dev_t)*2+2+DECIMAL_STR_MAX(unsigned long)*1,
                               DECIMAL_STR_MAX(dev_t)*2+2+DECIMAL_STR_MAX(uint64_t)+1)];
@@ -381,21 +381,30 @@ void cgroup_context_apply(CGroupContext *c, CGroupControllerMask mask, const cha
                 }
         }
 
-        if ((mask & CGROUP_MEMORY) && !is_root) {
+        if ((mask & CGROUP_MASK_MEMORY) && !is_root) {
                 if (c->memory_limit != (uint64_t) -1) {
                         char buf[DECIMAL_STR_MAX(uint64_t) + 1];
 
                         sprintf(buf, "%" PRIu64 "\n", c->memory_limit);
-                        r = cg_set_attribute("memory", path, "memory.limit_in_bytes", buf);
-                } else
-                        r = cg_set_attribute("memory", path, "memory.limit_in_bytes", "-1");
+
+                        if (cg_unified() <= 0)
+                                r = cg_set_attribute("memory", path, "memory.limit_in_bytes", buf);
+                        else
+                                r = cg_set_attribute("memory", path, "memory.max", buf);
+
+                } else {
+                        if (cg_unified() <= 0)
+                                r = cg_set_attribute("memory", path, "memory.limit_in_bytes", "-1");
+                        else
+                                r = cg_set_attribute("memory", path, "memory.max", "max");
+                }
 
                 if (r < 0)
                         log_full_errno(IN_SET(r, -ENOENT, -EROFS) ? LOG_DEBUG : LOG_WARNING, r,
-                                       "Failed to set memory.limit_in_bytes on %s: %m", path);
+                                       "Failed to set memory.limit_in_bytes/memory.max on %s: %m", path);
         }
 
-        if ((mask & CGROUP_DEVICE) && !is_root) {
+        if ((mask & CGROUP_MASK_DEVICE) && !is_root) {
                 CGroupDeviceAllow *a;
 
                 /* Changing the devices list of a populated cgroup
@@ -459,8 +468,8 @@ void cgroup_context_apply(CGroupContext *c, CGroupControllerMask mask, const cha
         }
 }
 
-CGroupControllerMask cgroup_context_get_mask(CGroupContext *c) {
-        CGroupControllerMask mask = 0;
+CGroupMask cgroup_context_get_mask(CGroupContext *c) {
+        CGroupMask mask = 0;
 
         /* Figure out which controllers we need */
 
@@ -468,29 +477,31 @@ CGroupControllerMask cgroup_context_get_mask(CGroupContext *c) {
             c->cpu_shares != (unsigned long) -1 ||
             c->startup_cpu_shares != (unsigned long) -1 ||
             c->cpu_quota_per_sec_usec != USEC_INFINITY)
-                mask |= CGROUP_CPUACCT | CGROUP_CPU;
+                mask |= CGROUP_MASK_CPUACCT | CGROUP_MASK_CPU;
 
         if (c->blockio_accounting ||
             c->blockio_weight != (unsigned long) -1 ||
             c->startup_blockio_weight != (unsigned long) -1 ||
             c->blockio_device_weights ||
             c->blockio_device_bandwidths)
-                mask |= CGROUP_BLKIO;
+                mask |= CGROUP_MASK_BLKIO;
 
         if (c->memory_accounting ||
             c->memory_limit != (uint64_t) -1)
-                mask |= CGROUP_MEMORY;
+                mask |= CGROUP_MASK_MEMORY;
 
         if (c->device_allow ||
             c->device_policy != CGROUP_AUTO)
-                mask |= CGROUP_DEVICE;
+                mask |= CGROUP_MASK_DEVICE;
 
         return mask;
 }
 
-CGroupControllerMask unit_get_cgroup_mask(Unit *u) {
+CGroupMask unit_get_own_mask(Unit *u) {
         CGroupContext *c;
 
+        /* Returns the mask of controllers the unit needs for itself */
+
         c = unit_get_cgroup_context(u);
         if (!c)
                 return 0;
@@ -505,15 +516,18 @@ CGroupControllerMask unit_get_cgroup_mask(Unit *u) {
 
                 e = unit_get_exec_context(u);
                 if (!e || exec_context_maintains_privileges(e))
-                        return _CGROUP_CONTROLLER_MASK_ALL;
+                        return _CGROUP_MASK_ALL;
         }
 
         return cgroup_context_get_mask(c);
 }
 
-CGroupControllerMask unit_get_members_mask(Unit *u) {
+CGroupMask unit_get_members_mask(Unit *u) {
         assert(u);
 
+        /* Returns the mask of controllers all of the unit's children
+         * require, merged */
+
         if (u->cgroup_members_mask_valid)
                 return u->cgroup_members_mask;
 
@@ -532,7 +546,7 @@ CGroupControllerMask unit_get_members_mask(Unit *u) {
                                 continue;
 
                         u->cgroup_members_mask |=
-                                unit_get_cgroup_mask(member) |
+                                unit_get_own_mask(member) |
                                 unit_get_members_mask(member);
                 }
         }
@@ -541,19 +555,52 @@ CGroupControllerMask unit_get_members_mask(Unit *u) {
         return u->cgroup_members_mask;
 }
 
-CGroupControllerMask unit_get_siblings_mask(Unit *u) {
+CGroupMask unit_get_siblings_mask(Unit *u) {
         assert(u);
 
+        /* Returns the mask of controllers all of the unit's siblings
+         * require, i.e. the members mask of the unit's parent slice
+         * if there is one. */
+
         if (UNIT_ISSET(u->slice))
                 return unit_get_members_mask(UNIT_DEREF(u->slice));
 
-        return unit_get_cgroup_mask(u) | unit_get_members_mask(u);
+        return unit_get_own_mask(u) | unit_get_members_mask(u);
 }
 
-CGroupControllerMask unit_get_target_mask(Unit *u) {
-        CGroupControllerMask mask;
+CGroupMask unit_get_subtree_mask(Unit *u) {
+
+        /* Returns the mask of this subtree, meaning of the group
+         * itself and its children. */
+
+        return unit_get_own_mask(u) | unit_get_members_mask(u);
+}
+
+CGroupMask unit_get_target_mask(Unit *u) {
+        CGroupMask mask;
+
+        /* This returns the cgroup mask of all controllers to enable
+         * for a specific cgroup, i.e. everything it needs itself,
+         * plus all that its children need, plus all that its siblings
+         * need. This is primarily useful on the legacy cgroup
+         * hierarchy, where we need to duplicate each cgroup in each
+         * hierarchy that shall be enabled for it. */
 
-        mask = unit_get_cgroup_mask(u) | unit_get_members_mask(u) | unit_get_siblings_mask(u);
+        mask = unit_get_own_mask(u) | unit_get_members_mask(u) | unit_get_siblings_mask(u);
+        mask &= u->manager->cgroup_supported;
+
+        return mask;
+}
+
+CGroupMask unit_get_enable_mask(Unit *u) {
+        CGroupMask mask;
+
+        /* This returns the cgroup mask of all controllers to enable
+         * for the children of a specific cgroup. This is primarily
+         * useful for the unified cgroup hierarchy, where each cgroup
+         * controls which controllers are enabled for its children. */
+
+        mask = unit_get_members_mask(u);
         mask &= u->manager->cgroup_supported;
 
         return mask;
@@ -562,13 +609,13 @@ CGroupControllerMask unit_get_target_mask(Unit *u) {
 /* Recurse from a unit up through its containing slices, propagating
  * mask bits upward. A unit is also member of itself. */
 void unit_update_cgroup_members_masks(Unit *u) {
-        CGroupControllerMask m;
+        CGroupMask m;
         bool more;
 
         assert(u);
 
         /* Calculate subtree mask */
-        m = unit_get_cgroup_mask(u) | unit_get_members_mask(u);
+        m = unit_get_subtree_mask(u);
 
         /* See if anything changed from the previous invocation. If
          * not, we're done. */
@@ -608,7 +655,7 @@ void unit_update_cgroup_members_masks(Unit *u) {
         }
 }
 
-static const char *migrate_callback(CGroupControllerMask mask, void *userdata) {
+static const char *migrate_callback(CGroupMask mask, void *userdata) {
         Unit *u = userdata;
 
         assert(mask != 0);
@@ -626,7 +673,115 @@ static const char *migrate_callback(CGroupControllerMask mask, void *userdata) {
         return NULL;
 }
 
-static int unit_create_cgroups(Unit *u, CGroupControllerMask mask) {
+char *unit_default_cgroup_path(Unit *u) {
+        _cleanup_free_ char *escaped = NULL, *slice = NULL;
+        int r;
+
+        assert(u);
+
+        if (unit_has_name(u, SPECIAL_ROOT_SLICE))
+                return strdup(u->manager->cgroup_root);
+
+        if (UNIT_ISSET(u->slice) && !unit_has_name(UNIT_DEREF(u->slice), SPECIAL_ROOT_SLICE)) {
+                r = cg_slice_to_path(UNIT_DEREF(u->slice)->id, &slice);
+                if (r < 0)
+                        return NULL;
+        }
+
+        escaped = cg_escape(u->id);
+        if (!escaped)
+                return NULL;
+
+        if (slice)
+                return strjoin(u->manager->cgroup_root, "/", slice, "/", escaped, NULL);
+        else
+                return strjoin(u->manager->cgroup_root, "/", escaped, NULL);
+}
+
+int unit_set_cgroup_path(Unit *u, const char *path) {
+        _cleanup_free_ char *p = NULL;
+        int r;
+
+        assert(u);
+
+        if (path) {
+                p = strdup(path);
+                if (!p)
+                        return -ENOMEM;
+        } else
+                p = NULL;
+
+        if (streq_ptr(u->cgroup_path, p))
+                return 0;
+
+        if (p) {
+                r = hashmap_put(u->manager->cgroup_unit, p, u);
+                if (r < 0)
+                        return r;
+        }
+
+        unit_release_cgroup(u);
+
+        u->cgroup_path = p;
+        p = NULL;
+
+        return 1;
+}
+
+int unit_watch_cgroup(Unit *u) {
+        _cleanup_free_ char *populated = NULL;
+        int r;
+
+        assert(u);
+
+        if (!u->cgroup_path)
+                return 0;
+
+        if (u->cgroup_inotify_wd >= 0)
+                return 0;
+
+        /* Only applies to the unified hierarchy */
+        r = cg_unified();
+        if (r < 0)
+                return log_unit_error_errno(u, r, "Failed detect wether the unified hierarchy is used: %m");
+        if (r == 0)
+                return 0;
+
+        /* Don't watch the root slice, it's pointless. */
+        if (unit_has_name(u, SPECIAL_ROOT_SLICE))
+                return 0;
+
+        r = hashmap_ensure_allocated(&u->manager->cgroup_inotify_wd_unit, &trivial_hash_ops);
+        if (r < 0)
+                return log_oom();
+
+        r = cg_get_path(SYSTEMD_CGROUP_CONTROLLER, u->cgroup_path, "cgroup.populated", &populated);
+        if (r < 0)
+                return log_oom();
+
+        u->cgroup_inotify_wd = inotify_add_watch(u->manager->cgroup_inotify_fd, populated, IN_MODIFY);
+        if (u->cgroup_inotify_wd < 0) {
+
+                if (errno == ENOENT) /* If the directory is already
+                                      * gone we don't need to track
+                                      * it, so this is not an error */
+                        return 0;
+
+                return log_unit_error_errno(u, errno, "Failed to add inotify watch descriptor for control group %s: %m", u->cgroup_path);
+        }
+
+        r = hashmap_put(u->manager->cgroup_inotify_wd_unit, INT_TO_PTR(u->cgroup_inotify_wd), u);
+        if (r < 0)
+                return log_unit_error_errno(u, r, "Failed to add inotify watch descriptor to hash map: %m");
+
+        return 0;
+}
+
+static int unit_create_cgroup(
+                Unit *u,
+                CGroupMask target_mask,
+                CGroupMask enable_mask) {
+
         CGroupContext *c;
         int r;
 
@@ -643,25 +798,29 @@ static int unit_create_cgroups(Unit *u, CGroupControllerMask mask) {
                 if (!path)
                         return log_oom();
 
-                r = hashmap_put(u->manager->cgroup_unit, path, u);
-                if (r < 0) {
-                        log_error(r == -EEXIST ? "cgroup %s exists already: %s" : "hashmap_put failed for %s: %s", path, strerror(-r));
-                        return r;
-                }
-                if (r > 0) {
-                        u->cgroup_path = path;
-                        path = NULL;
-                }
+                r = unit_set_cgroup_path(u, path);
+                if (r == -EEXIST)
+                        return log_unit_error_errno(u, r, "Control group %s exists already.", path);
+                if (r < 0)
+                        return log_unit_error_errno(u, r, "Failed to set unit's control group path to %s: %m", path);
         }
 
         /* First, create our own group */
-        r = cg_create_everywhere(u->manager->cgroup_supported, mask, u->cgroup_path);
+        r = cg_create_everywhere(u->manager->cgroup_supported, target_mask, u->cgroup_path);
+        if (r < 0)
+                return log_unit_error_errno(u, r, "Failed to create cgroup %s: %m", u->cgroup_path);
+
+        /* Start watching it */
+        (void) unit_watch_cgroup(u);
+
+        /* Enable all controllers we need */
+        r = cg_enable_everywhere(u->manager->cgroup_supported, enable_mask, u->cgroup_path);
         if (r < 0)
-                return log_error_errno(r, "Failed to create cgroup %s: %m", u->cgroup_path);
+                log_unit_warning_errno(u, r, "Failed to enable controllers on cgroup %s, ignoring: %m", u->cgroup_path);
 
         /* Keep track that this is now realized */
         u->cgroup_realized = true;
-        u->cgroup_realized_mask = mask;
+        u->cgroup_realized_mask = target_mask;
 
         if (u->type != UNIT_SLICE && !c->delegate) {
 
@@ -670,7 +829,7 @@ static int unit_create_cgroups(Unit *u, CGroupControllerMask mask) {
                  * for slice and delegation units. */
                 r = cg_migrate_everywhere(u->manager->cgroup_supported, u->cgroup_path, u->cgroup_path, migrate_callback, u);
                 if (r < 0)
-                        log_warning_errno(r, "Failed to migrate cgroup from to %s: %m", u->cgroup_path);
+                        log_unit_warning_errno(u, r, "Failed to migrate cgroup from to %s, ignoring: %m", u->cgroup_path);
         }
 
         return 0;
@@ -691,10 +850,10 @@ int unit_attach_pids_to_cgroup(Unit *u) {
         return 0;
 }
 
-static bool unit_has_mask_realized(Unit *u, CGroupControllerMask mask) {
+static bool unit_has_mask_realized(Unit *u, CGroupMask target_mask) {
         assert(u);
 
-        return u->cgroup_realized && u->cgroup_realized_mask == mask;
+        return u->cgroup_realized && u->cgroup_realized_mask == target_mask;
 }
 
 /* Check if necessary controllers and attributes for a unit are in place.
@@ -704,7 +863,7 @@ static bool unit_has_mask_realized(Unit *u, CGroupControllerMask mask) {
  *
  * Returns 0 on success and < 0 on failure. */
 static int unit_realize_cgroup_now(Unit *u, ManagerState state) {
-        CGroupControllerMask mask;
+        CGroupMask target_mask, enable_mask;
         int r;
 
         assert(u);
@@ -714,9 +873,8 @@ static int unit_realize_cgroup_now(Unit *u, ManagerState state) {
                 u->in_cgroup_queue = false;
         }
 
-        mask = unit_get_target_mask(u);
-
-        if (unit_has_mask_realized(u, mask))
+        target_mask = unit_get_target_mask(u);
+        if (unit_has_mask_realized(u, target_mask))
                 return 0;
 
         /* First, realize parents */
@@ -727,12 +885,13 @@ static int unit_realize_cgroup_now(Unit *u, ManagerState state) {
         }
 
         /* And then do the real work */
-        r = unit_create_cgroups(u, mask);
+        enable_mask = unit_get_enable_mask(u);
+        r = unit_create_cgroup(u, target_mask, enable_mask);
         if (r < 0)
                 return r;
 
         /* Finally, apply the necessary attributes. */
-        cgroup_context_apply(unit_get_cgroup_context(u), mask, u->cgroup_path, state);
+        cgroup_context_apply(unit_get_cgroup_context(u), target_mask, u->cgroup_path, state);
 
         return 0;
 }
@@ -759,7 +918,7 @@ unsigned manager_dispatch_cgroup_queue(Manager *m) {
 
                 r = unit_realize_cgroup_now(i, state);
                 if (r < 0)
-                        log_warning_errno(r, "Failed to realize cgroups for queued unit %s: %m", i->id);
+                        log_warning_errno(r, "Failed to realize cgroups for queued unit %s, ignoring: %m", i->id);
 
                 n++;
         }
@@ -829,39 +988,67 @@ int unit_realize_cgroup(Unit *u) {
         return unit_realize_cgroup_now(u, manager_state(u->manager));
 }
 
-void unit_destroy_cgroup_if_empty(Unit *u) {
+void unit_release_cgroup(Unit *u) {
+        assert(u);
+
+        /* Forgets all cgroup details for this cgroup */
+
+        if (u->cgroup_path) {
+                (void) hashmap_remove(u->manager->cgroup_unit, u->cgroup_path);
+                u->cgroup_path = mfree(u->cgroup_path);
+        }
+
+        if (u->cgroup_inotify_wd >= 0) {
+                if (inotify_rm_watch(u->manager->cgroup_inotify_fd, u->cgroup_inotify_wd) < 0)
+                        log_unit_debug_errno(u, errno, "Failed to remove cgroup inotify watch %i for %s, ignoring", u->cgroup_inotify_wd, u->id);
+
+                (void) hashmap_remove(u->manager->cgroup_inotify_wd_unit, INT_TO_PTR(u->cgroup_inotify_wd));
+                u->cgroup_inotify_wd = -1;
+        }
+}
+
+void unit_prune_cgroup(Unit *u) {
         int r;
+        bool is_root_slice;
 
         assert(u);
 
+        /* Removes the cgroup, if empty and possible, and stops watching it. */
+
         if (!u->cgroup_path)
                 return;
 
-        r = cg_trim_everywhere(u->manager->cgroup_supported, u->cgroup_path, !unit_has_name(u, SPECIAL_ROOT_SLICE));
+        is_root_slice = unit_has_name(u, SPECIAL_ROOT_SLICE);
+
+        r = cg_trim_everywhere(u->manager->cgroup_supported, u->cgroup_path, !is_root_slice);
         if (r < 0) {
-                log_debug_errno(r, "Failed to destroy cgroup %s: %m", u->cgroup_path);
+                log_debug_errno(r, "Failed to destroy cgroup %s, ignoring: %m", u->cgroup_path);
                 return;
         }
 
-        hashmap_remove(u->manager->cgroup_unit, u->cgroup_path);
+        if (is_root_slice)
+                return;
+
+        unit_release_cgroup(u);
 
-        free(u->cgroup_path);
-        u->cgroup_path = NULL;
         u->cgroup_realized = false;
         u->cgroup_realized_mask = 0;
 }
 
-pid_t unit_search_main_pid(Unit *u) {
+int unit_search_main_pid(Unit *u, pid_t *ret) {
         _cleanup_fclose_ FILE *f = NULL;
         pid_t pid = 0, npid, mypid;
+        int r;
 
         assert(u);
+        assert(ret);
 
         if (!u->cgroup_path)
-                return 0;
+                return -ENXIO;
 
-        if (cg_enumerate_processes(SYSTEMD_CGROUP_CONTROLLER, u->cgroup_path, &f) < 0)
-                return 0;
+        r = cg_enumerate_processes(SYSTEMD_CGROUP_CONTROLLER, u->cgroup_path, &f);
+        if (r < 0)
+                return r;
 
         mypid = getpid();
         while (cg_read_pid(f, &npid) > 0)  {
@@ -874,90 +1061,274 @@ pid_t unit_search_main_pid(Unit *u) {
                 if (get_parent_of_pid(npid, &ppid) >= 0 && ppid != mypid)
                         continue;
 
-                if (pid != 0) {
+                if (pid != 0)
                         /* Dang, there's more than one daemonized PID
                         in this group, so we don't know what process
                         is the main process. */
-                        pid = 0;
-                        break;
-                }
+
+                        return -ENODATA;
 
                 pid = npid;
         }
 
-        return pid;
+        *ret = pid;
+        return 0;
+}
+
+static int unit_watch_pids_in_path(Unit *u, const char *path) {
+       _cleanup_closedir_ DIR *d = NULL;
+        _cleanup_fclose_ FILE *f = NULL;
+        int ret = 0, r;
+
+        assert(u);
+        assert(path);
+
+        r = cg_enumerate_processes(SYSTEMD_CGROUP_CONTROLLER, path, &f);
+        if (r < 0)
+                ret = r;
+        else {
+                pid_t pid;
+
+                while ((r = cg_read_pid(f, &pid)) > 0) {
+                        r = unit_watch_pid(u, pid);
+                        if (r < 0 && ret >= 0)
+                                ret = r;
+                }
+
+                if (r < 0 && ret >= 0)
+                        ret = r;
+        }
+
+        r = cg_enumerate_subgroups(SYSTEMD_CGROUP_CONTROLLER, path, &d);
+        if (r < 0) {
+                if (ret >= 0)
+                        ret = r;
+        } else {
+                char *fn;
+
+                while ((r = cg_read_subgroup(d, &fn)) > 0) {
+                        _cleanup_free_ char *p = NULL;
+
+                        p = strjoin(path, "/", fn, NULL);
+                        free(fn);
+
+                        if (!p)
+                                return -ENOMEM;
+
+                        r = unit_watch_pids_in_path(u, p);
+                        if (r < 0 && ret >= 0)
+                                ret = r;
+                }
+
+                if (r < 0 && ret >= 0)
+                        ret = r;
+        }
+
+        return ret;
+}
+
+int unit_watch_all_pids(Unit *u) {
+        assert(u);
+
+        /* Adds all PIDs from our cgroup to the set of PIDs we
+         * watch. This is a fallback logic for cases where we do not
+         * get reliable cgroup empty notifications: we try to use
+         * SIGCHLD as replacement. */
+
+        if (!u->cgroup_path)
+                return -ENOENT;
+
+        if (cg_unified() > 0) /* On unified we can use proper notifications */
+                return 0;
+
+        return unit_watch_pids_in_path(u, u->cgroup_path);
+}
+
+int unit_notify_cgroup_empty(Unit *u) {
+        int r;
+
+        assert(u);
+
+        if (!u->cgroup_path)
+                return 0;
+
+        r = cg_is_empty_recursive(SYSTEMD_CGROUP_CONTROLLER, u->cgroup_path);
+        if (r <= 0)
+                return r;
+
+        unit_add_to_gc_queue(u);
+
+        if (UNIT_VTABLE(u)->notify_cgroup_empty)
+                UNIT_VTABLE(u)->notify_cgroup_empty(u);
+
+        return 0;
+}
+
+static int on_cgroup_inotify_event(sd_event_source *s, int fd, uint32_t revents, void *userdata) {
+        Manager *m = userdata;
+
+        assert(s);
+        assert(fd >= 0);
+        assert(m);
+
+        for (;;) {
+                union inotify_event_buffer buffer;
+                struct inotify_event *e;
+                ssize_t l;
+
+                l = read(fd, &buffer, sizeof(buffer));
+                if (l < 0) {
+                        if (errno == EINTR || errno == EAGAIN)
+                                return 0;
+
+                        return log_error_errno(errno, "Failed to read control group inotify events: %m");
+                }
+
+                FOREACH_INOTIFY_EVENT(e, buffer, l) {
+                        Unit *u;
+
+                        if (e->wd < 0)
+                                /* Queue overflow has no watch descriptor */
+                                continue;
+
+                        if (e->mask & IN_IGNORED)
+                                /* The watch was just removed */
+                                continue;
+
+                        u = hashmap_get(m->cgroup_inotify_wd_unit, INT_TO_PTR(e->wd));
+                        if (!u) /* Not that inotify might deliver
+                                 * events for a watch even after it
+                                 * was removed, because it was queued
+                                 * before the removal. Let's ignore
+                                 * this here safely. */
+                                continue;
+
+                        (void) unit_notify_cgroup_empty(u);
+                }
+        }
 }
 
 int manager_setup_cgroup(Manager *m) {
         _cleanup_free_ char *path = NULL;
-        int r;
+        CGroupController c;
+        int r, unified;
+        char *e;
 
         assert(m);
 
         /* 1. Determine hierarchy */
-        free(m->cgroup_root);
-        m->cgroup_root = NULL;
-
+        m->cgroup_root = mfree(m->cgroup_root);
         r = cg_pid_get_path(SYSTEMD_CGROUP_CONTROLLER, 0, &m->cgroup_root);
         if (r < 0)
                 return log_error_errno(r, "Cannot determine cgroup we are running in: %m");
 
-        /* LEGACY: Already in /system.slice? If so, let's cut this
-         * off. This is to support live upgrades from older systemd
-         * versions where PID 1 was moved there. */
-        if (m->running_as == MANAGER_SYSTEM) {
-                char *e;
+        /* Chop off the init scope, if we are already located in it */
+        e = endswith(m->cgroup_root, "/" SPECIAL_INIT_SCOPE);
 
+        /* LEGACY: Also chop off the system slice if we are in
+         * it. This is to support live upgrades from older systemd
+         * versions where PID 1 was moved there. Also see
+         * cg_get_root_path(). */
+        if (!e && m->running_as == MANAGER_SYSTEM) {
                 e = endswith(m->cgroup_root, "/" SPECIAL_SYSTEM_SLICE);
                 if (!e)
-                        e = endswith(m->cgroup_root, "/system");
-                if (e)
-                        *e = 0;
+                        e = endswith(m->cgroup_root, "/system"); /* even more legacy */
         }
+        if (e)
+                *e = 0;
 
         /* And make sure to store away the root value without trailing
          * slash, even for the root dir, so that we can easily prepend
          * it everywhere. */
-        if (streq(m->cgroup_root, "/"))
-                m->cgroup_root[0] = 0;
+        while ((e = endswith(m->cgroup_root, "/")))
+                *e = 0;
 
         /* 2. Show data */
         r = cg_get_path(SYSTEMD_CGROUP_CONTROLLER, m->cgroup_root, NULL, &path);
         if (r < 0)
                 return log_error_errno(r, "Cannot find cgroup mount point: %m");
 
-        log_debug("Using cgroup controller " SYSTEMD_CGROUP_CONTROLLER ". File system hierarchy is at %s.", path);
+        unified = cg_unified();
+        if (unified < 0)
+                return log_error_errno(r, "Couldn't determine if we are running in the unified hierarchy: %m");
+        if (unified > 0)
+                log_debug("Unified cgroup hierarchy is located at %s.", path);
+        else
+                log_debug("Using cgroup controller " SYSTEMD_CGROUP_CONTROLLER ". File system hierarchy is at %s.", path);
+
         if (!m->test_run) {
+                const char *scope_path;
 
                 /* 3. Install agent */
-                if (m->running_as == MANAGER_SYSTEM) {
+                if (unified) {
+
+                        /* In the unified hierarchy we can can get
+                         * cgroup empty notifications via inotify. */
+
+                        m->cgroup_inotify_event_source = sd_event_source_unref(m->cgroup_inotify_event_source);
+                        safe_close(m->cgroup_inotify_fd);
+
+                        m->cgroup_inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
+                        if (m->cgroup_inotify_fd < 0)
+                                return log_error_errno(errno, "Failed to create control group inotify object: %m");
+
+                        r = sd_event_add_io(m->event, &m->cgroup_inotify_event_source, m->cgroup_inotify_fd, EPOLLIN, on_cgroup_inotify_event, m);
+                        if (r < 0)
+                                return log_error_errno(r, "Failed to watch control group inotify object: %m");
+
+                        r = sd_event_source_set_priority(m->cgroup_inotify_event_source, SD_EVENT_PRIORITY_IDLE - 5);
+                        if (r < 0)
+                                return log_error_errno(r, "Failed to set priority of inotify event source: %m");
+
+                        (void) sd_event_source_set_description(m->cgroup_inotify_event_source, "cgroup-inotify");
+
+                } else if (m->running_as == MANAGER_SYSTEM) {
+
+                        /* On the legacy hierarchy we only get
+                         * notifications via cgroup agents. (Which
+                         * isn't really reliable, since it does not
+                         * generate events when control groups with
+                         * children run empty. */
+
                         r = cg_install_release_agent(SYSTEMD_CGROUP_CONTROLLER, SYSTEMD_CGROUP_AGENT_PATH);
                         if (r < 0)
                                 log_warning_errno(r, "Failed to install release agent, ignoring: %m");
                         else if (r > 0)
                                 log_debug("Installed release agent.");
-                        else
+                        else if (r == 0)
                                 log_debug("Release agent already installed.");
                 }
 
-                /* 4. Make sure we are in the root cgroup */
-                r = cg_create_and_attach(SYSTEMD_CGROUP_CONTROLLER, m->cgroup_root, 0);
+                /* 4. Make sure we are in the special "init.scope" unit in the root slice. */
+                scope_path = strjoina(m->cgroup_root, "/" SPECIAL_INIT_SCOPE);
+                r = cg_create_and_attach(SYSTEMD_CGROUP_CONTROLLER, scope_path, 0);
                 if (r < 0)
-                        return log_error_errno(r, "Failed to create root cgroup hierarchy: %m");
+                        return log_error_errno(r, "Failed to create %s control group: %m", scope_path);
+
+                /* also, move all other userspace processes remaining
+                 * in the root cgroup into that scope. */
+                r = cg_migrate(SYSTEMD_CGROUP_CONTROLLER, m->cgroup_root, SYSTEMD_CGROUP_CONTROLLER, scope_path, false);
+                if (r < 0)
+                        log_warning_errno(r, "Couldn't move remaining userspace processes, ignoring: %m");
 
                 /* 5. And pin it, so that it cannot be unmounted */
                 safe_close(m->pin_cgroupfs_fd);
-
                 m->pin_cgroupfs_fd = open(path, O_RDONLY|O_CLOEXEC|O_DIRECTORY|O_NOCTTY|O_NONBLOCK);
                 if (m->pin_cgroupfs_fd < 0)
                         return log_error_errno(errno, "Failed to open pin file: %m");
 
                 /* 6.  Always enable hierarchical support if it exists... */
-                cg_set_attribute("memory", "/", "memory.use_hierarchy", "1");
+                if (!unified)
+                        (void) cg_set_attribute("memory", "/", "memory.use_hierarchy", "1");
         }
 
         /* 7. Figure out which controllers are supported */
-        m->cgroup_supported = cg_mask_supported();
+        r = cg_mask_supported(&m->cgroup_supported);
+        if (r < 0)
+                return log_error_errno(r, "Failed to determine supported controllers: %m");
+
+        for (c = 0; c < _CGROUP_CONTROLLER_MAX; c++)
+                log_debug("Controller '%s' supported: %s", cgroup_controller_to_string(c), yes_no(m->cgroup_supported & c));
 
         return 0;
 }
@@ -968,12 +1339,16 @@ void manager_shutdown_cgroup(Manager *m, bool delete) {
         /* We can't really delete the group, since we are in it. But
          * let's trim it. */
         if (delete && m->cgroup_root)
-                cg_trim(SYSTEMD_CGROUP_CONTROLLER, m->cgroup_root, false);
+                (void) cg_trim(SYSTEMD_CGROUP_CONTROLLER, m->cgroup_root, false);
+
+        m->cgroup_inotify_wd_unit = hashmap_free(m->cgroup_inotify_wd_unit);
+
+        m->cgroup_inotify_event_source = sd_event_source_unref(m->cgroup_inotify_event_source);
+        m->cgroup_inotify_fd = safe_close(m->cgroup_inotify_fd);
 
         m->pin_cgroupfs_fd = safe_close(m->pin_cgroupfs_fd);
 
-        free(m->cgroup_root);
-        m->cgroup_root = NULL;
+        m->cgroup_root = mfree(m->cgroup_root);
 }
 
 Unit* manager_get_unit_by_cgroup(Manager *m, const char *cgroup) {
@@ -992,8 +1367,8 @@ Unit* manager_get_unit_by_cgroup(Manager *m, const char *cgroup) {
                 char *e;
 
                 e = strrchr(p, '/');
-                if (e == p || !e)
-                        return NULL;
+                if (!e || e == p)
+                        return hashmap_get(m->cgroup_unit, SPECIAL_ROOT_SLICE);
 
                 *e = 0;
 
@@ -1010,9 +1385,12 @@ Unit *manager_get_unit_by_pid(Manager *m, pid_t pid) {
 
         assert(m);
 
-        if (pid <= 1)
+        if (pid <= 0)
                 return NULL;
 
+        if (pid == 1)
+                return hashmap_get(m->units, SPECIAL_INIT_SCOPE);
+
         u = hashmap_get(m->watch_pids1, LONG_TO_PTR(pid));
         if (u)
                 return u;
@@ -1030,7 +1408,6 @@ Unit *manager_get_unit_by_pid(Manager *m, pid_t pid) {
 
 int manager_notify_cgroup_empty(Manager *m, const char *cgroup) {
         Unit *u;
-        int r;
 
         assert(m);
         assert(cgroup);
@@ -1039,15 +1416,7 @@ int manager_notify_cgroup_empty(Manager *m, const char *cgroup) {
         if (!u)
                 return 0;
 
-        r = cg_is_empty_recursive(SYSTEMD_CGROUP_CONTROLLER, u->cgroup_path);
-        if (r <= 0)
-                return r;
-
-        if (UNIT_VTABLE(u)->notify_cgroup_empty)
-                UNIT_VTABLE(u)->notify_cgroup_empty(u);
-
-        unit_add_to_gc_queue(u);
-        return 0;
+        return unit_notify_cgroup_empty(u);
 }
 
 int unit_get_memory_current(Unit *u, uint64_t *ret) {
@@ -1060,10 +1429,13 @@ int unit_get_memory_current(Unit *u, uint64_t *ret) {
         if (!u->cgroup_path)
                 return -ENODATA;
 
-        if ((u->cgroup_realized_mask & CGROUP_MEMORY) == 0)
+        if ((u->cgroup_realized_mask & CGROUP_MASK_MEMORY) == 0)
                 return -ENODATA;
 
-        r = cg_get_attribute("memory", u->cgroup_path, "memory.usage_in_bytes", &v);
+        if (cg_unified() <= 0)
+                r = cg_get_attribute("memory", u->cgroup_path, "memory.usage_in_bytes", &v);
+        else
+                r = cg_get_attribute("memory", u->cgroup_path, "memory.current", &v);
         if (r == -ENOENT)
                 return -ENODATA;
         if (r < 0)
@@ -1083,7 +1455,7 @@ static int unit_get_cpu_usage_raw(Unit *u, nsec_t *ret) {
         if (!u->cgroup_path)
                 return -ENODATA;
 
-        if ((u->cgroup_realized_mask & CGROUP_CPUACCT) == 0)
+        if ((u->cgroup_realized_mask & CGROUP_MASK_CPUACCT) == 0)
                 return -ENODATA;
 
         r = cg_get_attribute("cpuacct", u->cgroup_path, "cpuacct.usage", &v);