core: introduce new Restart=on-abnormal setting

Restart=on-abnormal is similar to Restart=on-failure, but avoids
restarts on unclean exit codes (but still doing restarts on all
obviously unclean exits, such as timeouts, signals, coredumps, watchdog
timeouts).

Also see:

https://fedorahosted.org/fpc/ticket/191

3 files changed, 129 insertions, 32 deletions

diff --git a/man/systemd.service.xml b/man/systemd.service.xml
index 74d974e6e9..640318ba44 100644
--- a/man/systemd.service.xml
+++ b/man/systemd.service.xml
@@ -703,51 +703,146 @@ ExecStart=/bin/echo $ONE $TWO ${TWO}</programlisting>
                                 <option>no</option>,
                                 <option>on-success</option>,
                                 <option>on-failure</option>,
+                                <option>on-abnormal</option>,
                                 <option>on-watchdog</option>,
                                 <option>on-abort</option>, or
                                 <option>always</option>. If set to
                                 <option>no</option> (the default), the
-                                service will not be restarted. If set to
-                                <option>on-success</option>, it will be
-                                restarted only when the service process
-                                exits cleanly.
-                                In this context, a clean exit means
-                                an exit code of 0, or one of the signals
+                                service will not be restarted. If set
+                                to <option>on-success</option>, it
+                                will be restarted only when the
+                                service process exits cleanly.  In
+                                this context, a clean exit means an
+                                exit code of 0, or one of the signals
                                 <constant>SIGHUP</constant>,
                                 <constant>SIGINT</constant>,
-                                <constant>SIGTERM</constant>,
-                                or <constant>SIGPIPE</constant>, and
-                                additionally, exit statuses and signals
-                                specified in <varname>SuccessExitStatus=</varname>.
+                                <constant>SIGTERM</constant> or
+                                <constant>SIGPIPE</constant>, and
+                                additionally, exit statuses and
+                                signals specified in
+                                <varname>SuccessExitStatus=</varname>.
                                 If set to <option>on-failure</option>,
                                 the service will be restarted when the
-                                process exits with a non-zero exit code,
-                                is terminated by a signal (including on
-                                core dump), when an operation (such as
-                                service reload) times out, and when the
-                                configured watchdog timeout is triggered.
-                                If set to
-                                <option>on-abort</option>, the service
-                                will be restarted only if the service
-                                process exits due to an uncaught
-                                signal not specified as a clean exit
-                                status.
-                                If set to
-                                <option>on-watchdog</option>, the service
-                                will be restarted only if the watchdog
-                                timeout for the service expires.
-                                If set to
+                                process exits with a non-zero exit
+                                code, is terminated by a signal
+                                (including on core dump, but excluding
+                                the aforementiond four signals), when
+                                an operation (such as service reload)
+                                times out, and when the configured
+                                watchdog timeout is triggered.  If set
+                                to <option>on-abnormal</option>, the
+                                service will be restarted when the
+                                process is terminated by a signal
+                                (including on core dump, excluding the
+                                aforementioned four signals), when an
+                                operation times out, or when the
+                                watchdog timeout is triggered. If set
+                                to <option>on-abort</option>, the
+                                service will be restarted only if the
+                                service process exits due to an
+                                uncaught signal not specified as a
+                                clean exit status.  If set to
+                                <option>on-watchdog</option>, the
+                                service will be restarted only if the
+                                watchdog timeout for the service
+                                expires.  If set to
                                 <option>always</option>, the service
-                                will be restarted regardless of whether
-                                it exited cleanly or not, got
+                                will be restarted regardless of
+                                whether it exited cleanly or not, got
                                 terminated abnormally by a signal, or
                                 hit a timeout.</para>
 
+                                <table>
+                                        <title>Exit causes and the effect of the <varname>Restart=</varname> settings on them</title>
+
+                                        <tgroup cols='2'>
+                                                <colspec colname='path' />
+                                                <colspec colname='expl' />
+                                                <thead>
+                                                        <row>
+                                                                <entry>Restart settings/Exit causes</entry>
+                                                                <entry><option>no</option></entry>
+                                                                <entry><option>always</option></entry>
+                                                                <entry><option>on-success</option></entry>
+                                                                <entry><option>on-failure</option></entry>
+                                                                <entry><option>on-abnormal</option></entry>
+                                                                <entry><option>on-abort</option></entry>
+                                                                <entry><option>on-watchdog</option></entry>
+                                                        </row>
+                                                </thead>
+                                                <tbody>
+                                                        <row>
+                                                                <entry>Clean exit code or signal</entry>
+                                                                <entry/>
+                                                                <entry>X</entry>
+                                                                <entry>X</entry>
+                                                                <entry/>
+                                                                <entry/>
+                                                                <entry/>
+                                                                <entry/>
+                                                        </row>
+                                                        <row>
+                                                                <entry>Unclean exit code</entry>
+                                                                <entry/>
+                                                                <entry>X</entry>
+                                                                <entry/>
+                                                                <entry>X</entry>
+                                                                <entry/>
+                                                                <entry/>
+                                                                <entry/>
+                                                        </row>
+                                                        <row>
+                                                                <entry>Unclean signal</entry>
+                                                                <entry/>
+                                                                <entry>X</entry>
+                                                                <entry/>
+                                                                <entry>X</entry>
+                                                                <entry>X</entry>
+                                                                <entry>X</entry>
+                                                                <entry/>
+                                                        </row>
+                                                        <row>
+                                                                <entry>Timeout</entry>
+                                                                <entry/>
+                                                                <entry>X</entry>
+                                                                <entry/>
+                                                                <entry>X</entry>
+                                                                <entry>X</entry>
+                                                                <entry/>
+                                                                <entry/>
+                                                        </row>
+                                                        <row>
+                                                                <entry>Watchdog</entry>
+                                                                <entry/>
+                                                                <entry>X</entry>
+                                                                <entry/>
+                                                                <entry>X</entry>
+                                                                <entry>X</entry>
+                                                                <entry/>
+                                                                <entry>X</entry>
+                                                        </row>
+                                                </tbody>
+                                        </tgroup>
+                                </table>
+
                                 <para>In addition to the above settings,
                                 the service will not be restarted if the
                                 exit code or signal is specified in
                                 <varname>RestartPreventExitStatus=</varname>
-                                (see below).</para></listitem>
+                                (see below).</para>
+
+                                <para>Setting this to
+                                <option>on-failure</option> is the
+                                recommended choice for long-running
+                                services, in order to increase
+                                reliability by attempting automatic
+                                recovery from errors. For services
+                                that shall be able to terminate on
+                                their own choice (and avoiding
+                                immediate restart)
+                                <option>on-abnormal</option> is an
+                                alternative choice.</para>
+                                </listitem>
                         </varlistentry>
 
                         <varlistentry>
diff --git a/src/core/service.c b/src/core/service.c
index 70e7635114..244ec11f75 100644
--- a/src/core/service.c
+++ b/src/core/service.c
@@ -1867,9 +1867,9 @@ static void service_enter_dead(Service *s, ServiceResult f, bool allow_restart)
             (s->restart == SERVICE_RESTART_ALWAYS ||
              (s->restart == SERVICE_RESTART_ON_SUCCESS && s->result == SERVICE_SUCCESS) ||
              (s->restart == SERVICE_RESTART_ON_FAILURE && s->result != SERVICE_SUCCESS) ||
+             (s->restart == SERVICE_RESTART_ON_ABNORMAL && !IN_SET(s->result, SERVICE_SUCCESS, SERVICE_FAILURE_EXIT_CODE)) ||
              (s->restart == SERVICE_RESTART_ON_WATCHDOG && s->result == SERVICE_FAILURE_WATCHDOG) ||
-             (s->restart == SERVICE_RESTART_ON_ABORT && (s->result == SERVICE_FAILURE_SIGNAL ||
-                                                         s->result == SERVICE_FAILURE_CORE_DUMP))) &&
+             (s->restart == SERVICE_RESTART_ON_ABORT && IN_SET(s->result, SERVICE_FAILURE_SIGNAL, SERVICE_FAILURE_CORE_DUMP))) &&
             (s->result != SERVICE_FAILURE_EXIT_CODE ||
              !set_contains(s->restart_ignore_status.code, INT_TO_PTR(s->main_exec_status.status))) &&
             (s->result != SERVICE_FAILURE_SIGNAL ||
@@ -3790,9 +3790,10 @@ static const char* const service_restart_table[_SERVICE_RESTART_MAX] = {
         [SERVICE_RESTART_NO] = "no",
         [SERVICE_RESTART_ON_SUCCESS] = "on-success",
         [SERVICE_RESTART_ON_FAILURE] = "on-failure",
+        [SERVICE_RESTART_ON_ABNORMAL] = "on-abnormal",
         [SERVICE_RESTART_ON_WATCHDOG] = "on-watchdog",
         [SERVICE_RESTART_ON_ABORT] = "on-abort",
-        [SERVICE_RESTART_ALWAYS] = "always"
+        [SERVICE_RESTART_ALWAYS] = "always",
 };
 
 DEFINE_STRING_TABLE_LOOKUP(service_restart, ServiceRestart);
diff --git a/src/core/service.h b/src/core/service.h
index 2254c3e0bf..8c2c819256 100644
--- a/src/core/service.h
+++ b/src/core/service.h
@@ -53,6 +53,7 @@ typedef enum ServiceRestart {
         SERVICE_RESTART_NO,
         SERVICE_RESTART_ON_SUCCESS,
         SERVICE_RESTART_ON_FAILURE,
+        SERVICE_RESTART_ON_ABNORMAL,
         SERVICE_RESTART_ON_WATCHDOG,
         SERVICE_RESTART_ON_ABORT,
         SERVICE_RESTART_ALWAYS,