Merge pull request #2839 from keszybz/use-sendfile-smarter

Use sendfile smarter

7 files changed, 438 insertions, 230 deletions

diff --git a/Makefile.am b/Makefile.am
index 0f1f79e62d..97e2c26442 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -744,6 +744,7 @@ noinst_LTLIBRARIES += \
 
 libbasic_la_SOURCES = \
 	src/basic/missing.h \
+	src/basic/missing_syscall.h \
 	src/basic/capability-util.c \
 	src/basic/capability-util.h \
 	src/basic/c-rbtree.c \
diff --git a/TODO b/TODO
index 08b74083d3..b2840ba4ab 100644
--- a/TODO
+++ b/TODO
@@ -49,8 +49,6 @@ Features:
 
 * cache sd_event_now() result from before the first iteration...
 
-* support for the new copy_file_range() syscall
-
 * add systemctl stop --job-mode=triggering that follows TRIGGERED_BY deps and adds them to the same transaction
 
 * Maybe add a way how users can "pin" units into memory, so that they are not subject to automatic GC?
diff --git a/configure.ac b/configure.ac
index 79340bcca9..cb14abda05 100644
--- a/configure.ac
+++ b/configure.ac
@@ -296,8 +296,19 @@ LIBS="$save_LIBS"
 AC_SUBST(CAP_LIBS)
 
 AC_CHECK_FUNCS([__secure_getenv secure_getenv])
-AC_CHECK_DECLS([memfd_create, gettid, pivot_root, name_to_handle_at, setns, getrandom, renameat2, kcmp, keyctl, LO_FLAGS_PARTSCAN],
-               [], [], [[
+AC_CHECK_DECLS([
+        memfd_create,
+        gettid,
+        pivot_root,
+        name_to_handle_at,
+        setns,
+        getrandom,
+        renameat2,
+        kcmp,
+        keyctl,
+        LO_FLAGS_PARTSCAN,
+        copy_file_range],
+        [], [], [[
 #include <sys/types.h>
 #include <unistd.h>
 #include <sys/mount.h>
diff --git a/src/basic/copy.c b/src/basic/copy.c
index 519b412941..41dc8ca79a 100644
--- a/src/basic/copy.c
+++ b/src/basic/copy.c
@@ -40,17 +40,38 @@
 #include "fs-util.h"
 #include "io-util.h"
 #include "macro.h"
+#include "missing.h"
 #include "string-util.h"
 #include "strv.h"
 #include "time-util.h"
 #include "umask-util.h"
 #include "xattr-util.h"
 
-#define COPY_BUFFER_SIZE (16*1024)
+#define COPY_BUFFER_SIZE (16*1024u)
+
+static ssize_t try_copy_file_range(int fd_in, loff_t *off_in,
+                                   int fd_out, loff_t *off_out,
+                                   size_t len,
+                                   unsigned int flags) {
+        static int have = -1;
+        ssize_t r;
+
+        if (have == false)
+                return -ENOSYS;
+
+        r = copy_file_range(fd_in, off_in, fd_out, off_out, len, flags);
+        if (_unlikely_(have < 0))
+                have = r >= 0 || errno != ENOSYS;
+        if (r >= 0)
+                return r;
+        else
+                return -errno;
+}
 
 int copy_bytes(int fdf, int fdt, uint64_t max_bytes, bool try_reflink) {
-        bool try_sendfile = true, try_splice = true;
+        bool try_cfr = true, try_sendfile = true, try_splice = true;
         int r;
+        size_t m = SSIZE_MAX; /* that the maximum that sendfile and c_f_r accept */
 
         assert(fdf >= 0);
         assert(fdt >= 0);
@@ -67,11 +88,9 @@ int copy_bytes(int fdf, int fdt, uint64_t max_bytes, bool try_reflink) {
         }
 
         for (;;) {
-                size_t m = COPY_BUFFER_SIZE;
                 ssize_t n;
 
                 if (max_bytes != (uint64_t) -1) {
-
                         if (max_bytes <= 0)
                                 return 1; /* return > 0 if we hit the max_bytes limit */
 
@@ -79,44 +98,59 @@ int copy_bytes(int fdf, int fdt, uint64_t max_bytes, bool try_reflink) {
                                 m = (size_t) max_bytes;
                 }
 
+                /* First try copy_file_range(), unless we already tried */
+                if (try_cfr) {
+                        n = try_copy_file_range(fdf, NULL, fdt, NULL, m, 0u);
+                        if (n < 0) {
+                                if (!IN_SET(n, -EINVAL, -ENOSYS, -EXDEV))
+                                        return n;
+
+                                try_cfr = false;
+                                /* use fallback below */
+                        } else if (n == 0) /* EOF */
+                                break;
+                        else
+                                /* Success! */
+                                goto next;
+                }
+
                 /* First try sendfile(), unless we already tried */
                 if (try_sendfile) {
-
                         n = sendfile(fdt, fdf, NULL, m);
                         if (n < 0) {
-                                if (errno != EINVAL && errno != ENOSYS)
+                                if (!IN_SET(errno, EINVAL, ENOSYS))
                                         return -errno;
 
                                 try_sendfile = false;
                                 /* use fallback below */
                         } else if (n == 0) /* EOF */
                                 break;
-                        else if (n > 0)
+                        else
                                 /* Success! */
                                 goto next;
                 }
 
-                /* The try splice, unless we already tried */
+                /* Then try splice, unless we already tried */
                 if (try_splice) {
                         n = splice(fdf, NULL, fdt, NULL, m, 0);
                         if (n < 0) {
-                                if (errno != EINVAL && errno != ENOSYS)
+                                if (!IN_SET(errno, EINVAL, ENOSYS))
                                         return -errno;
 
                                 try_splice = false;
                                 /* use fallback below */
                         } else if (n == 0) /* EOF */
                                 break;
-                        else if (n > 0)
+                        else
                                 /* Success! */
                                 goto next;
                 }
 
                 /* As a fallback just copy bits by hand */
                 {
-                        uint8_t buf[m];
+                        uint8_t buf[MIN(m, COPY_BUFFER_SIZE)];
 
-                        n = read(fdf, buf, m);
+                        n = read(fdf, buf, sizeof buf);
                         if (n < 0)
                                 return -errno;
                         if (n == 0) /* EOF */
@@ -132,6 +166,11 @@ int copy_bytes(int fdf, int fdt, uint64_t max_bytes, bool try_reflink) {
                         assert(max_bytes >= (uint64_t) n);
                         max_bytes -= n;
                 }
+                /* sendfile accepts at most SSIZE_MAX-offset bytes to copy,
+                 * so reduce our maximum by the amount we already copied,
+                 * but don't go below our copy buffer size, unless we are
+                 * close the the limit of bytes we are allowed to copy. */
+                m = MAX(MIN(COPY_BUFFER_SIZE, max_bytes), m - n);
         }
 
         return 0; /* return 0 if we hit EOF earlier than the size limit */
diff --git a/src/basic/missing.h b/src/basic/missing.h
index 417604aa64..034e334e66 100644
--- a/src/basic/missing.h
+++ b/src/basic/missing.h
@@ -135,84 +135,6 @@
 #define SOL_SCTP 132
 #endif
 
-#if !HAVE_DECL_PIVOT_ROOT
-static inline int pivot_root(const char *new_root, const char *put_old) {
-        return syscall(SYS_pivot_root, new_root, put_old);
-}
-#endif
-
-#ifndef __NR_memfd_create
-#  if defined __x86_64__
-#    define __NR_memfd_create 319
-#  elif defined __arm__
-#    define __NR_memfd_create 385
-#  elif defined __aarch64__
-#    define __NR_memfd_create 279
-#  elif defined __s390__
-#    define __NR_memfd_create 350
-#  elif defined _MIPS_SIM
-#    if _MIPS_SIM == _MIPS_SIM_ABI32
-#      define __NR_memfd_create 4354
-#    endif
-#    if _MIPS_SIM == _MIPS_SIM_NABI32
-#      define __NR_memfd_create 6318
-#    endif
-#    if _MIPS_SIM == _MIPS_SIM_ABI64
-#      define __NR_memfd_create 5314
-#    endif
-#  elif defined __i386__
-#    define __NR_memfd_create 356
-#  else
-#    warning "__NR_memfd_create unknown for your architecture"
-#    define __NR_memfd_create 0xffffffff
-#  endif
-#endif
-
-#if !HAVE_DECL_MEMFD_CREATE
-static inline int memfd_create(const char *name, unsigned int flags) {
-        return syscall(__NR_memfd_create, name, flags);
-}
-#endif
-
-#ifndef __NR_getrandom
-#  if defined __x86_64__
-#    define __NR_getrandom 318
-#  elif defined(__i386__)
-#    define __NR_getrandom 355
-#  elif defined(__arm__)
-#    define __NR_getrandom 384
-# elif defined(__aarch64__)
-#    define __NR_getrandom 278
-#  elif defined(__ia64__)
-#    define __NR_getrandom 1339
-#  elif defined(__m68k__)
-#    define __NR_getrandom 352
-#  elif defined(__s390x__)
-#    define __NR_getrandom 349
-#  elif defined(__powerpc__)
-#    define __NR_getrandom 359
-#  elif defined _MIPS_SIM
-#    if _MIPS_SIM == _MIPS_SIM_ABI32
-#      define __NR_getrandom 4353
-#    endif
-#    if _MIPS_SIM == _MIPS_SIM_NABI32
-#      define __NR_getrandom 6317
-#    endif
-#    if _MIPS_SIM == _MIPS_SIM_ABI64
-#      define __NR_getrandom 5313
-#    endif
-#  else
-#    warning "__NR_getrandom unknown for your architecture"
-#    define __NR_getrandom 0xffffffff
-#  endif
-#endif
-
-#if !HAVE_DECL_GETRANDOM
-static inline int getrandom(void *buffer, size_t count, unsigned flags) {
-        return syscall(__NR_getrandom, buffer, count, flags);
-}
-#endif
-
 #ifndef GRND_NONBLOCK
 #define GRND_NONBLOCK 0x0001
 #endif
@@ -527,12 +449,6 @@ struct btrfs_ioctl_quota_ctl_args {
 #define MS_PRIVATE  (1 << 18)
 #endif
 
-#if !HAVE_DECL_GETTID
-static inline pid_t gettid(void) {
-        return (pid_t) syscall(SYS_gettid);
-}
-#endif
-
 #ifndef SCM_SECURITY
 #define SCM_SECURITY 0x03
 #endif
@@ -561,32 +477,6 @@ static inline pid_t gettid(void) {
 #define MAX_HANDLE_SZ 128
 #endif
 
-#ifndef __NR_name_to_handle_at
-#  if defined(__x86_64__)
-#    define __NR_name_to_handle_at 303
-#  elif defined(__i386__)
-#    define __NR_name_to_handle_at 341
-#  elif defined(__arm__)
-#    define __NR_name_to_handle_at 370
-#  elif defined(__powerpc__)
-#    define __NR_name_to_handle_at 345
-#  else
-#    error "__NR_name_to_handle_at is not defined"
-#  endif
-#endif
-
-#if !HAVE_DECL_NAME_TO_HANDLE_AT
-struct file_handle {
-        unsigned int handle_bytes;
-        int handle_type;
-        unsigned char f_handle[0];
-};
-
-static inline int name_to_handle_at(int fd, const char *name, struct file_handle *handle, int *mnt_id, int flags) {
-        return syscall(__NR_name_to_handle_at, fd, name, handle, mnt_id, flags);
-}
-#endif
-
 #ifndef HAVE_SECURE_GETENV
 #  ifdef HAVE___SECURE_GETENV
 #    define secure_getenv __secure_getenv
@@ -635,22 +525,6 @@ static inline int name_to_handle_at(int fd, const char *name, struct file_handle
 
 #endif
 
-#ifndef __NR_setns
-#  if defined(__x86_64__)
-#    define __NR_setns 308
-#  elif defined(__i386__)
-#    define __NR_setns 346
-#  else
-#    error "__NR_setns is not defined"
-#  endif
-#endif
-
-#if !HAVE_DECL_SETNS
-static inline int setns(int fd, int nstype) {
-        return syscall(__NR_setns, fd, nstype);
-}
-#endif
-
 #if !HAVE_DECL_LO_FLAGS_PARTSCAN
 #define LO_FLAGS_PARTSCAN 8
 #endif
@@ -1018,69 +892,10 @@ static inline int setns(int fd, int nstype) {
 #define CAP_AUDIT_READ 37
 #endif
 
-static inline int raw_clone(unsigned long flags, void *child_stack) {
-#if defined(__s390__) || defined(__CRIS__)
-        /* On s390 and cris the order of the first and second arguments
-         * of the raw clone() system call is reversed. */
-        return (int) syscall(__NR_clone, child_stack, flags);
-#else
-        return (int) syscall(__NR_clone, flags, child_stack);
-#endif
-}
-
-static inline pid_t raw_getpid(void) {
-#if defined(__alpha__)
-        return (pid_t) syscall(__NR_getxpid);
-#else
-        return (pid_t) syscall(__NR_getpid);
-#endif
-}
-
-#if !HAVE_DECL_RENAMEAT2
-
-#ifndef __NR_renameat2
-#  if defined __x86_64__
-#    define __NR_renameat2 316
-#  elif defined __arm__
-#    define __NR_renameat2 382
-#  elif defined _MIPS_SIM
-#    if _MIPS_SIM == _MIPS_SIM_ABI32
-#      define __NR_renameat2 4351
-#    endif
-#    if _MIPS_SIM == _MIPS_SIM_NABI32
-#      define __NR_renameat2 6315
-#    endif
-#    if _MIPS_SIM == _MIPS_SIM_ABI64
-#      define __NR_renameat2 5311
-#    endif
-#  elif defined __i386__
-#    define __NR_renameat2 353
-#  else
-#    warning "__NR_renameat2 unknown for your architecture"
-#    define __NR_renameat2 0xffffffff
-#  endif
-#endif
-
-static inline int renameat2(int oldfd, const char *oldname, int newfd, const char *newname, unsigned flags) {
-        return syscall(__NR_renameat2, oldfd, oldname, newfd, newname, flags);
-}
-#endif
-
 #ifndef RENAME_NOREPLACE
 #define RENAME_NOREPLACE (1 << 0)
 #endif
 
-#if !HAVE_DECL_KCMP
-static inline int kcmp(pid_t pid1, pid_t pid2, int type, unsigned long idx1, unsigned long idx2) {
-#if defined(__NR_kcmp)
-        return syscall(__NR_kcmp, pid1, pid2, type, idx1, idx2);
-#else
-        errno = ENOSYS;
-        return -1;
-#endif
-}
-#endif
-
 #ifndef KCMP_FILE
 #define KCMP_FILE 0
 #endif
@@ -1097,35 +912,6 @@ static inline int kcmp(pid_t pid1, pid_t pid2, int type, unsigned long idx1, uns
 typedef int32_t key_serial_t;
 #endif
 
-#if !HAVE_DECL_KEYCTL
-static inline long keyctl(int cmd, unsigned long arg2, unsigned long arg3, unsigned long arg4,unsigned long arg5) {
-#if defined(__NR_keyctl)
-        return syscall(__NR_keyctl, cmd, arg2, arg3, arg4, arg5);
-#else
-        errno = ENOSYS;
-        return -1;
-#endif
-}
-
-static inline key_serial_t add_key(const char *type, const char *description, const void *payload, size_t plen, key_serial_t ringid) {
-#if defined (__NR_add_key)
-        return syscall(__NR_add_key, type, description, payload, plen, ringid);
-#else
-        errno = ENOSYS;
-        return -1;
-#endif
-}
-
-static inline key_serial_t request_key(const char *type, const char *description, const char * callout_info, key_serial_t destringid) {
-#if defined (__NR_request_key)
-        return syscall(__NR_request_key, type, description, callout_info, destringid);
-#else
-        errno = ENOSYS;
-        return -1;
-#endif
-}
-#endif
-
 #ifndef KEYCTL_READ
 #define KEYCTL_READ 11
 #endif
@@ -1177,3 +963,5 @@ static inline key_serial_t request_key(const char *type, const char *description
 #endif
 
 #endif
+
+#include "missing_syscall.h"
diff --git a/src/basic/missing_syscall.h b/src/basic/missing_syscall.h
new file mode 100644
index 0000000000..d502d3b9ca
--- /dev/null
+++ b/src/basic/missing_syscall.h
@@ -0,0 +1,310 @@
+#pragma once
+
+/***
+  This file is part of systemd.
+
+  Copyright 2010 Lennart Poettering
+  Copyright 2016 Zbigniew Jędrzejewski-Szmek
+
+  systemd is free software; you can redistribute it and/or modify it
+  under the terms of the GNU Lesser General Public License as published by
+  the Free Software Foundation; either version 2.1 of the License, or
+  (at your option) any later version.
+
+  systemd is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public License
+  along with systemd; If not, see <http://www.gnu.org/licenses/>.
+***/
+
+/* Missing glibc definitions to access certain kernel APIs */
+
+#if !HAVE_DECL_PIVOT_ROOT
+static inline int pivot_root(const char *new_root, const char *put_old) {
+        return syscall(SYS_pivot_root, new_root, put_old);
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_MEMFD_CREATE
+#  ifndef __NR_memfd_create
+#    if defined __x86_64__
+#      define __NR_memfd_create 319
+#    elif defined __arm__
+#      define __NR_memfd_create 385
+#    elif defined __aarch64__
+#      define __NR_memfd_create 279
+#    elif defined __s390__
+#      define __NR_memfd_create 350
+#    elif defined _MIPS_SIM
+#      if _MIPS_SIM == _MIPS_SIM_ABI32
+#        define __NR_memfd_create 4354
+#      endif
+#      if _MIPS_SIM == _MIPS_SIM_NABI32
+#        define __NR_memfd_create 6318
+#      endif
+#      if _MIPS_SIM == _MIPS_SIM_ABI64
+#        define __NR_memfd_create 5314
+#      endif
+#    elif defined __i386__
+#      define __NR_memfd_create 356
+#    else
+#      warning "__NR_memfd_create unknown for your architecture"
+#    endif
+#  endif
+
+static inline int memfd_create(const char *name, unsigned int flags) {
+#  ifdef __NR_memfd_create
+        return syscall(__NR_memfd_create, name, flags);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_GETRANDOM
+#  ifndef __NR_getrandom
+#    if defined __x86_64__
+#      define __NR_getrandom 318
+#    elif defined(__i386__)
+#      define __NR_getrandom 355
+#    elif defined(__arm__)
+#      define __NR_getrandom 384
+#   elif defined(__aarch64__)
+#      define __NR_getrandom 278
+#    elif defined(__ia64__)
+#      define __NR_getrandom 1339
+#    elif defined(__m68k__)
+#      define __NR_getrandom 352
+#    elif defined(__s390x__)
+#      define __NR_getrandom 349
+#    elif defined(__powerpc__)
+#      define __NR_getrandom 359
+#    elif defined _MIPS_SIM
+#      if _MIPS_SIM == _MIPS_SIM_ABI32
+#        define __NR_getrandom 4353
+#      endif
+#      if _MIPS_SIM == _MIPS_SIM_NABI32
+#        define __NR_getrandom 6317
+#      endif
+#      if _MIPS_SIM == _MIPS_SIM_ABI64
+#        define __NR_getrandom 5313
+#      endif
+#    else
+#      warning "__NR_getrandom unknown for your architecture"
+#    endif
+#  endif
+
+static inline int getrandom(void *buffer, size_t count, unsigned flags) {
+#  ifdef __NR_getrandom
+        return syscall(__NR_getrandom, buffer, count, flags);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_GETTID
+static inline pid_t gettid(void) {
+        return (pid_t) syscall(SYS_gettid);
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_NAME_TO_HANDLE_AT
+#  ifndef __NR_name_to_handle_at
+#    if defined(__x86_64__)
+#      define __NR_name_to_handle_at 303
+#    elif defined(__i386__)
+#      define __NR_name_to_handle_at 341
+#    elif defined(__arm__)
+#      define __NR_name_to_handle_at 370
+#    elif defined(__powerpc__)
+#      define __NR_name_to_handle_at 345
+#    else
+#      error "__NR_name_to_handle_at is not defined"
+#    endif
+#  endif
+
+struct file_handle {
+        unsigned int handle_bytes;
+        int handle_type;
+        unsigned char f_handle[0];
+};
+
+static inline int name_to_handle_at(int fd, const char *name, struct file_handle *handle, int *mnt_id, int flags) {
+#  ifdef __NR_name_to_handle_at
+        return syscall(__NR_name_to_handle_at, fd, name, handle, mnt_id, flags);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_SETNS
+#  ifndef __NR_setns
+#    if defined(__x86_64__)
+#      define __NR_setns 308
+#    elif defined(__i386__)
+#      define __NR_setns 346
+#    else
+#      error "__NR_setns is not defined"
+#    endif
+#  endif
+
+static inline int setns(int fd, int nstype) {
+#  ifdef __NR_setns
+        return syscall(__NR_setns, fd, nstype);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+#endif
+
+/* ======================================================================= */
+
+static inline int raw_clone(unsigned long flags, void *child_stack) {
+#if defined(__s390__) || defined(__CRIS__)
+        /* On s390 and cris the order of the first and second arguments
+         * of the raw clone() system call is reversed. */
+        return (int) syscall(__NR_clone, child_stack, flags);
+#else
+        return (int) syscall(__NR_clone, flags, child_stack);
+#endif
+}
+
+/* ======================================================================= */
+
+static inline pid_t raw_getpid(void) {
+#if defined(__alpha__)
+        return (pid_t) syscall(__NR_getxpid);
+#else
+        return (pid_t) syscall(__NR_getpid);
+#endif
+}
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_RENAMEAT2
+#  ifndef __NR_renameat2
+#    if defined __x86_64__
+#      define __NR_renameat2 316
+#    elif defined __arm__
+#      define __NR_renameat2 382
+#    elif defined _MIPS_SIM
+#      if _MIPS_SIM == _MIPS_SIM_ABI32
+#        define __NR_renameat2 4351
+#      endif
+#      if _MIPS_SIM == _MIPS_SIM_NABI32
+#        define __NR_renameat2 6315
+#      endif
+#      if _MIPS_SIM == _MIPS_SIM_ABI64
+#        define __NR_renameat2 5311
+#      endif
+#    elif defined __i386__
+#      define __NR_renameat2 353
+#    else
+#      warning "__NR_renameat2 unknown for your architecture"
+#    endif
+#  endif
+
+static inline int renameat2(int oldfd, const char *oldname, int newfd, const char *newname, unsigned flags) {
+#  ifdef __NR_renameat2
+        return syscall(__NR_renameat2, oldfd, oldname, newfd, newname, flags);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_KCMP
+static inline int kcmp(pid_t pid1, pid_t pid2, int type, unsigned long idx1, unsigned long idx2) {
+#  ifdef __NR_kcmp
+        return syscall(__NR_kcmp, pid1, pid2, type, idx1, idx2);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_KEYCTL
+static inline long keyctl(int cmd, unsigned long arg2, unsigned long arg3, unsigned long arg4,unsigned long arg5) {
+#  ifdef __NR_keyctl
+        return syscall(__NR_keyctl, cmd, arg2, arg3, arg4, arg5);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+
+static inline key_serial_t add_key(const char *type, const char *description, const void *payload, size_t plen, key_serial_t ringid) {
+#  ifdef __NR_add_key
+        return syscall(__NR_add_key, type, description, payload, plen, ringid);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+
+static inline key_serial_t request_key(const char *type, const char *description, const char * callout_info, key_serial_t destringid) {
+#  ifdef __NR_request_key
+        return syscall(__NR_request_key, type, description, callout_info, destringid);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_COPY_FILE_RANGE
+#  ifndef __NR_copy_file_range
+#    if defined(__x86_64__)
+#      define __NR_copy_file_range 326
+#    elif defined(__i386__)
+#      define __NR_copy_file_range 377
+#    elif defined __s390__
+#      define __NR_copy_file_range 375
+#    elif defined __arm__
+#      define __NR_copy_file_range 391
+#    elif defined __aarch64__
+#      define __NR_copy_file_range 285
+#    else
+#      warning "__NR_copy_file_range not defined for your architecture"
+#    endif
+#  endif
+
+static inline ssize_t copy_file_range(int fd_in, loff_t *off_in,
+                                      int fd_out, loff_t *off_out,
+                                      size_t len,
+                                      unsigned int flags) {
+#  ifdef __NR_copy_file_range
+        return syscall(__NR_copy_file_range, fd_in, off_in, fd_out, off_out, len, flags);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+#endif
diff --git a/src/test/test-copy.c b/src/test/test-copy.c
index ad57cb0202..1462affbcf 100644
--- a/src/test/test-copy.c
+++ b/src/test/test-copy.c
@@ -24,6 +24,7 @@
 #include "fd-util.h"
 #include "fileio.h"
 #include "fs-util.h"
+#include "log.h"
 #include "macro.h"
 #include "mkdir.h"
 #include "path-util.h"
@@ -39,6 +40,8 @@ static void test_copy_file(void) {
         size_t sz = 0;
         int fd;
 
+        log_info("%s", __func__);
+
         fd = mkostemp_safe(fn, O_RDWR|O_CLOEXEC);
         assert_se(fd >= 0);
         close(fd);
@@ -66,6 +69,8 @@ static void test_copy_file_fd(void) {
         char text[] = "boohoo\nfoo\n\tbar\n";
         char buf[64] = {0};
 
+        log_info("%s", __func__);
+
         in_fd = mkostemp_safe(in_fn, O_RDWR);
         assert_se(in_fd >= 0);
         out_fd = mkostemp_safe(out_fn, O_RDWR);
@@ -91,6 +96,8 @@ static void test_copy_tree(void) {
                                  "link2", "dir1/file");
         char **p, **link;
 
+        log_info("%s", __func__);
+
         (void) rm_rf(copy_dir, REMOVE_ROOT|REMOVE_PHYSICAL);
         (void) rm_rf(original_dir, REMOVE_ROOT|REMOVE_PHYSICAL);
 
@@ -173,11 +180,65 @@ static void test_copy_bytes(void) {
         assert_se(r == -EBADF);
 }
 
+static void test_copy_bytes_regular_file(const char *src, bool try_reflink, size_t max_bytes) {
+        char fn2[] = "/tmp/test-copy-file-XXXXXX";
+        char fn3[] = "/tmp/test-copy-file-XXXXXX";
+        _cleanup_close_ int fd = -1, fd2 = -1, fd3 = -1;
+        int r;
+        struct stat buf, buf2, buf3;
+
+        log_info("%s try_reflink=%s max_bytes=%zu", __func__, yes_no(try_reflink), max_bytes);
+
+        fd = open(src, O_RDONLY | O_CLOEXEC | O_NOCTTY);
+        assert_se(fd >= 0);
+
+        fd2 = mkostemp_safe(fn2, O_RDWR);
+        assert_se(fd2 >= 0);
+
+        fd3 = mkostemp_safe(fn3, O_WRONLY);
+        assert_se(fd3 >= 0);
+
+        r = copy_bytes(fd, fd2, max_bytes, try_reflink);
+        if (max_bytes == (uint64_t) -1)
+                assert_se(r == 0);
+        else
+                assert_se(IN_SET(r, 0, 1));
+
+        assert_se(lseek(fd2, 0, SEEK_SET) == 0);
+
+        r = copy_bytes(fd2, fd3, max_bytes, try_reflink);
+        if (max_bytes == (uint64_t) -1)
+                assert_se(r == 0);
+        else
+                /* We cannot distinguish between the input being exactly max_bytes
+                 * or longer than max_bytes (without trying to read one more byte,
+                 * or calling stat, or FION_READ, etc, and we don't want to do any
+                 * of that). So we expect "truncation" since we know that file we
+                 * are copying is exactly max_bytes bytes. */
+                assert_se(r == 1);
+
+        assert_se(fstat(fd, &buf) == 0);
+        assert_se(fstat(fd2, &buf2) == 0);
+        assert_se(fstat(fd3, &buf3) == 0);
+
+        assert_se((size_t) buf2.st_size == MIN((size_t) buf.st_size, max_bytes));
+        assert_se(buf3.st_size == buf2.st_size);
+
+        unlink(fn2);
+        unlink(fn3);
+}
+
 int main(int argc, char *argv[]) {
         test_copy_file();
         test_copy_file_fd();
         test_copy_tree();
         test_copy_bytes();
+        test_copy_bytes_regular_file(argv[0], false, (uint64_t) -1);
+        test_copy_bytes_regular_file(argv[0], true, (uint64_t) -1);
+        test_copy_bytes_regular_file(argv[0], false, 1000); /* smaller than copy buffer size */
+        test_copy_bytes_regular_file(argv[0], true, 1000);
+        test_copy_bytes_regular_file(argv[0], false, 32000); /* larger than copy buffer size */
+        test_copy_bytes_regular_file(argv[0], true, 32000);
 
         return 0;
 }