Merge pull request #2839 from keszybz/use-sendfile-smarter

Use sendfile smarter

3 files changed, 363 insertions, 226 deletions

diff --git a/src/basic/copy.c b/src/basic/copy.c
index 519b412941..41dc8ca79a 100644
--- a/src/basic/copy.c
+++ b/src/basic/copy.c
@@ -40,17 +40,38 @@
 #include "fs-util.h"
 #include "io-util.h"
 #include "macro.h"
+#include "missing.h"
 #include "string-util.h"
 #include "strv.h"
 #include "time-util.h"
 #include "umask-util.h"
 #include "xattr-util.h"
 
-#define COPY_BUFFER_SIZE (16*1024)
+#define COPY_BUFFER_SIZE (16*1024u)
+
+static ssize_t try_copy_file_range(int fd_in, loff_t *off_in,
+                                   int fd_out, loff_t *off_out,
+                                   size_t len,
+                                   unsigned int flags) {
+        static int have = -1;
+        ssize_t r;
+
+        if (have == false)
+                return -ENOSYS;
+
+        r = copy_file_range(fd_in, off_in, fd_out, off_out, len, flags);
+        if (_unlikely_(have < 0))
+                have = r >= 0 || errno != ENOSYS;
+        if (r >= 0)
+                return r;
+        else
+                return -errno;
+}
 
 int copy_bytes(int fdf, int fdt, uint64_t max_bytes, bool try_reflink) {
-        bool try_sendfile = true, try_splice = true;
+        bool try_cfr = true, try_sendfile = true, try_splice = true;
         int r;
+        size_t m = SSIZE_MAX; /* that the maximum that sendfile and c_f_r accept */
 
         assert(fdf >= 0);
         assert(fdt >= 0);
@@ -67,11 +88,9 @@ int copy_bytes(int fdf, int fdt, uint64_t max_bytes, bool try_reflink) {
         }
 
         for (;;) {
-                size_t m = COPY_BUFFER_SIZE;
                 ssize_t n;
 
                 if (max_bytes != (uint64_t) -1) {
-
                         if (max_bytes <= 0)
                                 return 1; /* return > 0 if we hit the max_bytes limit */
 
@@ -79,44 +98,59 @@ int copy_bytes(int fdf, int fdt, uint64_t max_bytes, bool try_reflink) {
                                 m = (size_t) max_bytes;
                 }
 
+                /* First try copy_file_range(), unless we already tried */
+                if (try_cfr) {
+                        n = try_copy_file_range(fdf, NULL, fdt, NULL, m, 0u);
+                        if (n < 0) {
+                                if (!IN_SET(n, -EINVAL, -ENOSYS, -EXDEV))
+                                        return n;
+
+                                try_cfr = false;
+                                /* use fallback below */
+                        } else if (n == 0) /* EOF */
+                                break;
+                        else
+                                /* Success! */
+                                goto next;
+                }
+
                 /* First try sendfile(), unless we already tried */
                 if (try_sendfile) {
-
                         n = sendfile(fdt, fdf, NULL, m);
                         if (n < 0) {
-                                if (errno != EINVAL && errno != ENOSYS)
+                                if (!IN_SET(errno, EINVAL, ENOSYS))
                                         return -errno;
 
                                 try_sendfile = false;
                                 /* use fallback below */
                         } else if (n == 0) /* EOF */
                                 break;
-                        else if (n > 0)
+                        else
                                 /* Success! */
                                 goto next;
                 }
 
-                /* The try splice, unless we already tried */
+                /* Then try splice, unless we already tried */
                 if (try_splice) {
                         n = splice(fdf, NULL, fdt, NULL, m, 0);
                         if (n < 0) {
-                                if (errno != EINVAL && errno != ENOSYS)
+                                if (!IN_SET(errno, EINVAL, ENOSYS))
                                         return -errno;
 
                                 try_splice = false;
                                 /* use fallback below */
                         } else if (n == 0) /* EOF */
                                 break;
-                        else if (n > 0)
+                        else
                                 /* Success! */
                                 goto next;
                 }
 
                 /* As a fallback just copy bits by hand */
                 {
-                        uint8_t buf[m];
+                        uint8_t buf[MIN(m, COPY_BUFFER_SIZE)];
 
-                        n = read(fdf, buf, m);
+                        n = read(fdf, buf, sizeof buf);
                         if (n < 0)
                                 return -errno;
                         if (n == 0) /* EOF */
@@ -132,6 +166,11 @@ int copy_bytes(int fdf, int fdt, uint64_t max_bytes, bool try_reflink) {
                         assert(max_bytes >= (uint64_t) n);
                         max_bytes -= n;
                 }
+                /* sendfile accepts at most SSIZE_MAX-offset bytes to copy,
+                 * so reduce our maximum by the amount we already copied,
+                 * but don't go below our copy buffer size, unless we are
+                 * close the the limit of bytes we are allowed to copy. */
+                m = MAX(MIN(COPY_BUFFER_SIZE, max_bytes), m - n);
         }
 
         return 0; /* return 0 if we hit EOF earlier than the size limit */
diff --git a/src/basic/missing.h b/src/basic/missing.h
index 417604aa64..034e334e66 100644
--- a/src/basic/missing.h
+++ b/src/basic/missing.h
@@ -135,84 +135,6 @@
 #define SOL_SCTP 132
 #endif
 
-#if !HAVE_DECL_PIVOT_ROOT
-static inline int pivot_root(const char *new_root, const char *put_old) {
-        return syscall(SYS_pivot_root, new_root, put_old);
-}
-#endif
-
-#ifndef __NR_memfd_create
-#  if defined __x86_64__
-#    define __NR_memfd_create 319
-#  elif defined __arm__
-#    define __NR_memfd_create 385
-#  elif defined __aarch64__
-#    define __NR_memfd_create 279
-#  elif defined __s390__
-#    define __NR_memfd_create 350
-#  elif defined _MIPS_SIM
-#    if _MIPS_SIM == _MIPS_SIM_ABI32
-#      define __NR_memfd_create 4354
-#    endif
-#    if _MIPS_SIM == _MIPS_SIM_NABI32
-#      define __NR_memfd_create 6318
-#    endif
-#    if _MIPS_SIM == _MIPS_SIM_ABI64
-#      define __NR_memfd_create 5314
-#    endif
-#  elif defined __i386__
-#    define __NR_memfd_create 356
-#  else
-#    warning "__NR_memfd_create unknown for your architecture"
-#    define __NR_memfd_create 0xffffffff
-#  endif
-#endif
-
-#if !HAVE_DECL_MEMFD_CREATE
-static inline int memfd_create(const char *name, unsigned int flags) {
-        return syscall(__NR_memfd_create, name, flags);
-}
-#endif
-
-#ifndef __NR_getrandom
-#  if defined __x86_64__
-#    define __NR_getrandom 318
-#  elif defined(__i386__)
-#    define __NR_getrandom 355
-#  elif defined(__arm__)
-#    define __NR_getrandom 384
-# elif defined(__aarch64__)
-#    define __NR_getrandom 278
-#  elif defined(__ia64__)
-#    define __NR_getrandom 1339
-#  elif defined(__m68k__)
-#    define __NR_getrandom 352
-#  elif defined(__s390x__)
-#    define __NR_getrandom 349
-#  elif defined(__powerpc__)
-#    define __NR_getrandom 359
-#  elif defined _MIPS_SIM
-#    if _MIPS_SIM == _MIPS_SIM_ABI32
-#      define __NR_getrandom 4353
-#    endif
-#    if _MIPS_SIM == _MIPS_SIM_NABI32
-#      define __NR_getrandom 6317
-#    endif
-#    if _MIPS_SIM == _MIPS_SIM_ABI64
-#      define __NR_getrandom 5313
-#    endif
-#  else
-#    warning "__NR_getrandom unknown for your architecture"
-#    define __NR_getrandom 0xffffffff
-#  endif
-#endif
-
-#if !HAVE_DECL_GETRANDOM
-static inline int getrandom(void *buffer, size_t count, unsigned flags) {
-        return syscall(__NR_getrandom, buffer, count, flags);
-}
-#endif
-
 #ifndef GRND_NONBLOCK
 #define GRND_NONBLOCK 0x0001
 #endif
@@ -527,12 +449,6 @@ struct btrfs_ioctl_quota_ctl_args {
 #define MS_PRIVATE  (1 << 18)
 #endif
 
-#if !HAVE_DECL_GETTID
-static inline pid_t gettid(void) {
-        return (pid_t) syscall(SYS_gettid);
-}
-#endif
-
 #ifndef SCM_SECURITY
 #define SCM_SECURITY 0x03
 #endif
@@ -561,32 +477,6 @@ static inline pid_t gettid(void) {
 #define MAX_HANDLE_SZ 128
 #endif
 
-#ifndef __NR_name_to_handle_at
-#  if defined(__x86_64__)
-#    define __NR_name_to_handle_at 303
-#  elif defined(__i386__)
-#    define __NR_name_to_handle_at 341
-#  elif defined(__arm__)
-#    define __NR_name_to_handle_at 370
-#  elif defined(__powerpc__)
-#    define __NR_name_to_handle_at 345
-#  else
-#    error "__NR_name_to_handle_at is not defined"
-#  endif
-#endif
-
-#if !HAVE_DECL_NAME_TO_HANDLE_AT
-struct file_handle {
-        unsigned int handle_bytes;
-        int handle_type;
-        unsigned char f_handle[0];
-};
-
-static inline int name_to_handle_at(int fd, const char *name, struct file_handle *handle, int *mnt_id, int flags) {
-        return syscall(__NR_name_to_handle_at, fd, name, handle, mnt_id, flags);
-}
-#endif
-
 #ifndef HAVE_SECURE_GETENV
 #  ifdef HAVE___SECURE_GETENV
 #    define secure_getenv __secure_getenv
@@ -635,22 +525,6 @@ static inline int name_to_handle_at(int fd, const char *name, struct file_handle
 
 #endif
 
-#ifndef __NR_setns
-#  if defined(__x86_64__)
-#    define __NR_setns 308
-#  elif defined(__i386__)
-#    define __NR_setns 346
-#  else
-#    error "__NR_setns is not defined"
-#  endif
-#endif
-
-#if !HAVE_DECL_SETNS
-static inline int setns(int fd, int nstype) {
-        return syscall(__NR_setns, fd, nstype);
-}
-#endif
-
 #if !HAVE_DECL_LO_FLAGS_PARTSCAN
 #define LO_FLAGS_PARTSCAN 8
 #endif
@@ -1018,69 +892,10 @@ static inline int setns(int fd, int nstype) {
 #define CAP_AUDIT_READ 37
 #endif
 
-static inline int raw_clone(unsigned long flags, void *child_stack) {
-#if defined(__s390__) || defined(__CRIS__)
-        /* On s390 and cris the order of the first and second arguments
-         * of the raw clone() system call is reversed. */
-        return (int) syscall(__NR_clone, child_stack, flags);
-#else
-        return (int) syscall(__NR_clone, flags, child_stack);
-#endif
-}
-
-static inline pid_t raw_getpid(void) {
-#if defined(__alpha__)
-        return (pid_t) syscall(__NR_getxpid);
-#else
-        return (pid_t) syscall(__NR_getpid);
-#endif
-}
-
-#if !HAVE_DECL_RENAMEAT2
-
-#ifndef __NR_renameat2
-#  if defined __x86_64__
-#    define __NR_renameat2 316
-#  elif defined __arm__
-#    define __NR_renameat2 382
-#  elif defined _MIPS_SIM
-#    if _MIPS_SIM == _MIPS_SIM_ABI32
-#      define __NR_renameat2 4351
-#    endif
-#    if _MIPS_SIM == _MIPS_SIM_NABI32
-#      define __NR_renameat2 6315
-#    endif
-#    if _MIPS_SIM == _MIPS_SIM_ABI64
-#      define __NR_renameat2 5311
-#    endif
-#  elif defined __i386__
-#    define __NR_renameat2 353
-#  else
-#    warning "__NR_renameat2 unknown for your architecture"
-#    define __NR_renameat2 0xffffffff
-#  endif
-#endif
-
-static inline int renameat2(int oldfd, const char *oldname, int newfd, const char *newname, unsigned flags) {
-        return syscall(__NR_renameat2, oldfd, oldname, newfd, newname, flags);
-}
-#endif
-
 #ifndef RENAME_NOREPLACE
 #define RENAME_NOREPLACE (1 << 0)
 #endif
 
-#if !HAVE_DECL_KCMP
-static inline int kcmp(pid_t pid1, pid_t pid2, int type, unsigned long idx1, unsigned long idx2) {
-#if defined(__NR_kcmp)
-        return syscall(__NR_kcmp, pid1, pid2, type, idx1, idx2);
-#else
-        errno = ENOSYS;
-        return -1;
-#endif
-}
-#endif
-
 #ifndef KCMP_FILE
 #define KCMP_FILE 0
 #endif
@@ -1097,35 +912,6 @@ static inline int kcmp(pid_t pid1, pid_t pid2, int type, unsigned long idx1, uns
 typedef int32_t key_serial_t;
 #endif
 
-#if !HAVE_DECL_KEYCTL
-static inline long keyctl(int cmd, unsigned long arg2, unsigned long arg3, unsigned long arg4,unsigned long arg5) {
-#if defined(__NR_keyctl)
-        return syscall(__NR_keyctl, cmd, arg2, arg3, arg4, arg5);
-#else
-        errno = ENOSYS;
-        return -1;
-#endif
-}
-
-static inline key_serial_t add_key(const char *type, const char *description, const void *payload, size_t plen, key_serial_t ringid) {
-#if defined (__NR_add_key)
-        return syscall(__NR_add_key, type, description, payload, plen, ringid);
-#else
-        errno = ENOSYS;
-        return -1;
-#endif
-}
-
-static inline key_serial_t request_key(const char *type, const char *description, const char * callout_info, key_serial_t destringid) {
-#if defined (__NR_request_key)
-        return syscall(__NR_request_key, type, description, callout_info, destringid);
-#else
-        errno = ENOSYS;
-        return -1;
-#endif
-}
-#endif
-
 #ifndef KEYCTL_READ
 #define KEYCTL_READ 11
 #endif
@@ -1177,3 +963,5 @@ static inline key_serial_t request_key(const char *type, const char *description
 #endif
 
 #endif
+
+#include "missing_syscall.h"
diff --git a/src/basic/missing_syscall.h b/src/basic/missing_syscall.h
new file mode 100644
index 0000000000..d502d3b9ca
--- /dev/null
+++ b/src/basic/missing_syscall.h
@@ -0,0 +1,310 @@
+#pragma once
+
+/***
+  This file is part of systemd.
+
+  Copyright 2010 Lennart Poettering
+  Copyright 2016 Zbigniew Jędrzejewski-Szmek
+
+  systemd is free software; you can redistribute it and/or modify it
+  under the terms of the GNU Lesser General Public License as published by
+  the Free Software Foundation; either version 2.1 of the License, or
+  (at your option) any later version.
+
+  systemd is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public License
+  along with systemd; If not, see <http://www.gnu.org/licenses/>.
+***/
+
+/* Missing glibc definitions to access certain kernel APIs */
+
+#if !HAVE_DECL_PIVOT_ROOT
+static inline int pivot_root(const char *new_root, const char *put_old) {
+        return syscall(SYS_pivot_root, new_root, put_old);
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_MEMFD_CREATE
+#  ifndef __NR_memfd_create
+#    if defined __x86_64__
+#      define __NR_memfd_create 319
+#    elif defined __arm__
+#      define __NR_memfd_create 385
+#    elif defined __aarch64__
+#      define __NR_memfd_create 279
+#    elif defined __s390__
+#      define __NR_memfd_create 350
+#    elif defined _MIPS_SIM
+#      if _MIPS_SIM == _MIPS_SIM_ABI32
+#        define __NR_memfd_create 4354
+#      endif
+#      if _MIPS_SIM == _MIPS_SIM_NABI32
+#        define __NR_memfd_create 6318
+#      endif
+#      if _MIPS_SIM == _MIPS_SIM_ABI64
+#        define __NR_memfd_create 5314
+#      endif
+#    elif defined __i386__
+#      define __NR_memfd_create 356
+#    else
+#      warning "__NR_memfd_create unknown for your architecture"
+#    endif
+#  endif
+
+static inline int memfd_create(const char *name, unsigned int flags) {
+#  ifdef __NR_memfd_create
+        return syscall(__NR_memfd_create, name, flags);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_GETRANDOM
+#  ifndef __NR_getrandom
+#    if defined __x86_64__
+#      define __NR_getrandom 318
+#    elif defined(__i386__)
+#      define __NR_getrandom 355
+#    elif defined(__arm__)
+#      define __NR_getrandom 384
+#   elif defined(__aarch64__)
+#      define __NR_getrandom 278
+#    elif defined(__ia64__)
+#      define __NR_getrandom 1339
+#    elif defined(__m68k__)
+#      define __NR_getrandom 352
+#    elif defined(__s390x__)
+#      define __NR_getrandom 349
+#    elif defined(__powerpc__)
+#      define __NR_getrandom 359
+#    elif defined _MIPS_SIM
+#      if _MIPS_SIM == _MIPS_SIM_ABI32
+#        define __NR_getrandom 4353
+#      endif
+#      if _MIPS_SIM == _MIPS_SIM_NABI32
+#        define __NR_getrandom 6317
+#      endif
+#      if _MIPS_SIM == _MIPS_SIM_ABI64
+#        define __NR_getrandom 5313
+#      endif
+#    else
+#      warning "__NR_getrandom unknown for your architecture"
+#    endif
+#  endif
+
+static inline int getrandom(void *buffer, size_t count, unsigned flags) {
+#  ifdef __NR_getrandom
+        return syscall(__NR_getrandom, buffer, count, flags);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_GETTID
+static inline pid_t gettid(void) {
+        return (pid_t) syscall(SYS_gettid);
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_NAME_TO_HANDLE_AT
+#  ifndef __NR_name_to_handle_at
+#    if defined(__x86_64__)
+#      define __NR_name_to_handle_at 303
+#    elif defined(__i386__)
+#      define __NR_name_to_handle_at 341
+#    elif defined(__arm__)
+#      define __NR_name_to_handle_at 370
+#    elif defined(__powerpc__)
+#      define __NR_name_to_handle_at 345
+#    else
+#      error "__NR_name_to_handle_at is not defined"
+#    endif
+#  endif
+
+struct file_handle {
+        unsigned int handle_bytes;
+        int handle_type;
+        unsigned char f_handle[0];
+};
+
+static inline int name_to_handle_at(int fd, const char *name, struct file_handle *handle, int *mnt_id, int flags) {
+#  ifdef __NR_name_to_handle_at
+        return syscall(__NR_name_to_handle_at, fd, name, handle, mnt_id, flags);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_SETNS
+#  ifndef __NR_setns
+#    if defined(__x86_64__)
+#      define __NR_setns 308
+#    elif defined(__i386__)
+#      define __NR_setns 346
+#    else
+#      error "__NR_setns is not defined"
+#    endif
+#  endif
+
+static inline int setns(int fd, int nstype) {
+#  ifdef __NR_setns
+        return syscall(__NR_setns, fd, nstype);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+#endif
+
+/* ======================================================================= */
+
+static inline int raw_clone(unsigned long flags, void *child_stack) {
+#if defined(__s390__) || defined(__CRIS__)
+        /* On s390 and cris the order of the first and second arguments
+         * of the raw clone() system call is reversed. */
+        return (int) syscall(__NR_clone, child_stack, flags);
+#else
+        return (int) syscall(__NR_clone, flags, child_stack);
+#endif
+}
+
+/* ======================================================================= */
+
+static inline pid_t raw_getpid(void) {
+#if defined(__alpha__)
+        return (pid_t) syscall(__NR_getxpid);
+#else
+        return (pid_t) syscall(__NR_getpid);
+#endif
+}
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_RENAMEAT2
+#  ifndef __NR_renameat2
+#    if defined __x86_64__
+#      define __NR_renameat2 316
+#    elif defined __arm__
+#      define __NR_renameat2 382
+#    elif defined _MIPS_SIM
+#      if _MIPS_SIM == _MIPS_SIM_ABI32
+#        define __NR_renameat2 4351
+#      endif
+#      if _MIPS_SIM == _MIPS_SIM_NABI32
+#        define __NR_renameat2 6315
+#      endif
+#      if _MIPS_SIM == _MIPS_SIM_ABI64
+#        define __NR_renameat2 5311
+#      endif
+#    elif defined __i386__
+#      define __NR_renameat2 353
+#    else
+#      warning "__NR_renameat2 unknown for your architecture"
+#    endif
+#  endif
+
+static inline int renameat2(int oldfd, const char *oldname, int newfd, const char *newname, unsigned flags) {
+#  ifdef __NR_renameat2
+        return syscall(__NR_renameat2, oldfd, oldname, newfd, newname, flags);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_KCMP
+static inline int kcmp(pid_t pid1, pid_t pid2, int type, unsigned long idx1, unsigned long idx2) {
+#  ifdef __NR_kcmp
+        return syscall(__NR_kcmp, pid1, pid2, type, idx1, idx2);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_KEYCTL
+static inline long keyctl(int cmd, unsigned long arg2, unsigned long arg3, unsigned long arg4,unsigned long arg5) {
+#  ifdef __NR_keyctl
+        return syscall(__NR_keyctl, cmd, arg2, arg3, arg4, arg5);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+
+static inline key_serial_t add_key(const char *type, const char *description, const void *payload, size_t plen, key_serial_t ringid) {
+#  ifdef __NR_add_key
+        return syscall(__NR_add_key, type, description, payload, plen, ringid);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+
+static inline key_serial_t request_key(const char *type, const char *description, const char * callout_info, key_serial_t destringid) {
+#  ifdef __NR_request_key
+        return syscall(__NR_request_key, type, description, callout_info, destringid);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_COPY_FILE_RANGE
+#  ifndef __NR_copy_file_range
+#    if defined(__x86_64__)
+#      define __NR_copy_file_range 326
+#    elif defined(__i386__)
+#      define __NR_copy_file_range 377
+#    elif defined __s390__
+#      define __NR_copy_file_range 375
+#    elif defined __arm__
+#      define __NR_copy_file_range 391
+#    elif defined __aarch64__
+#      define __NR_copy_file_range 285
+#    else
+#      warning "__NR_copy_file_range not defined for your architecture"
+#    endif
+#  endif
+
+static inline ssize_t copy_file_range(int fd_in, loff_t *off_in,
+                                      int fd_out, loff_t *off_out,
+                                      size_t len,
+                                      unsigned int flags) {
+#  ifdef __NR_copy_file_range
+        return syscall(__NR_copy_file_range, fd_in, off_in, fd_out, off_out, len, flags);
+#  else
+        errno = ENOSYS;
+        return -1;
+#  endif
+}
+#endif