summaryrefslogtreecommitdiff
path: root/src/basic
diff options
context:
space:
mode:
authorDaniel Mack <github@zonque.org>2016-03-17 20:33:41 +0100
committerDaniel Mack <github@zonque.org>2016-03-17 20:33:41 +0100
commit5fec580c84b6f0ac055909ce1f9ce9efab079943 (patch)
tree9efcdd0eabc63ccebeb4814a2cb1fdb12839dd39 /src/basic
parentda9a4daa08f84db68f7620d2e926690542c31689 (diff)
parent2f368e4a114888dae8c7ebb1020efc3d922b61e9 (diff)
Merge pull request #2839 from keszybz/use-sendfile-smarter
Use sendfile smarter
Diffstat (limited to 'src/basic')
-rw-r--r--src/basic/copy.c63
-rw-r--r--src/basic/missing.h216
-rw-r--r--src/basic/missing_syscall.h310
3 files changed, 363 insertions, 226 deletions
diff --git a/src/basic/copy.c b/src/basic/copy.c
index 519b412941..41dc8ca79a 100644
--- a/src/basic/copy.c
+++ b/src/basic/copy.c
@@ -40,17 +40,38 @@
#include "fs-util.h"
#include "io-util.h"
#include "macro.h"
+#include "missing.h"
#include "string-util.h"
#include "strv.h"
#include "time-util.h"
#include "umask-util.h"
#include "xattr-util.h"
-#define COPY_BUFFER_SIZE (16*1024)
+#define COPY_BUFFER_SIZE (16*1024u)
+
+static ssize_t try_copy_file_range(int fd_in, loff_t *off_in,
+ int fd_out, loff_t *off_out,
+ size_t len,
+ unsigned int flags) {
+ static int have = -1;
+ ssize_t r;
+
+ if (have == false)
+ return -ENOSYS;
+
+ r = copy_file_range(fd_in, off_in, fd_out, off_out, len, flags);
+ if (_unlikely_(have < 0))
+ have = r >= 0 || errno != ENOSYS;
+ if (r >= 0)
+ return r;
+ else
+ return -errno;
+}
int copy_bytes(int fdf, int fdt, uint64_t max_bytes, bool try_reflink) {
- bool try_sendfile = true, try_splice = true;
+ bool try_cfr = true, try_sendfile = true, try_splice = true;
int r;
+ size_t m = SSIZE_MAX; /* that the maximum that sendfile and c_f_r accept */
assert(fdf >= 0);
assert(fdt >= 0);
@@ -67,11 +88,9 @@ int copy_bytes(int fdf, int fdt, uint64_t max_bytes, bool try_reflink) {
}
for (;;) {
- size_t m = COPY_BUFFER_SIZE;
ssize_t n;
if (max_bytes != (uint64_t) -1) {
-
if (max_bytes <= 0)
return 1; /* return > 0 if we hit the max_bytes limit */
@@ -79,44 +98,59 @@ int copy_bytes(int fdf, int fdt, uint64_t max_bytes, bool try_reflink) {
m = (size_t) max_bytes;
}
+ /* First try copy_file_range(), unless we already tried */
+ if (try_cfr) {
+ n = try_copy_file_range(fdf, NULL, fdt, NULL, m, 0u);
+ if (n < 0) {
+ if (!IN_SET(n, -EINVAL, -ENOSYS, -EXDEV))
+ return n;
+
+ try_cfr = false;
+ /* use fallback below */
+ } else if (n == 0) /* EOF */
+ break;
+ else
+ /* Success! */
+ goto next;
+ }
+
/* First try sendfile(), unless we already tried */
if (try_sendfile) {
-
n = sendfile(fdt, fdf, NULL, m);
if (n < 0) {
- if (errno != EINVAL && errno != ENOSYS)
+ if (!IN_SET(errno, EINVAL, ENOSYS))
return -errno;
try_sendfile = false;
/* use fallback below */
} else if (n == 0) /* EOF */
break;
- else if (n > 0)
+ else
/* Success! */
goto next;
}
- /* The try splice, unless we already tried */
+ /* Then try splice, unless we already tried */
if (try_splice) {
n = splice(fdf, NULL, fdt, NULL, m, 0);
if (n < 0) {
- if (errno != EINVAL && errno != ENOSYS)
+ if (!IN_SET(errno, EINVAL, ENOSYS))
return -errno;
try_splice = false;
/* use fallback below */
} else if (n == 0) /* EOF */
break;
- else if (n > 0)
+ else
/* Success! */
goto next;
}
/* As a fallback just copy bits by hand */
{
- uint8_t buf[m];
+ uint8_t buf[MIN(m, COPY_BUFFER_SIZE)];
- n = read(fdf, buf, m);
+ n = read(fdf, buf, sizeof buf);
if (n < 0)
return -errno;
if (n == 0) /* EOF */
@@ -132,6 +166,11 @@ int copy_bytes(int fdf, int fdt, uint64_t max_bytes, bool try_reflink) {
assert(max_bytes >= (uint64_t) n);
max_bytes -= n;
}
+ /* sendfile accepts at most SSIZE_MAX-offset bytes to copy,
+ * so reduce our maximum by the amount we already copied,
+ * but don't go below our copy buffer size, unless we are
+ * close the the limit of bytes we are allowed to copy. */
+ m = MAX(MIN(COPY_BUFFER_SIZE, max_bytes), m - n);
}
return 0; /* return 0 if we hit EOF earlier than the size limit */
diff --git a/src/basic/missing.h b/src/basic/missing.h
index 417604aa64..034e334e66 100644
--- a/src/basic/missing.h
+++ b/src/basic/missing.h
@@ -135,84 +135,6 @@
#define SOL_SCTP 132
#endif
-#if !HAVE_DECL_PIVOT_ROOT
-static inline int pivot_root(const char *new_root, const char *put_old) {
- return syscall(SYS_pivot_root, new_root, put_old);
-}
-#endif
-
-#ifndef __NR_memfd_create
-# if defined __x86_64__
-# define __NR_memfd_create 319
-# elif defined __arm__
-# define __NR_memfd_create 385
-# elif defined __aarch64__
-# define __NR_memfd_create 279
-# elif defined __s390__
-# define __NR_memfd_create 350
-# elif defined _MIPS_SIM
-# if _MIPS_SIM == _MIPS_SIM_ABI32
-# define __NR_memfd_create 4354
-# endif
-# if _MIPS_SIM == _MIPS_SIM_NABI32
-# define __NR_memfd_create 6318
-# endif
-# if _MIPS_SIM == _MIPS_SIM_ABI64
-# define __NR_memfd_create 5314
-# endif
-# elif defined __i386__
-# define __NR_memfd_create 356
-# else
-# warning "__NR_memfd_create unknown for your architecture"
-# define __NR_memfd_create 0xffffffff
-# endif
-#endif
-
-#if !HAVE_DECL_MEMFD_CREATE
-static inline int memfd_create(const char *name, unsigned int flags) {
- return syscall(__NR_memfd_create, name, flags);
-}
-#endif
-
-#ifndef __NR_getrandom
-# if defined __x86_64__
-# define __NR_getrandom 318
-# elif defined(__i386__)
-# define __NR_getrandom 355
-# elif defined(__arm__)
-# define __NR_getrandom 384
-# elif defined(__aarch64__)
-# define __NR_getrandom 278
-# elif defined(__ia64__)
-# define __NR_getrandom 1339
-# elif defined(__m68k__)
-# define __NR_getrandom 352
-# elif defined(__s390x__)
-# define __NR_getrandom 349
-# elif defined(__powerpc__)
-# define __NR_getrandom 359
-# elif defined _MIPS_SIM
-# if _MIPS_SIM == _MIPS_SIM_ABI32
-# define __NR_getrandom 4353
-# endif
-# if _MIPS_SIM == _MIPS_SIM_NABI32
-# define __NR_getrandom 6317
-# endif
-# if _MIPS_SIM == _MIPS_SIM_ABI64
-# define __NR_getrandom 5313
-# endif
-# else
-# warning "__NR_getrandom unknown for your architecture"
-# define __NR_getrandom 0xffffffff
-# endif
-#endif
-
-#if !HAVE_DECL_GETRANDOM
-static inline int getrandom(void *buffer, size_t count, unsigned flags) {
- return syscall(__NR_getrandom, buffer, count, flags);
-}
-#endif
-
#ifndef GRND_NONBLOCK
#define GRND_NONBLOCK 0x0001
#endif
@@ -527,12 +449,6 @@ struct btrfs_ioctl_quota_ctl_args {
#define MS_PRIVATE (1 << 18)
#endif
-#if !HAVE_DECL_GETTID
-static inline pid_t gettid(void) {
- return (pid_t) syscall(SYS_gettid);
-}
-#endif
-
#ifndef SCM_SECURITY
#define SCM_SECURITY 0x03
#endif
@@ -561,32 +477,6 @@ static inline pid_t gettid(void) {
#define MAX_HANDLE_SZ 128
#endif
-#ifndef __NR_name_to_handle_at
-# if defined(__x86_64__)
-# define __NR_name_to_handle_at 303
-# elif defined(__i386__)
-# define __NR_name_to_handle_at 341
-# elif defined(__arm__)
-# define __NR_name_to_handle_at 370
-# elif defined(__powerpc__)
-# define __NR_name_to_handle_at 345
-# else
-# error "__NR_name_to_handle_at is not defined"
-# endif
-#endif
-
-#if !HAVE_DECL_NAME_TO_HANDLE_AT
-struct file_handle {
- unsigned int handle_bytes;
- int handle_type;
- unsigned char f_handle[0];
-};
-
-static inline int name_to_handle_at(int fd, const char *name, struct file_handle *handle, int *mnt_id, int flags) {
- return syscall(__NR_name_to_handle_at, fd, name, handle, mnt_id, flags);
-}
-#endif
-
#ifndef HAVE_SECURE_GETENV
# ifdef HAVE___SECURE_GETENV
# define secure_getenv __secure_getenv
@@ -635,22 +525,6 @@ static inline int name_to_handle_at(int fd, const char *name, struct file_handle
#endif
-#ifndef __NR_setns
-# if defined(__x86_64__)
-# define __NR_setns 308
-# elif defined(__i386__)
-# define __NR_setns 346
-# else
-# error "__NR_setns is not defined"
-# endif
-#endif
-
-#if !HAVE_DECL_SETNS
-static inline int setns(int fd, int nstype) {
- return syscall(__NR_setns, fd, nstype);
-}
-#endif
-
#if !HAVE_DECL_LO_FLAGS_PARTSCAN
#define LO_FLAGS_PARTSCAN 8
#endif
@@ -1018,69 +892,10 @@ static inline int setns(int fd, int nstype) {
#define CAP_AUDIT_READ 37
#endif
-static inline int raw_clone(unsigned long flags, void *child_stack) {
-#if defined(__s390__) || defined(__CRIS__)
- /* On s390 and cris the order of the first and second arguments
- * of the raw clone() system call is reversed. */
- return (int) syscall(__NR_clone, child_stack, flags);
-#else
- return (int) syscall(__NR_clone, flags, child_stack);
-#endif
-}
-
-static inline pid_t raw_getpid(void) {
-#if defined(__alpha__)
- return (pid_t) syscall(__NR_getxpid);
-#else
- return (pid_t) syscall(__NR_getpid);
-#endif
-}
-
-#if !HAVE_DECL_RENAMEAT2
-
-#ifndef __NR_renameat2
-# if defined __x86_64__
-# define __NR_renameat2 316
-# elif defined __arm__
-# define __NR_renameat2 382
-# elif defined _MIPS_SIM
-# if _MIPS_SIM == _MIPS_SIM_ABI32
-# define __NR_renameat2 4351
-# endif
-# if _MIPS_SIM == _MIPS_SIM_NABI32
-# define __NR_renameat2 6315
-# endif
-# if _MIPS_SIM == _MIPS_SIM_ABI64
-# define __NR_renameat2 5311
-# endif
-# elif defined __i386__
-# define __NR_renameat2 353
-# else
-# warning "__NR_renameat2 unknown for your architecture"
-# define __NR_renameat2 0xffffffff
-# endif
-#endif
-
-static inline int renameat2(int oldfd, const char *oldname, int newfd, const char *newname, unsigned flags) {
- return syscall(__NR_renameat2, oldfd, oldname, newfd, newname, flags);
-}
-#endif
-
#ifndef RENAME_NOREPLACE
#define RENAME_NOREPLACE (1 << 0)
#endif
-#if !HAVE_DECL_KCMP
-static inline int kcmp(pid_t pid1, pid_t pid2, int type, unsigned long idx1, unsigned long idx2) {
-#if defined(__NR_kcmp)
- return syscall(__NR_kcmp, pid1, pid2, type, idx1, idx2);
-#else
- errno = ENOSYS;
- return -1;
-#endif
-}
-#endif
-
#ifndef KCMP_FILE
#define KCMP_FILE 0
#endif
@@ -1097,35 +912,6 @@ static inline int kcmp(pid_t pid1, pid_t pid2, int type, unsigned long idx1, uns
typedef int32_t key_serial_t;
#endif
-#if !HAVE_DECL_KEYCTL
-static inline long keyctl(int cmd, unsigned long arg2, unsigned long arg3, unsigned long arg4,unsigned long arg5) {
-#if defined(__NR_keyctl)
- return syscall(__NR_keyctl, cmd, arg2, arg3, arg4, arg5);
-#else
- errno = ENOSYS;
- return -1;
-#endif
-}
-
-static inline key_serial_t add_key(const char *type, const char *description, const void *payload, size_t plen, key_serial_t ringid) {
-#if defined (__NR_add_key)
- return syscall(__NR_add_key, type, description, payload, plen, ringid);
-#else
- errno = ENOSYS;
- return -1;
-#endif
-}
-
-static inline key_serial_t request_key(const char *type, const char *description, const char * callout_info, key_serial_t destringid) {
-#if defined (__NR_request_key)
- return syscall(__NR_request_key, type, description, callout_info, destringid);
-#else
- errno = ENOSYS;
- return -1;
-#endif
-}
-#endif
-
#ifndef KEYCTL_READ
#define KEYCTL_READ 11
#endif
@@ -1177,3 +963,5 @@ static inline key_serial_t request_key(const char *type, const char *description
#endif
#endif
+
+#include "missing_syscall.h"
diff --git a/src/basic/missing_syscall.h b/src/basic/missing_syscall.h
new file mode 100644
index 0000000000..d502d3b9ca
--- /dev/null
+++ b/src/basic/missing_syscall.h
@@ -0,0 +1,310 @@
+#pragma once
+
+/***
+ This file is part of systemd.
+
+ Copyright 2010 Lennart Poettering
+ Copyright 2016 Zbigniew Jędrzejewski-Szmek
+
+ systemd is free software; you can redistribute it and/or modify it
+ under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation; either version 2.1 of the License, or
+ (at your option) any later version.
+
+ systemd is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with systemd; If not, see <http://www.gnu.org/licenses/>.
+***/
+
+/* Missing glibc definitions to access certain kernel APIs */
+
+#if !HAVE_DECL_PIVOT_ROOT
+static inline int pivot_root(const char *new_root, const char *put_old) {
+ return syscall(SYS_pivot_root, new_root, put_old);
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_MEMFD_CREATE
+# ifndef __NR_memfd_create
+# if defined __x86_64__
+# define __NR_memfd_create 319
+# elif defined __arm__
+# define __NR_memfd_create 385
+# elif defined __aarch64__
+# define __NR_memfd_create 279
+# elif defined __s390__
+# define __NR_memfd_create 350
+# elif defined _MIPS_SIM
+# if _MIPS_SIM == _MIPS_SIM_ABI32
+# define __NR_memfd_create 4354
+# endif
+# if _MIPS_SIM == _MIPS_SIM_NABI32
+# define __NR_memfd_create 6318
+# endif
+# if _MIPS_SIM == _MIPS_SIM_ABI64
+# define __NR_memfd_create 5314
+# endif
+# elif defined __i386__
+# define __NR_memfd_create 356
+# else
+# warning "__NR_memfd_create unknown for your architecture"
+# endif
+# endif
+
+static inline int memfd_create(const char *name, unsigned int flags) {
+# ifdef __NR_memfd_create
+ return syscall(__NR_memfd_create, name, flags);
+# else
+ errno = ENOSYS;
+ return -1;
+# endif
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_GETRANDOM
+# ifndef __NR_getrandom
+# if defined __x86_64__
+# define __NR_getrandom 318
+# elif defined(__i386__)
+# define __NR_getrandom 355
+# elif defined(__arm__)
+# define __NR_getrandom 384
+# elif defined(__aarch64__)
+# define __NR_getrandom 278
+# elif defined(__ia64__)
+# define __NR_getrandom 1339
+# elif defined(__m68k__)
+# define __NR_getrandom 352
+# elif defined(__s390x__)
+# define __NR_getrandom 349
+# elif defined(__powerpc__)
+# define __NR_getrandom 359
+# elif defined _MIPS_SIM
+# if _MIPS_SIM == _MIPS_SIM_ABI32
+# define __NR_getrandom 4353
+# endif
+# if _MIPS_SIM == _MIPS_SIM_NABI32
+# define __NR_getrandom 6317
+# endif
+# if _MIPS_SIM == _MIPS_SIM_ABI64
+# define __NR_getrandom 5313
+# endif
+# else
+# warning "__NR_getrandom unknown for your architecture"
+# endif
+# endif
+
+static inline int getrandom(void *buffer, size_t count, unsigned flags) {
+# ifdef __NR_getrandom
+ return syscall(__NR_getrandom, buffer, count, flags);
+# else
+ errno = ENOSYS;
+ return -1;
+# endif
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_GETTID
+static inline pid_t gettid(void) {
+ return (pid_t) syscall(SYS_gettid);
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_NAME_TO_HANDLE_AT
+# ifndef __NR_name_to_handle_at
+# if defined(__x86_64__)
+# define __NR_name_to_handle_at 303
+# elif defined(__i386__)
+# define __NR_name_to_handle_at 341
+# elif defined(__arm__)
+# define __NR_name_to_handle_at 370
+# elif defined(__powerpc__)
+# define __NR_name_to_handle_at 345
+# else
+# error "__NR_name_to_handle_at is not defined"
+# endif
+# endif
+
+struct file_handle {
+ unsigned int handle_bytes;
+ int handle_type;
+ unsigned char f_handle[0];
+};
+
+static inline int name_to_handle_at(int fd, const char *name, struct file_handle *handle, int *mnt_id, int flags) {
+# ifdef __NR_name_to_handle_at
+ return syscall(__NR_name_to_handle_at, fd, name, handle, mnt_id, flags);
+# else
+ errno = ENOSYS;
+ return -1;
+# endif
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_SETNS
+# ifndef __NR_setns
+# if defined(__x86_64__)
+# define __NR_setns 308
+# elif defined(__i386__)
+# define __NR_setns 346
+# else
+# error "__NR_setns is not defined"
+# endif
+# endif
+
+static inline int setns(int fd, int nstype) {
+# ifdef __NR_setns
+ return syscall(__NR_setns, fd, nstype);
+# else
+ errno = ENOSYS;
+ return -1;
+# endif
+}
+#endif
+
+/* ======================================================================= */
+
+static inline int raw_clone(unsigned long flags, void *child_stack) {
+#if defined(__s390__) || defined(__CRIS__)
+ /* On s390 and cris the order of the first and second arguments
+ * of the raw clone() system call is reversed. */
+ return (int) syscall(__NR_clone, child_stack, flags);
+#else
+ return (int) syscall(__NR_clone, flags, child_stack);
+#endif
+}
+
+/* ======================================================================= */
+
+static inline pid_t raw_getpid(void) {
+#if defined(__alpha__)
+ return (pid_t) syscall(__NR_getxpid);
+#else
+ return (pid_t) syscall(__NR_getpid);
+#endif
+}
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_RENAMEAT2
+# ifndef __NR_renameat2
+# if defined __x86_64__
+# define __NR_renameat2 316
+# elif defined __arm__
+# define __NR_renameat2 382
+# elif defined _MIPS_SIM
+# if _MIPS_SIM == _MIPS_SIM_ABI32
+# define __NR_renameat2 4351
+# endif
+# if _MIPS_SIM == _MIPS_SIM_NABI32
+# define __NR_renameat2 6315
+# endif
+# if _MIPS_SIM == _MIPS_SIM_ABI64
+# define __NR_renameat2 5311
+# endif
+# elif defined __i386__
+# define __NR_renameat2 353
+# else
+# warning "__NR_renameat2 unknown for your architecture"
+# endif
+# endif
+
+static inline int renameat2(int oldfd, const char *oldname, int newfd, const char *newname, unsigned flags) {
+# ifdef __NR_renameat2
+ return syscall(__NR_renameat2, oldfd, oldname, newfd, newname, flags);
+# else
+ errno = ENOSYS;
+ return -1;
+# endif
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_KCMP
+static inline int kcmp(pid_t pid1, pid_t pid2, int type, unsigned long idx1, unsigned long idx2) {
+# ifdef __NR_kcmp
+ return syscall(__NR_kcmp, pid1, pid2, type, idx1, idx2);
+# else
+ errno = ENOSYS;
+ return -1;
+# endif
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_KEYCTL
+static inline long keyctl(int cmd, unsigned long arg2, unsigned long arg3, unsigned long arg4,unsigned long arg5) {
+# ifdef __NR_keyctl
+ return syscall(__NR_keyctl, cmd, arg2, arg3, arg4, arg5);
+# else
+ errno = ENOSYS;
+ return -1;
+# endif
+}
+
+static inline key_serial_t add_key(const char *type, const char *description, const void *payload, size_t plen, key_serial_t ringid) {
+# ifdef __NR_add_key
+ return syscall(__NR_add_key, type, description, payload, plen, ringid);
+# else
+ errno = ENOSYS;
+ return -1;
+# endif
+}
+
+static inline key_serial_t request_key(const char *type, const char *description, const char * callout_info, key_serial_t destringid) {
+# ifdef __NR_request_key
+ return syscall(__NR_request_key, type, description, callout_info, destringid);
+# else
+ errno = ENOSYS;
+ return -1;
+# endif
+}
+#endif
+
+/* ======================================================================= */
+
+#if !HAVE_DECL_COPY_FILE_RANGE
+# ifndef __NR_copy_file_range
+# if defined(__x86_64__)
+# define __NR_copy_file_range 326
+# elif defined(__i386__)
+# define __NR_copy_file_range 377
+# elif defined __s390__
+# define __NR_copy_file_range 375
+# elif defined __arm__
+# define __NR_copy_file_range 391
+# elif defined __aarch64__
+# define __NR_copy_file_range 285
+# else
+# warning "__NR_copy_file_range not defined for your architecture"
+# endif
+# endif
+
+static inline ssize_t copy_file_range(int fd_in, loff_t *off_in,
+ int fd_out, loff_t *off_out,
+ size_t len,
+ unsigned int flags) {
+# ifdef __NR_copy_file_range
+ return syscall(__NR_copy_file_range, fd_in, off_in, fd_out, off_out, len, flags);
+# else
+ errno = ENOSYS;
+ return -1;
+# endif
+}
+#endif