Use copy_file_range() for transfers if it is available (#196).

This commit is contained in:
Andrew Wood
2026-05-31 22:18:18 +01:00
parent 85918358a5
commit 6130d65249
9 changed files with 557 additions and 185 deletions
+13
View File
@@ -143,6 +143,15 @@ AC_ARG_ENABLE([splice],
SPLICE_SUPPORT="yes"
)
COPY_FILE_RANGE_SUPPORT="no"
AC_ARG_ENABLE([copy-file-range],
[AS_HELP_STRING([--disable-copy-file-range], [do not use copy_file_range system call])],
if test "$enable_copy_file_range" = "yes"; then
COPY_FILE_RANGE_SUPPORT="yes"
fi,
COPY_FILE_RANGE_SUPPORT="yes"
)
IPC_SUPPORT="no"
AC_ARG_ENABLE([ipc],
[AS_HELP_STRING([--disable-ipc], [turn off IPC messaging])],
@@ -197,6 +206,10 @@ if test "$SPLICE_SUPPORT" = "yes"; then
AC_CHECK_FUNCS([splice])
fi
if test "$COPY_FILE_RANGE_SUPPORT" = "yes"; then
AC_CHECK_FUNCS([copy_file_range])
fi
CPPFLAGS="$CPPFLAGS -I\$(top_srcdir)/src/include"
dnl This must go after all the compiler based tests above.
+24 -14
View File
@@ -290,23 +290,26 @@ better with specific buffer sizes such as 1024.
Implies \*(lq\fB\-\-no\-splice\fR\*(rq.
.TP
.B \-C, \-\-no-splice
Prevent the use of \fBsplice\fR(2), a system call normally used on systems
that support it to improve efficiency by eliminating the need for a transfer
buffer.
Prevent the use of \fBsplice\fR(2) and \fBcopy_file_range\fR(2), which are
normally used on systems that support it to improve efficiency by
eliminating the need for a transfer buffer.
.IP
In the default case that \fBsplice\fR(2) is used, the lack of a transfer
buffer prevents \*(lq\fB\-\-buffer\-percent\fR\*(rq and
In the default case that \fBsplice\fR(2) or \fBcopy_file_range\fR(2) are
used, the lack of a transfer buffer prevents
\*(lq\fB\-\-buffer\-percent\fR\*(rq and
\*(lq\fB\-\-last\-written\fR\*(rq from working and makes
\*(lq\fB\-\-buffer\-size\fR\*(rq redundant, so using any of those options
automatically switches on \*(lq\fB\-\-no\-splice\fR\*(rq.
.IP
Sparse files cannot be written with \fBsplice\fR(2), so using
Sparse files cannot be written with \fBsplice\fR(2) or
\fBcopy_file_range\fR(2), so using
\*(lq\fB\-\-sparse\fR\*(rq automatically switches on
\*(lq\fB\-\-no\-splice\fR\*(rq.
.IP
Switching on \*(lq\fB\-\-no\-splice\fR\*(rq results in a small loss of
transfer efficiency.
It has no effect on systems where \fBsplice\fR(2) is unavailable.
It has no effect on systems where \fBsplice\fR(2) and
\fBcopy_file_range\fR(2) are unavailable.
.TP
.BI \-J\ BYTES \fR,\ \fB\-\-pipe\-buffer\-size\ BYTES
Attempt to set the size of the output pipe buffer, and the intermediate
@@ -349,7 +352,7 @@ end of input.
.TP
.B \-Y, \-\-sync
After every write operation, synchronise the buffer caches to disk with
\fBfdatasync\fR(2).
\fBsync\fR(2), or \fBfdatasync\fR(2) if possible.
This has no effect when the output is a pipe.
Using \*(lq\fB\-\-sync\fR\*(rq may improve the accuracy of the progress bar
when writing to a slow disk.
@@ -562,21 +565,26 @@ transferred by this side, in \*(lq\fB\-\-monitor\~both\fR\*(rq mode.
.TP
.BR %T ", " %{buffer\-percent}
Percentage of the transfer buffer in use.
Equivalent to \*(lq\fB\-\-buffer\-percent\fR\*(rq.
Displays \*(lq{\-\-\-\-}\*(rq if the transfer is being done with
\fBsplice\fR(2), since splicing to or from pipes does not use the buffer.
Equivalent to \*(lq\fB\-\-buffer\-percent\fR\*(rq, but using it does not
turn on \*(lq\fB\-\-no\-splice\fR\*(rq.
.IP
If the transfer is using \fBsplice\fR(2), displays \*(lq{\-\-\-\-}\*(rq; if
the transfer is using \fBsplice\fR(2) via an intermediate input pipe,
displays \*(lq{\-||\-}\*(rq; and if the transfer is using
\fBcopy_file_range\fR(2), displays \*(lq{\-\-\->}\*(rq.
.TP
.BR %nA ", " %n{last\-written}
Show the last \fIn\fR bytes written (for example, \*(lq\fB%16A\fR\*(rq shows
the last 16 bytes).
Shows only dots if the transfer is being done with \fBsplice\fR(2), since
splicing to or from pipes does not use the buffer.
Shows only dots if the transfer is using \fBsplice\fR(2) or
\fBcopy_file_range\fR(2), since these transfer modes do not use the buffer.
.TP
.BR %nL ", " %n{previous\-line}
Show the first \fIn\fR bytes of the most recently written line (for example,
\*(lq\fB%40L\fR\*(rq shows the first 40 bytes).
If no \fIn\fR is given, then this expands to fill the available space.
Shows only spaces if the transfer is being done with \fBsplice\fR(2).
Shows only spaces if the transfer is using \fBsplice\fR(2) or
\fBcopy_file_range\fR(2).
.TP
.BR %N ", " %{name}
Show the name prefix given by \*(lq\fB\-\-name\fR\*(rq.
@@ -912,6 +920,8 @@ Please report bugs or feature requests via the issue tracker linked from the
.SH "SEE ALSO"
.BR cat (1),
.BR splice (2),
.BR copy_file_range (2),
.BR fsync (2),
.BR fdatasync (2),
.BR open (2)
(for \fBO_DIRECT\fR),
+29 -24
View File
@@ -297,22 +297,23 @@ are explicitly switched on will be shown.
**-C, \--no-splice**
: Prevent the use of **splice**(2), a system call normally used on
systems that support it to improve efficiency by eliminating the
need for a transfer buffer.
: Prevent the use of **splice**(2) and **copy_file_range**(2), which
are normally used on systems that support it to improve efficiency
by eliminating the need for a transfer buffer.
In the default case that **splice**(2) is used, the lack of a
transfer buffer prevents "**\--buffer-percent**" and
"**\--last-written**" from working and makes "**\--buffer-size**"
redundant, so using any of those options automatically switches on
"**\--no-splice**".
In the default case that **splice**(2) or **copy_file_range**(2) are
used, the lack of a transfer buffer prevents "**\--buffer-percent**"
and "**\--last-written**" from working and makes
"**\--buffer-size**" redundant, so using any of those options
automatically switches on "**\--no-splice**".
Sparse files cannot be written with **splice**(2), so using
"**\--sparse**" automatically switches on "**\--no-splice**".
Sparse files cannot be written with **splice**(2) or
**copy_file_range**(2), so using "**\--sparse**" automatically
switches on "**\--no-splice**".
Switching on "**\--no-splice**" results in a small loss of transfer
efficiency. It has no effect on systems where **splice**(2) is
unavailable.
efficiency. It has no effect on systems where **splice**(2) and
**copy_file_range**(2) are unavailable.
**-J BYTES, \--pipe-buffer-size BYTES**
@@ -360,9 +361,9 @@ are explicitly switched on will be shown.
**-Y, \--sync**
: After every write operation, synchronise the buffer caches to disk
with **fdatasync**(2). This has no effect when the output is a pipe.
Using "**\--sync**" may improve the accuracy of the progress bar
when writing to a slow disk.
with **sync**(2), or **fdatasync**(2) if possible. This has no
effect when the output is a pipe. Using "**\--sync**" may improve
the accuracy of the progress bar when writing to a slow disk.
**-K, \--direct-io**
@@ -585,23 +586,27 @@ contain the following sequences:
**%T**, **%{buffer-percent}**
: Percentage of the transfer buffer in use. Equivalent to
"**\--buffer-percent**". Displays "{\-\-\--}" if the transfer is
being done with **splice**(2), since splicing to or from pipes does
not use the buffer.
"**\--buffer-percent**", but using it does not turn on
"**\--no-splice**".
If the transfer is using **splice**(2), displays "{\-\-\--}"; if the
transfer is using **splice**(2) via an intermediate input pipe,
displays "{-\|\|-}"; and if the transfer is using
**copy_file_range**(2), displays "{\-\--\>}".
**%nA**, **%n{last-written}**
: Show the last *n* bytes written (for example, "**%16A**" shows the
last 16 bytes). Shows only dots if the transfer is being done with
**splice**(2), since splicing to or from pipes does not use the
buffer.
last 16 bytes). Shows only dots if the transfer is using
**splice**(2) or **copy_file_range**(2), since these transfer modes
do not use the buffer.
**%nL**, **%n{previous-line}**
: Show the first *n* bytes of the most recently written line (for
example, "**%40L**" shows the first 40 bytes). If no *n* is given,
then this expands to fill the available space. Shows only spaces if
the transfer is being done with **splice**(2).
the transfer is using **splice**(2) or **copy_file_range**(2).
**%N**, **%{name}**
@@ -862,8 +867,8 @@ the [**pv** home page](https://ivarch.com/p/pv).
# SEE ALSO
**cat**(1), **splice**(2), **fdatasync**(2), **open**(2) (for
**O_DIRECT**), **console_codes**(4)
**cat**(1), **splice**(2), **copy_file_range**(2), **fsync**(2),
**fdatasync**(2), **open**(2) (for **O_DIRECT**), **console_codes**(4)
# COPYRIGHT
+22 -7
View File
@@ -78,6 +78,13 @@ typedef enum {
PV_TRANSFERCOUNT_LINES
} pvtransfercount_t;
/* Methods used to transfer bytes from input to output. */
typedef enum {
PV_TRANSFERMETHOD_READWRITE, /* read() + write() */
PV_TRANSFERMETHOD_SPLICE, /* splice() */
PV_TRANSFERMETHOD_SPLICE_INTERMEDIATE, /* splice() through an intermediate input pipe */
PV_TRANSFERMETHOD_COPY_FILE_RANGE /* copy_file_range() */
} pvtransfermethod_t;
/*
* Structure describing a short string used as part of a progress bar, whose
@@ -161,7 +168,9 @@ struct pvstate_s {
bool terminal_supports_colour; /* whether the terminal supports colour */
bool checked_colour_support; /* whether we have checked colour support yet */
bool current_input_is_pipe; /* whether the current input file is a pipe */
bool current_input_is_file; /* whether the current input is a regular file */
bool output_is_pipe; /* whether the output is a pipe */
bool output_is_file; /* whether the output is a regular file */
} status;
/***************
@@ -472,16 +481,22 @@ struct pvstate_s {
/* File descriptor to /dev/null for splicing with -X. */
int discard_fd;
/*
* These variables are used to keep track of whether
* splice() was used; splice_failed_fd is the file
* descriptor that splice() last failed on, to avoid
* continuing to try using it on an fd that doesn't support
* it, and splice_used is set to true if splice() was used
* this time within pv_transfer().
* splice_failed_fd is the input file descriptor that
* splice() last failed on, to avoid continuing to try using
* it on a file descriptor that doesn't support it.
*/
int splice_failed_fd;
bool splice_used;
#endif
#ifdef HAVE_COPY_FILE_RANGE
/*
* copy_file_range_failed_fd is the input file descriptor
* that copy_file_range() last failed on, to avoid
* continuing to try using it on a file descriptor that
* doesn't support it.
*/
int copy_file_range_failed_fd;
#endif
pvtransfermethod_t method; /* method used for most recent transfer */
bool read_error_warning_shown;
bool output_not_seekable; /* set if lseek() fails on output */
} transfer;
+41 -26
View File
@@ -442,14 +442,19 @@ int pv_next_file(pvstate_t state, unsigned int filenum, int oldfd)
}
/*
* Detect whether the input file is a pipe. This is used later, in
* pv__transfer_read(), to decide whether an intermediate pipe needs
* to be used with splice().
* Detect whether the input file is a pipe, a regular file, or
* neither. This is used later, in pv__transfer_read(), when
* determining the transfer method - read/write, splice(),
* copy_file_range().
*/
state->status.current_input_is_pipe = false;
state->status.current_input_is_file = false;
if ((isb.st_mode & S_IFMT) == S_IFIFO) {
state->status.current_input_is_pipe = true;
debug("%s (fd %d)", "input is a pipe", fd);
} else if ((isb.st_mode & S_IFMT) == S_IFREG) {
state->status.current_input_is_file = true;
debug("%s (fd %d)", "input is a regular file", fd);
}
state->status.current_input_file = filenum;
@@ -473,9 +478,41 @@ int pv_next_file(pvstate_t state, unsigned int filenum, int oldfd)
*/
#endif /* O_DIRECT */
debug("%s: %d: %s: fd=%d", "next file opened", filenum, pv_current_file_name(state), fd);
debug("%s: %d: %s: fd=%d, is_pipe=%s, is_file=%s", "next file opened", filenum, pv_current_file_name(state), fd,
state->status.current_input_is_pipe ? "true" : "false",
state->status.current_input_is_file ? "true" : "false");
#ifdef HAVE_SPLICE
if (state->control.discard_input && !state->control.no_splice && state->transfer.discard_fd < 0) {
/*
* Open a file descriptor to /dev/null, so that input can be
* spliced to it to implement -X.
*
* If this fails, transfer.discard_fd will be left at -1,
* which means the transfer functions will not use splice().
*/
state->transfer.discard_fd = open("/dev/null", O_WRONLY); /* flawfinder: ignore */
/* flawfinder: /dev/null is trusted. */
if (state->transfer.discard_fd < 0) {
debug("%s: %s", "/dev/null", strerror(errno));
state->transfer.discard_fd = -1;
}
if (!pv_fd_is_dev_null(state->transfer.discard_fd, false)) {
if (state->transfer.discard_fd >= 0)
(void) close(state->transfer.discard_fd);
state->transfer.discard_fd = -1;
}
debug("%s: %d", "discard_fd", state->transfer.discard_fd);
/*
* If discard_fd was successfully opened, it will be used as
* the output, so set the output status flags appropriately.
*/
if (state->transfer.discard_fd >= 0) {
state->status.output_is_pipe = false;
state->status.output_is_file = false;
}
}
if (!(state->status.output_is_pipe || state->status.current_input_is_pipe || state->control.no_splice)
&& (-1 == state->transfer.intermediate_pipe[0])) {
/*
@@ -528,28 +565,6 @@ int pv_next_file(pvstate_t state, unsigned int filenum, int oldfd)
state->transfer.intermediate_pipe[1]);
debug("%s: %d", "intermediate pipe buffer size", state->transfer.intermediate_pipe_buffer_size);
}
if (state->control.discard_input && !state->control.no_splice && state->transfer.discard_fd < 0) {
/*
* Open a file descriptor to /dev/null, so that input can be
* spliced to it to implement -X.
*
* If this fails, transfer.discard_fd will be left at -1,
* which means the transfer functions will not use splice().
*/
state->transfer.discard_fd = open("/dev/null", O_WRONLY); /* flawfinder: ignore */
/* flawfinder: /dev/null is trusted. */
if (state->transfer.discard_fd < 0) {
debug("%s: %s", "/dev/null", strerror(errno));
state->transfer.discard_fd = -1;
}
if (!pv_fd_is_dev_null(state->transfer.discard_fd, false)) {
if (state->transfer.discard_fd >= 0)
(void) close(state->transfer.discard_fd);
state->transfer.discard_fd = -1;
}
debug("%s: %d", "discard_fd", state->transfer.discard_fd);
}
#endif /* HAVE_SPLICE */
return fd;
+12 -3
View File
@@ -30,10 +30,19 @@ pvdisplay_bytecount_t pv_formatter_buffer_percent(pvformatter_args_t args)
(args->transfer->buffer_size));
(void) pv_snprintf(content, sizeof(content), "{%3.0f%%}", pct_used);
}
#ifdef HAVE_SPLICE
if (args->transfer->splice_used)
switch (args->transfer->method) {
case PV_TRANSFERMETHOD_READWRITE:
break;
case PV_TRANSFERMETHOD_SPLICE:
(void) pv_snprintf(content, sizeof(content), "{%s}", "----");
#endif
break;
case PV_TRANSFERMETHOD_SPLICE_INTERMEDIATE:
(void) pv_snprintf(content, sizeof(content), "{%s}", "-||-");
break;
case PV_TRANSFERMETHOD_COPY_FILE_RANGE:
(void) pv_snprintf(content, sizeof(content), "{%s}", "--->");
break;
}
return pv_formatter_segmentcontent(content, args);
}
+5 -1
View File
@@ -375,8 +375,9 @@ int pv_main_loop(pvstate_t state)
if (output_fd < 0)
output_fd = STDOUT_FILENO;
/* Determine whether the output is a pipe. */
/* Determine whether the output is a regular file, a pipe, or neither. */
state->status.output_is_pipe = false;
state->status.output_is_file = false;
{
struct stat sb;
memset(&sb, 0, sizeof(sb));
@@ -385,6 +386,9 @@ int pv_main_loop(pvstate_t state)
if ((sb.st_mode & S_IFMT) == S_IFIFO) {
state->status.output_is_pipe = true;
debug("%s (fd %d)", "output is a pipe", output_fd);
} else if ((sb.st_mode & S_IFMT) == S_IFREG) {
state->status.output_is_file = true;
debug("%s (fd %d)", "output is a regular file", output_fd);
}
/*@+type@ *//* splint says st_mode is __mode_t, not mode_t */
} else {
+3
View File
@@ -131,6 +131,9 @@ void pv_reset_transfer(pvtransferstate_t transfer)
transfer->written_but_not_consumed = 0;
transfer->read_errors_in_a_row = 0;
transfer->last_read_skip_fd = 0;
#ifdef HAVE_COPY_FILE_RANGE
transfer->copy_file_range_failed_fd = -1;
#endif /* HAVE_COPY_FILE_RANGE */
#ifdef HAVE_SPLICE
transfer->splice_failed_fd = -1;
transfer->intermediate_pipe[0] = -1;
+408 -110
View File
@@ -114,13 +114,19 @@ static int is_data_ready(int fd_in, /*@null@ */ bool *fd_in_ready, int fd_out, /
* TRANSFER_READ_TIMEOUT seconds have not yet elapsed, and more data is
* available to read according to is_data_ready().
*
* Sets state->transfer.method to PV_TRANSFERMETHOD_READWRITE as a side
* effect, so that if this is called as a fallback from another transfer
* method, the original caller can find out.
*
* Returns the total number of bytes read, or negative on error.
*/
static ssize_t pv__transfer__read_repeated(int fd, char *buf, size_t count)
static ssize_t pv__transfer__read_repeated(pvstate_t state, int fd, char *buf, size_t count)
{
struct timespec start_time;
ssize_t total_read;
state->transfer.method = PV_TRANSFERMETHOD_READWRITE;
memset(&start_time, 0, sizeof(start_time));
pv_elapsedtime_read(&start_time);
@@ -173,7 +179,7 @@ static ssize_t pv__transfer__read_repeated(int fd, char *buf, size_t count)
}
if (count > 0) {
debug("%s %d: %s (%ld %s, %ld %s)", "fd", fd,
debug("%s %d: %s (%ld %s, %lu %s)", "fd", fd,
"trying another read after partial buffer fill", nread, "read", count, "remaining");
if (is_data_ready(fd, NULL, -1, NULL, 0) < 1)
break;
@@ -224,24 +230,22 @@ static ssize_t pv__transfer__write_repeated(int fd, char *buf, size_t count, boo
nwritten = write(fd, buf, asked_to_write);
#ifdef HAVE_FDATASYNC
if (sync_after_write && nwritten >= 0) {
/*
* Ignore non I/O errors, such as EBADFD (bad file
* descriptor), EINVAL (non syncable fd, such as a
* pipe), etc - only return an error on EIO.
*/
#if defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO > 0
#if defined(HAVE_FDATASYNC) && defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO > 0
if ((fdatasync(fd) < 0) && (EIO == errno)) {
return -1;
}
#else
#else /* !HAVE_FDATASYNC et al */
if ((fsync(fd) < 0) && (EIO == errno)) {
return -1;
}
#endif
#endif /* HAVE_FDATASYNC et al */
}
#endif /* HAVE_FDATASYNC */
if (nwritten < 0) {
if ((EINTR == errno) || (EAGAIN == errno)) {
@@ -294,7 +298,7 @@ static ssize_t pv__transfer__write_repeated(int fd, char *buf, size_t count, boo
* prevent endless retries.
*/
if (count > 0) {
debug("%s %d: %s (%ld %s, %ld %s)", "fd", fd,
debug("%s %d: %s (%ld %s, %lu %s)", "fd", fd,
"trying another write after partial buffer flush",
nwritten, "written", count, "remaining");
@@ -414,30 +418,29 @@ static ssize_t pv__transfer__splice_via_intermediate(pvstate_t state, int input_
* TRANSFER_READ_TIMEOUT seconds have elapsed, as long as more data is
* available to read according to is_data_ready().
*
* If splice() was successfully used, sets state->transfer.splice_used to
* true. Does not modify it otherwise.
*
* If splice() could not be used, sets state->transfer.splice_failed_fd to
* fd so splice() won't be tried again until the next input file, and then
* calls pv__transfer__read_repeated() to read into the buffer, first
* capping "max_to_read" to "max_buffer_available", returning its result.
* input_fd so splice() won't be tried again until the next input file, and
* then calls pv__transfer__read_repeated() to read up to
* "fallback_read_amount" bytes into the buffer, returning its result.
*
* Sets state->transfer.method to either PV_TRANSFERMETHOD_SPLICE or
* PV_TRANSFERMETHOD_SPLICE_INTERMEDIATE as a side effect; if it falls back
* to pv__transfer__read_repeated(), that function then sets
* state->transfer.method to PV_TRANSFERMETHOD_READWRITE, so the caller can
* check which transfer method was actually used.
*
* Returns the total number of bytes transferred, or negative on error.
*/
static ssize_t pv__transfer__splice_repeated(pvstate_t state, int input_fd, int output_fd, char *buf,
size_t max_buffer_available, off_t max_to_read, off_t max_to_write)
size_t fallback_read_amount, off_t max_to_read, off_t max_to_write)
{
struct timespec start_time;
size_t bytes_to_splice;
ssize_t total_spliced;
bool use_intermediate_pipe;
size_t fallback_read_amount;
/* Amount to read_repeated() if splice() can't be used. */
fallback_read_amount = max_buffer_available;
if ((max_to_read >= 0) && (fallback_read_amount > (size_t) max_to_read)) {
fallback_read_amount = (size_t) max_to_read;
}
debug("%d->%d, fallback_read_amount=%lu, max_to_read=%ld, max_to_write=%ld", input_fd, output_fd,
fallback_read_amount, max_to_read, max_to_write);
/*
* Early return via pv__transfer__read_repeated() if splice() is
@@ -448,7 +451,7 @@ static ssize_t pv__transfer__splice_repeated(pvstate_t state, int input_fd, int
if (state->control.no_splice || state->control.linemode || (output_fd < 0)
|| (input_fd == state->transfer.splice_failed_fd)
|| (state->transfer.to_write > 0)) {
return pv__transfer__read_repeated(input_fd, buf, fallback_read_amount);
return pv__transfer__read_repeated(state, input_fd, buf, fallback_read_amount);
}
/*
@@ -474,18 +477,23 @@ static ssize_t pv__transfer__splice_repeated(pvstate_t state, int input_fd, int
/*@+unrecog@ *//* splint doesn't know of SIZE_MAX. */
/* Early return with 0 if the transfer cap is zero. */
if (0 == bytes_to_splice)
if (0 == bytes_to_splice) {
debug("bytes_to_splice=%lu", bytes_to_splice);
return 0;
}
/*
* Use an intermediate pipe if one has been created already, the
* input is not a pipe, and the output is not a pipe - because
* splice() only works if one or both file descriptors are pipes.
* Use an intermediate pipe if one is available and that transfer
* method is selected. Either way, update state->transfer.method to
* the method being used.
*/
use_intermediate_pipe = false;
if (!(state->status.output_is_pipe || state->status.current_input_is_pipe)
if ((PV_TRANSFERMETHOD_SPLICE_INTERMEDIATE == state->transfer.method)
&& (-1 != state->transfer.intermediate_pipe[0]) && (-1 != state->transfer.intermediate_pipe[1])) {
use_intermediate_pipe = true;
state->transfer.method = PV_TRANSFERMETHOD_SPLICE_INTERMEDIATE;
} else {
state->transfer.method = PV_TRANSFERMETHOD_SPLICE;
}
memset(&start_time, 0, sizeof(start_time));
@@ -494,6 +502,8 @@ static ssize_t pv__transfer__splice_repeated(pvstate_t state, int input_fd, int
total_spliced = 0;
debug("bytes_to_splice=%lu", bytes_to_splice);
while (bytes_to_splice > 0) {
ssize_t nspliced;
struct timespec cur_time, transfer_elapsed;
@@ -511,6 +521,9 @@ static ssize_t pv__transfer__splice_repeated(pvstate_t state, int input_fd, int
/*@+type@ *//*@+nullpass@ */
}
debug("bytes_to_splice=%lu, nspliced=%ld, use_intermediate_pipe=%s", bytes_to_splice, nspliced,
use_intermediate_pipe ? "true" : "false");
/*
* Early return on signal interrupt, returning -1 if nothing
* was transferred yet, or the amount transferred otherwise.
@@ -549,15 +562,12 @@ static ssize_t pv__transfer__splice_repeated(pvstate_t state, int input_fd, int
}
state->transfer.splice_failed_fd = input_fd;
if (0 == total_spliced) {
return pv__transfer__read_repeated(input_fd, buf, fallback_read_amount);
return pv__transfer__read_repeated(state, input_fd, buf, fallback_read_amount);
} else {
return total_spliced;
}
}
/* Flag the fact that splice() has been used successfully. */
state->transfer.splice_used = true;
total_spliced += nspliced;
/* Unsigned type - guard against underflow. */
if (bytes_to_splice > (size_t) nspliced) {
@@ -600,30 +610,202 @@ static ssize_t pv__transfer__splice_repeated(pvstate_t state, int input_fd, int
#endif /* HAVE_SPLICE */
#ifdef HAVE_COPY_FILE_RANGE
/*
* Copy bytes from file descriptor "input_fd" to "output_fd" using the
* in-kernel copy function copy_file_range().
*
* The number of bytes copied is capped to "max_to_read", or if
* "max_to_write" is greater than zero and less than "max_to_read", capped
* to "max_to_write". A "max_to_read" of less than zero indicates no
* maximum.
*
* If state->control.rate_limit_active is true and "max_to_write" is zero,
* performs no action and returns zero. Since this looks the same as EOF,
* the caller must trap for this condition.
*
* Keeps reading until the target amount has been copied or until
* TRANSFER_READ_TIMEOUT seconds have elapsed, as long as more data is
* available to read according to is_data_ready().
*
* If copy_file_range() could not be used, sets
* state->transfer.copy_file_range_failed_fd to input_fd so it won't be
* tried again until the next input file, and then calls
* pv__transfer__read_repeated() to read up to "fallback_read_amount" bytes
* into the buffer, returning its result.
*
* Sets state->transfer.method to PV_TRANSFERMETHOD_COPY_FILE_RANGE as a
* side effect; if it falls back to pv__transfer__read_repeated(), that
* function then sets state->transfer.method to PV_TRANSFERMETHOD_READWRITE,
* so the caller can check which transfer method was actually used.
*
* Returns the total number of bytes transferred, or negative on error.
*/
static ssize_t pv__transfer__copy_file_range_repeated(pvstate_t state, int input_fd, int output_fd, char *buf,
size_t fallback_read_amount, off_t max_to_read,
off_t max_to_write)
{
struct timespec start_time;
size_t bytes_to_copy;
ssize_t total_copied;
debug("%d->%d, fallback_read_amount=%lu, max_to_read=%ld, max_to_write=%ld", input_fd, output_fd,
fallback_read_amount, max_to_read, max_to_write);
/*
* Early return via pv__transfer__read_repeated() if this feature is
* turned off, or if line mode is active, or if there's no output
* fd, or if it already failed on this input file descriptor, or if
* there's anything waiting in the transfer buffer.
*/
if (state->control.no_splice || state->control.linemode || (output_fd < 0)
|| (input_fd == state->transfer.copy_file_range_failed_fd)
|| (state->transfer.to_write > 0)) {
return pv__transfer__read_repeated(state, input_fd, buf, fallback_read_amount);
}
/*
* Cap the transfer amount if applicable.
*
* Note that max_to_write is an off_t (file size / offset), which
* may not fit into a size_t (byte count), so check it against
* SIZE_MAX before trying a comparison otherwise on 32-bit systems
* it might appear to be 0.
*/
/*@-unrecog@ */
bytes_to_copy = SIZE_MAX;
if ((max_to_read >= 0) && ((unsigned long) max_to_read <= (unsigned long) SIZE_MAX)) {
bytes_to_copy = (size_t) max_to_read;
}
if ((state->control.rate_limit_active || max_to_write > 0)
&& ((unsigned long) max_to_write <= (unsigned long) SIZE_MAX)
&& ((max_to_read < 0) || ((unsigned long) max_to_read >= (unsigned long) SIZE_MAX)
|| (max_to_read > max_to_write))
) {
bytes_to_copy = (size_t) max_to_write;
}
/*@+unrecog@ *//* splint doesn't know of SIZE_MAX. */
/* Early return with 0 if the transfer cap is zero. */
if (0 == bytes_to_copy) {
debug("bytes_to_copy=%lu", bytes_to_copy);
return 0;
}
/* Explicitly set the transfer method being used. */
state->transfer.method = PV_TRANSFERMETHOD_COPY_FILE_RANGE;
memset(&start_time, 0, sizeof(start_time));
pv_elapsedtime_read(&start_time);
total_copied = 0;
debug("bytes_to_copy=%lu", bytes_to_copy);
while (bytes_to_copy > 0) {
ssize_t ncopied;
struct timespec cur_time, transfer_elapsed;
long double elapsed_seconds;
/*@-nullpass@ *//*@-type@ *//* splint doesn't know about copy_file_range. */
ncopied = copy_file_range(input_fd, NULL, output_fd, NULL, bytes_to_copy, 0);
/*@+type@ *//*@+nullpass@ */
debug("bytes_to_copy=%lu, ncopied=%ld", bytes_to_copy, ncopied);
/*
* Early return on signal interrupt, returning -1 if nothing
* was transferred yet, or the amount transferred otherwise.
*/
if (ncopied < 0 && ((EINTR == errno) || (EAGAIN == errno))) {
if (0 == total_copied)
return -1;
return total_copied;
}
/* TODO: possibly check for ENOSPC. */
/*
* If the copy failed, turn it off for this input file
* descriptor. Then, if nothing has been copied so far,
* return the result of an ordinary read, otherwise return
* the amount copied.
*/
if (ncopied < 0) {
debug("%s %d: %s: %s", "fd", input_fd, "disabling copy_file_range after failure",
strerror(errno));
state->transfer.copy_file_range_failed_fd = input_fd;
if (0 == total_copied) {
return pv__transfer__read_repeated(state, input_fd, buf, fallback_read_amount);
} else {
return total_copied;
}
}
total_copied += ncopied;
/* Unsigned type - guard against underflow. */
if (bytes_to_copy > (size_t) ncopied) {
bytes_to_copy -= ncopied;
} else {
bytes_to_copy = 0;
}
/* Early return on EOF. */
if (0 == ncopied) {
debug("%s %d: %s (%lu/%lu)", "fd", input_fd, "reached EOF", total_copied, bytes_to_copy);
return total_copied;
}
/* Keep trying while data is ready. */
memset(&cur_time, 0, sizeof(cur_time));
memset(&transfer_elapsed, 0, sizeof(transfer_elapsed));
elapsed_seconds = 0.0;
pv_elapsedtime_read(&cur_time);
pv_elapsedtime_subtract(&transfer_elapsed, &cur_time, &start_time);
elapsed_seconds = pv_elapsedtime_seconds(&transfer_elapsed);
if (elapsed_seconds > TRANSFER_READ_TIMEOUT) {
debug("%s %d: %s (%f %s)", "fd", input_fd,
"stopping copy_file_range - timer expired", (double) elapsed_seconds, "sec elapsed");
return total_copied;
}
if (bytes_to_copy > 0) {
debug("%s %d: %s (%ld %s, %lu %s)", "fd", input_fd,
"trying another copy_file_range", ncopied, "transferred this time", bytes_to_copy,
"remaining");
if (is_data_ready(input_fd, NULL, -1, NULL, 0) < 1)
break;
}
}
return total_copied;
}
#endif /* HAVE_COPY_FILE_RANGE */
/*
* Read some data from the given file descriptor, updating the state.
*
* At most, the number of bytes read will be the number of bytes remaining
* in the input buffer, capped to the number of bytes left until
* state->control.size is reached if state->control.stop_at_size is true.
* state->control.size is reached if state->control.stop_at_size is true.
* If state->control.rate_limit_active is true, and/or "max_to_write" is >0,
* and splice() is used, then the maximum number of bytes read will be
* further capped to the value of "max_to_write", since splice() writes as
* well as reads.
* and a non-read/write transfer method (splice or copy_file_range) is used,
* then the maximum number of bytes read will be further capped to the value
* of "max_to_write", since those functions write as well as read.
*
* If splice() was successfully used, sets state->transfer.splice_used to
* true; if it was unsuccessfully used, then
* state->transfer.splice_failed_fd is updated to the current fd so splice()
* won't be tried again until the next input file.
*
* Updates state->transfer.read_position by the number of bytes read, unless
* splice() was used, in which case it does not since there's nothing in the
* buffer (and it also adds the bytes to state->transfer.written since
* they've been written to the output).
* If a read/write transfer method was used, updates
* state->transfer.read_position by the number of bytes read. If not, the
* amount read by splice/copy_file_range is placed in
* state->transfer.written since that amount will have been written to the
* output.
*
* Also increases state->transfer.total_bytes_read by the number of bytes
* read, regardless of whether splice() was used, since total_bytes_read is
* what it says it is, rather than being a buffer indicator.
* read, regardless of the transfer method, since total_bytes_read is what
* it says it is, rather than being a buffer indicator.
*
* If there is a non-transient read error, updates
* state->status.exit_status, and tries to skip past the problem if
@@ -640,18 +822,18 @@ static ssize_t pv__transfer__splice_repeated(pvstate_t state, int input_fd, int
* Returns false if there was a transient error and so the transfer should
* not continue, but be retried shortly instead.
*/
static bool pv__transfer_read(pvstate_t state, int fd, bool *eof_in, bool *eof_out, off_t max_to_write)
static bool pv__transfer_read(pvstate_t state, int input_fd, bool *eof_in, bool *eof_out, off_t max_to_write)
{
bool do_not_skip_errors;
bool zero_transfer_cap;
size_t max_buffer_available;
off_t max_to_read;
off_t max_to_read, max_to_read_into_buffer;
off_t amount_to_skip, amount_skipped, orig_offset, skip_offset;
ssize_t nread;
#ifdef HAVE_SPLICE
int output_fd;
output_fd = state->control.output_fd;
#ifdef HAVE_SPLICE
if (state->control.discard_input && !state->control.no_splice)
output_fd = state->transfer.discard_fd;
#endif /* HAVE_SPLICE */
@@ -674,55 +856,176 @@ static bool pv__transfer_read(pvstate_t state, int fd, bool *eof_in, bool *eof_o
}
nread = 0;
/*
* Flag if the amount to read is going to be zero, so that a
* zero-sized read is not then mistakenly taken as EOF.
*/
zero_transfer_cap = false;
if (0 == max_to_read)
zero_transfer_cap = true;
/*
* Restricted version of max_to_read limited to the available buffer
* space, to be used by the read/write transfer method.
*/
max_to_read_into_buffer = max_to_read;
/* Clamp to a maximum of available buffer space. */
if (max_to_read_into_buffer > 0 && (size_t) max_to_read_into_buffer > max_buffer_available)
max_to_read_into_buffer = (off_t) max_buffer_available;
/* If "unlimited" (-1), use the maximum buffer available. */
if (max_to_read_into_buffer < 0)
max_to_read_into_buffer = (off_t) max_buffer_available;
/* Clamp to a minimum of 0 in case max_buffer_available underflowed. */
if (max_to_read_into_buffer < 0)
max_to_read_into_buffer = 0;
/* Determine which transfer method to attempt. */
state->transfer.method = PV_TRANSFERMETHOD_READWRITE;
#ifdef HAVE_SPLICE
state->transfer.splice_used = false;
if (state->control.rate_limit_active && 0 == max_to_write) {
/*
* Attempt to use splice() only if splice() is not turned off, and
* line mode is inactive, and there's an output fd, and there's
* nothing waiting to be written from the transfer buffer, and
* splice() hasn't already failed on this input file descriptor.
*/
if (!(state->control.no_splice || state->control.linemode || (output_fd < 0)
|| (state->transfer.to_write > 0)
|| (input_fd == state->transfer.splice_failed_fd)
)) {
state->transfer.method = PV_TRANSFERMETHOD_SPLICE;
/*
* Flag that no attempt was made to transfer because of rate
* limiting, so the zero nread is not because of EOF; and
* wait briefly to avoid a busy-wait when rate limiting.
* Splice through an intermediate input pipe if neither the
* input nor the output is a pipe, and an intermediate pipe
* is ready to use.
*/
if (!(state->status.output_is_pipe || state->status.current_input_is_pipe)
&& (-1 != state->transfer.intermediate_pipe[0]) && (-1 != state->transfer.intermediate_pipe[1])) {
state->transfer.method = PV_TRANSFERMETHOD_SPLICE_INTERMEDIATE;
}
}
#endif /* HAVE_SPLICE */
#ifdef HAVE_COPY_FILE_RANGE
/*
* Attempt to use copy_file_range() only if both the input and the
* output are regular files, and splice() is not turned off (since
* it's one option for both this and splice), and line mode is
* inactive, and there's an output fd, and there's nothing waiting
* to be written from the transfer buffer, and copy_file_range()
* hasn't already failed on this input file descriptor.
*/
if (state->status.current_input_is_file && state->status.output_is_file
&& !(state->control.no_splice || state->control.linemode || (output_fd < 0)
|| (state->transfer.to_write > 0)
|| (input_fd == state->transfer.copy_file_range_failed_fd)
)) {
state->transfer.method = PV_TRANSFERMETHOD_COPY_FILE_RANGE;
}
#endif /* HAVE_COPY_FILE_RANGE */
debug("%d->%d, max_to_write=%ld, max_to_read=%ld, max_to_read_into_buffer=%ld, method=%d", input_fd, output_fd,
max_to_write, max_to_read, max_to_read_into_buffer, state->transfer.method);
/*
* Transfer data using the appropriate method.
*
* If one method falls back to another, it will update
* state->transfer.method, so that variable may have changed after
* this block.
*/
switch (state->transfer.method) {
case PV_TRANSFERMETHOD_READWRITE:
nread =
pv__transfer__read_repeated(state, input_fd,
state->transfer.transfer_buffer + state->transfer.read_position,
(size_t) max_to_read_into_buffer);
break;
#ifdef HAVE_SPLICE
case PV_TRANSFERMETHOD_SPLICE:
case PV_TRANSFERMETHOD_SPLICE_INTERMEDIATE:
nread =
pv__transfer__splice_repeated(state, input_fd, output_fd,
state->transfer.transfer_buffer + state->transfer.read_position,
(size_t) max_to_read_into_buffer, max_to_read, max_to_write);
break;
#else /* !HAVE_SPLICE */
case PV_TRANSFERMETHOD_SPLICE:
case PV_TRANSFERMETHOD_SPLICE_INTERMEDIATE:
nread =
pv__transfer__read_repeated(state, input_fd,
state->transfer.transfer_buffer + state->transfer.read_position,
(size_t) max_to_read_into_buffer);
break;
#endif /* HAVE_SPLICE */
#ifdef HAVE_COPY_FILE_RANGE
case PV_TRANSFERMETHOD_COPY_FILE_RANGE:
nread =
pv__transfer__copy_file_range_repeated(state, input_fd, output_fd,
state->transfer.transfer_buffer +
state->transfer.read_position,
(size_t) max_to_read_into_buffer, max_to_read, max_to_write);
break;
#else /* !HAVE_COPY_FILE_RANGE */
case PV_TRANSFERMETHOD_COPY_FILE_RANGE:
nread =
pv__transfer__read_repeated(state, input_fd,
state->transfer.transfer_buffer + state->transfer.read_position,
(size_t) max_to_read_into_buffer);
break;
#endif /* HAVE_COPY_FILE_RANGE */
}
/*
* If the actual transfer method (after any fallbacks) was
* read/write, cap max_to_read to the amount that was left in the
* buffer.
*/
if (PV_TRANSFERMETHOD_READWRITE == state->transfer.method)
max_to_read = max_to_read_into_buffer;
/*
* If max_to_write is zero, state->control.rate_limit_active is
* true, and the read/write transfer method was not used, wait
* briefly to avoid a busy-wait, since non-read/write transfer
* methods may not have called select() to wait for input.
*
* Also set zero_transfer_cap, since a write limit of zero means
* nothing was read either, and this doesn't mean EOF.
*/
if (PV_TRANSFERMETHOD_READWRITE != state->transfer.method && state->control.rate_limit_active
&& 0 == max_to_write) {
zero_transfer_cap = true;
(void) is_data_ready(-1, NULL, -1, NULL, 10000);
} else {
nread =
pv__transfer__splice_repeated(state, fd, output_fd,
state->transfer.transfer_buffer + state->transfer.read_position,
max_buffer_available, max_to_read, max_to_write);
#ifdef HAVE_FDATASYNC
if (nread > 0 && state->control.sync_after_write) {
/*
* Sync after a successful splice, if enabled.
*
* Ignore non I/O errors, such as EBADFD (bad file
* descriptor), EINVAL (non syncable fd, such as a
* pipe), etc - only treat EIO as a failure.
*
* Since this is a write error, not a read error, it
* can't be skipped, so set "do_not_skip_errors".
*/
if ((fdatasync(output_fd) < 0)
&& (EIO == errno)) {
nread = -1;
do_not_skip_errors = true;
}
}
/*
* If the transfer wasn't a read/write type, then data has been
* written, so sync if sync_after_write is true.
*
* Ignore non I/O errors, such as EBADFD (bad file descriptor),
* EINVAL (non syncable fd, such as a pipe), etc - only treat EIO as
* a failure.
*
* Since this is a write error, not a read error, it can't be
* skipped, so set "do_not_skip_errors".
*/
if (PV_TRANSFERMETHOD_READWRITE != state->transfer.method && nread > 0 && state->control.sync_after_write) {
#if defined(HAVE_FDATASYNC) && defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO > 0
if ((fdatasync(output_fd) < 0)
&& (EIO == errno)) {
nread = -1;
do_not_skip_errors = true;
}
#endif /* HAVE_FDATASYNC */
#else /* !HAVE_FDATASYNC et al */
if ((fsync(output_fd) < 0)
&& (EIO == errno)) {
nread = -1;
do_not_skip_errors = true;
}
#endif /* HAVE_FDATASYNC et al */
}
#else /* !HAVE_SPLICE */
if (max_to_read < 0) {
max_to_read = max_buffer_available;
}
if (max_to_read < 0)
max_to_read = 0;
nread =
pv__transfer__read_repeated(fd, state->transfer.transfer_buffer + state->transfer.read_position,
(size_t) max_to_read);
#endif /* HAVE_SPLICE */
if (0 == nread) {
/*
@@ -735,7 +1038,7 @@ static bool pv__transfer_read(pvstate_t state, int fd, bool *eof_in, bool *eof_o
* loop can move on to the next input file.
*/
if (!zero_transfer_cap) {
debug("%s %d: %s", "fd", fd, "reached EOF");
debug("%s %d: %s", "input_fd", input_fd, "reached EOF");
*eof_in = true;
if (state->transfer.write_position >= state->transfer.read_position) {
*eof_out = true;
@@ -749,26 +1052,21 @@ static bool pv__transfer_read(pvstate_t state, int fd, bool *eof_in, bool *eof_o
* is in the buffer.
*/
state->transfer.read_errors_in_a_row = 0;
#ifdef HAVE_SPLICE
/*
* If splice() was used, the amount written was the amount
* read.
* If the transfer wasn't read/write, then the amount
* written was the amount read.
*/
if (state->transfer.splice_used) {
if (PV_TRANSFERMETHOD_READWRITE != state->transfer.method) {
state->transfer.written = nread;
} else {
/*
* When splice() wasn't used, the buffer has "nread"
* more bytes in it.
* The transfer was read/write, so the buffer
* now has "nread" more bytes in it.
*/
state->transfer.read_position += nread;
}
debug("%s %d: %s: %ld, %s=%s", "fd", fd, "transferred", nread, "splice_used",
state->transfer.splice_used ? "true" : "false");
#else
state->transfer.read_position += nread;
debug("%s %d: %s: %ld", "fd", fd, "transferred", nread);
#endif /* HAVE_SPLICE */
debug("%s %d: %s: %ld, %s=%d", "input_fd", input_fd, "transferred", nread, "transfer.method",
state->transfer.method);
/* Update the counter of all bytes read so far. */
state->transfer.total_bytes_read += nread;
return true;
@@ -783,7 +1081,7 @@ static bool pv__transfer_read(pvstate_t state, int fd, bool *eof_in, bool *eof_o
* and return false, since this was a transient error.
*/
if ((EINTR == errno) || (EAGAIN == errno)) {
debug("%s %d: %s: %s", "fd", fd, "transient error - waiting briefly", strerror(errno));
debug("%s %d: %s: %s", "input_fd", input_fd, "transient error - waiting briefly", strerror(errno));
(void) is_data_ready(-1, NULL, -1, NULL, 10000);
return false;
}
@@ -820,7 +1118,7 @@ static bool pv__transfer_read(pvstate_t state, int fd, bool *eof_in, bool *eof_o
state->transfer.read_error_warning_shown = true;
}
orig_offset = (off_t) lseek(fd, 0, SEEK_CUR);
orig_offset = (off_t) lseek(input_fd, 0, SEEK_CUR);
/*
* If the file is not seekable, the error can't be skipped, so
@@ -881,7 +1179,7 @@ static bool pv__transfer_read(pvstate_t state, int fd, bool *eof_in, bool *eof_o
/*@+longintegral@ */
/* splint complains about __off_t vs off_t */
skip_offset = (off_t) lseek(fd, (off_t) (orig_offset + amount_to_skip), SEEK_SET);
skip_offset = (off_t) lseek(input_fd, (off_t) (orig_offset + amount_to_skip), SEEK_SET);
/*@-longintegral@ */
/*
@@ -893,7 +1191,7 @@ static bool pv__transfer_read(pvstate_t state, int fd, bool *eof_in, bool *eof_o
amount_to_skip = 1;
/*@+longintegral@ */
/* see above */
skip_offset = (off_t) lseek(fd, (off_t) (orig_offset + amount_to_skip), SEEK_SET);
skip_offset = (off_t) lseek(input_fd, (off_t) (orig_offset + amount_to_skip), SEEK_SET);
/*@-longintegral@ */
}
@@ -1585,7 +1883,9 @@ ssize_t pv_transfer(pvstate_t state, int fd, bool *eof_in, bool *eof_out, off_t
* If there is data to read, try to read some in. Return early if
* there was a transient read error.
*
* NB this can update state->transfer.written because of splice().
* NB this can update state->transfer.written because of
* non-read/write transfer methods such as splice() or
* copy_file_range().
*/
if (ready_to_read) {
if (!pv__transfer_read(state, fd, eof_in, eof_out, allowed)) {
@@ -1614,14 +1914,12 @@ ssize_t pv_transfer(pvstate_t state, int fd, bool *eof_in, bool *eof_out, off_t
/*
* If there is data to write, and the output is ready to receive it,
* and splice() wasn't used this time, write some data.
* and the transfer method is read/write rather than splice() or
* copy_file_range(), write some data.
*
* Return early if there was a transient write error.
*/
if (ready_to_write
#ifdef HAVE_SPLICE
&& (!state->transfer.splice_used)
#endif /* HAVE_SPLICE */
if (ready_to_write && (PV_TRANSFERMETHOD_READWRITE == state->transfer.method)
&& (state->transfer.read_position > state->transfer.write_position)
&& (state->transfer.to_write > 0)
&& (NULL != lineswritten)) {