In discard mode, if splice() is available, open a file descriptor to /dev/null and splice the input to that, to improve performance (#191).

This commit is contained in:
Andrew Wood
2026-05-03 14:53:45 +01:00
parent b75ecd79d5
commit 11d727a88d
7 changed files with 37 additions and 13 deletions
-1
View File
@@ -356,7 +356,6 @@ seekable, this option will have no effect other than to turn on
Instead of transferring input data to standard output, discard it.
This is equivalent to redirecting standard output to \fI/dev/null\fR,
except that \fBwrite\fR(2) is never called.
Implies \*(lq\fB\-\-no\-splice\fR\*(rq.
.TP
.BI \-U\ FILE \fR,\ \fB\-\-store\-and\-forward\ FILE
Instead of passing data through immediately, do it in two stages - first
+1 -2
View File
@@ -369,8 +369,7 @@ are explicitly switched on will be shown.
: Instead of transferring input data to standard output, discard it.
This is equivalent to redirecting standard output to */dev/null*,
except that **write**(2) is never called. Implies
"**\--no-splice**".
except that **write**(2) is never called.
**-U FILE, \--store-and-forward FILE**
+2
View File
@@ -467,6 +467,8 @@ struct pvstate_s {
int intermediate_pipe[2];
int intermediate_pipe_buffer_size;
int intermediate_pipe_buffer_used;
/* File descriptor to /dev/null for splicing with -X. */
int discard_fd;
/*
* These variables are used to keep track of whether
* splice() was used; splice_failed_fd is the file
-1
View File
@@ -1084,7 +1084,6 @@ opts_t opts_parse(unsigned int argc, char **argv)
break;
case 'X':
opts->discard_input = true;
opts->no_splice = true;
break;
case 'U':
opts->store_and_forward_file = pv_strdup(optarg);
+15 -1
View File
@@ -462,12 +462,26 @@ int pv_next_file(pvstate_t state, unsigned int filenum, int oldfd)
state->transfer.intermediate_pipe_buffer_size = new_size;
}
#endif
#endif /* defined F_SETPIPE_SZ && defined F_GETPIPE_SZ */
}
debug("%s: [%d,%d]", "intermediate pipe fds", state->transfer.intermediate_pipe[0],
state->transfer.intermediate_pipe[1]);
debug("%s: %d", "intermediate pipe buffer size", state->transfer.intermediate_pipe_buffer_size);
}
if (state->control.discard_input && !state->control.no_splice && state->transfer.discard_fd < 0) {
/*
* Open a file descriptor to /dev/null, so that input can be
* spliced to it to implement -X.
*/
state->transfer.discard_fd = open("/dev/null", O_WRONLY); /* flawfinder: ignore */
/* flawfinder: /dev/null is trusted. */
if (state->transfer.discard_fd < 0) {
pv_perror("%s", "/dev/null");
(void) close(fd);
fd = -1;
}
}
#endif /* HAVE_SPLICE */
return fd;
+9 -5
View File
@@ -133,6 +133,9 @@ void pv_reset_transfer(pvtransferstate_t transfer)
transfer->last_read_skip_fd = 0;
#ifdef HAVE_SPLICE
transfer->splice_failed_fd = -1;
transfer->intermediate_pipe[0] = -1;
transfer->intermediate_pipe[1] = -1;
transfer->discard_fd = -1;
#endif /* HAVE_SPLICE */
transfer->line_positions_length = 0;
@@ -208,11 +211,6 @@ pvstate_t pv_state_alloc(void)
#endif /* HAVE_IPC */
state->cursor.lock_fd = -1;
#ifdef HAVE_SPLICE
state->transfer.intermediate_pipe[0] = -1;
state->transfer.intermediate_pipe[1] = -1;
#endif /* HAVE_SPLICE */
pv_state_reset(state);
#ifdef HAVE_GETCWD
@@ -287,6 +285,12 @@ void pv_freecontents_transfer(pvtransferstate_t transfer)
}
}
}
/* Close the discard file descriptor, if there was one. */
if (transfer->discard_fd >= 0) {
(void) close(transfer->discard_fd);
transfer->discard_fd = -1;
}
#endif /* HAVE_SPLICE */
if (NULL != transfer->line_positions)
+10 -3
View File
@@ -341,6 +341,13 @@ static bool pv__transfer_read(pvstate_t state, int fd, bool *eof_in, bool *eof_o
size_t bytes_can_read;
off_t amount_to_skip, amount_skipped, orig_offset, skip_offset;
ssize_t nread;
int output_fd;
output_fd = state->control.output_fd;
#ifdef HAVE_SPLICE
if (state->control.discard_input && !state->control.no_splice)
output_fd = state->transfer.discard_fd;
#endif /* HAVE_SPLICE */
do_not_skip_errors = false;
if (0 == state->control.skip_errors)
@@ -437,7 +444,7 @@ static bool pv__transfer_read(pvstate_t state, int fd, bool *eof_in, bool *eof_o
bytes_to_splice_out = bytes_to_splice;
spliced_out =
splice(state->transfer.intermediate_pipe[0], NULL, state->control.output_fd, NULL,
splice(state->transfer.intermediate_pipe[0], NULL, output_fd, NULL,
bytes_to_splice_out, SPLICE_F_MORE);
if (spliced_out > 0) {
state->transfer.intermediate_pipe_buffer_used -= (int) spliced_out;
@@ -446,7 +453,7 @@ static bool pv__transfer_read(pvstate_t state, int fd, bool *eof_in, bool *eof_o
}
} else {
/* Normal splice() from input to output. */
nread = splice(fd, NULL, state->control.output_fd, NULL, bytes_to_splice, SPLICE_F_MORE);
nread = splice(fd, NULL, output_fd, NULL, bytes_to_splice, SPLICE_F_MORE);
}
/*@+type@ */
/*@+nullpass@ */
@@ -474,7 +481,7 @@ static bool pv__transfer_read(pvstate_t state, int fd, bool *eof_in, bool *eof_o
* error, it can't be skipped, so set
* "do_not_skip_errors".
*/
if ((fdatasync(state->control.output_fd) < 0)
if ((fdatasync(output_fd) < 0)
&& (EIO == errno)) {
nread = -1;
do_not_skip_errors = true;