From 17cbab3045f6af54529f147802f40d0ec00ec58c Mon Sep 17 00:00:00 2001 From: Andrew Wood Date: Sat, 2 May 2026 16:42:35 +0100 Subject: [PATCH] Add an option "-J" / "--pipe-buffer-size" to set the size of the output pipe buffer when the output is a pipe, and when that option is not specified, automatically increase the output pipe buffer size to match that of the input, when the first input is also a pipe (#188). --- docs/pv.1 | 16 ++++++++++++---- docs/pv.1.md | 16 +++++++++++++--- src/include/options.h | 1 + src/include/pv-internal.h | 3 ++- src/include/pv.h | 1 + src/main/help.c | 3 +++ src/main/main.c | 1 + src/main/options.c | 7 ++++++- src/pv/loop.c | 40 +++++++++++++++++++++++++++++++++++++++ src/pv/state.c | 5 +++++ 10 files changed, 84 insertions(+), 9 deletions(-) diff --git a/docs/pv.1 b/docs/pv.1 index 10237ce..697cdcd 100644 --- a/docs/pv.1 +++ b/docs/pv.1 @@ -291,6 +291,14 @@ automatically switches on \*(lq\fB\-\-no\-splice\fR\*(rq. Switching on this option results in a small loss of transfer efficiency. It has no effect on systems where \fBsplice\fR(2) is unavailable. .TP +.BI \-J\ BYTES \fR,\ \fB\-\-pipe\-buffer\-size\ BYTES +Attempt to set the size of the output pipe buffer to \fIBYTES\fR bytes. +This will only have an effect if the output is a pipe, and will silently do +nothing if the requested size cannot be achieved. +If this option is not specified, and both the output and the first input are +pipes, then the output pipe buffer size will automatically be increased to +match the input pipe buffer, if it was smaller. +.TP .B \-E, \-\-skip-errors Ignore read errors by attempting to skip past the offending sections. The corresponding parts of the output will be null bytes. @@ -857,10 +865,10 @@ effective with small input files, and \*(lq\fB%nL\fR\*(rq may be a few lines out due to buffering within the pipeline itself. .PP Numbers passed to \*(lq\fB\-\-size\fR\*(rq, \*(lq\fB\-\-rate\-limit\fR\*(rq, -\*(lq\fB\-\-buffer\-size\fR\*(rq, and \*(lq\fB\-\-error\-skip\-block\fR\*(rq -may all be expressed as decimals if followed by a suffix, so for example -\*(lq\fI\-\-size\~1.5G\fR\*(rq is equivalent to -\*(lq\fI\-\-size\~1536M\fR\*(rq. +\*(lq\fB\-\-buffer\-size\fR\*(rq, \*(lq\fB\-\-pipe\-buffer\-size\fR\*(rq, +and \*(lq\fB\-\-error\-skip\-block\fR\*(rq may all be expressed as decimals +if followed by a suffix, so for example \*(lq\fI\-\-size\~1.5G\fR\*(rq is +equivalent to \*(lq\fI\-\-size\~1536M\fR\*(rq. .PP Numbers passed to \*(lq\fB\-\-interval\fR\*(rq and \*(lq\fB\-\-delay\-start\fR\*(rq may be integers or decimals, but may not diff --git a/docs/pv.1.md b/docs/pv.1.md index fb0f874..b821b6d 100644 --- a/docs/pv.1.md +++ b/docs/pv.1.md @@ -300,6 +300,15 @@ are explicitly switched on will be shown. option results in a small loss of transfer efficiency. It has no effect on systems where **splice**(2) is unavailable. +**-J BYTES, \--pipe-buffer-size BYTES** + +: Attempt to set the size of the output pipe buffer to *BYTES* bytes. + This will only have an effect if the output is a pipe, and will + silently do nothing if the requested size cannot be achieved. If + this option is not specified, and both the output and the first + input are pipes, then the output pipe buffer size will automatically + be increased to match the input pipe buffer, if it was smaller. + **-E, \--skip-errors** : Ignore read errors by attempting to skip past the offending @@ -814,9 +823,10 @@ small input files, and "**%nL**" may be a few lines out due to buffering within the pipeline itself. Numbers passed to "**\--size**", "**\--rate-limit**", -"**\--buffer-size**", and "**\--error-skip-block**" may all be expressed -as decimals if followed by a suffix, so for example "*\--size 1.5G*" is -equivalent to "*\--size 1536M*". +"**\--buffer-size**", "**\--pipe-buffer-size**", and +"**\--error-skip-block**" may all be expressed as decimals if followed +by a suffix, so for example "*\--size 1.5G*" is equivalent to +"*\--size 1536M*". Numbers passed to "**\--interval**" and "**\--delay-start**" may be integers or decimals, but may not have a suffix. diff --git a/src/include/options.h b/src/include/options.h index 0c173a6..a8719ce 100644 --- a/src/include/options.h +++ b/src/include/options.h @@ -72,6 +72,7 @@ struct opts_s { size_t lastwritten; /* show N bytes last written */ off_t rate_limit; /* rate limit, in bytes per second */ size_t buffer_size; /* buffer size, in bytes (0=default) */ + size_t pipe_buffer_size; /* pipe buffer size, in bytes (0=default) */ off_t size; /* total size of data */ off_t error_skip_block; /* skip block size, 0 for adaptive */ pid_t remote; /* PID of pv to update settings of */ diff --git a/src/include/pv-internal.h b/src/include/pv-internal.h index d395746..65502b0 100644 --- a/src/include/pv-internal.h +++ b/src/include/pv-internal.h @@ -201,7 +201,8 @@ struct pvstate_s { /*@null@*/ char *default_bar_style; /* which bar style to use by default */ off_t error_skip_block; /* skip block size, 0 for adaptive */ off_t rate_limit; /* rate limit, in bytes per second */ - size_t target_buffer_size; /* buffer size (0=default) */ + size_t target_buffer_size; /* transfer buffer size (0=default) */ + size_t pipe_buffer_size; /* pipe buffer size (0=default) */ off_t size; /* total size of data */ unsigned int skip_errors; /* skip read errors counter */ int output_fd; /* fd to write output to */ diff --git a/src/include/pv.h b/src/include/pv.h index 571f49c..cca38c1 100644 --- a/src/include/pv.h +++ b/src/include/pv.h @@ -260,6 +260,7 @@ extern void pv_state_sparse_output_set(pvstate_t, bool); extern void pv_state_rate_limit_set(pvstate_t, off_t); extern void pv_state_target_buffer_size_set(pvstate_t, size_t); extern void pv_state_no_splice_set(pvstate_t, bool); +extern void pv_state_pipe_buffer_size_set(pvstate_t, size_t); extern void pv_state_discard_input_set(pvstate_t, bool); extern void pv_state_size_set(pvstate_t, off_t); extern void pv_state_interval_set(pvstate_t, double); diff --git a/src/main/help.c b/src/main/help.c index ea04225..2dc8983 100644 --- a/src/main/help.c +++ b/src/main/help.c @@ -400,6 +400,9 @@ void display_help(void) { "-C", "--no-splice", NULL, N_("never use splice(), always use read/write"), { 0, 0, 0, 0} }, + { "-J", "--pipe-buffer-size", N_("BYTES"), + N_("set the pipe buffer size to BYTES"), + { 0, 0, 0, 0} }, { "-E", "--skip-errors", NULL, N_("skip read errors in input"), { 0, 0, 0, 0} }, diff --git a/src/main/main.c b/src/main/main.c index 7b44b11..4eadfe9 100644 --- a/src/main/main.c +++ b/src/main/main.c @@ -906,6 +906,7 @@ int main(int argc, char **argv) pv_state_rate_limit_set(state, opts->rate_limit); pv_state_target_buffer_size_set(state, opts->buffer_size); pv_state_no_splice_set(state, opts->no_splice); + pv_state_pipe_buffer_size_set(state, opts->pipe_buffer_size); pv_state_size_set(state, opts->size); pv_state_name_set(state, opts->name); pv_state_default_bar_style_set(state, opts->default_bar_style); diff --git a/src/main/options.c b/src/main/options.c index 5faa5d5..2538354 100644 --- a/src/main/options.c +++ b/src/main/options.c @@ -721,6 +721,7 @@ opts_t opts_parse(unsigned int argc, char **argv) { "no-splice", 0, NULL, (int) 'C' }, { "skip-errors", 0, NULL, (int) 'E' }, { "error-skip-block", 1, NULL, (int) 'Z' }, + { "pipe-buffer-size", 1, NULL, (int) 'J' }, { "stop-at-size", 0, NULL, (int) 'S' }, { "sync", 0, NULL, (int) 'Y' }, { "direct-io", 0, NULL, (int) 'K' }, @@ -743,7 +744,7 @@ opts_t opts_parse(unsigned int argc, char **argv) /*@+nullassign@ */ int option_index = 0; #endif /* HAVE_GETOPT_LONG */ - char *short_options = "hVpteIrab8kTA:fvnqcWD:s:gl0i:w:H:N:u:F:x:L:B:CEZ:SYKOXU:R:Q:P:d:m:o:M:" + char *short_options = "hVpteIrab8kTA:fvnqcWD:s:gl0i:w:H:N:u:F:x:L:B:CEZ:J:SYKOXU:R:Q:P:d:m:o:M:" #ifdef ENABLE_DEBUGGING "!:" #endif @@ -825,6 +826,7 @@ opts_t opts_parse(unsigned int argc, char **argv) case 'L': case 'B': case 'Z': + case 'J': if (!pv_getnum_check(optarg, PV_NUMTYPE_ANY_WITH_SUFFIX)) { /*@-mustfreefresh@ *//* see above */ pv_error("-%c: %s: %s", c, optarg, _("numeric value not understood")); @@ -1064,6 +1066,9 @@ opts_t opts_parse(unsigned int argc, char **argv) case 'Z': opts->error_skip_block = pv_getnum_size(optarg, opts->decimal_units); break; + case 'J': + opts->pipe_buffer_size = (size_t) pv_getnum_size(optarg, opts->decimal_units); + break; case 'S': opts->stop_at_size = true; break; diff --git a/src/pv/loop.c b/src/pv/loop.c index 4e566c3..52f5709 100644 --- a/src/pv/loop.c +++ b/src/pv/loop.c @@ -503,6 +503,46 @@ int pv_main_loop(pvstate_t state) if (0 == state->control.target_buffer_size) state->control.target_buffer_size = BUFFER_SIZE; +#ifdef F_GETPIPE_SZ +#ifdef F_SETPIPE_SZ + /* + * If a pipe buffer size was explicitly set, then set the output + * pipe buffer size to that. + * + * If no pipe buffer size was set, and the first input is a pipe, + * then increase the output pipe buffer size to the same as the + * input pipe buffer size, if the output buffer was smaller. + */ + if (output_is_pipe) { + size_t target_pipe_buffer_size = state->control.pipe_buffer_size; + + if (0 == target_pipe_buffer_size) { + int input_pipe_size; + input_pipe_size = fcntl(input_fd, F_GETPIPE_SZ); + + debug("%s %d: %s=%d", "input fd", input_fd, "pipe size", input_pipe_size); + if (input_pipe_size > 0) { + int output_pipe_size; + output_pipe_size = fcntl(output_fd, F_GETPIPE_SZ); + debug("%s %d: %s=%d", "output fd", output_fd, "pipe size", output_pipe_size); + if (output_pipe_size > 0 && output_pipe_size < input_pipe_size) { + target_pipe_buffer_size = (size_t) input_pipe_size; + } + } + } + if (target_pipe_buffer_size > 0) { + int output_pipe_size; + + output_pipe_size = fcntl(output_fd, F_SETPIPE_SZ, (int) target_pipe_buffer_size); + debug("%s %d: %s=%d", "output fd", output_fd, "updated pipe size", output_pipe_size); + if (output_pipe_size < 0) { + debug("%s: %s", "failed to set pipe buffer size", strerror(errno)); + } + } + } +#endif /* F_SETPIPE_SZ */ +#endif /* F_GETPIPE_SZ */ + /* * Repeat until eof_in is true, eof_out is true, and final_update is * true. diff --git a/src/pv/state.c b/src/pv/state.c index 2ed4eb4..8e79f78 100644 --- a/src/pv/state.c +++ b/src/pv/state.c @@ -642,6 +642,11 @@ void pv_state_no_splice_set(pvstate_t state, bool val) state->control.no_splice = val; } +void pv_state_pipe_buffer_size_set(pvstate_t state, size_t val) +{ + state->control.pipe_buffer_size = val; +} + void pv_state_size_set(pvstate_t state, off_t val) { state->control.size = val;