diff --git a/Makefile.am b/Makefile.am index a5a0c35..ae3803e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -95,6 +95,7 @@ tests/Modifiers_-_--line-mode.test \ tests/Modifiers_-_--size_from_file_size.test \ tests/Modifiers_-_--size.test \ tests/Modifiers_-_--sync.test \ +tests/Sparse_-_Basic.test \ tests/Terminal_-_Detect_width.test \ tests/Transfer_-_--rate-limit.test \ tests/Transfer_-_--remote.test \ diff --git a/docs/pv.1 b/docs/pv.1 index 09aa53f..2c89f55 100644 --- a/docs/pv.1 +++ b/docs/pv.1 @@ -273,9 +273,9 @@ from or to a pipe than regular \fBread\fR(2) and \fBwrite\fR(2), but means that the transfer buffer may not be used. This prevents \*(lq\fB\-\-buffer\-percent\fR\*(rq and \*(lq\fB\-\-last\-written\fR\*(rq from working, cannot work with -\*(lq\fB\-\-discard\fR\*(rq, and makes \*(lq\fB\-\-buffer\-size\fR\*(rq -redundant, so using any of those options automatically switches on -\*(lq\fB\-\-no\-splice\fR\*(rq. +\*(lq\fB\-\-sparse\fR\*(rq or \*(lq\fB\-\-discard\fR\*(rq, and makes +\*(lq\fB\-\-buffer\-size\fR\*(rq redundant, so using any of those options +automatically switches on \*(lq\fB\-\-no\-splice\fR\*(rq. Switching on this option results in a small loss of transfer efficiency. It has no effect on systems where \fBsplice\fR(2) is unavailable. .TP @@ -325,6 +325,13 @@ failures with an error of \*(lqInvalid argument\*(rq, especially if reading and writing files across a variety of filesystems in a single \fBpv\fR call. Use this option with caution. .TP +.B \-O, \-\-sparse +When writing null bytes, try to seek, producing a sparse output file. +Implies \*(lq\fB\-\-no\-splice\fR\*(rq. +On filesystems without sparse file support, or when the output is not +seekable, this option will have no effect other than to turn on +\*(lq\fB\-\-no\-splice\fR\*(rq. +.TP .B \-X, \-\-discard Instead of transferring input data to standard output, discard it. This is equivalent to redirecting standard output to \fI/dev/null\fR, diff --git a/docs/pv.1.md b/docs/pv.1.md index dedc28c..553dbbd 100644 --- a/docs/pv.1.md +++ b/docs/pv.1.md @@ -281,11 +281,11 @@ are explicitly switched on will be shown. data from or to a pipe than regular **read**(2) and **write**(2), but means that the transfer buffer may not be used. This prevents "**\--buffer-percent**" and "**\--last-written**" from working, - cannot work with "**\--discard**", and makes "**\--buffer-size**" - redundant, so using any of those options automatically switches on - "**\--no-splice**". Switching on this option results in a small loss - of transfer efficiency. It has no effect on systems where - **splice**(2) is unavailable. + cannot work with "**\--sparse**" or "**\--discard**", and makes + "**\--buffer-size**" redundant, so using any of those options + automatically switches on "**\--no-splice**". Switching on this + option results in a small loss of transfer efficiency. It has no + effect on systems where **splice**(2) is unavailable. **-E, \--skip-errors** @@ -336,6 +336,13 @@ are explicitly switched on will be shown. especially if reading and writing files across a variety of filesystems in a single **pv** call. Use this option with caution. +**-O, \--sparse** + +: When writing null bytes, try to seek, producing a sparse output + file. Implies "**\--no-splice**". On filesystems without sparse file + support, or when the output is not seekable, this option will have + no effect other than to turn on "**\--no-splice**". + **-X, \--discard** : Instead of transferring input data to standard output, discard it. diff --git a/src/include/pv.h b/src/include/pv.h index ccef7c3..a3e04d2 100644 --- a/src/include/pv.h +++ b/src/include/pv.h @@ -221,6 +221,7 @@ extern void pv_state_error_skip_block_set(pvstate_t, off_t); extern void pv_state_stop_at_size_set(pvstate_t, bool); extern void pv_state_sync_after_write_set(pvstate_t, bool); extern void pv_state_direct_io_set(pvstate_t, bool); +extern void pv_state_sparse_output_set(pvstate_t, bool); extern void pv_state_rate_limit_set(pvstate_t, off_t); extern void pv_state_target_buffer_size_set(pvstate_t, size_t); extern void pv_state_no_splice_set(pvstate_t, bool); diff --git a/src/main/help.c b/src/main/help.c index cd74f73..5706896 100644 --- a/src/main/help.c +++ b/src/main/help.c @@ -351,6 +351,9 @@ void display_help(void) { "-K", "--direct-io", NULL, N_("use direct I/O to bypass cache"), { 0, 0, 0, 0} }, + { "-O", "--sparse", NULL, + N_("try to seek instead of writing null bytes"), + { 0, 0, 0, 0} }, { "-X", "--discard", NULL, N_("discard input instead of writing to output"), { 0, 0, 0, 0} }, diff --git a/src/main/main.c b/src/main/main.c index 742327c..1345ccd 100644 --- a/src/main/main.c +++ b/src/main/main.c @@ -448,7 +448,12 @@ int main(int argc, char **argv) * do this before looking at setting the size, as the size * calculation looks at the output file if the input size can't be * calculated (issue #91). + * + * We have to set the sparse output flag before doing this, so that + * in sparse mode the lseek() on O_APPEND can be done (issue #45); + * see the comments in pv_state_output_set() in src/pv/state.c. */ + pv_state_sparse_output_set(state, opts->sparse_output); retcode = pv__set_output(state, opts, opts->output); if (0 != retcode) { pv_state_free(state); diff --git a/src/main/options.c b/src/main/options.c index 13f27ed..c1b0671 100644 --- a/src/main/options.c +++ b/src/main/options.c @@ -612,6 +612,8 @@ opts_t opts_parse(unsigned int argc, char **argv) { "stop-at-size", 0, NULL, (int) 'S' }, { "sync", 0, NULL, (int) 'Y' }, { "direct-io", 0, NULL, (int) 'K' }, + { "sparse", 0, NULL, (int) 'O' }, + { "sparse-output", 0, NULL, (int) 'O' }, { "discard", 0, NULL, (int) 'X' }, { "store-and-forward", 1, NULL, (int) 'U' }, { "remote", 1, NULL, (int) 'R' }, @@ -627,7 +629,7 @@ opts_t opts_parse(unsigned int argc, char **argv) /*@+nullassign@ */ int option_index = 0; #endif /* HAVE_GETOPT_LONG */ - char *short_options = "hVpteIrab8kTA:fvnqcWD:s:gl0i:w:H:N:u:F:x:L:B:CEZ:SYKXU:R:P:d:m:o:" + char *short_options = "hVpteIrab8kTA:fvnqcWD:s:gl0i:w:H:N:u:F:x:L:B:CEZ:SYKOXU:R:P:d:m:o:" #ifdef ENABLE_DEBUGGING "!:" #endif @@ -972,6 +974,10 @@ opts_t opts_parse(unsigned int argc, char **argv) case 'K': opts->direct_io = true; break; + case 'O': + opts->sparse_output = true; + opts->no_splice = true; + break; case 'X': opts->discard_input = true; opts->no_splice = true; diff --git a/src/pv/state.c b/src/pv/state.c index 8163af7..800b401 100644 --- a/src/pv/state.c +++ b/src/pv/state.c @@ -131,6 +131,7 @@ void pv_reset_transfer(pvtransferstate_t transfer) transfer->line_positions_length = 0; transfer->line_positions_head = 0; transfer->last_output_position = 0; + transfer->output_not_seekable = false; } @@ -256,6 +257,38 @@ void pv_freecontents_calc(pvtransfercalc_t calc) } +/* + * Truncate the output file descriptor to its current position, if it's a + * valid fd, we're in sparse output mode, and no lseek() failed. + */ +static void pv_truncate_output(pvstate_t state) +{ + off_t current_offset; + + if (!state->control.sparse_output) + return; + if (state->transfer.output_not_seekable) + return; + if (state->control.output_fd < 0) + return; + + current_offset = (off_t) lseek(state->control.output_fd, (off_t) 0, SEEK_CUR); + if (current_offset == (off_t) - 1) + return; + + debug("%s: %ld", "truncating to current offset", (long) current_offset); + + /*@+longintegral@ */ + /* + * splint has trouble with off_t / __off_t. + */ + if (0 != ftruncate(state->control.output_fd, current_offset)) { + debug("%s: %s", "output ftruncate() failed", strerror(errno)); + } + /*@-longintegral@ */ +} + + /* * Free a state structure, after which it can no longer be used. */ @@ -269,6 +302,7 @@ void pv_state_free(pvstate_t state) * still know the program name and output filename. */ if (state->control.output_fd >= 0) { + pv_truncate_output(state); if (STDOUT_FILENO != state->control.output_fd) { if (close(state->control.output_fd) < 0) { pv_error("%s: %s", @@ -494,6 +528,11 @@ void pv_state_direct_io_set(pvstate_t state, bool val) state->control.direct_io_changed = true; } +void pv_state_sparse_output_set(pvstate_t state, bool val) +{ + state->control.sparse_output = val; +} + void pv_state_discard_input_set(pvstate_t state, bool val) { state->control.discard_input = val; @@ -625,6 +664,7 @@ void pv_state_output_set(pvstate_t state, int fd, const char *name) * Close any previous output file first, so we can report any errors * before we store the new output filename. */ + pv_truncate_output(state); if (state->control.output_fd >= 0 && state->control.output_fd != STDOUT_FILENO) { if (close(state->control.output_fd) < 0) { pv_error("%s: %s", @@ -645,6 +685,22 @@ void pv_state_output_set(pvstate_t state, int fd, const char *name) */ fcntl(state->control.output_fd, F_SETFL, O_NONBLOCK | fcntl(state->control.output_fd, F_GETFL)); #endif /* MAKE_OUTPUT_NONBLOCKING */ + + /* + * In sparse output mode, if the output is in append mode (>>), + * explicitly lseek() to the end of the file. Otherwise, the file + * offset is not set until the first write(), which means that if + * the input starts with null bytes, when we lseek() past them + * relative to the current position, the "current position" is 0 + * rather than the end of the file, and the file gets truncated on + * exit to the wrong size. + */ + if (state->control.sparse_output && 0 != (fcntl(fd, F_GETFL) & O_APPEND)) { + debug("%s", "sparse output mode, and appending - seeking output to the end"); + if ((off_t) lseek(fd, 0, SEEK_END) == (off_t) - 1) { + debug("%s: %s", "lseek failed", strerror(errno)); + } + } } void pv_state_average_rate_window_set(pvstate_t state, unsigned int val) diff --git a/src/pv/transfer.c b/src/pv/transfer.c index 1f65ca1..8d70da3 100644 --- a/src/pv/transfer.c +++ b/src/pv/transfer.c @@ -655,6 +655,9 @@ static int pv__transfer_write(pvstate_t state, bool *eof_in, bool *eof_out, long { ssize_t nwritten; int write_errno; + bool all_nulls; + off_t output_offset; + size_t write_check_position, write_end_position; if (NULL == state->transfer.transfer_buffer) { pv_error("%s", _("no transfer buffer allocated")); @@ -671,6 +674,52 @@ static int pv__transfer_write(pvstate_t state, bool *eof_in, bool *eof_out, long nwritten = state->transfer.to_write; } else if (state->transfer.to_write > 0) { + /* + * In sparse output mode, check whether all of the bytes to + * be written are null, and if so, try to seek the output + * instead of writing the null bytes. + */ + if (state->control.sparse_output && !state->transfer.output_not_seekable) { + write_check_position = state->transfer.write_position; + write_end_position = write_check_position + (size_t) (state->transfer.to_write); + all_nulls = true; + while (all_nulls && write_check_position < write_end_position) { + if ('\0' == state->transfer.transfer_buffer[write_check_position]) { + write_check_position++; + } else { + all_nulls = false; + } + } + if (all_nulls) { + + /* + * Use lseek() to move forward in the file, + * and then ftruncate() it to its new size. + * + * If any step fails, mark the output not + * seekable so we stop trying. + */ + + /*@+longintegral@ */ + /* + * splint has trouble with off_t / __off_t, in the lseek() call. + */ + output_offset = + lseek(state->control.output_fd, (off_t) (state->transfer.to_write), SEEK_CUR); + if (output_offset == (off_t) - 1) { + debug("%s: %s", "output lseek() failed", strerror(errno)); + state->transfer.output_not_seekable = true; + } else { + /* Seek successful - skip write. */ + debug("%s (%ld) -> %s: %ld", "skipped null writes", + (long) (state->transfer.to_write), "new position", (long) output_offset); + nwritten = state->transfer.to_write; + goto pv__transfer_write_completed; + } + /*@-longintegral@ */ + } + } + /* * Set an interval timer or an alarm to interrupt the write * with a signal if the write takes too long, so we can @@ -735,6 +784,9 @@ static int pv__transfer_write(pvstate_t state, bool *eof_in, bool *eof_out, long #endif /* HAVE_SETITIMER */ } + pv__transfer_write_completed: + /* If lseek() worked for sparse output, it jumps down here. */ + if (nwritten > 0) { bool tracking_lines = false; diff --git a/tests/Sparse_-_Basic.test b/tests/Sparse_-_Basic.test new file mode 100755 index 0000000..1fefc87 --- /dev/null +++ b/tests/Sparse_-_Basic.test @@ -0,0 +1,89 @@ +#!/bin/sh +# +# Transfer a large chunk of data through pv using pipes, sending it in a +# bursty fashion, and make sure some of it consists of blocks of null bytes. +# Check data correctness afterwards, and if dd can produce a sparse file, +# check that pv's output is also sparse. + +# Allow all tests to be skipped, e.g. during a release build +test "${SKIP_ALL_TESTS}" = "1" && exit 77 + +true "${testSubject:?not set - call this from 'make check'}" +true "${workFile1:?not set - call this from 'make check'}" +true "${workFile2:?not set - call this from 'make check'}" +true "${workFile3:?not set - call this from 'make check'}" + +# generate some data, with a hole in the middle. +dd if=/dev/urandom of="${workFile1}" bs=4096 count=640 2>/dev/null +dd if=/dev/null of="${workFile1}" bs=4096 count=0 seek=1280 2>/dev/null +dd if=/dev/urandom of="${workFile1}" bs=4096 count=1280 seek=1280 2>/dev/null + +# check that sparse files are supported. +cat < "${workFile1}" > "${workFile3}" +inputBlocks="$(ls -1s "${workFile1}" | awk '{print $1}')" +nonSparseBlocks="$(ls -1s "${workFile3}" | awk '{print $1}')" +if test "${inputBlocks}" -eq "${nonSparseBlocks}"; then + echo 'sparse files not supported' + exit 77 +fi + +inputChecksum=$(cksum "${workFile1}" | awk '{print $1}') + +# read through pv and test afterwards +( +dd if="${workFile1}" bs=1 count=9000 +sleep 1 +dd if="${workFile1}" bs=1 skip=9000 count=1240 +sleep 1 +dd if="${workFile1}" bs=1024 skip=10 count=1014 +sleep 1 +dd if="${workFile1}" bs=1024 skip=1024 count=1024 +sleep 1 +dd if="${workFile1}" bs=1024 skip=2048 +) 2>/dev/null | "${testSubject}" -q -O -L 2M | cat > "${workFile2}" + +outputChecksum=$(cksum "${workFile2}" | awk '{print $1}') + +if ! test "${inputChecksum}" = "${outputChecksum}"; then + echo "checksum mismatch with dd | pv | cat" + exit 1 +fi + +# same again but with one less pipe +( +dd if="${workFile1}" bs=1 count=9000 +sleep 1 +dd if="${workFile1}" bs=1 skip=9000 count=1240 +sleep 1 +dd if="${workFile1}" bs=1024 skip=10 count=1014 +sleep 1 +dd if="${workFile1}" bs=1024 skip=1024 count=1024 +sleep 1 +dd if="${workFile1}" bs=1024 skip=2048 +) 2>/dev/null | "${testSubject}" -q -O -L 2M > "${workFile2}" + +outputChecksum=$(cksum "${workFile2}" | awk '{print $1}') + +if ! test "${inputChecksum}" = "${outputChecksum}"; then + echo "checksum mismatch with dd | pv > file" + exit 1 +fi + +outputBlocks="$(ls -1s "${workFile2}" | awk '{print $1}')" + +# Note that we only check that the output has any holes at all (and that the +# input had holes). We don't compare pv's output with dd's output directly, +# because the exact number of holes might differ intermittently. + +if test "${nonSparseBlocks}" -gt "${inputBlocks}" && ! test "${outputBlocks}" -lt "${nonSparseBlocks}"; then + echo "output is not sparse" + echo "input file:" + ls -ls "${workFile1}" + echo "output file:" + ls -ls "${workFile2}" + echo "input file without holes:" + ls -ls "${workFile3}" + exit 1 +fi + +exit 0