Merge branch 'issue45_sparse' (#45).

Makefile.am

@@ -95,6 +95,7 @@ tests/Modifiers_-_--line-mode.test \
 tests/Modifiers_-_--size_from_file_size.test \
 tests/Modifiers_-_--size.test \
 tests/Modifiers_-_--sync.test \
+tests/Sparse_-_Basic.test \
 tests/Terminal_-_Detect_width.test \
 tests/Transfer_-_--rate-limit.test \
 tests/Transfer_-_--remote.test \

docs/pv.1

@@ -273,9 +273,9 @@ from or to a pipe than regular \fBread\fR(2) and \fBwrite\fR(2), but means
 that the transfer buffer may not be used.
 This prevents \*(lq\fB\-\-buffer\-percent\fR\*(rq and
 \*(lq\fB\-\-last\-written\fR\*(rq from working, cannot work with
-\*(lq\fB\-\-discard\fR\*(rq, and makes \*(lq\fB\-\-buffer\-size\fR\*(rq
-redundant, so using any of those options automatically switches on
-\*(lq\fB\-\-no\-splice\fR\*(rq.
+\*(lq\fB\-\-sparse\fR\*(rq or \*(lq\fB\-\-discard\fR\*(rq, and makes
+\*(lq\fB\-\-buffer\-size\fR\*(rq redundant, so using any of those options
+automatically switches on \*(lq\fB\-\-no\-splice\fR\*(rq.
 Switching on this option results in a small loss of transfer efficiency.
 It has no effect on systems where \fBsplice\fR(2) is unavailable.
 .TP
@@ -325,6 +325,13 @@ failures with an error of \*(lqInvalid argument\*(rq, especially if reading
 and writing files across a variety of filesystems in a single \fBpv\fR call.
 Use this option with caution.
 .TP
+.B \-O, \-\-sparse
+When writing null bytes, try to seek, producing a sparse output file.
+Implies \*(lq\fB\-\-no\-splice\fR\*(rq.
+On filesystems without sparse file support, or when the output is not
+seekable, this option will have no effect other than to turn on
+\*(lq\fB\-\-no\-splice\fR\*(rq.
+.TP
 .B \-X, \-\-discard
 Instead of transferring input data to standard output, discard it.
 This is equivalent to redirecting standard output to \fI/dev/null\fR,

docs/pv.1.md

@@ -281,11 +281,11 @@ are explicitly switched on will be shown.
     data from or to a pipe than regular **read**(2) and **write**(2),
     but means that the transfer buffer may not be used. This prevents
     "**\--buffer-percent**" and "**\--last-written**" from working,
-    cannot work with "**\--discard**", and makes "**\--buffer-size**"
-    redundant, so using any of those options automatically switches on
-    "**\--no-splice**". Switching on this option results in a small loss
-    of transfer efficiency. It has no effect on systems where
-    **splice**(2) is unavailable.
+    cannot work with "**\--sparse**" or "**\--discard**", and makes
+    "**\--buffer-size**" redundant, so using any of those options
+    automatically switches on "**\--no-splice**". Switching on this
+    option results in a small loss of transfer efficiency. It has no
+    effect on systems where **splice**(2) is unavailable.
 
 **-E, \--skip-errors**
 
@@ -336,6 +336,13 @@ are explicitly switched on will be shown.
     especially if reading and writing files across a variety of
     filesystems in a single **pv** call. Use this option with caution.
 
+**-O, \--sparse**
+
+:   When writing null bytes, try to seek, producing a sparse output
+    file. Implies "**\--no-splice**". On filesystems without sparse file
+    support, or when the output is not seekable, this option will have
+    no effect other than to turn on "**\--no-splice**".
+
 **-X, \--discard**
 
 :   Instead of transferring input data to standard output, discard it.

src/include/pv.h

@@ -221,6 +221,7 @@ extern void pv_state_error_skip_block_set(pvstate_t, off_t);
 extern void pv_state_stop_at_size_set(pvstate_t, bool);
 extern void pv_state_sync_after_write_set(pvstate_t, bool);
 extern void pv_state_direct_io_set(pvstate_t, bool);
+extern void pv_state_sparse_output_set(pvstate_t, bool);
 extern void pv_state_rate_limit_set(pvstate_t, off_t);
 extern void pv_state_target_buffer_size_set(pvstate_t, size_t);
 extern void pv_state_no_splice_set(pvstate_t, bool);

src/main/help.c

@@ -351,6 +351,9 @@ void display_help(void)
 		{ "-K", "--direct-io", NULL,
 		 N_("use direct I/O to bypass cache"),
 		 { 0, 0, 0, 0} },
+		{ "-O", "--sparse", NULL,
+		 N_("try to seek instead of writing null bytes"),
+		 { 0, 0, 0, 0} },
 		{ "-X", "--discard", NULL,
 		 N_("discard input instead of writing to output"),
 		 { 0, 0, 0, 0} },

src/main/main.c

@@ -448,7 +448,12 @@ int main(int argc, char **argv)
 	 * do this before looking at setting the size, as the size
 	 * calculation looks at the output file if the input size can't be
 	 * calculated (issue #91).
+	 *
+	 * We have to set the sparse output flag before doing this, so that
+	 * in sparse mode the lseek() on O_APPEND can be done (issue #45);
+	 * see the comments in pv_state_output_set() in src/pv/state.c.
 	 */
+	pv_state_sparse_output_set(state, opts->sparse_output);
 	retcode = pv__set_output(state, opts, opts->output);
 	if (0 != retcode) {
 		pv_state_free(state);

src/main/options.c

@@ -612,6 +612,8 @@ opts_t opts_parse(unsigned int argc, char **argv)
 		{ "stop-at-size", 0, NULL, (int) 'S' },
 		{ "sync", 0, NULL, (int) 'Y' },
 		{ "direct-io", 0, NULL, (int) 'K' },
+		{ "sparse", 0, NULL, (int) 'O' },
+		{ "sparse-output", 0, NULL, (int) 'O' },
 		{ "discard", 0, NULL, (int) 'X' },
 		{ "store-and-forward", 1, NULL, (int) 'U' },
 		{ "remote", 1, NULL, (int) 'R' },
@@ -627,7 +629,7 @@ opts_t opts_parse(unsigned int argc, char **argv)
 	/*@+nullassign@ */
 	int option_index = 0;
 #endif				/* HAVE_GETOPT_LONG */
-	char *short_options = "hVpteIrab8kTA:fvnqcWD:s:gl0i:w:H:N:u:F:x:L:B:CEZ:SYKXU:R:P:d:m:o:"
+	char *short_options = "hVpteIrab8kTA:fvnqcWD:s:gl0i:w:H:N:u:F:x:L:B:CEZ:SYKOXU:R:P:d:m:o:"
 #ifdef ENABLE_DEBUGGING
 	    "!:"
 #endif
@@ -972,6 +974,10 @@ opts_t opts_parse(unsigned int argc, char **argv)
 		case 'K':
 			opts->direct_io = true;
 			break;
+		case 'O':
+			opts->sparse_output = true;
+			opts->no_splice = true;
+			break;
 		case 'X':
 			opts->discard_input = true;
 			opts->no_splice = true;

src/pv/state.c

@@ -131,6 +131,7 @@ void pv_reset_transfer(pvtransferstate_t transfer)
 	transfer->line_positions_length = 0;
 	transfer->line_positions_head = 0;
 	transfer->last_output_position = 0;
+	transfer->output_not_seekable = false;
 }
 
 
@@ -256,6 +257,38 @@ void pv_freecontents_calc(pvtransfercalc_t calc)
 }
 
 
+/*
+ * Truncate the output file descriptor to its current position, if it's a
+ * valid fd, we're in sparse output mode, and no lseek() failed.
+ */
+static void pv_truncate_output(pvstate_t state)
+{
+	off_t current_offset;
+
+	if (!state->control.sparse_output)
+		return;
+	if (state->transfer.output_not_seekable)
+		return;
+	if (state->control.output_fd < 0)
+		return;
+
+	current_offset = (off_t) lseek(state->control.output_fd, (off_t) 0, SEEK_CUR);
+	if (current_offset == (off_t) - 1)
+		return;
+
+	debug("%s: %ld", "truncating to current offset", (long) current_offset);
+
+	/*@+longintegral@ */
+	/*
+	 * splint has trouble with off_t / __off_t.
+	 */
+	if (0 != ftruncate(state->control.output_fd, current_offset)) {
+		debug("%s: %s", "output ftruncate() failed", strerror(errno));
+	}
+	/*@-longintegral@ */
+}
+
+
 /*
  * Free a state structure, after which it can no longer be used.
  */
@@ -269,6 +302,7 @@ void pv_state_free(pvstate_t state)
 	 * still know the program name and output filename.
 	 */
 	if (state->control.output_fd >= 0) {
+		pv_truncate_output(state);
 		if (STDOUT_FILENO != state->control.output_fd) {
 			if (close(state->control.output_fd) < 0) {
 				pv_error("%s: %s",
@@ -494,6 +528,11 @@ void pv_state_direct_io_set(pvstate_t state, bool val)
 	state->control.direct_io_changed = true;
 }
 
+void pv_state_sparse_output_set(pvstate_t state, bool val)
+{
+	state->control.sparse_output = val;
+}
+
 void pv_state_discard_input_set(pvstate_t state, bool val)
 {
 	state->control.discard_input = val;
@@ -625,6 +664,7 @@ void pv_state_output_set(pvstate_t state, int fd, const char *name)
 	 * Close any previous output file first, so we can report any errors
 	 * before we store the new output filename.
 	 */
+	pv_truncate_output(state);
 	if (state->control.output_fd >= 0 && state->control.output_fd != STDOUT_FILENO) {
 		if (close(state->control.output_fd) < 0) {
 			pv_error("%s: %s",
@@ -645,6 +685,22 @@ void pv_state_output_set(pvstate_t state, int fd, const char *name)
 	 */
 	fcntl(state->control.output_fd, F_SETFL, O_NONBLOCK | fcntl(state->control.output_fd, F_GETFL));
 #endif				/* MAKE_OUTPUT_NONBLOCKING */
+
+	/*
+	 * In sparse output mode, if the output is in append mode (>>),
+	 * explicitly lseek() to the end of the file.  Otherwise, the file
+	 * offset is not set until the first write(), which means that if
+	 * the input starts with null bytes, when we lseek() past them
+	 * relative to the current position, the "current position" is 0
+	 * rather than the end of the file, and the file gets truncated on
+	 * exit to the wrong size.
+	 */
+	if (state->control.sparse_output && 0 != (fcntl(fd, F_GETFL) & O_APPEND)) {
+		debug("%s", "sparse output mode, and appending - seeking output to the end");
+		if ((off_t) lseek(fd, 0, SEEK_END) == (off_t) - 1) {
+			debug("%s: %s", "lseek failed", strerror(errno));
+		}
+	}
 }
 
 void pv_state_average_rate_window_set(pvstate_t state, unsigned int val)

src/pv/transfer.c

@@ -655,6 +655,9 @@ static int pv__transfer_write(pvstate_t state, bool *eof_in, bool *eof_out, long
 {
 	ssize_t nwritten;
 	int write_errno;
+	bool all_nulls;
+	off_t output_offset;
+	size_t write_check_position, write_end_position;
 
 	if (NULL == state->transfer.transfer_buffer) {
 		pv_error("%s", _("no transfer buffer allocated"));
@@ -671,6 +674,52 @@ static int pv__transfer_write(pvstate_t state, bool *eof_in, bool *eof_out, long
 		nwritten = state->transfer.to_write;
 	} else if (state->transfer.to_write > 0) {
 
+		/*
+		 * In sparse output mode, check whether all of the bytes to
+		 * be written are null, and if so, try to seek the output
+		 * instead of writing the null bytes.
+		 */
+		if (state->control.sparse_output && !state->transfer.output_not_seekable) {
+			write_check_position = state->transfer.write_position;
+			write_end_position = write_check_position + (size_t) (state->transfer.to_write);
+			all_nulls = true;
+			while (all_nulls && write_check_position < write_end_position) {
+				if ('\0' == state->transfer.transfer_buffer[write_check_position]) {
+					write_check_position++;
+				} else {
+					all_nulls = false;
+				}
+			}
+			if (all_nulls) {
+
+				/*
+				 * Use lseek() to move forward in the file,
+				 * and then ftruncate() it to its new size.
+				 *
+				 * If any step fails, mark the output not
+				 * seekable so we stop trying.
+				 */
+
+				/*@+longintegral@ */
+				/*
+				 * splint has trouble with off_t / __off_t, in the lseek() call.
+				 */
+				output_offset =
+				    lseek(state->control.output_fd, (off_t) (state->transfer.to_write), SEEK_CUR);
+				if (output_offset == (off_t) - 1) {
+					debug("%s: %s", "output lseek() failed", strerror(errno));
+					state->transfer.output_not_seekable = true;
+				} else {
+					/* Seek successful - skip write. */
+					debug("%s (%ld) -> %s: %ld", "skipped null writes",
+					      (long) (state->transfer.to_write), "new position", (long) output_offset);
+					nwritten = state->transfer.to_write;
+					goto pv__transfer_write_completed;
+				}
+				/*@-longintegral@ */
+			}
+		}
+
 		/*
 		 * Set an interval timer or an alarm to interrupt the write
 		 * with a signal if the write takes too long, so we can
@@ -735,6 +784,9 @@ static int pv__transfer_write(pvstate_t state, bool *eof_in, bool *eof_out, long
 #endif				/* HAVE_SETITIMER */
 	}
 
+      pv__transfer_write_completed:
+	/* If lseek() worked for sparse output, it jumps down here. */
+
 	if (nwritten > 0) {
 		bool tracking_lines = false;
 

tests/Sparse_-_Basic.test

@@ -0,0 +1,89 @@
+#!/bin/sh
+#
+# Transfer a large chunk of data through pv using pipes, sending it in a
+# bursty fashion, and make sure some of it consists of blocks of null bytes. 
+# Check data correctness afterwards, and if dd can produce a sparse file,
+# check that pv's output is also sparse.
+
+# Allow all tests to be skipped, e.g. during a release build
+test "${SKIP_ALL_TESTS}" = "1" && exit 77
+
+true "${testSubject:?not set - call this from 'make check'}"
+true "${workFile1:?not set - call this from 'make check'}"
+true "${workFile2:?not set - call this from 'make check'}"
+true "${workFile3:?not set - call this from 'make check'}"
+
+# generate some data, with a hole in the middle.
+dd if=/dev/urandom of="${workFile1}" bs=4096 count=640 2>/dev/null
+dd if=/dev/null of="${workFile1}" bs=4096 count=0 seek=1280 2>/dev/null
+dd if=/dev/urandom of="${workFile1}" bs=4096 count=1280 seek=1280 2>/dev/null
+
+# check that sparse files are supported.
+cat < "${workFile1}" > "${workFile3}"
+inputBlocks="$(ls -1s "${workFile1}" | awk '{print $1}')"
+nonSparseBlocks="$(ls -1s "${workFile3}" | awk '{print $1}')"
+if test "${inputBlocks}" -eq "${nonSparseBlocks}"; then
+	echo 'sparse files not supported'
+	exit 77
+fi
+
+inputChecksum=$(cksum "${workFile1}" | awk '{print $1}')
+
+# read through pv and test afterwards
+(
+dd if="${workFile1}" bs=1 count=9000
+sleep 1
+dd if="${workFile1}" bs=1 skip=9000 count=1240
+sleep 1
+dd if="${workFile1}" bs=1024 skip=10 count=1014
+sleep 1
+dd if="${workFile1}" bs=1024 skip=1024 count=1024
+sleep 1
+dd if="${workFile1}" bs=1024 skip=2048
+) 2>/dev/null | "${testSubject}" -q -O -L 2M | cat > "${workFile2}"
+
+outputChecksum=$(cksum "${workFile2}" | awk '{print $1}')
+
+if ! test "${inputChecksum}" = "${outputChecksum}"; then
+	echo "checksum mismatch with dd | pv | cat"
+	exit 1
+fi
+
+# same again but with one less pipe
+(
+dd if="${workFile1}" bs=1 count=9000
+sleep 1
+dd if="${workFile1}" bs=1 skip=9000 count=1240
+sleep 1
+dd if="${workFile1}" bs=1024 skip=10 count=1014
+sleep 1
+dd if="${workFile1}" bs=1024 skip=1024 count=1024
+sleep 1
+dd if="${workFile1}" bs=1024 skip=2048
+) 2>/dev/null | "${testSubject}" -q -O -L 2M > "${workFile2}"
+
+outputChecksum=$(cksum "${workFile2}" | awk '{print $1}')
+
+if ! test "${inputChecksum}" = "${outputChecksum}"; then
+	echo "checksum mismatch with dd | pv > file"
+	exit 1
+fi
+
+outputBlocks="$(ls -1s "${workFile2}" | awk '{print $1}')"
+
+# Note that we only check that the output has any holes at all (and that the
+# input had holes).  We don't compare pv's output with dd's output directly,
+# because the exact number of holes might differ intermittently.
+
+if test "${nonSparseBlocks}" -gt "${inputBlocks}" && ! test "${outputBlocks}" -lt "${nonSparseBlocks}"; then
+	echo "output is not sparse"
+	echo "input file:"
+	ls -ls "${workFile1}"
+	echo "output file:"
+	ls -ls "${workFile2}"
+	echo "input file without holes:"
+	ls -ls "${workFile3}"
+	exit 1
+fi
+
+exit 0