mirrors/pv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'issue45_sparse' (#45).

Browse Source
This commit is contained in:
Andrew Wood
2025-10-15 23:17:05 +01:00
parent f0895177e7 3c9ab822fe
commit a40bdfb55f
10 changed files with 236 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/include/pv.h
+1
View File
@@ -221,6 +221,7 @@ extern void pv_state_error_skip_block_set(pvstate_t, off_t);
extern void pv_state_stop_at_size_set(pvstate_t, bool);
extern void pv_state_sync_after_write_set(pvstate_t, bool);
extern void pv_state_direct_io_set(pvstate_t, bool);
extern void pv_state_sparse_output_set(pvstate_t, bool);
extern void pv_state_rate_limit_set(pvstate_t, off_t);
extern void pv_state_target_buffer_size_set(pvstate_t, size_t);
extern void pv_state_no_splice_set(pvstate_t, bool);
src/main/help.c
+3
View File
@@ -351,6 +351,9 @@ void display_help(void)
{ "-K", "--direct-io", NULL,
N_("use direct I/O to bypass cache"),
{ 0, 0, 0, 0} },
{ "-O", "--sparse", NULL,
N_("try to seek instead of writing null bytes"),
{ 0, 0, 0, 0} },
{ "-X", "--discard", NULL,
N_("discard input instead of writing to output"),
{ 0, 0, 0, 0} },
src/main/main.c
+5
View File
@@ -448,7 +448,12 @@ int main(int argc, char **argv)
* do this before looking at setting the size, as the size
* calculation looks at the output file if the input size can't be
* calculated (issue #91).
*
* We have to set the sparse output flag before doing this, so that
* in sparse mode the lseek() on O_APPEND can be done (issue #45);
* see the comments in pv_state_output_set() in src/pv/state.c.
*/
pv_state_sparse_output_set(state, opts->sparse_output);
retcode = pv__set_output(state, opts, opts->output);
if (0 != retcode) {
pv_state_free(state);
src/main/options.c
+7 -1
View File
@@ -612,6 +612,8 @@ opts_t opts_parse(unsigned int argc, char **argv)
{ "stop-at-size", 0, NULL, (int) 'S' },
{ "sync", 0, NULL, (int) 'Y' },
{ "direct-io", 0, NULL, (int) 'K' },
{ "sparse", 0, NULL, (int) 'O' },
{ "sparse-output", 0, NULL, (int) 'O' },
{ "discard", 0, NULL, (int) 'X' },
{ "store-and-forward", 1, NULL, (int) 'U' },
{ "remote", 1, NULL, (int) 'R' },
@@ -627,7 +629,7 @@ opts_t opts_parse(unsigned int argc, char **argv)
/*@+nullassign@ */
int option_index = 0;
#endif /* HAVE_GETOPT_LONG */
char *short_options = "hVpteIrab8kTA:fvnqcWD:s:gl0i:w:H:N:u:F:x:L:B:CEZ:SYKXU:R:P:d:m:o:"
char *short_options = "hVpteIrab8kTA:fvnqcWD:s:gl0i:w:H:N:u:F:x:L:B:CEZ:SYKOXU:R:P:d:m:o:"
#ifdef ENABLE_DEBUGGING
"!:"
#endif
@@ -972,6 +974,10 @@ opts_t opts_parse(unsigned int argc, char **argv)
case 'K':
opts->direct_io = true;
break;
case 'O':
opts->sparse_output = true;
opts->no_splice = true;
break;
case 'X':
opts->discard_input = true;
opts->no_splice = true;
src/pv/state.c
+56
View File
@@ -131,6 +131,7 @@ void pv_reset_transfer(pvtransferstate_t transfer)
transfer->line_positions_length = 0;
transfer->line_positions_head = 0;
transfer->last_output_position = 0;
transfer->output_not_seekable = false;
}
@@ -256,6 +257,38 @@ void pv_freecontents_calc(pvtransfercalc_t calc)
}
/*
* Truncate the output file descriptor to its current position, if it's a
* valid fd, we're in sparse output mode, and no lseek() failed.
*/
static void pv_truncate_output(pvstate_t state)
{
off_t current_offset;
if (!state->control.sparse_output)
return;
if (state->transfer.output_not_seekable)
return;
if (state->control.output_fd < 0)
return;
current_offset = (off_t) lseek(state->control.output_fd, (off_t) 0, SEEK_CUR);
if (current_offset == (off_t) - 1)
return;
debug("%s: %ld", "truncating to current offset", (long) current_offset);
/*@+longintegral@ */
/*
* splint has trouble with off_t / __off_t.
*/
if (0 != ftruncate(state->control.output_fd, current_offset)) {
debug("%s: %s", "output ftruncate() failed", strerror(errno));
}
/*@-longintegral@ */
}
/*
* Free a state structure, after which it can no longer be used.
*/
@@ -269,6 +302,7 @@ void pv_state_free(pvstate_t state)
* still know the program name and output filename.
*/
if (state->control.output_fd >= 0) {
pv_truncate_output(state);
if (STDOUT_FILENO != state->control.output_fd) {
if (close(state->control.output_fd) < 0) {
pv_error("%s: %s",
@@ -494,6 +528,11 @@ void pv_state_direct_io_set(pvstate_t state, bool val)
state->control.direct_io_changed = true;
}
void pv_state_sparse_output_set(pvstate_t state, bool val)
{
state->control.sparse_output = val;
}
void pv_state_discard_input_set(pvstate_t state, bool val)
{
state->control.discard_input = val;
@@ -625,6 +664,7 @@ void pv_state_output_set(pvstate_t state, int fd, const char *name)
* Close any previous output file first, so we can report any errors
* before we store the new output filename.
*/
pv_truncate_output(state);
if (state->control.output_fd >= 0 && state->control.output_fd != STDOUT_FILENO) {
if (close(state->control.output_fd) < 0) {
pv_error("%s: %s",
@@ -645,6 +685,22 @@ void pv_state_output_set(pvstate_t state, int fd, const char *name)
*/
fcntl(state->control.output_fd, F_SETFL, O_NONBLOCK | fcntl(state->control.output_fd, F_GETFL));
#endif /* MAKE_OUTPUT_NONBLOCKING */
/*
* In sparse output mode, if the output is in append mode (>>),
* explicitly lseek() to the end of the file. Otherwise, the file
* offset is not set until the first write(), which means that if
* the input starts with null bytes, when we lseek() past them
* relative to the current position, the "current position" is 0
* rather than the end of the file, and the file gets truncated on
* exit to the wrong size.
*/
if (state->control.sparse_output && 0 != (fcntl(fd, F_GETFL) & O_APPEND)) {
debug("%s", "sparse output mode, and appending - seeking output to the end");
if ((off_t) lseek(fd, 0, SEEK_END) == (off_t) - 1) {
debug("%s: %s", "lseek failed", strerror(errno));
}
}
}
void pv_state_average_rate_window_set(pvstate_t state, unsigned int val)
src/pv/transfer.c
+52
View File
@@ -655,6 +655,9 @@ static int pv__transfer_write(pvstate_t state, bool *eof_in, bool *eof_out, long
{
ssize_t nwritten;
int write_errno;
bool all_nulls;
off_t output_offset;
size_t write_check_position, write_end_position;
if (NULL == state->transfer.transfer_buffer) {
pv_error("%s", _("no transfer buffer allocated"));
@@ -671,6 +674,52 @@ static int pv__transfer_write(pvstate_t state, bool *eof_in, bool *eof_out, long
nwritten = state->transfer.to_write;
} else if (state->transfer.to_write > 0) {
/*
* In sparse output mode, check whether all of the bytes to
* be written are null, and if so, try to seek the output
* instead of writing the null bytes.
*/
if (state->control.sparse_output && !state->transfer.output_not_seekable) {
write_check_position = state->transfer.write_position;
write_end_position = write_check_position + (size_t) (state->transfer.to_write);
all_nulls = true;
while (all_nulls && write_check_position < write_end_position) {
if ('\0' == state->transfer.transfer_buffer[write_check_position]) {
write_check_position++;
} else {
all_nulls = false;
}
}
if (all_nulls) {
/*
* Use lseek() to move forward in the file,
* and then ftruncate() it to its new size.
*
* If any step fails, mark the output not
* seekable so we stop trying.
*/
/*@+longintegral@ */
/*
* splint has trouble with off_t / __off_t, in the lseek() call.
*/
output_offset =
lseek(state->control.output_fd, (off_t) (state->transfer.to_write), SEEK_CUR);
if (output_offset == (off_t) - 1) {
debug("%s: %s", "output lseek() failed", strerror(errno));
state->transfer.output_not_seekable = true;
} else {
/* Seek successful - skip write. */
debug("%s (%ld) -> %s: %ld", "skipped null writes",
(long) (state->transfer.to_write), "new position", (long) output_offset);
nwritten = state->transfer.to_write;
goto pv__transfer_write_completed;
}
/*@-longintegral@ */
}
}
/*
* Set an interval timer or an alarm to interrupt the write
* with a signal if the write takes too long, so we can
@@ -735,6 +784,9 @@ static int pv__transfer_write(pvstate_t state, bool *eof_in, bool *eof_out, long
#endif /* HAVE_SETITIMER */
}
pv__transfer_write_completed:
/* If lseek() worked for sparse output, it jumps down here. */
if (nwritten > 0) {
bool tracking_lines = false;