Fix handling of files removed during incremental dumps.

Changes to src/create.c and src/incremen.c are partially based on patch from Alexander Peslyak <solar at openwall.com>. The new testcases require paxutils commit f653a2b or later. * src/common.h (struct name): New member `cmdline'. (dump_file): Change type of the 2nd argument to bool. (file_removed_diag, dir_removed_diag): New prototypes. (addname): New argument `cmdline'. (name_from_list): Change return value. * src/create.c (dump_dir0, dump_dir): top_level is bool. (create_archive): Update calls to name_from_list. Take advantage of the name->cmdline to set top_level argument during incremental backups. (dump_file0): top_level is bool. Do not bail out if a no-top-level file disappears during incremental backup, use file_removed_diag instead. (dump_filed): top_level is bool. * src/incremen.c (update_parent_directory): Silently ignore ENOENT. It should have already been reported elsewhere. (scan_directory): Use dir_removed_diag to report missing directories. * src/misc.c (file_removed_diag, dir_removed_diag): New functions. * src/names.c (name_gather): Set ->cmdname. (addname): Likewise. All uses updated. (name_from_list): Return struct name const *. All uses updated. * tests/filerem01.at: New testcase. * tests/filerem02.at: New testcase. * tests/Makefile.am, tests/testsuite.at: Add filerem01.at, filerem02.at * tests/grow.at, test/truncate.at: Use new syntax for genfile --run. * NEWS: Update. * doc/tar.texi: Minor fix.
2009-08-08 19:53:54 +03:00
parent 2b1bffbad6
commit 51aee274e8
15 changed files with 250 additions and 51 deletions
--- a/17
+++ b/17
@@ -31,6 +31,23 @@ options.  So far the only meaningful value for N is 0.  The
 `--level=0' option forces creating the level 0 dump, by truncating
 the snapshot file if it exists.

+* Files removed during incremental dumps
+
+If a file or directory is removed while incremental dump is
+in progress, tar exact actions depend on whether this file
+was explicitly listed in the command line, or was gathered
+during file system scan.
+
+If the file was explicitly listed in the command line, tar
+issues error messages and exits with the code 2, meaning
+fatal error.
+
+Otherwise, if the file was gathered during the file system
+scan, tar issues a warning, saying "File removed before we read it",
+and sets the exit code to 1, which means "some files differ".
+If the --warning=no-file-removed option is given, no warning
+is issued and the exit code remains 0.
+
 * Bugfixes
 ** Fix handling of hard link targets by -c --transform.
 ** Fix hard links recognition with -c --remove-files.
--- a/doc/tar.texi
+++ b/doc/tar.texi
@@ -2472,7 +2472,7 @@ messages as it reads through the archive.  It is intended for when you
 want a visual indication that @command{tar} is still running, but
 don't want to see @option{--verbose} output.  You can also instruct
@command{tar} to execute a list of actions on each checkpoint, see
-@option{--checklist-action} below.  For a detailed description, see
+@option{--checkpoint-action} below.  For a detailed description, see
@ref{checkpoints}.

@opsummary{checkpoint-action}
--- a/src/common.h
+++ b/src/common.h
@@ -331,8 +331,10 @@ struct name

    char *name;                 /* File name or globbing pattern */
    size_t length;		/* cached strlen (name) */
-    int matching_flags;		/* wildcard flags if name is a pattern */
-
+    int matching_flags;         /* wildcard flags if name is a pattern */
+    bool cmdline;               /* true if this name was given in the
+				   command line */
+    
    int change_dir;		/* Number of the directory to change to.
 				   Set with the -C option. */
    uintmax_t found_count;	/* number of times a matching file has
@@ -443,7 +445,7 @@ bool cachedir_file_p (const char *name);
 bool file_dumpable_p (struct tar_stat_info *st);
 void create_archive (void);
 void pad_archive (off_t size_left);
-void dump_file (const char *st, int top_level, dev_t parent_device);
+void dump_file (const char *st, bool top_level, dev_t parent_device);
 union block *start_header (struct tar_stat_info *st);
 void finish_header (struct tar_stat_info *st, union block *header,
 		    off_t block_ordinal);
@@ -629,6 +631,10 @@ void readlink_diag (char const *name);
 void savedir_diag (char const *name);
 void seek_diag_details (char const *name, off_t offset);
 void stat_diag (char const *name);
+void file_removed_diag (const char *name, bool top_level,
+			void (*diagfn) (char const *name));
+void dir_removed_diag (char const *name, bool top_level,
+		       void (*diagfn) (char const *name));
 void write_error_details (char const *name, size_t status, size_t size);
 void write_fatal (char const *name) __attribute__ ((noreturn));
 void write_fatal_details (char const *name, ssize_t status, size_t size)
@@ -656,12 +662,13 @@ void name_add_dir (const char *name);
 void name_term (void);
 const char *name_next (int change_dirs);
 void name_gather (void);
-struct name *addname (char const *string, int change_dir, struct name *parent);
+struct name *addname (char const *string, int change_dir,
+		      bool cmdline, struct name *parent);
 bool name_match (const char *name);
 void names_notfound (void);
 void collect_and_sort_names (void);
 struct name *name_scan (const char *name);
-char *name_from_list (void);
+struct name const *name_from_list (void);
 void blank_name_list (void);
 char *new_name (const char *dir_name, const char *name);
 size_t stripped_prefix_len (char const *file_name, size_t num);
--- a/src/create.c
+++ b/src/create.c
@@ -1092,7 +1092,7 @@ dump_regular_file (int fd, struct tar_stat_info *st)

 static void
 dump_dir0 (char *directory,
-	   struct tar_stat_info *st, int top_level, dev_t parent_device)
+	   struct tar_stat_info *st, bool top_level, dev_t parent_device)
 {
  dev_t our_device = st->stat.st_dev;
  const char *tag_file_name;
@@ -1210,7 +1210,7 @@ dump_dir0 (char *directory,
 		  }
 		strcpy (name_buf + name_len, entry);
 		if (!excluded_name (name_buf))
-		  dump_file (name_buf, 0, our_device);
+		  dump_file (name_buf, false, our_device);
 	      }
 	    
 	    free (name_buf);
@@ -1224,7 +1224,7 @@ dump_dir0 (char *directory,
 	  name_buf = xmalloc (name_size);
 	  strcpy (name_buf, st->orig_file_name);
 	  strcat (name_buf, tag_file_name);
-	  dump_file (name_buf, 0, our_device);
+	  dump_file (name_buf, false, our_device);
 	  free (name_buf);
 	  break;
      
@@ -1250,7 +1250,8 @@ ensure_slash (char **pstr)
 }

 static bool
-dump_dir (int fd, struct tar_stat_info *st, int top_level, dev_t parent_device)
+dump_dir (int fd, struct tar_stat_info *st, bool top_level,
+	  dev_t parent_device)
 {
  char *directory = fdsavedir (fd);
  if (!directory)
@@ -1271,7 +1272,7 @@ dump_dir (int fd, struct tar_stat_info *st, int top_level, dev_t parent_device)
 void
 create_archive (void)
 {
-  const char *p;
+  struct name const *p;

  open_archive (ACCESS_WRITE);
  buffer_write_global_xheader ();
@@ -1285,21 +1286,21 @@ create_archive (void)
      collect_and_sort_names ();

      while ((p = name_from_list ()) != NULL)
-	if (!excluded_name (p))
-	  dump_file (p, -1, (dev_t) 0);
+	if (!excluded_name (p->name))
+	  dump_file (p->name, p->cmdline, (dev_t) 0);

      blank_name_list ();
      while ((p = name_from_list ()) != NULL)
-	if (!excluded_name (p))
+	if (!excluded_name (p->name))
 	  {
-	    size_t plen = strlen (p);
+	    size_t plen = strlen (p->name);
 	    if (buffer_size <= plen)
 	      {
 		while ((buffer_size *= 2) <= plen)
 		  continue;
 		buffer = xrealloc (buffer, buffer_size);
 	      }
-	    memcpy (buffer, p, plen);
+	    memcpy (buffer, p->name, plen);
 	    if (! ISSLASH (buffer[plen - 1]))
 	      buffer[plen++] = DIRECTORY_SEPARATOR;
 	    q = directory_contents (gnu_list_name->directory);
@@ -1316,7 +1317,7 @@ create_archive (void)
 			  buffer = xrealloc (buffer, buffer_size);
 			}
 		      strcpy (buffer + plen, q + 1);
-		      dump_file (buffer, -1, (dev_t) 0);
+		      dump_file (buffer, false, (dev_t) 0);
 		    }
 		  q += qlen + 1;
 		}
@@ -1325,9 +1326,10 @@ create_archive (void)
    }
  else
    {
-      while ((p = name_next (1)) != NULL)
-	if (!excluded_name (p))
-	  dump_file (p, 1, (dev_t) 0);
+      const char *name;
+      while ((name = name_next (1)) != NULL)
+	if (!excluded_name (name))
+	  dump_file (name, true, (dev_t) 0);
    }

  write_eot ();
@@ -1475,7 +1477,6 @@ check_links (void)
    }
 }

-
 /* Dump a single file, recursing on directories.  P is the file name
   to dump.  TOP_LEVEL tells whether this is a top-level call; zero
   means no, positive means yes, and negative means the top level
@@ -1487,7 +1488,7 @@ check_links (void)

 static void
 dump_file0 (struct tar_stat_info *st, const char *p,
-	    int top_level, dev_t parent_device)
+	    bool top_level, dev_t parent_device)
 {
  union block *header;
  char type;
@@ -1508,7 +1509,7 @@ dump_file0 (struct tar_stat_info *st, const char *p,

  if (deref_stat (dereference_option, p, &st->stat) != 0)
    {
-      stat_diag (p);
+      file_removed_diag (p, top_level, stat_diag);
      return;
    }
  st->archive_file_size = original_size = st->stat.st_size;
@@ -1580,12 +1581,7 @@ dump_file0 (struct tar_stat_info *st, const char *p,
 			 : 0)));
 	  if (fd < 0)
 	    {
-	      if (!top_level && errno == ENOENT)
-		WARNOPT (WARN_FILE_REMOVED,
-			 (0, 0, _("%s: File removed before we read it"),
-			  quotearg_colon (p)));
-	      else
-		open_diag (p);
+	      file_removed_diag (p, top_level, open_diag);
 	      return;
 	    }
 	}
@@ -1655,7 +1651,7 @@ dump_file0 (struct tar_stat_info *st, const char *p,
 	       : fstat (fd, &final_stat))
 	      != 0)
 	    {
-	      stat_diag (p);
+	      file_removed_diag (p, top_level, stat_diag);
 	      ok = false;
 	    }
 	}
@@ -1713,7 +1709,7 @@ dump_file0 (struct tar_stat_info *st, const char *p,
      size = readlink (p, buffer, linklen + 1);
      if (size < 0)
 	{
-	  readlink_diag (p);
+	  file_removed_diag (p, top_level, readlink_diag);
 	  return;
 	}
      buffer[size] = '\0';
@@ -1795,7 +1791,7 @@ dump_file0 (struct tar_stat_info *st, const char *p,
 }

 void
-dump_file (const char *p, int top_level, dev_t parent_device)
+dump_file (const char *p, bool top_level, dev_t parent_device)
 {
  struct tar_stat_info st;
  tar_stat_init (&st);
--- a/src/incremen.c
+++ b/src/incremen.c
@@ -413,7 +413,11 @@ update_parent_directory (const char *name)
    {
      struct stat st;
      if (deref_stat (dereference_option, p, &st) != 0)
-	stat_diag (name);
+	{
+	  if (errno != ENOENT) 
+	    stat_diag (directory->name);
+	  /* else: should have been already reported */
+	}
      else
 	directory->mtime = get_stat_mtime (&st);
    }
@@ -549,6 +553,12 @@ procdir (const char *name_buffer, struct stat *stat_data,
  if (one_file_system_option && device != stat_data->st_dev
      /* ... except if it was explicitely given in the command line */
      && !is_individual_file (name_buffer))
+    /* FIXME: 
+	WARNOPT (WARN_XDEV,
+		 (0, 0,
+		  _("%s: directory is on a different filesystem; not dumped"),
+		  quotearg_colon (directory->name)));
+    */
    directory->children = NO_CHILDREN;
  else if (flag & PD_FORCE_CHILDREN)
    {
@@ -699,7 +709,7 @@ scan_directory (char *dir, dev_t device, bool cmdline)
  
  if (deref_stat (dereference_option, name_buffer, &stat_data))
    {
-      stat_diag (name_buffer);
+      dir_removed_diag (name_buffer, false, stat_diag);
      /* FIXME: used to be
           children = CHANGED_CHILDREN;
 	 but changed to: */
--- a/src/misc.c
+++ b/src/misc.c
@@ -745,6 +745,30 @@ stat_diag (char const *name)
    stat_error (name);
 }

+void
+file_removed_diag (const char *name, bool top_level,
+		   void (*diagfn) (char const *name))
+{
+  if (!top_level && errno == ENOENT)
+    WARNOPT (WARN_FILE_REMOVED,
+	     (0, 0, _("%s: File removed before we read it"),
+	      quotearg_colon (name)));
+  else
+    diagfn (name);
+}
+
+void
+dir_removed_diag (const char *name, bool top_level,
+		   void (*diagfn) (char const *name))
+{
+  if (!top_level && errno == ENOENT)
+    WARNOPT (WARN_FILE_REMOVED,
+	     (0, 0, _("%s: Directory removed before we read it"),
+	      quotearg_colon (name)));
+  else
+    diagfn (name);
+}
+
 void
 write_fatal_details (char const *name, ssize_t status, size_t size)
 {
--- a/src/names.c
+++ b/src/names.c
@@ -420,12 +420,13 @@ name_gather (void)
 	  buffer->matching_flags = matching_flags;
 	  buffer->directory = NULL;
 	  buffer->parent = NULL;
+	  buffer->cmdline = true;
 	  
 	  namelist = buffer;
 	  nametail = &namelist->next;
 	}
      else if (change_dir)
-	addname (0, change_dir, NULL);
+	addname (0, change_dir, false, NULL);
    }
  else
    {
@@ -439,11 +440,11 @@ name_gather (void)
 	    change_dir = chdir_arg (xstrdup (ep->v.name));

 	  if (ep)
-	    addname (ep->v.name, change_dir, NULL);
+	    addname (ep->v.name, change_dir, true, NULL);
 	  else
 	    {
 	      if (change_dir != change_dir0)
-		addname (0, change_dir, NULL);
+		addname (NULL, change_dir, false, NULL);
 	      break;
 	    }
 	}
@@ -452,7 +453,7 @@ name_gather (void)

 /*  Add a name to the namelist.  */
 struct name *
-addname (char const *string, int change_dir, struct name *parent)
+addname (char const *string, int change_dir, bool cmdline, struct name *parent)
 {
  struct name *name = make_name (string);

@@ -463,6 +464,7 @@ addname (char const *string, int change_dir, struct name *parent)
  name->change_dir = change_dir;
  name->directory = NULL;
  name->parent = parent;
+  name->cmdline = cmdline;
  
  *nametail = name;
  nametail = &name->next;
@@ -811,7 +813,7 @@ add_hierarchy_to_namelist (struct name *name, dev_t device, bool cmdline)
 		  namebuf = xrealloc (namebuf, allocated_length + 1);
 		}
 	      strcpy (namebuf + name_length, string + 1);
-	      np = addname (namebuf, change_dir, name);
+	      np = addname (namebuf, change_dir, false, name);
 	      if (!child_head)
 		child_head = np;
 	      else
@@ -886,7 +888,7 @@ collect_and_sort_names (void)
  name_gather ();

  if (!namelist)
-    addname (".", 0, NULL);
+    addname (".", 0, false, NULL);

  if (listed_incremental_option)
    {
@@ -1030,8 +1032,8 @@ name_scan (const char *file_name)
   find and return all the non-found names in the namelist.  */
 struct name *gnu_list_name;

-char *
-name_from_list (void)
+struct name const *
+name_from_list ()
 {
  if (!gnu_list_name)
    gnu_list_name = namelist;
@@ -1042,9 +1044,9 @@ name_from_list (void)
    {
      gnu_list_name->found_count++;
      chdir_do (gnu_list_name->change_dir);
-      return gnu_list_name->name;
+      return gnu_list_name;
    }
-  return 0;
+  return NULL;
 }

 void
--- a/src/tar.c
+++ b/src/tar.c
@@ -1388,7 +1388,7 @@ parse_opt (int key, char *arg, struct argp_state *state)

    case 'K':
      starting_file_option = true;
-      addname (arg, 0, NULL);
+      addname (arg, 0, true, NULL);
      break;

    case ONE_FILE_SYSTEM_OPTION:
--- a/src/update.c
+++ b/src/update.c
@@ -189,10 +189,10 @@ update_archive (void)
  output_start = current_block->buffer;

  {
-    char *file_name;
-
-    while ((file_name = name_from_list ()) != NULL)
+    struct name const *p;
+    while ((p = name_from_list ()) != NULL)
      {
+	char *file_name = p->name;
 	if (excluded_name (file_name))
 	  continue;
 	if (interactive_option && !confirm ("add", file_name))
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -69,6 +69,8 @@ TESTSUITE_AT = \
 extrac06.at\
 extrac07.at\
 extrac08.at\
+ filerem01.at\
+ filerem02.at\
 gzip.at\
 grow.at\
 incremental.at\
--- a/tests/filerem01.at
+++ b/tests/filerem01.at
@@ -0,0 +1,88 @@
+# Process this file with autom4te to create testsuite. -*- Autotest -*-
+
+# Test suite for GNU tar.
+# Copyright (C) 2009 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+AT_SETUP([file removed as we read it (ca. 22 seconds)])
+AT_KEYWORDS([create incremental filechange filerem filerem01])
+
+AT_TAR_CHECK([
+mkdir dir
+mkdir dir/sub
+genfile --file dir/file1
+genfile --file dir/sub/file2
+
+genfile --run --checkpoint=3 --unlink dir/file1 -- \
+       tar --blocking-factor=1 --checkpoint=1 --checkpoint-action='sleep=1' \
+       --checkpoint-action='echo' -c -f archive.tar \
+       --listed-incremental db -v dir >/dev/null
+],
+[0],
+[ignore],
+[tar: dir: Directory is new
+tar: dir/sub: Directory is new
+tar: dir/file1: File removed before we read it
+],[],[],[gnu, posix])
+
+# Timing information:
+#
+# For -Hgnu the above command line takes about 8 seconds to execute and
+# produces:
+#
+# tar: dir: Directory is new
+# tar: dir/sub: Directory is new
+# dir/
+# tar: Write checkpoint 1
+# tar: Write checkpoint 2
+# dir/sub/
+# tar: Write checkpoint 3
+# tar: Write checkpoint 4
+# dir/file1
+# tar: Write checkpoint 5
+# dir/sub/file2
+# tar: Write checkpoint 6
+# tar: Write checkpoint 7
+# tar: Write checkpoint 8
+#
+# For -Hposix the above command line takes about 14 seconds to execute and
+# produces:
+#
+# ./tar: dir: Directory is new
+# ./tar: dir/sub: Directory is new
+# dir/
+# ./tar: Write checkpoint 1
+# ./tar: Write checkpoint 2
+# ./tar: Write checkpoint 3
+# dir/sub/
+# ./tar: Write checkpoint 4
+# ./tar: Write checkpoint 5
+# ./tar: Write checkpoint 6
+# dir/file1
+# ./tar: Write checkpoint 7
+# ./tar: Write checkpoint 8
+# ./tar: Write checkpoint 9
+# dir/sub/file2
+# ./tar: Write checkpoint 10
+# ./tar: Write checkpoint 11
+# ./tar: Write checkpoint 12
+# ./tar: Write checkpoint 13
+# ./tar: Write checkpoint 14
+
+
+AT_CLEANUP
+
--- a/tests/filerem02.at
+++ b/tests/filerem02.at
@@ -0,0 +1,50 @@
+# Process this file with autom4te to create testsuite. -*- Autotest -*-
+
+# Test suite for GNU tar.
+# Copyright (C) 2009 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Description: see filerem01.at
+# This test case checks if the tar exit code is still 2 if a
+# file or directory disappears that is explicitly mentioned
+# in the command line.
+
+AT_SETUP([toplevel file removed (ca. 24 seconds)])
+AT_KEYWORDS([create incremental filechange filerem filerem02])
+
+AT_TAR_CHECK([
+mkdir dir
+mkdir dir/sub
+genfile --file dir/file1
+genfile --file dir/sub/file2
+mkdir dir2
+genfile --file dir2/file1
+
+genfile --run --checkpoint=3 --exec 'rm -rf dir2' -- \
+       tar --blocking-factor=1 --checkpoint=1 --checkpoint-action='sleep=1' \
+       --checkpoint-action='echo' -c -f archive.tar \
+       --listed-incremental db -v --warning=no-new-dir dir dir2 >/dev/null
+],
+[2],
+[ignore],
+[tar: dir2: Cannot stat: No such file or directory
+tar: dir2/file1: File removed before we read it
+tar: Exiting with failure status due to previous errors
+],[],[],[gnu, posix])
+
+# Timing information: see filerem01.at
+
+AT_CLEANUP
+
--- a/tests/grow.at
+++ b/tests/grow.at
@@ -27,8 +27,7 @@ AT_KEYWORDS([grow filechange])
 AT_TAR_CHECK([
 genfile --file foo --length 50000k
 genfile --file baz
-genfile --run 'tar -vcf bar foo baz' --checkpoint 10 --length 1024 \
-        --append foo
+genfile --run --checkpoint 10 --length 1024 --append foo -- tar --checkpoint -vcf bar foo baz
 ],
 [1],
 [foo
--- a/tests/testsuite.at
+++ b/tests/testsuite.at
@@ -154,6 +154,10 @@ m4_include([incr03.at])
 m4_include([incr04.at])
 m4_include([incr05.at])
 m4_include([incr06.at])
+
+m4_include([filerem01.at])
+m4_include([filerem02.at])
+
 m4_include([rename01.at])
 m4_include([rename02.at])
 m4_include([rename03.at])
--- a/tests/truncate.at
+++ b/tests/truncate.at
@@ -32,7 +32,7 @@ AT_KEYWORDS([truncate filechange])
 AT_TAR_CHECK([
 genfile --file foo --length 50000k
 genfile --file baz
-genfile --run 'tar -vcf bar foo baz' --checkpoint 10 --length 49995k --truncate foo
+genfile --run --checkpoint 10 --length 49995k --truncate foo -- tar --checkpoint -vcf bar foo baz
 echo Exit status: $?
 echo separator
 sleep 1