Add fiemap output through scoutfs util.

There's filefrag already, and that works, but, it's output is very inconsistent between various OS release versions, and it has already meant that we'd needed to adjust tests to account for these little but insignificant changes. A lot more work than useful. It's even more changed in el9. This adds `scoutfs get-fiemap FILE` and prints out block extent info with flags that we care about as an abbreviated letter: U for Unwritten, L for Last, and O for Unknown (as in, "offline"). The -P/--physical and -L/--logical options turn off logical or physical offset display, in case you only want to see the offsets in either units. You can pass -b/--byte to display offsets and lengths in byte values. The block size will then be obtained from fstat() of the queried file (4096 for scoutfs). I've removed all uses of filefrag from our scoutfs tests. Xfstests still calls it but their internal diff takes care of that issue. Where needed and appropriate, the tests are adjusted so that the output of `scoutfs get-fiemap` is as close as it can to what it used to be, so that reading the test results allows the quick view of what might have been going wrong. There are some output strings I have not bothered to update because there's no real value to updating every output string to match, and we just adjust the golden file accordingly. Signed-off-by: Auke Kok <auke.kok@versity.com>
2025-12-23 05:25:18 +00:00 · 2024-08-30 17:27:07 -07:00
parent 606c519e96
commit 8a4b0967cb
9 changed files with 259 additions and 84 deletions
--- a/tests/golden/data-prealloc
+++ b/tests/golden/data-prealloc
@@ -1,29 +1,29 @@
 == initial writes smaller than prealloc grow to prealloc size
-/mnt/test/test/data-prealloc/file-1: 7 extents found
-/mnt/test/test/data-prealloc/file-2: 7 extents found
+/mnt/test/test/data-prealloc/file-1: extents: 7
+/mnt/test/test/data-prealloc/file-2: extents: 7
 == larger files get full prealloc extents
-/mnt/test/test/data-prealloc/file-1: 9 extents found
-/mnt/test/test/data-prealloc/file-2: 9 extents found
+/mnt/test/test/data-prealloc/file-1: extents: 9
+/mnt/test/test/data-prealloc/file-2: extents: 9
 == non-streaming writes with contig have per-block extents
-/mnt/test/test/data-prealloc/file-1: 32 extents found
-/mnt/test/test/data-prealloc/file-2: 32 extents found
+/mnt/test/test/data-prealloc/file-1: extents: 32
+/mnt/test/test/data-prealloc/file-2: extents: 32
 == any writes to region prealloc get full extents
-/mnt/test/test/data-prealloc/file-1: 4 extents found
-/mnt/test/test/data-prealloc/file-2: 4 extents found
-/mnt/test/test/data-prealloc/file-1: 4 extents found
-/mnt/test/test/data-prealloc/file-2: 4 extents found
+/mnt/test/test/data-prealloc/file-1: extents: 4
+/mnt/test/test/data-prealloc/file-2: extents: 4
+/mnt/test/test/data-prealloc/file-1: extents: 4
+/mnt/test/test/data-prealloc/file-2: extents: 4
 == streaming offline writes get full extents either way
-/mnt/test/test/data-prealloc/file-1: 4 extents found
-/mnt/test/test/data-prealloc/file-2: 4 extents found
-/mnt/test/test/data-prealloc/file-1: 4 extents found
-/mnt/test/test/data-prealloc/file-2: 4 extents found
+/mnt/test/test/data-prealloc/file-1: extents: 4
+/mnt/test/test/data-prealloc/file-2: extents: 4
+/mnt/test/test/data-prealloc/file-1: extents: 4
+/mnt/test/test/data-prealloc/file-2: extents: 4
 == goofy preallocation amounts work
-/mnt/test/test/data-prealloc/file-1: 5 extents found
-/mnt/test/test/data-prealloc/file-2: 5 extents found
-/mnt/test/test/data-prealloc/file-1: 5 extents found
-/mnt/test/test/data-prealloc/file-2: 5 extents found
-/mnt/test/test/data-prealloc/file-1: 3 extents found
-/mnt/test/test/data-prealloc/file-2: 3 extents found
+/mnt/test/test/data-prealloc/file-1: extents: 6
+/mnt/test/test/data-prealloc/file-2: extents: 6
+/mnt/test/test/data-prealloc/file-1: extents: 6
+/mnt/test/test/data-prealloc/file-2: extents: 6
+/mnt/test/test/data-prealloc/file-1: extents: 3
+/mnt/test/test/data-prealloc/file-2: extents: 3
 == block writes into region allocs hole
 wrote blk 24
 wrote blk 32
--- a/tests/golden/setattr_more
+++ b/tests/golden/setattr_more
@@ -22,11 +22,8 @@ scoutfs: setattr failed: Invalid argument (22)
 == large ctime is set
 1972-02-19 00:06:25.999999999 +0000
 == large offline extents are created
-Filesystem type is: 554f4353
-File size of /mnt/test/test/setattr_more/file is 40988672 (10007 blocks of 4096 bytes)
- ext:     logical_offset:        physical_offset: length:   expected: flags:
-   0:        0..   10006:          0..     10006:  10007:             unknown,eof
-/mnt/test/test/setattr_more/file: 1 extent found
+0: offset: 0 0 length: 10007 flags: O.L
+extents: 1
 == correct offline extent length
 976563
 == omitting data_version should not fail
--- a/tests/golden/simple-staging
+++ b/tests/golden/simple-staging
@@ -1,5 +1,9 @@
 == create/release/stage single block file
+0: offset: 0 0 length: 1 flags: O.L
+extents: 1
 == create/release/stage larger file
+0: offset: 0 0 length: 4096 flags: O.L
+extents: 1
 == multiple release,drop_cache,stage cycles
 == release+stage shouldn't change stat, data seq or vers
 == stage does change meta_seq
@@ -12,11 +16,17 @@ scoutfs: stage failed: Input/output error (5)
 == non-block aligned offset fails
 stage returned -1, not 4095: error Invalid argument (22)
 scoutfs: stage failed: Input/output error (5)
+0: offset: 0 0 length: 1 flags: O.L
+extents: 1
 == non-block aligned len within block fails
 stage returned -1, not 1024: error Invalid argument (22)
 scoutfs: stage failed: Input/output error (5)
+0: offset: 0 0 length: 1 flags: O.L
+extents: 1
 == partial final block that writes to i_size does work
 == zero length stage doesn't bring blocks online
+0: offset: 0 0 length: 100 flags: O.L
+extents: 1
 == stage of non-regular file fails
 ioctl failed: Inappropriate ioctl for device (25)
 stage: must provide file version with --data-version
--- a/tests/tests/basic-posix-consistency.sh
+++ b/tests/tests/basic-posix-consistency.sh
@@ -3,13 +3,13 @@
 # operations in one mount and verify the results in another.
 #

-t_require_commands getfattr setfattr dd filefrag diff touch stat scoutfs
+t_require_commands getfattr setfattr dd diff touch stat scoutfs
 t_require_mounts 2

 GETFATTR="getfattr --absolute-names"
 SETFATTR="setfattr"
 DD="dd status=none"
-FILEFRAG="filefrag -v -b4096"
+FIEMAP="scoutfs get-fiemap"

 echo "== root inode updates flow back and forth"
 sleep 1
@@ -55,8 +55,8 @@ for i in $(seq 1 10); do
 		conv=notrunc oflag=append &
 	wait
 done
-$FILEFRAG "$T_D0/file" | t_filter_fs > "$T_TMP.0"
-$FILEFRAG "$T_D1/file" | t_filter_fs > "$T_TMP.1"
+$FIEMAP "$T_D0/file" > "$T_TMP.0"
+$FIEMAP "$T_D1/file" > "$T_TMP.1"
 diff -u "$T_TMP.0" "$T_TMP.1"

 echo "== unlinked file isn't found"
--- a/tests/tests/data-prealloc.sh
+++ b/tests/tests/data-prealloc.sh
@@ -4,7 +4,16 @@
 # merge adjacent consecutive allocations.  (we don't have multiple
 # allocation cursors)
 #
-t_require_commands scoutfs stat filefrag dd touch truncate
+t_require_commands scoutfs stat dd touch truncate
+
+get_fiemap()
+{
+	scoutfs get-fiemap "$1" | awk '($1 != "extents:") {
+			unwritten = (substr($8, 2, 1) == "U") ? "unwritten" : "";
+			eof = (substr($8, 3, 1) == "L") ? "eof" : "";
+			print $3 ".. " $6 ": " unwritten eof;
+			};'
+}

 write_block()
 {
@@ -76,26 +85,9 @@ print_extents_found()
 {
 	local prefix="$1"

-	filefrag "$prefix"* 2>&1 | grep "extent.*found" | t_filter_fs
-}
-
-#
-# print the logical start, len, and flags if they're there.
-#
-print_logical_extents()
-{
-	local file="$1"
-
-	filefrag -v -b4096 "$file" 2>&1 | t_filter_fs | awk '
-		($1 ~ /[0-9]+:/) {
-			if ($NF !~  /[0-9]+:/) {
-				flags=$NF
-			} else {
-				flags=""
-			}
-			print $2, $6, flags
-		}
-	' | sed 's/last,eof/eof/'
+	for f in "$prefix"-*; do
+		echo "$f: $(scoutfs get-fiemap "$f" | tail -n 1)" | t_filter_fs
+	done
 }

 t_save_all_sysfs_mount_options data_prealloc_blocks
@@ -197,7 +189,7 @@ for sides in 0 1 2 3; do
 done

 echo before:
-print_logical_extents "$prefix"
+get_fiemap "$prefix"

 # now write into the first, middle, and last empty block of each
 t_set_sysfs_mount_option 0 data_prealloc_contig_only 0
@@ -223,7 +215,7 @@ for sides in 0 1 2 3; do
 			# mid (both has 6 blocks internally)
 			2) write_block $prefix $((left + 3)) ;;
 		esac
-		print_logical_extents "$prefix"
+		get_fiemap "$prefix"
 		((base+=8))
 	done
 done
--- a/tests/tests/setattr_more.sh
+++ b/tests/tests/setattr_more.sh
@@ -2,7 +2,7 @@
 # Test correctness of the setattr_more ioctl.
 #

-t_require_commands filefrag scoutfs touch mkdir rm stat mknod
+t_require_commands scoutfs touch mkdir rm stat mknod

 FILE="$T_D0/file"

@@ -55,17 +55,10 @@ scoutfs setattr -t 67305985.999999999 -V 1 -s 1 "$FILE" 2>&1 | t_filter_fs
 TZ=GMT stat -c "%z" "$FILE"
 rm "$FILE"

-#
-# With e2fsprogs-v1.42.10-10-g29758d2f, the output of filefrag 'flags' changes
-# significantly. First, the _LAST flag is now output. Second, the 'unknown'
-# flag is now printed out as 'unknown_loc'. To compensate for this, we check
-# and replace the "correct" output for new versions here with the expected
-# value.
-#
 echo "== large offline extents are created"
 touch "$FILE"
 scoutfs setattr -V 1 -o -s $((10007 * 4096)) "$FILE" 2>&1 | t_filter_fs
-filefrag -v -b4096 "$FILE" 2>&1 | sed 's/last,unknown_loc,eof$/unknown,eof/' | t_filter_fs
+scoutfs get-fiemap "$FILE"
 rm "$FILE"

 # had a bug where we were creating extents that were too long
--- a/tests/tests/simple-release-extents.sh
+++ b/tests/tests/simple-release-extents.sh
@@ -7,6 +7,7 @@ t_require_commands xfs_io filefrag scoutfs mknod
 # this test wants to ignore unwritten extents
 fiemap_file() {
 	filefrag -v -b4096 "$1" | grep -v "unwritten"
+	scoutfs get-fiemap "$1" | grep -v 'flags:.*U'
 }

 create_file() {
@@ -108,25 +109,20 @@ for c in $(seq 0 4); do
 		fi
 	done

-	start=$(fiemap_file "$FILE" | \
-		awk '($1 == "0:"){print substr($4, 0, length($4)- 2)}')
-
 	release_vers "$FILE" stat $(($a * 4))K 4K
 	release_vers "$FILE" stat $(($b * 4))K 4K
 	release_vers "$FILE" stat $(($c * 4))K 4K

 	echo -n "$a $b $c:"

-	fiemap_file "$FILE" | \
-		awk 'BEGIN{ORS=""}($1 == (NR - 4)":") {
-			off=substr($2, 0, length($2)- 2);
-			phys=substr($4, 0, length($4)- 2);
-			if (phys > 100) {
-				phys = phys - phys + 100 + off;
-			}
-			len=substr($6, 0, length($6)- 1);
-			print "  (" off, phys, len ")";
-		}'
+	scoutfs get-fiemap "$FILE" | \
+		awk 'BEGIN{ORS=""}($1 != "extents:") {
+			off=$3;
+			len=$6;
+			phys=substr($8, 0, 1);
+			phys = (phys == ".") ? off + 100 : 0;
+			print "  (" off, phys, len ")"
+		};'
 	echo

 	rm "$FILE"
--- a/tests/tests/simple-staging.sh
+++ b/tests/tests/simple-staging.sh
@@ -2,11 +2,7 @@
 # Test correctness of the staging operation
 #

-t_require_commands filefrag dd scoutfs cp cmp rm
-
-fiemap_file() {
-	filefrag -v -b4096 "$1"
-}
+t_require_commands dd scoutfs cp cmp rm

 create_file() {
 	local file="$1"
@@ -62,7 +58,7 @@ create_file "$FILE" 4096
 cp "$FILE"  "$T_TMP"
 release_vers "$FILE" stat 0 4K
 # make sure there only offline extents
-fiemap_file "$FILE" | grep "^[ 0-9]*:" | grep -v "unknown"
+scoutfs get-fiemap "$FILE"
 stage_vers "$FILE" stat 0 4096 "$T_TMP"
 cmp "$FILE" "$T_TMP"
 rm -f "$FILE"
@@ -72,7 +68,7 @@ create_file "$FILE" $((4096 * 4096))
 cp "$FILE"  "$T_TMP"
 release_vers "$FILE" stat 0 16M
 # make sure there only offline extents
-fiemap_file "$FILE" | grep "^[ 0-9]*:" | grep -v "unknown"
+scoutfs get-fiemap "$FILE"
 stage_vers "$FILE" stat 0 $((4096 * 4096)) "$T_TMP"
 cmp "$FILE" "$T_TMP"
 rm -f "$FILE"
@@ -152,7 +148,7 @@ create_file "$FILE" 4096
 cp "$FILE"  "$T_TMP"
 release_vers "$FILE" stat 0 4K
 stage_vers "$FILE" stat 1 4095 "$T_TMP"
-fiemap_file "$FILE" | grep "^[ 0-9]*:" | grep -v "unknown"
+scoutfs get-fiemap "$FILE"
 rm -f "$FILE"

 echo "== non-block aligned len within block fails"
@@ -160,7 +156,7 @@ create_file "$FILE" 4096
 cp "$FILE"  "$T_TMP"
 release_vers "$FILE" stat 0 4K
 stage_vers "$FILE" stat 0 1024 "$T_TMP"
-fiemap_file "$FILE" | grep "^[ 0-9]*:" | grep -v "unknown"
+scoutfs get-fiemap "$FILE"
 rm -f "$FILE"

 echo "== partial final block that writes to i_size does work"
@@ -175,7 +171,7 @@ echo "== zero length stage doesn't bring blocks online"
 create_file "$FILE" $((4096 * 100))
 release_vers "$FILE" stat 0 400K
 stage_vers "$FILE" stat 4096 0 /dev/zero
-fiemap_file "$FILE" | grep "^[ 0-9]*:" | grep -v "unknown"
+scoutfs get-fiemap "$FILE"
 rm -f "$FILE"

 # XXX yup, needs to be updated for demand staging
--- a/utils/src/fiemap.c
+++ b/utils/src/fiemap.c
@@ -0,0 +1,191 @@
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include <linux/fs.h>
+#include <linux/fiemap.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <string.h>
+#include <argp.h>
+
+#include "sparse.h"
+#include "parse.h"
+#include "util.h"
+#include "format.h"
+#include "ioctl.h"
+#include "cmd.h"
+
+/*
+ * This is wholly modeled after e2fsprogs/filefrag.c from tso
+ */
+
+struct get_fiemap_args {
+	char *filename;
+	bool phys;
+	bool logical;
+	bool byte;
+};
+
+static int do_get_fiemap(struct get_fiemap_args *args)
+{
+	__u64 buf[2048];        /* __u64 for proper field alignment */
+	struct stat st;
+	struct fiemap *fiemap = (struct fiemap *)buf;
+	struct fiemap_extent *fm_ext = &fiemap->fm_extents[0];
+	int count = (sizeof(buf) - sizeof(*fiemap)) /
+			sizeof(struct fiemap_extent);
+	int fd;
+	int ret;
+	int i;
+	u64 nr = 0; /* XXX we could put this in fm_start to make start/count an option */
+	int last = 0;
+	u64 off_p, off_l;
+	u64 len;
+
+	memset(fiemap, 0, sizeof(struct fiemap));
+
+	fd = open(args->filename, O_RDONLY);
+	if (fd < 0) {
+		ret = -errno;
+		fprintf(stderr, "failed to open '%s': %s (%d)\n",
+			args->filename, strerror(errno), errno);
+		goto out;
+	}
+
+	/* get block size from stat */
+	if (fstat(fd, &st) != 0) {
+		ret = -errno;
+		fprintf(stderr, "stat failed on '%s': %s (%d)\n",
+			args->filename, strerror(errno), errno);
+		goto out;
+	};
+
+	do {
+		fiemap->fm_length = ~0ULL;
+		fiemap->fm_extent_count = count;
+
+		ret = ioctl(fd, FS_IOC_FIEMAP, (unsigned long) fiemap);
+		if (ret < 0) {
+			ret = -errno;
+			fprintf(stderr, "get_fiemap ioctl failed: "
+				"%s (%d)\n", strerror(errno), errno);
+			goto out;
+		}
+
+		/* nothing returned, so exit */
+		if (fiemap->fm_mapped_extents == 0)
+			break;
+
+		for (i = 0; i < fiemap->fm_mapped_extents; i++) {
+			off_p = fm_ext[i].fe_physical;
+			off_l = fm_ext[i].fe_logical;
+			len = fm_ext[i].fe_length;
+
+			if (!args->byte) {
+				off_p /= st.st_blksize;
+				off_l /= st.st_blksize;
+				len /= st.st_blksize;
+			}
+
+			printf("%llu: offset: ", nr++);
+
+			if (!args->phys)
+				printf("%llu ", off_l);
+			else if (!args->logical)
+				printf("%llu ", off_p);
+			else
+				printf("%llu %llu ", off_l, off_p);
+
+			printf("length: %llu flags: %c%c%c\n",
+				len,
+				(fm_ext[i].fe_flags & FIEMAP_EXTENT_UNKNOWN) ? 'O' : '.',
+				(fm_ext[i].fe_flags & FIEMAP_EXTENT_UNWRITTEN) ? 'U' : '.',
+				(fm_ext[i].fe_flags & FIEMAP_EXTENT_LAST) ? 'L' : '.');
+
+			if (fm_ext[i].fe_flags & FIEMAP_EXTENT_LAST)
+				last = 1;
+		}
+
+		/* fm_start from the next logical extent */
+		fiemap->fm_start = fm_ext[i-1].fe_logical + fm_ext[i-1].fe_length;
+	} while (last == 0);
+
+	printf("extents: %llu\n", nr);
+
+out:
+	if (fd >= 0)
+		close(fd);
+
+	return ret;
+};
+
+static int parse_opt(int key, char *arg, struct argp_state *state)
+{
+	struct get_fiemap_args *args = state->input;
+
+	switch (key) {
+	case 'P':
+		args->logical = false;
+		break;
+	case 'L':
+		args->phys = false;
+		break;
+	case 'b':
+		args->byte = true;
+		break;
+	case ARGP_KEY_ARG:
+		if (!args->filename)
+			args->filename = strdup_or_error(state, arg);
+		else
+			argp_error(state, "more than one argument given");
+		break;
+	case ARGP_KEY_FINI:
+		if ((!args->logical) && (!args->phys))
+			argp_error(state, "can't pass both -P and -L options");
+		if (!args->filename)
+			argp_error(state, "no filename given");
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static struct argp_option options[] = {
+	{ "physical", 'P', NULL, 0, "Output physical offsets only"},
+	{ "logical", 'L', NULL, 0, "Output logical offsets only"},
+	{ "byte", 'b', NULL, 0, "Output byte values instead of blocks"},
+	{ NULL }
+};
+
+static struct argp argp = {
+	options,
+	parse_opt,
+	"FILE",
+	"Print fiemap extent mapping"
+};
+
+static int get_fiemap_cmd(int argc, char **argv)
+{
+	struct get_fiemap_args get_fiemap_args = {NULL};
+	int ret;
+
+	get_fiemap_args.phys = true;
+	get_fiemap_args.logical = true;
+
+	ret = argp_parse(&argp, argc, argv, 0, NULL, &get_fiemap_args);
+	if (ret)
+		return ret;
+
+	return do_get_fiemap(&get_fiemap_args);
+}
+
+static void __attribute__((constructor)) get_fiemap_ctor(void)
+{
+	cmd_register_argp("get-fiemap", &argp, GROUP_DEBUG, get_fiemap_cmd);
+}