From 311bf75902a4a32c6fb7486f6122dbd9b63712bd Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Mon, 10 Jun 2024 18:32:39 -0400 Subject: [PATCH] Add mmap tests. Two test programs are added. The run time is about 1min on my el7 instance. The test script finishes up with a read/write mmap test on offline extents to verify the data wait paths in those functions. One program will perform vfs read/write and mmap read/write calls on the same file from across 5 threads (mounts) repeatedly. The goal is to assure there are no locking issues between read/write paths. The second test program performs consistency checking on a file that is repeatedly written/read using memory maps and normal reads and writes, and the content is verified after every operation. Signed-off-by: Auke Kok --- tests/.gitignore | 2 + tests/Makefile | 8 +- tests/golden/mmap | 27 ++++++ tests/sequence | 1 + tests/src/mmap_stress.c | 181 ++++++++++++++++++++++++++++++++++++++ tests/src/mmap_validate.c | 159 +++++++++++++++++++++++++++++++++ tests/tests/mmap.sh | 54 ++++++++++++ 7 files changed, 430 insertions(+), 2 deletions(-) create mode 100644 tests/golden/mmap create mode 100644 tests/src/mmap_stress.c create mode 100644 tests/src/mmap_validate.c create mode 100644 tests/tests/mmap.sh diff --git a/tests/.gitignore b/tests/.gitignore index b19b962a..32ad161c 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -10,3 +10,5 @@ src/stage_tmpfile src/create_xattr_loop src/o_tmpfile_umask src/o_tmpfile_linkat +src/mmap_stress +src/mmap_validate diff --git a/tests/Makefile b/tests/Makefile index 4c61a0b3..3a2380dc 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -13,7 +13,9 @@ BIN := src/createmany \ src/create_xattr_loop \ src/fragmented_data_extents \ src/o_tmpfile_umask \ - src/o_tmpfile_linkat + src/o_tmpfile_linkat \ + src/mmap_stress \ + src/mmap_validate DEPS := $(wildcard src/*.d) @@ -23,8 +25,10 @@ ifneq ($(DEPS),) -include $(DEPS) endif +src/mmap_stress: LIBS+=-lpthread + $(BIN): %: %.c Makefile - gcc $(CFLAGS) -MD -MP -MF $*.d $< -o $@ + gcc $(CFLAGS) -MD -MP -MF $*.d $< -o $@ $(LIBS) .PHONY: clean clean: diff --git a/tests/golden/mmap b/tests/golden/mmap new file mode 100644 index 00000000..8d5a058e --- /dev/null +++ b/tests/golden/mmap @@ -0,0 +1,27 @@ +== mmap_stress +thread 0 complete +thread 1 complete +thread 2 complete +thread 3 complete +thread 4 complete +== basic mmap/read/write consistency checks +== mmap read from offline extent +0: offset: 0 length: 2 flags: O.L +extents: 1 +1 +00000200: ea ea ea ea ea ea ea ea ea ea ea ea ea ea ea ea ................ +0 +0: offset: 0 length: 2 flags: ..L +extents: 1 +== mmap write to an offline extent +0: offset: 0 length: 2 flags: O.L +extents: 1 +1 +0 +0: offset: 0 length: 2 flags: ..L +extents: 1 +00000000 ea ea ea ea ea ea ea ea ea ea ea ea ea ea ea ea |................| +00000010 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 |................| +00000020 ea ea ea ea ea ea ea ea ea ea ea ea ea ea ea ea |................| +00000030 +== done diff --git a/tests/sequence b/tests/sequence index 78001e59..6b78ff41 100644 --- a/tests/sequence +++ b/tests/sequence @@ -17,6 +17,7 @@ projects.sh large-fragmented-free.sh format-version-forward-back.sh enospc.sh +mmap.sh srch-safe-merge-pos.sh srch-basic-functionality.sh simple-xattr-unit.sh diff --git a/tests/src/mmap_stress.c b/tests/src/mmap_stress.c new file mode 100644 index 00000000..94a41484 --- /dev/null +++ b/tests/src/mmap_stress.c @@ -0,0 +1,181 @@ +#define _GNU_SOURCE +/* + * mmap() stress test for scoutfs + * + * This test exercises the scoutfs kernel module's locking by + * repeatedly reading/writing using mmap and pread/write calls + * across 5 clients (mounts). + * + * Each thread operates on a single thread/client, and performs + * operations in a random order on the file. + * + * The goal is to assure that locking between _page_mkwrite vfs + * calls and the normal read/write paths do not cause deadlocks. + * + * There is no content validation performed. All that is done is + * assure that the programs continues without errors. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int size = 0; +static int count = 0; /* XXX make this duration instead */ + +struct thread_info { + int nr; + int fd; +}; + +static void *run_test_func(void *ptr) +{ + void *buf = NULL; + char *addr = NULL; + struct thread_info *tinfo = ptr; + int c = 0; + int fd; + ssize_t read, written, ret; + int preads = 0, pwrites = 0, mreads = 0, mwrites = 0; + + fd = tinfo->fd; + + if (posix_memalign(&buf, 4096, size) != 0) { + perror("calloc"); + exit(-1); + } + + addr = mmap(NULL, size, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0); + if (addr == MAP_FAILED) { + perror("mmap"); + exit(-1); + } + + usleep(100000); /* 0.1sec to allow all threads to start roughly at the same time */ + + for (;;) { + if (++c > count) + break; + + switch (rand() % 4) { + case 0: /* pread */ + preads++; + for (read = 0; read < size;) { + ret = pread(fd, buf, size - read, read); + if (ret < 0) { + perror("pwrite"); + exit(-1); + } + read += ret; + } + break; + case 1: /* pwrite */ + pwrites++; + memset(buf, (char)(c & 0xff), size); + for (written = 0; written < size;) { + ret = pwrite(fd, buf, size - written, written); + if (ret < 0) { + perror("pwrite"); + exit(-1); + } + written += ret; + } + break; + case 2: /* mmap read */ + mreads++; + memcpy(buf, addr, size); /* noerr */ + break; + case 3: /* mmap write */ + mwrites++; + memset(buf, (char)(c & 0xff), size); + memcpy(addr, buf, size); /* noerr */ + break; + } + } + + munmap(addr, size); + + free(buf); + + printf("thread %u complete: preads %u pwrites %u mreads %u mwrites %u\n", tinfo->nr, + mreads, mwrites, preads, pwrites); + + return NULL; +} + +int main(int argc, char **argv) +{ + pthread_t thread[5]; + struct thread_info tinfo[5]; + int fd[5]; + int ret; + int i; + + if (argc != 8) { + fprintf(stderr, "%s requires 7 arguments - size count file1 file2 file3 file4 file5\n", argv[0]); + exit(-1); + } + + size = atoi(argv[1]); + if (size <= 0) { + fprintf(stderr, "invalid size, must be greater than 0\n"); + exit(-1); + } + + count = atoi(argv[2]); + if (count < 0) { + fprintf(stderr, "invalid count, must be greater than 0\n"); + exit(-1); + } + + /* create and truncate one fd */ + fd[0] = open(argv[3], O_RDWR | O_CREAT | O_TRUNC, 00644); + if (fd[0] < 0) { + perror("open"); + exit(-1); + } + + /* make it the test size */ + if (posix_fallocate(fd[0], 0, size) != 0) { + perror("fallocate"); + exit(-1); + } + + /* now open the rest of the fds */ + for (i = 1; i < 5; i++) { + fd[i] = open(argv[3+i], O_RDWR); + if (fd[i] < 0) { + perror("open"); + exit(-1); + } + } + + /* start threads */ + for (i = 0; i < 5; i++) { + tinfo[i].fd = fd[i]; + tinfo[i].nr = i; + ret = pthread_create(&thread[i], NULL, run_test_func, (void*)&tinfo[i]); + + if (ret) { + perror("pthread_create"); + exit(-1); + } + } + + /* wait for complete */ + for (i = 0; i < 5; i++) + pthread_join(thread[i], NULL); + + for (i = 0; i < 5; i++) + close(fd[i]); + + exit(0); +} diff --git a/tests/src/mmap_validate.c b/tests/src/mmap_validate.c new file mode 100644 index 00000000..40f7435d --- /dev/null +++ b/tests/src/mmap_validate.c @@ -0,0 +1,159 @@ +#define _GNU_SOURCE +/* + * mmap() content consistency checking for scoutfs + * + * This test program validates that content from memory mappings + * are consistent across clients, whether written/read with mmap or + * normal writes/reads. + * + * One side of (read/write) will always be memory mapped. It may + * be that both sides do memory mapped (33% of the time). + */ + +#include +#include +#include +#include +#include +#include +#include + +static int count = 0; +static int size = 0; + +static void run_test_func(int fd1, int fd2) +{ + void *buf1 = NULL; + void *buf2 = NULL; + char *addr1 = NULL; + char *addr2 = NULL; + int c = 0; + ssize_t read, written, ret; + + /* buffers for both sides to compare */ + if (posix_memalign(&buf1, 4096, size) != 0) { + perror("calloc1"); + exit(-1); + } + + if (posix_memalign(&buf2, 4096, size) != 0) { + perror("calloc1"); + exit(-1); + } + + /* memory maps for both sides */ + addr1 = mmap(NULL, size, PROT_WRITE | PROT_READ, MAP_SHARED, fd1, 0); + if (addr1 == MAP_FAILED) { + perror("mmap1"); + exit(-1); + } + + addr2 = mmap(NULL, size, PROT_WRITE | PROT_READ, MAP_SHARED, fd2, 0); + if (addr2 == MAP_FAILED) { + perror("mmap2"); + exit(-1); + } + + for (;;) { + if (++c > count) /* 10k iterations */ + break; + + /* put a pattern in buf1 */ + memset(buf1, c & 0xff, size); + + /* pwrite or mmap write from buf1 */ + switch (c % 3) { + case 0: /* pwrite */ + for (written = 0; written < size;) { + ret = pwrite(fd1, buf1, size - written, written); + if (ret < 0) { + perror("pwrite"); + exit(-1); + } + written += ret; + } + break; + default: /* mmap write */ + memcpy(addr1, buf1, size); + break; + } + + /* pread or mmap read to buf2 */ + switch (c % 3) { + case 2: /* pread */ + for (read = 0; read < size;) { + ret = pread(fd2, buf2, size - read, read); + if (ret < 0) { + perror("pwrite"); + exit(-1); + } + read += ret; + } + break; + default: /* mmap read */ + memcpy(buf2, addr2, size); + break; + } + + /* compare bufs */ + if (memcmp(buf1, buf2, size) != 0) { + fprintf(stderr, "memcmp() failed\n"); + exit(-1); + } + } + + munmap(addr1, size); + munmap(addr2, size); + + free(buf1); + free(buf2); +} + +int main(int argc, char **argv) +{ + int fd[1]; + + if (argc != 5) { + fprintf(stderr, "%s requires 4 arguments - size count file1 file2\n", argv[0]); + exit(-1); + } + + size = atoi(argv[1]); + if (size <= 0) { + fprintf(stderr, "invalid size, must be greater than 0\n"); + exit(-1); + } + + count = atoi(argv[2]); + if (count < 3) { + fprintf(stderr, "invalid count, must be greater than 3\n"); + exit(-1); + } + + /* create and truncate one fd */ + fd[0] = open(argv[3], O_RDWR | O_CREAT | O_TRUNC, 00644); + if (fd[0] < 0) { + perror("open"); + exit(-1); + } + + fd[1] = open(argv[4], O_RDWR , 00644); + if (fd[1] < 0) { + perror("open"); + exit(-1); + } + + /* make it the test size */ + if (posix_fallocate(fd[0], 0, size) != 0) { + perror("fallocate"); + exit(-1); + } + + /* run the test function */ + run_test_func(fd[0], fd[1]); + + close(fd[0]); + close(fd[1]); + + exit(0); +} diff --git a/tests/tests/mmap.sh b/tests/tests/mmap.sh new file mode 100644 index 00000000..bf465ce9 --- /dev/null +++ b/tests/tests/mmap.sh @@ -0,0 +1,54 @@ +# +# test mmap() and normal read/write consistency between different nodes +# + +t_require_commands mmap_stress mmap_validate scoutfs xfs_io + +echo "== mmap_stress" +mmap_stress 8192 2000 "$T_D0/mmap_stress" "$T_D1/mmap_stress" "$T_D2/mmap_stress" "$T_D3/mmap_stress" "$T_D4/mmap_stress" | sed 's/:.*//g' | sort + +echo "== basic mmap/read/write consistency checks" +mmap_validate 256 1000 "$T_D0/mmap_val1" "$T_D1/mmap_val1" +mmap_validate 8192 1000 "$T_D0/mmap_val2" "$T_D1/mmap_val2" +mmap_validate 88400 1000 "$T_D0/mmap_val3" "$T_D1/mmap_val3" + +echo "== mmap read from offline extent" +F="$T_D0/mmap-offline" +touch "$F" +xfs_io -c "pwrite -S 0xEA 0 8192" "$F" > /dev/null +cp "$F" "${F}-stage" +vers=$(scoutfs stat -s data_version "$F") +scoutfs release "$F" -V "$vers" -o 0 -l 8192 +scoutfs get-fiemap -L "$F" +xfs_io -c "mmap -rwx 0 8192" \ + -c "mread -v 512 16" "$F" & +sleep 1 +# should be 1 - data waiting +jobs | wc -l +scoutfs stage "${F}-stage" "$F" -V "$vers" -o 0 -l 8192 +# xfs_io thread will output 16 bytes of read data +sleep 1 +# should be 0 - no more waiting jobs, xfs_io should have exited +jobs | wc -l +scoutfs get-fiemap -L "$F" + +echo "== mmap write to an offline extent" +# reuse the same file +scoutfs release "$F" -V "$vers" -o 0 -l 8192 +scoutfs get-fiemap -L "$F" +xfs_io -c "mmap -rwx 0 8192" \ + -c "mwrite -S 0x11 528 16" "$F" & +sleep 1 +# should be 1 job waiting +jobs | wc -l +scoutfs stage "${F}-stage" "$F" -V "$vers" -o 0 -l 8192 +# no output here from write +sleep 1 +# should be 0 - no more waiting jobs, xfs_io should have exited +jobs | wc -l +scoutfs get-fiemap -L "$F" +# read back contents to assure write changed the file +dd status=none if="$F" bs=1 count=48 skip=512 | hexdump -C + +echo "== done" +t_pass