From 9e47a32257babfb543d4285b3654122d75b8d5ed Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Thu, 19 Nov 2020 09:08:20 -0800 Subject: [PATCH 01/21] Add get_path() Implement a fallback mechanism for opening paths to a filesystem. If explicitly given, use that. If env var is set, use that. Otherwise, use current working directory. Use wordexp to expand ~, $HOME, etc. Signed-off-by: Andy Grover --- utils/src/dev.h | 2 +- utils/src/util.c | 70 ++++++++++++++++++++++++++++++++++++++++++++++++ utils/src/util.h | 2 ++ 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 utils/src/util.c diff --git a/utils/src/dev.h b/utils/src/dev.h index 83dfffb4..f7017da4 100644 --- a/utils/src/dev.h +++ b/utils/src/dev.h @@ -1,7 +1,7 @@ #ifndef _DEV_H_ #define _DEV_H_ -#define BASE_SIZE_FMT "%.2f %s" +#define BASE_SIZE_FMT "%.2f%s" #define BASE_SIZE_ARGS(sz) size_flt(sz, 1), size_str(sz, 1) #define SIZE_FMT "%llu (%.2f %s)" diff --git a/utils/src/util.c b/utils/src/util.c new file mode 100644 index 00000000..dfa948ff --- /dev/null +++ b/utils/src/util.c @@ -0,0 +1,70 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "util.h" + +static int open_path(char *path, int flags) +{ + wordexp_t exp_result; + int ret; + + ret = wordexp(path, &exp_result, WRDE_NOCMD | WRDE_SHOWERR | WRDE_UNDEF); + if (ret) { + fprintf(stderr, "wordexp() failure for \"%s\": %d\n", path, ret); + ret = -EINVAL; + goto out; + } + + ret = open(exp_result.we_wordv[0], flags); + if (ret < 0) { + ret = -errno; + fprintf(stderr, "failed to open '%s': %s (%d)\n", + path, strerror(errno), errno); + } + +out: + wordfree(&exp_result); + + return ret; +} + +/* + * 1. if path option given, use that + * 2. if env var, use that + * 3. if cwd is in a scoutfs fs, use that + * 4. else error + */ +int get_path(char *path, int flags) +{ + char *env_path; + char *cur_dir_path; + int ret; + + if (path) + return open_path(path, flags); + + env_path = getenv("SCOUTFS_PATH"); + if (env_path) + return open_path(path, flags); + + cur_dir_path = get_current_dir_name(); + if (!cur_dir_path) { + ret = -errno; + return ret; + } + + ret = open_path(cur_dir_path, flags); + free(cur_dir_path); + + // TODO: check this is within a scoutfs mount? + + return ret; +} diff --git a/utils/src/util.h b/utils/src/util.h index 637fdf5b..c70ddbb5 100644 --- a/utils/src/util.h +++ b/utils/src/util.h @@ -111,4 +111,6 @@ static inline int memcmp_lens(const void *a, int a_len, return memcmp(a, b, len) ?: a_len - b_len; } +int get_path(char *path, int flags); + #endif From a3035582d38b9f12393e91b1e25d8f9a77aad27a Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Thu, 3 Dec 2020 15:49:43 -0800 Subject: [PATCH 02/21] Add strdup_or_error() Add a helper function to handle the impossible event that strdup fails. Signed-off-by: Andy Grover --- utils/src/parse.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/utils/src/parse.h b/utils/src/parse.h index 0a0d9ac4..b7ed51f4 100644 --- a/utils/src/parse.h +++ b/utils/src/parse.h @@ -2,6 +2,7 @@ #define _PARSE_H_ #include +#include int parse_human(char* str, u64 *val_ret); int parse_u64(char *str, u64 *val_ret); @@ -9,4 +10,13 @@ int parse_s64(char *str, s64 *val_ret); int parse_u32(char *str, u32 *val_ret); int parse_timespec(char *str, struct timespec *ts); +static inline char* strdup_or_error(const struct argp_state *state, char *str) +{ + char *new = strdup(str); + if (!new) + argp_error(state, "memory allocation failed"); + + return new; +} + #endif From 5701184324f184e2d8912dd83893bb82080bdf52 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Fri, 4 Dec 2020 09:32:25 -0800 Subject: [PATCH 03/21] Implement argp support for df Convert arg parsing to use argp. Use new get_path() helper fn. Add -h human-readable option. Signed-off-by: Andy Grover --- utils/src/df.c | 105 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 21 deletions(-) diff --git a/utils/src/df.c b/utils/src/df.c index 96bec369..4c8be0ef 100644 --- a/utils/src/df.c +++ b/utils/src/df.c @@ -7,20 +7,29 @@ #include #include #include -#include #include +#include +#include #include "sparse.h" +#include "parse.h" #include "util.h" #include "format.h" #include "ioctl.h" #include "cmd.h" +#include "dev.h" #define ROWS 3 #define COLS 6 #define CHARS 20 -static int df_cmd(int argc, char **argv) +struct df_args { + char *path; + bool human_readable; + u8 __pad[7]; +}; + +static int do_df(struct df_args *args) { struct scoutfs_ioctl_alloc_detail ad; struct scoutfs_ioctl_alloc_detail_entry *ade = NULL; @@ -36,18 +45,9 @@ static int df_cmd(int argc, char **argv) int r; int c; - if (argc != 2) { - fprintf(stderr, "must specify path\n"); - return -EINVAL; - } - - fd = open(argv[1], O_RDONLY); - if (fd < 0) { - ret = -errno; - fprintf(stderr, "failed to open '%s': %s (%d)\n", - argv[1], strerror(errno), errno); - return ret; - } + fd = get_path(args->path, O_RDONLY); + if (fd < 0) + return fd; sfm.valid_bytes = sizeof(struct scoutfs_ioctl_statfs_more); ret = ioctl(fd, SCOUTFS_IOC_STATFS_MORE, &sfm); @@ -96,18 +96,38 @@ static int df_cmd(int argc, char **argv) snprintf(cells[1][0], CHARS, "MetaData"); snprintf(cells[1][1], CHARS, "64KB"); - snprintf(cells[1][2], CHARS, "%llu", sfm.total_meta_blocks); - snprintf(cells[1][3], CHARS, "%llu", sfm.total_meta_blocks - meta_free); - snprintf(cells[1][4], CHARS, "%llu", meta_free); + if (args->human_readable) { + snprintf(cells[1][2], CHARS, BASE_SIZE_FMT, + BASE_SIZE_ARGS(sfm.total_meta_blocks * SCOUTFS_BLOCK_LG_SIZE)); + snprintf(cells[1][3], CHARS, BASE_SIZE_FMT, + BASE_SIZE_ARGS((sfm.total_meta_blocks - meta_free) + * SCOUTFS_BLOCK_LG_SIZE)); + snprintf(cells[1][4], CHARS, BASE_SIZE_FMT, + BASE_SIZE_ARGS(meta_free * SCOUTFS_BLOCK_LG_SIZE)); + } else { + snprintf(cells[1][2], CHARS, "%llu", sfm.total_meta_blocks); + snprintf(cells[1][3], CHARS, "%llu", sfm.total_meta_blocks - meta_free); + snprintf(cells[1][4], CHARS, "%llu", meta_free); + } snprintf(cells[1][5], CHARS, "%llu", ((sfm.total_meta_blocks - meta_free) * 100) / sfm.total_meta_blocks); snprintf(cells[2][0], CHARS, "Data"); snprintf(cells[2][1], CHARS, "4KB"); - snprintf(cells[2][2], CHARS, "%llu", sfm.total_data_blocks); - snprintf(cells[2][3], CHARS, "%llu", sfm.total_data_blocks - data_free); - snprintf(cells[2][4], CHARS, "%llu", data_free); + if (args->human_readable) { + snprintf(cells[2][2], CHARS, BASE_SIZE_FMT, + BASE_SIZE_ARGS(sfm.total_data_blocks * SCOUTFS_BLOCK_SM_SIZE)); + snprintf(cells[2][3], CHARS, BASE_SIZE_FMT, + BASE_SIZE_ARGS((sfm.total_data_blocks - data_free) + * SCOUTFS_BLOCK_SM_SIZE)); + snprintf(cells[2][4], CHARS, BASE_SIZE_FMT, + BASE_SIZE_ARGS(data_free * SCOUTFS_BLOCK_SM_SIZE)); + } else { + snprintf(cells[2][2], CHARS, "%llu", sfm.total_data_blocks); + snprintf(cells[2][3], CHARS, "%llu", sfm.total_data_blocks - data_free); + snprintf(cells[2][4], CHARS, "%llu", data_free); + } snprintf(cells[2][5], CHARS, "%llu", ((sfm.total_data_blocks - data_free) * 100) / sfm.total_data_blocks); @@ -131,8 +151,51 @@ out: return ret; } +static int parse_opt(int key, char *arg, struct argp_state *state) +{ + struct df_args *args = state->input; + + switch (key) { + case 'p': + args->path = strdup_or_error(state, arg); + break; + case 'h': + args->human_readable = true; + break; + default: + break; + } + + return 0; +} + +static struct argp_option options[] = { + { "path", 'p', "PATH", 0, "Path to ScoutFS filesystem"}, + { "human-readable", 'h', NULL, 0, "Print sizes in human readable format (e.g., 1KB 234MB 2GB)"}, + { NULL } +}; + +static int df_cmd(int argc, char **argv) +{ + struct argp argp = { + options, + parse_opt, + NULL, + "Show metadata and data block usage" + }; + struct df_args df_args = {NULL}; + int ret; + + ret = argp_parse(&argp, argc, argv, 0, NULL, &df_args); + if (ret) + return ret; + + return do_df(&df_args); + +} + static void __attribute__((constructor)) df_ctor(void) { cmd_register("df", "", - "show metadata and data block usage", df_cmd); + "Show metadata and data block usage", df_cmd); } From 68b8e4098d3a202a49648cf92ca78e30bc659cf7 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Fri, 20 Nov 2020 14:34:12 -0800 Subject: [PATCH 04/21] Implement argp support for stat and statfs Signed-off-by: Andy Grover --- utils/src/stat.c | 171 ++++++++++++++++++++++++++++++----------------- 1 file changed, 111 insertions(+), 60 deletions(-) diff --git a/utils/src/stat.c b/utils/src/stat.c index ef9c6247..3e20d1d0 100644 --- a/utils/src/stat.c +++ b/utils/src/stat.c @@ -7,10 +7,12 @@ #include #include #include -#include #include +#include +#include #include "sparse.h" +#include "parse.h" #include "util.h" #include "format.h" #include "ioctl.h" @@ -101,12 +103,14 @@ static void print_fs_field(void *st, size_t off) typedef void (*print_field_t)(void *st, size_t off); -static struct option long_ops[] = { - { "single_field", 1, NULL, 's' }, - { NULL, 0, NULL, 0} +struct stat_args { + char *path; + char *single_field; + bool is_inode; + u8 __pad[7]; }; -static int do_stat(int argc, char **argv, int is_inode) +static int do_stat(struct stat_args *args) { union { struct scoutfs_ioctl_stat_more stm; @@ -115,17 +119,13 @@ static int do_stat(int argc, char **argv, int is_inode) struct stat_more_field *single = NULL; struct stat_more_field *fields; struct stat_more_field *fi; - char *single_name = NULL; print_field_t pr = NULL; - char *path; int cmd; int ret; int fd; - int i; - int c; memset(&st, 0, sizeof(st)); - if (is_inode) { + if (args->is_inode) { cmd = SCOUTFS_IOC_STAT_MORE; fields = inode_fields; st.stm.valid_bytes = sizeof(struct scoutfs_ioctl_stat_more); @@ -137,79 +137,130 @@ static int do_stat(int argc, char **argv, int is_inode) pr = print_fs_field; } - while ((c = getopt_long(argc, argv, "s:", long_ops, NULL)) != -1) { - switch (c) { - case 's': - single_name = strdup(optarg); - assert(single_name); - break; - case '?': - default: - return -EINVAL; - } - } - - if (single_name) { + if (args->single_field) { for_each_field(fi, fields) { - if (strcmp(fi->name, single_name) == 0) { + if (strcmp(fi->name, args->single_field) == 0) { single = fi; break; } } if (!single) { - fprintf(stderr, "unknown field: '%s'\n", single_name); + fprintf(stderr, "unknown field: '%s'\n", args->single_field); return -EINVAL; } } - if (optind >= argc) { - fprintf(stderr, "must specify at least one path argument\n"); - return -EINVAL; - } + fd = get_path(args->path, O_RDONLY); + if (fd < 0) + return fd; - for (i = optind; i < argc; i++) { - path = argv[i]; - - fd = open(path, O_RDONLY); - if (fd < 0) { - ret = -errno; - fprintf(stderr, "failed to open '%s': %s (%d)\n", - path, strerror(errno), errno); - continue; - } - - ret = ioctl(fd, cmd, &st); - if (ret < 0) { - ret = -errno; - fprintf(stderr, "ioctl failed on '%s': " - "%s (%d)\n", path, strerror(errno), errno); - - } else if (single) { - pr(&st, single->offset); + ret = ioctl(fd, cmd, &st); + if (ret < 0) { + ret = -errno; + fprintf(stderr, "ioctl failed: %s (%d)\n", strerror(errno), errno); + } else if (single) { + pr(&st, single->offset); + printf("\n"); + } else { + for_each_field(fi, fields) { + printf("%-17s ", fi->name); + pr(&st, fi->offset); printf("\n"); - } else { - printf("%-17s %s\n", "path", path); - for_each_field(fi, fields) { - printf("%-17s ", fi->name); - pr(&st, fi->offset); - printf("\n"); - } } - - close(fd); } return 0; } -static int stat_more_cmd(int argc, char **argv) +static int stat_parse_opt(int key, char *arg, struct argp_state *state) { - return do_stat(argc, argv, 1); + struct stat_args *args = state->input; + + switch (key) { + case 's': + args->single_field = strdup_or_error(state, arg); + break; + case ARGP_KEY_ARG: + if (!args->path) + args->path = strdup_or_error(state, arg); + else + argp_error(state, "more than one argument"); + break; + case ARGP_KEY_FINI: + if (!args->path) + argp_error(state, "missing operand"); + break; + default: + break; + } + + return 0; } +static struct argp_option stat_options[] = { + { "single-field", 's', "FIELD-NAME", 0, "Specify single field to print" }, + { NULL } +}; + +static int stat_more_cmd(int argc, char **argv) +{ + struct argp argp = { + stat_options, + stat_parse_opt, + "FILE", + "Show ScoutFS extra inode information" + }; + struct stat_args stat_args = {NULL}; + int ret; + + ret = argp_parse(&argp, argc, argv, 0, NULL, &stat_args); + if (ret) + return ret; + stat_args.is_inode = true; + + return do_stat(&stat_args); +} + +static struct argp_option statfs_options[] = { + { "path", 'p', "PATH", 0, "Path to ScoutFS filesystem"}, + { "single-field", 's', "FIELD-NAME", 0, "Specify single field to print" }, + { NULL } +}; + +static int statfs_parse_opt(int key, char *arg, struct argp_state *state) +{ + struct stat_args *args = state->input; + + switch (key) { + case 'p': + args->path = strdup_or_error(state, arg); + break; + case 's': + args->single_field = strdup_or_error(state, arg); + break; + default: + break; + } + + return 0; +} static int statfs_more_cmd(int argc, char **argv) { - return do_stat(argc, argv, 0); + struct argp argp = { + statfs_options, + statfs_parse_opt, + NULL, + "Show ScoutFS file system information" + }; + struct stat_args stat_args = {NULL}; + int ret; + + ret = argp_parse(&argp, argc, argv, 0, NULL, &stat_args); + if (ret) + return ret; + stat_args.is_inode = false; + + return do_stat(&stat_args); } static void __attribute__((constructor)) stat_more_ctor(void) From 10df01eb7afead5980165e2ea6a9c6b7032b6b20 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Fri, 20 Nov 2020 16:14:01 -0800 Subject: [PATCH 05/21] Implement argp support for ino-path Signed-off-by: Andy Grover --- utils/src/ino_path.c | 110 +++++++++++++++++++++++++++++-------------- 1 file changed, 75 insertions(+), 35 deletions(-) diff --git a/utils/src/ino_path.c b/utils/src/ino_path.c index 91cbd184..aaa00f2d 100644 --- a/utils/src/ino_path.c +++ b/utils/src/ino_path.c @@ -8,44 +8,32 @@ #include #include #include +#include #include "sparse.h" +#include "parse.h" #include "util.h" #include "format.h" #include "ioctl.h" +#include "parse.h" #include "cmd.h" -static int ino_path_cmd(int argc, char **argv) +struct ino_args { + char *path; + u64 ino; +}; + +static int do_ino_path(struct ino_args *args) { - struct scoutfs_ioctl_ino_path args; + struct scoutfs_ioctl_ino_path ioctl_args; struct scoutfs_ioctl_ino_path_result *res; unsigned int result_bytes; - char *endptr = NULL; - u64 ino; int ret; int fd; - if (argc != 3) { - fprintf(stderr, "must specify ino and path\n"); - return -EINVAL; - } - - ino = strtoull(argv[1], &endptr, 0); - if (*endptr != '\0' || - ((ino == LLONG_MIN || ino == LLONG_MAX) && errno == ERANGE)) { - fprintf(stderr, "error parsing inode number '%s'\n", - argv[1]); - return -EINVAL; - } - - - fd = open(argv[2], O_RDONLY); - if (fd < 0) { - ret = -errno; - fprintf(stderr, "failed to open '%s': %s (%d)\n", - argv[2], strerror(errno), errno); - return ret; - } + fd = get_path(args->path, O_RDONLY); + if (fd < 0) + return fd; result_bytes = offsetof(struct scoutfs_ioctl_ino_path_result, path[PATH_MAX]); @@ -57,13 +45,13 @@ static int ino_path_cmd(int argc, char **argv) goto out; } - args.ino = ino; - args.dir_ino = 0; - args.dir_pos = 0; - args.result_ptr = (intptr_t)res; - args.result_bytes = result_bytes; + ioctl_args.ino = args->ino; + ioctl_args.dir_ino = 0; + ioctl_args.dir_pos = 0; + ioctl_args.result_ptr = (intptr_t)res; + ioctl_args.result_bytes = result_bytes; for (;;) { - ret = ioctl(fd, SCOUTFS_IOC_INO_PATH, &args); + ret = ioctl(fd, SCOUTFS_IOC_INO_PATH, &ioctl_args); if (ret < 0) { ret = -errno; if (ret == -ENOENT) @@ -73,10 +61,10 @@ static int ino_path_cmd(int argc, char **argv) printf("%.*s\n", res->path_bytes, res->path); - args.dir_ino = res->dir_ino; - args.dir_pos = res->dir_pos; - if (++args.dir_pos == 0) { - if (++args.dir_ino == 0) + ioctl_args.dir_ino = res->dir_ino; + ioctl_args.dir_pos = res->dir_pos; + if (++ioctl_args.dir_pos == 0) { + if (++ioctl_args.dir_ino == 0) break; } } @@ -92,6 +80,58 @@ out: return ret; }; +static int parse_opt(int key, char *arg, struct argp_state *state) +{ + struct ino_args *args = state->input; + int ret; + + switch (key) { + case 'p': + args->path = strdup_or_error(state, arg); + break; + case ARGP_KEY_ARG: + if (args->ino) + argp_error(state, "more than one argument given"); + ret = parse_u64(arg, &args->ino); + if (ret) + argp_error(state, "inode parse error"); + break; + case ARGP_KEY_FINI: + if (!args->ino) { + argp_error(state, "must provide inode number"); + } + break; + default: + break; + } + + return 0; +} + +static struct argp_option options[] = { + { "path", 'p', "PATH", 0, "Path to ScoutFS filesystem"}, + { NULL } +}; + +static int ino_path_cmd(int argc, char **argv) +{ + struct argp argp = { + options, + parse_opt, + "INODE-NUM", + "Print paths that refer to inode number" + }; + struct ino_args ino_args = {NULL}; + int ret; + + ret = argp_parse(&argp, argc, argv, 0, NULL, &ino_args); + if (ret) + return ret; + + return do_ino_path(&ino_args); +} + + static void __attribute__((constructor)) ino_path_ctor(void) { cmd_register("ino-path", " ", From 0f17ecb9e39fd24bd3bec5acf0251d901e3ad6f6 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Mon, 30 Nov 2020 16:07:41 -0800 Subject: [PATCH 06/21] Implement argp support for stage/release Make offset and length optional. Allow size units (KMGTP) to be used for offset/length. release: Since off/len no longer given in 4k blocks, round offset and length to to 4KiB, down and up respectively. Emit a message if rounding occurs. Make version a required option. stage: change ordering to src (the archive file) then the dest (the staged file). Signed-off-by: Andy Grover --- utils/src/stage_release.c | 316 ++++++++++++++++++++++++-------------- 1 file changed, 204 insertions(+), 112 deletions(-) diff --git a/utils/src/stage_release.c b/utils/src/stage_release.c index 2e1ec342..8683afd1 100644 --- a/utils/src/stage_release.c +++ b/utils/src/stage_release.c @@ -8,82 +8,50 @@ #include #include #include +#include +#include #include "sparse.h" #include "util.h" #include "format.h" #include "ioctl.h" +#include "parse.h" #include "cmd.h" -static int stage_cmd(int argc, char **argv) +struct stage_args { + char *archive_path; + char *path; + u64 data_version; + u64 offset; + u64 length; +}; + +static int do_stage(struct stage_args *args) { - struct scoutfs_ioctl_stage args; + struct scoutfs_ioctl_stage ioctl_args; unsigned int buf_len = 1024 * 1024; unsigned int bytes; - char *endptr = NULL; char *buf = NULL; int afd = -1; int fd = -1; - u64 offset; - u64 count; - u64 vers; int ret; - if (argc != 6) { - fprintf(stderr, "must specify moar args\n"); - return -EINVAL; - } - - fd = open(argv[1], O_RDWR); - if (fd < 0) { - ret = -errno; - fprintf(stderr, "failed to open '%s': %s (%d)\n", - argv[1], strerror(errno), errno); - return ret; - } - - vers = strtoull(argv[2], &endptr, 0); - if (*endptr != '\0' || - ((vers == LLONG_MIN || vers == LLONG_MAX) && errno == ERANGE)) { - fprintf(stderr, "error parsing data version '%s'\n", - argv[2]); - ret = -EINVAL; - goto out; - } - - offset = strtoull(argv[3], &endptr, 0); - if (*endptr != '\0' || - ((offset == LLONG_MIN || offset == LLONG_MAX) && errno == ERANGE)) { - fprintf(stderr, "error parsing offset '%s'\n", - argv[3]); - ret = -EINVAL; - goto out; - } - - count = strtoull(argv[4], &endptr, 0); - if (*endptr != '\0' || - ((count == LLONG_MIN || count == LLONG_MAX) && errno == ERANGE)) { - fprintf(stderr, "error parsing count '%s'\n", - argv[4]); - ret = -EINVAL; - goto out; - } - - if (count > INT_MAX) { - fprintf(stderr, "count %llu too large, limited to %d\n", - count, INT_MAX); - ret = -EINVAL; - goto out; - } - - afd = open(argv[5], O_RDONLY); + afd = open(args->archive_path, O_RDONLY); if (afd < 0) { ret = -errno; fprintf(stderr, "failed to open '%s': %s (%d)\n", - argv[5], strerror(errno), errno); + args->archive_path, strerror(errno), errno); goto out; } + fd = open(args->path, O_RDWR); + if (fd < 0) { + ret = -errno; + fprintf(stderr, "failed to open '%s': %s (%d)\n", + args->path, strerror(errno), errno); + return ret; + } + buf = malloc(buf_len); if (!buf) { fprintf(stderr, "couldn't allocate %u byte buffer\n", buf_len); @@ -91,9 +59,9 @@ static int stage_cmd(int argc, char **argv) goto out; } - while (count) { + while (args->length) { - bytes = min(count, buf_len); + bytes = min(args->length, buf_len); ret = read(afd, buf, bytes); if (ret <= 0) { @@ -105,15 +73,15 @@ static int stage_cmd(int argc, char **argv) bytes = ret; - args.data_version = vers; - args.buf_ptr = (unsigned long)buf; - args.offset = offset; - args.count = bytes; + ioctl_args.data_version = args->data_version; + ioctl_args.buf_ptr = (unsigned long)buf; + ioctl_args.offset = args->offset; + ioctl_args.count = bytes; - count -= bytes; - offset += bytes; + args->length -= bytes; + args->offset += bytes; - ret = ioctl(fd, SCOUTFS_IOC_STAGE, &args); + ret = ioctl(fd, SCOUTFS_IOC_STAGE, &ioctl_args); if (ret != bytes) { fprintf(stderr, "stage returned %d, not %u: error %s (%d)\n", ret, bytes, strerror(errno), errno); @@ -132,79 +100,203 @@ out: return ret; }; -static void __attribute__((constructor)) stage_ctor(void) +static int parse_stage_opts(int key, char *arg, struct argp_state *state) { - cmd_register("stage", " ", - "write archive file contents to offline region", stage_cmd); + struct stage_args *args = state->input; + int ret; + + switch (key) { + case 'V': + ret = parse_u64(arg, &args->data_version); + if (ret) + return ret; + break; + case 'o': /* offset */ + ret = parse_human(arg, &args->offset); + if (ret) + return ret; + break; + case 'l': /* length */ + ret = parse_human(arg, &args->length); + if (ret) + return ret; + break; + case ARGP_KEY_ARG: + if (!args->archive_path) + args->archive_path = strdup_or_error(state, arg); + else if (!args->path) + args->path = strdup_or_error(state, arg); + else + argp_error(state, "more than two arguments given"); + break; + case ARGP_KEY_FINI: + if (!args->archive_path) { + argp_error(state, "must provide archive file path"); + } + if (!args->path) { + argp_error(state, "must provide to-stage file path"); + } + if (!args->data_version) { + argp_error(state, "must provide file version with --data-version"); + } + if (!args->length) { + struct stat statbuf = {0}; + + ret = stat(args->archive_path, &statbuf); + if (ret < 0) + argp_failure(state, 1, -errno, "Could not get file size"); + + args->length = statbuf.st_size; + } + break; + default: + break; + } + + return 0; } -static int release_cmd(int argc, char **argv) +static struct argp_option options[] = { + { "data-version", 'V', "VERSION", 0, "Data version of the file [Required]"}, + { "offset", 'o', "OFFSET", 0, "Offset (bytes or KMGTP units) in file to stage (default: 0)"}, + { "length", 'l', "LENGTH", 0, "Length of range (bytes or KMGTP units) of file to stage. (default: size of ARCHIVE-FILE)"}, + { NULL } +}; + +static int stage_cmd(int argc, char **argv) { - struct scoutfs_ioctl_release args; - char *endptr = NULL; - u64 block; - u64 count; - u64 vers; + struct argp argp = { + options, + parse_stage_opts, + "ARCHIVE-FILE STAGE-FILE --data-version VERSION", + "Write archive file contents to an offline file" + }; + struct stage_args stage_args = {NULL}; + int ret; + + ret = argp_parse(&argp, argc, argv, 0, NULL, &stage_args); + if (ret) + return ret; + + return do_stage(&stage_args); +} + +static void __attribute__((constructor)) stage_ctor(void) +{ + cmd_register("stage", " -V ", + "write archive file contents to an offline file", stage_cmd); +} + +struct release_args { + char *path; + u64 data_version; + u64 offset; + u64 length; +}; + +static int do_release(struct release_args *args) +{ + struct scoutfs_ioctl_release ioctl_args = {0}; int ret; int fd; - if (argc != 5) { - fprintf(stderr, "must specify path, data version, offset, and count\n"); - return -EINVAL; - } - - fd = open(argv[1], O_RDWR); + fd = open(args->path, O_RDWR); if (fd < 0) { ret = -errno; fprintf(stderr, "failed to open '%s': %s (%d)\n", - argv[1], strerror(errno), errno); + args->path, strerror(errno), errno); return ret; } - vers = strtoull(argv[2], &endptr, 0); - if (*endptr != '\0' || - ((vers == LLONG_MIN || vers == LLONG_MAX) && errno == ERANGE)) { - fprintf(stderr, "error parsing data version '%s'\n", - argv[2]); - ret = -EINVAL; - goto out; - } + assert(args->offset % SCOUTFS_BLOCK_SM_SIZE == 0); + assert(args->length % SCOUTFS_BLOCK_SM_SIZE == 0); - block = strtoull(argv[3], &endptr, 0); - if (*endptr != '\0' || - ((block == LLONG_MIN || block == LLONG_MAX) && errno == ERANGE)) { - fprintf(stderr, "error parsing starting 4K block offset '%s'\n", - argv[3]); - ret = -EINVAL; - goto out; - } + ioctl_args.block = args->offset / SCOUTFS_BLOCK_SM_SIZE; + ioctl_args.count = args->length / SCOUTFS_BLOCK_SM_SIZE; + ioctl_args.data_version = args->data_version; - count = strtoull(argv[4], &endptr, 0); - if (*endptr != '\0' || - ((count == LLONG_MIN || count == LLONG_MAX) && errno == ERANGE)) { - fprintf(stderr, "error parsing length '%s'\n", - argv[4]); - ret = -EINVAL; - goto out; - } - - args.block = block; - args.count = count; - args.data_version = vers; - - ret = ioctl(fd, SCOUTFS_IOC_RELEASE, &args); + ret = ioctl(fd, SCOUTFS_IOC_RELEASE, &ioctl_args); if (ret < 0) { ret = -errno; fprintf(stderr, "release ioctl failed: %s (%d)\n", strerror(errno), errno); } -out: + close(fd); return ret; }; +static int parse_release_opts(int key, char *arg, struct argp_state *state) +{ + struct release_args *args = state->input; + int ret; + + switch (key) { + case 'V': + ret = parse_u64(arg, &args->data_version); + if (ret) + return ret; + break; + case 'o': /* offset */ + ret = parse_human(arg, &args->offset); + if (ret) + return ret; + break; + case 'l': /* length */ + ret = parse_human(arg, &args->length); + if (ret) + return ret; + break; + case ARGP_KEY_ARG: + if (args->path) + argp_error(state, "more than one argument given"); + args->path = strdup_or_error(state, arg); + break; + case ARGP_KEY_FINI: + if (!args->path) { + argp_error(state, "must provide file path"); + } + if (!args->data_version) { + argp_error(state, "must provide file version --data-version"); + } + if (!args->length) { + int ret; + struct stat statbuf = {0}; + + ret = stat(args->path, &statbuf); + if (ret < 0) + argp_failure(state, 1, -errno, "Could not get file size"); + + args->length = round_up(statbuf.st_size, SCOUTFS_BLOCK_SM_SIZE); + } + break; + default: + break; + } + + return 0; +} + +static int release_cmd(int argc, char **argv) +{ + struct argp argp = { + options, + parse_release_opts, + "FILE --data-version VERSION", + "Mark file region offline and free extents" + }; + struct release_args release_args = {NULL}; + int ret; + + ret = argp_parse(&argp, argc, argv, 0, NULL, &release_args); + if (ret) + return ret; + + return do_release(&release_args); +} + static void __attribute__((constructor)) release_ctor(void) { - cmd_register("release", " <4K block offset> ", + cmd_register("release", " ", "mark file region offline and free extents", release_cmd); } From 706fe9a30ef6143dabdc1d403086b9ecd9cf4ae3 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Tue, 1 Dec 2020 16:44:27 -0800 Subject: [PATCH 07/21] Implement argp support for search-xattrs Get fs path via normal methods, and make xattr an argument not an option. Signed-off-by: Andy Grover --- utils/src/search_xattrs.c | 124 ++++++++++++++++++++------------------ 1 file changed, 64 insertions(+), 60 deletions(-) diff --git a/utils/src/search_xattrs.c b/utils/src/search_xattrs.c index 4d7e86e8..a079a13f 100644 --- a/utils/src/search_xattrs.c +++ b/utils/src/search_xattrs.c @@ -7,38 +7,36 @@ #include #include #include -#include +#include #include "sparse.h" +#include "parse.h" #include "util.h" #include "format.h" #include "ioctl.h" #include "cmd.h" -static struct option long_ops[] = { - { "name", 1, NULL, 'n' }, - { "file", 1, NULL, 'f' }, - { NULL, 0, NULL, 0} -}; - /* * There are significant constant costs to each search call, we * want to get the inodes in as few calls as possible. */ #define BATCH_SIZE 1000000 -static int search_xattrs_cmd(int argc, char **argv) +struct xattr_args { + char *name; + char *path; +}; + +static int do_search_xattrs(struct xattr_args *args) { - struct scoutfs_ioctl_search_xattrs sx; - char *path = NULL; - char *name = NULL; + struct scoutfs_ioctl_search_xattrs sx = {0}; u64 *inos = NULL; int fd = -1; int ret; - int c; int i; memset(&sx, 0, sizeof(sx)); + inos = malloc(BATCH_SIZE * sizeof(inos[0])); if (!inos) { fprintf(stderr, "inos mem alloc failed\n"); @@ -46,56 +44,15 @@ static int search_xattrs_cmd(int argc, char **argv) goto out; } - while ((c = getopt_long(argc, argv, "f:n:", long_ops, NULL)) != -1) { - switch (c) { - case 'f': - path = strdup(optarg); - if (!path) { - fprintf(stderr, "path mem alloc failed\n"); - ret = -ENOMEM; - goto out; - } - break; - case 'n': - name = strdup(optarg); - if (!name) { - fprintf(stderr, "name mem alloc failed\n"); - ret = -ENOMEM; - goto out; - } - break; - case '?': - default: - ret = -EINVAL; - goto out; - } - } - - if (path == NULL) { - fprintf(stderr, "must specify -f path to file\n"); - ret = -EINVAL; - goto out; - } - - if (name == NULL) { - fprintf(stderr, "must specify -n xattr name to search for\n"); - ret = -EINVAL; - goto out; - } - - fd = open(path, O_RDONLY); - if (fd < 0) { - ret = -errno; - fprintf(stderr, "failed to open '%s': %s (%d)\n", - path, strerror(errno), errno); - goto out; - } + fd = get_path(args->path, O_RDONLY); + if (fd < 0) + return fd; sx.next_ino = 0; sx.last_ino = U64_MAX; - sx.name_ptr = (unsigned long)name; + sx.name_ptr = (unsigned long)args->name; sx.inodes_ptr = (unsigned long)inos; - sx.name_bytes = strlen(name); + sx.name_bytes = strlen(args->name); sx.nr_inodes = BATCH_SIZE; do { @@ -119,13 +76,60 @@ static int search_xattrs_cmd(int argc, char **argv) out: if (fd >= 0) close(fd); - free(path); - free(name); free(inos); return ret; }; +static int parse_opt(int key, char *arg, struct argp_state *state) +{ + struct xattr_args *args = state->input; + + switch (key) { + case 'p': + args->path = strdup_or_error(state, arg); + break; + case ARGP_KEY_ARG: + if (args->name) + argp_error(state, "more than one name argument given"); + + args->name = strdup_or_error(state, arg); + break; + case ARGP_KEY_FINI: + if (!args->name) { + argp_error(state, "must provide xattr containing .srch. scoutfs tag"); + } + break; + default: + break; + } + + return 0; +} + +static struct argp_option options[] = { + { "path", 'p', "PATH", 0, "Path to ScoutFS filesystem"}, + { NULL } +}; + +static int search_xattrs_cmd(int argc, char **argv) +{ + struct argp argp = { + options, + parse_opt, + "XATTR-NAME", + "Print inode numbers of inodes which may have given xattr" + }; + struct xattr_args xattr_args = {NULL}; + int ret; + + ret = argp_parse(&argp, argc, argv, 0, NULL, &xattr_args); + if (ret) + return ret; + + return do_search_xattrs(&xattr_args); +} + static void __attribute__((constructor)) search_xattrs_ctor(void) { cmd_register("search-xattrs", "-n name -f ", From d025122fdd0ad9737ee3dd5ae1c4542435f55f59 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Wed, 2 Dec 2020 11:39:28 -0800 Subject: [PATCH 08/21] Implement argp support for listxaddr-hidden Rename to list-hidden-xaddrs. Signed-off-by: Andy Grover --- utils/src/listxattr_hidden.c | 87 +++++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 35 deletions(-) diff --git a/utils/src/listxattr_hidden.c b/utils/src/listxattr_hidden.c index a98426aa..10130b3c 100644 --- a/utils/src/listxattr_hidden.c +++ b/utils/src/listxattr_hidden.c @@ -7,56 +7,31 @@ #include #include #include -#include #include +#include #include "sparse.h" +#include "parse.h" #include "util.h" #include "format.h" #include "ioctl.h" #include "cmd.h" -static struct option long_ops[] = { - { "file", 1, NULL, 'f' }, - { NULL, 0, NULL, 0} +struct list_hidden_xattr_args { + char *filename; }; -static int listxattr_hidden_cmd(int argc, char **argv) +static int do_list_hidden_xattrs(struct list_hidden_xattr_args *args) { struct scoutfs_ioctl_listxattr_hidden lxh; - char *path = NULL; char *buf = NULL; char *name; int fd = -1; int bytes; int len; int ret; - int c; int i; - while ((c = getopt_long(argc, argv, "f:", long_ops, NULL)) != -1) { - switch (c) { - case 'f': - path = strdup(optarg); - if (!path) { - fprintf(stderr, "path mem alloc failed\n"); - ret = -ENOMEM; - goto out; - } - break; - case '?': - default: - ret = -EINVAL; - goto out; - } - } - - if (path == NULL) { - fprintf(stderr, "must specify -f path to file\n"); - ret = -EINVAL; - goto out; - } - memset(&lxh, 0, sizeof(lxh)); lxh.id_pos = 0; lxh.hash_pos = 0; @@ -69,11 +44,11 @@ static int listxattr_hidden_cmd(int argc, char **argv) } lxh.buf_ptr = (unsigned long)buf; - fd = open(path, O_RDONLY); + fd = open(args->filename, O_RDONLY); if (fd < 0) { ret = -errno; fprintf(stderr, "failed to open '%s': %s (%d)\n", - path, strerror(errno), errno); + args->filename, strerror(errno), errno); goto out; } @@ -139,9 +114,51 @@ out: return ret; }; +static int parse_opt(int key, char *arg, struct argp_state *state) +{ + struct list_hidden_xattr_args *args = state->input; + + switch (key) { + case ARGP_KEY_ARG: + if (args->filename) + argp_error(state, "more than one filename argument given"); + + args->filename = strdup_or_error(state, arg); + break; + case ARGP_KEY_FINI: + if (!args->filename) { + argp_error(state, "must specify filename"); + } + break; + default: + break; + } + + return 0; +} + +static int list_hidden_xattrs_cmd(int argc, char **argv) +{ + struct argp argp = { + NULL, + parse_opt, + "FILE", + "Print the names of hidden xattrs on a file" + }; + struct list_hidden_xattr_args list_hidden_xattr_args = {NULL}; + int ret; + + ret = argp_parse(&argp, argc, argv, 0, NULL, &list_hidden_xattr_args); + if (ret) + return ret; + + return do_list_hidden_xattrs(&list_hidden_xattr_args); +} + + static void __attribute__((constructor)) listxattr_hidden_ctor(void) { - cmd_register("listxattr-hidden", "-f ", - "print the names of hidden xattrs on the file", - listxattr_hidden_cmd); + cmd_register("list-hidden-xattrs", "", + "print the names of hidden xattrs on a file", + list_hidden_xattrs_cmd); } From 6b5ddf2b3aa47ff27673e5c31143866a6c8bcacd Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Wed, 2 Dec 2020 17:21:35 -0800 Subject: [PATCH 09/21] Implement argp support for print Print warning if printing a data dev, you probably wanted the meta dev. Change read_block to return err value. Otherwise there are confusing ENOMEM messages when pread() fails. e.g. try to print /dev/null. Signed-off-by: Andy Grover --- utils/src/print.c | 127 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 88 insertions(+), 39 deletions(-) diff --git a/utils/src/print.c b/utils/src/print.c index 7f713848..d7e548ef 100644 --- a/utils/src/print.c +++ b/utils/src/print.c @@ -12,8 +12,10 @@ #include #include #include +#include #include "sparse.h" +#include "parse.h" #include "util.h" #include "format.h" #include "bitmap.h" @@ -24,25 +26,32 @@ #include "srch.h" #include "leaf_item_hash.h" -static void *read_block(int fd, u64 blkno, int shift) +static int read_block(int fd, u64 blkno, int shift, void **ret_val) { size_t size = 1ULL << shift; - ssize_t ret; void *buf; + int ret; + + *ret_val = NULL; buf = malloc(size); if (!buf) - return NULL; + return -ENOMEM; ret = pread(fd, buf, size, blkno << shift); - if (ret != size) { - fprintf(stderr, "read blkno %llu returned %zd: %s (%d)\n", + if (ret == -1) { + fprintf(stderr, "read blkno %llu returned %d: %s (%d)\n", blkno, ret, strerror(errno), errno); free(buf); - buf = NULL; + return -errno; + } else if (ret != size) { + fprintf(stderr, "incomplete pread\n"); + free(buf); + return -EINVAL; + } else { + *ret_val = buf; + return 0; } - - return buf; } static void print_block_header(struct scoutfs_block_header *hdr, int size) @@ -465,9 +474,9 @@ static int print_btree_block(int fd, struct scoutfs_super_block *super, int ret; int i; - bt = read_block(fd, le64_to_cpu(ref->blkno), SCOUTFS_BLOCK_LG_SHIFT); - if (!bt) - return -ENOMEM; + ret = read_block(fd, le64_to_cpu(ref->blkno), SCOUTFS_BLOCK_LG_SHIFT, (void **)&bt); + if (ret) + return ret; if (bt->level == level) { printf("%s btree blkno %llu\n" @@ -559,15 +568,16 @@ static int print_alloc_list_block(int fd, char *str, u64 start; u64 len; int wid; + int ret; int i; blkno = le64_to_cpu(ref->blkno); if (blkno == 0) return 0; - lblk = read_block(fd, blkno, SCOUTFS_BLOCK_LG_SHIFT); - if (!lblk) - return -ENOMEM; + ret = read_block(fd, blkno, SCOUTFS_BLOCK_LG_SHIFT, (void **)&lblk); + if (ret) + return ret; printf("%s alloc_list_block blkno %llu\n", str, blkno); print_block_header(&lblk->hdr, SCOUTFS_BLOCK_LG_SIZE); @@ -617,11 +627,10 @@ static int print_srch_block(int fd, struct scoutfs_srch_ref *ref, int level) if (blkno == 0) return 0; - srp = read_block(fd, blkno, SCOUTFS_BLOCK_LG_SHIFT); - if (!srp) { - ret = -ENOMEM; + ret = read_block(fd, blkno, SCOUTFS_BLOCK_LG_SHIFT, (void **)&srp); + if (ret) goto out; - } + srb = (void *)srp; printf("srch %sblock blkno %llu\n", level ? "parent " : "", blkno); @@ -763,9 +772,9 @@ static int print_btree_leaf_items(int fd, struct scoutfs_super_block *super, if (ref->blkno == 0) return 0; - bt = read_block(fd, le64_to_cpu(ref->blkno), SCOUTFS_BLOCK_LG_SHIFT); - if (!bt) - return -ENOMEM; + ret = read_block(fd, le64_to_cpu(ref->blkno), SCOUTFS_BLOCK_LG_SHIFT, (void **)&bt); + if (ret) + return ret; node = avl_first(&bt->item_root); while (node) { @@ -828,11 +837,9 @@ static int print_quorum_blocks(int fd, struct scoutfs_super_block *super) for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) { blkno = SCOUTFS_QUORUM_BLKNO + i; free(blk); - blk = read_block(fd, blkno, SCOUTFS_BLOCK_SM_SHIFT); - if (!blk) { - ret = -ENOMEM; + ret = read_block(fd, blkno, SCOUTFS_BLOCK_SM_SHIFT, (void **)&blk); + if (ret) goto out; - } if (blk->voter_rid != 0) { printf("quorum block blkno %llu\n" @@ -876,11 +883,15 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno) uuid_unparse(super->uuid, uuid_str); + if (!(le64_to_cpu(super->flags) && SCOUTFS_FLAG_IS_META_BDEV)) + fprintf(stderr, + "**** Printing metadata from a data device! Did you mean to do this? ****\n"); + printf("super blkno %llu\n", blkno); print_block_header(&super->hdr, SCOUTFS_BLOCK_SM_SIZE); printf(" format_hash %llx uuid %s\n", le64_to_cpu(super->format_hash), uuid_str); - printf(" flags: 0x%016llx\n", super->flags); + printf(" flags: 0x%016llx\n", le64_to_cpu(super->flags)); server_addr = alloc_addr_str(&super->server_addr); if (!server_addr) @@ -943,6 +954,10 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno) free(server_addr); } +struct print_args { + char *meta_device; +}; + static int print_volume(int fd) { struct scoutfs_super_block *super = NULL; @@ -952,9 +967,9 @@ static int print_volume(int fd) int err; int i; - super = read_block(fd, SCOUTFS_SUPER_BLKNO, SCOUTFS_BLOCK_SM_SHIFT); - if (!super) - return -ENOMEM; + ret = read_block(fd, SCOUTFS_SUPER_BLKNO, SCOUTFS_BLOCK_SM_SHIFT, (void **)&super); + if (ret) + return ret; print_super_block(super, SCOUTFS_SUPER_BLKNO); @@ -1034,23 +1049,16 @@ static int print_volume(int fd) return ret; } -static int print_cmd(int argc, char **argv) +static int do_print(struct print_args *args) { - char *path; int ret; int fd; - if (argc != 2) { - printf("scoutfs print: a single path argument is required\n"); - return -EINVAL; - } - path = argv[1]; - - fd = open(path, O_RDONLY); + fd = open(args->meta_device, O_RDONLY); if (fd < 0) { ret = -errno; fprintf(stderr, "failed to open '%s': %s (%d)\n", - path, strerror(errno), errno); + args->meta_device, strerror(errno), errno); return ret; } @@ -1059,6 +1067,47 @@ static int print_cmd(int argc, char **argv) return ret; }; +static int parse_opt(int key, char *arg, struct argp_state *state) +{ + struct print_args *args = state->input; + + switch (key) { + case ARGP_KEY_ARG: + if (!args->meta_device) + args->meta_device = strdup_or_error(state, arg); + else + argp_error(state, "more than one argument given"); + break; + case ARGP_KEY_FINI: + if (!args->meta_device) + argp_error(state, "no metadata device argument given"); + break; + default: + break; + } + + return 0; +} + +static int print_cmd(int argc, char **argv) +{ + struct argp argp = { + NULL, + parse_opt, + "META-DEV", + "Print metadata structures" + }; + struct print_args print_args = {NULL}; + int ret; + + ret = argp_parse(&argp, argc, argv, 0, NULL, &print_args); + if (ret) + return ret; + + return do_print(&print_args); +} + + static void __attribute__((constructor)) print_ctor(void) { cmd_register("print", "", "print metadata structures", From 7befc6148274ac57b786cd3397c165fa3da0eaf3 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Wed, 2 Dec 2020 12:48:55 -0800 Subject: [PATCH 10/21] Implement argp support for mkfs and add --force Support max-meta-size and max-data-size using KMGTP units with rounding. Detect other fs signatures using blkid library. Detect ScoutFS super using magic value. Move read_block() from print.c into util.c since blkid also needs it. Signed-off-by: Andy Grover --- utils/Makefile | 2 +- utils/scoutfs-utils.spec.in | 1 + utils/src/blkid.c | 94 ++++++++++++++ utils/src/blkid.h | 6 + utils/src/mkfs.c | 238 +++++++++++++++++++++--------------- utils/src/print.c | 28 ----- utils/src/util.c | 28 +++++ utils/src/util.h | 1 + 8 files changed, 270 insertions(+), 128 deletions(-) create mode 100644 utils/src/blkid.c create mode 100644 utils/src/blkid.h diff --git a/utils/Makefile b/utils/Makefile index ad146e5b..8116bcd8 100644 --- a/utils/Makefile +++ b/utils/Makefile @@ -47,7 +47,7 @@ endif $(BIN): $(OBJ) $(QU) [BIN $@] - $(VE)gcc -o $@ $^ -luuid -lm -lcrypto + $(VE)gcc -o $@ $^ -luuid -lm -lcrypto -lblkid %.o %.d: %.c Makefile sparse.sh $(QU) [CC $<] diff --git a/utils/scoutfs-utils.spec.in b/utils/scoutfs-utils.spec.in index 35219721..42bb86c5 100644 --- a/utils/scoutfs-utils.spec.in +++ b/utils/scoutfs-utils.spec.in @@ -16,6 +16,7 @@ BuildRequires: git BuildRequires: gzip BuildRequires: libuuid-devel BuildRequires: openssl-devel +BuildRequires: libblkid-devel #Requires: kmod-scoutfs = %{version} diff --git a/utils/src/blkid.c b/utils/src/blkid.c new file mode 100644 index 00000000..7dd19574 --- /dev/null +++ b/utils/src/blkid.c @@ -0,0 +1,94 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "util.h" +#include "format.h" +#include "blkid.h" + +static int check_bdev_blkid(int fd, char *devname, char *usage) +{ + blkid_probe pr; + int ret = 0; + + pr = blkid_new_probe_from_filename(devname); + if (!pr) { + fprintf(stderr, "%s: failed to create a new libblkid probe\n", devname); + goto out; + } + + /* enable partitions probing (superblocks are enabled by default) */ + ret = blkid_probe_enable_partitions(pr, true); + if (ret == -1) { + fprintf(stderr, "%s: blkid_probe_enable_partitions() failed\n", devname); + goto out; + } + + ret = blkid_do_fullprobe(pr); + if (ret == -1) { + fprintf(stderr, "%s: blkid_do_fullprobe() failed", devname); + goto out; + } else if (ret == 0) { + const char *type; + + if (!blkid_probe_lookup_value(pr, "TYPE", &type, NULL)) { + fprintf(stderr, "%s: appears to contain an existing " + "%s superblock\n", devname, type); + ret = -1; + goto out; + } + + if (!blkid_probe_lookup_value(pr, "PTTYPE", &type, NULL)) { + fprintf(stderr, "%s: appears to contain a partition " + "table (%s)\n", devname, type); + ret = -1; + goto out; + } + } else { + /* return 0 if ok */ + ret = 0; + } + +out: + blkid_free_probe(pr); + + return ret; +} + +static int check_bdev_scoutfs(int fd, char *devname, char *usage) +{ + struct scoutfs_super_block *super = NULL; + int ret; + + ret = read_block(fd, SCOUTFS_SUPER_BLKNO, SCOUTFS_BLOCK_SM_SHIFT, (void **)&super); + if (ret) + return ret; + + if (le32_to_cpu(super->hdr.magic) == SCOUTFS_SUPER_MAGIC) { + fprintf(stderr, "%s: appears to contain an existing " + "ScoutFS superblock\n", devname); + ret = -EINVAL; + } + + free(super); + + return ret; +} + + +/* + * Returns -1 on error, 0 otherwise. + */ +int check_bdev(int fd, char *devname, char *usage) +{ + return check_bdev_blkid(fd, devname, usage) ?: + /* Our sig is not in blkid (yet) so check explicitly for us. */ + check_bdev_scoutfs(fd, devname, usage); +} diff --git a/utils/src/blkid.h b/utils/src/blkid.h new file mode 100644 index 00000000..ddb708c4 --- /dev/null +++ b/utils/src/blkid.h @@ -0,0 +1,6 @@ +#ifndef _BLKID_H_ +#define _BLKID_H_ + +int check_bdev(int fd, char *path, char *usage); + +#endif diff --git a/utils/src/mkfs.c b/utils/src/mkfs.c index 8d5e4dff..d9c04dd4 100644 --- a/utils/src/mkfs.c +++ b/utils/src/mkfs.c @@ -11,12 +11,12 @@ #include #include #include -#include #include #include #include #include #include +#include #include "sparse.h" #include "cmd.h" @@ -30,6 +30,7 @@ #include "bitops.h" #include "btree.h" #include "leaf_item_hash.h" +#include "blkid.h" static int write_raw_block(int fd, u64 blkno, int shift, void *blk) { @@ -99,6 +100,16 @@ static int write_alloc_root(struct scoutfs_super_block *super, int fd, return write_raw_block(fd, blkno, SCOUTFS_BLOCK_LG_SHIFT, bt); } +struct mkfs_args { + unsigned long long quorum_count; + char *meta_device; + char *data_device; + unsigned long long max_meta_size; + unsigned long long max_data_size; + bool force; + char __pad[7]; +}; + /* * Make a new file system by writing: * - super blocks @@ -108,19 +119,18 @@ static int write_alloc_root(struct scoutfs_super_block *super, int fd, * Superblock is written to both metadata and data devices, everything else is * written only to the metadata device. */ -static int write_new_fs(char *meta_path, char *data_path, - int meta_fd, int data_fd, - u8 quorum_count, - u64 max_meta_size, u64 max_data_size) +static int do_mkfs(struct mkfs_args *args) { - struct scoutfs_super_block *super; + struct scoutfs_super_block *super = NULL; struct scoutfs_inode inode; struct scoutfs_alloc_list_block *lblk; - struct scoutfs_btree_block *bt; + struct scoutfs_btree_block *bt = NULL; struct scoutfs_key key; struct timeval tv; + int meta_fd = -1; + int data_fd = -1; char uuid_str[37]; - void *zeros; + void *zeros = NULL; u64 blkno; u64 meta_size; u64 data_size; @@ -135,6 +145,33 @@ static int write_new_fs(char *meta_path, char *data_path, gettimeofday(&tv, NULL); + meta_fd = open(args->meta_device, O_RDWR | O_EXCL); + if (meta_fd < 0) { + ret = -errno; + fprintf(stderr, "failed to open '%s': %s (%d)\n", + args->meta_device, strerror(errno), errno); + goto out; + } + if (!args->force) { + ret = check_bdev(meta_fd, args->meta_device, "meta"); + if (ret) + return ret; + } + + data_fd = open(args->data_device, O_RDWR | O_EXCL); + if (data_fd < 0) { + ret = -errno; + fprintf(stderr, "failed to open '%s': %s (%d)\n", + args->data_device, strerror(errno), errno); + goto out; + } + if (!args->force) { + ret = check_bdev(data_fd, args->data_device, "data"); + if (ret) + return ret; + } + + super = calloc(1, SCOUTFS_BLOCK_SM_SIZE); bt = calloc(1, SCOUTFS_BLOCK_LG_SIZE); zeros = calloc(1, SCOUTFS_BLOCK_SM_SIZE); @@ -145,13 +182,13 @@ static int write_new_fs(char *meta_path, char *data_path, goto out; } - ret = device_size(meta_path, meta_fd, 2ULL * (1024 * 1024 * 1024), - max_meta_size, "meta", &meta_size); + ret = device_size(args->meta_device, meta_fd, 2ULL * (1024 * 1024 * 1024), + args->max_meta_size, "meta", &meta_size); if (ret) goto out; - ret = device_size(data_path, data_fd, 8ULL * (1024 * 1024 * 1024), - max_data_size, "data", &data_size); + ret = device_size(args->data_device, data_fd, 8ULL * (1024 * 1024 * 1024), + args->max_data_size, "data", &data_size); if (ret) goto out; @@ -179,7 +216,7 @@ static int write_new_fs(char *meta_path, char *data_path, super->total_data_blocks = cpu_to_le64(last_data - first_data + 1); super->first_data_blkno = cpu_to_le64(first_data); super->last_data_blkno = cpu_to_le64(last_data); - super->quorum_count = quorum_count; + super->quorum_count = args->quorum_count; /* fs root starts with root inode and its index items */ blkno = next_meta++; @@ -293,7 +330,7 @@ static int write_new_fs(char *meta_path, char *data_path, if (fsync(data_fd)) { ret = -errno; fprintf(stderr, "failed to fsync '%s': %s (%d)\n", - data_path, strerror(errno), errno); + args->data_device, strerror(errno), errno); goto out; } @@ -306,7 +343,7 @@ static int write_new_fs(char *meta_path, char *data_path, if (fsync(meta_fd)) { ret = -errno; fprintf(stderr, "failed to fsync '%s': %s (%d)\n", - meta_path, strerror(errno), errno); + args->meta_device, strerror(errno), errno); goto out; } @@ -321,8 +358,8 @@ static int write_new_fs(char *meta_path, char *data_path, " 64KB metadata blocks: "SIZE_FMT"\n" " 4KB data blocks: "SIZE_FMT"\n" " quorum count: %u\n", - meta_path, - data_path, + args->meta_device, + args->data_device, le64_to_cpu(super->hdr.fsid), le64_to_cpu(super->format_hash), uuid_str, @@ -340,102 +377,105 @@ out: free(bt); if (zeros) free(zeros); + if (meta_fd != -1) + close(meta_fd); + if (data_fd != -1) + close(data_fd); return ret; } -static struct option long_ops[] = { - { "quorum_count", 1, NULL, 'Q' }, - { NULL, 0, NULL, 0} +static int parse_opt(int key, char *arg, struct argp_state *state) +{ + struct mkfs_args *args = state->input; + int ret; + + switch (key) { + case 'Q': + ret = parse_u64(arg, &args->quorum_count); + if (ret) + return ret; + break; + case 'f': + args->force = true; + break; + case 'm': /* max-meta-size */ + { + u64 prev_val; + ret = parse_human(arg, &args->max_meta_size); + if (ret) + return ret; + prev_val = args->max_meta_size; + args->max_meta_size = round_down(args->max_meta_size, SCOUTFS_BLOCK_LG_SIZE); + if (args->max_meta_size != prev_val) + fprintf(stderr, "Meta dev size %llu rounded down to %llu bytes\n", + prev_val, args->max_meta_size); + break; + } + case 'd': /* max-data-size */ + { + u64 prev_val; + ret = parse_human(arg, &args->max_data_size); + if (ret) + return ret; + prev_val = args->max_data_size; + args->max_data_size = round_down(args->max_data_size, SCOUTFS_BLOCK_SM_SIZE); + if (args->max_data_size != prev_val) + fprintf(stderr, "Data dev size %llu rounded down to %llu bytes\n", + prev_val, args->max_data_size); + break; + } + case ARGP_KEY_ARG: + if (!args->meta_device) + args->meta_device = strdup_or_error(state, arg); + else if (!args->data_device) + args->data_device = strdup_or_error(state, arg); + else + argp_error(state, "more than two arguments given"); + break; + case ARGP_KEY_FINI: + if (!args->quorum_count) + argp_error(state, "must provide nonzero quorum count with --quorum-count|-Q option"); + if (!args->meta_device) + argp_error(state, "no metadata device argument given"); + if (!args->data_device) + argp_error(state, "no data device argument given"); + break; + default: + break; + } + + return 0; +} + +static struct argp_option options[] = { + { "quorum-count", 'Q', "NUM", 0, "Number of voters required to use the filesystem [Required]"}, + { "force", 'f', NULL, 0, "Overwrite existing data on block devices"}, + { "max-meta-size", 'm', "SIZE", 0, "Use a size less than the base metadata device size (bytes or KMGTP units)"}, + { "max-data-size", 'd', "SIZE", 0, "Use a size less than the base data device size (bytes or KMGTP units)"}, + { NULL } }; -static int mkfs_func(int argc, char *argv[]) +static int mkfs_cmd(int argc, char *argv[]) { - unsigned long long ull; - u8 quorum_count = 0; - u64 max_data_size = 0; - u64 max_meta_size = 0; - char *end = NULL; - char *meta_path; - char *data_path; - int meta_fd; - int data_fd; + struct argp argp = { + options, + parse_opt, + "META-DEVICE DATA-DEVICE", + "Initialize a new ScoutFS filesystem" + }; + struct mkfs_args mkfs_args = {0}; int ret; - int c; - while ((c = getopt_long(argc, argv, "Q:D:M:", long_ops, NULL)) != -1) { - switch (c) { - case 'Q': - ull = strtoull(optarg, &end, 0); - if (*end != '\0' || ull == 0 || - ull > SCOUTFS_QUORUM_MAX_COUNT) { - printf("scoutfs: invalid quorum count '%s'\n", - optarg); - return -EINVAL; - } - quorum_count = ull; - break; - case 'D': - ret = parse_human(optarg, &max_data_size); - if (ret < 0) { - printf("scoutfs: invalid data device size '%s'\n", - optarg); - return ret; - } - break; - case 'M': - ret = parse_human(optarg, &max_meta_size); - if (ret < 0) { - printf("scoutfs: invalid meta device size '%s'\n", - optarg); - return ret; - } - break; - case '?': - default: - return -EINVAL; - } - } - - if (optind + 2 != argc) { - printf("scoutfs: mkfs: paths to metadata and data devices are required\n"); - return -EINVAL; - } - - meta_path = argv[optind]; - data_path = argv[optind + 1]; - - if (!quorum_count) { - printf("provide quorum count with --quorum_count|-Q option\n"); - return -EINVAL; - } - - meta_fd = open(meta_path, O_RDWR | O_EXCL); - if (meta_fd < 0) { - ret = -errno; - fprintf(stderr, "failed to open metadata device '%s': %s (%d)\n", - meta_path, strerror(errno), errno); + ret = argp_parse(&argp, argc, argv, 0, NULL, &mkfs_args); + if (ret) return ret; - } - data_fd = open(data_path, O_RDWR | O_EXCL); - if (data_fd < 0) { - ret = -errno; - fprintf(stderr, "failed to open data device '%s': %s (%d)\n", - data_path, strerror(errno), errno); - return ret; - } - - ret = write_new_fs(meta_path, data_path, meta_fd, data_fd, - quorum_count, max_meta_size, max_data_size); - close(meta_fd); - close(data_fd); - - return ret; + return do_mkfs(&mkfs_args); } static void __attribute__((constructor)) mkfs_ctor(void) { - cmd_register("mkfs", "", "write a new file system", mkfs_func); + cmd_register("mkfs", " ", "write a new file system", mkfs_cmd); /* for lack of some other place to put these.. */ build_assert(sizeof(uuid_t) == SCOUTFS_UUID_BYTES); diff --git a/utils/src/print.c b/utils/src/print.c index d7e548ef..faacb467 100644 --- a/utils/src/print.c +++ b/utils/src/print.c @@ -26,34 +26,6 @@ #include "srch.h" #include "leaf_item_hash.h" -static int read_block(int fd, u64 blkno, int shift, void **ret_val) -{ - size_t size = 1ULL << shift; - void *buf; - int ret; - - *ret_val = NULL; - - buf = malloc(size); - if (!buf) - return -ENOMEM; - - ret = pread(fd, buf, size, blkno << shift); - if (ret == -1) { - fprintf(stderr, "read blkno %llu returned %d: %s (%d)\n", - blkno, ret, strerror(errno), errno); - free(buf); - return -errno; - } else if (ret != size) { - fprintf(stderr, "incomplete pread\n"); - free(buf); - return -EINVAL; - } else { - *ret_val = buf; - return 0; - } -} - static void print_block_header(struct scoutfs_block_header *hdr, int size) { u32 crc = crc_block(hdr, size); diff --git a/utils/src/util.c b/utils/src/util.c index dfa948ff..d30e44ba 100644 --- a/utils/src/util.c +++ b/utils/src/util.c @@ -68,3 +68,31 @@ int get_path(char *path, int flags) return ret; } + +int read_block(int fd, u64 blkno, int shift, void **ret_val) +{ + size_t size = 1ULL << shift; + void *buf; + int ret; + + *ret_val = NULL; + + buf = malloc(size); + if (!buf) + return -ENOMEM; + + ret = pread(fd, buf, size, blkno << shift); + if (ret == -1) { + fprintf(stderr, "read blkno %llu returned %d: %s (%d)\n", + blkno, ret, strerror(errno), errno); + free(buf); + return -errno; + } else if (ret != size) { + fprintf(stderr, "incomplete pread\n"); + free(buf); + return -EINVAL; + } else { + *ret_val = buf; + return 0; + } +} diff --git a/utils/src/util.h b/utils/src/util.h index c70ddbb5..ab195d9d 100644 --- a/utils/src/util.h +++ b/utils/src/util.h @@ -112,5 +112,6 @@ static inline int memcmp_lens(const void *a, int a_len, } int get_path(char *path, int flags); +int read_block(int fd, u64 blkno, int shift, void **ret_val); #endif From f35154eb1913973bdf956694d1c153aac29d2361 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Wed, 2 Dec 2020 14:47:40 -0800 Subject: [PATCH 11/21] counters: Ensure name_wid[0] is initialized to zero I was seeing some segfaults and other weirdness without this. Signed-off-by: Andy Grover --- utils/src/counters.c | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/src/counters.c b/utils/src/counters.c index 676226dc..831688e1 100644 --- a/utils/src/counters.c +++ b/utils/src/counters.c @@ -120,6 +120,7 @@ static int counters_cmd(int argc, char **argv) goto out; } memset(&ctrs[nr], 0, (alloced - nr) * sizeof(*ctrs)); + memset(&name_wid[nr], 0, (alloced - nr) * sizeof(*name_wid)); } ctr = &ctrs[nr]; From e1ba50830140ccd7f2b348067fd43a4bba6a49d2 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Wed, 2 Dec 2020 14:47:49 -0800 Subject: [PATCH 12/21] Implement argp support for counters Signed-off-by: Andy Grover --- utils/src/counters.c | 85 +++++++++++++++++++++++++++++++++----------- 1 file changed, 65 insertions(+), 20 deletions(-) diff --git a/utils/src/counters.c b/utils/src/counters.c index 831688e1..b57ecb54 100644 --- a/utils/src/counters.c +++ b/utils/src/counters.c @@ -12,7 +12,10 @@ #include #include #include +#include +#include "sparse.h" +#include "parse.h" #include "util.h" #include "cmd.h" @@ -37,7 +40,13 @@ static int cmp_counter_names(const void *A, const void *B) return strcmp(a->name, b->name); } -static int counters_cmd(int argc, char **argv) +struct counters_args { + char *sysfs_path; + bool tabular; + char __pad[7]; +}; + +static int do_counters(struct counters_args *args) { unsigned int *name_wid = NULL; unsigned int *val_wid = NULL; @@ -50,9 +59,7 @@ static int counters_cmd(int argc, char **argv) unsigned int rows = 0; unsigned int cols = 0; unsigned int nr = 0; - char *dir_arg = NULL; struct dirent *dent; - bool table = false; struct winsize ws; DIR *dirp = NULL; int dir_fd = -1; @@ -64,28 +71,16 @@ static int counters_cmd(int argc, char **argv) int r; int c; - for (i = 1; i < argc; i++) { - if (strcmp(argv[i], "-t") == 0) - table = true; - else - dir_arg = argv[i]; - } - ret = ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws); if (ret < 0) ret = ioctl(STDIN_FILENO, TIOCGWINSZ, &ws); if (ret < 0) - table = false; + args->tabular = false; - if (dir_arg == NULL) { - printf("scoutfs counter-table: need mount sysfs dir (i.e. /sys/fs/scoutfs/$fr)\n"); - return -EINVAL; - } - - ret = snprintf(path, PATH_MAX, "%s/counters", dir_arg); + ret = snprintf(path, PATH_MAX, "%s/counters", args->sysfs_path); if (ret < 1 || ret >= PATH_MAX) { ret = -EINVAL; - fprintf(stderr, "invalid counter dir path '%s'\n", dir_arg); + fprintf(stderr, "invalid counter dir path '%s'\n", args->sysfs_path); goto out; } @@ -192,7 +187,7 @@ static int counters_cmd(int argc, char **argv) * one column of counters and use the max field widths from the * initial counter reads. */ - if (table) { + if (args->tabular) { min_rows = 1; cols = ws.ws_col / (name_wid[0] + 1 + val_wid[0] + 2); max_rows = nr / cols; @@ -277,9 +272,59 @@ out: return ret; }; +static int parse_opt(int key, char *arg, struct argp_state *state) +{ + struct counters_args *args = state->input; + + switch (key) { + case 't': + args->tabular = true; + break; + case ARGP_KEY_ARG: + if (!args->sysfs_path) + args->sysfs_path = strdup_or_error(state, arg); + else + argp_error(state, "more than one argument given"); + break; + case ARGP_KEY_FINI: + if (!args->sysfs_path) + argp_error(state, "no sysfs path argument given"); + break; + default: + break; + } + + return 0; +} + + +static struct argp_option options[] = { + { "table", 't', NULL, 0, "Output in table format" }, + { NULL } +}; + +static int counters_cmd(int argc, char *argv[]) +{ + struct argp argp = { + options, + parse_opt, + "SYSFS-DIR", + "Show counters for a mounted volume" + }; + struct counters_args counters_args = {NULL}; + int ret; + + ret = argp_parse(&argp, argc, argv, 0, NULL, &counters_args); + if (ret) + return ret; + + return do_counters(&counters_args); +} + + static void __attribute__((constructor)) counters_ctor(void) { cmd_register("counters", "[-t] ", - "show [tablular] counters for a given mounted volume", + "show [tabular] counters for a given mounted volume", counters_cmd); } From 7c54c86c38801a05f3bf9baea3171eb84329b37c Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Thu, 3 Dec 2020 13:57:26 -0800 Subject: [PATCH 13/21] Implement argp support for setattr Signed-off-by: Andy Grover --- utils/src/setattr.c | 164 ++++++++++++++++++++++++++------------------ 1 file changed, 99 insertions(+), 65 deletions(-) diff --git a/utils/src/setattr.c b/utils/src/setattr.c index e9ab3b34..23922531 100644 --- a/utils/src/setattr.c +++ b/utils/src/setattr.c @@ -7,8 +7,9 @@ #include #include #include -#include #include +#include +#include #include "sparse.h" #include "util.h" @@ -17,83 +18,41 @@ #include "parse.h" #include "cmd.h" -static struct option long_ops[] = { - { "ctime", 1, NULL, 'c' }, - { "data_version", 1, NULL, 'd' }, - { "file", 1, NULL, 'f' }, - { "offline", 0, NULL, 'o' }, - { "i_size", 1, NULL, 's' }, - { NULL, 0, NULL, 0} +struct setattr_args { + char *filename; + struct timespec ctime; + u64 data_version; + u64 i_size; + bool offline; + char __pad[7]; }; -static int setattr_more_cmd(int argc, char **argv) +static int do_setattr(struct setattr_args *args) { - struct scoutfs_ioctl_setattr_more sm; - struct timespec ctime; - char *path = NULL; - int ret; + struct scoutfs_ioctl_setattr_more sm = {0}; int fd = -1; - int c; + int ret; - memset(&sm, 0, sizeof(sm)); - - while ((c = getopt_long(argc, argv, "c:d:f:os:", long_ops, NULL)) != -1) { - switch (c) { - case 'c': - ret = parse_timespec(optarg, &ctime); - if (ret) - goto out; - break; - case 'd': - ret = parse_u64(optarg, &sm.data_version); - if (ret) - goto out; - break; - case 'f': - path = strdup(optarg); - if (!path) { - fprintf(stderr, "path mem alloc failed\n"); - ret = -ENOMEM; - goto out; - } - break; - case 'o': - sm.flags |= SCOUTFS_IOC_SETATTR_MORE_OFFLINE; - break; - case 's': - ret = parse_u64(optarg, &sm.i_size); - if (ret) - goto out; - break; - case '?': - default: - ret = -EINVAL; - goto out; - } - } - - if (path == NULL) { - fprintf(stderr, "must specify -f path to file\n"); - ret = -EINVAL; - goto out; - } - - fd = open(path, O_WRONLY); + fd = open(args->filename, O_WRONLY); if (fd < 0) { ret = -errno; fprintf(stderr, "failed to open '%s': %s (%d)\n", - path, strerror(errno), errno); + args->filename, strerror(errno), errno); goto out; } - sm.ctime_sec = ctime.tv_sec; - sm.ctime_nsec = ctime.tv_nsec; + sm.ctime_sec = args->ctime.tv_sec; + sm.ctime_nsec = args->ctime.tv_nsec; + sm.data_version = args->data_version; + if (args->offline) + sm.flags |= SCOUTFS_IOC_SETATTR_MORE_OFFLINE; + sm.i_size = args->i_size; ret = ioctl(fd, SCOUTFS_IOC_SETATTR_MORE, &sm); if (ret < 0) { ret = -errno; fprintf(stderr, "setattr_more ioctl failed on '%s': " - "%s (%d)\n", path, strerror(errno), errno); + "%s (%d)\n", args->filename, strerror(errno), errno); goto out; } @@ -104,9 +63,84 @@ out: return ret; } +static int parse_opt(int key, char *arg, struct argp_state *state) +{ + struct setattr_args *args = state->input; + int ret; + + switch (key) { + case 't': /* timespec */ + ret = parse_timespec(arg, &args->ctime); + if (ret) + return ret; + break; + case 'V': /* data version */ + ret = parse_u64(arg, &args->data_version); + if (ret) + return ret; + if (args->data_version == 0) + argp_error(state, "data version must not be 0"); + break; + case 's': /* size */ + ret = parse_human(arg, &args->i_size); + if (ret) + return ret; + break; + case 'o': /* offline */ + args->offline = true; + break; + case ARGP_KEY_ARG: + if (!args->filename) + args->filename = strdup_or_error(state, arg); + else + argp_error(state, "more than one argument given"); + break; + case ARGP_KEY_FINI: + if (!args->filename) + argp_error(state, "no filename given"); + if (args->i_size && !args->data_version) { + argp_error(state, "must provide data-version if using --size option"); + } + if (!args->i_size && args->offline) { + argp_error(state, "must provide size if using --offline option"); + } + break; + default: + break; + } + + return 0; +} + +static struct argp_option options[] = { + { "ctime", 't', "TIMESPEC", 0, "Set creation time using \".\" format"}, + { "data-version", 'V', "VERSION", 0, "Set data version"}, + { "size", 's', "SIZE", 0, "Set file size (bytes or KMGTP units). Requires --data-version"}, + { "offline", 'o', NULL, 0, "Set file contents as offline, not sparse. Requires --size"}, + { NULL } +}; + +static int setattr_cmd(int argc, char **argv) +{ + struct argp argp = { + options, + parse_opt, + "FILE", + "Set attributes on newly-created zero-length file" + }; + struct setattr_args setattr_args = {NULL}; + int ret; + + ret = argp_parse(&argp, argc, argv, 0, NULL, &setattr_args); + if (ret) + return ret; + + return do_setattr(&setattr_args); +} + static void __attribute__((constructor)) setattr_more_ctor(void) { - cmd_register("setattr", "-c ctime -d data_version -o -s i_size -f ", - "set attributes on file with no data", - setattr_more_cmd); + cmd_register("setattr", "", + "set attributes on newly-created zero-length file", + setattr_cmd); } From 97c6cc559e3eb799955a2d66e587b12da43cd946 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Thu, 3 Dec 2020 15:49:52 -0800 Subject: [PATCH 14/21] Implement argp support for data-waiting and data-wait-err These both have a lot of required options. Signed-off-by: Andy Grover --- utils/src/waiting.c | 274 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 216 insertions(+), 58 deletions(-) diff --git a/utils/src/waiting.c b/utils/src/waiting.c index 414a3c4c..35abdd8d 100644 --- a/utils/src/waiting.c +++ b/utils/src/waiting.c @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include "sparse.h" #include "util.h" @@ -31,7 +33,16 @@ (((ops) & (bit)) ? (str) : ""), \ (((ops) & (bit)) && ((ops) & ~(((bit) << 1) - 1)) ? "," : "") -static int waiting_cmd(int argc, char **argv) + +struct waiting_args { + char *path; + bool inode_set; + u64 inode; + bool blkno_set; + u64 blkno; +}; + +static int do_waiting(struct waiting_args *args) { struct scoutfs_ioctl_data_waiting_entry dwe[16]; struct scoutfs_ioctl_data_waiting idw; @@ -39,25 +50,13 @@ static int waiting_cmd(int argc, char **argv) int fd; int i; - if (argc != 4) { - fprintf(stderr, "must specify ino, iblock, and path\n"); - return -EINVAL; - } - - ret = parse_u64(argv[1], &idw.after_ino) ?: - parse_u64(argv[2], &idw.after_iblock); - if (ret) - return ret; - - fd = open(argv[3], O_RDONLY); - if (fd < 0) { - ret = -errno; - fprintf(stderr, "failed to open '%s': %s (%d)\n", - argv[3], strerror(errno), errno); - return ret; - } + fd = get_path(args->path, O_RDONLY); + if (fd < 0) + return fd; idw.flags = 0; + idw.after_ino = args->inode; + idw.after_iblock = args->blkno; idw.ents_ptr = (unsigned long)dwe; idw.ents_nr = array_size(dwe); @@ -91,59 +90,114 @@ static int waiting_cmd(int argc, char **argv) return ret; }; -static void __attribute__((constructor)) waiting_ctor(void) +static int waiting_parse_opt(int key, char *arg, struct argp_state *state) { - cmd_register("data-waiting", " ", - "print ops waiting for data blocks", waiting_cmd); -} - -static int data_wait_err_cmd(int argc, char **argv) -{ - struct scoutfs_ioctl_data_wait_err args; - int fd = -1; + struct waiting_args *args = state->input; int ret; - memset(&args, 0, sizeof(args)); - - if (argc != 8) { - fprintf(stderr, "must specify path, ino, version, offset, count,op, and err\n"); - return -EINVAL; + switch (key) { + case 'p': + args->path = strdup_or_error(state, arg); + break; + case 'I': /* inode */ + ret = parse_u64(arg, &args->inode); + if (ret) + argp_error(state, "inode parse error"); + args->inode_set = true; + break; + case 'B': /* blkno */ + ret = parse_u64(arg, &args->blkno); + if (ret) + argp_error(state, "blkno parse error"); + args->blkno_set = true; + break; + case ARGP_KEY_FINI: + if (!args->inode_set) + argp_error(state, "no inode given"); + if (!args->blkno_set) + argp_error(state, "no blkno given"); + break; + default: + break; } - ret = parse_u64(argv[2], &args.ino) ?: - parse_u64(argv[3], &args.data_version) ?: - parse_u64(argv[4], &args.offset) ?: - parse_u64(argv[5], &args.count) ?: - parse_s64(argv[7], &args.err); + return 0; +} + +static struct argp_option waiting_options[] = { + { "path", 'p', "PATH", 0, "Path to ScoutFS filesystem"}, + { "inode", 'I', "INODE-NUM", 0, "Inode number [Required]"}, + { "block", 'B', "BLKNO-NUM", 0, "Block number [Required]"}, + { NULL } +}; + +static int waiting_cmd(int argc, char **argv) +{ + struct argp argp = { + waiting_options, + waiting_parse_opt, + "--inode INODE-NUM --block BLOCK-NUM", + "Print operations waiting for data blocks" + }; + struct waiting_args waiting_args = {NULL}; + int ret; + + ret = argp_parse(&argp, argc, argv, 0, NULL, &waiting_args); if (ret) return ret; - if ((args.err >= 0) || (args.err < -MAX_ERRNO)) { - fprintf(stderr, "err %lld invalid\n", args.err); - ret = -EINVAL; - goto out; - } + return do_waiting(&waiting_args); +} - if (!strcmp(argv[6], "read")) { - args.op = SCOUTFS_IOC_DWO_READ; - } else if (!strcmp(argv[6], "write")) { - args.op = SCOUTFS_IOC_DWO_WRITE; - } else if (!strcmp(argv[6], "change_size")) { - args.op = SCOUTFS_IOC_DWO_CHANGE_SIZE; +static void __attribute__((constructor)) waiting_ctor(void) +{ + cmd_register("data-waiting", "--inode --blockno ", + "print ops waiting for data blocks", waiting_cmd); +} + +struct wait_err_args { + char *path; + bool inode_set; + u64 inode; + bool version_set; + u64 version; + bool offset_set; + u64 offset; + bool count_set; + u64 count; + char *op; + bool err_set; + s64 err; +}; + +static int do_wait_err(struct wait_err_args *args) +{ + struct scoutfs_ioctl_data_wait_err dwe = {0}; + int fd = -1; + int ret; + + + dwe.ino = args->inode; + dwe.data_version = args->version; + dwe.offset = args->offset; + dwe.count = args->count; + if (!strcmp(args->op, "read")) { + dwe.op = SCOUTFS_IOC_DWO_READ; + } else if (!strcmp(args->op, "write")) { + dwe.op = SCOUTFS_IOC_DWO_WRITE; + } else if (!strcmp(args->op, "change_size")) { + dwe.op = SCOUTFS_IOC_DWO_CHANGE_SIZE; } else { - fprintf(stderr, "invalid data wait op: '%s'\n", argv[6]); + fprintf(stderr, "invalid data wait op: '%s'\n", args->op); return -EINVAL; } + dwe.err = args->err; - fd = open(argv[1], O_RDONLY); - if (fd < 0) { - ret = -errno; - fprintf(stderr, "failed to open '%s': %s (%d)\n", - argv[1], strerror(errno), errno); - return ret; - } + fd = get_path(args->path, O_RDONLY); + if (fd < 0) + return fd; - ret = ioctl(fd, SCOUTFS_IOC_DATA_WAIT_ERR, &args); + ret = ioctl(fd, SCOUTFS_IOC_DATA_WAIT_ERR, &dwe); if (ret < 0) { fprintf(stderr, "data_wait_err returned %d: error %s (%d)\n", ret, strerror(errno), errno); @@ -158,9 +212,113 @@ out: return ret; }; +static int wait_err_parse_opt(int key, char *arg, struct argp_state *state) +{ + struct wait_err_args *args = state->input; + int ret; + + switch (key) { + case 'p': + args->path = strdup_or_error(state, arg); + break; + case 'I': /* inode */ + ret = parse_u64(arg, &args->inode); + if (ret) + argp_error(state, "inode parse error"); + args->inode_set = true; + break; + case 'V': /* version */ + ret = parse_u64(arg, &args->version); + if (ret) + argp_error(state, "version parse error"); + args->version_set = true; + break; + case 'F': /* offset */ + ret = parse_human(arg, &args->offset); + if (ret) + argp_error(state, "version parse error"); + args->offset_set = true; + break; + case 'C': /* count */ + ret = parse_u64(arg, &args->count); + if (ret) + argp_error(state, "count parse error"); + args->count_set = true; + break; + case 'O': /* op */ + args->op = strdup_or_error(state, arg); + break; + case 'E': /* err */ + ret = parse_s64(arg, &args->err); + if (ret) + argp_error(state, "error parse error"); + if ((args->err >= 0) || (args->err < -MAX_ERRNO)) + argp_error(state, "errno out of range"); + args->err_set = true; + break; + case ARGP_KEY_FINI: + if (!args->inode_set) + argp_error(state, "no inode given"); + if (!args->version_set) + argp_error(state, "no version given"); + if (!args->offset_set) + argp_error(state, "no offset given"); + if (!args->count_set) + argp_error(state, "no count given"); + if (!args->op) + argp_error(state, "no operation given"); + if (!args->err_set) + argp_error(state, "no error given"); + break; + default: + break; + } + + return 0; +} + +static struct argp_option wait_err_options[] = { + { "path", 'p', "PATH", 0, "Path to ScoutFS filesystem"}, + { "inode", 'I', "INODE-NUM", 0, "Inode number [Required]"}, + { "version", 'V', "VER-NUM", 0, "Version [Required]"}, + { "offset", 'F', "OFF-NUM", 0, "Offset (bytes or KMGTP units) [Required]"}, + { "count", 'C', "COUNT", 0, "Count [Required]"}, + { "op", 'O', "OP", 0, "Operation: \"read\", \"write\", \"change_size\" [Required]"}, + { "err", 'E', "ERR", 0, "Error [Required]"}, + { NULL } +}; + +static struct argp wait_err_argp = { + wait_err_options, + wait_err_parse_opt, + "--inode INODE-NUM --version VER-NUM " + "--offset OFF-NUM --count COUNT --op OP --err ERR", + "Return error from matching waiters" +}; + +static int wait_err_cmd(int argc, char **argv) +{ + struct argp argp = { + wait_err_options, + wait_err_parse_opt, + "--inode INODE-NUM --block BLOCK-NUM --version VER-NUM " + "--offset OFF-NUM --count COUNT --op OP --err ERR", + "Return error from matching waiters" + }; + struct wait_err_args wait_err_args = {NULL}; + int ret; + + ret = argp_parse(&argp, argc, argv, 0, NULL, &wait_err_args); + if (ret) + return ret; + + return do_wait_err(&wait_err_args); +} + + static void __attribute__((constructor)) data_wait_err_ctor(void) { cmd_register("data-wait-err", " ", "return error from matching waiters", - data_wait_err_cmd); + wait_err_cmd); } From f2cd1003f6c84ec206790c6430329826f6f929b2 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Thu, 3 Dec 2020 16:27:26 -0800 Subject: [PATCH 15/21] Implement argp support for walk-inodes This has some fancy parsing going on, and I decided to just leave it in the main function instead of going to the effort to move it all to the parsing function. Signed-off-by: Andy Grover --- utils/src/walk_inodes.c | 98 ++++++++++++++++++++++++++++++++--------- 1 file changed, 77 insertions(+), 21 deletions(-) diff --git a/utils/src/walk_inodes.c b/utils/src/walk_inodes.c index 6cf36d5a..e528991a 100644 --- a/utils/src/walk_inodes.c +++ b/utils/src/walk_inodes.c @@ -8,8 +8,10 @@ #include #include #include +#include #include "sparse.h" +#include "parse.h" #include "util.h" #include "format.h" #include "ioctl.h" @@ -66,7 +68,14 @@ static int parse_walk_entry(struct scoutfs_ioctl_walk_inodes_entry *ent, return 0; } -static int walk_inodes_cmd(int argc, char **argv) +struct walk_inodes_args { + char *path; + char *index; + char *first_entry; + char *last_entry; +}; + +static int do_walk_inodes(struct walk_inodes_args *args) { struct scoutfs_ioctl_walk_inodes_entry ents[128]; struct scoutfs_ioctl_walk_inodes walk; @@ -75,44 +84,35 @@ static int walk_inodes_cmd(int argc, char **argv) int fd; int i; - if (argc != 5) { - fprintf(stderr, "must specify seq and path\n"); - return -EINVAL; - } - - if (!strcasecmp(argv[1], "meta_seq")) + if (!strcasecmp(args->index, "meta_seq")) walk.index = SCOUTFS_IOC_WALK_INODES_META_SEQ; - else if (!strcasecmp(argv[1], "data_seq")) + else if (!strcasecmp(args->index, "data_seq")) walk.index = SCOUTFS_IOC_WALK_INODES_DATA_SEQ; else { fprintf(stderr, "unknown index '%s', try 'meta_seq' or " - "'data_seq'\n", argv[1]); + "'data_seq'\n", args->index); return -EINVAL; } - ret = parse_walk_entry(&walk.first, argv[2]); + ret = parse_walk_entry(&walk.first, args->first_entry); if (ret) { fprintf(stderr, "invalid first position '%s', try '1.2.3' or " - "'-1'\n", argv[2]); + "'-1'\n", args->first_entry); return -EINVAL; } - ret = parse_walk_entry(&walk.last, argv[3]); + ret = parse_walk_entry(&walk.last, args->last_entry); if (ret) { fprintf(stderr, "invalid last position '%s', try '1.2.3' or " - "'-1'\n", argv[3]); + "'-1'\n", args->last_entry); return -EINVAL; } - fd = open(argv[4], O_RDONLY); - if (fd < 0) { - ret = -errno; - fprintf(stderr, "failed to open '%s': %s (%d)\n", - argv[4], strerror(errno), errno); - return ret; - } + fd = get_path(args->path, O_RDONLY); + if (fd < 0) + return fd; walk.entries_ptr = (unsigned long)ents; walk.nr_entries = array_size(ents); @@ -149,8 +149,64 @@ static int walk_inodes_cmd(int argc, char **argv) return ret; }; +static int walk_inodes_parse_opt(int key, char *arg, struct argp_state *state) +{ + struct walk_inodes_args *args = state->input; + + switch (key) { + case 'p': + args->path = strdup_or_error(state, arg); + break; + case ARGP_KEY_ARG: + if (!args->index) + args->index = strdup_or_error(state, arg); + else if (!args->first_entry) + args->first_entry = strdup_or_error(state, arg); + else if (!args->last_entry) + args->last_entry = strdup_or_error(state, arg); + else + argp_error(state, "more than three arguments given"); + break; + case ARGP_KEY_FINI: + if (!args->index) + argp_error(state, "no index given"); + if (!args->first_entry) + argp_error(state, "no first entry given"); + if (!args->last_entry) + argp_error(state, "no last entry given"); + break; + default: + break; + } + + return 0; +} + +static struct argp_option options[] = { + { "path", 'p', "PATH", 0, "Path to ScoutFS filesystem"}, + { NULL } +}; + +static int walk_inodes_cmd(int argc, char **argv) +{ + struct argp argp = { + options, + walk_inodes_parse_opt, + " FIRST-ENTRY LAST-ENTRY" + }; + struct walk_inodes_args walk_inodes_args = {NULL}; + int ret; + + ret = argp_parse(&argp, argc, argv, 0, NULL, &walk_inodes_args); + if (ret) + return ret; + + return do_walk_inodes(&walk_inodes_args); +} + + static void __attribute__((constructor)) walk_inodes_ctor(void) { - cmd_register("walk-inodes", " ", + cmd_register("walk-inodes", " ", "print range of indexed inodes", walk_inodes_cmd); } From e0a2175c2e0e18367b0378163e822a276b05d3b0 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Fri, 4 Dec 2020 14:32:59 -0800 Subject: [PATCH 16/21] Use argp info instead of duplicating for cmd_register() Make it static and then use it both for argp_parse as well as cmd_register_argp. Split commands into five groups, to help understanding of their usefulness. Mention that each command has its own help text, and that we are being fancy to keep the user from having to give fs path. Signed-off-by: Andy Grover --- utils/src/cmd.c | 64 ++++++++++++++++++++++++------------ utils/src/cmd.h | 8 ++++- utils/src/counters.c | 17 +++++----- utils/src/df.c | 16 ++++----- utils/src/ino_path.c | 16 ++++----- utils/src/listxattr_hidden.c | 17 +++++----- utils/src/main.c | 1 + utils/src/mkfs.c | 15 +++++---- utils/src/print.c | 16 ++++----- utils/src/search_xattrs.c | 18 +++++----- utils/src/setattr.c | 17 +++++----- utils/src/stage_release.c | 30 ++++++++--------- utils/src/stat.c | 37 +++++++++++---------- utils/src/util.c | 4 ++- utils/src/waiting.c | 31 +++++++---------- utils/src/walk_inodes.c | 15 +++++---- 16 files changed, 172 insertions(+), 150 deletions(-) diff --git a/utils/src/cmd.c b/utils/src/cmd.c index 607f12ec..a5e590aa 100644 --- a/utils/src/cmd.c +++ b/utils/src/cmd.c @@ -4,35 +4,38 @@ #include #include #include +#include #include "cmd.h" #include "util.h" -static struct command { +static struct argp_command { char *name; - char *opts; - char *summary; + struct argp *argp; + int group; + char __pad[4]; int (*func)(int argc, char **argv); -} cmds[100], *next_cmd = cmds; +} argp_cmds[100], *next_argp_cmd = argp_cmds; -#define cmd_for_each(com) for (com = cmds; com->func; com++) +#define cmd_for_each(com) for (com = argp_cmds; com->func; com++) -void cmd_register(char *name, char *opts, char *summary, +void cmd_register_argp(char *name, struct argp *argp, int group, int (*func)(int argc, char **argv)) { - struct command *com = next_cmd++; + struct argp_command *com = next_argp_cmd++; - assert((com - cmds) < array_size(cmds)); + assert((com - argp_cmds) < array_size(argp_cmds)); com->name = name; - com->opts = opts; - com->summary = summary; + com->argp = argp; + com->group = group; com->func = func; } -static struct command *find_command(char *name) + +static struct argp_command *find_command(char *name) { - struct command *com; + struct argp_command *com; cmd_for_each(com) { if (!strcmp(name, com->name)) @@ -42,28 +45,47 @@ static struct command *find_command(char *name) return NULL; } -static void usage(void) +static void print_cmds_for_group(int group) { - struct command *com; + struct argp_command *com; int largest = 0; - fprintf(stderr, "usage: scoutfs []\n" - "Commands:\n"); - + /* Base alignment on all groups */ cmd_for_each(com) largest = max(strlen(com->name), largest); cmd_for_each(com) { - fprintf(stderr, " %*s %s\n %*s %s\n", - largest, com->name, com->opts, - largest, "", com->summary); + if (com->group == group) { + fprintf(stderr, " %*s %s\n %*s %s\n", + largest, com->name, com->argp->args_doc, + largest, "", com->argp->doc); + } } + +} + +static void usage(void) +{ + fprintf(stderr, "usage: scoutfs []\n\n"); + fprintf(stderr, "Selected fs defaults to current working directory.\n"); + fprintf(stderr, "See --help for more details.\n"); + + fprintf(stderr, "\nCore admin:\n"); + print_cmds_for_group(GROUP_CORE); + fprintf(stderr, "\nAdditional Information:\n"); + print_cmds_for_group(GROUP_INFO); + fprintf(stderr, "\nSearch Acceleration:\n"); + print_cmds_for_group(GROUP_SEARCH); + fprintf(stderr, "\nArchival Agent Support:\n"); + print_cmds_for_group(GROUP_AGENT); + fprintf(stderr, "\nDebugging commands:\n"); + print_cmds_for_group(GROUP_DEBUG); } /* this returns a positive unix return code on error for some reason */ char cmd_execute(int argc, char **argv) { - struct command *com = NULL; + struct argp_command *com = NULL; int ret; if (argc > 1) { diff --git a/utils/src/cmd.h b/utils/src/cmd.h index 084590e9..b225e22e 100644 --- a/utils/src/cmd.h +++ b/utils/src/cmd.h @@ -1,7 +1,13 @@ #ifndef _CMD_H_ #define _CMD_H_ -void cmd_register(char *name, char *opts, char *summary, +#define GROUP_CORE 0 +#define GROUP_INFO 1 +#define GROUP_SEARCH 2 +#define GROUP_AGENT 3 +#define GROUP_DEBUG 4 + +void cmd_register_argp(char *name, struct argp *argp, int group, int (*func)(int argc, char **argv)); char cmd_execute(int argc, char **argv); diff --git a/utils/src/counters.c b/utils/src/counters.c index b57ecb54..c71fa8bd 100644 --- a/utils/src/counters.c +++ b/utils/src/counters.c @@ -303,14 +303,15 @@ static struct argp_option options[] = { { NULL } }; +static struct argp argp = { + options, + parse_opt, + "SYSFS-DIR", + "Show counters for a mounted volume" +}; + static int counters_cmd(int argc, char *argv[]) { - struct argp argp = { - options, - parse_opt, - "SYSFS-DIR", - "Show counters for a mounted volume" - }; struct counters_args counters_args = {NULL}; int ret; @@ -324,7 +325,5 @@ static int counters_cmd(int argc, char *argv[]) static void __attribute__((constructor)) counters_ctor(void) { - cmd_register("counters", "[-t] ", - "show [tabular] counters for a given mounted volume", - counters_cmd); + cmd_register_argp("counters", &argp, GROUP_INFO, counters_cmd); } diff --git a/utils/src/df.c b/utils/src/df.c index 4c8be0ef..a6ca0d12 100644 --- a/utils/src/df.c +++ b/utils/src/df.c @@ -175,14 +175,15 @@ static struct argp_option options[] = { { NULL } }; +static struct argp argp = { + options, + parse_opt, + "", + "Show metadata and data block usage" +}; + static int df_cmd(int argc, char **argv) { - struct argp argp = { - options, - parse_opt, - NULL, - "Show metadata and data block usage" - }; struct df_args df_args = {NULL}; int ret; @@ -196,6 +197,5 @@ static int df_cmd(int argc, char **argv) static void __attribute__((constructor)) df_ctor(void) { - cmd_register("df", "", - "Show metadata and data block usage", df_cmd); + cmd_register_argp("df", &argp, GROUP_CORE, df_cmd); } diff --git a/utils/src/ino_path.c b/utils/src/ino_path.c index aaa00f2d..99d05de2 100644 --- a/utils/src/ino_path.c +++ b/utils/src/ino_path.c @@ -113,14 +113,15 @@ static struct argp_option options[] = { { NULL } }; +static struct argp argp = { + options, + parse_opt, + "INODE-NUM", + "Print paths that refer to inode number" +}; + static int ino_path_cmd(int argc, char **argv) { - struct argp argp = { - options, - parse_opt, - "INODE-NUM", - "Print paths that refer to inode number" - }; struct ino_args ino_args = {NULL}; int ret; @@ -134,6 +135,5 @@ static int ino_path_cmd(int argc, char **argv) static void __attribute__((constructor)) ino_path_ctor(void) { - cmd_register("ino-path", " ", - "print paths that refer to inode #", ino_path_cmd); + cmd_register_argp("ino-path", &argp, GROUP_SEARCH, ino_path_cmd); } diff --git a/utils/src/listxattr_hidden.c b/utils/src/listxattr_hidden.c index 10130b3c..da389ab2 100644 --- a/utils/src/listxattr_hidden.c +++ b/utils/src/listxattr_hidden.c @@ -137,14 +137,15 @@ static int parse_opt(int key, char *arg, struct argp_state *state) return 0; } +static struct argp argp = { + NULL, + parse_opt, + "FILE", + "Print the names of hidden xattrs on a file" +}; + static int list_hidden_xattrs_cmd(int argc, char **argv) { - struct argp argp = { - NULL, - parse_opt, - "FILE", - "Print the names of hidden xattrs on a file" - }; struct list_hidden_xattr_args list_hidden_xattr_args = {NULL}; int ret; @@ -158,7 +159,5 @@ static int list_hidden_xattrs_cmd(int argc, char **argv) static void __attribute__((constructor)) listxattr_hidden_ctor(void) { - cmd_register("list-hidden-xattrs", "", - "print the names of hidden xattrs on a file", - list_hidden_xattrs_cmd); + cmd_register_argp("list-hidden-xattrs", &argp, GROUP_INFO, list_hidden_xattrs_cmd); } diff --git a/utils/src/main.c b/utils/src/main.c index 369b4ece..277b8ec2 100644 --- a/utils/src/main.c +++ b/utils/src/main.c @@ -4,6 +4,7 @@ #include #include #include +#include #include "cmd.h" #include "util.h" diff --git a/utils/src/mkfs.c b/utils/src/mkfs.c index d9c04dd4..2c1936fb 100644 --- a/utils/src/mkfs.c +++ b/utils/src/mkfs.c @@ -455,14 +455,15 @@ static struct argp_option options[] = { { NULL } }; +static struct argp argp = { + options, + parse_opt, + "META-DEVICE DATA-DEVICE", + "Initialize a new ScoutFS filesystem" +}; + static int mkfs_cmd(int argc, char *argv[]) { - struct argp argp = { - options, - parse_opt, - "META-DEVICE DATA-DEVICE", - "Initialize a new ScoutFS filesystem" - }; struct mkfs_args mkfs_args = {0}; int ret; @@ -475,7 +476,7 @@ static int mkfs_cmd(int argc, char *argv[]) static void __attribute__((constructor)) mkfs_ctor(void) { - cmd_register("mkfs", " ", "write a new file system", mkfs_cmd); + cmd_register_argp("mkfs", &argp, GROUP_CORE, mkfs_cmd); /* for lack of some other place to put these.. */ build_assert(sizeof(uuid_t) == SCOUTFS_UUID_BYTES); diff --git a/utils/src/print.c b/utils/src/print.c index faacb467..aff6110a 100644 --- a/utils/src/print.c +++ b/utils/src/print.c @@ -1061,14 +1061,15 @@ static int parse_opt(int key, char *arg, struct argp_state *state) return 0; } +static struct argp argp = { + NULL, + parse_opt, + "META-DEV", + "Print metadata structures" +}; + static int print_cmd(int argc, char **argv) { - struct argp argp = { - NULL, - parse_opt, - "META-DEV", - "Print metadata structures" - }; struct print_args print_args = {NULL}; int ret; @@ -1082,6 +1083,5 @@ static int print_cmd(int argc, char **argv) static void __attribute__((constructor)) print_ctor(void) { - cmd_register("print", "", "print metadata structures", - print_cmd); + cmd_register_argp("print", &argp, GROUP_DEBUG, print_cmd); } diff --git a/utils/src/search_xattrs.c b/utils/src/search_xattrs.c index a079a13f..bb242f06 100644 --- a/utils/src/search_xattrs.c +++ b/utils/src/search_xattrs.c @@ -112,14 +112,16 @@ static struct argp_option options[] = { { NULL } }; +static struct argp argp = { + options, + parse_opt, + "XATTR-NAME", + "Print inode numbers of inodes which may have given xattr" +}; + static int search_xattrs_cmd(int argc, char **argv) { - struct argp argp = { - options, - parse_opt, - "XATTR-NAME", - "Print inode numbers of inodes which may have given xattr" - }; + struct xattr_args xattr_args = {NULL}; int ret; @@ -132,7 +134,5 @@ static int search_xattrs_cmd(int argc, char **argv) static void __attribute__((constructor)) search_xattrs_ctor(void) { - cmd_register("search-xattrs", "-n name -f ", - "print inode numbers of inodes which may have given xattr", - search_xattrs_cmd); + cmd_register_argp("search-xattrs", &argp, GROUP_INFO, search_xattrs_cmd); } diff --git a/utils/src/setattr.c b/utils/src/setattr.c index 23922531..e4178f6c 100644 --- a/utils/src/setattr.c +++ b/utils/src/setattr.c @@ -120,14 +120,15 @@ static struct argp_option options[] = { { NULL } }; +static struct argp argp = { + options, + parse_opt, + "FILE", + "Set attributes on newly-created zero-length file" +}; + static int setattr_cmd(int argc, char **argv) { - struct argp argp = { - options, - parse_opt, - "FILE", - "Set attributes on newly-created zero-length file" - }; struct setattr_args setattr_args = {NULL}; int ret; @@ -140,7 +141,5 @@ static int setattr_cmd(int argc, char **argv) static void __attribute__((constructor)) setattr_more_ctor(void) { - cmd_register("setattr", "", - "set attributes on newly-created zero-length file", - setattr_cmd); + cmd_register_argp("setattr", &argp, GROUP_AGENT, setattr_cmd); } diff --git a/utils/src/stage_release.c b/utils/src/stage_release.c index 8683afd1..38b62196 100644 --- a/utils/src/stage_release.c +++ b/utils/src/stage_release.c @@ -163,18 +163,19 @@ static struct argp_option options[] = { { NULL } }; -static int stage_cmd(int argc, char **argv) -{ - struct argp argp = { +static struct argp stage_argp = { options, parse_stage_opts, "ARCHIVE-FILE STAGE-FILE --data-version VERSION", "Write archive file contents to an offline file" }; + +static int stage_cmd(int argc, char **argv) +{ struct stage_args stage_args = {NULL}; int ret; - ret = argp_parse(&argp, argc, argv, 0, NULL, &stage_args); + ret = argp_parse(&stage_argp, argc, argv, 0, NULL, &stage_args); if (ret) return ret; @@ -183,8 +184,7 @@ static int stage_cmd(int argc, char **argv) static void __attribute__((constructor)) stage_ctor(void) { - cmd_register("stage", " -V ", - "write archive file contents to an offline file", stage_cmd); + cmd_register_argp("stage", &stage_argp, GROUP_AGENT, stage_cmd); } struct release_args { @@ -277,18 +277,19 @@ static int parse_release_opts(int key, char *arg, struct argp_state *state) return 0; } +static struct argp release_argp = { + options, + parse_release_opts, + "FILE --data-version VERSION", + "Mark file region offline and free extents" +}; + static int release_cmd(int argc, char **argv) { - struct argp argp = { - options, - parse_release_opts, - "FILE --data-version VERSION", - "Mark file region offline and free extents" - }; struct release_args release_args = {NULL}; int ret; - ret = argp_parse(&argp, argc, argv, 0, NULL, &release_args); + ret = argp_parse(&release_argp, argc, argv, 0, NULL, &release_args); if (ret) return ret; @@ -297,6 +298,5 @@ static int release_cmd(int argc, char **argv) static void __attribute__((constructor)) release_ctor(void) { - cmd_register("release", " ", - "mark file region offline and free extents", release_cmd); + cmd_register_argp("release", &release_argp, GROUP_AGENT, release_cmd); } diff --git a/utils/src/stat.c b/utils/src/stat.c index 3e20d1d0..996d4106 100644 --- a/utils/src/stat.c +++ b/utils/src/stat.c @@ -202,18 +202,19 @@ static struct argp_option stat_options[] = { { NULL } }; +static struct argp stat_argp = { + stat_options, + stat_parse_opt, + "FILE", + "Show ScoutFS extra inode information" +}; + static int stat_more_cmd(int argc, char **argv) { - struct argp argp = { - stat_options, - stat_parse_opt, - "FILE", - "Show ScoutFS extra inode information" - }; struct stat_args stat_args = {NULL}; int ret; - ret = argp_parse(&argp, argc, argv, 0, NULL, &stat_args); + ret = argp_parse(&stat_argp, argc, argv, 0, NULL, &stat_args); if (ret) return ret; stat_args.is_inode = true; @@ -244,18 +245,20 @@ static int statfs_parse_opt(int key, char *arg, struct argp_state *state) return 0; } + +static struct argp statfs_argp = { + statfs_options, + statfs_parse_opt, + "", + "Show ScoutFS file system information" +}; + static int statfs_more_cmd(int argc, char **argv) { - struct argp argp = { - statfs_options, - statfs_parse_opt, - NULL, - "Show ScoutFS file system information" - }; struct stat_args stat_args = {NULL}; int ret; - ret = argp_parse(&argp, argc, argv, 0, NULL, &stat_args); + ret = argp_parse(&statfs_argp, argc, argv, 0, NULL, &stat_args); if (ret) return ret; stat_args.is_inode = false; @@ -265,12 +268,10 @@ static int statfs_more_cmd(int argc, char **argv) static void __attribute__((constructor)) stat_more_ctor(void) { - cmd_register("stat", "", - "show scoutfs inode information", stat_more_cmd); + cmd_register_argp("stat", &stat_argp, GROUP_INFO, stat_more_cmd); } static void __attribute__((constructor)) statfs_more_ctor(void) { - cmd_register("statfs", "", - "show scoutfs file system information", statfs_more_cmd); + cmd_register_argp("statfs", &statfs_argp, GROUP_INFO, statfs_more_cmd); } diff --git a/utils/src/util.c b/utils/src/util.c index d30e44ba..9c3aa6ef 100644 --- a/utils/src/util.c +++ b/utils/src/util.c @@ -11,6 +11,8 @@ #include "util.h" +#define ENV_PATH "SCOUTFS_MOUNT_PATH" + static int open_path(char *path, int flags) { wordexp_t exp_result; @@ -51,7 +53,7 @@ int get_path(char *path, int flags) if (path) return open_path(path, flags); - env_path = getenv("SCOUTFS_PATH"); + env_path = getenv(ENV_PATH); if (env_path) return open_path(path, flags); diff --git a/utils/src/waiting.c b/utils/src/waiting.c index 35abdd8d..92b31f6b 100644 --- a/utils/src/waiting.c +++ b/utils/src/waiting.c @@ -131,18 +131,19 @@ static struct argp_option waiting_options[] = { { NULL } }; +static struct argp waiting_argp = { + waiting_options, + waiting_parse_opt, + "--inode INODE-NUM --block BLOCK-NUM", + "Print operations waiting for data blocks" +}; + static int waiting_cmd(int argc, char **argv) { - struct argp argp = { - waiting_options, - waiting_parse_opt, - "--inode INODE-NUM --block BLOCK-NUM", - "Print operations waiting for data blocks" - }; struct waiting_args waiting_args = {NULL}; int ret; - ret = argp_parse(&argp, argc, argv, 0, NULL, &waiting_args); + ret = argp_parse(&waiting_argp, argc, argv, 0, NULL, &waiting_args); if (ret) return ret; @@ -151,8 +152,7 @@ static int waiting_cmd(int argc, char **argv) static void __attribute__((constructor)) waiting_ctor(void) { - cmd_register("data-waiting", "--inode --blockno ", - "print ops waiting for data blocks", waiting_cmd); + cmd_register_argp("data-waiting", &waiting_argp, GROUP_AGENT, waiting_cmd); } struct wait_err_args { @@ -298,17 +298,10 @@ static struct argp wait_err_argp = { static int wait_err_cmd(int argc, char **argv) { - struct argp argp = { - wait_err_options, - wait_err_parse_opt, - "--inode INODE-NUM --block BLOCK-NUM --version VER-NUM " - "--offset OFF-NUM --count COUNT --op OP --err ERR", - "Return error from matching waiters" - }; struct wait_err_args wait_err_args = {NULL}; int ret; - ret = argp_parse(&argp, argc, argv, 0, NULL, &wait_err_args); + ret = argp_parse(&wait_err_argp, argc, argv, 0, NULL, &wait_err_args); if (ret) return ret; @@ -318,7 +311,5 @@ static int wait_err_cmd(int argc, char **argv) static void __attribute__((constructor)) data_wait_err_ctor(void) { - cmd_register("data-wait-err", " ", - "return error from matching waiters", - wait_err_cmd); + cmd_register_argp("data-wait-err", &wait_err_argp, GROUP_AGENT, wait_err_cmd); } diff --git a/utils/src/walk_inodes.c b/utils/src/walk_inodes.c index e528991a..7dcc4377 100644 --- a/utils/src/walk_inodes.c +++ b/utils/src/walk_inodes.c @@ -187,13 +187,15 @@ static struct argp_option options[] = { { NULL } }; +static struct argp argp = { + options, + walk_inodes_parse_opt, + " FIRST-ENTRY LAST-ENTRY", + "Print range of indexed inodes" +}; + static int walk_inodes_cmd(int argc, char **argv) { - struct argp argp = { - options, - walk_inodes_parse_opt, - " FIRST-ENTRY LAST-ENTRY" - }; struct walk_inodes_args walk_inodes_args = {NULL}; int ret; @@ -207,6 +209,5 @@ static int walk_inodes_cmd(int argc, char **argv) static void __attribute__((constructor)) walk_inodes_ctor(void) { - cmd_register("walk-inodes", " ", - "print range of indexed inodes", walk_inodes_cmd); + cmd_register_argp("walk-inodes", &argp, GROUP_SEARCH, walk_inodes_cmd); } From 5241bba7f675352832cf5e61656deeca0ecd442f Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Wed, 9 Dec 2020 11:24:50 -0800 Subject: [PATCH 17/21] Update scoutfs.8 man page Update for cli args and options changes. Reorder subcommands to match scoutfs built-in help. Consistent ScoutFS capitalization. Tighten up some descriptions and verbiage for consistency and omit descriptions of internals in a few spots. Add SEE ALSO for blockdev(8) and wipefs(8). Signed-off-by: Andy Grover --- utils/man/scoutfs.8 | 787 +++++++++++++++++++++++--------------------- 1 file changed, 409 insertions(+), 378 deletions(-) diff --git a/utils/man/scoutfs.8 b/utils/man/scoutfs.8 index 1e8721a2..a43d4ff7 100644 --- a/utils/man/scoutfs.8 +++ b/utils/man/scoutfs.8 @@ -3,51 +3,302 @@ scoutfs \- scoutfs management utility .SH DESCRIPTION The -.b -scoutfs -utility provides commands to manage a scoutfs filesystem. +.B scoutfs +utility provides commands to create and manage a ScoutFS filesystem. .SH COMMANDS + +Note: Commands taking the +.B --path +option will, when the option is omitted, fall back to using the value of the +.I SCOUTFS_MOUNT_PATH +environment variable. If that variable is also absent the current working +directory will be used. + .TP -.BI "counters [\-t\] " +.BI "df [-h|--human-readable] [-p|--path PATH]" .sp -Displays the counters and their values for a mounted scoutfs filesystem. -Each counter and its value are printed on a line to stdout with -sufficient spaces seperating the name and value to align the values -after +Display available and used space on the ScoutFS data and metadata devices. .RS 1.0i .PD 0 .TP .sp -.B "\-t" -Format the counters into a table that fills the display instead of -printing one counter per line. The names and values are padded to -create columns that fill the current width of the terminal. +.B "-h, --human-readable" +Output sizes in human-readable size units (e.g. 500G, 1.2P) rather than number +of ScoutFS allocation blocks. .TP -.B "sysfs topdir" -Specify the mount's sysfs directory in which to find the -.B counters/ -directory when then contains files for each counter. -The sysfs directory is typically -of the form -.I /sys/fs/scoutfs/f..r./ -\&. +.B "-p, --path PATH" +A path within a ScoutFS filesystem. .RE .PD .TP -.BI "data-waiting " +.BI "mkfs META-DEVICE DATA-DEVICE {-Q|--quorum-count} NUM [-m|--max-meta-size SIZE] [-d|--max-data-size SIZE] [-f|--force]" .sp -Displays all the files and blocks for which there is a task blocked waiting on +Initialize a new ScoutFS filesystem on the target devices. Since ScoutFS uses +separate block devices for its metadata and data storage, two are required. +.sp +If +.B --force +option is not given, mkfs will check for existing filesystem signatures. It is +recommended to use +.B wipefs(8) +to remove non-ScoutFS filesystem signatures before proceeding, and +.B --force +to overwrite a previous ScoutFS filesystem. +.RS 1.0i +.PD 0 +.TP +.sp +.B META-DEVICE +The path to the block device to be used for ScoutFS metadata. If possible, use +a faster block device for the metadata device. +.TP +.B DATA-DEVICE +The path to the block device to be used for ScoutFS file data. If possible, use +a larger block device for the data device. +.TP +.B "-Q, --quorum-count NUM" +The number of mounts needed to reach quorum and elect one +to be the server. Mounts of the filesystem will hang until a quorum of +mounts are operational. +.sp +Mounts with the +.B server_addr +mount option participate in quorum. The safest quorum number is the +smallest majority of an odd number of participating mounts. For +example, +two out of three total mounts. This ensures that there can only be one +set of mounts that can establish quorum. +.TP +.B "-m, --max-meta-size SIZE" +Limit the space used by ScoutFS on the metadata device to the +given size, rather than using the entire block device. Size is given as +an integer followed by a units digit: "K", "M", "G", "T", "P", to denote +kibibytes, mebibytes, etc. +.TP +.B "-d, --max-data-size SIZE" +Same as previous, but for limiting the size of the data device. +.TP +.B "-f, --force" +Ignore presence of existing data on the data and metadata devices. +.RE +.PD + +.TP +.BI "stat FILE [-s|--single-field FIELD-NAME]" +.sp +Display ScoutFS-specific metadata fields for the given file. +.RS 1.0i +.PD 0 +.TP +.sp +.B "FILE" +Path to the file. +.TP +.B "-s, --single-field FIELD-NAME" +Only output a single field's value instead of the default: all the stats with +one stat per line. +.sp +.TP +.RE +.PD +The fields are: +.RS 1.0i +.PD 0 +.TP +.B "meta_seq" +The metadata change sequence. This changes each time the inode's metadata +is changed. +.TP +.B "data_seq" +The data change sequence. This changes each time the inode's data +is changed. +.TP +.B "data_version" +The data version changes every time the contents of the file changes, +or the file grows or shrinks. +.TP +.B "online_blocks" +The number of 4Kb data blocks that contain data and can be read. +.TP +.B "offline_blocks" +The number of 4Kb data blocks that are offline and would need to be +staged to be read. +.RE +.PD + +.TP +.BI "statfs [-s|--single-field FIELD-NAME] [-p|--path PATH]" +.sp +Display ScoutFS-specific filesystem-wide metadata fields. +.RS 1.0i +.PD 0 +.TP +.sp +.B "-s, --single-field FIELD-NAME" +Only ontput a single stat instead of all the stats with one stat per +line. The possible stat names are those given in the output. +.TP +.B "-p, --path PATH" +A path within a ScoutFS filesystem. +.sp +.TP +.RE +.PD +The fields are: +.RS 1.0i +.PD 0 +.TP +.B "fsid" +The unique 64bit filesystem identifier for this filesystem. +.TP +.B "rid" +The unique 64bit random identifier for this mount of the filesystem. +This is generated for every new mount of the file system. +.TP +.B "committed_seq" +All seqs up to and including this seq have been +committed. Can be compared with meta_seq and data_seq from inodes in +.B stat +to discover if changes to a file have been committed to disk. +.TP +.B "total_meta_blocks" +The total number of 64K metadata blocks in the filesystem. +.TP +.B "total_data_blocks" +The total number of 4K data blocks in the filesystem. +.RE +.PD + +.TP +.BI "counters [-t|--table] SYSFS-DIR" +.sp +Display the counters and their values for a mounted ScoutFS filesystem. +.RS 1.0i +.PD 0 +.sp +.TP +.B SYSFS-DIR +The mount's sysfs directory in which to find the +.B counters/ +directory when then contains files for each counter. +The sysfs directory is +of the form +.I /sys/fs/scoutfs/f..r./ +\&. +.TP +.B "-t, --table" +Format the counters into a columnar table that fills the width of the display +instead of printing one counter per line. +.RE +.PD + +.TP +.BI "search-xattrs XATTR-NAME [-p|--path PATH]" +.sp +Display the inode numbers of inodes in the filesystem which may have +an extended attribute with the given name. +.sp +The results may contain false positives. The returned inode numbers +should be checked to verify that the extended attribute is in fact +present on the inode. +.RS 1.0i +.PD 0 +.TP +.sp +.B XATTR-NAME +The full name of the extended attribute to search for as +described in the +.BR xattr (7) +manual page. +.TP +.B "-p|--path PATH" +A path within a ScoutFS filesystem. +.RE +.PD + +.TP +.BI "list-hidden-xattrs FILE" +.sp +Display extended attributes starting with the +.BR scoutfs. +prefix and containing the +.BR hide. +tag +which makes them invisible to +.BR listxattr (2) . +The names of each attribute are output, one per line. Their order +is not specified. +.RS 1.0i +.PD 0 +.TP +.sp +.B "FILE" +The path to a file within a ScoutFS filesystem. File permissions must allow +reading. +.RE +.PD + +.TP +.BI "walk-inodes {meta_seq|data_seq} FIRST-INODE LAST-INODE [-p|--path PATH]" +.sp +Walk an inode index in the file system and output the inode numbers +that are found between the first and last positions in the index. +.RS 1.0i +.PD 0 +.sp +.TP +.BR meta_seq , data_seq +Which index to walk. +.TP +.B "FIRST-INODE" +An integer index value giving starting position of the index walk. +.I 0 +is the first possible position. +.TP +.B "LAST-INODE" +An integer index value giving the last position to include in the index walk. +.I \-1 +can be given to indicate the last possible position. +.TP +.B "-p|--path PATH" +A path within a ScoutFS filesystem. +.RE +.PD + +.TP +.BI "ino-path INODE-NUM [-p|--path PATH]" +.sp +Display all paths that reference an inode number. +.sp +Ongoing filesystem changes, such as renaming a common parent of multiple paths, +can cause displayed paths to be inconsistent. +.RS 1.0i +.PD 0 +.sp +.TP +.B "INODE-NUM" +The inode number of the target inode. +.TP +.B "-p|--path PATH" +A path within a ScoutFS filesystem. +.RE +.PD + +.TP +.BI "data-waiting {-I|--inode} INODE-NUM {-B|--block} BLOCK-NUM [-p|--path PATH]" +.sp +Display all the files and blocks for which there is a task blocked waiting on offline data. .sp The results are sorted by the file's inode number and the logical block offset that is being waited on. .sp -Each line of output specifies a block in a file that has a task waiting +Each line of output describes a block in a file that has a task waiting and is formatted as: .I "ino iblock ops [str]" \&. The ops string indicates blocked operations seperated by commas and can -include +include .B read for a read operation, .B write @@ -58,156 +309,151 @@ for a truncate or extending write. .PD 0 .sp .TP -.B "ino" +.B "-I, --inode INODE-NUM" Start iterating over waiting tasks from the given inode number. -Specifying 0 will show all waiting tasks. +Value of 0 will show all waiting tasks. .TP -.B "iblock" +.B "-B, --block BLOCK-NUM" Start iterating over waiting tasks from the given logical block number -in the starting inode. Specifying 0 will show blocks in the first inode +in the starting inode. Value of 0 will show blocks in the first inode and then continue to show all blocks with tasks waiting in all the remaining inodes. .TP +.B "-p, --path PATH" +A path within a ScoutFS filesystem. +.RE +.PD + +.TP +.BI "data-wait-err {-I|--inode} INODE-NUM {-V|--version} VER-NUM {-F|--offset} OFF-NUM {-C|--count} COUNT {-O|--op} OP {-E|--err} ERR [-p|--path PATH]" +.sp +Return error from matching waiters. +.RS 1.0i +.PD 0 +.sp +.TP +.B "-C, --count COUNT" +Count. +.TP +.B "-E, --err ERR" +Error. +.TP +.B "-F, --offset OFF-NUM" +Offset. May be expressed in bytes, or with KMGTP (Kibi, Mibi, etc.) size +suffixes. +.TP +.B "-I, --inode INODE-NUM" +Inode number. +.TP +.B "-O, --op OP" +Operation. One of: "read", "write", "change_size". +.TP +.B "-p, --path PATH" +A path within a ScoutFS filesystem. +.RE +.PD + +.TP +.BI "stage ARCHIVE-FILE FILE {-V|--version} VERSION [-o, --offset OFF-NUM] [-l, --length LENGTH]" +.sp +.B Stage +(i.e. return to online) the previously-offline contents of a file by copying a +region from another file, the archive, and without updating regular inode +metadata. Any operations that are blocked by the existence of an offline +region will proceed once the region has been staged. +.RS 1.0i +.PD 0 +.TP +.sp +.B "ARCHIVE-FILE" +The source file for the file contents being staged. +.TP +.B "FILE" +The regular file whose contents will be staged. +.TP +.B "-V, --version VERSION" +The data_version of the contents to be staged. It must match the +current data_version of the file. +.TP +.B "-o, --offset OFF-NUM" +The starting byte offset of the region to write. May be expressed in bytes, or with +KMGTP (Kibi, Mibi, etc.) size suffixes. Default is 0. +.TP +.B "-l, --length LENGTH" +Length of range (bytes or KMGTP units) of file to stage. Default is the file's +total size. +.RE +.PD + +.TP +.BI "release FILE {-V|--version} VERSION [-o, --offset OFF-NUM] [-l, --length LENGTH]" +.sp +.B Release +the given region of the file. That is, remove the region's backing data and +leave an offline data region. Future attempts to read or write the offline +region will block until the region is restored by a +.B stage +write. This is used by userspace archive managers to free data space in the +ScoutFS filesystem once the file data has been archived. +.sp +Note: This only works on regular files with write permission. Releasing regions +that are already offline or sparse, including regions extending past the end of +the file, will silently succeed. +.RS 1.0i +.PD 0 +.TP +.sp .B "path" -A path to any inode in the target filesystem, typically the root -directory. +The path to the regular file whose region will be released. +.TP +.B "-V, --version VERSION" +The data_version of the contents to be released. It must match the current +data_version of the file. This ensures that a release operation is truncating +the same version of the data that was archived. (Use the +.BI "stat" +subcommand to obtain data version for a file.) +.TP +.B "-o, --offset OFF-NUM" +The starting byte offset of the region to write. May be expressed in bytes, or with +KMGTP (Kibi, Mibi, etc.) size suffixes. Default is 0. +.TP +.B "-l, --length LENGTH" +Length of range (bytes or KMGTP units) of file to stage. Default is the file's +total size. .RE .PD .TP -.BI "find-xattrs <\-n\ name> <\-f path>" +.BI "setattr FILE [-d, --data-version=VERSION [-s, --size=SIZE [-o, --offline]]] [-t, --ctime=TIMESPEC]" .sp -Displays the inode numbers of inodes in the filesystem which may have -an extended attribute with the given name. -.sp -The results may contain false positives. The returned inode numbers -should be checked to verify that the extended attribute is in fact -present on the inode. -.RS 1.0i -.PD 0 -.TP -.sp -.B "-n name" -Specifies the full name of the extended attribute to search for as -described in the -.BR xattr (7) -manual page. -.TP -.B "-f path" -Specifies the path to any inode in the filesystem to search. -.RE -.PD - -.TP -.BI "ino-path " -.sp -Displays all the paths to links to the given inode number. -.sp -All the relative paths from the root directory to each link of the -target inode are output, one result per line. Each output path is -guaranteed to have been a valid path to a link at some point in the -past. An individual path won't be corrupted by a rename that occurs -during the search. The set of paths can be modified while the search is -running. A rename of a parent directory of all the paths, for example, -can result in output where the parent directory name component changes -in the middle of outputting all the paths. +Set ScoutFS-specific attributes on a newly created zero-length file. .RS 1.0i .PD 0 .sp .TP -.B "ino" -The inode number of the target inode to resolve. +.B "-V, --data-version=VERSION" +Set data version. .TP -.B "path" -A path to any inode in the target filesystem, typically the root -directory. +.B "-o, --offline" +Set file contents as offline, not sparse. Requires +.I --size +option also be present. +.TP +.B "-s, --size=SIZE" +Set file size. May be expressed in bytes, or with +KMGTP (Kibi, Mibi, etc.) size suffixes. Requires +.I --data-version +option also be present. +.TP +.B "-t, --ctime=TIMESPEC" +Set creation time using +.I "." +format. .RE .PD .TP -.BI "listxattr-hidden <\-f path>" -.sp -Displays all the extended attributes starting with the -.BR scoutfs. -prefix and which contain the -.BR hide. -tag -which makes them invisible to -.BR listxattr (2) -\&. -The names of each attribute are output, one name per line. Their order -is determined by internal indexing implementation details and should not -be relied on. -.RS 1.0i -.PD 0 -.TP -.sp -.B "-f path" -The path to the file whose extended attributes will be listed. The -user must have read permission to the inode. -.RE -.PD - -.TP -.BI "mkfs <\-Q nr> [-M meta_size] [-D data_size]" -.sp -Initialize a new empty filesystem in the target devices by writing empty -structures and a new superblock. Since ScoutFS uses separate block -devices for its metadata and data storage, both must be given. -.sp -This -.B unconditionally destroys -the contents of the devices, regardless of what they contain or who may be -using them. It simply writes new data structures into known offsets. -.B Be very careful that the devices do not contain data and are not actively in use. -.RS 1.0i -.PD 0 -.TP -.sp -.B "-Q nr" -Specify the number of mounts needed to reach quorum and elect a mount -to start the server. Mounts of the filesystem will hang until this many -mounts are operational and can elect a server amongst themselves. -.sp -Mounts with the -.B server_addr -mount option participate in quorum. The safest quorum number is the -smallest majority of an odd number of participating mounts. For -example, -two out of three total mounts. This ensures that there can only be one -set of mounts that can establish quorum. -.sp -Degenerate quorums are possible, for example by specifying half of an -even number of mounts or less than half of the mount count, down to even -just one mount establishing quorum. These minority quorums carry the -risk of multiple quorums being established concurrently. Each quorum's -elected servers race to fence each other and can have the unlikely -outcome of continually racing to fence each other resulting in a -persistent loss of service. -.TP -.B "meta_dev_path" -The path to the device to be used for ScoutFS metadata. If possible, -use a faster block device for the metadata device. Its contents will be -unconditionally destroyed. -.TP -.B "data_dev_path" -The path to the device to be used for ScoutFS file data. If possible, -use a larger block device for the data device. Its contents will be -unconditionally destroyed. -.TP -.B "-M meta_size" -Limit the space used by the filesystem on the metadata device to the -given size, rather than using the entire block device. Size is given as -an integer followed by a units digit: "K", "M", "G", "T", "P", to denote -kibibytes, mebibytes, etc. -.TP -.B "-D data_size" -Same as previous, but for limiting the size of the data device. -.RE -.PD - -.TP -.BI "print " +.BI "print META-DEVICE" .sp Prints out all of the metadata in the file system. This makes no effort to ensure that the structures are consistent as they're traversed and @@ -217,236 +463,21 @@ output. .PD 0 .TP .sp -.B "path" -The path to the metadata device for filesystem whose metadata will -be printed. The command reads from the buffer cache of the device which -may not reflect the current blocks in the filesystem that may have been -written through another host or device. The local device's cache can be -manually flushed before printing, perhaps with the -.B \--flushbufs -command in the -.BR blockdev (8) -command. -.RE -.PD - -.TP -.BI "release <4KB block offset> <4KB block count>" -.sp -.B Release -the given logical block region of the file. That is, truncate away -any data blocks but leave behind offline data regions and do not change -the main inode metadata. Future attempts to read or write the block -region -will block until the region is restored by a -.B stage -write. This is used by userspace archive managers to store file data -in a remote archive tier. -.sp -This only works on regular files and with write permission. Releasing -regions that are already offline or are sparse, including past the end -of the file, silently succeed. -.RS 1.0i -.PD 0 -.TP -.sp -.B "path" -The path to the regular file whose region will be released. -.TP -.B "version" -The current data version of the contents of the file. This ensures -that a release operation is truncating the version of the data that it -expects. It can't throw away data that was newly written while it was -performing its release operation. An inode's data_version is read -by the SCOUTFS_IOC_STATFS_MORE -ioctl. -.TP -.B "4KB block offset" -The 64bit logical block offset of the start of the region in units of 4KB. -.TP -.B "4KB block count" -The 64bit length of the region to release in units of 4KB blocks. -.RE -.PD - -.TP -.BI "setattr <\-c ctime> <\-d data_version> -o <\-s i_size> <\-f path> -.sp -Set scoutfs specific metadata on a newly created inode without updating -other inode metadata. -.RS 1.0i -.PD 0 -.TP -.sp -.B "-c ctime" -Specify the inode's creation GMT timespec with 64bit seconds and 32bit -nanoseconds formatted as -.B sec.nsec -\&. -.TP -.B "-d data_version" -Specify the inode's data version. This can only be set on regular files whose -current data_version is 0. -.TP -.B "-o" -Create an offline region for all of the file's data up to the specified -file size. This can only be set on regular files whose data_version is -0 and i_size must also be specified. -.TP -.B "-s i_size" -Set the inode's i_size. This can only be set on regular files whose -data_version is 0. -.TP -.B "-f path" -The file whose metadata will be set. -.RE -.PD - -.TP -.BI "stage " -.sp -.B Stage -the contents of the file by reading a region of another archive file and writing it -into the file region without updating regular inode metadata. Any tasks -that are blocked by the offline region will proceed once it has been -staged. -.RS 1.0i -.PD 0 -.TP -.sp -.B "file" -The regular file whose contents will be staged. -.TP -.B "vers" -The data_version of the contents to be staged. It must match the -current data_version of the file. -.TP -.B "offset" -The starting byte offset of the region to write. This must be aligned -to 4KB blocks. -.TP -.B "count" -The length of the region to write in bytes. A length of 0 is a noop -and will immediately return success. The length must be a multiple -of 4KB blocks unless it is writing the final partial block in which -case it must end at i_size. -.TP -.B "archive file" -A file whose contents will be read and written as the staged region. -The start of the archive file will be used as the start of the region. -.RE -.PD - -.TP -.BI "stat [-s single] " -.sp -Display scoutfs metadata fields for the given inode. -.RS 1.0i -.PD 0 -.TP -.sp -.B "-s single" -Only ontput a single stat instead of all the stats with one stat per -line. The possible stat names are those given in the output. -.TP -.B "path" -The path to the file whose inode field will be output. -.sp -.TP -.RE -.PD -The fields are as follows: -.RS 1.0i -.PD 0 -.TP -.B "meta_seq" -The metadata change sequence. This changes each time the inode's metadata -is changed during a mount's transaction. -.TP -.B "data_seq" -The data change sequence. This changes each time the inode's data -is changed during a mount's transaction. -.TP -.B "data_version" -The data version changes every time any contents of the file changes, -including size changes. It can change many times during a syscall in a -transactions. -.TP -.B "online_blocks" -The number of 4Kb data blocks that contain data and can be read. -.TP -.B "online_blocks" -The number of 4Kb data blocks that are offline and would need to be -staged to be read. -.RE -.PD - -.TP -.BI "statfs [-s single] " -.sp -Display scoutfs metadata fields for a scoutfs filesystem. -.RS 1.0i -.PD 0 -.TP -.sp -.B "-s single" -Only ontput a single stat instead of all the stats with one stat per -line. The possible stat names are those given in the output. -.TP -.B "path" -The path to any inode in the filesystem. -.sp -.TP -.RE -.PD -The fields are as follows: -.RS 1.0i -.PD 0 -.TP -.B "fsid" -The unique 64bit filesystem identifier for this filesystem. -.TP -.B "rid" -The unique 64bit random identifier for this mount of the filesystem. -This is generated for every new mount of the file system. -.RE -.PD - -.TP -.BI "walk-inodes " -.sp -Walks an inode index in the file system and outputs the inode numbers -that are found within the first and last positions in the index. -.RS 1.0i -.PD 0 -.sp -.TP -.B "index" -Specifies the index to walk. The currently supported indices are -.B meta_seq -and -.B data_seq -\&. -.TP -.B "first" -The starting position of the index walk. -.I 0 -is the first possible position in every index. -.TP -.B "last" -The last position to include in the index walk. -.I \-1 -can be given as shorthand for the U64_MAX last possible position in -every index. -.TP -.B "path" -A path to any inode in the filesystem, typically the root directory. +.B "META-DEVICE" +The path to the metadata device for the filesystem whose metadata will be +printed. Since this command reads via the host's buffer cache, it may not +reflect the current blocks in the filesystem possibly written to the shared +block devices from another host, unless +.B blockdev \--flushbufs +command is used first. .RE .PD .SH SEE ALSO .BR scoutfs (5), -.BR xattr (7). +.BR xattr (7), +.BR blockdev (8), +.BR wipefs (8) .SH AUTHORS Zach Brown From d48b447e75b0d25738cb322566c8250e38216621 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Thu, 10 Dec 2020 13:57:16 -0800 Subject: [PATCH 18/21] Do not set -Wpadded except for checking kmod-shared headers Remove now-unneeded manual padding in arg structs. Signed-off-by: Andy Grover --- utils/Makefile | 1 - utils/src/cmd.c | 1 - utils/src/counters.c | 1 - utils/src/df.c | 1 - utils/src/main.c | 9 +++++++++ utils/src/mkfs.c | 1 - utils/src/print.c | 1 - utils/src/setattr.c | 1 - utils/src/stat.c | 1 - 9 files changed, 9 insertions(+), 8 deletions(-) diff --git a/utils/Makefile b/utils/Makefile index 8116bcd8..10eb66aa 100644 --- a/utils/Makefile +++ b/utils/Makefile @@ -19,7 +19,6 @@ endif SCOUTFS_FORMAT_HASH := $(shell cat $(HASH_FILES) | md5sum | cut -b1-16) CFLAGS := -Wall -O2 -Werror -D_FILE_OFFSET_BITS=64 -g -msse4.2 \ - -Wpadded \ -fno-strict-aliasing \ -DSCOUTFS_FORMAT_HASH=0x$(SCOUTFS_FORMAT_HASH)LLU diff --git a/utils/src/cmd.c b/utils/src/cmd.c index a5e590aa..ddd49710 100644 --- a/utils/src/cmd.c +++ b/utils/src/cmd.c @@ -13,7 +13,6 @@ static struct argp_command { char *name; struct argp *argp; int group; - char __pad[4]; int (*func)(int argc, char **argv); } argp_cmds[100], *next_argp_cmd = argp_cmds; diff --git a/utils/src/counters.c b/utils/src/counters.c index c71fa8bd..38105b34 100644 --- a/utils/src/counters.c +++ b/utils/src/counters.c @@ -43,7 +43,6 @@ static int cmp_counter_names(const void *A, const void *B) struct counters_args { char *sysfs_path; bool tabular; - char __pad[7]; }; static int do_counters(struct counters_args *args) diff --git a/utils/src/df.c b/utils/src/df.c index a6ca0d12..21ea9f04 100644 --- a/utils/src/df.c +++ b/utils/src/df.c @@ -26,7 +26,6 @@ struct df_args { char *path; bool human_readable; - u8 __pad[7]; }; static int do_df(struct df_args *args) diff --git a/utils/src/main.c b/utils/src/main.c index 277b8ec2..15903125 100644 --- a/utils/src/main.c +++ b/utils/src/main.c @@ -9,6 +9,15 @@ #include "cmd.h" #include "util.h" +/* + * Ensure no compiler-added padding sneaks into structs defined in these + * headers. + */ +#pragma GCC diagnostic error "-Wpadded" +#include "format.h" +#include "ioctl.h" +#pragma GCC diagnostic pop + int main(int argc, char **argv) { /* diff --git a/utils/src/mkfs.c b/utils/src/mkfs.c index 2c1936fb..7d3b582b 100644 --- a/utils/src/mkfs.c +++ b/utils/src/mkfs.c @@ -107,7 +107,6 @@ struct mkfs_args { unsigned long long max_meta_size; unsigned long long max_data_size; bool force; - char __pad[7]; }; /* diff --git a/utils/src/print.c b/utils/src/print.c index aff6110a..968068a5 100644 --- a/utils/src/print.c +++ b/utils/src/print.c @@ -658,7 +658,6 @@ out: struct print_recursion_args { struct scoutfs_super_block *super; int fd; - u8 __pad[4]; }; /* same as fs item but with a small header in the value */ diff --git a/utils/src/setattr.c b/utils/src/setattr.c index e4178f6c..dafa52a9 100644 --- a/utils/src/setattr.c +++ b/utils/src/setattr.c @@ -24,7 +24,6 @@ struct setattr_args { u64 data_version; u64 i_size; bool offline; - char __pad[7]; }; static int do_setattr(struct setattr_args *args) diff --git a/utils/src/stat.c b/utils/src/stat.c index 996d4106..812c38ef 100644 --- a/utils/src/stat.c +++ b/utils/src/stat.c @@ -107,7 +107,6 @@ struct stat_args { char *path; char *single_field; bool is_inode; - u8 __pad[7]; }; static int do_stat(struct stat_args *args) From 64a698aa937aabf619d065b652392020b7873518 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Mon, 4 Jan 2021 16:27:27 -0800 Subject: [PATCH 19/21] Make changes to tests for new scoutfs cmdline syntax Some different error message require changes to golden/* Signed-off-by: Andy Grover --- tests/funcs/fs.sh | 4 +- tests/golden/offline-extent-waiting | 4 +- tests/golden/setattr_more | 8 ++-- tests/golden/simple-release-extents | 16 ++++---- tests/golden/simple-staging | 10 ++--- tests/tests/archive-light-cycle.sh | 4 +- tests/tests/basic-block-counts.sh | 16 ++++---- tests/tests/basic-posix-consistency.sh | 24 +++++------ tests/tests/inode-items-updated.sh | 6 +-- tests/tests/lock-pr-cw-conflict.sh | 2 +- tests/tests/offline-extent-waiting.sh | 54 ++++++++++++------------- tests/tests/setattr_more.sh | 22 +++++----- tests/tests/simple-inode-index.sh | 4 +- tests/tests/simple-release-extents.sh | 28 ++++++------- tests/tests/simple-staging.sh | 34 ++++++++-------- tests/tests/srch-basic-functionality.sh | 2 +- tests/tests/stage-multi-part.sh | 6 +-- tests/tests/stage-release-race-alloc.sh | 6 +-- 18 files changed, 125 insertions(+), 125 deletions(-) diff --git a/tests/funcs/fs.sh b/tests/funcs/fs.sh index 84c1adb4..160f00df 100644 --- a/tests/funcs/fs.sh +++ b/tests/funcs/fs.sh @@ -28,8 +28,8 @@ t_ident() local fsid local rid - fsid=$(scoutfs statfs -s fsid "$mnt") - rid=$(scoutfs statfs -s rid "$mnt") + fsid=$(scoutfs statfs -s fsid -p "$mnt") + rid=$(scoutfs statfs -s rid -p "$mnt") echo "f.${fsid:0:6}.r.${rid:0:6}" } diff --git a/tests/golden/offline-extent-waiting b/tests/golden/offline-extent-waiting index 5b163cc9..5b4af2d2 100644 --- a/tests/golden/offline-extent-waiting +++ b/tests/golden/offline-extent-waiting @@ -1,6 +1,6 @@ == create files == waiter shows up in ioctl -offline wating should be empty: +offline waiting should be empty: 0 offline waiting should now have one known entry: == multiple waiters on same block listed once @@ -8,7 +8,7 @@ offline waiting still has one known entry: == different blocks show up offline waiting now has two known entries: == staging wakes everyone -offline wating should be empty again: +offline waiting should be empty again: 0 == interruption does no harm offline waiting should now have one known entry: diff --git a/tests/golden/setattr_more b/tests/golden/setattr_more index dd80461f..040ffe07 100644 --- a/tests/golden/setattr_more +++ b/tests/golden/setattr_more @@ -1,9 +1,9 @@ == 0 data_version arg fails -setattr_more ioctl failed on '/mnt/test/test/setattr_more/file': Invalid argument (22) -scoutfs: setattr failed: Invalid argument (22) +setattr: data version must not be 0 +Try `setattr --help' or `setattr --usage' for more information. == args must specify size and offline -setattr_more ioctl failed on '/mnt/test/test/setattr_more/file': Invalid argument (22) -scoutfs: setattr failed: Invalid argument (22) +setattr: must provide size if using --offline option +Try `setattr --help' or `setattr --usage' for more information. == only works on regular files failed to open '/mnt/test/test/setattr_more/dir': Is a directory (21) scoutfs: setattr failed: Is a directory (21) diff --git a/tests/golden/simple-release-extents b/tests/golden/simple-release-extents index c28511cc..44305fd1 100644 --- a/tests/golden/simple-release-extents +++ b/tests/golden/simple-release-extents @@ -8,16 +8,16 @@ release ioctl failed: Invalid argument (22) scoutfs: release failed: Invalid argument (22) == releasing non-file fails -ioctl failed on '/mnt/test/test/simple-release-extents/file-char': Inappropriate ioctl for device (25) -release ioctl failed: Inappropriate ioctl for device (25) -scoutfs: release failed: Inappropriate ioctl for device (25) +ioctl failed: Inappropriate ioctl for device (25) +release: must provide file version --data-version +Try `release --help' or `release --usage' for more information. == releasing a non-scoutfs file fails -ioctl failed on '/dev/null': Inappropriate ioctl for device (25) -release ioctl failed: Inappropriate ioctl for device (25) -scoutfs: release failed: Inappropriate ioctl for device (25) +ioctl failed: Inappropriate ioctl for device (25) +release: must provide file version --data-version +Try `release --help' or `release --usage' for more information. == releasing bad version fails -release ioctl failed: Stale file handle (116) -scoutfs: release failed: Stale file handle (116) +release: must provide file version --data-version +Try `release --help' or `release --usage' for more information. == verify small release merging 0 0 0: (0 0 1) (1 101 4) 0 0 1: (0 0 2) (2 102 3) diff --git a/tests/golden/simple-staging b/tests/golden/simple-staging index bc3d10ca..d05c8496 100644 --- a/tests/golden/simple-staging +++ b/tests/golden/simple-staging @@ -4,8 +4,8 @@ == release+stage shouldn't change stat, data seq or vers == stage does change meta_seq == can't use stage to extend online file -stage returned -1, not 4096: error Invalid argument (22) -scoutfs: stage failed: Input/output error (5) +stage: must provide file version with --data-version +Try `stage --help' or `stage --usage' for more information. == wrapped region fails stage returned -1, not 4096: error Invalid argument (22) scoutfs: stage failed: Input/output error (5) @@ -18,6 +18,6 @@ scoutfs: stage failed: Input/output error (5) == partial final block that writes to i_size does work == zero length stage doesn't bring blocks online == stage of non-regular file fails -ioctl failed on '/mnt/test/test/simple-staging/file-char': Inappropriate ioctl for device (25) -stage returned -1, not 1: error Inappropriate ioctl for device (25) -scoutfs: stage failed: Input/output error (5) +ioctl failed: Inappropriate ioctl for device (25) +stage: must provide file version with --data-version +Try `stage --help' or `stage --usage' for more information. diff --git a/tests/tests/archive-light-cycle.sh b/tests/tests/archive-light-cycle.sh index 8321bd71..a02cfab4 100644 --- a/tests/tests/archive-light-cycle.sh +++ b/tests/tests/archive-light-cycle.sh @@ -161,9 +161,9 @@ for n in $(t_fs_nrs); do echo "bash $gen $blocks $n $p $f > $path" >> $create echo "cmp $path <(bash $gen $blocks $n $p $f)" >> $verify echo "vers=\$(scoutfs stat -s data_version $path)" >> $release - echo "scoutfs release $path \$vers 0 $blocks" >> $release + echo "scoutfs release $path -V \$vers -o 0 -l $bytes" >> $release echo "vers=\$(scoutfs stat -s data_version $path)" >> $stage - echo "scoutfs stage $path \$vers 0 $bytes <(bash $gen $blocks $n $p $f)" >> $stage + echo "scoutfs stage <(bash $gen $blocks $n $p $f) $path -V \$vers -o 0 -l $bytes " >> $stage echo "rm -f $path" >> $unlink echo "x=\$(scoutfs stat -s online_blocks $path)" >> $online diff --git a/tests/tests/basic-block-counts.sh b/tests/tests/basic-block-counts.sh index 6a9e49c3..1b95610e 100644 --- a/tests/tests/basic-block-counts.sh +++ b/tests/tests/basic-block-counts.sh @@ -9,14 +9,14 @@ t_require_commands scoutfs dd truncate touch mkdir rm rmdir release_vers() { local file="$1" local vers="$2" - local block="$3" - local count="$4" + local offset="$3" + local length="$4" if [ "$vers" == "stat" ]; then vers=$(scoutfs stat -s data_version "$file") fi - scoutfs release "$file" "$vers" "$block" "$count" + scoutfs release "$file" -V "$vers" -o "$offset" -l "$length" } # if vers is "stat" then we ask stat_more for the data_version @@ -24,14 +24,14 @@ stage_vers() { local file="$1" local vers="$2" local offset="$3" - local count="$4" + local length="$4" local contents="$5" if [ "$vers" == "stat" ]; then vers=$(scoutfs stat -s data_version "$file") fi - scoutfs stage "$file" "$vers" "$offset" "$count" "$contents" + scoutfs stage "$contents" "$file" -V "$vers" -o "$offset" -l "$length" } echo_blocks() @@ -57,15 +57,15 @@ dd if=/dev/zero of="$FILE" bs=4K count=1 conv=notrunc oflag=append status=none echo_blocks "$FILE" echo "== release" -release_vers "$FILE" stat 0 2 +release_vers "$FILE" stat 0 8K echo_blocks "$FILE" echo "== duplicate release" -release_vers "$FILE" stat 0 2 +release_vers "$FILE" stat 0 8K echo_blocks "$FILE" echo "== duplicate release past i_size" -release_vers "$FILE" stat 0 16 +release_vers "$FILE" stat 0 64K echo_blocks "$FILE" echo "== stage" diff --git a/tests/tests/basic-posix-consistency.sh b/tests/tests/basic-posix-consistency.sh index 0444a3aa..9c0d2398 100644 --- a/tests/tests/basic-posix-consistency.sh +++ b/tests/tests/basic-posix-consistency.sh @@ -169,32 +169,32 @@ rm -rf "$T_D0/dir" echo "== inode indexes match after syncing existing" t_sync_seq_index -scoutfs walk-inodes meta_seq 0 -1 "$T_M0" > "$T_TMP.0" -scoutfs walk-inodes meta_seq 0 -1 "$T_M1" > "$T_TMP.1" +scoutfs walk-inodes -p "$T_M0" -- meta_seq 0 -1 > "$T_TMP.0" +scoutfs walk-inodes -p "$T_M1" -- meta_seq 0 -1 > "$T_TMP.1" diff -u "$T_TMP.0" "$T_TMP.1" -scoutfs walk-inodes data_seq 0 -1 "$T_M0" > "$T_TMP.0" -scoutfs walk-inodes data_seq 0 -1 "$T_M1" > "$T_TMP.1" +scoutfs walk-inodes -p "$T_M0" -- data_seq 0 -1 > "$T_TMP.0" +scoutfs walk-inodes -p "$T_M1" -- data_seq 0 -1 > "$T_TMP.1" diff -u "$T_TMP.0" "$T_TMP.1" echo "== inode indexes match after copying and syncing" mkdir "$T_D0/dir" cp -ar /boot/conf* "$T_D0/dir" t_sync_seq_index -scoutfs walk-inodes meta_seq 0 -1 "$T_M0" > "$T_TMP.0" -scoutfs walk-inodes meta_seq 0 -1 "$T_M1" > "$T_TMP.1" +scoutfs walk-inodes -p "$T_M0" -- meta_seq 0 -1 > "$T_TMP.0" +scoutfs walk-inodes -p "$T_M1" -- meta_seq 0 -1 > "$T_TMP.1" diff -u "$T_TMP.0" "$T_TMP.1" -scoutfs walk-inodes data_seq 0 -1 "$T_M0" > "$T_TMP.0" -scoutfs walk-inodes data_seq 0 -1 "$T_M1" > "$T_TMP.1" +scoutfs walk-inodes -p "$T_M0" -- data_seq 0 -1 > "$T_TMP.0" +scoutfs walk-inodes -p "$T_M1" -- data_seq 0 -1 > "$T_TMP.1" diff -u "$T_TMP.0" "$T_TMP.1" echo "== inode indexes match after removing and syncing" rm -f "$T_D1/dir/conf*" t_sync_seq_index -scoutfs walk-inodes meta_seq 0 -1 "$T_M0" > "$T_TMP.0" -scoutfs walk-inodes meta_seq 0 -1 "$T_M1" > "$T_TMP.1" +scoutfs walk-inodes -p "$T_M0" -- meta_seq 0 -1 > "$T_TMP.0" +scoutfs walk-inodes -p "$T_M1" -- meta_seq 0 -1 > "$T_TMP.1" diff -u "$T_TMP.0" "$T_TMP.1" -scoutfs walk-inodes data_seq 0 -1 "$T_M0" > "$T_TMP.0" -scoutfs walk-inodes data_seq 0 -1 "$T_M1" > "$T_TMP.1" +scoutfs walk-inodes -p "$T_M0" -- data_seq 0 -1 > "$T_TMP.0" +scoutfs walk-inodes -p "$T_M1" -- data_seq 0 -1 > "$T_TMP.1" diff -u "$T_TMP.0" "$T_TMP.1" t_pass diff --git a/tests/tests/inode-items-updated.sh b/tests/tests/inode-items-updated.sh index 810e3ac3..193a626a 100644 --- a/tests/tests/inode-items-updated.sh +++ b/tests/tests/inode-items-updated.sh @@ -30,7 +30,7 @@ echo "== create files and sync" dd if=/dev/zero of="$DIR/truncate" bs=4096 count=1 status=none dd if=/dev/zero of="$DIR/stage" bs=4096 count=1 status=none vers=$(scoutfs stat -s data_version "$DIR/stage") -scoutfs release "$DIR/stage" $vers 0 1 +scoutfs release "$DIR/stage" -V $vers -o 0 -l 4K dd if=/dev/zero of="$DIR/release" bs=4096 count=1 status=none touch "$DIR/write_end" mkdir "$DIR"/{mknod_dir,link_dir,unlink_dir,symlink_dir,rename_dir} @@ -41,9 +41,9 @@ sync; sync echo "== modify files" truncate -s 0 "$DIR/truncate" vers=$(scoutfs stat -s data_version "$DIR/stage") -scoutfs stage "$DIR/stage" $vers 0 4096 /dev/zero +scoutfs stage /dev/zero "$DIR/stage" -V $vers -o 0 -l 4096 vers=$(scoutfs stat -s data_version "$DIR/release") -scoutfs release "$DIR/release" $vers 0 1 +scoutfs release "$DIR/release" -V $vers -o 0 -l 4K dd if=/dev/zero of="$DIR/write_end" bs=4096 count=1 status=none conv=notrunc touch $DIR/mknod_dir/mknod_file touch $DIR/link_dir/link_targ diff --git a/tests/tests/lock-pr-cw-conflict.sh b/tests/tests/lock-pr-cw-conflict.sh index 09210a39..96ce7b9f 100644 --- a/tests/tests/lock-pr-cw-conflict.sh +++ b/tests/tests/lock-pr-cw-conflict.sh @@ -9,7 +9,7 @@ FILE="$T_D0/file" echo "== race writing and index walking" for i in $(seq 1 10); do dd if=/dev/zero of="$FILE" bs=4K count=1 status=none conv=notrunc & - scoutfs walk-inodes data_seq 0 -1 "$T_M0" > /dev/null & + scoutfs walk-inodes -p "$T_M0" -- data_seq 0 -1 > /dev/null & wait done diff --git a/tests/tests/offline-extent-waiting.sh b/tests/tests/offline-extent-waiting.sh index fb413562..31b04b59 100644 --- a/tests/tests/offline-extent-waiting.sh +++ b/tests/tests/offline-extent-waiting.sh @@ -24,7 +24,7 @@ expect_wait() shift done - scoutfs data-waiting 0 0 "$file" > $T_TMP.wait.output + scoutfs data-waiting -B 0 -I 0 -p "$file" > $T_TMP.wait.output diff -u $T_TMP.wait.expected $T_TMP.wait.output } @@ -37,9 +37,9 @@ ino=$(stat -c "%i" "$DIR/file") vers=$(scoutfs stat -s data_version "$DIR/file") echo "== waiter shows up in ioctl" -echo "offline wating should be empty:" -scoutfs data-waiting 0 0 "$DIR" | wc -l -scoutfs release "$DIR/file" "$vers" 0 $BLOCKS +echo "offline waiting should be empty:" +scoutfs data-waiting -B 0 -I 0 -p "$DIR" | wc -l +scoutfs release "$DIR/file" -V "$vers" -o 0 -l $BYTES cat "$DIR/file" > /dev/null & sleep .1 echo "offline waiting should now have one known entry:" @@ -58,13 +58,13 @@ echo "offline waiting now has two known entries:" expect_wait "$DIR/file" "read" $ino 0 $ino 1 echo "== staging wakes everyone" -scoutfs stage "$DIR/file" "$vers" 0 $BYTES "$DIR/golden" +scoutfs stage "$DIR/golden" "$DIR/file" -V "$vers" -o 0 -l $BYTES sleep .1 -echo "offline wating should be empty again:" -scoutfs data-waiting 0 0 "$DIR" | wc -l +echo "offline waiting should be empty again:" +scoutfs data-waiting -B 0 -I 0 -p "$DIR" | wc -l echo "== interruption does no harm" -scoutfs release "$DIR/file" "$vers" 0 $BLOCKS +scoutfs release "$DIR/file" -V "$vers" -o 0 -l $BYTES cat "$DIR/file" > /dev/null 2>&1 & pid="$!" sleep .1 @@ -74,7 +74,7 @@ kill "$pid" # silence terminated message wait "$pid" 2> /dev/null echo "offline waiting should be empty again:" -scoutfs data-waiting 0 0 "$DIR" | wc -l +scoutfs data-waiting -B 0 -I 0 -p "$DIR" | wc -l echo "== EIO injection for waiting readers works" ino=$(stat -c "%i" "$DIR/file") @@ -86,23 +86,23 @@ dd if="$DIR/file" bs=$BS skip=1 of=/dev/null 2>&1 | \ pid2="$!" sleep .1 echo "offline waiting should now have two known entries:" -scoutfs data-waiting 0 0 "$DIR" | wc -l +scoutfs data-waiting -B 0 -I 0 -p "$DIR" | wc -l expect_wait "$DIR/file" "read" $ino 0 $ino 1 -scoutfs data-wait-err "$DIR" "$ino" "$vers" 0 $((BS*2)) read -5 +scoutfs data-wait-err -p "$DIR" -I "$ino" -V "$vers" -F 0 -C $((BS*2)) -O read -E -5 sleep .1 echo "offline waiting should now have 0 known entries:" -scoutfs data-waiting 0 0 "$DIR" | wc -l +scoutfs data-waiting -B 0 -I 0 -p "$DIR" | wc -l # silence terminated message wait "$pid" 2> /dev/null wait "$pid2" 2> /dev/null cat $T_TMP.cat1 cat $T_TMP.cat2 echo "offline waiting should be empty again:" -scoutfs data-waiting 0 0 "$DIR" | wc -l +scoutfs data-waiting -B 0 -I 0 -p "$DIR" | wc -l echo "== readahead while offline does no harm" xfs_io -c "fadvise -w 0 $BYTES" "$DIR/file" -scoutfs stage "$DIR/file" "$vers" 0 $BYTES "$DIR/golden" +scoutfs stage "$DIR/golden" "$DIR/file" -V "$vers" -o 0 -l $BYTES cmp "$DIR/file" "$DIR/golden" echo "== waiting on interesting blocks works" @@ -113,65 +113,65 @@ for base in $(echo 0 $(($BLOCKS / 2)) $(($BLOCKS - 2))); do done done for b in $blocks; do - scoutfs release "$DIR/file" "$vers" 0 $BLOCKS + scoutfs release "$DIR/file" -V "$vers" -o 0 -l $BYTES dd if="$DIR/file" of=/dev/null \ status=none bs=$BS count=1 skip=$b 2> /dev/null & sleep .1 - scoutfs stage "$DIR/file" "$vers" 0 $BYTES "$DIR/golden" + scoutfs stage "$DIR/golden" "$DIR/file" -V "$vers" -o 0 -l $BYTES sleep .1 echo "offline waiting is empty at block $b" - scoutfs data-waiting 0 0 "$DIR" | wc -l + scoutfs data-waiting -B 0 -I 0 -p "$DIR" | wc -l done echo "== contents match when staging blocks forward" -scoutfs release "$DIR/file" "$vers" 0 $BLOCKS +scoutfs release "$DIR/file" -V "$vers" -o 0 -l $BYTES cat "$DIR/file" > "$DIR/forward" & for b in $(seq 0 1 $((BLOCKS - 1))); do dd if="$DIR/golden" of="$DIR/block" status=none bs=$BS skip=$b count=1 - scoutfs stage "$DIR/file" "$vers" $((b * $BS)) $BS "$DIR/block" + scoutfs stage "$DIR/block" "$DIR/file" -V "$vers" -o $((b * $BS)) -l $BS done sleep .1 cmp "$DIR/golden" "$DIR/forward" echo "== contents match when staging blocks backwards" -scoutfs release "$DIR/file" "$vers" 0 $BLOCKS +scoutfs release "$DIR/file" -V "$vers" -o 0 -l $BYTES cat "$DIR/file" > "$DIR/backward" & for b in $(seq $((BLOCKS - 1)) -1 0); do dd if="$DIR/golden" of="$DIR/block" status=none bs=$BS skip=$b count=1 - scoutfs stage "$DIR/file" "$vers" $((b * $BS)) $BS "$DIR/block" + scoutfs stage "$DIR/block" "$DIR/file" -V "$vers" -o $((b * $BS)) -l $BS done sleep .1 cmp "$DIR/golden" "$DIR/backward" echo "== truncate to same size doesn't wait" -scoutfs release "$DIR/file" "$vers" 0 $BLOCKS +scoutfs release "$DIR/file" -V "$vers" -o 0 -l $BYTES truncate -s "$BYTES" "$DIR/file" & sleep .1 echo "offline wating should be empty:" -scoutfs data-waiting 0 0 "$DIR" | wc -l +scoutfs data-waiting -B 0 -I 0 -p "$DIR" | wc -l echo "== truncating does wait" truncate -s "$BS" "$DIR/file" & sleep .1 echo "truncate should be waiting for first block:" expect_wait "$DIR/file" "change_size" $ino 0 -scoutfs stage "$DIR/file" "$vers" 0 $BYTES "$DIR/golden" +scoutfs stage "$DIR/golden" "$DIR/file" -V "$vers" -o 0 -l $BYTES sleep .1 echo "trunate should no longer be waiting:" -scoutfs data-waiting 0 0 "$DIR" | wc -l +scoutfs data-waiting -B 0 -I 0 -p "$DIR" | wc -l cat "$DIR/golden" > "$DIR/file" vers=$(scoutfs stat -s data_version "$DIR/file") echo "== writing waits" dd if=/dev/urandom of="$DIR/other" bs=$BS count=$BLOCKS status=none -scoutfs release "$DIR/file" "$vers" 0 $BLOCKS +scoutfs release "$DIR/file" -V "$vers" -o 0 -l $BYTES # overwrite, not truncate+write dd if="$DIR/other" of="$DIR/file" \ bs=$BS count=$BLOCKS conv=notrunc status=none & sleep .1 echo "should be waiting for write" expect_wait "$DIR/file" "write" $ino 0 -scoutfs stage "$DIR/file" "$vers" 0 $BYTES "$DIR/golden" +scoutfs stage "$DIR/golden" "$DIR/file" -V "$vers" -o 0 -l $BYTES cmp "$DIR/file" "$DIR/other" echo "== cleanup" diff --git a/tests/tests/setattr_more.sh b/tests/tests/setattr_more.sh index 0aecd9be..af05b89f 100644 --- a/tests/tests/setattr_more.sh +++ b/tests/tests/setattr_more.sh @@ -8,63 +8,63 @@ FILE="$T_D0/file" echo "== 0 data_version arg fails" touch "$FILE" -scoutfs setattr -d 0 -s 1 -f "$FILE" 2>&1 | t_filter_fs +scoutfs setattr -V 0 -s 1 "$FILE" 2>&1 | t_filter_fs rm "$FILE" echo "== args must specify size and offline" touch "$FILE" -scoutfs setattr -d 1 -o -s 0 -f "$FILE" 2>&1 | t_filter_fs +scoutfs setattr -V 1 -o -s 0 "$FILE" 2>&1 | t_filter_fs rm "$FILE" echo "== only works on regular files" mkdir "$T_D0/dir" -scoutfs setattr -d 1 -s 1 -f "$T_D0/dir" 2>&1 | t_filter_fs +scoutfs setattr -V 1 -s 1 "$T_D0/dir" 2>&1 | t_filter_fs rmdir "$T_D0/dir" mknod "$T_D0/char" c 1 3 -scoutfs setattr -d 1 -s 1 -f "$T_D0/char" 2>&1 | t_filter_fs +scoutfs setattr -V 1 -s 1 "$T_D0/char" 2>&1 | t_filter_fs rm "$T_D0/char" echo "== non-zero file size fails" echo contents > "$FILE" -scoutfs setattr -d 1 -s 1 -f "$FILE" 2>&1 | t_filter_fs +scoutfs setattr -V 1 -s 1 "$FILE" 2>&1 | t_filter_fs rm "$FILE" echo "== non-zero file data_version fails" touch "$FILE" truncate -s 1M "$FILE" truncate -s 0 "$FILE" -scoutfs setattr -d 1 -o -s 1 -f "$FILE" 2>&1 | t_filter_fs +scoutfs setattr -V 1 -o -s 1 "$FILE" 2>&1 | t_filter_fs rm "$FILE" echo "== large size is set" touch "$FILE" -scoutfs setattr -d 1 -s 578437695752307201 -f "$FILE" 2>&1 | t_filter_fs +scoutfs setattr -V 1 -s 578437695752307201 "$FILE" 2>&1 | t_filter_fs stat -c "%s" "$FILE" rm "$FILE" echo "== large data_version is set" touch "$FILE" -scoutfs setattr -d 578437695752307201 -s 1 -f "$FILE" 2>&1 | t_filter_fs +scoutfs setattr -V 578437695752307201 -s 1 "$FILE" 2>&1 | t_filter_fs scoutfs stat -s data_version "$FILE" rm "$FILE" echo "== large ctime is set" touch "$FILE" # only doing 32bit sec 'cause stat gets confused -scoutfs setattr -c 67305985.999999999 -d 1 -s 1 -f "$FILE" 2>&1 | t_filter_fs +scoutfs setattr -t 67305985.999999999 -V 1 -s 1 "$FILE" 2>&1 | t_filter_fs TZ=GMT stat -c "%z" "$FILE" rm "$FILE" echo "== large offline extents are created" touch "$FILE" -scoutfs setattr -d 1 -o -s $((10007 * 4096)) -f "$FILE" 2>&1 | t_filter_fs +scoutfs setattr -V 1 -o -s $((10007 * 4096)) "$FILE" 2>&1 | t_filter_fs filefrag -v -b4096 "$FILE" 2>&1 | t_filter_fs rm "$FILE" # had a bug where we were creating extents that were too long echo "== correct offline extent length" touch "$FILE" -scoutfs setattr -d 1 -o -s 4000000000 -f "$FILE" 2>&1 | t_filter_fs +scoutfs setattr -V 1 -o -s 4000000000 "$FILE" 2>&1 | t_filter_fs scoutfs stat -s offline_blocks "$FILE" rm "$FILE" diff --git a/tests/tests/simple-inode-index.sh b/tests/tests/simple-inode-index.sh index bdd065d8..514a4348 100644 --- a/tests/tests/simple-inode-index.sh +++ b/tests/tests/simple-inode-index.sh @@ -14,7 +14,7 @@ query_index() { local first="${2:-0}" local last="${3:--1}" - scoutfs walk-inodes $which $first $last "$T_M0" + scoutfs walk-inodes -p "$T_M0" -- $which $first $last } # print the major in the index for the ino if it's found @@ -22,7 +22,7 @@ ino_major() { local which="$1" local ino="$2" - scoutfs walk-inodes $which 0 -1 "$T_M0" | \ + scoutfs walk-inodes -p "$T_M0" -- $which 0 -1 | \ awk '($4 == "'$ino'") {print $2}' } diff --git a/tests/tests/simple-release-extents.sh b/tests/tests/simple-release-extents.sh index 4fa9583b..da3eee3d 100644 --- a/tests/tests/simple-release-extents.sh +++ b/tests/tests/simple-release-extents.sh @@ -23,14 +23,14 @@ create_file() { release_vers() { local file="$1" local vers="$2" - local block="$3" - local count="$4" + local offset="$3" + local length="$4" if [ "$vers" == "stat" ]; then vers=$(scoutfs stat -s data_version "$file") fi - scoutfs release "$file" "$vers" "$block" "$count" + scoutfs release "$file" -V "$vers" -o "$offset" -l "$length" } FILE="$T_D0/file" @@ -38,41 +38,41 @@ CHAR="$FILE-char" echo "== simple whole file multi-block releasing" create_file "$FILE" 65536 -release_vers "$FILE" stat 0 16 +release_vers "$FILE" stat 0 64K rm "$FILE" echo "== release last block that straddles i_size" create_file "$FILE" 6144 -release_vers "$FILE" stat 1 1 +release_vers "$FILE" stat 4K 4K rm "$FILE" echo "== release entire file past i_size" create_file "$FILE" 8192 -release_vers "$FILE" stat 0 100 +release_vers "$FILE" stat 0 400K # not deleting for the following little tests echo "== releasing offline extents is fine" -release_vers "$FILE" stat 0 100 +release_vers "$FILE" stat 0 400K echo "== 0 count is fine" release_vers "$FILE" stat 0 0 echo "== release past i_size is fine" -release_vers "$FILE" stat 100 1 +release_vers "$FILE" stat 400K 4K echo "== wrapped blocks fails" release_vers "$FILE" stat $vers 0x8000000000000000 0x8000000000000000 echo "== releasing non-file fails" mknod "$CHAR" c 1 3 -release_vers "$CHAR" stat 0 1 2>&1 | t_filter_fs +release_vers "$CHAR" stat 0 4K 2>&1 | t_filter_fs rm "$CHAR" echo "== releasing a non-scoutfs file fails" -release_vers "/dev/null" stat 0 1 +release_vers "/dev/null" stat 0 4K echo "== releasing bad version fails" -release_vers "$FILE" 0 0 1 +release_vers "$FILE" 0 0 4K rm "$FILE" @@ -108,9 +108,9 @@ for c in $(seq 0 4); do start=$(fiemap_file "$FILE" | \ awk '($1 == "0:"){print substr($4, 0, length($4)- 2)}') - release_vers "$FILE" stat $a 1 - release_vers "$FILE" stat $b 1 - release_vers "$FILE" stat $c 1 + release_vers "$FILE" stat $(($a * 4))K 4K + release_vers "$FILE" stat $(($b * 4))K 4K + release_vers "$FILE" stat $(($c * 4))K 4K echo -n "$a $b $c:" diff --git a/tests/tests/simple-staging.sh b/tests/tests/simple-staging.sh index 1cd8e1e6..ad059bd0 100644 --- a/tests/tests/simple-staging.sh +++ b/tests/tests/simple-staging.sh @@ -29,14 +29,14 @@ create_file() { release_vers() { local file="$1" local vers="$2" - local block="$3" - local count="$4" + local offset="$3" + local length="$4" if [ "$vers" == "stat" ]; then vers=$(scoutfs stat -s data_version "$file") fi - scoutfs release "$file" "$vers" "$block" "$count" + scoutfs release "$file" -V "$vers" -o "$offset" -l "$length" } # if vers is "stat" then we ask stat_more for the data_version @@ -44,14 +44,14 @@ stage_vers() { local file="$1" local vers="$2" local offset="$3" - local count="$4" + local length="$4" local contents="$5" if [ "$vers" == "stat" ]; then vers=$(scoutfs stat -s data_version "$file") fi - scoutfs stage "$file" "$vers" "$offset" "$count" "$contents" + scoutfs stage "$contents" "$file" -V "$vers" -o "$offset" -l "$length" } FILE="$T_D0/file" @@ -60,7 +60,7 @@ CHAR="$FILE-char" echo "== create/release/stage single block file" create_file "$FILE" 4096 cp "$FILE" "$T_TMP" -release_vers "$FILE" stat 0 1 +release_vers "$FILE" stat 0 4K # make sure there only offline extents fiemap_file "$FILE" | grep "^[ 0-9]*:" | grep -v "unknown" stage_vers "$FILE" stat 0 4096 "$T_TMP" @@ -70,7 +70,7 @@ rm -f "$FILE" echo "== create/release/stage larger file" create_file "$FILE" $((4096 * 4096)) cp "$FILE" "$T_TMP" -release_vers "$FILE" stat 0 4096 +release_vers "$FILE" stat 0 16M # make sure there only offline extents fiemap_file "$FILE" | grep "^[ 0-9]*:" | grep -v "unknown" stage_vers "$FILE" stat 0 $((4096 * 4096)) "$T_TMP" @@ -83,7 +83,7 @@ cp "$FILE" "$T_TMP" nr=1 while [ "$nr" -lt 10 ]; do echo "attempt $nr" >> $seqres.full 2>&1 - release_vers "$FILE" stat 0 1024 + release_vers "$FILE" stat 0 4096K sync echo 3 > /proc/sys/vm/drop_caches stage_vers "$FILE" stat 0 $((4096 * 1024)) "$T_TMP" @@ -100,7 +100,7 @@ sync stat "$FILE" > "$T_TMP.before" scoutfs stat -s data_seq "$FILE" >> "$T_TMP.before" scoutfs stat -s data_version "$FILE" >> "$T_TMP.before" -release_vers "$FILE" stat 0 1 +release_vers "$FILE" stat 0 4K stage_vers "$FILE" stat 0 4096 "$T_TMP" stat "$FILE" > "$T_TMP.after" scoutfs stat -s data_seq "$FILE" >> "$T_TMP.after" @@ -110,7 +110,7 @@ rm -f "$FILE" echo "== stage does change meta_seq" create_file "$FILE" 4096 -release_vers "$FILE" stat 0 1 +release_vers "$FILE" stat 0 4K sync before=$(scoutfs stat -s meta_seq "$FILE") stage_vers "$FILE" stat 0 4096 "$T_TMP" @@ -121,7 +121,7 @@ rm -f "$FILE" # XXX this now waits, demand staging should be own test #echo "== can't write to offline" #create_file "$FILE" 4096 -#release_vers "$FILE" stat 0 1 +#release_vers "$FILE" stat 0 4K ## make sure there only offline extents #fiemap_file "$FILE" | grep "^[ 0-9]*:" | grep -v "unknown" #dd if=/dev/zero of="$FILE" conv=notrunc bs=4096 count=1 2>&1 | t_filter_fs @@ -144,13 +144,13 @@ rm -f "$FILE" echo "== wrapped region fails" create_file "$FILE" 4096 -stage_vers "$FILE" stat 0xFFFFFFFFFFFFFFFF 4096 /dev/zero +stage_vers "$FILE" stat 0xFFFFFFFFFFFFF000 4096 /dev/zero rm -f "$FILE" echo "== non-block aligned offset fails" create_file "$FILE" 4096 cp "$FILE" "$T_TMP" -release_vers "$FILE" stat 0 1 +release_vers "$FILE" stat 0 4K stage_vers "$FILE" stat 1 4095 "$T_TMP" fiemap_file "$FILE" | grep "^[ 0-9]*:" | grep -v "unknown" rm -f "$FILE" @@ -158,7 +158,7 @@ rm -f "$FILE" echo "== non-block aligned len within block fails" create_file "$FILE" 4096 cp "$FILE" "$T_TMP" -release_vers "$FILE" stat 0 1 +release_vers "$FILE" stat 0 4K stage_vers "$FILE" stat 0 1024 "$T_TMP" fiemap_file "$FILE" | grep "^[ 0-9]*:" | grep -v "unknown" rm -f "$FILE" @@ -166,14 +166,14 @@ rm -f "$FILE" echo "== partial final block that writes to i_size does work" create_file "$FILE" 2048 cp "$FILE" "$T_TMP" -release_vers "$FILE" stat 0 1 +release_vers "$FILE" stat 0 4K stage_vers "$FILE" stat 0 2048 "$T_TMP" cmp "$FILE" "$T_TMP" rm -f "$FILE" echo "== zero length stage doesn't bring blocks online" create_file "$FILE" $((4096 * 100)) -release_vers "$FILE" stat 0 100 +release_vers "$FILE" stat 0 400K stage_vers "$FILE" stat 4096 0 /dev/zero fiemap_file "$FILE" | grep "^[ 0-9]*:" | grep -v "unknown" rm -f "$FILE" @@ -188,7 +188,7 @@ rm -f "$FILE" #create_file "$FILE" 4096 #cp "$FILE" "$T_TMP" #sync -#release_vers "$FILE" stat 0 1 +#release_vers "$FILE" stat 0 4K #md5sum "$FILE" 2>&1 | t_filter_fs #stage_vers "$FILE" stat 0 4096 "$T_TMP" #cmp "$FILE" "$T_TMP" diff --git a/tests/tests/srch-basic-functionality.sh b/tests/tests/srch-basic-functionality.sh index d7d6eb66..b310ecd4 100644 --- a/tests/tests/srch-basic-functionality.sh +++ b/tests/tests/srch-basic-functionality.sh @@ -17,7 +17,7 @@ diff_srch_find() local n="$1" sync - scoutfs search-xattrs -n "$n" -f "$T_M0" > "$T_TMP.srch" + scoutfs search-xattrs "$n" -p "$T_M0" > "$T_TMP.srch" find_xattrs -d "$T_D0" -m "$T_M0" -n "$n" > "$T_TMP.find" diff -u "$T_TMP.srch" "$T_TMP.find" diff --git a/tests/tests/stage-multi-part.sh b/tests/tests/stage-multi-part.sh index 594a4f06..6ed1673c 100644 --- a/tests/tests/stage-multi-part.sh +++ b/tests/tests/stage-multi-part.sh @@ -29,7 +29,7 @@ release_file() { local path="$1" local vers=$(scoutfs stat -s data_version "$path") - scoutfs release "$path" "$vers" 0 $FILE_BLOCKS + scoutfs release "$path" -V "$vers" -o 0 -l $FILE_BYTES } stage_file() { @@ -38,8 +38,8 @@ stage_file() { local off=0 for a in $(seq 1 $NR_FRAGS); do - scoutfs stage "$path" "$vers" $off $FRAG_BYTES \ - <(gen $FRAG_BLOCKS $a $a $a) + scoutfs stage <(gen $FRAG_BLOCKS $a $a $a) "$path" -V "$vers" \ + -o $off -l $FRAG_BYTES ((off+=$FRAG_BYTES)) done } diff --git a/tests/tests/stage-release-race-alloc.sh b/tests/tests/stage-release-race-alloc.sh index 5dc3438d..b9c74de5 100644 --- a/tests/tests/stage-release-race-alloc.sh +++ b/tests/tests/stage-release-race-alloc.sh @@ -15,7 +15,7 @@ release_file() { local vers=$(scoutfs stat -s data_version "$path") echo "releasing $path" >> "$T_TMP.log" - scoutfs release "$path" "$vers" 0 $BLOCKS + scoutfs release "$path" -V "$vers" -o 0 -l $BYTES echo "released $path" >> "$T_TMP.log" } @@ -24,8 +24,8 @@ stage_file() { local vers=$(scoutfs stat -s data_version "$path") echo "staging $path" >> "$T_TMP.log" - scoutfs stage "$path" "$vers" 0 $BYTES \ - "$DIR/good/$(basename $path)" + scoutfs stage "$DIR/good/$(basename $path)" "$path" -V "$vers" -o 0 -l $BYTES + echo "staged $path" >> "$T_TMP.log" } From 2c5871c253628caa16b6b0f77dba5ae033b9766a Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Tue, 5 Jan 2021 11:59:54 -0800 Subject: [PATCH 20/21] Change release ioctl to be denominated in bytes not blocks This more closely matches stage ioctl and other conventions. Also change release code to use offset/length nomenclature for consistency. Signed-off-by: Andy Grover --- kmod/src/ioctl.c | 44 +++++++++++++++++++++------------------ kmod/src/ioctl.h | 10 ++++----- kmod/src/scoutfs_trace.h | 22 ++++++++++---------- utils/src/stage_release.c | 6 +++--- 4 files changed, 43 insertions(+), 39 deletions(-) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index cfb06462..3fcaae34 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -274,8 +274,8 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) struct super_block *sb = inode->i_sb; struct scoutfs_ioctl_release args; struct scoutfs_lock *lock = NULL; - loff_t start; - loff_t end_inc; + u64 sblock; + u64 eblock; u64 online; u64 offline; u64 isize; @@ -286,9 +286,11 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) trace_scoutfs_ioc_release(sb, scoutfs_ino(inode), &args); - if (args.count == 0) + if (args.length == 0) return 0; - if ((args.block + args.count) < args.block) + if (((args.offset + args.length) < args.offset) || + (args.offset & SCOUTFS_BLOCK_SM_MASK) || + (args.length & SCOUTFS_BLOCK_SM_MASK)) return -EINVAL; @@ -321,23 +323,24 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) inode_dio_wait(inode); /* drop all clean and dirty cached blocks in the range */ - start = args.block << SCOUTFS_BLOCK_SM_SHIFT; - end_inc = ((args.block + args.count) << SCOUTFS_BLOCK_SM_SHIFT) - 1; - truncate_inode_pages_range(&inode->i_data, start, end_inc); + truncate_inode_pages_range(&inode->i_data, args.offset, + args.offset + args.length - 1); + sblock = args.offset >> SCOUTFS_BLOCK_SM_SHIFT; + eblock = (args.offset + args.length - 1) >> SCOUTFS_BLOCK_SM_SHIFT; ret = scoutfs_data_truncate_items(sb, inode, scoutfs_ino(inode), - args.block, - args.block + args.count - 1, true, + sblock, + eblock, true, lock); if (ret == 0) { scoutfs_inode_get_onoff(inode, &online, &offline); isize = i_size_read(inode); if (online == 0 && isize) { - start = (isize + SCOUTFS_BLOCK_SM_SIZE - 1) + sblock = (isize + SCOUTFS_BLOCK_SM_SIZE - 1) >> SCOUTFS_BLOCK_SM_SHIFT; ret = scoutfs_data_truncate_items(sb, inode, scoutfs_ino(inode), - start, U64_MAX, + sblock, U64_MAX, false, lock); } } @@ -459,23 +462,24 @@ static long scoutfs_ioc_stage(struct file *file, unsigned long arg) trace_scoutfs_ioc_stage(sb, scoutfs_ino(inode), &args); - end_size = args.offset + args.count; + end_size = args.offset + args.length; /* verify arg constraints that aren't dependent on file */ - if (args.count < 0 || (end_size < args.offset) || - args.offset & SCOUTFS_BLOCK_SM_MASK) + if (args.length < 0 || (end_size < args.offset) || + args.offset & SCOUTFS_BLOCK_SM_MASK) { return -EINVAL; + } - if (args.count == 0) + if (args.length == 0) return 0; /* the iocb is really only used for the file pointer :P */ init_sync_kiocb(&kiocb, file); kiocb.ki_pos = args.offset; - kiocb.ki_left = args.count; - kiocb.ki_nbytes = args.count; + kiocb.ki_left = args.length; + kiocb.ki_nbytes = args.length; iov.iov_base = (void __user *)(unsigned long)args.buf_ptr; - iov.iov_len = args.count; + iov.iov_len = args.length; ret = mnt_want_write_file(file); if (ret) @@ -514,11 +518,11 @@ static long scoutfs_ioc_stage(struct file *file, unsigned long arg) written = 0; do { ret = generic_file_buffered_write(&kiocb, &iov, 1, pos, &pos, - args.count, written); + args.length, written); BUG_ON(ret == -EIOCBQUEUED); if (ret > 0) written += ret; - } while (ret > 0 && written < args.count); + } while (ret > 0 && written < args.length); si->staging = false; current->backing_dev_info = NULL; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index a53626a0..081baa3e 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -176,8 +176,8 @@ struct scoutfs_ioctl_ino_path_result { * an offline record is left behind to trigger demand staging if the * file is read. * - * The starting block offset and number of blocks to release are in - * units 4KB blocks. + * The starting file offset and number of bytes to release must be in + * multiples of 4KB. * * The specified range can extend past i_size and can straddle sparse * regions or blocks that are already offline. The only change it makes @@ -193,8 +193,8 @@ struct scoutfs_ioctl_ino_path_result { * presentation of the data in the file. */ struct scoutfs_ioctl_release { - __u64 block; - __u64 count; + __u64 offset; + __u64 length; __u64 data_version; }; @@ -205,7 +205,7 @@ struct scoutfs_ioctl_stage { __u64 data_version; __u64 buf_ptr; __u64 offset; - __s32 count; + __s32 length; __u32 _pad; }; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index aa376dab..a2e1cae8 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -530,22 +530,22 @@ TRACE_EVENT(scoutfs_ioc_release, TP_STRUCT__entry( SCSB_TRACE_FIELDS __field(__u64, ino) - __field(__u64, block) - __field(__u64, count) + __field(__u64, offset) + __field(__u64, length) __field(__u64, vers) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); __entry->ino = ino; - __entry->block = args->block; - __entry->count = args->count; + __entry->offset = args->offset; + __entry->length = args->length; __entry->vers = args->data_version; ), - TP_printk(SCSBF" ino %llu block %llu count %llu vers %llu", - SCSB_TRACE_ARGS, __entry->ino, __entry->block, - __entry->count, __entry->vers) + TP_printk(SCSBF" ino %llu offset %llu length %llu vers %llu", + SCSB_TRACE_ARGS, __entry->ino, __entry->offset, + __entry->length, __entry->vers) ); DEFINE_EVENT(scoutfs_ino_ret_class, scoutfs_ioc_release_ret, @@ -564,7 +564,7 @@ TRACE_EVENT(scoutfs_ioc_stage, __field(__u64, ino) __field(__u64, vers) __field(__u64, offset) - __field(__s32, count) + __field(__s32, length) ), TP_fast_assign( @@ -572,12 +572,12 @@ TRACE_EVENT(scoutfs_ioc_stage, __entry->ino = ino; __entry->vers = args->data_version; __entry->offset = args->offset; - __entry->count = args->count; + __entry->length = args->length; ), - TP_printk(SCSBF" ino %llu vers %llu offset %llu count %d", + TP_printk(SCSBF" ino %llu vers %llu offset %llu length %d", SCSB_TRACE_ARGS, __entry->ino, __entry->vers, - __entry->offset, __entry->count) + __entry->offset, __entry->length) ); TRACE_EVENT(scoutfs_ioc_data_wait_err, diff --git a/utils/src/stage_release.c b/utils/src/stage_release.c index 38b62196..e4bc7bc0 100644 --- a/utils/src/stage_release.c +++ b/utils/src/stage_release.c @@ -76,7 +76,7 @@ static int do_stage(struct stage_args *args) ioctl_args.data_version = args->data_version; ioctl_args.buf_ptr = (unsigned long)buf; ioctl_args.offset = args->offset; - ioctl_args.count = bytes; + ioctl_args.length = bytes; args->length -= bytes; args->offset += bytes; @@ -211,8 +211,8 @@ static int do_release(struct release_args *args) assert(args->offset % SCOUTFS_BLOCK_SM_SIZE == 0); assert(args->length % SCOUTFS_BLOCK_SM_SIZE == 0); - ioctl_args.block = args->offset / SCOUTFS_BLOCK_SM_SIZE; - ioctl_args.count = args->length / SCOUTFS_BLOCK_SM_SIZE; + ioctl_args.offset = args->offset; + ioctl_args.length = args->length; ioctl_args.data_version = args->data_version; ret = ioctl(fd, SCOUTFS_IOC_RELEASE, &ioctl_args); From 454dbebf595a1ca2de6c55361c97fc0168fdba39 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Sun, 10 Jan 2021 13:20:52 -0800 Subject: [PATCH 21/21] Categorize not enough mounts as skip, not fail Signed-off-by: Andy Grover --- tests/funcs/require.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/funcs/require.sh b/tests/funcs/require.sh index 878fd33f..17b757cf 100644 --- a/tests/funcs/require.sh +++ b/tests/funcs/require.sh @@ -21,5 +21,5 @@ t_require_mounts() { local req="$1" test "$T_NR_MOUNTS" -ge "$req" || \ - t_fail "$req mounts required, only have $T_NR_MOUNTS" + t_skip "$req mounts required, only have $T_NR_MOUNTS" }