scoutfs-utils: Use separate block devices for metadata and data

mkfs: Take two block devices as arguments. Write everything to metadata
dev, and the superblock to the data dev. UUIDs match. Differentiate by
checking a bit in a new "flags" field in the superblock.

Refactor device_size() a little. Convert spaces to tabs.

Move code to pretty-print sizes to dev.c so we can use it in error
messages there, as well as in mkfs.c.

print: Include flags in output.

Add -D and -M options for setting max dev sizes

Allow sizes to be specified using units like "K", "G" etc.

Note: -D option replaces -S option, and uses above units rather than
the number of 4k data blocks.

Update man pages for cmdline changes.

Signed-off-by: Andy Grover <agrover@versity.com>
This commit is contained in:
Andy Grover
2020-11-19 11:41:54 -08:00
committed by Zach Brown
parent f46ab548a4
commit 8f72d16609
9 changed files with 288 additions and 127 deletions
+7 -1
View File
@@ -2,7 +2,7 @@
.SH NAME
scoutfs \- overview and mount options for the scoutfs filesystem
.SH DESCRIPTION
A scoutfs filesystem is stored on a block device. Multiple mounts of
A scoutfs filesystem is stored on two block devices. Multiple mounts of
the filesystem are supported between hosts that share access to the
block device. A new filesystem is created with the
.B mkfs
@@ -15,6 +15,12 @@ general mount options described in the
.BR mount (8)
manual page.
.TP
.B metadev_path=<device>
The metadev_path option specifies the path to the block device that
contains the filesystem's metadata.
.sp
This option is required.
.TP
.B server_addr=<ipv4:port>
The server_addr option indicates that this mount will participate in
quorum election to try and run a server for all the mounts of its
+27 -15
View File
@@ -149,23 +149,24 @@ user must have read permission to the inode.
.PD
.TP
.BI "mkfs <\-Q nr> <path>"
.BI "mkfs <\-Q nr> <meta_dev_path> <data_dev_path> [-M meta_size] [-D data_size]"
.sp
Initialize a new empty filesystem in the target device by writing empty
structures and a new superblock.
Initialize a new empty filesystem in the target devices by writing empty
structures and a new superblock. Since ScoutFS uses separate block
devices for its metadata and data storage, both must be given.
.sp
This
.B unconditionally destroys
the contents of the device, regardless of what it contains or who may be
using it. It simply writes new data structures into known offsets.
.B Be very careful that the device does not contain data and is not actively in use.
the contents of the devices, regardless of what they contain or who may be
using them. It simply writes new data structures into known offsets.
.B Be very careful that the devices do not contain data and are not actively in use.
.RS 1.0i
.PD 0
.TP
.sp
.B "-Q nr"
Specify the number of mounts needed to reach quorum and elect a mount
to start the server. Mounts of the device will hang until this many
to start the server. Mounts of the filesystem will hang until this many
mounts are operational and can elect a server amongst themselves.
.sp
Mounts with the
@@ -184,13 +185,24 @@ elected servers race to fence each other and can have the unlikely
outcome of continually racing to fence each other resulting in a
persistent loss of service.
.TP
.B "-S 4KB_blocks"
Limit the device size used by the filesystem to the given size in units
of 4KB blocks. It must be larger than the mkfs minimum size and fit
within the device.
.B "meta_dev_path"
The path to the device to be used for ScoutFS metadata. If possible,
use a faster block device for the metadata device. Its contents will be
unconditionally destroyed.
.TP
.B "path"
The path to the device whose contents will be unconditionally destroyed.
.B "data_dev_path"
The path to the device to be used for ScoutFS file data. If possible,
use a larger block device for the data device. Its contents will be
unconditionally destroyed.
.TP
.B "-M meta_size"
Limit the space used by the filesystem on the metadata device to the
given size, rather than using the entire block device. Size is given as
an integer followed by a units digit: "K", "M", "G", "T", "P", to denote
kibibytes, mebibytes, etc.
.TP
.B "-D data_size"
Same as previous, but for limiting the size of the data device.
.RE
.PD
@@ -206,11 +218,11 @@ output.
.TP
.sp
.B "path"
The path to the device that contains the filesystem whose metadata will
The path to the metadata device for filesystem whose metadata will
be printed. The command reads from the buffer cache of the device which
may not reflect the current blocks in the filesystem that may have been
written through another host or device. The local device's cache can be
manually flused before printing, perhaps with the
manually flushed before printing, perhaps with the
.B \--flushbufs
command in the
.BR blockdev (8)
+83 -24
View File
@@ -10,33 +10,92 @@
#include "sparse.h"
#include "dev.h"
int device_size(char *path, int fd, u64 *size)
int device_size(char *path, int fd,
u64 min_size, u64 max_size,
char *use_type, u64 *size_ret)
{
struct stat st;
int ret;
struct stat st;
u64 size;
char *target_type;
int ret;
if (fstat(fd, &st)) {
ret = -errno;
fprintf(stderr, "failed to stat '%s': %s (%d)\n",
path, strerror(errno), errno);
return ret;
}
if (fstat(fd, &st)) {
ret = -errno;
fprintf(stderr, "failed to stat '%s': %s (%d)\n",
path, strerror(errno), errno);
return ret;
}
if (S_ISREG(st.st_mode)) {
*size = st.st_size;
} else if (S_ISBLK(st.st_mode)) {
if (ioctl(fd, BLKGETSIZE64, size)) {
ret = -errno;
fprintf(stderr, "BLKGETSIZE64 failed '%s': %s (%d)\n",
path, strerror(errno), errno);
return ret;
}
} else {
fprintf(stderr, "path isn't regular or device file '%s'\n",
path);
return -EINVAL;
}
if (S_ISREG(st.st_mode)) {
size = st.st_size;
target_type = "file";
} else if (S_ISBLK(st.st_mode)) {
if (ioctl(fd, BLKGETSIZE64, &size)) {
ret = -errno;
fprintf(stderr, "BLKGETSIZE64 failed '%s': %s (%d)\n",
path, strerror(errno), errno);
return ret;
}
target_type = "device";
} else {
fprintf(stderr, "path isn't regular or device file '%s'\n",
path);
return -EINVAL;
}
return 0;
if (max_size) {
if (size > max_size) {
printf("Limiting use of "BASE_SIZE_FMT
" %s device to "BASE_SIZE_FMT"\n",
BASE_SIZE_ARGS(size), use_type,
BASE_SIZE_ARGS(max_size));
size = max_size;
} else if (size < max_size) {
printf("Device size limit of "BASE_SIZE_FMT
" for %s device"
" is greater than "BASE_SIZE_FMT
" available, ignored.\n",
BASE_SIZE_ARGS(max_size), use_type,
BASE_SIZE_ARGS(size));
}
}
if (size < min_size) {
fprintf(stderr,
BASE_SIZE_FMT" %s too small for min "
BASE_SIZE_FMT" %s device\n",
BASE_SIZE_ARGS(size), target_type,
BASE_SIZE_ARGS(min_size), use_type);
return -EINVAL;
}
*size_ret = size;
return 0;
}
float size_flt(u64 nr, unsigned size)
{
float x = (float)nr * (float)size;
while (x >= 1024)
x /= 1024;
return x;
}
char *size_str(u64 nr, unsigned size)
{
float x = (float)nr * (float)size;
static char *suffixes[] = {
"B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB",
};
int i = 0;
while (x >= 1024) {
x /= 1024;
i++;
}
return suffixes[i];
}
+11 -1
View File
@@ -1,6 +1,16 @@
#ifndef _DEV_H_
#define _DEV_H_
int device_size(char *path, int fd, u64 *size);
#define BASE_SIZE_FMT "%.2f %s"
#define BASE_SIZE_ARGS(sz) size_flt(sz, 1), size_str(sz, 1)
#define SIZE_FMT "%llu (%.2f %s)"
#define SIZE_ARGS(nr, sz) (nr), size_flt(nr, sz), size_str(nr, sz)
int device_size(char *path, int fd,
u64 min_size, u64 max_size,
char *use_type, u64 *size_ret);
float size_flt(u64 nr, unsigned size);
char *size_str(u64 nr, unsigned size);
#endif
+9
View File
@@ -61,6 +61,12 @@
#define SCOUTFS_QUORUM_BLKNO ((256ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT)
#define SCOUTFS_QUORUM_BLOCKS ((256ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT)
/*
* Start data on the data device aligned as well.
*/
#define SCOUTFS_DATA_DEV_START_BLKNO ((256ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT)
#define SCOUTFS_UNIQUE_NAME_MAX_BYTES 64 /* includes null */
/*
@@ -585,10 +591,13 @@ struct scoutfs_quorum_block {
((SCOUTFS_BLOCK_SM_SIZE - sizeof(struct scoutfs_quorum_block)) / \
sizeof(struct scoutfs_quorum_log))
#define SCOUTFS_FLAG_IS_META_BDEV 0x01
struct scoutfs_super_block {
struct scoutfs_block_header hdr;
__le64 id;
__le64 format_hash;
__le64 flags;
__u8 uuid[SCOUTFS_UUID_BYTES];
__le64 next_ino;
__le64 next_trans_seq;
+85 -86
View File
@@ -16,6 +16,7 @@
#include <netinet/in.h>
#include <arpa/inet.h>
#include <ctype.h>
#include <inttypes.h>
#include "sparse.h"
#include "cmd.h"
@@ -62,35 +63,6 @@ static int write_block(int fd, u64 blkno, int shift,
return write_raw_block(fd, blkno, shift, hdr);
}
static float size_flt(u64 nr, unsigned size)
{
float x = (float)nr * (float)size;
while (x >= 1024)
x /= 1024;
return x;
}
static char *size_str(u64 nr, unsigned size)
{
float x = (float)nr * (float)size;
static char *suffixes[] = {
"B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB",
};
int i = 0;
while (x >= 1024) {
x /= 1024;
i++;
}
return suffixes[i];
}
#define SIZE_FMT "%llu (%.2f %s)"
#define SIZE_ARGS(nr, sz) (nr), size_flt(nr, sz), size_str(nr, sz)
/*
* Write the single btree block that contains the blkno and len indexed
* items to store the given extent, and update the root to point to it.
@@ -132,8 +104,14 @@ static int write_alloc_root(struct scoutfs_super_block *super, int fd,
* - super blocks
* - btree ring blocks with manifest and allocator btree blocks
* - segment with root inode items
*
* Superblock is written to both metadata and data devices, everything else is
* written only to the metadata device.
*/
static int write_new_fs(char *path, int fd, u8 quorum_count, u64 dev_blocks)
static int write_new_fs(char *meta_path, char *data_path,
int meta_fd, int data_fd,
u8 quorum_count,
u64 max_meta_size, u64 max_data_size)
{
struct scoutfs_super_block *super;
struct scoutfs_inode inode;
@@ -144,8 +122,8 @@ static int write_new_fs(char *path, int fd, u8 quorum_count, u64 dev_blocks)
char uuid_str[37];
void *zeros;
u64 blkno;
u64 limit;
u64 size;
u64 meta_size;
u64 data_size;
u64 next_meta;
u64 last_meta;
u64 first_data;
@@ -167,40 +145,24 @@ static int write_new_fs(char *path, int fd, u8 quorum_count, u64 dev_blocks)
goto out;
}
ret = device_size(path, fd, &size);
if (ret) {
fprintf(stderr, "failed to stat '%s': %s (%d)\n",
path, strerror(errno), errno);
ret = device_size(meta_path, meta_fd, 2ULL * (1024 * 1024 * 1024),
max_meta_size, "meta", &meta_size);
if (ret)
goto out;
}
if (dev_blocks > 0 && size < (dev_blocks << SCOUTFS_BLOCK_SM_SHIFT)) {
fprintf(stderr, "device size limit %llu in 4KB blocks given with -S is greater than device byte size %llu\n",
dev_blocks, size);
ret = -EINVAL;
ret = device_size(data_path, data_fd, 8ULL * (1024 * 1024 * 1024),
max_data_size, "data", &data_size);
if (ret)
goto out;
}
if (dev_blocks > 0 && size > (dev_blocks << SCOUTFS_BLOCK_SM_SHIFT))
size = dev_blocks << SCOUTFS_BLOCK_SM_SHIFT;
/* arbitrarily require a reasonably large device */
limit = 8ULL * (1024 * 1024 * 1024);
if (size < limit) {
fprintf(stderr, "%llu byte device too small for min %llu byte fs\n",
size, limit);
ret = -EINVAL;
goto out;
}
/* metadata blocks start after the quorum blocks */
next_meta = (SCOUTFS_QUORUM_BLKNO + SCOUTFS_QUORUM_BLOCKS) >>
SCOUTFS_BLOCK_SM_LG_SHIFT;
/* use about 1/5 of the device for metadata blocks */
last_meta = next_meta + ((size / 5) >> SCOUTFS_BLOCK_LG_SHIFT);
/* The rest of the device is data blocks */
first_data = (last_meta + 1) << SCOUTFS_BLOCK_SM_LG_SHIFT;
last_data = (size >> SCOUTFS_BLOCK_SM_SHIFT) - 1;
/* rest of meta dev is available for metadata blocks */
last_meta = (meta_size >> SCOUTFS_BLOCK_LG_SHIFT) - 1;
/* Data blocks go on the data dev */
first_data = SCOUTFS_DATA_DEV_START_BLKNO;
last_data = (data_size >> SCOUTFS_BLOCK_SM_SHIFT) - 1;
/* partially initialize the super so we can use it to init others */
memset(super, 0, SCOUTFS_BLOCK_SM_SIZE);
@@ -249,7 +211,7 @@ static int write_new_fs(char *path, int fd, u8 quorum_count, u64 dev_blocks)
bt->hdr.crc = cpu_to_le32(crc_block(&bt->hdr,
SCOUTFS_BLOCK_LG_SIZE));
ret = write_raw_block(fd, blkno, SCOUTFS_BLOCK_LG_SHIFT, bt);
ret = write_raw_block(meta_fd, blkno, SCOUTFS_BLOCK_LG_SHIFT, bt);
if (ret)
goto out;
@@ -276,13 +238,13 @@ static int write_new_fs(char *path, int fd, u8 quorum_count, u64 dev_blocks)
super->server_meta_avail[0].first_nr = lblk->nr;
lblk->hdr.crc = cpu_to_le32(crc_block(&bt->hdr, SCOUTFS_BLOCK_LG_SIZE));
ret = write_raw_block(fd, blkno, SCOUTFS_BLOCK_LG_SHIFT, lblk);
ret = write_raw_block(meta_fd, blkno, SCOUTFS_BLOCK_LG_SHIFT, lblk);
if (ret)
goto out;
/* the data allocator has a single extent */
blkno = next_meta++;
ret = write_alloc_root(super, fd, &super->data_alloc, bt,
ret = write_alloc_root(super, meta_fd, &super->data_alloc, bt,
blkno, first_data,
le64_to_cpu(super->total_data_blocks));
if (ret < 0)
@@ -300,7 +262,7 @@ static int write_new_fs(char *path, int fd, u8 quorum_count, u64 dev_blocks)
/* each meta alloc root contains a portion of free metadata extents */
for (i = 0; i < array_size(super->meta_alloc); i++) {
blkno = next_meta++;
ret = write_alloc_root(super, fd, &super->meta_alloc[i], bt,
ret = write_alloc_root(super, meta_fd, &super->meta_alloc[i], bt,
blkno, meta_start,
min(meta_len,
last_meta - meta_start + 1));
@@ -312,7 +274,7 @@ static int write_new_fs(char *path, int fd, u8 quorum_count, u64 dev_blocks)
/* zero out quorum blocks */
for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) {
ret = write_raw_block(fd, SCOUTFS_QUORUM_BLKNO + i,
ret = write_raw_block(meta_fd, SCOUTFS_QUORUM_BLKNO + i,
SCOUTFS_BLOCK_SM_SHIFT, zeros);
if (ret < 0) {
fprintf(stderr, "error zeroing quorum block: %s (%d)\n",
@@ -321,31 +283,46 @@ static int write_new_fs(char *path, int fd, u8 quorum_count, u64 dev_blocks)
}
}
/* write the super block */
/* write the super block to data dev and meta dev*/
super->hdr.seq = cpu_to_le64(1);
ret = write_block(fd, SCOUTFS_SUPER_BLKNO, SCOUTFS_BLOCK_SM_SHIFT,
ret = write_block(data_fd, SCOUTFS_SUPER_BLKNO, SCOUTFS_BLOCK_SM_SHIFT,
NULL, &super->hdr);
if (ret)
goto out;
if (fsync(fd)) {
if (fsync(data_fd)) {
ret = -errno;
fprintf(stderr, "failed to fsync '%s': %s (%d)\n",
path, strerror(errno), errno);
data_path, strerror(errno), errno);
goto out;
}
super->flags |= cpu_to_le64(SCOUTFS_FLAG_IS_META_BDEV);
ret = write_block(meta_fd, SCOUTFS_SUPER_BLKNO, SCOUTFS_BLOCK_SM_SHIFT,
NULL, &super->hdr);
if (ret)
goto out;
if (fsync(meta_fd)) {
ret = -errno;
fprintf(stderr, "failed to fsync '%s': %s (%d)\n",
meta_path, strerror(errno), errno);
goto out;
}
uuid_unparse(super->uuid, uuid_str);
printf("Created scoutfs filesystem:\n"
" device path: %s\n"
" meta device path: %s\n"
" data device path: %s\n"
" fsid: %llx\n"
" format hash: %llx\n"
" uuid: %s\n"
" 64KB metadata blocks: "SIZE_FMT"\n"
" 4KB data blocks: "SIZE_FMT"\n"
" quorum count: %u\n",
path,
meta_path,
data_path,
le64_to_cpu(super->hdr.fsid),
le64_to_cpu(super->format_hash),
uuid_str,
@@ -374,15 +351,18 @@ static struct option long_ops[] = {
static int mkfs_func(int argc, char *argv[])
{
unsigned long long ull;
char *path = argv[1];
u8 quorum_count = 0;
u64 dev_blocks = 0;
u64 max_data_size = 0;
u64 max_meta_size = 0;
char *end = NULL;
char *meta_path;
char *data_path;
int meta_fd;
int data_fd;
int ret;
int fd;
int c;
while ((c = getopt_long(argc, argv, "Q:S:", long_ops, NULL)) != -1) {
while ((c = getopt_long(argc, argv, "Q:D:M:", long_ops, NULL)) != -1) {
switch (c) {
case 'Q':
ull = strtoull(optarg, &end, 0);
@@ -394,10 +374,18 @@ static int mkfs_func(int argc, char *argv[])
}
quorum_count = ull;
break;
case 'S':
ret = parse_u64(optarg, &dev_blocks);
case 'D':
ret = parse_human(optarg, &max_data_size);
if (ret < 0) {
printf("scoutfs: invalid device blocks count '%s'\n",
printf("scoutfs: invalid data device size '%s'\n",
optarg);
return ret;
}
break;
case 'M':
ret = parse_human(optarg, &max_meta_size);
if (ret < 0) {
printf("scoutfs: invalid meta device size '%s'\n",
optarg);
return ret;
}
@@ -408,28 +396,39 @@ static int mkfs_func(int argc, char *argv[])
}
}
if (optind >= argc) {
printf("scoutfs: mkfs: a single path argument is required\n");
if (optind + 2 != argc) {
printf("scoutfs: mkfs: paths to metadata and data devices are required\n");
return -EINVAL;
}
path = argv[optind];
meta_path = argv[optind];
data_path = argv[optind + 1];
if (!quorum_count) {
printf("provide quorum count with --quorum_count|-Q option\n");
return -EINVAL;
}
fd = open(path, O_RDWR | O_EXCL);
if (fd < 0) {
meta_fd = open(meta_path, O_RDWR | O_EXCL);
if (meta_fd < 0) {
ret = -errno;
fprintf(stderr, "failed to open '%s': %s (%d)\n",
path, strerror(errno), errno);
fprintf(stderr, "failed to open metadata device '%s': %s (%d)\n",
meta_path, strerror(errno), errno);
return ret;
}
ret = write_new_fs(path, fd, quorum_count, dev_blocks);
close(fd);
data_fd = open(data_path, O_RDWR | O_EXCL);
if (data_fd < 0) {
ret = -errno;
fprintf(stderr, "failed to open data device '%s': %s (%d)\n",
data_path, strerror(errno), errno);
return ret;
}
ret = write_new_fs(meta_path, data_path, meta_fd, data_fd,
quorum_count, max_meta_size, max_data_size);
close(meta_fd);
close(data_fd);
return ret;
}
+64
View File
@@ -10,6 +10,70 @@
#include "parse.h"
/*
* Convert size with multiplicative suffix to bytes.
* e.g. "40M", "10G", "4T"
*
* These are powers-of-two prefixes - K means 1024 not 1000.
*
* One can go pretty far with variations but keeping relatively simple for
* now: commas, decimals, and multichar suffixes not handled.
*/
int parse_human(char* str, u64 *val_ret)
{
unsigned long long ull;
char *endptr = NULL;
int sh;
int ret = 0;
ull = strtoull(str, &endptr, 0);
if (((ull == LLONG_MIN || ull == LLONG_MAX) &&
errno == ERANGE)) {
fprintf(stderr, "invalid 64bit value: '%s'\n", str);
*val_ret = 0;
ret = -EINVAL;
goto error;
}
switch (*endptr) {
case 'K':
sh = 10;
break;
case 'M':
sh = 20;
break;
case 'G':
sh = 30;
break;
case 'T':
sh = 40;
break;
case 'P':
sh = 50;
break;
case '\0':
sh = 0;
break;
default:
fprintf(stderr, "unknown suffix: '%s'\n", endptr);
ret = -ERANGE;
goto error;
}
if (ull > (SIZE_MAX >> sh)) {
fprintf(stderr, "size too big: '%s'\n", str);
ret = -ERANGE;
goto error;
}
ull <<= sh;
*val_ret = ull;
error:
return ret;
}
int parse_u64(char *str, u64 *val_ret)
{
unsigned long long ull;
+1
View File
@@ -3,6 +3,7 @@
#include <sys/time.h>
int parse_human(char* str, u64 *val_ret);
int parse_u64(char *str, u64 *val_ret);
int parse_s64(char *str, s64 *val_ret);
int parse_u32(char *str, u32 *val_ret);
+1
View File
@@ -880,6 +880,7 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno)
print_block_header(&super->hdr, SCOUTFS_BLOCK_SM_SIZE);
printf(" format_hash %llx uuid %s\n",
le64_to_cpu(super->format_hash), uuid_str);
printf(" flags: 0x%016llx\n", super->flags);
server_addr = alloc_addr_str(&super->server_addr);
if (!server_addr)