diff --git a/utils/src/crc.c b/utils/src/crc.c index 8380cf37..1d027a32 100644 --- a/utils/src/crc.c +++ b/utils/src/crc.c @@ -38,9 +38,15 @@ u32 crc_block(struct scoutfs_block_header *hdr) SCOUTFS_BLOCK_SIZE - sizeof(hdr->crc)); } -u32 crc_ring_block(struct scoutfs_ring_block *rblk) +u32 crc_btree_block(struct scoutfs_btree_block *bt) { - unsigned long skip = (char *)(&rblk->crc + 1) - (char *)rblk; + __le32 old; + u32 crc; - return crc32c(~0, (char *)rblk + skip, SCOUTFS_BLOCK_SIZE - skip); + old = bt->crc; + bt->crc = 0; + crc = crc32c(~0, bt, SCOUTFS_BLOCK_SIZE); + bt->crc = old; + + return crc; } diff --git a/utils/src/crc.h b/utils/src/crc.h index 1006871e..03ce2891 100644 --- a/utils/src/crc.h +++ b/utils/src/crc.h @@ -8,6 +8,6 @@ u32 crc32c(u32 crc, const void *data, unsigned int len); u64 crc32c_64(u32 crc, const void *data, unsigned int len); u32 crc_block(struct scoutfs_block_header *hdr); -u32 crc_ring_block(struct scoutfs_ring_block *rblk); +u32 crc_btree_block(struct scoutfs_btree_block *bt); #endif diff --git a/utils/src/format.h b/utils/src/format.h index a59ca4ca..b1bbdbbd 100644 --- a/utils/src/format.h +++ b/utils/src/format.h @@ -7,7 +7,7 @@ #define SCOUTFS_SUPER_ID 0x2e736674756f6373ULL /* "scoutfs." */ /* - * The super block and ring blocks are fixed 4k. + * The super block and btree blocks are fixed 4k. */ #define SCOUTFS_BLOCK_SHIFT 12 #define SCOUTFS_BLOCK_SIZE (1 << SCOUTFS_BLOCK_SHIFT) @@ -50,30 +50,97 @@ struct scoutfs_block_header { __le64 blkno; } __packed; -struct scoutfs_ring_entry { - __le16 data_len; - __u8 flags; +/* + * The largest possible btree has 2^64 bytes worth of segments with + * the largest possible keys in a pathologically sparse btree where + * all the nodes are half full. + */ + +/* + * Assert that we'll be able to represent all possible keys with 8 64bit + * primary sort values. + */ +#define SCOUTFS_BTREE_GREATEST_KEY_LEN 32 +/* level >0 segments can have a full key and some metadata */ +#define SCOUTFS_BTREE_MAX_KEY_LEN 320 +/* level 0 segments can have two full keys in the value :/ */ +#define SCOUTFS_BTREE_MAX_VAL_LEN 768 + +/* + * A 4EB test image measured a worst case height of 17. This is plenty + * generous. + */ +#define SCOUTFS_BTREE_MAX_HEIGHT 20 + +/* btree blocks (beyond the first) need to be at least half full */ +#define SCOUTFS_BTREE_FREE_LIMIT \ + ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block)) / 2) + +#define SCOUTFS_BTREE_BITS 8 + +/* + * Btree items can have bits associated with them. Their parent items + * reflect all the bits that their child block contain. Thus searches + * can find items with bits set. + * + * @SCOUTFS_BTREE_BIT_HALF1: Tracks blocks found in the first half of + * the ring. It's used to migrate blocks from the old half of the ring + * into the current half as blocks are dirtied. It's not found in leaf + * items but is calculated based on the block number of referenced + * blocks. _HALF2 is identical but for the second half of the ring. + */ +enum { + SCOUTFS_BTREE_BIT_HALF1 = (1 << 0), + SCOUTFS_BTREE_BIT_HALF2 = (1 << 1), +}; + +#define SCOUTFS_BTREE_HALF_BITS \ + (SCOUTFS_BTREE_BIT_HALF1 | SCOUTFS_BTREE_BIT_HALF2) + +struct scoutfs_btree_ref { + __le64 blkno; + __le64 seq; +} __packed; + +/* + * A height of X means that the first block read will have level X-1 and + * the leaves will have level 0. + */ +struct scoutfs_btree_root { + struct scoutfs_btree_ref ref; + __u8 height; +} __packed; + +struct scoutfs_btree_item_header { + __le16 off; + __u8 bits; +} __packed; + +struct scoutfs_btree_item { + __le16 key_len; + __le16 val_len; __u8 data[0]; } __packed; -#define SCOUTFS_RING_ENTRY_FLAG_DELETION (1 << 0) - -struct scoutfs_ring_block { - __le32 crc; - __le32 pad; +struct scoutfs_btree_block { __le64 fsid; + __le64 blkno; __le64 seq; - __le64 block; - __le32 nr_entries; - struct scoutfs_ring_entry entries[0]; + __le32 crc; + __le32 _pad; + __le16 free_end; + __le16 free_reclaim; + __le16 nr_items; + __le16 bit_counts[SCOUTFS_BTREE_BITS]; + __u8 level; + struct scoutfs_btree_item_header item_hdrs[0]; } __packed; -struct scoutfs_ring_descriptor { - __le64 blkno; - __le64 total_blocks; - __le64 first_block; - __le64 first_seq; +struct scoutfs_btree_ring { + __le64 first_blkno; __le64 nr_blocks; + __le64 next_block; + __le64 next_seq; } __packed; /* @@ -85,16 +152,39 @@ struct scoutfs_ring_descriptor { #define SCOUTFS_MANIFEST_FANOUT 10 struct scoutfs_manifest { - struct scoutfs_ring_descriptor ring; + struct scoutfs_btree_root root; __le64 level_counts[SCOUTFS_MANIFEST_MAX_LEVEL]; } __packed; -struct scoutfs_manifest_entry { +/* + * Manifest entries are packed into btree keys and values in a very + * fiddly way so that we can sort them with memcmp first by level then + * by their position in the level. First comes the level. + * + * Level 0 segments are sorted by their seq so they don't have the first + * segment key in the manifest btree key. Both of their keys are in the + * value. + * + * Level 1 segments are sorted by their key before their seq so the + * btree header has the key and the seq is in the footer. Only their + * last key is in the value. + * + * We go to all this trouble so that we can communicate a version of the + * manifest with one btree root, have dense btree keys which are used as + * seperators in parent blocks, and don't duplicate the large keys in + * the manifest btree key and value. + */ + +struct scoutfs_manifest_btree_key { + __u8 level; + __u8 bkey[0]; +} __packed; + +struct scoutfs_manifest_btree_val { __le64 segno; __le64 seq; __le16 first_key_len; __le16 last_key_len; - __u8 level; __u8 keys[0]; } __packed; @@ -102,12 +192,12 @@ struct scoutfs_manifest_entry { #define SCOUTFS_ALLOC_REGION_BITS (1 << SCOUTFS_ALLOC_REGION_SHIFT) #define SCOUTFS_ALLOC_REGION_MASK (SCOUTFS_ALLOC_REGION_BITS - 1) -/* - * The bits need to be aligned so that the host can use native long - * bitops on the bits in memory. - */ -struct scoutfs_alloc_region { - __le64 index; +struct scoutfs_alloc_region_btree_key { + __be64 index; +} __packed; + +/* The bits need to be aligned so that the hosts can use native long bit ops */ +struct scoutfs_alloc_region_btree_val { __le64 bits[SCOUTFS_ALLOC_REGION_BITS / 64]; } __packed; @@ -270,8 +360,6 @@ struct scoutfs_symlink_key { __u8 nr; } __packed; -#define SCOUTFS_SYMLINK_MAX_VAL_SIZE 200 - struct scoutfs_betimespec { __be64 sec; __be32 nsec; @@ -289,12 +377,14 @@ struct scoutfs_inode_index_key { #define SCOUTFS_UUID_BYTES 16 +/* XXX ipv6 */ +struct scoutfs_inet_addr { + __le32 addr; + __le16 port; +} __packed; + +#define SCOUTFS_DEFAULT_PORT 12345 -/* - * The ring fields describe the statically allocated ring log. The - * head and tail indexes are logical 4k blocks offsets inside the ring. - * The head block should contain the seq. - */ struct scoutfs_super_block { struct scoutfs_block_header hdr; __le64 id; @@ -304,13 +394,11 @@ struct scoutfs_super_block { __le64 alloc_uninit; __le64 total_segs; __le64 free_segs; - __le64 ring_blkno; - __le64 ring_blocks; - __le64 ring_tail_block; - __le64 ring_gen; + struct scoutfs_btree_ring bring; __le64 next_seg_seq; - struct scoutfs_ring_descriptor alloc_ring; + struct scoutfs_btree_root alloc_root; struct scoutfs_manifest manifest; + struct scoutfs_inet_addr server_addr; } __packed; #define SCOUTFS_ROOT_INO 1 @@ -379,13 +467,6 @@ struct scoutfs_dirent { /* S32_MAX avoids the (int) sign bit and might avoid sloppy bugs */ #define SCOUTFS_LINK_MAX S32_MAX -#define SCOUTFS_XATTR_MAX_NAME_LEN 255 -#define SCOUTFS_XATTR_MAX_SIZE 65536 -#define SCOUTFS_XATTR_PART_SIZE \ - (SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_xattr_val_header)) -#define SCOUTFS_XATTR_MAX_PARTS \ - DIV_ROUND_UP(SCOUTFS_XATTR_MAX_SIZE, SCOUTFS_XATTR_PART_SIZE) - /* entries begin after . and .. */ #define SCOUTFS_DIRENT_FIRST_POS 2 /* getdents returns next pos with an entry, no entry at (f_pos)~0 */ @@ -410,16 +491,18 @@ enum { #define SCOUTFS_MAX_VAL_SIZE \ offsetof(struct scoutfs_dirent, name[SCOUTFS_NAME_LEN]) +#define SCOUTFS_XATTR_MAX_NAME_LEN 255 +#define SCOUTFS_XATTR_MAX_SIZE 65536 +#define SCOUTFS_XATTR_PART_SIZE \ + (SCOUTFS_MAX_VAL_SIZE - sizeof(struct scoutfs_xattr_val_header)) +#define SCOUTFS_XATTR_MAX_PARTS \ + DIV_ROUND_UP(SCOUTFS_XATTR_MAX_SIZE, SCOUTFS_XATTR_PART_SIZE) + + /* * messages over the wire. */ -/* XXX ipv6 */ -struct scoutfs_inet_addr { - __le32 addr; - __le16 port; -} __packed; - /* * This header precedes and describes all network messages sent over * sockets. The id is set by the request and sent in the reply. The @@ -450,9 +533,13 @@ struct scoutfs_net_key_range { __u8 key_bytes[0]; } __packed; -struct scoutfs_net_manifest_entries { - __le16 nr; - struct scoutfs_manifest_entry ments[0]; +struct scoutfs_net_manifest_entry { + __le64 segno; + __le64 seq; + __le16 first_key_len; + __le16 last_key_len; + __u8 level; + __u8 keys[0]; } __packed; /* XXX I dunno, totally made up */ @@ -475,12 +562,12 @@ struct scoutfs_net_segnos { enum { SCOUTFS_NET_ALLOC_INODES = 0, - SCOUTFS_NET_MANIFEST_RANGE_ENTRIES, SCOUTFS_NET_ALLOC_SEGNO, SCOUTFS_NET_RECORD_SEGMENT, SCOUTFS_NET_BULK_ALLOC, SCOUTFS_NET_ADVANCE_SEQ, SCOUTFS_NET_GET_LAST_SEQ, + SCOUTFS_NET_GET_MANIFEST_ROOT, SCOUTFS_NET_UNKNOWN, }; diff --git a/utils/src/mkfs.c b/utils/src/mkfs.c index 07723de2..1388009e 100644 --- a/utils/src/mkfs.c +++ b/utils/src/mkfs.c @@ -47,30 +47,106 @@ static int write_block(int fd, u64 blkno, struct scoutfs_super_block *super, } /* - * Figure out how many blocks a given ring will need given a max number - * of entries up to a given max size. We figure out how many blocks it - * could take to store these maximal entries given unused tail space and - * block header overheads. Then we (wastefully) multiply by three to - * ensure that the ring won't consume itself as it wraps. The caller - * aligns the ring size to a segment size depending on where it starts. + * Calculate the greatest number of btree blocks that might be needed to + * store the given item population. At most all blocks will be half + * full. All keys will be the max size including parent items which + * determines the fanout. + * + * We will never hit this in practice. But some joker *could* fill a + * filesystem with empty files with enormous file names. */ -static u64 calc_ring_blocks(u64 max_nr, u64 max_size) +static u64 calc_btree_blocks(u64 nr, u64 max_key, u64 max_val) { - u64 block_bytes; + u64 item_bytes; + u64 fanout; + u64 block_items; + u64 leaf_blocks; + u64 level_blocks; + u64 total_blocks; - max_size += sizeof(struct scoutfs_ring_entry); + /* figure out the parent fanout for these silly huge possible items */ + item_bytes = sizeof(struct scoutfs_btree_item_header) + + sizeof(struct scoutfs_btree_item) + + max_key + sizeof(struct scoutfs_btree_ref); + fanout = (SCOUTFS_BLOCK_SIZE - SCOUTFS_BTREE_FREE_LIMIT) / item_bytes; - block_bytes = SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_ring_block) - - (max_size - 1); + /* figure out how many items we have to store */ + item_bytes = sizeof(struct scoutfs_btree_item_header) + + sizeof(struct scoutfs_btree_item) + + max_key + max_val; + block_items = (SCOUTFS_BLOCK_SIZE - SCOUTFS_BTREE_FREE_LIMIT) / item_bytes; + leaf_blocks = DIV_ROUND_UP(nr, block_items); - return DIV_ROUND_UP(max_nr * max_size, block_bytes) * 3; + /* then calc total blocks as we grow to have enough blocks for items */ + level_blocks = 1; + total_blocks = level_blocks; + while (level_blocks < leaf_blocks) { + level_blocks *= fanout; + level_blocks = min(leaf_blocks, level_blocks); + total_blocks += level_blocks; + } + + return total_blocks; } +/* + * Figure out how many btree ring blocks we'll need for all the btree + * items that could be needed to describe this many segments. The + * allocator regions are nice and dense but the manifest entries can be + * absolutely enormous. + */ +static u64 calc_btree_ring_blocks(u64 total_segs) +{ + u64 blocks; + + blocks = calc_btree_blocks(DIV_ROUND_UP(total_segs, + SCOUTFS_ALLOC_REGION_BITS), + sizeof(struct scoutfs_alloc_region_btree_key), + sizeof(struct scoutfs_alloc_region_btree_val)); + + blocks += calc_btree_blocks(total_segs, + sizeof(struct scoutfs_manifest_btree_key) + + SCOUTFS_MAX_KEY_SIZE, + sizeof(struct scoutfs_manifest_btree_val) + + SCOUTFS_MAX_KEY_SIZE); + + return round_up(blocks * 4, SCOUTFS_SEGMENT_BLOCKS); +} + +static float size_flt(u64 nr, unsigned size) +{ + float x = (float)nr * (float)size; + + while (x >= 1024) + x /= 1024; + + return x; +} + +static char *size_str(u64 nr, unsigned size) +{ + float x = (float)nr * (float)size; + static char *suffixes[] = { + "B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB", + }; + int i = 0; + + while (x >= 1024) { + x /= 1024; + i++; + } + + return suffixes[i]; +} + +#define SIZE_FMT "%llu (%.2f %s)" +#define SIZE_ARGS(nr, sz) (nr), size_flt(nr, sz), size_str(nr, sz) + /* * Make a new file system by writing: * - super blocks - * - ring block with manifest node - * - segment with root inode + * - btree ring blocks with manifest and allocator btree blocks + * - segment with root inode items */ static int write_new_fs(char *path, int fd) { @@ -79,10 +155,10 @@ static int write_new_fs(char *path, int fd) struct scoutfs_inode_index_key *idx_key; struct scoutfs_inode *inode; struct scoutfs_segment_block *sblk; - struct scoutfs_manifest_entry *ment; - struct scoutfs_ring_descriptor *rdesc; - struct scoutfs_ring_block *rblk; - struct scoutfs_ring_entry *rent; + struct scoutfs_manifest_btree_key *mkey; + struct scoutfs_manifest_btree_val *mval; + struct scoutfs_btree_block *bt; + struct scoutfs_btree_item *btitem; struct scoutfs_segment_item *item; __le32 *prev_link; struct timeval tv; @@ -99,9 +175,9 @@ static int write_new_fs(char *path, int fd) gettimeofday(&tv, NULL); super = calloc(1, SCOUTFS_BLOCK_SIZE); - rblk = calloc(1, SCOUTFS_BLOCK_SIZE); + bt = calloc(1, SCOUTFS_BLOCK_SIZE); sblk = calloc(1, SCOUTFS_SEGMENT_SIZE); - if (!super || !rblk || !sblk) { + if (!super || !bt || !sblk) { ret = -errno; fprintf(stderr, "failed to allocate block mem: %s (%d)\n", strerror(errno), errno); @@ -136,73 +212,66 @@ static int write_new_fs(char *path, int fd) super->total_segs = cpu_to_le64(total_segs); super->next_seg_seq = cpu_to_le64(2); - /* start writing rings after the super */ - blkno = SCOUTFS_SUPER_BLKNO + SCOUTFS_SUPER_NR; - - /* allocator ring is empty, allocations start from super fields */ - ring_blocks = calc_ring_blocks(DIV_ROUND_UP(total_segs, - SCOUTFS_ALLOC_REGION_BITS), - sizeof(struct scoutfs_alloc_region)); - ring_blocks = round_up(blkno + ring_blocks, SCOUTFS_SEGMENT_BLOCKS) - - blkno; - - rdesc = &super->alloc_ring; - rdesc->blkno = cpu_to_le64(blkno); - rdesc->total_blocks = cpu_to_le64(ring_blocks); - rdesc->first_block = cpu_to_le64(0); - rdesc->first_seq = cpu_to_le64(0); - rdesc->nr_blocks = cpu_to_le64(0); - - blkno += ring_blocks; - - /* manifest ring has a block with an entry for the segment */ - ring_blocks = calc_ring_blocks(total_segs, - sizeof(struct scoutfs_manifest_entry) + - (2 * SCOUTFS_MAX_KEY_SIZE)); - ring_blocks = round_up(ring_blocks, SCOUTFS_SEGMENT_BLOCKS); - + /* align the btree ring to the segment after the supers */ + blkno = round_up(SCOUTFS_SUPER_BLKNO + SCOUTFS_SUPER_NR, + SCOUTFS_SEGMENT_BLOCKS); /* first usable segno follows manifest ring */ + ring_blocks = calc_btree_ring_blocks(total_segs); first_segno = (blkno + ring_blocks) / SCOUTFS_SEGMENT_BLOCKS; + super->bring.first_blkno = cpu_to_le64(blkno); + super->bring.nr_blocks = cpu_to_le64(ring_blocks); + super->bring.next_block = cpu_to_le64(1); + super->bring.next_seq = cpu_to_le64(2); + + /* allocator btree is empty, allocations start from super fields */ + super->alloc_root.ref.blkno = cpu_to_le64(0); + super->alloc_root.ref.seq = cpu_to_le64(0); + super->alloc_root.height = 0; + + /* manifest btree has a block with an item for the segment */ + super->manifest.root.ref.blkno = cpu_to_le64(blkno); + super->manifest.root.ref.seq = cpu_to_le64(1); + super->manifest.root.height = 1; super->manifest.level_counts[1] = cpu_to_le64(1); - rdesc = &super->manifest.ring; - rdesc->blkno = cpu_to_le64(blkno); - rdesc->total_blocks = cpu_to_le64(ring_blocks); - rdesc->first_seq = cpu_to_le64(1); - rdesc->nr_blocks = cpu_to_le64(1); + memset(bt, 0, SCOUTFS_BLOCK_SIZE); + bt->fsid = super->hdr.fsid; + bt->blkno = cpu_to_le64(blkno); + bt->seq = cpu_to_le64(1); + bt->nr_items = cpu_to_le16(1); - memset(rblk, 0, SCOUTFS_BLOCK_SIZE); - rblk->pad = 0; - rblk->fsid = super->hdr.fsid; - rblk->seq = cpu_to_le64(1); - rblk->block = 0; - rblk->nr_entries = cpu_to_le32(1); + /* btree item allocated from the back of the block */ + idx_key = (void *)bt + SCOUTFS_BLOCK_SIZE - sizeof(*idx_key); + mval = (void *)idx_key - sizeof(*mval); + ikey = (void *)mval - sizeof(*ikey); + mkey = (void *)ikey - sizeof(*mkey); + btitem = (void *)mkey - sizeof(*btitem); - rent = rblk->entries; - rent->flags = 0; - rent->data_len = cpu_to_le16(sizeof(struct scoutfs_manifest_entry) + - sizeof(struct scoutfs_inode_key) + - sizeof(struct scoutfs_inode_index_key)); + bt->item_hdrs[0].off = cpu_to_le16((long)btitem - (long)bt); + bt->free_end = bt->item_hdrs[0].off; - ment = (void *)rent->data; - ment->segno = cpu_to_le64(first_segno); - ment->seq = cpu_to_le64(1); - ment->first_key_len = cpu_to_le16(sizeof(struct scoutfs_inode_key)); - ment->last_key_len = cpu_to_le16(sizeof(struct scoutfs_inode_index_key)); - ment->level = 1; - ikey = (void *)ment->keys; + btitem->key_len = cpu_to_le16(sizeof(struct scoutfs_manifest_btree_key) + + sizeof(struct scoutfs_inode_key)); + btitem->val_len = cpu_to_le16(sizeof(struct scoutfs_manifest_btree_val) + + sizeof(struct scoutfs_inode_index_key)); + + mkey->level = 1; ikey->type = SCOUTFS_INODE_KEY; ikey->ino = cpu_to_be64(SCOUTFS_ROOT_INO); - idx_key = (void *)(ikey + 1); + + mval->segno = cpu_to_le64(first_segno); + mval->seq = cpu_to_le64(1); + mval->first_key_len = cpu_to_le16(sizeof(struct scoutfs_inode_key)); + mval->last_key_len = cpu_to_le16(sizeof(struct scoutfs_inode_index_key)); idx_key->type = SCOUTFS_INODE_INDEX_META_SEQ_KEY; idx_key->major = cpu_to_be64(0); idx_key->minor = 0; idx_key->ino = cpu_to_be64(SCOUTFS_ROOT_INO); - rblk->crc = cpu_to_le32(crc_ring_block(rblk)); + bt->crc = cpu_to_le32(crc_btree_block(bt)); - ret = write_raw_block(fd, blkno, rblk); + ret = write_raw_block(fd, blkno, bt); if (ret) goto out; blkno += ring_blocks; @@ -304,17 +373,27 @@ static int write_new_fs(char *path, int fd) uuid_unparse(super->uuid, uuid_str); printf("Created scoutfs filesystem:\n" - " fsid: %llx\n" - " uuid: %s\n", + " device path: %s\n" + " fsid: %llx\n" + " uuid: %s\n" + " device bytes: "SIZE_FMT"\n" + " btree ring blocks: "SIZE_FMT"\n" + " usable segments: "SIZE_FMT"\n", + path, le64_to_cpu(super->hdr.fsid), - uuid_str); + uuid_str, + SIZE_ARGS(size, 1), + SIZE_ARGS(le64_to_cpu(super->bring.nr_blocks), + SCOUTFS_BLOCK_SIZE), + SIZE_ARGS(le64_to_cpu(super->free_segs) + 1, + SCOUTFS_SEGMENT_SIZE)); ret = 0; out: if (super) free(super); - if (rblk) - free(rblk); + if (bt) + free(bt); if (sblk) free(sblk); return ret; diff --git a/utils/src/print.c b/utils/src/print.c index 2e2185af..6462e161 100644 --- a/utils/src/print.c +++ b/utils/src/print.c @@ -338,116 +338,189 @@ static int print_segments(int fd, unsigned long *seg_map, u64 total) return 0; } -static void print_ring_descriptor(struct scoutfs_ring_descriptor *rdesc, - char *which) +static int print_manifest_entry(void *key, unsigned key_len, void *val, + unsigned val_len, void *arg) { - printf(" %s ring:\n blkno %llu total_blocks %llu first_block %llu " - "first_seq %llu nr_blocks %llu\n", - which, le64_to_cpu(rdesc->blkno), - le64_to_cpu(rdesc->total_blocks), - le64_to_cpu(rdesc->first_block), - le64_to_cpu(rdesc->first_seq), - le64_to_cpu(rdesc->nr_blocks)); -} - -static int print_manifest_entry(int fd, struct scoutfs_ring_entry *rent, - void *arg) -{ - struct scoutfs_manifest_entry *ment = (void *)rent->data; + struct scoutfs_manifest_btree_key *mkey = key; + struct scoutfs_manifest_btree_val *mval = val; unsigned long *seg_map = arg; + unsigned first_len; + unsigned last_len; + void *first; + void *last; + __be64 seq; - printf(" segno %llu seq %llu first_len %u last_len %u level %u\n", - le64_to_cpu(ment->segno), - le64_to_cpu(ment->seq), - le16_to_cpu(ment->first_key_len), - le16_to_cpu(ment->last_key_len), - ment->level); - printf(" first: "); - print_key(ment->keys, le16_to_cpu(ment->first_key_len)); - printf("\n last: "); - print_key(ment->keys + le16_to_cpu(ment->first_key_len), - le16_to_cpu(ment->last_key_len)); - printf("\n"); - - if (rent->flags & SCOUTFS_RING_ENTRY_FLAG_DELETION) - clear_bit(seg_map, le64_to_cpu(ment->segno)); - else - set_bit(seg_map, le64_to_cpu(ment->segno)); - - return 0; -} - -static int print_alloc_region(int fd, struct scoutfs_ring_entry *rent, - void *arg) -{ - struct scoutfs_alloc_region *reg = (void *)rent->data; - int i; - - printf(" index %llu bits", le64_to_cpu(reg->index)); - for (i = 0; i < array_size(reg->bits); i++) - printf(" %016llx", le64_to_cpu(reg->bits[i])); - printf("\n"); - - return 0; -} - -typedef int (*rent_func)(int fd, struct scoutfs_ring_entry *rent, void *arg); - -static int print_ring(int fd, struct scoutfs_super_block *super, - char *which, struct scoutfs_ring_descriptor *rdesc, - rent_func func, void *arg) -{ - struct scoutfs_ring_block *rblk; - struct scoutfs_ring_entry *rent; - u64 block; - u64 blkno; - int ret; - u64 i; - u32 e; - - block = le64_to_cpu(rdesc->first_block); - for (i = 0; i < le64_to_cpu(rdesc->nr_blocks); i++) { - blkno = le64_to_cpu(rdesc->blkno) + block; - - rblk = read_block(fd, blkno); - if (!rblk) - return -ENOMEM; - - printf("%s ring blkno %llu\n" - " crc %08x fsid %llx seq %llu block %llu " - "nr_entries %u\n", - which, blkno, le32_to_cpu(rblk->crc), - le64_to_cpu(rblk->fsid), - le64_to_cpu(rblk->seq), - le64_to_cpu(rblk->block), - le32_to_cpu(rblk->nr_entries)); - - rent = rblk->entries; - for (e = 0; e < le32_to_cpu(rblk->nr_entries); e++) { - - printf(" entry [%u] off %lu data_len %u flags %x\n", - e, (char *)rent - (char *)rblk->entries, - le16_to_cpu(rent->data_len), rent->flags); - - ret = func(fd, rent, arg); - if (ret) { - free(rblk); - return ret; - } - - rent = (void *)&rent->data[le16_to_cpu(rent->data_len)]; + /* parent items only have the key */ + if (val == NULL) { + if (mkey->level == 0) { + memcpy(&seq, mkey->bkey, sizeof(seq)); + printf(" level %u seq %llu\n", + mkey->level, be64_to_cpu(seq)); + } else { + printf(" level %u first ", mkey->level); + print_key(mkey->bkey, key_len - sizeof(mkey->level)); + printf("\n"); } - - block++; - if (block == le64_to_cpu(rdesc->total_blocks)) - block = 0; - - free(rblk); + return 0; } + /* leaf items print the whole entry */ + first_len = le16_to_cpu(mval->first_key_len); + last_len = le16_to_cpu(mval->last_key_len); + + if (mkey->level == 0) { + first = mval->keys; + last = mval->keys + first_len; + } else { + first = mkey->bkey; + last = mval->keys; + } + + printf(" level %u segno %llu seq %llu first_len %u last_len %u\n", + mkey->level, le64_to_cpu(mval->segno), le64_to_cpu(mval->seq), + first_len, last_len); + + printf(" first "); + print_key(first, first_len); + printf("\n last "); + print_key(last, last_len); + printf("\n"); + + set_bit(seg_map, le64_to_cpu(mval->segno)); + return 0; } +static int print_alloc_region(void *key, unsigned key_len, void *val, + unsigned val_len, void *arg) +{ + struct scoutfs_alloc_region_btree_key *reg_key = key; + struct scoutfs_alloc_region_btree_val *reg_val = val; + int i; + + /* XXX check sizes */ + + printf(" index %llu bits", be64_to_cpu(reg_key->index)); + + if (val == NULL) + return 0; + + for (i = 0; i < array_size(reg_val->bits); i++) + printf(" %016llx", le64_to_cpu(reg_val->bits[i])); + printf("\n"); + + return 0; +} + +typedef int (*print_item_func)(void *key, unsigned key_len, void *val, + unsigned val_len, void *arg); + +static int print_btree_ref(void *key, unsigned key_len, void *val, + unsigned val_len, print_item_func func, void *arg) +{ + struct scoutfs_btree_ref *ref = val; + + func(key, key_len, NULL, 0, arg); + printf(" ref blkno %llu seq %llu\n", + le64_to_cpu(ref->blkno), le64_to_cpu(ref->seq)); + + return 0; +} + +static int print_btree_block(int fd, struct scoutfs_super_block *super, + char *which, struct scoutfs_btree_ref *ref, + print_item_func func, void *arg, u8 level) +{ + struct scoutfs_btree_item *item; + struct scoutfs_btree_block *bt; + unsigned key_len; + unsigned val_len; + void *key; + void *val; + int ret; + int i; + + bt = read_block(fd, le64_to_cpu(ref->blkno)); + if (!bt) + return -ENOMEM; + + if (bt->level == level) { + printf("%s btree blkno %llu\n" + " fsid %llx blkno %llu seq %llu crc %08x \n" + " level %u free_end %u free_reclaim %u nr_items %u\n" + " bit_counts:", + which, le64_to_cpu(ref->blkno), + le64_to_cpu(bt->fsid), + le64_to_cpu(bt->blkno), + le64_to_cpu(bt->seq), + le32_to_cpu(bt->crc), + bt->level, + le16_to_cpu(bt->free_end), + le16_to_cpu(bt->free_reclaim), + le16_to_cpu(bt->nr_items)); + for (i = 0; i < array_size(bt->bit_counts); i++) { + if (bt->bit_counts[i]) + printf(" %u:%u", + i, le16_to_cpu(bt->bit_counts[i])); + } + printf("\n"); + } + + for (i = 0; i < le16_to_cpu(bt->nr_items); i++) { + item = (void *)bt + le16_to_cpu(bt->item_hdrs[i].off); + key_len = le16_to_cpu(item->key_len); + val_len = le16_to_cpu(item->val_len); + key = (void *)(item + 1); + val = (void *)key + key_len; + + if (level < bt->level) { + ref = val; + /* XXX check len */ + if (ref->blkno) { + ret = print_btree_block(fd, super, which, ref, + func, arg, level); + if (ret) + break; + } + continue; + } + + printf(" item [%u] off %u bits %02x key_len %u val_len %u\n", + i, le16_to_cpu(bt->item_hdrs[i].off), + bt->item_hdrs[i].bits, key_len, val_len); + + if (level) + print_btree_ref(key, key_len, val, val_len, func, arg); + else + func(key, key_len, val, val_len, arg); + } + + free(bt); + return 0; +} + +/* + * We print btrees by a breadth-first search. This way all the parent + * blocks are printed before the factor of fanout more numerous leaf + * blocks and their included items. + */ +static int print_btree(int fd, struct scoutfs_super_block *super, char *which, + struct scoutfs_btree_root *root, + print_item_func func, void *arg) +{ + int ret = 0; + int i; + + for (i = root->height - 1; i >= 0; i--) { + ret = print_btree_block(fd, super, which, &root->ref, + func, arg, i); + if (ret) + break; + } + + return ret; +} + static void print_super_block(struct scoutfs_super_block *super, u64 blkno) { char uuid_str[37]; @@ -460,24 +533,30 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno) print_block_header(&super->hdr); printf(" id %llx uuid %s\n", le64_to_cpu(super->id), uuid_str); + /* XXX these are all in a crazy order */ - printf(" next_ino %llu next_seq %llu\n" - " ring_blkno %llu ring_blocks %llu ring_tail_block %llu\n" - " ring_gen %llu alloc_uninit %llu total_segs %llu\n" - " next_seg_seq %llu free_segs %llu\n", + printf(" next_ino %llu next_seq %llu next_seg_seq %llu\n" + " alloc_uninit %llu total_segs %llu free_segs %llu\n" + " btree ring: first_blkno %llu nr_blocks %llu next_block %llu " + "next_seq %llu\n" + " alloc btree root: height %u blkno %llu seq %llu\n" + " manifest btree root: height %u blkno %llu seq %llu\n", le64_to_cpu(super->next_ino), le64_to_cpu(super->next_seq), - le64_to_cpu(super->ring_blkno), - le64_to_cpu(super->ring_blocks), - le64_to_cpu(super->ring_tail_block), - le64_to_cpu(super->ring_gen), + le64_to_cpu(super->next_seg_seq), le64_to_cpu(super->alloc_uninit), le64_to_cpu(super->total_segs), - le64_to_cpu(super->next_seg_seq), - le64_to_cpu(super->free_segs)); - - print_ring_descriptor(&super->alloc_ring, "alloc"); - print_ring_descriptor(&super->manifest.ring, "manifest"); + le64_to_cpu(super->free_segs), + le64_to_cpu(super->bring.first_blkno), + le64_to_cpu(super->bring.nr_blocks), + le64_to_cpu(super->bring.next_block), + le64_to_cpu(super->bring.next_seq), + super->alloc_root.height, + le64_to_cpu(super->alloc_root.ref.blkno), + le64_to_cpu(super->alloc_root.ref.seq), + super->manifest.root.height, + le64_to_cpu(super->manifest.root.ref.blkno), + le64_to_cpu(super->manifest.root.ref.seq)); printf(" level_counts:"); counts = super->manifest.level_counts; @@ -524,11 +603,11 @@ static int print_super_blocks(int fd) return ret; } - ret = print_ring(fd, super, "alloc", &super->alloc_ring, - print_alloc_region, NULL); + ret = print_btree(fd, super, "alloc", &super->alloc_root, + print_alloc_region, NULL); - err = print_ring(fd, super, "manifest", &super->manifest.ring, - print_manifest_entry, seg_map); + err = print_btree(fd, super, "manifest", &super->manifest.root, + print_manifest_entry, seg_map); if (err && !ret) ret = err;