diff --git a/utils/.gitignore b/utils/.gitignore new file mode 100644 index 00000000..3d9129b6 --- /dev/null +++ b/utils/.gitignore @@ -0,0 +1,9 @@ +*.o +*.d +*.swp +src/scoutfs +.sparse* +.mock.build* +cscope.* +scoutfs-utils.spec +scoutfs-utils-*.tar diff --git a/utils/Makefile b/utils/Makefile new file mode 100644 index 00000000..81386c83 --- /dev/null +++ b/utils/Makefile @@ -0,0 +1,55 @@ +SCOUTFS_FORMAT_HASH := \ + $(shell cat src/format.h src/ioctl.h | md5sum | cut -b1-16) + +CFLAGS := -Wall -O2 -Werror -D_FILE_OFFSET_BITS=64 -g -msse4.2 \ + -Wpadded \ + -fno-strict-aliasing \ + -DSCOUTFS_FORMAT_HASH=0x$(SCOUTFS_FORMAT_HASH)LLU + +BIN := src/scoutfs +OBJ := $(patsubst %.c,%.o,$(wildcard src/*.c)) +DEPS := $(wildcard */*.d) + +all: $(BIN) + +ifneq ($(DEPS),) +-include $(DEPS) +endif + +ifeq ($(V), ) +QU = @echo +VE = @ +else +QU = @: +VE = +endif + +$(BIN): $(OBJ) + $(QU) [BIN $@] + $(VE)gcc -o $@ $^ -luuid -lm -lcrypto + +%.o %.d: %.c Makefile sparse.sh + $(QU) [CC $<] + $(VE)gcc $(CFLAGS) -MD -MP -MF $*.d -c $< -o $*.o + $(QU) [SP $<] + $(VE)./sparse.sh -Wbitwise -D__CHECKER__ $(CFLAGS) $< + +.PHONY: .FORCE + +# - We use the git describe from tags to set up the RPM versioning +RPM_VERSION := $(shell git describe --long --tags | awk -F '-' '{gsub(/^v/,""); print $$1}') +RPM_GITHASH := $(shell git rev-parse --short HEAD) + +%.spec: %.spec.in .FORCE + sed -e 's/@@VERSION@@/$(RPM_VERSION)/g' \ + -e 's/@@GITHASH@@/$(RPM_GITHASH)/g' < $< > $@+ + mv $@+ $@ + +TARFILE = scoutfs-utils-$(RPM_VERSION).tar + +dist: $(RPM_DIR) scoutfs-utils.spec + git archive --format=tar --prefix scoutfs-utils-$(RPM_VERSION)/ HEAD^{tree} > $(TARFILE) + @ tar rf $(TARFILE) --transform="s@\(.*\)@scoutfs-utils-$(RPM_VERSION)/\1@" scoutfs-utils.spec + +clean: + @rm -f $(BIN) $(OBJ) $(DEPS) .sparse.* diff --git a/utils/README.md b/utils/README.md new file mode 100644 index 00000000..4eb0650e --- /dev/null +++ b/utils/README.md @@ -0,0 +1,7 @@ +This repository contains the userspace software for the scoutfs +clustered filesystem. + +More context and instructions can be found on the https://scoutfs.org/ +community site or in the +[scoutfs-kmod-dev](https://github.com/versity/scoutfs-kmod-dev) git +repository which houses the scoutfs Linux kernel module. diff --git a/utils/man/scoutfs-corruption.7 b/utils/man/scoutfs-corruption.7 new file mode 100644 index 00000000..99e797b3 --- /dev/null +++ b/utils/man/scoutfs-corruption.7 @@ -0,0 +1,169 @@ +.TH scoutfs-corruption 7 +.SH NAME +scoutfs-corruption \- corruption message details +.SH DESCRIPTION +If scoutfs detects corruption during operation it will output an error +message describing the corruption. This document gives details of the +corruption described by the messages. +.SH CORRUPTION MESSAGE IDENTIFIERS +.TP +.B SC_DIRENT_NAME_LEN +A directory entry with an invalid name length was found during lookup. + +Directory entries are stored in the values of metadata items. The item +value contains a small header and the full entry name. The length of +the entry name is calculated by substracting the size of the header from +the length of the item value. This corruption is detected if the length +of the calculated name length is invalid by being less than 1 or greater +than 255. + +.BR dir_ino " - inode number of directory that contains the item" +.br +.BR hash " - hash value of search name" +.br +.BR key " - identifies the item with the invalid name length" +.br +.BR len " - the invalid calculaged name length" +.sp +.TP +.B SC_DIRENT_READDIR_NAME_LEN +A directory entry with an invalid name length was found during readdir. + +This corruption is very similar to +.B SC_DIRENT_NAME_LEN +except that the corruption is discovered during readdir instead of +lookup. The readdir search key is formed from the file position instead +of from the hashed name as in lookup. The dirent structure stored in +the item value is the same. + +.BR dir_ino " - inode number of directory that contains the item" +.br +.BR pos " - the file position readdir was searching from" +.br +.BR key " - identifies the item with the invalid name length" +.br +.BR len " - the invalid calculaged name length" +.sp + +.TP +.B SC_DIRENT_BACKREF_NAME_LEN +A directory entry with an invalid name length was found while finding +entries that point to an inode. + +This corruption is very similar to +.B SC_DIRENT_NAME_LEN +except that the +corruption is discovered while finding entries that refer to a specific +inode. The search key is formed from the inode and position of the +referring entry instead of from the hashed name as in lookup. The +dirent structure stored in the item value is the same. + +.BR ino " - target inode number we're finding entries to" +.br +.BR dir_ino " - inode number of directory containing entries to search" +.br +.BR pos " - position in directory containing entries to search" +.br +.BR key " - identifies the item with the invalid name length" +.br +.BR len " - the invalid calculaged name length" +.sp + +.TP +.B SC_SYMLINK_INODE_SIZE +The items that contain a symlink target path weren't found. + +The target path of a symlink is stored in a series of metadata items. +The number of items can be calculated from the size of the path. While +trying to resolve a symlink one of the items wasn't found. + +.BR ino " - inode number of the symlink with the invalid size" +.br +.BR size " - the invalid size found in the inode" +.sp + +.TP +.B SC_SYMLINK_MISSING_ITEM +A symlink inode contained an invalid size. + +The i_size field of the inode that stores a symlink records the length +of the path of the symlink target. The path length can't be less than 1 +or greater than the max size which is around 4KiB. + +.BR ino " - inode number of the symlink with the invalid size" +.br +.BR size " - the length of the target path" +.sp + +.TP +.B SC_SYMLINK_NOT_NULL_TERM +A symlink target path wasn't null terminated. + +The target path stored in a symlink's metadata items wasn't null +terminated. + +.BR ino " - inode number of the symlink with the invalid size" +.br +.BR last " - the value of the final byte of the path" +.sp + +.TP +.B SC_BTREE_BLOCK_LEVEL +A btree block's header did not contain the expected level field. + +The btree root stores the height of the btree and each btree block +stores its level in the tree. During descent the level is loaded from +the root and decremented as each block is traveresed. This corruption +occurs when a btree block's level field didn't match the level that was +being calculated during descent. + +.BR root_height " - height of the tree in the root" +.br +.BR root_blkno " - block number of the first block in the root" +.br +.BR root_seq " - sequence number of the first block in the root" +.br +.BR blkno " - block number of the block with mismatched level" +.br +.BR seq " - sequence number of the block with mismatched level" +.br +.BR level " - level of the block with mismatched level" +.br +.BR expected " - expected level that was calculated during descent" +.sp + +.TP +.B SC_BTREE_NO_CHILD_REF +A btree parent block didn't have a child item for a key. + +Each child reference in a parent btree block contains the greatest key +that will be stored in the subtree rooted in the child. The child +references down the right side of the tree must have a key that is +greater than all possible keys. + +This corruption occurs during descent when the search key was greater +than the last child reference's key. + +.BR root_height " - height of the tree in the root" +.br +.BR root_blkno " - block number of the first block in the root" +.br +.BR root_seq " - sequence number of the first block in the root" +.br +.BR blkno " - block number of the block with mismatched level" +.br +.BR seq " - sequence number of the block with mismatched level" +.br +.BR level " - level of the block with mismatched level" +.br +.BR nr " - number of items in the parent block" +.br +.BR pos " - child item index that search found" +.br +.BR cmp " - comparison of search key and found" +.sp + +.SH AUTHORS +Zach Brown + + diff --git a/utils/man/scoutfs.5 b/utils/man/scoutfs.5 new file mode 100644 index 00000000..b85571cb --- /dev/null +++ b/utils/man/scoutfs.5 @@ -0,0 +1,54 @@ +.TH scoutfs 5 +.SH NAME +scoutfs \- overview and mount options for the scoutfs filesystem +.SH DESCRIPTION +A scoutfs filesystem is stored on two block devices. Multiple mounts of +the filesystem are supported between hosts that share access to the +block device. A new filesystem is created with the +.B mkfs +command in the +.BR scoutfs (8) +utility. +.SH MOUNT OPTIONS +The following mount options are supported by scoutfs in addition to the +general mount options described in the +.BR mount (8) +manual page. +.TP +.B metadev_path= +The metadev_path option specifies the path to the block device that +contains the filesystem's metadata. +.sp +This option is required. +.TP +.B server_addr= +The server_addr option indicates that this mount will participate in +quorum election to try and run a server for all the mounts of its +filesystem. The option specifies the local TCP IPv4 address that the +mount's elected server will listen on for connections from all other +mounts of the filesystem. +.sp +The IPv4 address must be specified as a dotted quad, name resolution is +not supported. A specific port may be provided after a seperating +colon. If no port is specified then a random port will be chosen. The +address will be used for the lifetime of the mount and can not be +changed. The mount must be unmounted to specify a different address. +.sp +If server_addr is not specified then the mount will read the filesystem +until it sees the address of an elected server to connect to. +.SH FURTHER READING +A +.B scoutfs +filesystem can detect corruption at runtime. A catalog of kernel log +messages that indicate corruption can be found in +.BR scoutfs-corruption (8) +\&. + +.SH SEE ALSO +.BR scoutfs (8), +.BR scoutfs-corruption (7). + +.SH AUTHORS +Zach Brown + + diff --git a/utils/man/scoutfs.8 b/utils/man/scoutfs.8 new file mode 100644 index 00000000..1e8721a2 --- /dev/null +++ b/utils/man/scoutfs.8 @@ -0,0 +1,452 @@ +.TH scoutfs 8 +.SH NAME +scoutfs \- scoutfs management utility +.SH DESCRIPTION +The +.b +scoutfs +utility provides commands to manage a scoutfs filesystem. +.SH COMMANDS +.TP +.BI "counters [\-t\] " +.sp +Displays the counters and their values for a mounted scoutfs filesystem. +Each counter and its value are printed on a line to stdout with +sufficient spaces seperating the name and value to align the values +after +.RS 1.0i +.PD 0 +.TP +.sp +.B "\-t" +Format the counters into a table that fills the display instead of +printing one counter per line. The names and values are padded to +create columns that fill the current width of the terminal. +.TP +.B "sysfs topdir" +Specify the mount's sysfs directory in which to find the +.B counters/ +directory when then contains files for each counter. +The sysfs directory is typically +of the form +.I /sys/fs/scoutfs/f..r./ +\&. +.RE +.PD + +.TP +.BI "data-waiting " +.sp +Displays all the files and blocks for which there is a task blocked waiting on +offline data. +.sp +The results are sorted by the file's inode number and the +logical block offset that is being waited on. +.sp +Each line of output specifies a block in a file that has a task waiting +and is formatted as: +.I "ino iblock ops [str]" +\&. The ops string indicates blocked operations seperated by commas and can +include +.B read +for a read operation, +.B write +for a write operation, and +.B change_size +for a truncate or extending write. +.RS 1.0i +.PD 0 +.sp +.TP +.B "ino" +Start iterating over waiting tasks from the given inode number. +Specifying 0 will show all waiting tasks. +.TP +.B "iblock" +Start iterating over waiting tasks from the given logical block number +in the starting inode. Specifying 0 will show blocks in the first inode +and then continue to show all blocks with tasks waiting in all the +remaining inodes. +.TP +.B "path" +A path to any inode in the target filesystem, typically the root +directory. +.RE +.PD + +.TP +.BI "find-xattrs <\-n\ name> <\-f path>" +.sp +Displays the inode numbers of inodes in the filesystem which may have +an extended attribute with the given name. +.sp +The results may contain false positives. The returned inode numbers +should be checked to verify that the extended attribute is in fact +present on the inode. +.RS 1.0i +.PD 0 +.TP +.sp +.B "-n name" +Specifies the full name of the extended attribute to search for as +described in the +.BR xattr (7) +manual page. +.TP +.B "-f path" +Specifies the path to any inode in the filesystem to search. +.RE +.PD + +.TP +.BI "ino-path " +.sp +Displays all the paths to links to the given inode number. +.sp +All the relative paths from the root directory to each link of the +target inode are output, one result per line. Each output path is +guaranteed to have been a valid path to a link at some point in the +past. An individual path won't be corrupted by a rename that occurs +during the search. The set of paths can be modified while the search is +running. A rename of a parent directory of all the paths, for example, +can result in output where the parent directory name component changes +in the middle of outputting all the paths. +.RS 1.0i +.PD 0 +.sp +.TP +.B "ino" +The inode number of the target inode to resolve. +.TP +.B "path" +A path to any inode in the target filesystem, typically the root +directory. +.RE +.PD + +.TP +.BI "listxattr-hidden <\-f path>" +.sp +Displays all the extended attributes starting with the +.BR scoutfs. +prefix and which contain the +.BR hide. +tag +which makes them invisible to +.BR listxattr (2) +\&. +The names of each attribute are output, one name per line. Their order +is determined by internal indexing implementation details and should not +be relied on. +.RS 1.0i +.PD 0 +.TP +.sp +.B "-f path" +The path to the file whose extended attributes will be listed. The +user must have read permission to the inode. +.RE +.PD + +.TP +.BI "mkfs <\-Q nr> [-M meta_size] [-D data_size]" +.sp +Initialize a new empty filesystem in the target devices by writing empty +structures and a new superblock. Since ScoutFS uses separate block +devices for its metadata and data storage, both must be given. +.sp +This +.B unconditionally destroys +the contents of the devices, regardless of what they contain or who may be +using them. It simply writes new data structures into known offsets. +.B Be very careful that the devices do not contain data and are not actively in use. +.RS 1.0i +.PD 0 +.TP +.sp +.B "-Q nr" +Specify the number of mounts needed to reach quorum and elect a mount +to start the server. Mounts of the filesystem will hang until this many +mounts are operational and can elect a server amongst themselves. +.sp +Mounts with the +.B server_addr +mount option participate in quorum. The safest quorum number is the +smallest majority of an odd number of participating mounts. For +example, +two out of three total mounts. This ensures that there can only be one +set of mounts that can establish quorum. +.sp +Degenerate quorums are possible, for example by specifying half of an +even number of mounts or less than half of the mount count, down to even +just one mount establishing quorum. These minority quorums carry the +risk of multiple quorums being established concurrently. Each quorum's +elected servers race to fence each other and can have the unlikely +outcome of continually racing to fence each other resulting in a +persistent loss of service. +.TP +.B "meta_dev_path" +The path to the device to be used for ScoutFS metadata. If possible, +use a faster block device for the metadata device. Its contents will be +unconditionally destroyed. +.TP +.B "data_dev_path" +The path to the device to be used for ScoutFS file data. If possible, +use a larger block device for the data device. Its contents will be +unconditionally destroyed. +.TP +.B "-M meta_size" +Limit the space used by the filesystem on the metadata device to the +given size, rather than using the entire block device. Size is given as +an integer followed by a units digit: "K", "M", "G", "T", "P", to denote +kibibytes, mebibytes, etc. +.TP +.B "-D data_size" +Same as previous, but for limiting the size of the data device. +.RE +.PD + +.TP +.BI "print " +.sp +Prints out all of the metadata in the file system. This makes no effort +to ensure that the structures are consistent as they're traversed and +can present structures that seem corrupt as they change as they're +output. +.RS 1.0i +.PD 0 +.TP +.sp +.B "path" +The path to the metadata device for filesystem whose metadata will +be printed. The command reads from the buffer cache of the device which +may not reflect the current blocks in the filesystem that may have been +written through another host or device. The local device's cache can be +manually flushed before printing, perhaps with the +.B \--flushbufs +command in the +.BR blockdev (8) +command. +.RE +.PD + +.TP +.BI "release <4KB block offset> <4KB block count>" +.sp +.B Release +the given logical block region of the file. That is, truncate away +any data blocks but leave behind offline data regions and do not change +the main inode metadata. Future attempts to read or write the block +region +will block until the region is restored by a +.B stage +write. This is used by userspace archive managers to store file data +in a remote archive tier. +.sp +This only works on regular files and with write permission. Releasing +regions that are already offline or are sparse, including past the end +of the file, silently succeed. +.RS 1.0i +.PD 0 +.TP +.sp +.B "path" +The path to the regular file whose region will be released. +.TP +.B "version" +The current data version of the contents of the file. This ensures +that a release operation is truncating the version of the data that it +expects. It can't throw away data that was newly written while it was +performing its release operation. An inode's data_version is read +by the SCOUTFS_IOC_STATFS_MORE +ioctl. +.TP +.B "4KB block offset" +The 64bit logical block offset of the start of the region in units of 4KB. +.TP +.B "4KB block count" +The 64bit length of the region to release in units of 4KB blocks. +.RE +.PD + +.TP +.BI "setattr <\-c ctime> <\-d data_version> -o <\-s i_size> <\-f path> +.sp +Set scoutfs specific metadata on a newly created inode without updating +other inode metadata. +.RS 1.0i +.PD 0 +.TP +.sp +.B "-c ctime" +Specify the inode's creation GMT timespec with 64bit seconds and 32bit +nanoseconds formatted as +.B sec.nsec +\&. +.TP +.B "-d data_version" +Specify the inode's data version. This can only be set on regular files whose +current data_version is 0. +.TP +.B "-o" +Create an offline region for all of the file's data up to the specified +file size. This can only be set on regular files whose data_version is +0 and i_size must also be specified. +.TP +.B "-s i_size" +Set the inode's i_size. This can only be set on regular files whose +data_version is 0. +.TP +.B "-f path" +The file whose metadata will be set. +.RE +.PD + +.TP +.BI "stage " +.sp +.B Stage +the contents of the file by reading a region of another archive file and writing it +into the file region without updating regular inode metadata. Any tasks +that are blocked by the offline region will proceed once it has been +staged. +.RS 1.0i +.PD 0 +.TP +.sp +.B "file" +The regular file whose contents will be staged. +.TP +.B "vers" +The data_version of the contents to be staged. It must match the +current data_version of the file. +.TP +.B "offset" +The starting byte offset of the region to write. This must be aligned +to 4KB blocks. +.TP +.B "count" +The length of the region to write in bytes. A length of 0 is a noop +and will immediately return success. The length must be a multiple +of 4KB blocks unless it is writing the final partial block in which +case it must end at i_size. +.TP +.B "archive file" +A file whose contents will be read and written as the staged region. +The start of the archive file will be used as the start of the region. +.RE +.PD + +.TP +.BI "stat [-s single] " +.sp +Display scoutfs metadata fields for the given inode. +.RS 1.0i +.PD 0 +.TP +.sp +.B "-s single" +Only ontput a single stat instead of all the stats with one stat per +line. The possible stat names are those given in the output. +.TP +.B "path" +The path to the file whose inode field will be output. +.sp +.TP +.RE +.PD +The fields are as follows: +.RS 1.0i +.PD 0 +.TP +.B "meta_seq" +The metadata change sequence. This changes each time the inode's metadata +is changed during a mount's transaction. +.TP +.B "data_seq" +The data change sequence. This changes each time the inode's data +is changed during a mount's transaction. +.TP +.B "data_version" +The data version changes every time any contents of the file changes, +including size changes. It can change many times during a syscall in a +transactions. +.TP +.B "online_blocks" +The number of 4Kb data blocks that contain data and can be read. +.TP +.B "online_blocks" +The number of 4Kb data blocks that are offline and would need to be +staged to be read. +.RE +.PD + +.TP +.BI "statfs [-s single] " +.sp +Display scoutfs metadata fields for a scoutfs filesystem. +.RS 1.0i +.PD 0 +.TP +.sp +.B "-s single" +Only ontput a single stat instead of all the stats with one stat per +line. The possible stat names are those given in the output. +.TP +.B "path" +The path to any inode in the filesystem. +.sp +.TP +.RE +.PD +The fields are as follows: +.RS 1.0i +.PD 0 +.TP +.B "fsid" +The unique 64bit filesystem identifier for this filesystem. +.TP +.B "rid" +The unique 64bit random identifier for this mount of the filesystem. +This is generated for every new mount of the file system. +.RE +.PD + +.TP +.BI "walk-inodes " +.sp +Walks an inode index in the file system and outputs the inode numbers +that are found within the first and last positions in the index. +.RS 1.0i +.PD 0 +.sp +.TP +.B "index" +Specifies the index to walk. The currently supported indices are +.B meta_seq +and +.B data_seq +\&. +.TP +.B "first" +The starting position of the index walk. +.I 0 +is the first possible position in every index. +.TP +.B "last" +The last position to include in the index walk. +.I \-1 +can be given as shorthand for the U64_MAX last possible position in +every index. +.TP +.B "path" +A path to any inode in the filesystem, typically the root directory. +.RE +.PD + +.SH SEE ALSO +.BR scoutfs (5), +.BR xattr (7). + +.SH AUTHORS +Zach Brown diff --git a/utils/scoutfs-utils.spec.in b/utils/scoutfs-utils.spec.in new file mode 100644 index 00000000..35219721 --- /dev/null +++ b/utils/scoutfs-utils.spec.in @@ -0,0 +1,69 @@ +%define pkg_version @@VERSION@@ +%define pkg_git_hash @@GITHASH@@ +%define pkg_date %(date +%%Y%%m%%d) + +%{!?_release: %global _release 0.%{pkg_date}git%{pkg_git_hash}} + +Name: scoutfs-utils +Summary: scoutfs user space utilities +Version: %{pkg_version} +Release: %{_release}%{?dist} +License: GPLv2 +Group: System Environment/Base +URL: http://scoutfs.org/ + +BuildRequires: git +BuildRequires: gzip +BuildRequires: libuuid-devel +BuildRequires: openssl-devel + +#Requires: kmod-scoutfs = %{version} + +Source: scoutfs-utils-%{pkg_version}.tar + +# Disable the building of the debug package(s). +%define debug_package %{nil} + +%description +scoutfs - user space utilities + +%package -n scoutfs-devel +Summary: scoutfs devel headers +Version: %{pkg_version} +Release: %{_release}%{?dist} +License: GPLv2 +Group: Development/Libraries +URL: http://scoutfs.org/ + +%description -n scoutfs-devel +scoutfs - development headers + +%prep +%setup -q -n scoutfs-utils-%{pkg_version} + +%build +make +gzip man/*.? + +%install +mkdir -p $RPM_BUILD_ROOT%{_mandir}/man{5,7,8} +cp man/*.5.gz $RPM_BUILD_ROOT%{_mandir}/man5/. +cp man/*.7.gz $RPM_BUILD_ROOT%{_mandir}/man7/. +cp man/*.8.gz $RPM_BUILD_ROOT%{_mandir}/man8/. +install -m 755 -D src/scoutfs $RPM_BUILD_ROOT%{_sbindir}/scoutfs +install -m 644 -D src/ioctl.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/ioctl.h +install -m 644 -D src/format.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/format.h + +%files +%defattr(644,root,root,755) +%{_mandir}/man*/scoutfs*.gz +%defattr(755,root,root,755) +%{_sbindir}/scoutfs + +%files -n scoutfs-devel +%defattr(644,root,root,755) +%{_includedir}/scoutfs + +%clean +rm -rf %{buildroot} + diff --git a/utils/sparse.sh b/utils/sparse.sh new file mode 100755 index 00000000..93791617 --- /dev/null +++ b/utils/sparse.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# can we find sparse? If not, we're done. +which sparse > /dev/null 2>&1 || exit 0 + +# +# one of the problems with using sparse in userspace is that it picks up +# things in system headers that we don't care about. We're willing to +# take on the burden of filtering them out so that we can have it tell +# us about problems in our code. +# +# system headers using __transparent_union__ +RE="^/.*error: ignoring attribute __transparent_union__" + +# we don't care if system headers have gcc attributes sparse doesn't +# know about +RE="$RE|error: attribute '__leaf__': unknown attribute" + +# yes, sparse, that's the size of memseting a 4 meg buffer all right +RE="$RE|warning: memset with byte count of 4194304" + +# some sparse versions don't know about some builtins +RE="$RE|error: undefined identifier '__builtin_fpclassify'" + +# +# don't filter out 'too many errors' here, it can signify that +# sparse doesn't understand something and is throwing a *ton* +# of useless errors before giving up and existing. Check +# unfiltered sparse output. +# + +# +# I'm not sure this is needed. +# +search=$(gcc -print-search-dirs | awk '($1 == "install:"){print "-I" $2}') + +# +# We're trying to use sparse against glibc headers which go wild trying to +# use internal compiler macros to test features. We copy gcc's and give +# them to sparse. But not __SIZE_TYPE__ 'cause sparse defines that one. +# +defines=".sparse.gcc-defines.h" +gcc -dM -E -x c - < /dev/null | grep -v __SIZE_TYPE__ > $defines +include="-include $defines" + +# +# sparse doesn't seem to notice when it's on a 64bit host. It warns that +# 64bit values don't fit in 'unsigned long' without this. +# +if grep -q "__LP64__ 1" $defines; then + m64="-m64" +else + m64="" +fi + +sparse $m64 $include $search/include "$@" 2>&1 | egrep -v "($RE)" | tee .sparse.output +if [ -s .sparse.output ]; then + exit 1 +else + exit 0 +fi diff --git a/utils/src/avl.c b/utils/src/avl.c new file mode 100644 index 00000000..3d86c17f --- /dev/null +++ b/utils/src/avl.c @@ -0,0 +1,48 @@ +#include "sparse.h" +#include "util.h" +#include "format.h" +#include "avl.h" + +static struct scoutfs_avl_node *node_ptr(struct scoutfs_avl_root *root, + + __le16 off) +{ + return off ? (void *)root + le16_to_cpu(off) : NULL; +} + +__le16 avl_node_off(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node) +{ + if (!node) + return 0; + return cpu_to_le16((void *)node - (void *)root); +} + +struct scoutfs_avl_node *avl_first(struct scoutfs_avl_root *root) +{ + struct scoutfs_avl_node *node = node_ptr(root, root->node); + + while (node && node->left) + node = node_ptr(root, node->left); + + return node; +} + +struct scoutfs_avl_node *avl_next(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node) +{ + struct scoutfs_avl_node *parent; + + if (node->right) { + node = node_ptr(root, node->right); + while (node->left) + node = node_ptr(root, node->left); + return node; + } + + while ((parent = node_ptr(root, node->parent)) && + node == node_ptr(root, parent->right)) + node = parent; + + return parent; +} diff --git a/utils/src/avl.h b/utils/src/avl.h new file mode 100644 index 00000000..b72e8e3d --- /dev/null +++ b/utils/src/avl.h @@ -0,0 +1,10 @@ +#ifndef _AVL_H_ +#define _AVL_H_ + +__le16 avl_node_off(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node); +struct scoutfs_avl_node *avl_first(struct scoutfs_avl_root *root); +struct scoutfs_avl_node *avl_next(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node); + +#endif diff --git a/utils/src/bitmap.c b/utils/src/bitmap.c new file mode 100644 index 00000000..5e9ab615 --- /dev/null +++ b/utils/src/bitmap.c @@ -0,0 +1,57 @@ +#define _GNU_SOURCE +#include +#include + +#include "sparse.h" +#include "util.h" +#include "bitmap.h" + +/* + * Just a quick simple native bitmap. + */ + +void set_bit(unsigned long *bits, u64 nr) +{ + bits[nr / BITS_PER_LONG] |= 1UL << (nr & (BITS_PER_LONG - 1)); +} + +void clear_bit(unsigned long *bits, u64 nr) +{ + bits[nr / BITS_PER_LONG] &= ~(1UL << (nr & (BITS_PER_LONG - 1))); +} + +u64 find_next_set_bit(unsigned long *map, u64 from, u64 total) +{ + unsigned long bits; + u64 base; + u64 nr; + int bit; + + base = from & ~((unsigned long)BITS_PER_LONG - 1); + map += from / BITS_PER_LONG; + + while (base < total) { + bits = *map; + + while (bits) { + bit = ffsl(bits) - 1; + nr = base + bit; + + if (nr >= from) + return min(nr, total); + + bits &= ~(1UL << bit); + } + + base += BITS_PER_LONG; + map++; + } + + return total; +} + +unsigned long *alloc_bits(u64 max) +{ + return calloc(DIV_ROUND_UP(max, BITS_PER_LONG), sizeof(unsigned long)); +} + diff --git a/utils/src/bitmap.h b/utils/src/bitmap.h new file mode 100644 index 00000000..993bad4e --- /dev/null +++ b/utils/src/bitmap.h @@ -0,0 +1,9 @@ +#ifndef _BITMAP_H_ +#define _BITMAP_H_ + +void set_bit(unsigned long *bits, u64 nr); +void clear_bit(unsigned long *bits, u64 nr); +u64 find_next_set_bit(unsigned long *start, u64 from, u64 total); +unsigned long *alloc_bits(u64 max); + +#endif diff --git a/utils/src/bitops.h b/utils/src/bitops.h new file mode 100644 index 00000000..69605fed --- /dev/null +++ b/utils/src/bitops.h @@ -0,0 +1,101 @@ +#ifndef _BITOPS_H_ +#define _BITOPS_H_ + +#include "sparse.h" + +/* + * Implement little endian bitmaps in terms of native longs. __packed + * is used to avoid unaligned accesses. These are neither atomic nor + * particularly efficient. + */ + +#define BITS_PER_LONG (sizeof(long) * 8) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define BITOP_LE_SWIZZLE 0 +#else +#define BITOP_LE_SWIZZLE ((BITS_PER_LONG-1) & ~0x7) +#endif + +static inline unsigned long get_nr_word(int nr, void *addr) +{ + unsigned long *longs = addr; + unsigned long ind = nr / BITS_PER_LONG; + unsigned long val; + + memcpy(&val, &longs[ind], sizeof(val)); + + return val; +} + +static inline void put_nr_word(int nr, void *addr, unsigned long val) +{ + unsigned long *longs = addr; + unsigned long ind = nr / BITS_PER_LONG; + + memcpy(&longs[ind], &val, sizeof(val)); +} + +static inline unsigned long nr_mask(int nr) +{ + return 1UL << (nr % BITS_PER_LONG); +} + +static inline int test_bit(int nr, void *addr) +{ + unsigned long val = get_nr_word(nr, addr); + + return !!(val & nr_mask(nr)); +} + +static inline void set_bit(int nr, void *addr) +{ + unsigned long val = get_nr_word(nr, addr); + + val |= nr_mask(nr); + put_nr_word(nr, addr, val); +} + +static inline void clear_bit(int nr, void *addr) +{ + unsigned long val = get_nr_word(nr, addr); + + val &= ~nr_mask(nr); + put_nr_word(nr, addr, val); +} + +static inline int test_bit_le(int nr, void *addr) +{ + return test_bit(nr ^ BITOP_LE_SWIZZLE, addr); +} + +static inline int test_and_set_bit_le(int nr, void *addr) +{ + int ret; + + nr ^= BITOP_LE_SWIZZLE; + ret = test_bit(nr, addr); + set_bit(nr, addr); + return ret; +} + +static inline void set_bit_le(int nr, void *addr) +{ + set_bit(nr ^ BITOP_LE_SWIZZLE, addr); +} + +static inline void clear_bit_le(int nr, void *addr) +{ + clear_bit(nr ^ BITOP_LE_SWIZZLE, addr); +} + +static inline int test_and_clear_bit_le(int nr, void *addr) +{ + int ret; + + nr ^= BITOP_LE_SWIZZLE; + ret = test_bit(nr, addr); + clear_bit(nr, addr); + return ret; +} + +#endif diff --git a/utils/src/btree.c b/utils/src/btree.c new file mode 100644 index 00000000..4008af63 --- /dev/null +++ b/utils/src/btree.c @@ -0,0 +1,91 @@ +#include + +#include "sparse.h" +#include "util.h" +#include "format.h" +#include "key.h" +#include "avl.h" +#include "leaf_item_hash.h" +#include "btree.h" + +static void init_block(struct scoutfs_btree_block *bt, int level) +{ + int free; + + free = SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_btree_block); + if (level == 0) + free -= SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES; + + bt->level = level; + bt->mid_free_len = cpu_to_le16(free); +} + +/* + * Point the root at the single leaf block that makes up a btree. + */ +void btree_init_root_single(struct scoutfs_btree_root *root, + struct scoutfs_btree_block *bt, + u64 blkno, u64 seq, __le64 fsid) +{ + root->ref.blkno = cpu_to_le64(blkno); + root->ref.seq = cpu_to_le64(1); + root->height = 1; + + memset(bt, 0, SCOUTFS_BLOCK_LG_SIZE); + bt->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_BTREE); + bt->hdr.fsid = fsid; + bt->hdr.blkno = cpu_to_le64(blkno); + bt->hdr.seq = cpu_to_le64(1); + + init_block(bt, 0); +} + +static void *alloc_val(struct scoutfs_btree_block *bt, int len) +{ + le16_add_cpu(&bt->mid_free_len, -len); + le16_add_cpu(&bt->total_item_bytes, len); + return (void *)bt + le16_to_cpu(bt->mid_free_len); +} + +/* + * Add a sorted item after all the items in the block. + * + * We simply implement the special case of a wildly imbalanced avl tree. + * Mkfs only ever inserts a handful of items and they'll be rebalanced + * over time. + */ +void btree_append_item(struct scoutfs_btree_block *bt, + struct scoutfs_key *key, void *val, int val_len) +{ + struct scoutfs_btree_item *item; + struct scoutfs_avl_node *prev; + void *val_buf; + + item = &bt->items[le16_to_cpu(bt->nr_items)]; + + if (bt->nr_items) { + assert(scoutfs_key_compare(key, &(item - 1)->key) > 0); + prev = &(item - 1)->node; + + item->node.height = prev->height++; + item->node.left = avl_node_off(&bt->item_root, prev); + prev->parent = avl_node_off(&bt->item_root, &item->node); + } + + bt->item_root.node = avl_node_off(&bt->item_root, &item->node); + le16_add_cpu(&bt->nr_items, 1); + le16_add_cpu(&bt->mid_free_len, + -(u16)sizeof(struct scoutfs_btree_item)); + le16_add_cpu(&bt->total_item_bytes, sizeof(struct scoutfs_btree_item)); + + item->key = *key; + leaf_item_hash_insert(bt, &item->key, + cpu_to_le16((void *)item - (void *)bt)); + if (val_len == 0) + return; + + val_buf = alloc_val(bt, val_len); + item->val_off = cpu_to_le16((void *)val_buf - (void *)bt); + item->val_len = cpu_to_le16(val_len); + memcpy(val_buf, val, val_len); +} diff --git a/utils/src/btree.h b/utils/src/btree.h new file mode 100644 index 00000000..1f176a8b --- /dev/null +++ b/utils/src/btree.h @@ -0,0 +1,11 @@ +#ifndef _BTREE_H_ +#define _BTREE_H_ + +void btree_init_root_single(struct scoutfs_btree_root *root, + struct scoutfs_btree_block *bt, + u64 blkno, u64 seq, __le64 fsid); + +void btree_append_item(struct scoutfs_btree_block *bt, + struct scoutfs_key *key, void *val, int val_len); + +#endif diff --git a/utils/src/cmd.c b/utils/src/cmd.c new file mode 100644 index 00000000..607f12ec --- /dev/null +++ b/utils/src/cmd.c @@ -0,0 +1,88 @@ +#include +#include +#include +#include +#include +#include + +#include "cmd.h" +#include "util.h" + +static struct command { + char *name; + char *opts; + char *summary; + int (*func)(int argc, char **argv); +} cmds[100], *next_cmd = cmds; + +#define cmd_for_each(com) for (com = cmds; com->func; com++) + +void cmd_register(char *name, char *opts, char *summary, + int (*func)(int argc, char **argv)) +{ + struct command *com = next_cmd++; + + assert((com - cmds) < array_size(cmds)); + + com->name = name; + com->opts = opts; + com->summary = summary; + com->func = func; +} + +static struct command *find_command(char *name) +{ + struct command *com; + + cmd_for_each(com) { + if (!strcmp(name, com->name)) + return com; + } + + return NULL; +} + +static void usage(void) +{ + struct command *com; + int largest = 0; + + fprintf(stderr, "usage: scoutfs []\n" + "Commands:\n"); + + cmd_for_each(com) + largest = max(strlen(com->name), largest); + + cmd_for_each(com) { + fprintf(stderr, " %*s %s\n %*s %s\n", + largest, com->name, com->opts, + largest, "", com->summary); + } +} + +/* this returns a positive unix return code on error for some reason */ +char cmd_execute(int argc, char **argv) +{ + struct command *com = NULL; + int ret; + + if (argc > 1) { + com = find_command(argv[1]); + if (!com) + fprintf(stderr, "scoutfs: unrecognized command: '%s'\n", + argv[1]); + } + if (!com) { + usage(); + return 1; + } + + ret = com->func(argc - 1, argv + 1); + if (ret < 0) { + fprintf(stderr, "scoutfs: %s failed: %s (%d)\n", + com->name, strerror(-ret), -ret); + return 1; + } + + return 0; +} diff --git a/utils/src/cmd.h b/utils/src/cmd.h new file mode 100644 index 00000000..084590e9 --- /dev/null +++ b/utils/src/cmd.h @@ -0,0 +1,9 @@ +#ifndef _CMD_H_ +#define _CMD_H_ + +void cmd_register(char *name, char *opts, char *summary, + int (*func)(int argc, char **argv)); + +char cmd_execute(int argc, char **argv); + +#endif diff --git a/utils/src/cmp.h b/utils/src/cmp.h new file mode 100644 index 00000000..23c6d8a6 --- /dev/null +++ b/utils/src/cmp.h @@ -0,0 +1,23 @@ +#ifndef _SCOUTFS_CMP_H_ +#define _SCOUTFS_CMP_H_ + +/* + * A generic ternary comparison macro with strict type checking. + */ +#define scoutfs_cmp(a, b) \ +({ \ + __typeof__(a) _a = (a); \ + __typeof__(b) _b = (b); \ + int _ret; \ + \ + (void) (&_a == &_b); \ + _ret = _a < _b ? -1 : _a > _b ? 1 : 0; \ + _ret; \ +}) + +static inline int scoutfs_cmp_u64s(u64 a, u64 b) +{ + return a < b ? -1 : a > b ? 1 : 0; +} + +#endif diff --git a/utils/src/counters.c b/utils/src/counters.c new file mode 100644 index 00000000..676226dc --- /dev/null +++ b/utils/src/counters.c @@ -0,0 +1,284 @@ +#define _XOPEN_SOURCE 700 /* openat */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "util.h" +#include "cmd.h" + +struct counter { + char *name; + char *val; + unsigned int name_wid; + unsigned int val_wid; +}; + +static int dots(char *name) +{ + return name[0] == '.' && + (name[1] == '\0' || (name[1] == '.' && name[2] == '\0')); +} + +static int cmp_counter_names(const void *A, const void *B) +{ + const struct counter *a = A; + const struct counter *b = B; + + return strcmp(a->name, b->name); +} + +static int counters_cmd(int argc, char **argv) +{ + unsigned int *name_wid = NULL; + unsigned int *val_wid = NULL; + struct counter *ctrs = NULL; + struct counter *ctr; + char path[PATH_MAX + 1]; + unsigned int alloced = 0; + unsigned int min_rows; + unsigned int max_rows; + unsigned int rows = 0; + unsigned int cols = 0; + unsigned int nr = 0; + char *dir_arg = NULL; + struct dirent *dent; + bool table = false; + struct winsize ws; + DIR *dirp = NULL; + int dir_fd = -1; + char buf[25]; + int room; + int ret; + int fd; + int i; + int r; + int c; + + for (i = 1; i < argc; i++) { + if (strcmp(argv[i], "-t") == 0) + table = true; + else + dir_arg = argv[i]; + } + + ret = ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws); + if (ret < 0) + ret = ioctl(STDIN_FILENO, TIOCGWINSZ, &ws); + if (ret < 0) + table = false; + + if (dir_arg == NULL) { + printf("scoutfs counter-table: need mount sysfs dir (i.e. /sys/fs/scoutfs/$fr)\n"); + return -EINVAL; + } + + ret = snprintf(path, PATH_MAX, "%s/counters", dir_arg); + if (ret < 1 || ret >= PATH_MAX) { + ret = -EINVAL; + fprintf(stderr, "invalid counter dir path '%s'\n", dir_arg); + goto out; + } + + dirp = opendir(path); + if (!dirp) { + ret = -errno; + fprintf(stderr, "failed to open sysfs counter dir '%s': %s (%d)\n", + path, strerror(errno), errno); + goto out; + } + + dir_fd = dup(dirfd(dirp)); + if (dir_fd < 0) { + ret = -errno; + fprintf(stderr, "couldn't dup fd for path '%s': %s (%d)\n", + path, strerror(errno), errno); + goto out; + } + + /* read all the counters */ + while ((dent = readdir(dirp))) { + if (dots(dent->d_name)) + continue; + if (nr == alloced) { + alloced += 100; + ctrs = realloc(ctrs, alloced * sizeof(*ctrs)); + name_wid = realloc(name_wid, alloced * sizeof(*name_wid)); + val_wid = realloc(val_wid, alloced * sizeof(*val_wid)); + if (!ctrs || !name_wid || !val_wid) { + fprintf(stderr, "counter array allocation error\n"); + ret = -ENOMEM; + goto out; + } + memset(&ctrs[nr], 0, (alloced - nr) * sizeof(*ctrs)); + } + + ctr = &ctrs[nr]; + + ctr->name = strdup(dent->d_name); + if (ctr->name == NULL) { + fprintf(stderr, "name string allocation error\n"); + ret = -ENOMEM; + goto out; + } + + fd = openat(dir_fd, ctr->name, O_RDONLY); + if (fd < 0) { + ret = -errno; + fprintf(stderr, "failed to open counter file '%s/%s': %s (%d)\n", + path, ctr->name, strerror(errno), errno); + goto out; + } + + ret = pread(fd, buf, sizeof(buf), 0); + close(fd); + + if (ret <= 1 || ret >= sizeof(buf) || buf[ret - 1] != '\n') { + fprintf(stderr, "counter file %s/%s read returned %d\n", + path, ctr->name, ret); + ret = -EIO; + goto out; + } + + buf[ret - 1] = '\0'; + ctr->val = strdup(buf); + if (ctr->val == NULL) { + fprintf(stderr, "value string allocation error\n"); + ret = -ENOMEM; + goto out; + } + + ctr->name_wid = strlen(ctr->name); + ctr->val_wid = strlen(ctr->val); + + name_wid[0] = max(ctr->name_wid, name_wid[0]); + val_wid[0] = max(ctr->val_wid, val_wid[0]); + + nr++; + } + closedir(dirp); + dirp = NULL; + close(dir_fd); + dir_fd = -1; + + /* huh, empty counter dir */ + if (nr == 0) { + ret = 0; + goto out; + } + + /* sort counters by name */ + qsort(ctrs, nr, sizeof(ctrs[0]), cmp_counter_names); + + /* + * If we're packing the counters into a table that fills the + * width of the terminal then there will be a smallest number of + * rows in the table that packs counters into columns that fill + * the width of the terminal. We perform a binary search for + * that smallest number of rows that doesn't fill too many + * columns. + * + * Unless we're not outputting a table, then we just spit out + * one column of counters and use the max field widths from the + * initial counter reads. + */ + if (table) { + min_rows = 1; + cols = ws.ws_col / (name_wid[0] + 1 + val_wid[0] + 2); + max_rows = nr / cols; + } else { + rows = nr; + cols = 1; + min_rows = nr + 1; + max_rows = nr - 1; + } + + while (min_rows <= max_rows) { + rows = min_rows + ((max_rows - min_rows) / 2); + i = 0; + room = ws.ws_col; + + /* + * Iterate over counters, storing the max field widths + * of each column, recording the column chars left in + * the terminal, stopping if we fill too many columns + * for the terminal. + */ + for (c = 0; i < nr && room >= 0; c++) { + name_wid[c] = 0; + val_wid[c] = 0; + + for (r = 0; r < rows && i < nr; r++, i++) { + ctr = &ctrs[i]; + + name_wid[c] = max(ctr->name_wid, name_wid[c]); + val_wid[c] = max(ctr->val_wid, val_wid[c]); + } + + cols = c + 1; + if (c > 0) + room -= 2; + room -= name_wid[c] + 1 + val_wid[c]; + } + + if (room < 0) { + /* need more rows if we ran out of cols */ + min_rows = rows + 1; + } else { + /* see if we can get away with fewer */ + if (max_rows == rows) + break; + max_rows = rows; + } + } + + /* finally output the columns in each row */ + for (r = 0; r < rows; r++) { + for (c = 0; c < cols; c++) { + i = (c * rows) + r; + if (i >= nr) + break; + ctr = &ctrs[i]; + + printf("%s%-*s %*s", + c > 0 ? " " : "", + name_wid[c], ctr->name, + val_wid[c], ctr->val); + } + printf("\n"); + } + + ret = 0; +out: + if (dirp) + closedir(dirp); + if (dir_fd >= 0) + close(dir_fd); + if (ctrs) { + for (i = 0; i < alloced; i++) { + free(ctrs[i].name); + free(ctrs[i].val); + } + free(ctrs); + } + free(name_wid); + free(val_wid); + + return ret; +}; + +static void __attribute__((constructor)) counters_ctor(void) +{ + cmd_register("counters", "[-t] ", + "show [tablular] counters for a given mounted volume", + counters_cmd); +} diff --git a/utils/src/crc.c b/utils/src/crc.c new file mode 100644 index 00000000..0562e580 --- /dev/null +++ b/utils/src/crc.c @@ -0,0 +1,39 @@ +#include "crc.h" +#include "util.h" +#include "format.h" + +u32 crc32c(u32 crc, const void *data, unsigned int len) +{ + while (len >= 8) { + crc = __builtin_ia32_crc32di(crc, *(u64 *)data); + len -= 8; + data += 8; + } + if (len & 4) { + crc = __builtin_ia32_crc32si(crc, *(u32 *)data); + data += 4; + } + if (len & 2) { + crc = __builtin_ia32_crc32hi(crc, *(u16 *)data); + data += 2; + } + if (len & 1) + crc = __builtin_ia32_crc32qi(crc, *(u8 *)data); + + return crc; +} + +/* A simple hack to get reasonably solid 64bit hash values */ +u64 crc32c_64(u32 crc, const void *data, unsigned int len) +{ + unsigned int half = (len + 1) / 2; + + return ((u64)crc32c(crc, data, half) << 32) | + crc32c(~crc, data + len - half, half); +} + +u32 crc_block(struct scoutfs_block_header *hdr, u32 size) +{ + return crc32c(~0, (char *)hdr + sizeof(hdr->crc), + size - sizeof(hdr->crc)); +} diff --git a/utils/src/crc.h b/utils/src/crc.h new file mode 100644 index 00000000..a6e85ae0 --- /dev/null +++ b/utils/src/crc.h @@ -0,0 +1,12 @@ +#ifndef _CRC_H_ +#define _CRC_H_ + +#include "sparse.h" +#include "util.h" +#include "format.h" + +u32 crc32c(u32 crc, const void *data, unsigned int len); +u64 crc32c_64(u32 crc, const void *data, unsigned int len); +u32 crc_block(struct scoutfs_block_header *hdr, u32 size); + +#endif diff --git a/utils/src/dev.c b/utils/src/dev.c new file mode 100644 index 00000000..303e6438 --- /dev/null +++ b/utils/src/dev.c @@ -0,0 +1,101 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sparse.h" +#include "dev.h" + +int device_size(char *path, int fd, + u64 min_size, u64 max_size, + char *use_type, u64 *size_ret) +{ + struct stat st; + u64 size; + char *target_type; + int ret; + + if (fstat(fd, &st)) { + ret = -errno; + fprintf(stderr, "failed to stat '%s': %s (%d)\n", + path, strerror(errno), errno); + return ret; + } + + if (S_ISREG(st.st_mode)) { + size = st.st_size; + target_type = "file"; + } else if (S_ISBLK(st.st_mode)) { + if (ioctl(fd, BLKGETSIZE64, &size)) { + ret = -errno; + fprintf(stderr, "BLKGETSIZE64 failed '%s': %s (%d)\n", + path, strerror(errno), errno); + return ret; + } + target_type = "device"; + } else { + fprintf(stderr, "path isn't regular or device file '%s'\n", + path); + return -EINVAL; + } + + if (max_size) { + if (size > max_size) { + printf("Limiting use of "BASE_SIZE_FMT + " %s device to "BASE_SIZE_FMT"\n", + BASE_SIZE_ARGS(size), use_type, + BASE_SIZE_ARGS(max_size)); + size = max_size; + } else if (size < max_size) { + printf("Device size limit of "BASE_SIZE_FMT + " for %s device" + " is greater than "BASE_SIZE_FMT + " available, ignored.\n", + BASE_SIZE_ARGS(max_size), use_type, + BASE_SIZE_ARGS(size)); + } + } + + if (size < min_size) { + fprintf(stderr, + BASE_SIZE_FMT" %s too small for min " + BASE_SIZE_FMT" %s device\n", + BASE_SIZE_ARGS(size), target_type, + BASE_SIZE_ARGS(min_size), use_type); + return -EINVAL; + } + + *size_ret = size; + + return 0; +} + +float size_flt(u64 nr, unsigned size) +{ + float x = (float)nr * (float)size; + + while (x >= 1024) + x /= 1024; + + return x; +} + +char *size_str(u64 nr, unsigned size) +{ + float x = (float)nr * (float)size; + static char *suffixes[] = { + "B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB", + }; + int i = 0; + + while (x >= 1024) { + x /= 1024; + i++; + } + + return suffixes[i]; +} diff --git a/utils/src/dev.h b/utils/src/dev.h new file mode 100644 index 00000000..83dfffb4 --- /dev/null +++ b/utils/src/dev.h @@ -0,0 +1,16 @@ +#ifndef _DEV_H_ +#define _DEV_H_ + +#define BASE_SIZE_FMT "%.2f %s" +#define BASE_SIZE_ARGS(sz) size_flt(sz, 1), size_str(sz, 1) + +#define SIZE_FMT "%llu (%.2f %s)" +#define SIZE_ARGS(nr, sz) (nr), size_flt(nr, sz), size_str(nr, sz) + +int device_size(char *path, int fd, + u64 min_size, u64 max_size, + char *use_type, u64 *size_ret); +float size_flt(u64 nr, unsigned size); +char *size_str(u64 nr, unsigned size); + +#endif diff --git a/utils/src/df.c b/utils/src/df.c new file mode 100644 index 00000000..96bec369 --- /dev/null +++ b/utils/src/df.c @@ -0,0 +1,138 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sparse.h" +#include "util.h" +#include "format.h" +#include "ioctl.h" +#include "cmd.h" + +#define ROWS 3 +#define COLS 6 +#define CHARS 20 + +static int df_cmd(int argc, char **argv) +{ + struct scoutfs_ioctl_alloc_detail ad; + struct scoutfs_ioctl_alloc_detail_entry *ade = NULL; + struct scoutfs_ioctl_statfs_more sfm; + static char cells[ROWS][COLS][CHARS]; + int wid[COLS] = {0}; + u64 nr = 4096 / sizeof(*ade); + u64 meta_free = 0; + u64 data_free = 0; + int ret; + int fd; + int i; + int r; + int c; + + if (argc != 2) { + fprintf(stderr, "must specify path\n"); + return -EINVAL; + } + + fd = open(argv[1], O_RDONLY); + if (fd < 0) { + ret = -errno; + fprintf(stderr, "failed to open '%s': %s (%d)\n", + argv[1], strerror(errno), errno); + return ret; + } + + sfm.valid_bytes = sizeof(struct scoutfs_ioctl_statfs_more); + ret = ioctl(fd, SCOUTFS_IOC_STATFS_MORE, &sfm); + if (ret < 0) { + fprintf(stderr, "statfs_more returned %d: error %s (%d)\n", + ret, strerror(errno), errno); + ret = -EIO; + goto out; + } + + do { + free(ade); + ade = calloc(nr, sizeof(*ade)); + if (!ade) { + ret = -ENOMEM; + goto out; + } + + ad.entries_ptr = (intptr_t)ade; + ad.entries_nr = nr; + ret = ioctl(fd, SCOUTFS_IOC_ALLOC_DETAIL, &ad); + if (ret < 0 && errno == EOVERFLOW) + nr = nr + (nr >> 2); + } while (ret < 0 && errno == EOVERFLOW); + + if (ret < 0) { + fprintf(stderr, "alloc_detail returned %d: error %s (%d)\n", + ret, strerror(errno), errno); + ret = -EIO; + goto out; + } + + for (i = 0; i < ret; i++) { + if (ade[i].meta) + meta_free += ade[i].blocks; + else + data_free += ade[i].blocks; + } + + snprintf(cells[0][0], CHARS, "Type"); + snprintf(cells[0][1], CHARS, "Size"); + snprintf(cells[0][2], CHARS, "Total"); + snprintf(cells[0][3], CHARS, "Used"); + snprintf(cells[0][4], CHARS, "Free"); + snprintf(cells[0][5], CHARS, "Use%%"); + + snprintf(cells[1][0], CHARS, "MetaData"); + snprintf(cells[1][1], CHARS, "64KB"); + snprintf(cells[1][2], CHARS, "%llu", sfm.total_meta_blocks); + snprintf(cells[1][3], CHARS, "%llu", sfm.total_meta_blocks - meta_free); + snprintf(cells[1][4], CHARS, "%llu", meta_free); + snprintf(cells[1][5], CHARS, "%llu", + ((sfm.total_meta_blocks - meta_free) * 100) / + sfm.total_meta_blocks); + + snprintf(cells[2][0], CHARS, "Data"); + snprintf(cells[2][1], CHARS, "4KB"); + snprintf(cells[2][2], CHARS, "%llu", sfm.total_data_blocks); + snprintf(cells[2][3], CHARS, "%llu", sfm.total_data_blocks - data_free); + snprintf(cells[2][4], CHARS, "%llu", data_free); + snprintf(cells[2][5], CHARS, "%llu", + ((sfm.total_data_blocks - data_free) * 100) / + sfm.total_data_blocks); + + for (r = 0; r < ROWS; r++) { + for (c = 0; c < COLS; c++) { + wid[c] = max(wid[c], strlen(cells[r][c])); + } + } + + for (r = 0; r < ROWS; r++) { + for (c = 0; c < COLS; c++) { + printf("%*s ", wid[c], cells[r][c]); + } + printf("\n"); + } + + ret = 0; +out: + free(ade); + return ret; +} + +static void __attribute__((constructor)) df_ctor(void) +{ + cmd_register("df", "", + "show metadata and data block usage", df_cmd); +} diff --git a/utils/src/endian_swap.h b/utils/src/endian_swap.h new file mode 100644 index 00000000..fae7d8df --- /dev/null +++ b/utils/src/endian_swap.h @@ -0,0 +1,15 @@ +#ifndef _SCOUTFS_ENDIAN_SWAP_H_ +#define _SCOUTFS_ENDIAN_SWAP_H_ + +#define le64_to_be64(x) cpu_to_be64(le64_to_cpu(x)) +#define le32_to_be32(x) cpu_to_be32(le32_to_cpu(x)) +#define le16_to_be16(x) cpu_to_be16(le16_to_cpu(x)) + +#define be64_to_le64(x) cpu_to_le64(be64_to_cpu(x)) +#define be32_to_le32(x) cpu_to_le32(be32_to_cpu(x)) +#define be16_to_le16(x) cpu_to_le16(be16_to_cpu(x)) + +#define le16_to_le64(x) cpu_to_le64(le16_to_cpu(x)) +#define le32_to_le64(x) cpu_to_le64(le32_to_cpu(x)) + +#endif diff --git a/utils/src/format.h b/utils/src/format.h new file mode 100644 index 00000000..033552bf --- /dev/null +++ b/utils/src/format.h @@ -0,0 +1,946 @@ +#ifndef _SCOUTFS_FORMAT_H_ +#define _SCOUTFS_FORMAT_H_ + +/* statfs(2) f_type */ +#define SCOUTFS_SUPER_MAGIC 0x554f4353 /* "SCOU" */ + +/* block header magic values, chosen at random */ +#define SCOUTFS_BLOCK_MAGIC_SUPER 0x103c428b +#define SCOUTFS_BLOCK_MAGIC_BTREE 0xe597f96d +#define SCOUTFS_BLOCK_MAGIC_BLOOM 0x31995604 +#define SCOUTFS_BLOCK_MAGIC_SRCH_BLOCK 0x897e4a7d +#define SCOUTFS_BLOCK_MAGIC_SRCH_PARENT 0xb23a2a05 +#define SCOUTFS_BLOCK_MAGIC_ALLOC_LIST 0x8a93ac83 + +/* + * The super block, quorum block, and file data allocation granularity + * use the smaller 4KB block. + */ +#define SCOUTFS_BLOCK_SM_SHIFT 12 +#define SCOUTFS_BLOCK_SM_SIZE (1 << SCOUTFS_BLOCK_SM_SHIFT) +#define SCOUTFS_BLOCK_SM_MASK (SCOUTFS_BLOCK_SM_SIZE - 1) +#define SCOUTFS_BLOCK_SM_PER_PAGE (PAGE_SIZE / SCOUTFS_BLOCK_SM_SIZE) +#define SCOUTFS_BLOCK_SM_SECTOR_SHIFT (SCOUTFS_BLOCK_SM_SHIFT - 9) +#define SCOUTFS_BLOCK_SM_SECTORS (1 << SCOUTFS_BLOCK_SM_SECTOR_SHIFT) +#define SCOUTFS_BLOCK_SM_MAX (U64_MAX >> SCOUTFS_BLOCK_SM_SHIFT) +#define SCOUTFS_BLOCK_SM_PAGES_PER (SCOUTFS_BLOCK_SM_SIZE / PAGE_SIZE) +#define SCOUTFS_BLOCK_SM_PAGE_ORDER (SCOUTFS_BLOCK_SM_SHIFT - PAGE_SHIFT) + +/* + * The radix and btree structures, and the forest bloom block, use the + * larger 64KB metadata block size. + */ +#define SCOUTFS_BLOCK_LG_SHIFT 16 +#define SCOUTFS_BLOCK_LG_SIZE (1 << SCOUTFS_BLOCK_LG_SHIFT) +#define SCOUTFS_BLOCK_LG_MASK (SCOUTFS_BLOCK_LG_SIZE - 1) +#define SCOUTFS_BLOCK_LG_PER_PAGE (PAGE_SIZE / SCOUTFS_BLOCK_LG_SIZE) +#define SCOUTFS_BLOCK_LG_SECTOR_SHIFT (SCOUTFS_BLOCK_LG_SHIFT - 9) +#define SCOUTFS_BLOCK_LG_SECTORS (1 << SCOUTFS_BLOCK_LG_SECTOR_SHIFT) +#define SCOUTFS_BLOCK_LG_MAX (U64_MAX >> SCOUTFS_BLOCK_LG_SHIFT) +#define SCOUTFS_BLOCK_LG_PAGES_PER (SCOUTFS_BLOCK_LG_SIZE / PAGE_SIZE) +#define SCOUTFS_BLOCK_LG_PAGE_ORDER (SCOUTFS_BLOCK_LG_SHIFT - PAGE_SHIFT) + +#define SCOUTFS_BLOCK_SM_LG_SHIFT (SCOUTFS_BLOCK_LG_SHIFT - \ + SCOUTFS_BLOCK_SM_SHIFT) + + +/* + * The super block leaves some room before the first block for platform + * structures like boot loaders. + */ +#define SCOUTFS_SUPER_BLKNO ((64ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT) + +/* + * A reasonably large region of aligned quorum blocks follow the super + * block. Each voting cycle reads the entire region so we don't want it + * to be too enormous. 256K seems like a reasonably chunky single IO. + * The number of blocks in the region also determines the number of + * mounts that have a reasonable probability of not overwriting each + * other's random block locations. + */ +#define SCOUTFS_QUORUM_BLKNO ((256ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT) +#define SCOUTFS_QUORUM_BLOCKS ((256ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT) + +/* + * Start data on the data device aligned as well. + */ +#define SCOUTFS_DATA_DEV_START_BLKNO ((256ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT) + + +#define SCOUTFS_UNIQUE_NAME_MAX_BYTES 64 /* includes null */ + +/* + * Base types used by other structures. + */ +struct scoutfs_timespec { + __le64 sec; + __le32 nsec; + __u8 __pad[4]; +}; + +/* XXX ipv6 */ +struct scoutfs_inet_addr { + __le32 addr; + __le16 port; + __u8 __pad[2]; +}; + +/* + * This header is stored at the start of btree blocks and the super + * block for verification. The crc field is not included in the + * calculation of the crc. + */ +struct scoutfs_block_header { + __le32 crc; + __le32 magic; + __le64 fsid; + __le64 seq; + __le64 blkno; +}; + +/* + * scoutfs identifies all file system metadata items by a small key + * struct. + * + * Each item type maps their logical structures to the fixed fields in + * sort order. This lets us print keys without needing per-type + * formats. + * + * The keys are compared by considering the fields in struct order from + * most to least significant. They are considered a multi precision + * value when navigating the keys in ordered key space. We can + * increment them, subtract them from each other, etc. + */ +struct scoutfs_key { + __le64 _sk_first; + __le64 _sk_second; + __le64 _sk_third; + __u8 _sk_fourth; + __u8 sk_zone; + __u8 sk_type; + __u8 __pad[5]; +}; + +/* inode index */ +#define skii_major _sk_second +#define skii_ino _sk_third + +/* node orphan inode */ +#define sko_rid _sk_first +#define sko_ino _sk_second + +/* inode */ +#define ski_ino _sk_first + +/* xattr parts */ +#define skx_ino _sk_first +#define skx_name_hash _sk_second +#define skx_id _sk_third +#define skx_part _sk_fourth + +/* directory entries */ +#define skd_ino _sk_first +#define skd_major _sk_second +#define skd_minor _sk_third + +/* symlink target */ +#define sks_ino _sk_first +#define sks_nr _sk_second + +/* data extents */ +#define skdx_ino _sk_first +#define skdx_end _sk_second +#define skdx_len _sk_third + +/* log trees */ +#define sklt_rid _sk_first +#define sklt_nr _sk_second + +/* lock clients */ +#define sklc_rid _sk_first + +/* seqs */ +#define skts_trans_seq _sk_first +#define skts_rid _sk_second + +/* mounted clients */ +#define skmc_rid _sk_first + +/* free extents by blkno */ +#define skfb_end _sk_second +#define skfb_len _sk_third +/* free extents by len */ +#define skfl_neglen _sk_second +#define skfl_blkno _sk_third + +struct scoutfs_radix_block { + struct scoutfs_block_header hdr; + union { + struct scoutfs_radix_ref { + __le64 blkno; + __le64 seq; + __le64 sm_total; + __le64 lg_total; + } refs[0]; + __le64 bits[0]; + }; +}; + +struct scoutfs_avl_root { + __le16 node; +}; + +struct scoutfs_avl_node { + __le16 parent; + __le16 left; + __le16 right; + __u8 height; + __u8 __pad[1]; +}; + +/* when we split we want to have multiple items on each side */ +#define SCOUTFS_BTREE_MAX_VAL_LEN 896 + +/* + * A 4EB test image measured a worst case height of 17. This is plenty + * generous. + */ +#define SCOUTFS_BTREE_MAX_HEIGHT 20 + +struct scoutfs_btree_ref { + __le64 blkno; + __le64 seq; +}; + +/* + * A height of X means that the first block read will have level X-1 and + * the leaves will have level 0. + */ +struct scoutfs_btree_root { + struct scoutfs_btree_ref ref; + __u8 height; + __u8 __pad[7]; +}; + +struct scoutfs_btree_item { + struct scoutfs_avl_node node; + struct scoutfs_key key; + __le16 val_off; + __le16 val_len; + __u8 __pad[4]; +}; + +struct scoutfs_btree_block { + struct scoutfs_block_header hdr; + struct scoutfs_avl_root item_root; + __le16 nr_items; + __le16 total_item_bytes; + __le16 mid_free_len; + __u8 level; + __u8 __pad[7]; + struct scoutfs_btree_item items[0]; + /* leaf blocks have a fixed size item offset hash table at the end */ +}; + +#define SCOUTFS_BTREE_VALUE_ALIGN 8 + +/* + * Try to aim for a 75% load in a leaf full of items with no value. + * We'll almost never see this because most items have values and most + * blocks aren't full. + */ +#define SCOUTFS_BTREE_LEAF_ITEM_HASH_NR_UNALIGNED \ + ((SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_btree_block)) / \ + (sizeof(struct scoutfs_btree_item) + (sizeof(__le16))) * 100 / 75) +#define SCOUTFS_BTREE_LEAF_ITEM_HASH_NR \ + (round_up(SCOUTFS_BTREE_LEAF_ITEM_HASH_NR_UNALIGNED, \ + SCOUTFS_BTREE_VALUE_ALIGN)) +#define SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES \ + (SCOUTFS_BTREE_LEAF_ITEM_HASH_NR * sizeof(__le16)) + +struct scoutfs_alloc_list_ref { + __le64 blkno; + __le64 seq; +}; + +/* + * first_nr tracks the nr of the first block in the list and is used for + * allocation sizing. total_nr is the sum of the nr of all the blocks in + * the list and is used for calculating total free block counts. + */ +struct scoutfs_alloc_list_head { + struct scoutfs_alloc_list_ref ref; + __le64 total_nr; + __le32 first_nr; + __u8 __pad[4]; +}; + +/* + * While the main allocator uses extent items in btree blocks, metadata + * allocations for a single transaction are recorded in arrays in + * blocks. This limits the number of allocations and frees needed to + * cow and modify the structure. The blocks can be stored in a list + * which lets us create a persistent log of pending frees that are + * generated as we cow btree blocks to insert freed extents. + * + * The array floats in the block so that both adding and removing blknos + * only modifies an index. + */ +struct scoutfs_alloc_list_block { + struct scoutfs_block_header hdr; + struct scoutfs_alloc_list_ref next; + __le32 start; + __le32 nr; + __le64 blknos[0]; /* naturally aligned for sorting */ +}; + +#define SCOUTFS_ALLOC_LIST_MAX_BLOCKS \ + ((SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_alloc_list_block)) / \ + (member_sizeof(struct scoutfs_alloc_list_block, blknos[0]))) + +/* + * These can safely be initialized to all-zeros. + */ +struct scoutfs_alloc_root { + __le64 total_len; + struct scoutfs_btree_root root; +}; + +/* types of allocators, exposed to alloc_detail ioctl */ +#define SCOUTFS_ALLOC_OWNER_NONE 0 +#define SCOUTFS_ALLOC_OWNER_SERVER 1 +#define SCOUTFS_ALLOC_OWNER_MOUNT 2 +#define SCOUTFS_ALLOC_OWNER_SRCH 3 + +struct scoutfs_mounted_client_btree_val { + __u8 flags; +}; + +#define SCOUTFS_MOUNTED_CLIENT_VOTER (1 << 0) + +/* + * srch files are a contiguous run of blocks with compressed entries + * described by a dense parent radix. The files can be stored in + * log_tree items when the files contain unsorted entries written by + * mounts during their transactions. Sorted files of increasing size + * are kept in a btree off the super for searching and further + * compacting. + */ +struct scoutfs_srch_entry { + __le64 hash; + __le64 ino; + __le64 id; +}; + +#define SCOUTFS_SRCH_ENTRY_MAX_BYTES (2 + (sizeof(__u64) * 3)) + +struct scoutfs_srch_ref { + __le64 blkno; + __le64 seq; +}; + +struct scoutfs_srch_file { + struct scoutfs_srch_entry first; + struct scoutfs_srch_entry last; + struct scoutfs_srch_ref ref; + __le64 blocks; + __le64 entries; + __u8 height; + __u8 __pad[7]; +}; + +struct scoutfs_srch_parent { + struct scoutfs_block_header hdr; + struct scoutfs_srch_ref refs[0]; +}; + +#define SCOUTFS_SRCH_PARENT_REFS \ + ((SCOUTFS_BLOCK_LG_SIZE - \ + offsetof(struct scoutfs_srch_parent, refs)) / \ + sizeof(struct scoutfs_srch_ref)) + +struct scoutfs_srch_block { + struct scoutfs_block_header hdr; + struct scoutfs_srch_entry first; + struct scoutfs_srch_entry last; + struct scoutfs_srch_entry tail; + __le32 entry_nr; + __le32 entry_bytes; + __u8 entries[0]; +}; + +/* + * Decoding loads final small deltas with full __u64 loads. Rather than + * check the size before each load we stop coding entries past the point + * where a full size entry could overflow the block. A final entry can + * start at this byte count and consume the rest of the block, though + * its unlikely. + */ +#define SCOUTFS_SRCH_BLOCK_SAFE_BYTES \ + (SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_srch_block) - \ + SCOUTFS_SRCH_ENTRY_MAX_BYTES) + +#define SCOUTFS_SRCH_LOG_BLOCK_LIMIT (1024 * 1024 / SCOUTFS_BLOCK_LG_SIZE) +#define SCOUTFS_SRCH_COMPACT_ORDER 2 +#define SCOUTFS_SRCH_COMPACT_NR (1 << SCOUTFS_SRCH_COMPACT_ORDER) + +/* + * A persistent record of a srch file compaction operation in progress. + * + * When compacting log files blk and pos aren't used. When compacting + * sorted files blk is the logical block number and pos is the byte + * offset of the next entry. When deleting files pos is the height of + * the level that we're deleting, and blk is the logical block offset of + * the next parent ref array index to descend through. + */ +struct scoutfs_srch_compact { + struct scoutfs_alloc_list_head meta_avail; + struct scoutfs_alloc_list_head meta_freed; + __le64 id; + __u8 nr; + __u8 flags; + __u8 __pad[6]; + struct scoutfs_srch_file out; + struct scoutfs_srch_compact_input { + struct scoutfs_srch_file sfl; + __le64 blk; + __le64 pos; + } in[SCOUTFS_SRCH_COMPACT_NR]; +}; + +/* server -> client: combine input log file entries into output file */ +#define SCOUTFS_SRCH_COMPACT_FLAG_LOG (1 << 0) +/* server -> client: combine input sorted file entries into output file */ +#define SCOUTFS_SRCH_COMPACT_FLAG_SORTED (1 << 1) +/* server -> client: delete input files */ +#define SCOUTFS_SRCH_COMPACT_FLAG_DELETE (1 << 2) +/* client -> server: compaction phase (LOG,SORTED,DELETE) done */ +#define SCOUTFS_SRCH_COMPACT_FLAG_DONE (1 << 4) +/* client -> server: compaction failed */ +#define SCOUTFS_SRCH_COMPACT_FLAG_ERROR (1 << 5) + +/* + * XXX I imagine we should rename these now that they've evolved to track + * all the btrees that clients use during a transaction. It's not just + * about item logs, it's about clients making changes to trees. + */ +struct scoutfs_log_trees { + struct scoutfs_alloc_list_head meta_avail; + struct scoutfs_alloc_list_head meta_freed; + struct scoutfs_btree_root item_root; + struct scoutfs_btree_ref bloom_ref; + struct scoutfs_alloc_root data_avail; + struct scoutfs_alloc_root data_freed; + struct scoutfs_srch_file srch_file; + __le64 max_item_vers; + __le64 rid; + __le64 nr; +}; + +struct scoutfs_log_item_value { + __le64 vers; + __u8 flags; + __u8 __pad[7]; + __u8 data[0]; +}; + +/* + * FS items are limited by the max btree value length with the log item + * value header. + */ +#define SCOUTFS_MAX_VAL_SIZE \ + (SCOUTFS_BTREE_MAX_VAL_LEN - sizeof(struct scoutfs_log_item_value)) + +#define SCOUTFS_LOG_ITEM_FLAG_DELETION (1 << 0) + +struct scoutfs_bloom_block { + struct scoutfs_block_header hdr; + __le64 total_set; + __le64 bits[0]; +}; + +/* + * Item log trees are accompanied by a block of bits that make up a + * bloom filter which indicate if the item log trees may contain items + * covered by a lock. The log trees should be finalized and merged long + * before the bloom filters fill up and start returning excessive false + * positives. + */ +#define SCOUTFS_FOREST_BLOOM_NRS 3 +#define SCOUTFS_FOREST_BLOOM_BITS \ + (((SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_bloom_block)) / \ + member_sizeof(struct scoutfs_bloom_block, bits[0])) * \ + member_sizeof(struct scoutfs_bloom_block, bits[0]) * 8) +#define SCOUTFS_FOREST_BLOOM_FUNC_BITS (SCOUTFS_BLOCK_LG_SHIFT + 3) + +/* + * Keys are first sorted by major key zones. + */ +#define SCOUTFS_INODE_INDEX_ZONE 1 +#define SCOUTFS_RID_ZONE 2 +#define SCOUTFS_FS_ZONE 3 +#define SCOUTFS_LOCK_ZONE 4 +/* Items only stored in server btrees */ +#define SCOUTFS_LOG_TREES_ZONE 6 +#define SCOUTFS_LOCK_CLIENTS_ZONE 7 +#define SCOUTFS_TRANS_SEQ_ZONE 8 +#define SCOUTFS_MOUNTED_CLIENT_ZONE 9 +#define SCOUTFS_SRCH_ZONE 10 +#define SCOUTFS_FREE_EXTENT_ZONE 11 + +/* inode index zone */ +#define SCOUTFS_INODE_INDEX_META_SEQ_TYPE 1 +#define SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE 2 +#define SCOUTFS_INODE_INDEX_NR 3 /* don't forget to update */ + +/* rid zone (also used in server alloc btree) */ +#define SCOUTFS_ORPHAN_TYPE 1 + +/* fs zone */ +#define SCOUTFS_INODE_TYPE 1 +#define SCOUTFS_XATTR_TYPE 2 +#define SCOUTFS_DIRENT_TYPE 3 +#define SCOUTFS_READDIR_TYPE 4 +#define SCOUTFS_LINK_BACKREF_TYPE 5 +#define SCOUTFS_SYMLINK_TYPE 6 +#define SCOUTFS_DATA_EXTENT_TYPE 7 + +/* lock zone, only ever found in lock ranges, never in persistent items */ +#define SCOUTFS_RENAME_TYPE 1 + +/* srch zone, only in server btrees */ +#define SCOUTFS_SRCH_LOG_TYPE 1 +#define SCOUTFS_SRCH_BLOCKS_TYPE 2 +#define SCOUTFS_SRCH_PENDING_TYPE 3 +#define SCOUTFS_SRCH_BUSY_TYPE 4 + +/* free extents in allocator btrees in client and server, by blkno or len */ +#define SCOUTFS_FREE_EXTENT_BLKNO_TYPE 1 +#define SCOUTFS_FREE_EXTENT_LEN_TYPE 2 + +/* file data extents have start and len in key */ +struct scoutfs_data_extent_val { + __le64 blkno; + __u8 flags; + __u8 __pad[7]; +}; + +#define SEF_OFFLINE (1 << 0) +#define SEF_UNWRITTEN (1 << 1) +#define SEF_UNKNOWN (U8_MAX << 2) + +/* + * The first xattr part item has a header that describes the xattr. The + * name and value are then packed into the following bytes in the first + * part item and overflow into the values of the rest of the part items. + */ +struct scoutfs_xattr { + __le16 val_len; + __u8 name_len; + __u8 __pad[5]; + __u8 name[0]; +}; + + +/* XXX does this exist upstream somewhere? */ +#define member_sizeof(TYPE, MEMBER) (sizeof(((TYPE *)0)->MEMBER)) + +#define SCOUTFS_UUID_BYTES 16 + +/* + * Mounts read all the quorum blocks and write to one random quorum + * block during a cycle. The min cycle time limits the per-mount iop + * load during elections. The random cycle delay makes it less likely + * that mounts will read and write at the same time and miss each + * other's writes. An election only completes if a quorum of mounts + * vote for a leader before any of their elections timeout. This is + * made less likely by the probability that mounts will overwrite each + * others random block locations. The max quorum count limits that + * probability. 9 mounts only have a 55% chance of writing to unique 4k + * blocks in a 256k region. The election timeout is set to include + * enough cycles to usually complete the election. Once a leader is + * elected it spends a number of cycles writing out blocks with itself + * logged as a leader. This reduces the possibility that servers + * will have their log entries overwritten and not be fenced. + */ +#define SCOUTFS_QUORUM_MAX_COUNT 9 +#define SCOUTFS_QUORUM_CYCLE_LO_MS 10 +#define SCOUTFS_QUORUM_CYCLE_HI_MS 20 +#define SCOUTFS_QUORUM_TERM_LO_MS 250 +#define SCOUTFS_QUORUM_TERM_HI_MS 500 +#define SCOUTFS_QUORUM_ELECTED_LOG_CYCLES 10 + +struct scoutfs_quorum_block { + __le64 fsid; + __le64 blkno; + __le64 term; + __le64 write_nr; + __le64 voter_rid; + __le64 vote_for_rid; + __le32 crc; + __u8 log_nr; + __u8 __pad[3]; + struct scoutfs_quorum_log { + __le64 term; + __le64 rid; + struct scoutfs_inet_addr addr; + } log[0]; +}; + +#define SCOUTFS_QUORUM_LOG_MAX \ + ((SCOUTFS_BLOCK_SM_SIZE - sizeof(struct scoutfs_quorum_block)) / \ + sizeof(struct scoutfs_quorum_log)) + +#define SCOUTFS_FLAG_IS_META_BDEV 0x01 + +struct scoutfs_super_block { + struct scoutfs_block_header hdr; + __le64 id; + __le64 format_hash; + __le64 flags; + __u8 uuid[SCOUTFS_UUID_BYTES]; + __le64 next_ino; + __le64 next_trans_seq; + __le64 total_meta_blocks; /* both static and dynamic */ + __le64 first_meta_blkno; /* first dynamically allocated */ + __le64 last_meta_blkno; + __le64 total_data_blocks; + __le64 first_data_blkno; + __le64 last_data_blkno; + __le64 quorum_fenced_term; + __le64 quorum_server_term; + __le64 unmount_barrier; + __u8 quorum_count; + __u8 __pad[7]; + struct scoutfs_inet_addr server_addr; + struct scoutfs_alloc_root meta_alloc[2]; + struct scoutfs_alloc_root data_alloc; + struct scoutfs_alloc_list_head server_meta_avail[2]; + struct scoutfs_alloc_list_head server_meta_freed[2]; + struct scoutfs_btree_root fs_root; + struct scoutfs_btree_root logs_root; + struct scoutfs_btree_root lock_clients; + struct scoutfs_btree_root trans_seqs; + struct scoutfs_btree_root mounted_clients; + struct scoutfs_btree_root srch_root; +}; + +#define SCOUTFS_ROOT_INO 1 + + +/* + * @meta_seq: advanced the first time an inode is updated in a given + * transaction. It can only advance again after the inode is written + * and a new transaction opens. + * + * @data_seq: advanced the first time a file's data (or size) is + * modified in a given transaction. It can only advance again after the + * file is written and a new transaction opens. + * + * @data_version: incremented every time the contents of a file could + * have changed. It is exposed via an ioctl and is then provided as an + * argument to data functions to protect racing modification. + * + * @online_blocks: The number of fixed 4k blocks currently allocated and + * storing data in the volume. + * + * @offline_blocks: The number of fixed 4k blocks that could be made + * online by staging. + * + * XXX + * - otime? + * - compat flags? + * - version? + * - generation? + * - be more careful with rdev? + */ +struct scoutfs_inode { + __le64 size; + __le64 meta_seq; + __le64 data_seq; + __le64 data_version; + __le64 online_blocks; + __le64 offline_blocks; + __le64 next_readdir_pos; + __le64 next_xattr_id; + __le32 nlink; + __le32 uid; + __le32 gid; + __le32 mode; + __le32 rdev; + __le32 flags; + struct scoutfs_timespec atime; + struct scoutfs_timespec ctime; + struct scoutfs_timespec mtime; +}; + +#define SCOUTFS_INO_FLAG_TRUNCATE 0x1 + +#define SCOUTFS_ROOT_INO 1 + +/* like the block size, a reasonable min PATH_MAX across platforms */ +#define SCOUTFS_SYMLINK_MAX_SIZE 4096 + +/* + * Dirents are stored in multiple places to isolate contention when + * performing different operations: hashed by name for creation and + * lookup, at incrementing positions for readdir and resolving inodes to + * paths. Each entry has all the metadata needed to reference all the + * items (so an entry cached by lookup can be used to unlink all the + * items). + */ +struct scoutfs_dirent { + __le64 ino; + __le64 hash; + __le64 pos; + __u8 type; + __u8 __pad[7]; + __u8 name[0]; +}; + +#define SCOUTFS_NAME_LEN 255 + +/* S32_MAX avoids the (int) sign bit and might avoid sloppy bugs */ +#define SCOUTFS_LINK_MAX S32_MAX + +/* entries begin after . and .. */ +#define SCOUTFS_DIRENT_FIRST_POS 2 +/* getdents returns next pos with an entry, no entry at (f_pos)~0 */ +#define SCOUTFS_DIRENT_LAST_POS (U64_MAX - 1) + +enum scoutfs_dentry_type { + SCOUTFS_DT_FIFO = 0, + SCOUTFS_DT_CHR, + SCOUTFS_DT_DIR, + SCOUTFS_DT_BLK, + SCOUTFS_DT_REG, + SCOUTFS_DT_LNK, + SCOUTFS_DT_SOCK, + SCOUTFS_DT_WHT, +}; + + +#define SCOUTFS_XATTR_MAX_NAME_LEN 255 +#define SCOUTFS_XATTR_MAX_VAL_LEN 65535 +#define SCOUTFS_XATTR_MAX_PART_SIZE SCOUTFS_MAX_VAL_SIZE + +#define SCOUTFS_XATTR_NR_PARTS(name_len, val_len) \ + DIV_ROUND_UP(sizeof(struct scoutfs_xattr) + name_len + val_len, \ + (unsigned int)SCOUTFS_XATTR_MAX_PART_SIZE) + +#define SCOUTFS_LOCK_INODE_GROUP_NR 1024 +#define SCOUTFS_LOCK_INODE_GROUP_MASK (SCOUTFS_LOCK_INODE_GROUP_NR - 1) +#define SCOUTFS_LOCK_SEQ_GROUP_MASK ((1ULL << 10) - 1) + +/* + * messages over the wire. + */ + +/* + * Greetings verify identity of communicating nodes. The sender sends + * their credentials and the receiver verifies them. + * + * @server_term: The raft term that elected the server. Initially 0 + * from the client, sent by the server, then sent by the client as it + * tries to reconnect. Used to identify a client reconnecting to both + * the same serer after receiving a greeting response and to a new + * server after failover. + * + * @unmount_barrier: Incremented every time the remaining majority of + * quorum members all agree to leave. The server tells a quorum member + * the value that it's connecting under so that if the client sees the + * value increase in the super block then it knows that the server has + * processed its farewell and can safely unmount. + * + * @rid: The client's random id that was generated once as the mount + * started up. This identifies a specific remote mount across + * connections and servers. It's set to the client's rid in both the + * request and response for consistency. + */ +struct scoutfs_net_greeting { + __le64 fsid; + __le64 format_hash; + __le64 server_term; + __le64 unmount_barrier; + __le64 rid; + __le64 flags; +}; + +#define SCOUTFS_NET_GREETING_FLAG_FAREWELL (1 << 0) +#define SCOUTFS_NET_GREETING_FLAG_VOTER (1 << 1) +#define SCOUTFS_NET_GREETING_FLAG_INVALID (~(__u64)0 << 2) + +/* + * This header precedes and describes all network messages sent over + * sockets. + * + * @seq: A sequence number that is increased for each message queued for + * send on the sender. The sender will never reorder messages in the + * send queue so this will always increase in recv on the receiver. The + * receiver can use this to drop messages that arrived twice after being + * resent across a newly connected socket for a given connection. + * + * @recv_seq: The sequence number of the last received message. The + * receiver is sending this to the sender in every message. The sender + * uses them to drop responses which have been delivered. + * + * @id: An increasing identifier that is set in each request. Responses + * specify the request that they're responding to. + * + * Error is only set to a translated errno and will only be found in + * response messages. + */ +struct scoutfs_net_header { + __le64 clock_sync_id; + __le64 seq; + __le64 recv_seq; + __le64 id; + __le16 data_len; + __u8 cmd; + __u8 flags; + __u8 error; + __u8 __pad[3]; + __u8 data[0]; +}; + +#define SCOUTFS_NET_FLAG_RESPONSE (1 << 0) +#define SCOUTFS_NET_FLAGS_UNKNOWN (U8_MAX << 1) + +enum scoutfs_net_cmd { + SCOUTFS_NET_CMD_GREETING = 0, + SCOUTFS_NET_CMD_ALLOC_INODES, + SCOUTFS_NET_CMD_GET_LOG_TREES, + SCOUTFS_NET_CMD_COMMIT_LOG_TREES, + SCOUTFS_NET_CMD_GET_ROOTS, + SCOUTFS_NET_CMD_ADVANCE_SEQ, + SCOUTFS_NET_CMD_GET_LAST_SEQ, + SCOUTFS_NET_CMD_LOCK, + SCOUTFS_NET_CMD_LOCK_RECOVER, + SCOUTFS_NET_CMD_SRCH_GET_COMPACT, + SCOUTFS_NET_CMD_SRCH_COMMIT_COMPACT, + SCOUTFS_NET_CMD_FAREWELL, + SCOUTFS_NET_CMD_UNKNOWN, +}; + +/* + * Define a macro to evaluate another macro for each of the errnos we + * translate over the wire. This lets us keep our enum in sync with the + * mapping arrays to and from host errnos. + */ +#define EXPAND_EACH_NET_ERRNO \ + EXPAND_NET_ERRNO(ENOENT) \ + EXPAND_NET_ERRNO(ENOMEM) \ + EXPAND_NET_ERRNO(EIO) \ + EXPAND_NET_ERRNO(ENOSPC) \ + EXPAND_NET_ERRNO(EINVAL) + +#undef EXPAND_NET_ERRNO +#define EXPAND_NET_ERRNO(which) SCOUTFS_NET_ERR_##which, +enum scoutfs_net_errors { + SCOUTFS_NET_ERR_NONE = 0, + EXPAND_EACH_NET_ERRNO + SCOUTFS_NET_ERR_UNKNOWN, +}; + +/* arbitrarily chosen to be safely less than mss and allow 1k with header */ +#define SCOUTFS_NET_MAX_DATA_LEN 1100 + +/* + * When there's no more free inodes this will be sent with ino = ~0 and + * nr = 0. + */ +struct scoutfs_net_inode_alloc { + __le64 ino; + __le64 nr; +}; + +struct scoutfs_net_roots { + struct scoutfs_btree_root fs_root; + struct scoutfs_btree_root logs_root; + struct scoutfs_btree_root srch_root; +}; + +struct scoutfs_net_lock { + struct scoutfs_key key; + __le64 write_version; + __u8 old_mode; + __u8 new_mode; + __u8 __pad[6]; +}; + +struct scoutfs_net_lock_grant_response { + struct scoutfs_net_lock nl; + struct scoutfs_net_roots roots; +}; + +struct scoutfs_net_lock_recover { + __le16 nr; + __u8 __pad[6]; + struct scoutfs_net_lock locks[0]; +}; + +#define SCOUTFS_NET_LOCK_MAX_RECOVER_NR \ + ((SCOUTFS_NET_MAX_DATA_LEN - sizeof(struct scoutfs_net_lock_recover)) /\ + sizeof(struct scoutfs_net_lock)) + +/* some enums for tracing */ +enum scoutfs_lock_trace { + SLT_CLIENT, + SLT_SERVER, + SLT_GRANT, + SLT_INVALIDATE, + SLT_REQUEST, + SLT_RESPONSE, +}; + +/* + * Read and write locks operate as you'd expect. Multiple readers can + * hold read locks while writers are excluded. A single writer can hold + * a write lock which excludes other readers and writers. Writers can + * read while holding a write lock. + * + * Multiple writers can hold write only locks but they can not read, + * they can only generate dirty items. It's used when the system has + * other means of knowing that it's safe to overwrite items. + * + * The null mode provides no access and is used to destroy locks. + */ +enum scoutfs_lock_mode { + SCOUTFS_LOCK_NULL = 0, + SCOUTFS_LOCK_READ, + SCOUTFS_LOCK_WRITE, + SCOUTFS_LOCK_WRITE_ONLY, + SCOUTFS_LOCK_INVALID, +}; + +/* + * Scoutfs file handle structure - this can be copied out to userspace + * via open by handle or put on the wire from NFS. + */ +struct scoutfs_fid { + __le64 ino; + __le64 parent_ino; +}; + +#define FILEID_SCOUTFS 0x81 +#define FILEID_SCOUTFS_WITH_PARENT 0x82 + +/* + * Identifiers for sources of corruption that can generate messages. + */ +enum scoutfs_corruption_sources { + SC_DIRENT_NAME_LEN = 0, + SC_DIRENT_BACKREF_NAME_LEN, + SC_DIRENT_READDIR_NAME_LEN, + SC_SYMLINK_INODE_SIZE, + SC_SYMLINK_MISSING_ITEM, + SC_SYMLINK_NOT_NULL_TERM, + SC_BTREE_BLOCK_LEVEL, + SC_BTREE_NO_CHILD_REF, + SC_INODE_BLOCK_COUNTS, + SC_NR_SOURCES, +}; + +#define SC_NR_LONGS DIV_ROUND_UP(SC_NR_SOURCES, BITS_PER_LONG) + +#endif diff --git a/utils/src/hash.h b/utils/src/hash.h new file mode 100644 index 00000000..cb50b99c --- /dev/null +++ b/utils/src/hash.h @@ -0,0 +1,49 @@ +#ifndef _SCOUTFS_HASH_H_ +#define _SCOUTFS_HASH_H_ + +/* + * We're using FNV1a for now. It's fine. Ish. + * + * The longer term plan is xxh3 but it looks like it'll take just a bit + * more time to be declared stable and then it needs to be ported to the + * kernel. + * + * - https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * - https://github.com/Cyan4973/xxHash/releases/tag/v0.7.4 + */ + +static inline u32 fnv1a32(const void *data, unsigned int len) +{ + u32 hash = 0x811c9dc5; + + while (len--) { + hash ^= *(u8 *)(data++); + hash *= 0x01000193; + } + + return hash; +} + +static inline u64 fnv1a64(const void *data, unsigned int len) +{ + u64 hash = 0xcbf29ce484222325ULL; + + while (len--) { + hash ^= *(u8 *)(data++); + hash *= 0x100000001b3ULL; + } + + return hash; +} + +static inline u32 scoutfs_hash32(const void *data, unsigned int len) +{ + return fnv1a32(data, len); +} + +static inline u64 scoutfs_hash64(const void *data, unsigned int len) +{ + return fnv1a64(data, len); +} + +#endif diff --git a/utils/src/ino_path.c b/utils/src/ino_path.c new file mode 100644 index 00000000..91cbd184 --- /dev/null +++ b/utils/src/ino_path.c @@ -0,0 +1,99 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sparse.h" +#include "util.h" +#include "format.h" +#include "ioctl.h" +#include "cmd.h" + +static int ino_path_cmd(int argc, char **argv) +{ + struct scoutfs_ioctl_ino_path args; + struct scoutfs_ioctl_ino_path_result *res; + unsigned int result_bytes; + char *endptr = NULL; + u64 ino; + int ret; + int fd; + + if (argc != 3) { + fprintf(stderr, "must specify ino and path\n"); + return -EINVAL; + } + + ino = strtoull(argv[1], &endptr, 0); + if (*endptr != '\0' || + ((ino == LLONG_MIN || ino == LLONG_MAX) && errno == ERANGE)) { + fprintf(stderr, "error parsing inode number '%s'\n", + argv[1]); + return -EINVAL; + } + + + fd = open(argv[2], O_RDONLY); + if (fd < 0) { + ret = -errno; + fprintf(stderr, "failed to open '%s': %s (%d)\n", + argv[2], strerror(errno), errno); + return ret; + } + + result_bytes = offsetof(struct scoutfs_ioctl_ino_path_result, + path[PATH_MAX]); + res = malloc(result_bytes); + if (!res) { + fprintf(stderr, "couldn't allocate %u byte buffer\n", + result_bytes); + ret = -ENOMEM; + goto out; + } + + args.ino = ino; + args.dir_ino = 0; + args.dir_pos = 0; + args.result_ptr = (intptr_t)res; + args.result_bytes = result_bytes; + for (;;) { + ret = ioctl(fd, SCOUTFS_IOC_INO_PATH, &args); + if (ret < 0) { + ret = -errno; + if (ret == -ENOENT) + ret = 0; + break; + } + + printf("%.*s\n", res->path_bytes, res->path); + + args.dir_ino = res->dir_ino; + args.dir_pos = res->dir_pos; + if (++args.dir_pos == 0) { + if (++args.dir_ino == 0) + break; + } + } + + if (ret < 0) { + ret = -errno; + fprintf(stderr, "inodes_since ioctl failed: %s (%d)\n", + strerror(errno), errno); + } +out: + free(res); + close(fd); + return ret; +}; + +static void __attribute__((constructor)) ino_path_ctor(void) +{ + cmd_register("ino-path", " ", + "print paths that refer to inode #", ino_path_cmd); +} diff --git a/utils/src/ioctl.h b/utils/src/ioctl.h new file mode 100644 index 00000000..a53626a0 --- /dev/null +++ b/utils/src/ioctl.h @@ -0,0 +1,416 @@ +#ifndef _SCOUTFS_IOCTL_H_ +#define _SCOUTFS_IOCTL_H_ + +/* + * We naturally align explicit width fields in the ioctl structs so that + * userspace doesn't need to deal with padding or unaligned packing and + * we don't have to deal with 32/64 compat. It makes it a little + * awkward to communicate persistent packed structs through the ioctls + * but that happens very rarely. An interesting special case are + * 0length arrays that follow the structs. We make those start at the + * next aligned offset of the struct to be safe. + * + * This is enforced by pahole scripting in external build environments. + */ + +/* XXX I have no idea how these are chosen. */ +#define SCOUTFS_IOCTL_MAGIC 's' + +/* + * Packed scoutfs keys rarely cross the ioctl boundary so we have a + * translation struct. + */ +struct scoutfs_ioctl_key { + __le64 _sk_first; + __le64 _sk_second; + __le64 _sk_third; + __u8 _sk_fourth; + __u8 sk_type; + __u8 sk_zone; + __u8 _pad[5]; +}; + +struct scoutfs_ioctl_walk_inodes_entry { + __u64 major; + __u64 ino; + __u32 minor; + __u8 _pad[4]; +}; + +/* + * Walk inodes in an index that is sorted by one of their fields. + * + * Each index is built from generic index items that have major and + * minor values that are set to the field being indexed. In time + * indices, for example, major is seconds and minor is nanoseconds. + * + * @first The first index entry that can be returned. + * @last The last index entry that can be returned. + * @entries_ptr Pointer to emory containing buffer for entry results. + * @nr_entries The number of entries that can fit in the buffer. + * @index Which index to walk, enumerated in _WALK_INODES_ constants. + * + * To start iterating first can be memset to 0 and last to 0xff. Then + * after each set of results first can be set to the last entry returned + * and then the fields can be incremented in reverse sort order (ino < + * minor < major) as each increasingly significant value wraps around to + * 0. + * + * These indexes are not strictly consistent. The items that back these + * index entries aren't updated with cluster locks so they're not + * guaranteed to be visible the moment you read after writing. They're + * only visible when the transaction that updated them is synced. + * + * In addition, the seq indexes will only allow walking through sequence + * space that has been consistent. This prevents old dirty entries from + * becoming visible after newer stable entries are displayed. + * + * If first is greater than last then the walk will return 0 entries. + * + * XXX invalidate before reading. + */ +struct scoutfs_ioctl_walk_inodes { + struct scoutfs_ioctl_walk_inodes_entry first; + struct scoutfs_ioctl_walk_inodes_entry last; + __u64 entries_ptr; + __u32 nr_entries; + __u8 index; + __u8 _pad[11]; /* padded to align walk_inodes_entry total size */ +}; + +enum scoutfs_ino_walk_seq_type { + SCOUTFS_IOC_WALK_INODES_META_SEQ = 0, + SCOUTFS_IOC_WALK_INODES_DATA_SEQ, + SCOUTFS_IOC_WALK_INODES_UNKNOWN, +}; + +/* + * Adds entries to the user's buffer for each inode that is found in the + * given index between the first and last positions. + */ +#define SCOUTFS_IOC_WALK_INODES _IOR(SCOUTFS_IOCTL_MAGIC, 1, \ + struct scoutfs_ioctl_walk_inodes) + +/* + * Fill the result buffer with the next absolute path to the target + * inode searching from a given position in a parent directory. + * + * @ino: The target ino that we're finding paths to. Constant across + * all the calls that make up an iteration over all the inode's paths. + * + * @dir_ino: The inode number of the directory containing the entry to + * our inode to search from. If this parent directory contains no more + * entries to our inode then we'll search through other parent directory + * inodes in inode order. + * + * @dir_pos: The position in the dir_ino parent directory of the entry + * to our inode to search from. If there is no entry at this position + * then we'll search through other entry positions in increasing order. + * If we exhaust the parent directory then we'll search through + * additional parent directories in inode order. + * + * @result_ptr: A pointer to the buffer where the result struct and + * absolute path will be stored. + * + * @result_bytes: The size of the buffer that will contain the result + * struct and the null terminated absolute path name. + * + * To start iterating set the desired target inode, dir_ino to 0, + * dir_pos to 0, and set result_ptr and _bytes to a sufficiently large + * buffeer (sizeof(result) + PATH_MAX is a solid choice). + * + * After each returned result set the next search dir_ino and dir_pos to + * the returned dir_ino and dir_pos. Then increment the search dir_pos, + * and if it wrapped to 0, increment dir_ino. + * + * This only walks back through full hard links. None of the returned + * paths will reflect symlinks to components in the path. + * + * This doesn't ensure that the caller has permissions to traverse the + * returned paths to the inode. It requires CAP_DAC_READ_SEARCH which + * bypasses permissions checking. + * + * This call is not serialized with any modification (create, rename, + * unlink) of the path components. It will return all the paths that + * were stable both before and after the call. It may or may not return + * paths which are created or unlinked during the call. + * + * On success 0 is returned and result struct is filled with the next + * absolute path. The path_bytes length of the path includes a null + * terminating byte. dir_ino and dir_pos refer to the position of the + * final component in its parent directory and can be advanced to search + * for the next terminal entry whose path is then built by walking up + * parent directories. + * + * ENOENT is returned when no paths are found. + * + * ENAMETOOLONG is returned when the result struct and path found + * doesn't fit in the result buffer. + * + * Many other errnos indicate hard failure to find the next path. + */ +struct scoutfs_ioctl_ino_path { + __u64 ino; + __u64 dir_ino; + __u64 dir_pos; + __u64 result_ptr; + __u16 result_bytes; + __u8 _pad[6]; +}; + +struct scoutfs_ioctl_ino_path_result { + __u64 dir_ino; + __u64 dir_pos; + __u16 path_bytes; + __u8 _pad[6]; + __u8 path[0]; +}; + +/* Get a single path from the root to the given inode number */ +#define SCOUTFS_IOC_INO_PATH _IOR(SCOUTFS_IOCTL_MAGIC, 2, \ + struct scoutfs_ioctl_ino_path) + +/* + * "Release" a contiguous range of logical blocks of file data. + * Released blocks are removed from the file system like truncation, but + * an offline record is left behind to trigger demand staging if the + * file is read. + * + * The starting block offset and number of blocks to release are in + * units 4KB blocks. + * + * The specified range can extend past i_size and can straddle sparse + * regions or blocks that are already offline. The only change it makes + * is to free and mark offline any existing blocks that intersect with + * the region. + * + * Returns 0 if the operation succeeds. If an error is returned then + * some partial region of the blocks in the region may have been marked + * offline. + * + * If the operation succeeds then inode metadata that reflects file data + * contents are not updated. This is intended to be transparent to the + * presentation of the data in the file. + */ +struct scoutfs_ioctl_release { + __u64 block; + __u64 count; + __u64 data_version; +}; + +#define SCOUTFS_IOC_RELEASE _IOW(SCOUTFS_IOCTL_MAGIC, 3, \ + struct scoutfs_ioctl_release) + +struct scoutfs_ioctl_stage { + __u64 data_version; + __u64 buf_ptr; + __u64 offset; + __s32 count; + __u32 _pad; +}; + +#define SCOUTFS_IOC_STAGE _IOW(SCOUTFS_IOCTL_MAGIC, 4, \ + struct scoutfs_ioctl_stage) + +/* + * Give the user inode fields that are not otherwise visible. statx() + * isn't always available and xattrs are relatively expensive. + * + * @valid_bytes stores the number of bytes that are valid in the + * structure. The caller sets this to the size of the struct that they + * understand. The kernel then fills and copies back the min of the + * size they and the user caller understand. The user can tell if a + * field is set if all of its bytes are within the valid_bytes that the + * kernel set on return. + * + * New fields are only added to the end of the struct. + */ +struct scoutfs_ioctl_stat_more { + __u64 valid_bytes; + __u64 meta_seq; + __u64 data_seq; + __u64 data_version; + __u64 online_blocks; + __u64 offline_blocks; +}; + +#define SCOUTFS_IOC_STAT_MORE _IOR(SCOUTFS_IOCTL_MAGIC, 5, \ + struct scoutfs_ioctl_stat_more) + + +struct scoutfs_ioctl_data_waiting_entry { + __u64 ino; + __u64 iblock; + __u8 op; + __u8 _pad[7]; +}; + +#define SCOUTFS_IOC_DWO_READ (1 << 0) +#define SCOUTFS_IOC_DWO_WRITE (1 << 1) +#define SCOUTFS_IOC_DWO_CHANGE_SIZE (1 << 2) +#define SCOUTFS_IOC_DWO_UNKNOWN (U8_MAX << 3) + +struct scoutfs_ioctl_data_waiting { + __u64 flags; + __u64 after_ino; + __u64 after_iblock; + __u64 ents_ptr; + __u16 ents_nr; + __u8 _pad[6]; +}; + +#define SCOUTFS_IOC_DATA_WAITING_FLAGS_UNKNOWN (U8_MAX << 0) + +#define SCOUTFS_IOC_DATA_WAITING _IOR(SCOUTFS_IOCTL_MAGIC, 6, \ + struct scoutfs_ioctl_data_waiting) + +/* + * If i_size is set then data_version must be non-zero. If the offline + * flag is set then i_size must be set and a offline extent will be + * created from offset 0 to i_size. + */ +struct scoutfs_ioctl_setattr_more { + __u64 data_version; + __u64 i_size; + __u64 flags; + __u64 ctime_sec; + __u32 ctime_nsec; + __u8 _pad[4]; +}; + +#define SCOUTFS_IOC_SETATTR_MORE_OFFLINE (1 << 0) +#define SCOUTFS_IOC_SETATTR_MORE_UNKNOWN (U8_MAX << 1) + +#define SCOUTFS_IOC_SETATTR_MORE _IOW(SCOUTFS_IOCTL_MAGIC, 7, \ + struct scoutfs_ioctl_setattr_more) + +struct scoutfs_ioctl_listxattr_hidden { + __u64 id_pos; + __u64 buf_ptr; + __u32 buf_bytes; + __u32 hash_pos; +}; + +#define SCOUTFS_IOC_LISTXATTR_HIDDEN _IOR(SCOUTFS_IOCTL_MAGIC, 8, \ + struct scoutfs_ioctl_listxattr_hidden) + +/* + * Return the inode numbers of inodes which might contain the given + * xattr. The inode may not have a set xattr with that name, the caller + * must check the returned inodes to see if they match. + * + * @next_ino: The next inode number that could be returned. Initialized + * to 0 when first searching and set to one past the last inode number + * returned to continue searching. + * @last_ino: The last inode number that could be returned. U64_MAX to + * find all inodes. + * @name_ptr: The address of the name of the xattr to search for. It is + * not null terminated. + * @inodes_ptr: The address of the array of uint64_t inode numbers in + * which to store inode numbers that may contain the xattr. EFAULT may + * be returned if this address is not naturally aligned. + * @output_flags: Set as success is returned. If an error is returned + * then this field is undefined and should not be read. + * @nr_inodes: The number of elements in the array found at inodes_ptr. + * @name_bytes: The number of non-null bytes found in the name at + * name_ptr. + * + * This requires the CAP_SYS_ADMIN capability and will return -EPERM if + * it's not granted. + * + * The number of inode numbers stored in the inodes_ptr array is + * returned. If nr_inodes is 0 or last_ino is less than next_ino then 0 + * will be immediately returned. + * + * Partial progress can be returned if an error is hit or if nr_inodes + * was larger than the internal limit on the number of inodes returned + * in a search pass. The _END output flag is set if all the results + * including last_ino were searched in this pass. + * + * It's valuable to provide a large inodes array so that all the results + * can be found in one search pass and _END can be set. There are + * significant constant costs for performing each search pass. + */ +struct scoutfs_ioctl_search_xattrs { + __u64 next_ino; + __u64 last_ino; + __u64 name_ptr; + __u64 inodes_ptr; + __u64 output_flags; + __u64 nr_inodes; + __u16 name_bytes; + __u8 _pad[6]; +}; + +/* set in output_flags if returned inodes reached last_ino */ +#define SCOUTFS_SEARCH_XATTRS_OFLAG_END (1ULL << 0) + +#define SCOUTFS_IOC_SEARCH_XATTRS _IOR(SCOUTFS_IOCTL_MAGIC, 9, \ + struct scoutfs_ioctl_search_xattrs) + +/* + * Give the user information about the filesystem. + * + * @valid_bytes stores the number of bytes that are valid in the + * structure. The caller sets this to the size of the struct that they + * understand. The kernel then fills and copies back the min of the + * size they and the user caller understand. The user can tell if a + * field is set if all of its bytes are within the valid_bytes that the + * kernel set on return. + * + * @committed_seq: All seqs up to and including this seq have been + * committed. Can be compared with meta_seq and data_seq from inodes in + * stat_more to discover if changes have been committed to disk. + * + * New fields are only added to the end of the struct. + */ +struct scoutfs_ioctl_statfs_more { + __u64 valid_bytes; + __u64 fsid; + __u64 rid; + __u64 committed_seq; + __u64 total_meta_blocks; + __u64 total_data_blocks; +}; + +#define SCOUTFS_IOC_STATFS_MORE _IOR(SCOUTFS_IOCTL_MAGIC, 10, \ + struct scoutfs_ioctl_statfs_more) + +/* + * Cause matching waiters to return an error. + * + * Find current waiters that match the inode, op, and block range to wake + * up and return an error. + */ +struct scoutfs_ioctl_data_wait_err { + __u64 ino; + __u64 data_version; + __u64 offset; + __u64 count; + __u64 op; + __s64 err; +}; + +#define SCOUTFS_IOC_DATA_WAIT_ERR _IOR(SCOUTFS_IOCTL_MAGIC, 11, \ + struct scoutfs_ioctl_data_wait_err) + + +#define SCOUTFS_IOC_ALLOC_DETAIL _IOR(SCOUTFS_IOCTL_MAGIC, 12, \ + struct scoutfs_ioctl_alloc_detail) + +struct scoutfs_ioctl_alloc_detail { + __u64 entries_ptr; + __u64 entries_nr; +}; + +struct scoutfs_ioctl_alloc_detail_entry { + __u64 id; + __u64 blocks; + __u8 type; + __u8 meta:1, + avail:1; + __u8 __bit_pad:6; + __u8 __pad[6]; +}; + +#endif diff --git a/utils/src/key.h b/utils/src/key.h new file mode 100644 index 00000000..7e3ad20d --- /dev/null +++ b/utils/src/key.h @@ -0,0 +1,144 @@ +#ifndef _SCOUTFS_KEY_H_ +#define _SCOUTFS_KEY_H_ + +#include "sparse.h" +#include "util.h" +#include "format.h" +#include "cmp.h" +#include "endian_swap.h" + +#define SK_FMT "%u.%llu.%u.%llu.%llu.%u" +/* This does not support null keys */ +#define SK_ARG(key) (key)->sk_zone, le64_to_cpu((key)->_sk_first), \ + (key)->sk_type, le64_to_cpu((key)->_sk_second), \ + le64_to_cpu((key)->_sk_third), \ + (key)->_sk_fourth + +/* + * copy fields between keys with the same fields but different types. + * The destination type might have internal padding so we zero it. + */ +#define scoutfs_key_copy_types(a, b) \ +do { \ + __typeof__(a) _to = (a); \ + __typeof__(b) _from = (b); \ + \ + memset(_to, 0, sizeof(*_to)); \ + _to->sk_zone = _from->sk_zone; \ + _to->_sk_first = _from->_sk_first; \ + _to->sk_type = _from->sk_type; \ + _to->_sk_second = _from->_sk_second; \ + _to->_sk_third = _from->_sk_third; \ + _to->_sk_fourth = _from->_sk_fourth; \ +} while (0) + +static inline void scoutfs_key_set_zeros(struct scoutfs_key *key) +{ + key->sk_zone = 0; + key->_sk_first = 0; + key->sk_type = 0; + key->_sk_second = 0; + key->_sk_third = 0; + key->_sk_fourth = 0; +} + +static inline void scoutfs_key_copy_or_zeros(struct scoutfs_key *dst, + struct scoutfs_key *src) +{ + if (src) + *dst = *src; + else + scoutfs_key_set_zeros(dst); +} + +static inline void scoutfs_key_set_ones(struct scoutfs_key *key) +{ + key->sk_zone = U8_MAX; + key->_sk_first = cpu_to_le64(U64_MAX); + key->sk_type = U8_MAX; + key->_sk_second = cpu_to_le64(U64_MAX); + key->_sk_third = cpu_to_le64(U64_MAX); + key->_sk_fourth = U8_MAX; +} + +/* + * Return a -1/0/1 comparison of keys. + * + * It turns out that these ternary chains are consistently cheaper than + * other alternatives across keys that first differ in any of the + * values. Say maybe 20% faster than memcmp. + */ +static inline int scoutfs_key_compare(struct scoutfs_key *a, + struct scoutfs_key *b) +{ + return scoutfs_cmp(a->sk_zone, b->sk_zone) ?: + scoutfs_cmp(le64_to_cpu(a->_sk_first), le64_to_cpu(b->_sk_first)) ?: + scoutfs_cmp(a->sk_type, b->sk_type) ?: + scoutfs_cmp(le64_to_cpu(a->_sk_second), le64_to_cpu(b->_sk_second)) ?: + scoutfs_cmp(le64_to_cpu(a->_sk_third), le64_to_cpu(b->_sk_third)) ?: + scoutfs_cmp(a->_sk_fourth, b->_sk_fourth); +} + +/* + * Compare ranges of keys where overlapping is equality. Returns: + * -1: a_end < b_start + * 1: a_start > b_end + * else 0: ranges overlap + */ +static inline int scoutfs_key_compare_ranges(struct scoutfs_key *a_start, + struct scoutfs_key *a_end, + struct scoutfs_key *b_start, + struct scoutfs_key *b_end) +{ + return scoutfs_key_compare(a_end, b_start) < 0 ? -1 : + scoutfs_key_compare(a_start, b_end) > 0 ? 1 : + 0; +} + +static inline void scoutfs_key_inc(struct scoutfs_key *key) +{ + if (++key->_sk_fourth != 0) + return; + + le64_add_cpu(&key->_sk_third, 1); + if (key->_sk_third != 0) + return; + + le64_add_cpu(&key->_sk_second, 1); + if (key->_sk_second != 0) + return; + + if (++key->sk_type != 0) + return; + + le64_add_cpu(&key->_sk_first, 1); + if (key->_sk_first != 0) + return; + + key->sk_zone++; +} + +static inline void scoutfs_key_dec(struct scoutfs_key *key) +{ + if (--key->_sk_fourth != U8_MAX) + return; + + le64_add_cpu(&key->_sk_third, -1); + if (key->_sk_third != cpu_to_le64(U64_MAX)) + return; + + le64_add_cpu(&key->_sk_second, -1); + if (key->_sk_second != cpu_to_le64(U64_MAX)) + return; + + if (--key->sk_type != U8_MAX) + return; + + le64_add_cpu(&key->_sk_first, -1); + if (key->_sk_first != cpu_to_le64(U64_MAX)) + return; + + key->sk_zone--; +} + +#endif diff --git a/utils/src/leaf_item_hash.c b/utils/src/leaf_item_hash.c new file mode 100644 index 00000000..b2946d4b --- /dev/null +++ b/utils/src/leaf_item_hash.c @@ -0,0 +1,39 @@ +#include "sparse.h" +#include "util.h" +#include "format.h" +#include "hash.h" +#include "leaf_item_hash.h" + +/* + * A minimal extraction of the leaf item hash from the kernel's btree. + */ + +int leaf_item_hash_ind(struct scoutfs_key *key) +{ + return scoutfs_hash32(key, sizeof(struct scoutfs_key)) % + SCOUTFS_BTREE_LEAF_ITEM_HASH_NR; +} + +__le16 *leaf_item_hash_buckets(struct scoutfs_btree_block *bt) +{ + return (void *)bt + SCOUTFS_BLOCK_LG_SIZE - + SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES; +} + +void leaf_item_hash_insert(struct scoutfs_btree_block *bt, + struct scoutfs_key *key, __le16 off) +{ + __le16 *buckets = leaf_item_hash_buckets(bt); + int i; + + if (bt->level > 0) + return; + + for (i = leaf_item_hash_ind(key); + i < SCOUTFS_BTREE_LEAF_ITEM_HASH_NR; i++) { + if (buckets[i] == 0) { + buckets[i] = off; + return; + } + } +} diff --git a/utils/src/leaf_item_hash.h b/utils/src/leaf_item_hash.h new file mode 100644 index 00000000..0408bc0e --- /dev/null +++ b/utils/src/leaf_item_hash.h @@ -0,0 +1,9 @@ +#ifndef _LEAF_ITEM_HASH_H_ +#define _LEAF_ITEM_HASH_H_ + +int leaf_item_hash_ind(struct scoutfs_key *key); +__le16 *leaf_item_hash_buckets(struct scoutfs_btree_block *bt); +void leaf_item_hash_insert(struct scoutfs_btree_block *bt, + struct scoutfs_key *key, __le16 off); + +#endif diff --git a/utils/src/list.h b/utils/src/list.h new file mode 100644 index 00000000..231807df --- /dev/null +++ b/utils/src/list.h @@ -0,0 +1,310 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + * yanked from the linux kernel.. + */ +#ifndef _LIST_H_ +#define _LIST_H_ +/* + * Simple doubly linked list implementation. + * + * Some of the internal functions ("__xxx") are useful when + * manipulating whole lists rather than single entries, as + * sometimes we already know the next/prev entries and we can + * generate better code by using them directly rather than + * using the generic single-entry routines. + */ + +struct list_head { + struct list_head *next, *prev; +}; + +#define LIST_HEAD_INIT(name) { &(name), &(name) } + +#define LIST_HEAD(name) \ + struct list_head name = LIST_HEAD_INIT(name) + +#define INIT_LIST_HEAD(ptr) do { \ + (ptr)->next = (ptr); (ptr)->prev = (ptr); \ +} while (0) + +/* + * Insert a new entry between two known consecutive entries. + * + * This is only for internal list manipulation where we know + * the prev/next entries already! + */ +static inline void __list_add(struct list_head *new, + struct list_head *prev, + struct list_head *next) +{ + next->prev = new; + new->next = next; + new->prev = prev; + prev->next = new; +} + +/** + * list_add - add a new entry + * @new: new entry to be added + * @head: list head to add it after + * + * Insert a new entry after the specified head. + * This is good for implementing stacks. + */ +static inline void list_add(struct list_head *new, struct list_head *head) +{ + __list_add(new, head, head->next); +} + +/** + * list_add_tail - add a new entry + * @new: new entry to be added + * @head: list head to add it before + * + * Insert a new entry before the specified head. + * This is useful for implementing queues. + */ +static inline void list_add_tail(struct list_head *new, struct list_head *head) +{ + __list_add(new, head->prev, head); +} + +/* + * Insert a new entry between two known consecutive entries. + * + * This is only for internal list manipulation where we know + * the prev/next entries already! + */ +static __inline__ void __list_add_rcu(struct list_head * new, + struct list_head * prev, + struct list_head * next) +{ + new->next = next; + new->prev = prev; + next->prev = new; + prev->next = new; +} + +/* + * Delete a list entry by making the prev/next entries + * point to each other. + * + * This is only for internal list manipulation where we know + * the prev/next entries already! + */ +static inline void __list_del(struct list_head * prev, struct list_head * next) +{ + next->prev = prev; + prev->next = next; +} + +/** + * list_del - deletes entry from list. + * @entry: the element to delete from the list. + * Note: list_empty on entry does not return true after this, the entry is + * in an undefined state. + */ +static inline void list_del(struct list_head *entry) +{ + __list_del(entry->prev, entry->next); +} + +/** + * list_del_init - deletes entry from list and reinitialize it. + * @entry: the element to delete from the list. + */ +static inline void list_del_init(struct list_head *entry) +{ + __list_del(entry->prev, entry->next); + INIT_LIST_HEAD(entry); +} + +/** + * list_move - delete from one list and add as another's head + * @list: the entry to move + * @head: the head that will precede our entry + */ +static inline void list_move(struct list_head *list, struct list_head *head) +{ + __list_del(list->prev, list->next); + list_add(list, head); +} + +/** + * list_move_tail - delete from one list and add as another's tail + * @list: the entry to move + * @head: the head that will follow our entry + */ +static inline void list_move_tail(struct list_head *list, + struct list_head *head) +{ + __list_del(list->prev, list->next); + list_add_tail(list, head); +} + +/** + * list_empty - tests whether a list is empty + * @head: the list to test. + */ +static inline int list_empty(struct list_head *head) +{ + return head->next == head; +} + +static inline void __list_splice(struct list_head *list, + struct list_head *head) +{ + struct list_head *first = list->next; + struct list_head *last = list->prev; + struct list_head *at = head->next; + + first->prev = head; + head->next = first; + + last->next = at; + at->prev = last; +} + +/** + * list_splice - join two lists + * @list: the new list to add. + * @head: the place to add it in the first list. + */ +static inline void list_splice(struct list_head *list, struct list_head *head) +{ + if (!list_empty(list)) + __list_splice(list, head); +} + +/** + * list_splice_init - join two lists and reinitialise the emptied list. + * @list: the new list to add. + * @head: the place to add it in the first list. + * + * The list at @list is reinitialised + */ +static inline void list_splice_init(struct list_head *list, + struct list_head *head) +{ + if (!list_empty(list)) { + __list_splice(list, head); + INIT_LIST_HEAD(list); + } +} + +/** + * list_entry - get the struct for this entry + * @ptr: the &struct list_head pointer. + * @type: the type of the struct this is embedded in. + * @member: the name of the list_struct within the struct. + */ +#define list_entry(ptr, type, member) \ + ((type *)((char *)(ptr)-(unsigned long)(&((type *)0)->member))) + + +/** + * list_for_each - iterate over a list + * @pos: the &struct list_head to use as a loop counter. + * @head: the head for your list. + */ +#define list_for_each(pos, head) \ + for (pos = (head)->next; pos != (head); pos = pos->next) + +/** + * list_for_each_prev - iterate over a list backwards + * @pos: the &struct list_head to use as a loop counter. + * @head: the head for your list. + */ +#define list_for_each_prev(pos, head) \ + for (pos = (head)->prev; pos != (head); pos = pos->prev) + +/** + * list_for_each_safe - iterate over a list safe against removal of list entry + * @pos: the &struct list_head to use as a loop counter. + * @n: another &struct list_head to use as temporary storage + * @head: the head for your list. + */ +#define list_for_each_safe(pos, n, head) \ + for (pos = (head)->next, n = pos->next; pos != (head); \ + pos = n, n = pos->next) + +/** + * list_for_each_entry - iterate over list of given type + * @pos: the type * to use as a loop counter. + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + */ +#define list_for_each_entry(pos, head, member) \ + for (pos = list_entry((head)->next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = list_entry(pos->member.next, typeof(*pos), member)) + +/** + * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry + * @pos: the type * to use as a loop cursor. + * @n: another type * to use as temporary storage + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + */ +#define list_for_each_entry_safe(pos, n, head, member) \ + for (pos = list_entry((head)->next, typeof(*pos), member), \ + n = list_entry(pos->member.next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = n, n = list_entry(n->member.next, typeof(*n), member)) + +/** + * list_first_entry - get the first element from a list + * @ptr: the list head to take the element from. + * @type: the type of the struct this is embedded in. + * @member: the name of the list_head within the struct. + * + * Note, that list is expected to be not empty. + */ +#define list_first_entry(ptr, type, member) \ + list_entry((ptr)->next, type, member) + +/** + * list_last_entry - get the last element from a list + * @ptr: the list head to take the element from. + * @type: the type of the struct this is embedded in. + * @member: the name of the list_head within the struct. + * + * Note, that list is expected to be not empty. + */ +#define list_last_entry(ptr, type, member) \ + list_entry((ptr)->prev, type, member) + +/** + * list_first_entry_or_null - get the first element from a list + * @ptr: the list head to take the element from. + * @type: the type of the struct this is embedded in. + * @member: the name of the list_head within the struct. + * + * Note that if the list is empty, it returns NULL. + */ +#define list_first_entry_or_null(ptr, type, member) \ + (!list_empty(ptr) ? list_first_entry(ptr, type, member) : NULL) + +/** + * list_next_entry - get the next element in list + * @pos: the type * to cursor + * @member: the name of the list_head within the struct. + */ +#define list_next_entry(pos, member) \ + list_entry((pos)->member.next, typeof(*(pos)), member) + +#endif diff --git a/utils/src/listxattr_hidden.c b/utils/src/listxattr_hidden.c new file mode 100644 index 00000000..a98426aa --- /dev/null +++ b/utils/src/listxattr_hidden.c @@ -0,0 +1,147 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sparse.h" +#include "util.h" +#include "format.h" +#include "ioctl.h" +#include "cmd.h" + +static struct option long_ops[] = { + { "file", 1, NULL, 'f' }, + { NULL, 0, NULL, 0} +}; + +static int listxattr_hidden_cmd(int argc, char **argv) +{ + struct scoutfs_ioctl_listxattr_hidden lxh; + char *path = NULL; + char *buf = NULL; + char *name; + int fd = -1; + int bytes; + int len; + int ret; + int c; + int i; + + while ((c = getopt_long(argc, argv, "f:", long_ops, NULL)) != -1) { + switch (c) { + case 'f': + path = strdup(optarg); + if (!path) { + fprintf(stderr, "path mem alloc failed\n"); + ret = -ENOMEM; + goto out; + } + break; + case '?': + default: + ret = -EINVAL; + goto out; + } + } + + if (path == NULL) { + fprintf(stderr, "must specify -f path to file\n"); + ret = -EINVAL; + goto out; + } + + memset(&lxh, 0, sizeof(lxh)); + lxh.id_pos = 0; + lxh.hash_pos = 0; + lxh.buf_bytes = 256 * 1024; + + buf = malloc(lxh.buf_bytes); + if (!buf) { + fprintf(stderr, "xattr name buf alloc failed\n"); + return -ENOMEM; + } + lxh.buf_ptr = (unsigned long)buf; + + fd = open(path, O_RDONLY); + if (fd < 0) { + ret = -errno; + fprintf(stderr, "failed to open '%s': %s (%d)\n", + path, strerror(errno), errno); + goto out; + } + + for (;;) { + + ret = ioctl(fd, SCOUTFS_IOC_LISTXATTR_HIDDEN, &lxh); + if (ret == 0) + break; + if (ret < 0) { + ret = -errno; + fprintf(stderr, "listxattr_hidden ioctl failed: " + "%s (%d)\n", strerror(errno), errno); + goto out; + } + + bytes = ret; + + if (bytes > lxh.buf_bytes) { + fprintf(stderr, "listxattr_hidden overflowed\n"); + ret = -EFAULT; + goto out; + } + if (buf[bytes - 1] != '\0') { + fprintf(stderr, "listxattr_hidden didn't term\n"); + ret = -EINVAL; + goto out; + } + + name = buf; + + do { + len = strlen(name); + if (len == 0) { + fprintf(stderr, "listxattr_hidden empty name\n"); + ret = -EINVAL; + goto out; + } + + if (len > SCOUTFS_XATTR_MAX_NAME_LEN) { + fprintf(stderr, "listxattr_hidden long name\n"); + ret = -EINVAL; + goto out; + } + + for (i = 0; i < len; i++) { + if (!isprint(name[i])) + name[i] = '?'; + } + + printf("%s\n", name); + name += len + 1; + bytes -= len + 1; + + } while (bytes > 0); + } + + ret = 0; +out: + if (fd >= 0) + close(fd); + free(buf); + + return ret; +}; + +static void __attribute__((constructor)) listxattr_hidden_ctor(void) +{ + cmd_register("listxattr-hidden", "-f ", + "print the names of hidden xattrs on the file", + listxattr_hidden_cmd); +} diff --git a/utils/src/main.c b/utils/src/main.c new file mode 100644 index 00000000..369b4ece --- /dev/null +++ b/utils/src/main.c @@ -0,0 +1,18 @@ +#include +#include +#include +#include +#include +#include + +#include "cmd.h" +#include "util.h" + +int main(int argc, char **argv) +{ + /* + * XXX parse global options, env, configs, etc. + */ + + return cmd_execute(argc, argv); +} diff --git a/utils/src/mkfs.c b/utils/src/mkfs.c new file mode 100644 index 00000000..8d5e4dff --- /dev/null +++ b/utils/src/mkfs.c @@ -0,0 +1,442 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sparse.h" +#include "cmd.h" +#include "util.h" +#include "format.h" +#include "parse.h" +#include "crc.h" +#include "rand.h" +#include "dev.h" +#include "key.h" +#include "bitops.h" +#include "btree.h" +#include "leaf_item_hash.h" + +static int write_raw_block(int fd, u64 blkno, int shift, void *blk) +{ + size_t size = 1ULL << shift; + ssize_t ret; + + ret = pwrite(fd, blk, size, blkno << shift); + if (ret != size) { + fprintf(stderr, "write to blkno %llu returned %zd: %s (%d)\n", + blkno, ret, strerror(errno), errno); + return -errno; + } + + return 0; +} + +/* + * Update the block's header and write it out. + */ +static int write_block(int fd, u64 blkno, int shift, + struct scoutfs_super_block *super, + struct scoutfs_block_header *hdr) +{ + size_t size = 1ULL << shift; + + if (super) + *hdr = super->hdr; + hdr->blkno = cpu_to_le64(blkno); + hdr->crc = cpu_to_le32(crc_block(hdr, size)); + + return write_raw_block(fd, blkno, shift, hdr); +} + +/* + * Write the single btree block that contains the blkno and len indexed + * items to store the given extent, and update the root to point to it. + */ +static int write_alloc_root(struct scoutfs_super_block *super, int fd, + struct scoutfs_alloc_root *root, + struct scoutfs_btree_block *bt, + u64 blkno, u64 start, u64 len) +{ + struct scoutfs_key key; + + btree_init_root_single(&root->root, bt, blkno, 1, super->hdr.fsid); + root->total_len = cpu_to_le64(len); + + memset(&key, 0, sizeof(key)); + key.sk_zone = SCOUTFS_FREE_EXTENT_ZONE; + key.sk_type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE; + key.skii_ino = cpu_to_le64(SCOUTFS_ROOT_INO); + key.skfb_end = cpu_to_le64(start + len - 1); + key.skfb_len = cpu_to_le64(len); + btree_append_item(bt, &key, NULL, 0); + + memset(&key, 0, sizeof(key)); + key.sk_zone = SCOUTFS_FREE_EXTENT_ZONE; + key.sk_type = SCOUTFS_FREE_EXTENT_LEN_TYPE; + key.skii_ino = cpu_to_le64(SCOUTFS_ROOT_INO); + key.skfl_neglen = cpu_to_le64(-len); + key.skfl_blkno = cpu_to_le64(start); + btree_append_item(bt, &key, NULL, 0); + + bt->hdr.crc = cpu_to_le32(crc_block(&bt->hdr, + SCOUTFS_BLOCK_LG_SIZE)); + + return write_raw_block(fd, blkno, SCOUTFS_BLOCK_LG_SHIFT, bt); +} + +/* + * Make a new file system by writing: + * - super blocks + * - btree ring blocks with manifest and allocator btree blocks + * - segment with root inode items + * + * Superblock is written to both metadata and data devices, everything else is + * written only to the metadata device. + */ +static int write_new_fs(char *meta_path, char *data_path, + int meta_fd, int data_fd, + u8 quorum_count, + u64 max_meta_size, u64 max_data_size) +{ + struct scoutfs_super_block *super; + struct scoutfs_inode inode; + struct scoutfs_alloc_list_block *lblk; + struct scoutfs_btree_block *bt; + struct scoutfs_key key; + struct timeval tv; + char uuid_str[37]; + void *zeros; + u64 blkno; + u64 meta_size; + u64 data_size; + u64 next_meta; + u64 last_meta; + u64 first_data; + u64 last_data; + u64 meta_start; + u64 meta_len; + int ret; + int i; + + gettimeofday(&tv, NULL); + + super = calloc(1, SCOUTFS_BLOCK_SM_SIZE); + bt = calloc(1, SCOUTFS_BLOCK_LG_SIZE); + zeros = calloc(1, SCOUTFS_BLOCK_SM_SIZE); + if (!super || !bt || !zeros) { + ret = -errno; + fprintf(stderr, "failed to allocate block mem: %s (%d)\n", + strerror(errno), errno); + goto out; + } + + ret = device_size(meta_path, meta_fd, 2ULL * (1024 * 1024 * 1024), + max_meta_size, "meta", &meta_size); + if (ret) + goto out; + + ret = device_size(data_path, data_fd, 8ULL * (1024 * 1024 * 1024), + max_data_size, "data", &data_size); + if (ret) + goto out; + + /* metadata blocks start after the quorum blocks */ + next_meta = (SCOUTFS_QUORUM_BLKNO + SCOUTFS_QUORUM_BLOCKS) >> + SCOUTFS_BLOCK_SM_LG_SHIFT; + /* rest of meta dev is available for metadata blocks */ + last_meta = (meta_size >> SCOUTFS_BLOCK_LG_SHIFT) - 1; + /* Data blocks go on the data dev */ + first_data = SCOUTFS_DATA_DEV_START_BLKNO; + last_data = (data_size >> SCOUTFS_BLOCK_SM_SHIFT) - 1; + + /* partially initialize the super so we can use it to init others */ + memset(super, 0, SCOUTFS_BLOCK_SM_SIZE); + pseudo_random_bytes(&super->hdr.fsid, sizeof(super->hdr.fsid)); + super->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_SUPER); + super->hdr.seq = cpu_to_le64(1); + super->format_hash = cpu_to_le64(SCOUTFS_FORMAT_HASH); + uuid_generate(super->uuid); + super->next_ino = cpu_to_le64(SCOUTFS_ROOT_INO + 1); + super->next_trans_seq = cpu_to_le64(1); + super->total_meta_blocks = cpu_to_le64(last_meta + 1); + super->first_meta_blkno = cpu_to_le64(next_meta); + super->last_meta_blkno = cpu_to_le64(last_meta); + super->total_data_blocks = cpu_to_le64(last_data - first_data + 1); + super->first_data_blkno = cpu_to_le64(first_data); + super->last_data_blkno = cpu_to_le64(last_data); + super->quorum_count = quorum_count; + + /* fs root starts with root inode and its index items */ + blkno = next_meta++; + btree_init_root_single(&super->fs_root, bt, blkno, 1, super->hdr.fsid); + + memset(&key, 0, sizeof(key)); + key.sk_zone = SCOUTFS_INODE_INDEX_ZONE; + key.sk_type = SCOUTFS_INODE_INDEX_META_SEQ_TYPE; + key.skii_ino = cpu_to_le64(SCOUTFS_ROOT_INO); + btree_append_item(bt, &key, NULL, 0); + + memset(&key, 0, sizeof(key)); + key.sk_zone = SCOUTFS_FS_ZONE; + key.ski_ino = cpu_to_le64(SCOUTFS_ROOT_INO); + key.sk_type = SCOUTFS_INODE_TYPE; + + memset(&inode, 0, sizeof(inode)); + inode.next_readdir_pos = cpu_to_le64(2); + inode.nlink = cpu_to_le32(SCOUTFS_DIRENT_FIRST_POS); + inode.mode = cpu_to_le32(0755 | 0040000); + inode.atime.sec = cpu_to_le64(tv.tv_sec); + inode.atime.nsec = cpu_to_le32(tv.tv_usec * 1000); + inode.ctime.sec = inode.atime.sec; + inode.ctime.nsec = inode.atime.nsec; + inode.mtime.sec = inode.atime.sec; + inode.mtime.nsec = inode.atime.nsec; + btree_append_item(bt, &key, &inode, sizeof(inode)); + + bt->hdr.crc = cpu_to_le32(crc_block(&bt->hdr, + SCOUTFS_BLOCK_LG_SIZE)); + + ret = write_raw_block(meta_fd, blkno, SCOUTFS_BLOCK_LG_SHIFT, bt); + if (ret) + goto out; + + /* fill an avail list block for the first server transaction */ + blkno = next_meta++; + lblk = (void *)bt; + memset(lblk, 0, SCOUTFS_BLOCK_LG_SIZE); + + lblk->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_ALLOC_LIST); + lblk->hdr.fsid = super->hdr.fsid; + lblk->hdr.blkno = cpu_to_le64(blkno); + lblk->hdr.seq = cpu_to_le64(1); + + meta_len = (64 * 1024 * 1024) >> SCOUTFS_BLOCK_LG_SHIFT; + for (i = 0; i < meta_len; i++) { + lblk->blknos[i] = cpu_to_le64(next_meta); + next_meta++; + } + lblk->nr = cpu_to_le32(i); + + super->server_meta_avail[0].ref.blkno = lblk->hdr.blkno; + super->server_meta_avail[0].ref.seq = lblk->hdr.seq; + super->server_meta_avail[0].total_nr = le32_to_le64(lblk->nr); + super->server_meta_avail[0].first_nr = lblk->nr; + + lblk->hdr.crc = cpu_to_le32(crc_block(&bt->hdr, SCOUTFS_BLOCK_LG_SIZE)); + ret = write_raw_block(meta_fd, blkno, SCOUTFS_BLOCK_LG_SHIFT, lblk); + if (ret) + goto out; + + /* the data allocator has a single extent */ + blkno = next_meta++; + ret = write_alloc_root(super, meta_fd, &super->data_alloc, bt, + blkno, first_data, + le64_to_cpu(super->total_data_blocks)); + if (ret < 0) + goto out; + + /* + * Initialize all the meta_alloc roots with an equal portion of + * the free metadata extents, excluding the blocks we're going + * to use for the allocators. + */ + meta_start = next_meta + array_size(super->meta_alloc); + meta_len = DIV_ROUND_UP(last_meta - meta_start + 1, + array_size(super->meta_alloc)); + + /* each meta alloc root contains a portion of free metadata extents */ + for (i = 0; i < array_size(super->meta_alloc); i++) { + blkno = next_meta++; + ret = write_alloc_root(super, meta_fd, &super->meta_alloc[i], bt, + blkno, meta_start, + min(meta_len, + last_meta - meta_start + 1)); + if (ret < 0) + goto out; + + meta_start += meta_len; + } + + /* zero out quorum blocks */ + for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) { + ret = write_raw_block(meta_fd, SCOUTFS_QUORUM_BLKNO + i, + SCOUTFS_BLOCK_SM_SHIFT, zeros); + if (ret < 0) { + fprintf(stderr, "error zeroing quorum block: %s (%d)\n", + strerror(-errno), -errno); + goto out; + } + } + + /* write the super block to data dev and meta dev*/ + super->hdr.seq = cpu_to_le64(1); + ret = write_block(data_fd, SCOUTFS_SUPER_BLKNO, SCOUTFS_BLOCK_SM_SHIFT, + NULL, &super->hdr); + if (ret) + goto out; + + if (fsync(data_fd)) { + ret = -errno; + fprintf(stderr, "failed to fsync '%s': %s (%d)\n", + data_path, strerror(errno), errno); + goto out; + } + + super->flags |= cpu_to_le64(SCOUTFS_FLAG_IS_META_BDEV); + ret = write_block(meta_fd, SCOUTFS_SUPER_BLKNO, SCOUTFS_BLOCK_SM_SHIFT, + NULL, &super->hdr); + if (ret) + goto out; + + if (fsync(meta_fd)) { + ret = -errno; + fprintf(stderr, "failed to fsync '%s': %s (%d)\n", + meta_path, strerror(errno), errno); + goto out; + } + + uuid_unparse(super->uuid, uuid_str); + + printf("Created scoutfs filesystem:\n" + " meta device path: %s\n" + " data device path: %s\n" + " fsid: %llx\n" + " format hash: %llx\n" + " uuid: %s\n" + " 64KB metadata blocks: "SIZE_FMT"\n" + " 4KB data blocks: "SIZE_FMT"\n" + " quorum count: %u\n", + meta_path, + data_path, + le64_to_cpu(super->hdr.fsid), + le64_to_cpu(super->format_hash), + uuid_str, + SIZE_ARGS(le64_to_cpu(super->total_meta_blocks), + SCOUTFS_BLOCK_LG_SIZE), + SIZE_ARGS(le64_to_cpu(super->total_data_blocks), + SCOUTFS_BLOCK_SM_SIZE), + super->quorum_count); + + ret = 0; +out: + if (super) + free(super); + if (bt) + free(bt); + if (zeros) + free(zeros); + return ret; +} + +static struct option long_ops[] = { + { "quorum_count", 1, NULL, 'Q' }, + { NULL, 0, NULL, 0} +}; + +static int mkfs_func(int argc, char *argv[]) +{ + unsigned long long ull; + u8 quorum_count = 0; + u64 max_data_size = 0; + u64 max_meta_size = 0; + char *end = NULL; + char *meta_path; + char *data_path; + int meta_fd; + int data_fd; + int ret; + int c; + + while ((c = getopt_long(argc, argv, "Q:D:M:", long_ops, NULL)) != -1) { + switch (c) { + case 'Q': + ull = strtoull(optarg, &end, 0); + if (*end != '\0' || ull == 0 || + ull > SCOUTFS_QUORUM_MAX_COUNT) { + printf("scoutfs: invalid quorum count '%s'\n", + optarg); + return -EINVAL; + } + quorum_count = ull; + break; + case 'D': + ret = parse_human(optarg, &max_data_size); + if (ret < 0) { + printf("scoutfs: invalid data device size '%s'\n", + optarg); + return ret; + } + break; + case 'M': + ret = parse_human(optarg, &max_meta_size); + if (ret < 0) { + printf("scoutfs: invalid meta device size '%s'\n", + optarg); + return ret; + } + break; + case '?': + default: + return -EINVAL; + } + } + + if (optind + 2 != argc) { + printf("scoutfs: mkfs: paths to metadata and data devices are required\n"); + return -EINVAL; + } + + meta_path = argv[optind]; + data_path = argv[optind + 1]; + + if (!quorum_count) { + printf("provide quorum count with --quorum_count|-Q option\n"); + return -EINVAL; + } + + meta_fd = open(meta_path, O_RDWR | O_EXCL); + if (meta_fd < 0) { + ret = -errno; + fprintf(stderr, "failed to open metadata device '%s': %s (%d)\n", + meta_path, strerror(errno), errno); + return ret; + } + + data_fd = open(data_path, O_RDWR | O_EXCL); + if (data_fd < 0) { + ret = -errno; + fprintf(stderr, "failed to open data device '%s': %s (%d)\n", + data_path, strerror(errno), errno); + return ret; + } + + ret = write_new_fs(meta_path, data_path, meta_fd, data_fd, + quorum_count, max_meta_size, max_data_size); + close(meta_fd); + close(data_fd); + + return ret; +} + +static void __attribute__((constructor)) mkfs_ctor(void) +{ + cmd_register("mkfs", "", "write a new file system", mkfs_func); + + /* for lack of some other place to put these.. */ + build_assert(sizeof(uuid_t) == SCOUTFS_UUID_BYTES); +} diff --git a/utils/src/parse.c b/utils/src/parse.c new file mode 100644 index 00000000..761129f9 --- /dev/null +++ b/utils/src/parse.c @@ -0,0 +1,154 @@ +#include +#include +#include +#include +#include + +#include "sparse.h" +#include "util.h" +#include "format.h" + +#include "parse.h" + +/* + * Convert size with multiplicative suffix to bytes. + * e.g. "40M", "10G", "4T" + * + * These are powers-of-two prefixes - K means 1024 not 1000. + * + * One can go pretty far with variations but keeping relatively simple for + * now: commas, decimals, and multichar suffixes not handled. + */ +int parse_human(char* str, u64 *val_ret) +{ + unsigned long long ull; + char *endptr = NULL; + int sh; + int ret = 0; + + ull = strtoull(str, &endptr, 0); + if (((ull == LLONG_MIN || ull == LLONG_MAX) && + errno == ERANGE)) { + fprintf(stderr, "invalid 64bit value: '%s'\n", str); + *val_ret = 0; + ret = -EINVAL; + goto error; + } + + switch (*endptr) { + case 'K': + sh = 10; + break; + case 'M': + sh = 20; + break; + case 'G': + sh = 30; + break; + case 'T': + sh = 40; + break; + case 'P': + sh = 50; + break; + case '\0': + sh = 0; + break; + default: + fprintf(stderr, "unknown suffix: '%s'\n", endptr); + ret = -ERANGE; + goto error; + } + + if (ull > (SIZE_MAX >> sh)) { + fprintf(stderr, "size too big: '%s'\n", str); + ret = -ERANGE; + goto error; + } + + ull <<= sh; + + *val_ret = ull; + +error: + return ret; +} + +int parse_u64(char *str, u64 *val_ret) +{ + unsigned long long ull; + char *endptr = NULL; + + ull = strtoull(str, &endptr, 0); + if (*endptr != '\0' || + ((ull == LLONG_MIN || ull == LLONG_MAX) && + errno == ERANGE)) { + fprintf(stderr, "invalid 64bit value: '%s'\n", str); + *val_ret = 0; + return -EINVAL; + } + + *val_ret = ull; + + return 0; +} + +int parse_s64(char *str, s64 *val_ret) +{ + long long ll; + char *endptr = NULL; + + ll = strtoll(str, &endptr, 0); + if (*endptr != '\0' || + ((ll == LLONG_MIN || ll == LLONG_MAX) && + errno == ERANGE)) { + fprintf(stderr, "invalid 64bit value: '%s'\n", str); + *val_ret = 0; + return -EINVAL; + } + + *val_ret = ll; + + return 0; +} + +int parse_u32(char *str, u32 *val_ret) +{ + u64 val; + int ret; + + ret = parse_u64(str, &val); + if (ret) + return ret; + + if (val > UINT_MAX) + return -EINVAL; + + *val_ret = val; + return 0; +} + +int parse_timespec(char *str, struct timespec *ts) +{ + unsigned long long sec; + unsigned int nsec; + int ret; + + memset(ts, 0, sizeof(struct timespec)); + + ret = sscanf(str, "%llu.%u", &sec, &nsec); + if (ret != 2) { + fprintf(stderr, "invalid timespec string: '%s'\n", str); + return -EINVAL; + } + + if (nsec > 1000000000) { + fprintf(stderr, "invalid timespec nsec value: '%s'\n", str); + return -EINVAL; + } + + ts->tv_sec = sec; + ts->tv_nsec = nsec; + + return 0; +} diff --git a/utils/src/parse.h b/utils/src/parse.h new file mode 100644 index 00000000..0a0d9ac4 --- /dev/null +++ b/utils/src/parse.h @@ -0,0 +1,12 @@ +#ifndef _PARSE_H_ +#define _PARSE_H_ + +#include + +int parse_human(char* str, u64 *val_ret); +int parse_u64(char *str, u64 *val_ret); +int parse_s64(char *str, s64 *val_ret); +int parse_u32(char *str, u32 *val_ret); +int parse_timespec(char *str, struct timespec *ts); + +#endif diff --git a/utils/src/print.c b/utils/src/print.c new file mode 100644 index 00000000..7f713848 --- /dev/null +++ b/utils/src/print.c @@ -0,0 +1,1066 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sparse.h" +#include "util.h" +#include "format.h" +#include "bitmap.h" +#include "cmd.h" +#include "crc.h" +#include "key.h" +#include "avl.h" +#include "srch.h" +#include "leaf_item_hash.h" + +static void *read_block(int fd, u64 blkno, int shift) +{ + size_t size = 1ULL << shift; + ssize_t ret; + void *buf; + + buf = malloc(size); + if (!buf) + return NULL; + + ret = pread(fd, buf, size, blkno << shift); + if (ret != size) { + fprintf(stderr, "read blkno %llu returned %zd: %s (%d)\n", + blkno, ret, strerror(errno), errno); + free(buf); + buf = NULL; + } + + return buf; +} + +static void print_block_header(struct scoutfs_block_header *hdr, int size) +{ + u32 crc = crc_block(hdr, size); + char valid_str[40]; + + if (crc != le32_to_cpu(hdr->crc)) + sprintf(valid_str, "(!= %08x) ", crc); + else + valid_str[0] = '\0'; + + printf(" hdr: crc %08x %smagic %08x fsid %llx blkno %llu seq %llu\n", + le32_to_cpu(hdr->crc), valid_str, le32_to_cpu(hdr->magic), + le64_to_cpu(hdr->fsid), le64_to_cpu(hdr->blkno), + le64_to_cpu(hdr->seq)); +} + +static void print_inode(struct scoutfs_key *key, void *val, int val_len) +{ + struct scoutfs_inode *inode = val; + + printf(" inode: ino %llu size %llu nlink %u\n" + " uid %u gid %u mode 0%o rdev 0x%x flags 0x%x\n" + " next_readdir_pos %llu meta_seq %llu data_seq %llu data_version %llu\n" + " atime %llu.%08u ctime %llu.%08u\n" + " mtime %llu.%08u\n", + le64_to_cpu(key->ski_ino), + le64_to_cpu(inode->size), + le32_to_cpu(inode->nlink), le32_to_cpu(inode->uid), + le32_to_cpu(inode->gid), le32_to_cpu(inode->mode), + le32_to_cpu(inode->rdev), + le32_to_cpu(inode->flags), + le64_to_cpu(inode->next_readdir_pos), + le64_to_cpu(inode->meta_seq), + le64_to_cpu(inode->data_seq), + le64_to_cpu(inode->data_version), + le64_to_cpu(inode->atime.sec), + le32_to_cpu(inode->atime.nsec), + le64_to_cpu(inode->ctime.sec), + le32_to_cpu(inode->ctime.nsec), + le64_to_cpu(inode->mtime.sec), + le32_to_cpu(inode->mtime.nsec)); +} + +static void print_orphan(struct scoutfs_key *key, void *val, int val_len) +{ + printf(" orphan: ino %llu\n", le64_to_cpu(key->sko_ino)); +} + +static u8 *global_printable_name(u8 *name, int name_len) +{ + static u8 name_buf[SCOUTFS_NAME_LEN + 1]; + int i; + + name_len = min(SCOUTFS_NAME_LEN, name_len); + for (i = 0; i < name_len; i++) + name_buf[i] = isprint(name[i]) ? name[i] : '.'; + name_buf[i] = '\0'; + + return name_buf; +} + +static void print_xattr(struct scoutfs_key *key, void *val, int val_len) +{ + struct scoutfs_xattr *xat = val; + + printf(" xattr: ino %llu name_hash %08x id %llu part %u\n", + le64_to_cpu(key->skx_ino), (u32)le64_to_cpu(key->skx_name_hash), + le64_to_cpu(key->skx_id), key->skx_part); + + if (key->skx_part == 0) + printf(" name_len %u val_len %u name %s\n", + xat->name_len, le16_to_cpu(xat->val_len), + global_printable_name(xat->name, xat->name_len)); +} + +static void print_dirent(struct scoutfs_key *key, void *val, int val_len) +{ + struct scoutfs_dirent *dent = val; + unsigned int name_len = val_len - sizeof(*dent); + u8 *name = global_printable_name(dent->name, name_len); + + printf(" dirent: dir %llu hash %016llx pos %llu type %u ino %llu\n" + " name %s\n", + le64_to_cpu(key->skd_ino), le64_to_cpu(dent->hash), + le64_to_cpu(dent->pos), dent->type, le64_to_cpu(dent->ino), + name); +} + +static void print_symlink(struct scoutfs_key *key, void *val, int val_len) +{ + u8 *frag = val; + u8 *name; + + /* don't try to print null term */ + if (frag[val_len - 1] == '\0') + val_len--; + name = global_printable_name(frag, val_len); + + printf(" symlink: ino %llu nr %llu\n" + " target %s\n", + le64_to_cpu(key->sks_ino), le64_to_cpu(key->sks_nr), name); +} + +static void print_data_extent(struct scoutfs_key *key, void *val, int val_len) +{ + struct scoutfs_data_extent_val *dv = val; + u64 iblock; + + iblock = le64_to_cpu(key->skdx_end) - le64_to_cpu(key->skdx_len) + 1; + + printf(" extent: ino %llu iblock %llu len %llu blkno %llu flags %x\n", + le64_to_cpu(key->skdx_ino), iblock, + le64_to_cpu(key->skdx_len), + le64_to_cpu(dv->blkno), dv->flags); +} + +static void print_inode_index(struct scoutfs_key *key, void *val, int val_len) +{ + printf(" index: major %llu ino %llu\n", + le64_to_cpu(key->skii_major), le64_to_cpu(key->skii_ino)); +} + +typedef void (*print_func_t)(struct scoutfs_key *key, void *val, int val_len); + +static print_func_t find_printer(u8 zone, u8 type) +{ + if (zone == SCOUTFS_INODE_INDEX_ZONE && + type >= SCOUTFS_INODE_INDEX_META_SEQ_TYPE && + type <= SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE) + return print_inode_index; + + if (zone == SCOUTFS_RID_ZONE) { + if (type == SCOUTFS_ORPHAN_TYPE) + return print_orphan; + } + + if (zone == SCOUTFS_FS_ZONE) { + switch(type) { + case SCOUTFS_INODE_TYPE: return print_inode; + case SCOUTFS_XATTR_TYPE: return print_xattr; + case SCOUTFS_DIRENT_TYPE: return print_dirent; + case SCOUTFS_READDIR_TYPE: return print_dirent; + case SCOUTFS_SYMLINK_TYPE: return print_symlink; + case SCOUTFS_LINK_BACKREF_TYPE: return print_dirent; + case SCOUTFS_DATA_EXTENT_TYPE: return print_data_extent; + } + } + + return NULL; +} + +static int print_fs_item(struct scoutfs_key *key, void *val, + unsigned val_len, void *arg) +{ + print_func_t printer; + + printf(" "SK_FMT"\n", SK_ARG(key)); + + /* only items in leaf blocks have values */ + if (val) { + printer = find_printer(key->sk_zone, key->sk_type); + if (printer) + printer(key, val, val_len); + else + printf(" (unknown zone %u type %u)\n", + key->sk_zone, key->sk_type); + } + + return 0; +} + +/* same as fs item but with a small header in the value */ +static int print_logs_item(struct scoutfs_key *key, void *val, + unsigned val_len, void *arg) +{ + struct scoutfs_log_item_value *liv; + print_func_t printer; + + printf(" "SK_FMT"\n", SK_ARG(key)); + + /* only items in leaf blocks have values */ + if (val) { + liv = val; + printf(" log_item_value: vers %llu flags %x\n", + le64_to_cpu(liv->vers), liv->flags); + + /* deletion items don't have values */ + if (!(liv->flags & SCOUTFS_LOG_ITEM_FLAG_DELETION)) { + printer = find_printer(key->sk_zone, + key->sk_type); + if (printer) + printer(key, val + sizeof(*liv), + val_len - sizeof(*liv)); + else + printf(" (unknown zone %u type %u)\n", + key->sk_zone, key->sk_type); + } + } + + return 0; +} + +#define BTREF_F \ + "blkno %llu seq %llu" +#define BTREF_A(ref) \ + le64_to_cpu((ref)->blkno), le64_to_cpu((ref)->seq) + +#define BTROOT_F \ + BTREF_F" height %u" +#define BTROOT_A(root) \ + BTREF_A(&(root)->ref), (root)->height + +#define AL_REF_F \ + "blkno %llu seq %llu" +#define AL_REF_A(p) \ + le64_to_cpu((p)->blkno), le64_to_cpu((p)->seq) + +#define AL_HEAD_F \ + AL_REF_F" total_nr %llu first_nr %u" +#define AL_HEAD_A(p) \ + AL_REF_A(&(p)->ref), le64_to_cpu((p)->total_nr),\ + le32_to_cpu((p)->first_nr) + +#define ALCROOT_F \ + BTROOT_F" total_len %llu" +#define ALCROOT_A(ar) \ + BTROOT_A(&(ar)->root), le64_to_cpu((ar)->total_len) + +#define SRE_FMT "%016llx.%llu.%llu" +#define SRE_A(sre) \ + le64_to_cpu((sre)->hash), le64_to_cpu((sre)->ino), \ + le64_to_cpu((sre)->id) + +#define SRF_FMT \ + "f "SRE_FMT" l "SRE_FMT" blks %llu ents %llu hei %u blkno %llu seq %016llx" +#define SRF_A(srf) \ + SRE_A(&(srf)->first), SRE_A(&(srf)->last), \ + le64_to_cpu((srf)->blocks), le64_to_cpu((srf)->entries), \ + (srf)->height, le64_to_cpu((srf)->ref.blkno), \ + le64_to_cpu((srf)->ref.seq) + +/* same as fs item but with a small header in the value */ +static int print_log_trees_item(struct scoutfs_key *key, void *val, + unsigned val_len, void *arg) +{ + struct scoutfs_log_trees *lt = val; + + printf(" rid %llu nr %llu\n", + le64_to_cpu(key->sklt_rid), le64_to_cpu(key->sklt_nr)); + + /* only items in leaf blocks have values */ + if (val) { + printf(" meta_avail: "AL_HEAD_F"\n" + " meta_freed: "AL_HEAD_F"\n" + " item_root: height %u blkno %llu seq %llu\n" + " bloom_ref: blkno %llu seq %llu\n" + " data_avail: "ALCROOT_F"\n" + " data_freed: "ALCROOT_F"\n" + " srch_file: "SRF_FMT"\n" + " max_item_vers: %llu\n" + " rid: %016llx\n" + " nr: %llu\n", + AL_HEAD_A(<->meta_avail), + AL_HEAD_A(<->meta_freed), + lt->item_root.height, + le64_to_cpu(lt->item_root.ref.blkno), + le64_to_cpu(lt->item_root.ref.seq), + le64_to_cpu(lt->bloom_ref.blkno), + le64_to_cpu(lt->bloom_ref.seq), + ALCROOT_A(<->data_avail), + ALCROOT_A(<->data_freed), + SRF_A(<->srch_file), + le64_to_cpu(lt->max_item_vers), + le64_to_cpu(lt->rid), + le64_to_cpu(lt->nr)); + } + + return 0; +} + +static int print_srch_root_item(struct scoutfs_key *key, void *val, + unsigned val_len, void *arg) +{ + struct scoutfs_srch_compact *sc; + struct scoutfs_srch_file *sfl; + int i; + + printf(" "SK_FMT"\n", SK_ARG(key)); + + /* only items in leaf blocks have values */ + if (val) { + if (key->sk_type == SCOUTFS_SRCH_PENDING_TYPE || + key->sk_type == SCOUTFS_SRCH_BUSY_TYPE) { + sc = val; + printf(" compact %s: nr %u flags 0x%x\n", + key->sk_type == SCOUTFS_SRCH_PENDING_TYPE ? + "pending" : "busy", + sc->nr, sc->flags); + for (i = 0; i < sc->nr; i++) { + printf(" [%u] blk %llu pos %llu sfl "SRF_FMT"\n", + i, le64_to_cpu(sc->in[i].blk), + le64_to_cpu(sc->in[i].pos), + SRF_A(&sc->in[i].sfl)); + } + } else { + sfl = val; + printf(" "SRF_FMT"\n", SRF_A(sfl)); + } + } + + return 0; +} + +static int print_lock_clients_entry(struct scoutfs_key *key, void *val, + unsigned val_len, void *arg) +{ + printf(" rid %016llx\n", le64_to_cpu(key->sklc_rid)); + + return 0; +} + +static int print_trans_seqs_entry(struct scoutfs_key *key, void *val, + unsigned val_len, void *arg) +{ + printf(" trans_seq %llu rid %016llx\n", + le64_to_cpu(key->skts_trans_seq), le64_to_cpu(key->skts_rid)); + + return 0; +} + +static int print_mounted_client_entry(struct scoutfs_key *key, void *val, + unsigned val_len, void *arg) +{ + struct scoutfs_mounted_client_btree_val *mcv = val; + + printf(" rid %016llx flags 0x%x\n", + le64_to_cpu(key->skmc_rid), mcv->flags); + + return 0; +} + +static int print_alloc_item(struct scoutfs_key *key, void *val, + unsigned val_len, void *arg) +{ + if (key->sk_type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) + printf(" free extent: blkno %llu len %llu end %llu\n", + le64_to_cpu(key->skfb_end) - + le64_to_cpu(key->skfb_len) + 1, + le64_to_cpu(key->skfb_len), + le64_to_cpu(key->skfb_end)); + else + printf(" free extent: blkno %llu len %llu neglen %lld\n", + le64_to_cpu(key->skfl_blkno), + -le64_to_cpu(key->skfl_neglen), + (long long)le64_to_cpu(key->skfl_neglen)); + + return 0; +} + +typedef int (*print_item_func)(struct scoutfs_key *key, void *val, + unsigned val_len, void *arg); + +static int print_btree_ref(struct scoutfs_key *key, void *val, + unsigned val_len, print_item_func func, void *arg) +{ + struct scoutfs_btree_ref *ref = val; + + func(key, NULL, 0, arg); + printf(" ref blkno %llu seq %llu\n", + le64_to_cpu(ref->blkno), le64_to_cpu(ref->seq)); + + return 0; +} + +static void print_leaf_item_hash(struct scoutfs_btree_block *bt) +{ + __le16 *b; + int col; + int nr; + int i; + + /* print the leaf item hash */ + printf(" item hash: "); + col = 13; + + b = leaf_item_hash_buckets(bt); + nr = 0; + for (i = 0; i < SCOUTFS_BTREE_LEAF_ITEM_HASH_NR; i++) { + if (b[i] == 0) + continue; + + nr++; + col += snprintf(NULL, 0, "%u,%u ", i, le16_to_cpu(b[i])); + if (col >= 78) { + printf("\n "); + col = 3; + } + printf("%u,%u ", i, le16_to_cpu(b[i])); + } + if (col != 3) + printf("\n"); + printf(" (%u / %u populated, %u%% load)\n", + nr, (int)SCOUTFS_BTREE_LEAF_ITEM_HASH_NR, + nr * 100 / (int)SCOUTFS_BTREE_LEAF_ITEM_HASH_NR); +} + +static int print_btree_block(int fd, struct scoutfs_super_block *super, + char *which, struct scoutfs_btree_ref *ref, + print_item_func func, void *arg, u8 level) +{ + struct scoutfs_btree_item *item; + struct scoutfs_avl_node *node; + struct scoutfs_btree_block *bt; + struct scoutfs_key *key; + unsigned int val_len; + unsigned int off; + void *val; + int ret; + int i; + + bt = read_block(fd, le64_to_cpu(ref->blkno), SCOUTFS_BLOCK_LG_SHIFT); + if (!bt) + return -ENOMEM; + + if (bt->level == level) { + printf("%s btree blkno %llu\n" + " crc %08x fsid %llx seq %llu blkno %llu \n" + " total_item_bytes %u mid_free_len %u\n" + " level %u nr_items %u item_root.node %u\n", + which, le64_to_cpu(ref->blkno), + le32_to_cpu(bt->hdr.crc), + le64_to_cpu(bt->hdr.fsid), + le64_to_cpu(bt->hdr.seq), + le64_to_cpu(bt->hdr.blkno), + le16_to_cpu(bt->total_item_bytes), + le16_to_cpu(bt->mid_free_len), + bt->level, + le16_to_cpu(bt->nr_items), + le16_to_cpu(bt->item_root.node)); + + if (bt->level == 0) + print_leaf_item_hash(bt); + } + + for (i = 0, node = avl_first(&bt->item_root); + node; + i++, node = avl_next(&bt->item_root, node)) { + + item = container_of(node, struct scoutfs_btree_item, node); + off = (void *)item - (void *)bt; + val_len = le16_to_cpu(item->val_len); + key = &item->key; + val = (void *)bt + le16_to_cpu(item->val_off); + + if (level < bt->level) { + ref = val; + /* XXX check len */ + if (ref->blkno) { + ret = print_btree_block(fd, super, which, ref, + func, arg, level); + if (ret) + break; + } + continue; + } + + printf(" [%u] off %u par %u l %u r %u h %u vo %u vl %u\n", + i, off, le16_to_cpu(item->node.parent), + le16_to_cpu(item->node.left), + le16_to_cpu(item->node.right), + item->node.height, le16_to_cpu(item->val_off), + val_len); + + if (level) + print_btree_ref(key, val, val_len, func, arg); + else + func(key, val, val_len, arg); + } + + free(bt); + return 0; +} + +/* + * We print btrees by a breadth-first search. This way all the parent + * blocks are printed before the factor of fanout more numerous leaf + * blocks and their included items. + */ +static int print_btree(int fd, struct scoutfs_super_block *super, char *which, + struct scoutfs_btree_root *root, + print_item_func func, void *arg) +{ + int ret = 0; + int i; + + for (i = root->height - 1; i >= 0; i--) { + ret = print_btree_block(fd, super, which, &root->ref, + func, arg, i); + if (ret) + break; + } + + return ret; +} + +static int print_alloc_list_block(int fd, char *str, + struct scoutfs_alloc_list_ref *ref) +{ + struct scoutfs_alloc_list_block *lblk; + struct scoutfs_alloc_list_ref next; + u64 blkno; + u64 start; + u64 len; + int wid; + int i; + + blkno = le64_to_cpu(ref->blkno); + if (blkno == 0) + return 0; + + lblk = read_block(fd, blkno, SCOUTFS_BLOCK_LG_SHIFT); + if (!lblk) + return -ENOMEM; + + printf("%s alloc_list_block blkno %llu\n", str, blkno); + print_block_header(&lblk->hdr, SCOUTFS_BLOCK_LG_SIZE); + printf(" next "AL_REF_F" start %u nr %u\n", + AL_REF_A(&lblk->next), le32_to_cpu(lblk->start), + le32_to_cpu(lblk->nr)); + + if (lblk->nr) { + wid = printf(" exts: "); + start = 0; + len = 0; + for (i = 0; i < le32_to_cpu(lblk->nr); i++) { + if (len == 0) + start = le64_to_cpu(lblk->blknos[i]); + len++; + + if (i == (le32_to_cpu(lblk->nr) - 1) || + start + len != le64_to_cpu(lblk->blknos[i + 1])) { + if (wid >= 72) + wid = printf("\n "); + + wid += printf("%llu,%llu ", start, len); + len = 0; + } + } + printf("\n"); + } + + next = lblk->next; + free(lblk); + return print_alloc_list_block(fd, str, &next); +} + +static int print_srch_block(int fd, struct scoutfs_srch_ref *ref, int level) +{ + struct scoutfs_srch_parent *srp; + struct scoutfs_srch_block *srb; + struct scoutfs_srch_entry sre; + struct scoutfs_srch_entry prev; + u64 blkno; + int pos; + int ret; + int err; + int i; + + blkno = le64_to_cpu(ref->blkno); + if (blkno == 0) + return 0; + + srp = read_block(fd, blkno, SCOUTFS_BLOCK_LG_SHIFT); + if (!srp) { + ret = -ENOMEM; + goto out; + } + srb = (void *)srp; + + printf("srch %sblock blkno %llu\n", level ? "parent " : "", blkno); + print_block_header(&srp->hdr, SCOUTFS_BLOCK_LG_SIZE); + + for (i = 0; level > 0 && i < SCOUTFS_SRCH_PARENT_REFS; i++) { + if (le64_to_cpu(srp->refs[i].blkno) == 0) + continue; + printf(" [%u]: blkno %llu seq %llu\n", + i, le64_to_cpu(srp->refs[i].blkno), + le64_to_cpu(srp->refs[i].seq)); + } + + ret = 0; + for (i = 0; level > 0 && i < SCOUTFS_SRCH_PARENT_REFS; i++) { + if (le64_to_cpu(srp->refs[i].blkno) == 0) + continue; + err = print_srch_block(fd, &srp->refs[i], level - 1); + if (err < 0 && ret == 0) + ret = err; + } + + if (level > 0) + goto out; + + printf(" first "SRE_FMT" last "SRE_FMT" tail "SRE_FMT"\n" + " entry_nr %u entry_bytes %u\n", + SRE_A(&srb->first), SRE_A(&srb->last), SRE_A(&srb->tail), + le32_to_cpu(srb->entry_nr), le32_to_cpu(srb->entry_bytes)); + + memset(&prev, 0, sizeof(prev)); + pos = 0; + for (i = 0; level == 0 && i < le32_to_cpu(srb->entry_nr); i++) { + if (pos > SCOUTFS_SRCH_BLOCK_SAFE_BYTES) { + ret = EIO; + break; + } + + ret = srch_decode_entry(srb->entries + pos, &sre, &prev); + if (ret < 0) + break; + pos += ret; + prev = sre; + printf(" [%u]: (%u) "SRE_FMT"\n", i, ret, SRE_A(&sre)); + } + +out: + free(srp); + + return ret; +} + +struct print_recursion_args { + struct scoutfs_super_block *super; + int fd; + u8 __pad[4]; +}; + +/* same as fs item but with a small header in the value */ +static int print_log_trees_roots(struct scoutfs_key *key, void *val, + unsigned val_len, void *arg) +{ + struct scoutfs_log_trees *lt = val; + struct print_recursion_args *pa = arg; + int ret = 0; + int err; + + /* XXX doesn't print the bloom block */ + + err = print_alloc_list_block(pa->fd, "lt_meta_avail", + <->meta_avail.ref); + if (err && !ret) + ret = err; + err = print_alloc_list_block(pa->fd, "lt_meta_freed", + <->meta_freed.ref); + if (err && !ret) + ret = err; + err = print_btree(pa->fd, pa->super, "data_avail", + <->data_avail.root, print_alloc_item, NULL); + if (err && !ret) + ret = err; + err = print_btree(pa->fd, pa->super, "data_freed", + <->data_freed.root, print_alloc_item, NULL); + if (err && !ret) + ret = err; + err = print_srch_block(pa->fd, <->srch_file.ref, + lt->srch_file.height - 1); + if (err && !ret) + ret = err; + + err = print_btree(pa->fd, pa->super, "", <->item_root, + print_logs_item, NULL); + if (err && !ret) + ret = err; + + return ret; +} + +static int print_srch_root_files(struct scoutfs_key *key, void *val, + unsigned val_len, void *arg) +{ + struct print_recursion_args *pa = arg; + struct scoutfs_srch_compact *sc; + struct scoutfs_srch_file *sfl; + int ret = 0; + int i; + + if (key->sk_type == SCOUTFS_SRCH_PENDING_TYPE || + key->sk_type == SCOUTFS_SRCH_BUSY_TYPE) { + sc = val; + for (i = 0; i < sc->nr; i++) { + sfl = &sc->in[i].sfl; + ret = print_srch_block(pa->fd, &sfl->ref, + sfl->height - 1); + if (ret < 0) + break; + } + + } else { + sfl = val; + ret = print_srch_block(pa->fd, &sfl->ref, sfl->height - 1); + } + + return ret; +} + +static int print_btree_leaf_items(int fd, struct scoutfs_super_block *super, + struct scoutfs_btree_ref *ref, + print_item_func func, void *arg) +{ + struct scoutfs_btree_item *item; + struct scoutfs_avl_node *node; + struct scoutfs_btree_block *bt; + unsigned val_len; + void *key; + void *val; + int ret; + + if (ref->blkno == 0) + return 0; + + bt = read_block(fd, le64_to_cpu(ref->blkno), SCOUTFS_BLOCK_LG_SHIFT); + if (!bt) + return -ENOMEM; + + node = avl_first(&bt->item_root); + while (node) { + item = container_of(node, struct scoutfs_btree_item, node); + val_len = le16_to_cpu(item->val_len); + key = &item->key; + val = (void *)bt + le16_to_cpu(item->val_off); + + if (bt->level > 0) { + ret = print_btree_leaf_items(fd, super, val, func, arg); + if (ret) + break; + continue; + } else { + func(key, val, val_len, arg); + } + + node = avl_next(&bt->item_root, node); + } + + free(bt); + return 0; +} + +static char *alloc_addr_str(struct scoutfs_inet_addr *ia) +{ + struct in_addr addr; + char *quad; + char *str; + int len; + + memset(&addr, 0, sizeof(addr)); + addr.s_addr = htonl(le32_to_cpu(ia->addr)); + quad = inet_ntoa(addr); + if (quad == NULL) + return NULL; + + len = snprintf(NULL, 0, "%s:%u", quad, le16_to_cpu(ia->port)); + if (len < 1 || len > 22) + return NULL; + + len++; /* null */ + str = malloc(len); + if (!str) + return NULL; + + snprintf(str, len, "%s:%u", quad, le16_to_cpu(ia->port)); + return str; +} + +static int print_quorum_blocks(int fd, struct scoutfs_super_block *super) +{ + struct scoutfs_quorum_block *blk = NULL; + char *log_addr = NULL; + u64 blkno; + int ret; + int i; + int j; + + for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) { + blkno = SCOUTFS_QUORUM_BLKNO + i; + free(blk); + blk = read_block(fd, blkno, SCOUTFS_BLOCK_SM_SHIFT); + if (!blk) { + ret = -ENOMEM; + goto out; + } + + if (blk->voter_rid != 0) { + printf("quorum block blkno %llu\n" + " fsid %llx blkno %llu crc 0x%08x\n" + " term %llu write_nr %llu voter_rid %016llx " + "vote_for_rid %016llx\n" + " log_nr %u\n", + blkno, le64_to_cpu(blk->fsid), + le64_to_cpu(blk->blkno), le32_to_cpu(blk->crc), + le64_to_cpu(blk->term), + le64_to_cpu(blk->write_nr), + le64_to_cpu(blk->voter_rid), + le64_to_cpu(blk->vote_for_rid), + blk->log_nr); + for (j = 0; j < blk->log_nr; j++) { + free(log_addr); + log_addr = alloc_addr_str(&blk->log[j].addr); + if (!log_addr) { + ret = -ENOMEM; + goto out; + } + printf(" [%u]: term %llu rid %llu addr %s\n", + j, le64_to_cpu(blk->log[j].term), + le64_to_cpu(blk->log[j].rid), + log_addr); + } + } + } + + ret = 0; +out: + free(log_addr); + + return ret; +} + +static void print_super_block(struct scoutfs_super_block *super, u64 blkno) +{ + char uuid_str[37]; + char *server_addr; + + uuid_unparse(super->uuid, uuid_str); + + printf("super blkno %llu\n", blkno); + print_block_header(&super->hdr, SCOUTFS_BLOCK_SM_SIZE); + printf(" format_hash %llx uuid %s\n", + le64_to_cpu(super->format_hash), uuid_str); + printf(" flags: 0x%016llx\n", super->flags); + + server_addr = alloc_addr_str(&super->server_addr); + if (!server_addr) + return; + + /* XXX these are all in a crazy order */ + printf(" next_ino %llu next_trans_seq %llu\n" + " total_meta_blocks %llu first_meta_blkno %llu last_meta_blkno %llu\n" + " total_data_blocks %llu first_data_blkno %llu last_data_blkno %llu\n" + " quorum_fenced_term %llu quorum_server_term %llu unmount_barrier %llu\n" + " quorum_count %u server_addr %s\n" + " meta_alloc[0]: "ALCROOT_F"\n" + " meta_alloc[1]: "ALCROOT_F"\n" + " data_alloc: "ALCROOT_F"\n" + " server_meta_avail[0]: "AL_HEAD_F"\n" + " server_meta_avail[1]: "AL_HEAD_F"\n" + " server_meta_freed[0]: "AL_HEAD_F"\n" + " server_meta_freed[1]: "AL_HEAD_F"\n" + " lock_clients root: height %u blkno %llu seq %llu\n" + " mounted_clients root: height %u blkno %llu seq %llu\n" + " srch_root root: height %u blkno %llu seq %llu\n" + " trans_seqs root: height %u blkno %llu seq %llu\n" + " fs_root btree root: height %u blkno %llu seq %llu\n", + le64_to_cpu(super->next_ino), + le64_to_cpu(super->next_trans_seq), + le64_to_cpu(super->total_meta_blocks), + le64_to_cpu(super->first_meta_blkno), + le64_to_cpu(super->last_meta_blkno), + le64_to_cpu(super->total_data_blocks), + le64_to_cpu(super->first_data_blkno), + le64_to_cpu(super->last_data_blkno), + le64_to_cpu(super->quorum_fenced_term), + le64_to_cpu(super->quorum_server_term), + le64_to_cpu(super->unmount_barrier), + super->quorum_count, + server_addr, + ALCROOT_A(&super->meta_alloc[0]), + ALCROOT_A(&super->meta_alloc[1]), + ALCROOT_A(&super->data_alloc), + AL_HEAD_A(&super->server_meta_avail[0]), + AL_HEAD_A(&super->server_meta_avail[1]), + AL_HEAD_A(&super->server_meta_freed[0]), + AL_HEAD_A(&super->server_meta_freed[1]), + super->lock_clients.height, + le64_to_cpu(super->lock_clients.ref.blkno), + le64_to_cpu(super->lock_clients.ref.seq), + super->mounted_clients.height, + le64_to_cpu(super->mounted_clients.ref.blkno), + le64_to_cpu(super->mounted_clients.ref.seq), + super->srch_root.height, + le64_to_cpu(super->srch_root.ref.blkno), + le64_to_cpu(super->srch_root.ref.seq), + super->trans_seqs.height, + le64_to_cpu(super->trans_seqs.ref.blkno), + le64_to_cpu(super->trans_seqs.ref.seq), + super->fs_root.height, + le64_to_cpu(super->fs_root.ref.blkno), + le64_to_cpu(super->fs_root.ref.seq)); + + free(server_addr); +} + +static int print_volume(int fd) +{ + struct scoutfs_super_block *super = NULL; + struct print_recursion_args pa; + char str[80]; + int ret = 0; + int err; + int i; + + super = read_block(fd, SCOUTFS_SUPER_BLKNO, SCOUTFS_BLOCK_SM_SHIFT); + if (!super) + return -ENOMEM; + + print_super_block(super, SCOUTFS_SUPER_BLKNO); + + ret = print_quorum_blocks(fd, super); + + err = print_btree(fd, super, "lock_clients", &super->lock_clients, + print_lock_clients_entry, NULL); + if (err && !ret) + ret = err; + + err = print_btree(fd, super, "mounted_clients", &super->mounted_clients, + print_mounted_client_entry, NULL); + if (err && !ret) + ret = err; + + err = print_btree(fd, super, "trans_seqs", &super->trans_seqs, + print_trans_seqs_entry, NULL); + if (err && !ret) + ret = err; + + for (i = 0; i < array_size(super->server_meta_avail); i++) { + snprintf(str, sizeof(str), "server_meta_avail[%u]", i); + err = print_alloc_list_block(fd, str, + &super->server_meta_avail[i].ref); + if (err && !ret) + ret = err; + } + + for (i = 0; i < array_size(super->server_meta_freed); i++) { + snprintf(str, sizeof(str), "server_meta_freed[%u]", i); + err = print_alloc_list_block(fd, str, + &super->server_meta_freed[i].ref); + if (err && !ret) + ret = err; + } + + for (i = 0; i < array_size(super->meta_alloc); i++) { + snprintf(str, sizeof(str), "meta_alloc[%u]", i); + err = print_btree(fd, super, str, &super->meta_alloc[i].root, + print_alloc_item, NULL); + if (err && !ret) + ret = err; + } + + err = print_btree(fd, super, "data_alloc", &super->data_alloc.root, + print_alloc_item, NULL); + if (err && !ret) + ret = err; + + err = print_btree(fd, super, "srch_root", &super->srch_root, + print_srch_root_item, NULL); + if (err && !ret) + ret = err; + err = print_btree(fd, super, "logs_root", &super->logs_root, + print_log_trees_item, NULL); + if (err && !ret) + ret = err; + + pa.super = super; + pa.fd = fd; + err = print_btree_leaf_items(fd, super, &super->srch_root.ref, + print_srch_root_files, &pa); + if (err && !ret) + ret = err; + err = print_btree_leaf_items(fd, super, &super->logs_root.ref, + print_log_trees_roots, &pa); + if (err && !ret) + ret = err; + + err = print_btree(fd, super, "fs_root", &super->fs_root, + print_fs_item, NULL); + if (err && !ret) + ret = err; + + free(super); + + return ret; +} + +static int print_cmd(int argc, char **argv) +{ + char *path; + int ret; + int fd; + + if (argc != 2) { + printf("scoutfs print: a single path argument is required\n"); + return -EINVAL; + } + path = argv[1]; + + fd = open(path, O_RDONLY); + if (fd < 0) { + ret = -errno; + fprintf(stderr, "failed to open '%s': %s (%d)\n", + path, strerror(errno), errno); + return ret; + } + + ret = print_volume(fd); + close(fd); + return ret; +}; + +static void __attribute__((constructor)) print_ctor(void) +{ + cmd_register("print", "", "print metadata structures", + print_cmd); +} diff --git a/utils/src/rand.c b/utils/src/rand.c new file mode 100644 index 00000000..8af89165 --- /dev/null +++ b/utils/src/rand.c @@ -0,0 +1,12 @@ +#include + +#include "rand.h" +#include "sparse.h" +#include "util.h" + +#include + +void pseudo_random_bytes(void *data, unsigned int len) +{ + RAND_bytes(data, len); +} diff --git a/utils/src/rand.h b/utils/src/rand.h new file mode 100644 index 00000000..cbc74eb7 --- /dev/null +++ b/utils/src/rand.h @@ -0,0 +1,10 @@ +#ifndef _RAND_H_ +#define _RAND_H_ + +/* + * We could play around a bit with some macros to get aligned constant + * word sized buffers filled by single instructions. + */ +void pseudo_random_bytes(void *data, unsigned int len); + +#endif diff --git a/utils/src/search_xattrs.c b/utils/src/search_xattrs.c new file mode 100644 index 00000000..4d7e86e8 --- /dev/null +++ b/utils/src/search_xattrs.c @@ -0,0 +1,134 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sparse.h" +#include "util.h" +#include "format.h" +#include "ioctl.h" +#include "cmd.h" + +static struct option long_ops[] = { + { "name", 1, NULL, 'n' }, + { "file", 1, NULL, 'f' }, + { NULL, 0, NULL, 0} +}; + +/* + * There are significant constant costs to each search call, we + * want to get the inodes in as few calls as possible. + */ +#define BATCH_SIZE 1000000 + +static int search_xattrs_cmd(int argc, char **argv) +{ + struct scoutfs_ioctl_search_xattrs sx; + char *path = NULL; + char *name = NULL; + u64 *inos = NULL; + int fd = -1; + int ret; + int c; + int i; + + memset(&sx, 0, sizeof(sx)); + inos = malloc(BATCH_SIZE * sizeof(inos[0])); + if (!inos) { + fprintf(stderr, "inos mem alloc failed\n"); + ret = -ENOMEM; + goto out; + } + + while ((c = getopt_long(argc, argv, "f:n:", long_ops, NULL)) != -1) { + switch (c) { + case 'f': + path = strdup(optarg); + if (!path) { + fprintf(stderr, "path mem alloc failed\n"); + ret = -ENOMEM; + goto out; + } + break; + case 'n': + name = strdup(optarg); + if (!name) { + fprintf(stderr, "name mem alloc failed\n"); + ret = -ENOMEM; + goto out; + } + break; + case '?': + default: + ret = -EINVAL; + goto out; + } + } + + if (path == NULL) { + fprintf(stderr, "must specify -f path to file\n"); + ret = -EINVAL; + goto out; + } + + if (name == NULL) { + fprintf(stderr, "must specify -n xattr name to search for\n"); + ret = -EINVAL; + goto out; + } + + fd = open(path, O_RDONLY); + if (fd < 0) { + ret = -errno; + fprintf(stderr, "failed to open '%s': %s (%d)\n", + path, strerror(errno), errno); + goto out; + } + + sx.next_ino = 0; + sx.last_ino = U64_MAX; + sx.name_ptr = (unsigned long)name; + sx.inodes_ptr = (unsigned long)inos; + sx.name_bytes = strlen(name); + sx.nr_inodes = BATCH_SIZE; + + do { + ret = ioctl(fd, SCOUTFS_IOC_SEARCH_XATTRS, &sx); + if (ret == 0) + break; + if (ret < 0) { + ret = -errno; + fprintf(stderr, "search_xattrs ioctl failed: " + "%s (%d)\n", strerror(errno), errno); + goto out; + } + + for (i = 0; i < ret; i++) + printf("%llu\n", inos[i]); + + sx.next_ino = inos[ret - 1] + 1; + } while (!(sx.output_flags & SCOUTFS_SEARCH_XATTRS_OFLAG_END)); + + ret = 0; +out: + if (fd >= 0) + close(fd); + free(path); + free(name); + free(inos); + + return ret; +}; + +static void __attribute__((constructor)) search_xattrs_ctor(void) +{ + cmd_register("search-xattrs", "-n name -f ", + "print inode numbers of inodes which may have given xattr", + search_xattrs_cmd); +} diff --git a/utils/src/setattr.c b/utils/src/setattr.c new file mode 100644 index 00000000..e9ab3b34 --- /dev/null +++ b/utils/src/setattr.c @@ -0,0 +1,112 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sparse.h" +#include "util.h" +#include "format.h" +#include "ioctl.h" +#include "parse.h" +#include "cmd.h" + +static struct option long_ops[] = { + { "ctime", 1, NULL, 'c' }, + { "data_version", 1, NULL, 'd' }, + { "file", 1, NULL, 'f' }, + { "offline", 0, NULL, 'o' }, + { "i_size", 1, NULL, 's' }, + { NULL, 0, NULL, 0} +}; + +static int setattr_more_cmd(int argc, char **argv) +{ + struct scoutfs_ioctl_setattr_more sm; + struct timespec ctime; + char *path = NULL; + int ret; + int fd = -1; + int c; + + memset(&sm, 0, sizeof(sm)); + + while ((c = getopt_long(argc, argv, "c:d:f:os:", long_ops, NULL)) != -1) { + switch (c) { + case 'c': + ret = parse_timespec(optarg, &ctime); + if (ret) + goto out; + break; + case 'd': + ret = parse_u64(optarg, &sm.data_version); + if (ret) + goto out; + break; + case 'f': + path = strdup(optarg); + if (!path) { + fprintf(stderr, "path mem alloc failed\n"); + ret = -ENOMEM; + goto out; + } + break; + case 'o': + sm.flags |= SCOUTFS_IOC_SETATTR_MORE_OFFLINE; + break; + case 's': + ret = parse_u64(optarg, &sm.i_size); + if (ret) + goto out; + break; + case '?': + default: + ret = -EINVAL; + goto out; + } + } + + if (path == NULL) { + fprintf(stderr, "must specify -f path to file\n"); + ret = -EINVAL; + goto out; + } + + fd = open(path, O_WRONLY); + if (fd < 0) { + ret = -errno; + fprintf(stderr, "failed to open '%s': %s (%d)\n", + path, strerror(errno), errno); + goto out; + } + + sm.ctime_sec = ctime.tv_sec; + sm.ctime_nsec = ctime.tv_nsec; + + ret = ioctl(fd, SCOUTFS_IOC_SETATTR_MORE, &sm); + if (ret < 0) { + ret = -errno; + fprintf(stderr, "setattr_more ioctl failed on '%s': " + "%s (%d)\n", path, strerror(errno), errno); + goto out; + } + + ret = 0; +out: + if (fd >= 0) + close(fd); + return ret; +} + +static void __attribute__((constructor)) setattr_more_ctor(void) +{ + cmd_register("setattr", "-c ctime -d data_version -o -s i_size -f ", + "set attributes on file with no data", + setattr_more_cmd); +} diff --git a/utils/src/sparse.h b/utils/src/sparse.h new file mode 100644 index 00000000..012f27dc --- /dev/null +++ b/utils/src/sparse.h @@ -0,0 +1,112 @@ +#ifndef _SPARSE_H_ +#define _SPARSE_H_ + +#include +#include +#include + +#ifdef __CHECKER__ +# undef __force +# define __force __attribute__((force)) +/* sparse seems to get confused by some builtins */ +extern int __builtin_ia32_rdrand64_step(unsigned long long *); +extern unsigned int __builtin_ia32_crc32di(unsigned int, unsigned long long); +extern unsigned int __builtin_ia32_crc32si(unsigned int, unsigned int); +extern unsigned int __builtin_ia32_crc32hi(unsigned int, unsigned short); +extern unsigned int __builtin_ia32_crc32qi(unsigned int, unsigned char); + +#else +# define __force +#endif + +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned int u32; +typedef int s32; +typedef unsigned long long u64; +typedef signed long long s64; + +typedef u8 __u8; +typedef u16 __u16; +typedef u32 __u32; +typedef s32 __s32; +typedef u64 __u64; +typedef s64 __s64; + +static inline u16 ___swab16(u16 x) +{ + return ((x & (u16)0x00ffU) << 8) | + ((x & (u16)0xff00U) >> 8); +} + +static inline u32 ___swab32(u32 x) +{ + return ((x & (u32)0x000000ffUL) << 24) | + ((x & (u32)0x0000ff00UL) << 8) | + ((x & (u32)0x00ff0000UL) >> 8) | + ((x & (u32)0xff000000UL) >> 24); +} + +static inline u64 ___swab64(u64 x) +{ + return (u64)((x & (u64)0x00000000000000ffULL) << 56) | + (u64)((x & (u64)0x000000000000ff00ULL) << 40) | + (u64)((x & (u64)0x0000000000ff0000ULL) << 24) | + (u64)((x & (u64)0x00000000ff000000ULL) << 8) | + (u64)((x & (u64)0x000000ff00000000ULL) >> 8) | + (u64)((x & (u64)0x0000ff0000000000ULL) >> 24) | + (u64)((x & (u64)0x00ff000000000000ULL) >> 40) | + (u64)((x & (u64)0xff00000000000000ULL) >> 56); +} + +#define __gen_cast_tofrom(end, size) \ +static inline __##end##size cpu_to_##end##size(u##size x) \ +{ \ + return (__force __##end##size)x; \ +} \ +static inline u##size end##size##_to_cpu(__##end##size x) \ +{ \ + return (__force u##size)x; \ +} + +#define __gen_swap_tofrom(end, size) \ +static inline __##end##size cpu_to_##end##size(u##size x) \ +{ \ + return (__force __##end##size)___swab##size(x); \ +} \ +static inline u##size end##size##_to_cpu(__##end##size x) \ +{ \ + return ___swab##size((__force u##size) x); \ +} + +#define __gen_functions(which, end) \ + __gen_##which##_tofrom(end, 16) \ + __gen_##which##_tofrom(end, 32) \ + __gen_##which##_tofrom(end, 64) + +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN_BITFIELD +__gen_functions(swap, be) +__gen_functions(cast, le) +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN_BITFIELD +__gen_functions(swap, le) +__gen_functions(cast, be) +#else +#error "machine is neither BIG_ENDIAN nor LITTLE_ENDIAN" +#endif + +#define __gen_add_funcs(end, size) \ +static inline void end##size##_add_cpu(__##end##size *val, u##size delta) \ +{ \ + *val = cpu_to_##end##size(end##size##_to_cpu(*val) + delta); \ +} + +__gen_add_funcs(le, 16) +__gen_add_funcs(le, 32) +__gen_add_funcs(le, 64) +__gen_add_funcs(be, 16) +__gen_add_funcs(be, 32) +__gen_add_funcs(be, 64) + +#endif diff --git a/utils/src/srch.c b/utils/src/srch.c new file mode 100644 index 00000000..b58075f7 --- /dev/null +++ b/utils/src/srch.c @@ -0,0 +1,46 @@ +#include + +#include "sparse.h" +#include "util.h" +#include "format.h" +#include "srch.h" + +/* shifting by width is undefined :/ */ +#define BYTE_MASK(b) ((1ULL << (b << 3)) - 1) +static u64 byte_masks[] = { + 0, BYTE_MASK(1), BYTE_MASK(2), BYTE_MASK(3), + BYTE_MASK(4), BYTE_MASK(5), BYTE_MASK(6), BYTE_MASK(7), U64_MAX, +}; + +static u64 decode_u64(void *buf, int bytes) +{ + u64 val = get_unaligned_le64(buf) & byte_masks[bytes]; + + return (val >> 1) ^ (-(val & 1)); +} + +int srch_decode_entry(void *buf, struct scoutfs_srch_entry *sre, + struct scoutfs_srch_entry *prev) +{ + u64 diffs[3]; + u16 lengths; + int bytes; + int tot; + int i; + + lengths = get_unaligned_le16(buf); + tot = 2; + + for (i = 0; i < array_size(diffs); i++) { + bytes = min(8, lengths & 15); + diffs[i] = decode_u64(buf + tot, bytes); + tot += bytes; + lengths >>= 4; + } + + sre->hash = cpu_to_le64(le64_to_cpu(prev->hash) + diffs[0]); + sre->ino = cpu_to_le64(le64_to_cpu(prev->ino) + diffs[1]); + sre->id = cpu_to_le64(le64_to_cpu(prev->id) + diffs[2]); + + return tot; +} diff --git a/utils/src/srch.h b/utils/src/srch.h new file mode 100644 index 00000000..c44c52f9 --- /dev/null +++ b/utils/src/srch.h @@ -0,0 +1,7 @@ +#ifndef _SRCH_H_ +#define _SRCH_H_ + +int srch_decode_entry(void *buf, struct scoutfs_srch_entry *sre, + struct scoutfs_srch_entry *prev); + +#endif diff --git a/utils/src/stage_release.c b/utils/src/stage_release.c new file mode 100644 index 00000000..2e1ec342 --- /dev/null +++ b/utils/src/stage_release.c @@ -0,0 +1,210 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sparse.h" +#include "util.h" +#include "format.h" +#include "ioctl.h" +#include "cmd.h" + +static int stage_cmd(int argc, char **argv) +{ + struct scoutfs_ioctl_stage args; + unsigned int buf_len = 1024 * 1024; + unsigned int bytes; + char *endptr = NULL; + char *buf = NULL; + int afd = -1; + int fd = -1; + u64 offset; + u64 count; + u64 vers; + int ret; + + if (argc != 6) { + fprintf(stderr, "must specify moar args\n"); + return -EINVAL; + } + + fd = open(argv[1], O_RDWR); + if (fd < 0) { + ret = -errno; + fprintf(stderr, "failed to open '%s': %s (%d)\n", + argv[1], strerror(errno), errno); + return ret; + } + + vers = strtoull(argv[2], &endptr, 0); + if (*endptr != '\0' || + ((vers == LLONG_MIN || vers == LLONG_MAX) && errno == ERANGE)) { + fprintf(stderr, "error parsing data version '%s'\n", + argv[2]); + ret = -EINVAL; + goto out; + } + + offset = strtoull(argv[3], &endptr, 0); + if (*endptr != '\0' || + ((offset == LLONG_MIN || offset == LLONG_MAX) && errno == ERANGE)) { + fprintf(stderr, "error parsing offset '%s'\n", + argv[3]); + ret = -EINVAL; + goto out; + } + + count = strtoull(argv[4], &endptr, 0); + if (*endptr != '\0' || + ((count == LLONG_MIN || count == LLONG_MAX) && errno == ERANGE)) { + fprintf(stderr, "error parsing count '%s'\n", + argv[4]); + ret = -EINVAL; + goto out; + } + + if (count > INT_MAX) { + fprintf(stderr, "count %llu too large, limited to %d\n", + count, INT_MAX); + ret = -EINVAL; + goto out; + } + + afd = open(argv[5], O_RDONLY); + if (afd < 0) { + ret = -errno; + fprintf(stderr, "failed to open '%s': %s (%d)\n", + argv[5], strerror(errno), errno); + goto out; + } + + buf = malloc(buf_len); + if (!buf) { + fprintf(stderr, "couldn't allocate %u byte buffer\n", buf_len); + ret = -ENOMEM; + goto out; + } + + while (count) { + + bytes = min(count, buf_len); + + ret = read(afd, buf, bytes); + if (ret <= 0) { + fprintf(stderr, "archive read returned %d: error %s (%d)\n", + ret, strerror(errno), errno); + ret = -EIO; + goto out; + } + + bytes = ret; + + args.data_version = vers; + args.buf_ptr = (unsigned long)buf; + args.offset = offset; + args.count = bytes; + + count -= bytes; + offset += bytes; + + ret = ioctl(fd, SCOUTFS_IOC_STAGE, &args); + if (ret != bytes) { + fprintf(stderr, "stage returned %d, not %u: error %s (%d)\n", + ret, bytes, strerror(errno), errno); + ret = -EIO; + goto out; + } + } + + ret = 0; +out: + free(buf); + if (fd > -1) + close(fd); + if (afd > -1) + close(afd); + return ret; +}; + +static void __attribute__((constructor)) stage_ctor(void) +{ + cmd_register("stage", " ", + "write archive file contents to offline region", stage_cmd); +} + +static int release_cmd(int argc, char **argv) +{ + struct scoutfs_ioctl_release args; + char *endptr = NULL; + u64 block; + u64 count; + u64 vers; + int ret; + int fd; + + if (argc != 5) { + fprintf(stderr, "must specify path, data version, offset, and count\n"); + return -EINVAL; + } + + fd = open(argv[1], O_RDWR); + if (fd < 0) { + ret = -errno; + fprintf(stderr, "failed to open '%s': %s (%d)\n", + argv[1], strerror(errno), errno); + return ret; + } + + vers = strtoull(argv[2], &endptr, 0); + if (*endptr != '\0' || + ((vers == LLONG_MIN || vers == LLONG_MAX) && errno == ERANGE)) { + fprintf(stderr, "error parsing data version '%s'\n", + argv[2]); + ret = -EINVAL; + goto out; + } + + block = strtoull(argv[3], &endptr, 0); + if (*endptr != '\0' || + ((block == LLONG_MIN || block == LLONG_MAX) && errno == ERANGE)) { + fprintf(stderr, "error parsing starting 4K block offset '%s'\n", + argv[3]); + ret = -EINVAL; + goto out; + } + + count = strtoull(argv[4], &endptr, 0); + if (*endptr != '\0' || + ((count == LLONG_MIN || count == LLONG_MAX) && errno == ERANGE)) { + fprintf(stderr, "error parsing length '%s'\n", + argv[4]); + ret = -EINVAL; + goto out; + } + + args.block = block; + args.count = count; + args.data_version = vers; + + ret = ioctl(fd, SCOUTFS_IOC_RELEASE, &args); + if (ret < 0) { + ret = -errno; + fprintf(stderr, "release ioctl failed: %s (%d)\n", + strerror(errno), errno); + } +out: + close(fd); + return ret; +}; + +static void __attribute__((constructor)) release_ctor(void) +{ + cmd_register("release", " <4K block offset> ", + "mark file region offline and free extents", release_cmd); +} diff --git a/utils/src/stat.c b/utils/src/stat.c new file mode 100644 index 00000000..ef9c6247 --- /dev/null +++ b/utils/src/stat.c @@ -0,0 +1,225 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sparse.h" +#include "util.h" +#include "format.h" +#include "ioctl.h" +#include "cmd.h" + +struct stat_more_field { + char *name; + size_t offset; +}; + +#define FIELD(f, o) { \ + .name = #f, \ + .offset = o, \ +} + +#define INODE_FIELD_OFF(f) offsetof(struct scoutfs_ioctl_stat_more, f) +#define INODE_FIELD(f) FIELD(f, INODE_FIELD_OFF(f)) + +static struct stat_more_field inode_fields[] = { + INODE_FIELD(meta_seq), + INODE_FIELD(data_seq), + INODE_FIELD(data_version), + INODE_FIELD(online_blocks), + INODE_FIELD(offline_blocks), + { NULL, } +}; + +static void print_inode_field(void *st, size_t off) +{ + struct scoutfs_ioctl_stat_more *stm = st; + + switch(off) { + case INODE_FIELD_OFF(meta_seq): + printf("%llu", stm->meta_seq); + break; + case INODE_FIELD_OFF(data_seq): + printf("%llu", stm->data_seq); + break; + case INODE_FIELD_OFF(data_version): + printf("%llu", stm->data_version); + break; + case INODE_FIELD_OFF(online_blocks): + printf("%llu", stm->online_blocks); + break; + case INODE_FIELD_OFF(offline_blocks): + printf("%llu", stm->offline_blocks); + break; + }; +} + +#define FS_FIELD_OFF(f) offsetof(struct scoutfs_ioctl_statfs_more, f) +#define FS_FIELD(f) FIELD(f, FS_FIELD_OFF(f)) + +static struct stat_more_field fs_fields[] = { + FS_FIELD(fsid), + FS_FIELD(rid), + FS_FIELD(committed_seq), + FS_FIELD(total_meta_blocks), + FS_FIELD(total_data_blocks), + { NULL, } +}; + +static void print_fs_field(void *st, size_t off) +{ + struct scoutfs_ioctl_statfs_more *sfm = st; + + switch(off) { + case FS_FIELD_OFF(fsid): + printf("%016llx", sfm->fsid); + break; + case FS_FIELD_OFF(rid): + printf("%016llx", sfm->rid); + break; + case FS_FIELD_OFF(committed_seq): + printf("%llu", sfm->committed_seq); + break; + case FS_FIELD_OFF(total_meta_blocks): + printf("%llu", sfm->total_meta_blocks); + break; + case FS_FIELD_OFF(total_data_blocks): + printf("%llu", sfm->total_data_blocks); + break; + }; +} + +#define for_each_field(f, fields) \ + for (f = fields; f->name; f++) + +typedef void (*print_field_t)(void *st, size_t off); + +static struct option long_ops[] = { + { "single_field", 1, NULL, 's' }, + { NULL, 0, NULL, 0} +}; + +static int do_stat(int argc, char **argv, int is_inode) +{ + union { + struct scoutfs_ioctl_stat_more stm; + struct scoutfs_ioctl_statfs_more sfm; + } st; + struct stat_more_field *single = NULL; + struct stat_more_field *fields; + struct stat_more_field *fi; + char *single_name = NULL; + print_field_t pr = NULL; + char *path; + int cmd; + int ret; + int fd; + int i; + int c; + + memset(&st, 0, sizeof(st)); + if (is_inode) { + cmd = SCOUTFS_IOC_STAT_MORE; + fields = inode_fields; + st.stm.valid_bytes = sizeof(struct scoutfs_ioctl_stat_more); + pr = print_inode_field; + } else { + cmd = SCOUTFS_IOC_STATFS_MORE; + fields = fs_fields; + st.sfm.valid_bytes = sizeof(struct scoutfs_ioctl_statfs_more); + pr = print_fs_field; + } + + while ((c = getopt_long(argc, argv, "s:", long_ops, NULL)) != -1) { + switch (c) { + case 's': + single_name = strdup(optarg); + assert(single_name); + break; + case '?': + default: + return -EINVAL; + } + } + + if (single_name) { + for_each_field(fi, fields) { + if (strcmp(fi->name, single_name) == 0) { + single = fi; + break; + } + } + if (!single) { + fprintf(stderr, "unknown field: '%s'\n", single_name); + return -EINVAL; + } + } + + if (optind >= argc) { + fprintf(stderr, "must specify at least one path argument\n"); + return -EINVAL; + } + + for (i = optind; i < argc; i++) { + path = argv[i]; + + fd = open(path, O_RDONLY); + if (fd < 0) { + ret = -errno; + fprintf(stderr, "failed to open '%s': %s (%d)\n", + path, strerror(errno), errno); + continue; + } + + ret = ioctl(fd, cmd, &st); + if (ret < 0) { + ret = -errno; + fprintf(stderr, "ioctl failed on '%s': " + "%s (%d)\n", path, strerror(errno), errno); + + } else if (single) { + pr(&st, single->offset); + printf("\n"); + } else { + printf("%-17s %s\n", "path", path); + for_each_field(fi, fields) { + printf("%-17s ", fi->name); + pr(&st, fi->offset); + printf("\n"); + } + } + + close(fd); + } + + return 0; +} + +static int stat_more_cmd(int argc, char **argv) +{ + return do_stat(argc, argv, 1); +} + +static int statfs_more_cmd(int argc, char **argv) +{ + return do_stat(argc, argv, 0); +} + +static void __attribute__((constructor)) stat_more_ctor(void) +{ + cmd_register("stat", "", + "show scoutfs inode information", stat_more_cmd); +} + +static void __attribute__((constructor)) statfs_more_ctor(void) +{ + cmd_register("statfs", "", + "show scoutfs file system information", statfs_more_cmd); +} diff --git a/utils/src/util.h b/utils/src/util.h new file mode 100644 index 00000000..637fdf5b --- /dev/null +++ b/utils/src/util.h @@ -0,0 +1,114 @@ +#ifndef _UTIL_H_ +#define _UTIL_H_ + +#include +#include +#include +#include + +#include "sparse.h" + +/* + * Generate build warnings if the condition is false but generate no + * code at run time if it's true. + */ +#define build_assert(cond) ((void)sizeof(char[1 - 2*!(cond)])) + +#define min(a, b) \ +({ \ + __typeof__(a) _a = (a); \ + __typeof__(b) _b = (b); \ + \ + _a < _b ? _a : _b; \ +}) + +#define max(a, b) \ +({ \ + __typeof__(a) _a = (a); \ + __typeof__(b) _b = (b); \ + \ + _a > _b ? _a : _b; \ +}) + +#define swap(a, b) \ +do { \ + __typeof__(a) _t = (a); \ + (a) = (b); \ + (b) = (_t); \ +} while (0) + +#define array_size(arr) (sizeof(arr) / sizeof(arr[0])) + +#define __packed __attribute__((packed)) + +/* + * Round the 'a' value up to the next 'b' power of two boundary. It + * casts the mask to the value type before masking to avoid truncation + * problems. + */ +#define round_up(a, b) \ +({ \ + __typeof__(a) _b = (b); \ + \ + ((a) + _b - 1) & ~(_b - 1); \ +}) +#define round_down(a, b) \ +({ \ + __typeof__(a) _b = (b); \ + \ + ((a) & ~(_b - 1)); \ +}) + +#define DIV_ROUND_UP(x, y) (((x) + (y) - 1) / (y)) +#define ALIGN(x, y) (((x) + (y) - 1) & ~((y) - 1)) + +#ifndef offsetof +#define offsetof(type, memb) ((unsigned long)&((type *)0)->memb) +#endif + +#define container_of(ptr, type, memb) \ + ((type *)((void *)(ptr) - offsetof(type, memb))) + +#define BITS_PER_LONG (sizeof(long) * 8) +#define U8_MAX ((u8)~0ULL) +#define U16_MAX ((u16)~0ULL) +#define U32_MAX ((u32)~0ULL) +#define U64_MAX ((u64)~0ULL) + +#define flsll(x) \ +({ \ + unsigned long long _x = (x); \ + \ + (_x == 0 ? 0 : 64 - __builtin_clzll(_x)); \ +}) + +#define ilog2(x) \ +({ \ + ((unsigned long)log2l((long double)x)); \ +}) + +#define emit_get_unaligned_le(nr) \ +static inline __u##nr get_unaligned_le##nr(void *buf) \ +{ \ + __le##nr x; \ + memcpy(&x, buf, sizeof(x)); \ + return le##nr##_to_cpu(x); \ +} +emit_get_unaligned_le(16) +emit_get_unaligned_le(32) +emit_get_unaligned_le(64) + +/* + * return -1,0,+1 based on the memcmp comparison of the minimum of their + * two lengths. If their min shared bytes are equal but the lengths + * are not then the larger length is considered greater. + */ +static inline int memcmp_lens(const void *a, int a_len, + const void *b, int b_len) +{ + unsigned int len = min(a_len, b_len); + + return memcmp(a, b, len) ?: a_len - b_len; +} + +#endif diff --git a/utils/src/waiting.c b/utils/src/waiting.c new file mode 100644 index 00000000..414a3c4c --- /dev/null +++ b/utils/src/waiting.c @@ -0,0 +1,166 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sparse.h" +#include "util.h" +#include "format.h" +#include "ioctl.h" +#include "cmd.h" +#include "parse.h" + +#ifndef MAX_ERRNO +#define MAX_ERRNO 4095 +#endif + +#define OP_FMT "%s%s" + +/* + * Print the caller's string for the bit if it's set, and if it's set + * and there are more significant bits coming then we also print a + * separating comma. + */ +#define op_str(ops, bit, str) \ + (((ops) & (bit)) ? (str) : ""), \ + (((ops) & (bit)) && ((ops) & ~(((bit) << 1) - 1)) ? "," : "") + +static int waiting_cmd(int argc, char **argv) +{ + struct scoutfs_ioctl_data_waiting_entry dwe[16]; + struct scoutfs_ioctl_data_waiting idw; + int ret; + int fd; + int i; + + if (argc != 4) { + fprintf(stderr, "must specify ino, iblock, and path\n"); + return -EINVAL; + } + + ret = parse_u64(argv[1], &idw.after_ino) ?: + parse_u64(argv[2], &idw.after_iblock); + if (ret) + return ret; + + fd = open(argv[3], O_RDONLY); + if (fd < 0) { + ret = -errno; + fprintf(stderr, "failed to open '%s': %s (%d)\n", + argv[3], strerror(errno), errno); + return ret; + } + + idw.flags = 0; + idw.ents_ptr = (unsigned long)dwe; + idw.ents_nr = array_size(dwe); + + for (;;) { + ret = ioctl(fd, SCOUTFS_IOC_DATA_WAITING, &idw); + if (ret < 0) { + ret = -errno; + fprintf(stderr, "waiting ioctl failed: %s (%d)\n", + strerror(errno), errno); + break; + } else if (ret == 0) { + break; + } + + for (i = 0; i < ret; i++) + printf("ino %llu iblock %llu ops " + OP_FMT OP_FMT OP_FMT"\n", + dwe[i].ino, dwe[i].iblock, + op_str(dwe[i].op, SCOUTFS_IOC_DWO_READ, + "read"), + op_str(dwe[i].op, SCOUTFS_IOC_DWO_WRITE, + "write"), + op_str(dwe[i].op, SCOUTFS_IOC_DWO_CHANGE_SIZE, + "change_size")); + + idw.after_ino = dwe[i - 1].ino; + idw.after_iblock = dwe[i - 1].iblock; + } + + close(fd); + return ret; +}; + +static void __attribute__((constructor)) waiting_ctor(void) +{ + cmd_register("data-waiting", " ", + "print ops waiting for data blocks", waiting_cmd); +} + +static int data_wait_err_cmd(int argc, char **argv) +{ + struct scoutfs_ioctl_data_wait_err args; + int fd = -1; + int ret; + + memset(&args, 0, sizeof(args)); + + if (argc != 8) { + fprintf(stderr, "must specify path, ino, version, offset, count,op, and err\n"); + return -EINVAL; + } + + ret = parse_u64(argv[2], &args.ino) ?: + parse_u64(argv[3], &args.data_version) ?: + parse_u64(argv[4], &args.offset) ?: + parse_u64(argv[5], &args.count) ?: + parse_s64(argv[7], &args.err); + if (ret) + return ret; + + if ((args.err >= 0) || (args.err < -MAX_ERRNO)) { + fprintf(stderr, "err %lld invalid\n", args.err); + ret = -EINVAL; + goto out; + } + + if (!strcmp(argv[6], "read")) { + args.op = SCOUTFS_IOC_DWO_READ; + } else if (!strcmp(argv[6], "write")) { + args.op = SCOUTFS_IOC_DWO_WRITE; + } else if (!strcmp(argv[6], "change_size")) { + args.op = SCOUTFS_IOC_DWO_CHANGE_SIZE; + } else { + fprintf(stderr, "invalid data wait op: '%s'\n", argv[6]); + return -EINVAL; + } + + fd = open(argv[1], O_RDONLY); + if (fd < 0) { + ret = -errno; + fprintf(stderr, "failed to open '%s': %s (%d)\n", + argv[1], strerror(errno), errno); + return ret; + } + + ret = ioctl(fd, SCOUTFS_IOC_DATA_WAIT_ERR, &args); + if (ret < 0) { + fprintf(stderr, "data_wait_err returned %d: error %s (%d)\n", + ret, strerror(errno), errno); + ret = -EIO; + goto out; + } + printf("data_wait_err found %d waiters.\n", ret); + +out: + if (fd > -1) + close(fd); + return ret; +}; + +static void __attribute__((constructor)) data_wait_err_ctor(void) +{ + cmd_register("data-wait-err", " ", + "return error from matching waiters", + data_wait_err_cmd); +} diff --git a/utils/src/walk_inodes.c b/utils/src/walk_inodes.c new file mode 100644 index 00000000..6cf36d5a --- /dev/null +++ b/utils/src/walk_inodes.c @@ -0,0 +1,156 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sparse.h" +#include "util.h" +#include "format.h" +#include "ioctl.h" +#include "cmd.h" + +/* + * Parse the command line specification of a walk inodes entry of the + * form "major.minor.ino". At least one value must be given, the rest + * default to 0. + */ +static int parse_walk_entry(struct scoutfs_ioctl_walk_inodes_entry *ent, + char *str) +{ + char *endptr; + char *c; + u64 ull; + u64 minor = 0; + u64 *val; + + memset(ent, 0, sizeof(*ent)); + val = &ent->major; + + for (;;) { + c = index(str, '.'); + if (c) + *c = '\0'; + + endptr = NULL; + ull = strtoull(str, &endptr, 0); + if (*endptr != '\0' || + ((ull == LLONG_MIN || ull == LLONG_MAX) && + errno == ERANGE) || + (val == &minor && (*val < INT_MIN || *val > INT_MAX))) { + fprintf(stderr, "bad index pos at '%s'\n", str); + return -EINVAL; + } + + *val = ull; + + if (val == &ent->major) + val = &minor; + else if (val == &minor) + val = &ent->ino; + else + break; + + if (c) + str = c + 1; + else + break; + } + + ent->minor = minor; + return 0; +} + +static int walk_inodes_cmd(int argc, char **argv) +{ + struct scoutfs_ioctl_walk_inodes_entry ents[128]; + struct scoutfs_ioctl_walk_inodes walk; + u64 total = 0; + int ret; + int fd; + int i; + + if (argc != 5) { + fprintf(stderr, "must specify seq and path\n"); + return -EINVAL; + } + + if (!strcasecmp(argv[1], "meta_seq")) + walk.index = SCOUTFS_IOC_WALK_INODES_META_SEQ; + else if (!strcasecmp(argv[1], "data_seq")) + walk.index = SCOUTFS_IOC_WALK_INODES_DATA_SEQ; + else { + fprintf(stderr, "unknown index '%s', try 'meta_seq' or " + "'data_seq'\n", argv[1]); + return -EINVAL; + } + + ret = parse_walk_entry(&walk.first, argv[2]); + if (ret) { + fprintf(stderr, "invalid first position '%s', try '1.2.3' or " + "'-1'\n", argv[2]); + return -EINVAL; + + } + + ret = parse_walk_entry(&walk.last, argv[3]); + if (ret) { + fprintf(stderr, "invalid last position '%s', try '1.2.3' or " + "'-1'\n", argv[3]); + return -EINVAL; + + } + + fd = open(argv[4], O_RDONLY); + if (fd < 0) { + ret = -errno; + fprintf(stderr, "failed to open '%s': %s (%d)\n", + argv[4], strerror(errno), errno); + return ret; + } + + walk.entries_ptr = (unsigned long)ents; + walk.nr_entries = array_size(ents); + + for (;;) { + ret = ioctl(fd, SCOUTFS_IOC_WALK_INODES, &walk); + if (ret < 0) { + ret = -errno; + fprintf(stderr, "walk_inodes ioctl failed: %s (%d)\n", + strerror(errno), errno); + break; + } else if (ret == 0) { + break; + } + + for (i = 0; i < ret; i++) { + if ((total + i) % 25 == 0) + printf("%-20s %-20s %-10s %-20s\n", + "#", "major", "minor", "ino"); + + printf("%-20llu %-20llu %-10u %-20llu\n", + total + i, ents[i].major, ents[i].minor, + ents[i].ino); + } + + total += i; + + walk.first = ents[i - 1]; + if (++walk.first.ino == 0 && ++walk.first.minor == 0) + walk.first.major++; + } + + close(fd); + return ret; +}; + +static void __attribute__((constructor)) walk_inodes_ctor(void) +{ + cmd_register("walk-inodes", " ", + "print range of indexed inodes", walk_inodes_cmd); +} diff --git a/utils/tex/.gitignore b/utils/tex/.gitignore new file mode 100644 index 00000000..853cdad5 --- /dev/null +++ b/utils/tex/.gitignore @@ -0,0 +1,8 @@ +missfont.log +*.fls +*.aux +*.d +*.d +*.fdb_latexmk +*.log +*.pdf diff --git a/utils/tex/Makefile b/utils/tex/Makefile new file mode 100644 index 00000000..35c59470 --- /dev/null +++ b/utils/tex/Makefile @@ -0,0 +1,33 @@ +# +# # dnf install latexmk texlive +# # make +# +# Tools +LATEXMK = latexmk +RM = rm -f + +# Project-specific settings +DOCNAME = scoutfs + +# Targets +all: doc +doc: pdf +pdf: $(DOCNAME).pdf + +# Rules +%.pdf: %.tex + $(LATEXMK) -pdf -M -MP -MF $*.d $* + +mostlyclean: + $(LATEXMK) -silent -c + $(RM) *.bbl + +clean: mostlyclean + $(LATEXMK) -silent -C + $(RM) *.run.xml *.synctex.gz + $(RM) *.d + +.PHONY: all clean doc mostlyclean pdf + +# Include auto-generated dependencies +-include *.d diff --git a/utils/tex/scoutfs.tex b/utils/tex/scoutfs.tex new file mode 100644 index 00000000..2c173be6 --- /dev/null +++ b/utils/tex/scoutfs.tex @@ -0,0 +1,221 @@ +% This was derived from the usenix templates, whose introductory +% comment is as follows: +% +% TEMPLATE for Usenix papers, specifically to meet requirements of +% USENIX '05 +% originally a template for producing IEEE-format articles using LaTeX. +% written by Matthew Ward, CS Department, Worcester Polytechnic Institute. +% adapted by David Beazley for his excellent SWIG paper in Proceedings, +% Tcl 96 +% turned into a smartass generic template by De Clarke, with thanks to +% both the above pioneers +% use at your own risk. Complaints to /dev/null. +% make it two column with no page numbering, default is 10 point + +% Munged by Fred Douglis 10/97 to separate +% the .sty file from the LaTeX source template, so that people can +% more easily include the .sty file into an existing document. Also +% changed to more closely follow the style guidelines as represented +% by the Word sample file. + +% Note that since 2010, USENIX does not require endnotes. If you want +% foot of page notes, don't include the endnotes package in the +% usepackage command, below. + +% This version uses the latex2e styles, not the very ancient 2.09 stuff. +\documentclass[letterpaper,twocolumn,10pt]{article} +\usepackage{usenix2019,epsfig} +\begin{document} + +%don't want date printed +\date{} + +%make title bold and 14 pt font (Latex default is non-bold, 16 pt) +\title{\Large \bf scoutfs : A Scalable Archival Filesystem} + +%for single author (just remove % characters) +\author{ +{\rm Zach Brown}\\ +Versity Software, Inc. +} + +\maketitle + +% Use the following at camera-ready time to suppress page numbers. +% Comment it out when you first submit the paper for review. +% \thispagestyle{empty} + +\section{Metadata Items} + +scoutfs stores filesystem metadata in items that are identified by a +key and contain a variable length value payload.\\ + +Every key uses a generic structure with a fixed number of fields. + +{\tt \small +\begin{verbatim} +struct scoutfs_key { + __u8 sk_zone; + __le64 _sk_first; + __u8 sk_type; + __le64 _sk_second; + __le64 _sk_third; + __u8 _sk_fourth; +}; +\end{verbatim} +} + +Using a shared key struct lets us sort all the metadata items in the +filesystem in one key space regardless of their form or function. The +generic keys are displayed, sorted, and computed (incrementing, finding +difference) without needing to know the specific fields of each item +type. + +Different structures are identified by their zone and type pair. They +then map their type's fields to the remaining generic fields to +determine the sorting of the item keys within their type. + +For example, when storing inodes we use the {\tt SCOUTFS\_FS\_ZONE} and +{\tt SCOUTFS\_INODE\_TYPE} and put the inode number in the first generic +key field. + +{\tt \small +\begin{verbatim} + #define ski_ino _sk_first +\end{verbatim} +} + +{\tt \small +\begin{verbatim} + key.sk_zone = SCOUTFS_FS_ZONE; + key.ski_ino = ino; + key.sk_type = SCOUTFS_INODE_TYPE; +\end{verbatim} +} + +Continuing this example, metadata that is associated with inodes also +use the {\tt SCOUTFS\_FS\_ZONE} and store the inode number in {\tt +\_sk\_first} but then have different type values. For example {\tt +SCOUTFS\_XATTR\_TYPE} or {\tt SCOUTFS\_SYMLINK\_TYPE}. When the items' +keys are sorted we end up with all the items for a given inode stored +near each other. + +\subsection{Directory Entries} + +A directory entry is stored in three different metadata items, each with +a different key and used for a different purpose. Each item shares the +same key format and directory entry value payload, however. + +The key stores the entry's directory inode number and major and minor +values associated with the type of directory entry being stored. + +{\tt \small +\begin{verbatim} + #define skd_ino _sk_first + #define skd_major _sk_second + #define skd_minor _sk_third +\end{verbatim} +} + +The value contains a directory entry struct with all the metadata +associated with a directory entry, including the full entry name. + +{\tt \small +\begin{verbatim} +struct scoutfs_dirent { + __le64 ino; + __le64 hash; + __le64 pos; + __u8 type; + __u8 name[0]; +}; +\end{verbatim} +} + +Each item contains a full copy of the item value. This duplicates +storage across each item type but also lets each operation be satisfied +by one item lookup. Once the item value is obtained its fields can be +used to construct the keys for each of the items associated with the +entry. + +\subsubsection{Directory Entry Lookup Items} + +{\tt \small +\begin{verbatim} + key.sk_zone = SCOUTFS_FS_ZONE; + key.skd_ino = dir_ino; + key.sk_type = SCOUTFS_DIRENT_TYPE; + key.skd_major = hash(entry_name); + key.skd_minor = dir_pos; +\end{verbatim} +} + +Lookup entries are stored in the parent directory at the hash of the +name of the entry. These entries are used to map names to inode numbers +during path traversal. + +The major key value is set to a 64bit hash of the file name. These hash +values can collide so the minor key value is set to the readdir position +in the directory of the entry. This readdir position is unique for +every entry and ensures that keys are unique when hash values collide. + +A name lookup is performed by iterating over all the keys with the major +that matches the hashed name. The full name in the dirent value struct +is compared to the search name. It will be very rare to have more than +one item with a given hash value. + +\subsubsection{Directory Entry Readdir Items} + +{\tt \small +\begin{verbatim} + key.sk_zone = SCOUTFS_FS_ZONE; + key.skd_ino = dir_ino; + key.sk_type = SCOUTFS_READDIR_TYPE; + key.skd_major = dir_pos; + key.skd_minor = 0; +\end{verbatim} +} + +Readdir entries are used to iterate over entries for the readdir() +call. By providing a unique 64bit {\tt dir\_pos} for each entry we avoid +having to track multiple entries for a given readdir position value. + +readdir() returns entries in {\tt dir\_pos} order which depends on entry +creation order and matches inode allocation order. Accessing the inodes +that are referenced by the entries returned from readdir() will result +in efficient forward iteration over the readdir and inode items, +assuming that files were simply created. + +Renaming files or creating hard links to existing files creates a new +entry but can't reassign the inode number and can result in mismatched +access patterns of the readdir entry items and the inode items. + +\subsubsection{Directory Entry Link Backref Items} + +{\tt \small +\begin{verbatim} + key.sk_zone = SCOUTFS_FS_ZONE; + key.skd_ino = target_ino; + key.sk_type = SCOUTFS_LINK_BACKREF_TYPE; + key.skd_major = dir_ino; + key.skd_minor = dir_pos; +\end{verbatim} +} + +Link backref entry items are stored with the target inode number and the +inode number and readdir position of the entry in its directory. +They're used to iterate over all the entries that refer to a given +inode. Full relative paths from the root directory to a target inode +can be constructed by walking up through each parent entry as its +discovered. + +Both inode numbers and readdir positions are allocated by strictly +increasing the next free number. Old inode numbers or readdir positions +are never reused. This means that resolving paths for existing inodes +will always walk keys that are strictly sorted less than the keys that +will be created as new files are created. This tends to isolate read +access patterns during backround archival policy processing from write +access patterns during new file creation and increases performance by +reducing contention. + +\end{document} diff --git a/utils/tex/usenix2019.sty b/utils/tex/usenix2019.sty new file mode 100644 index 00000000..dbab970c --- /dev/null +++ b/utils/tex/usenix2019.sty @@ -0,0 +1,97 @@ +% usenix.sty - to be used with latex2e for USENIX. +% To use this style file, look at the template usenix_template.tex +% +% $Id: usenix.sty,v 1.2 2005/02/16 22:30:47 maniatis Exp $ +% +% The following definitions are modifications of standard article.sty +% definitions, arranged to do a better job of matching the USENIX +% guidelines. +% It will automatically select two-column mode and the Times-Roman +% font. + +% +% USENIX papers are two-column. +% Times-Roman font is nice if you can get it (requires NFSS, +% which is in latex2e. + +\if@twocolumn\else\input twocolumn.sty\fi +\usepackage{mathptmx} % times roman, including math (where possible) + +% +% USENIX wants margins of: 0.75" sides, 1" bottom, and 1" top. +% 0.33" gutter between columns. +% Gives active areas of 7" x 9" +% +\setlength{\textheight}{9.0in} +\setlength{\columnsep}{0.33in} +\setlength{\textwidth}{7.00in} + +\setlength{\topmargin}{0.0in} + +\setlength{\headheight}{0.0in} + +\setlength{\headsep}{0.0in} + +\addtolength{\oddsidemargin}{-0.25in} +\addtolength{\evensidemargin}{-0.25in} + +% Usenix wants no page numbers for camera-ready papers, so that they can +% number them themselves. But submitted papers should have page numbers +% for the reviewers' convenience. +% +% +% \pagestyle{empty} + +% +% Usenix titles are in 14-point bold type, with no date, and with no +% change in the empty page headers. The whole author section is 12 point +% italic--- you must use {\rm } around the actual author names to get +% them in roman. +% +\def\maketitle{\par + \begingroup + \renewcommand\thefootnote{\fnsymbol{footnote}}% + \def\@makefnmark{\hbox to\z@{$\m@th^{\@thefnmark}$\hss}}% + \long\def\@makefntext##1{\parindent 1em\noindent + \hbox to1.8em{\hss$\m@th^{\@thefnmark}$}##1}% + \if@twocolumn + \twocolumn[\@maketitle]% + \else \newpage + \global\@topnum\z@ + \@maketitle \fi\@thanks + \endgroup + \setcounter{footnote}{0}% + \let\maketitle\relax + \let\@maketitle\relax + \gdef\@thanks{}\gdef\@author{}\gdef\@title{}\let\thanks\relax} + +\def\@maketitle{\newpage + \vbox to 2.5in{ + \vspace*{\fill} + \vskip 2em + \begin{center}% + {\Large\bf \@title \par}% + \vskip 0.375in minus 0.300in + {\large\it + \lineskip .5em + \begin{tabular}[t]{c}\@author + \end{tabular}\par}% + \end{center}% + \par + \vspace*{\fill} +% \vskip 1.5em + } +} + +% +% The abstract is preceded by a 12-pt bold centered heading +\def\abstract{\begin{center}% +{\large\bf \abstractname\vspace{-.5em}\vspace{\z@}}% +\end{center}} +\def\endabstract{} + +% +% Main section titles are 12-pt bold. Others can be same or smaller. +% +\def\section{\@startsection {section}{1}{\z@}{-3.5ex plus-1ex minus + -.2ex}{2.3ex plus.2ex}{\reset@font\large\bf}} diff --git a/utils/tex/usenix2019.tex b/utils/tex/usenix2019.tex new file mode 100644 index 00000000..b712783f --- /dev/null +++ b/utils/tex/usenix2019.tex @@ -0,0 +1,219 @@ +% TEMPLATE for Usenix papers, specifically to meet requirements of +% USENIX '05 +% originally a template for producing IEEE-format articles using LaTeX. +% written by Matthew Ward, CS Department, Worcester Polytechnic Institute. +% adapted by David Beazley for his excellent SWIG paper in Proceedings, +% Tcl 96 +% turned into a smartass generic template by De Clarke, with thanks to +% both the above pioneers +% use at your own risk. Complaints to /dev/null. +% make it two column with no page numbering, default is 10 point + +% Munged by Fred Douglis 10/97 to separate +% the .sty file from the LaTeX source template, so that people can +% more easily include the .sty file into an existing document. Also +% changed to more closely follow the style guidelines as represented +% by the Word sample file. + +% Note that since 2010, USENIX does not require endnotes. If you want +% foot of page notes, don't include the endnotes package in the +% usepackage command, below. + +% This version uses the latex2e styles, not the very ancient 2.09 stuff. +\documentclass[letterpaper,twocolumn,10pt]{article} +\usepackage{usenix2019,epsfig,endnotes} +\begin{document} + +%don't want date printed +\date{} + +%make title bold and 14 pt font (Latex default is non-bold, 16 pt) +\title{\Large \bf Wonderful : A Terrific Application and Fascinating Paper} + +%for single author (just remove % characters) +\author{ +{\rm Your N.\ Here}\\ +Your Institution +\and +{\rm Second Name}\\ +Second Institution +% copy the following lines to add more authors +% \and +% {\rm Name}\\ +%Name Institution +} % end author + +\maketitle + +% Use the following at camera-ready time to suppress page numbers. +% Comment it out when you first submit the paper for review. +\thispagestyle{empty} + + +\subsection*{Abstract} +Your Abstract Text Goes Here. Just a few facts. +Whet our appetites. + +\section{Introduction} + +A paragraph of text goes here. Lots of text. Plenty of interesting +text. \\ + +More fascinating text. Features\endnote{Remember to use endnotes, not footnotes!} galore, plethora of promises.\\ + +\section{This is Another Section} + +Some embedded literal typset code might +look like the following : + +{\tt \small +\begin{verbatim} +int wrap_fact(ClientData clientData, + Tcl_Interp *interp, + int argc, char *argv[]) { + int result; + int arg0; + if (argc != 2) { + interp->result = "wrong # args"; + return TCL_ERROR; + } + arg0 = atoi(argv[1]); + result = fact(arg0); + sprintf(interp->result,"%d",result); + return TCL_OK; +} +\end{verbatim} +} + +Now we're going to cite somebody. Watch for the cite tag. +Here it comes~\cite{Chaum1981,Diffie1976}. The tilde character (\~{}) +in the source means a non-breaking space. This way, your reference will +always be attached to the word that preceded it, instead of going to the +next line. + +\section{This Section has SubSections} +\subsection{First SubSection} + +Here's a typical figure reference. The figure is centered at the +top of the column. It's scaled. It's explicitly placed. You'll +have to tweak the numbers to get what you want.\\ + +% you can also use the wonderful epsfig package... +\begin{figure}[t] +\begin{center} +\begin{picture}(300,150)(0,200) +\put(-15,-30){\special{psfile = fig1.ps hscale = 50 vscale = 50}} +\end{picture}\\ +\end{center} +\caption{Wonderful Flowchart} +\end{figure} + +This text came after the figure, so we'll casually refer to Figure 1 +as we go on our merry way. + +\subsection{New Subsection} + +It can get tricky typesetting Tcl and C code in LaTeX because they share +a lot of mystical feelings about certain magic characters. You +will have to do a lot of escaping to typeset curly braces and percent +signs, for example, like this: +``The {\tt \%module} directive +sets the name of the initialization function. This is optional, but is +recommended if building a Tcl 7.5 module. +Everything inside the {\tt \%\{, \%\}} +block is copied directly into the output. allowing the inclusion of +header files and additional C code." \\ + +Sometimes you want to really call attention to a piece of text. You +can center it in the column like this: +\begin{center} +{\tt \_1008e614\_Vector\_p} +\end{center} +and people will really notice it.\\ + +\noindent +The noindent at the start of this paragraph makes it clear that it's +a continuation of the preceding text, not a new para in its own right. + + +Now this is an ingenious way to get a forced space. +{\tt Real~$*$} and {\tt double~$*$} are equivalent. + +Now here is another way to call attention to a line of code, but instead +of centering it, we noindent and bold it.\\ + +\noindent +{\bf \tt size\_t : fread ptr size nobj stream } \\ + +And here we have made an indented para like a definition tag (dt) +in HTML. You don't need a surrounding list macro pair. +\begin{itemize} +\item[] {\tt fread} reads from {\tt stream} into the array {\tt ptr} at +most {\tt nobj} objects of size {\tt size}. {\tt fread} returns +the number of objects read. +\end{itemize} +This concludes the definitions tag. + +\subsection{How to Build Your Paper} + +You have to run {\tt latex} once to prepare your references for +munging. Then run {\tt bibtex} to build your bibliography metadata. +Then run {\tt latex} twice to ensure all references have been resolved. +If your source file is called {\tt usenixTemplate.tex} and your {\tt + bibtex} file is called {\tt usenixTemplate.bib}, here's what you do: +{\tt \small +\begin{verbatim} +latex usenixTemplate +bibtex usenixTemplate +latex usenixTemplate +latex usenixTemplate +\end{verbatim} +} + + +\subsection{Last SubSection} + +Well, it's getting boring isn't it. This is the last subsection +before we wrap it up. + +\section{Acknowledgments} + +A polite author always includes acknowledgments. Thank everyone, +especially those who funded the work. + +\section{Availability} + +It's great when this section says that MyWonderfulApp is free software, +available via anonymous FTP from + +\begin{center} +{\tt ftp.site.dom/pub/myname/Wonderful}\\ +\end{center} + +Also, it's even greater when you can write that information is also +available on the Wonderful homepage at + +\begin{center} +{\tt http://www.site.dom/\~{}myname/SWIG} +\end{center} + +Now we get serious and fill in those references. Remember you will +have to run latex twice on the document in order to resolve those +cite tags you met earlier. This is where they get resolved. +We've preserved some real ones in addition to the template-speak. +After the bibliography you are DONE. + +{\footnotesize \bibliographystyle{acm} +\bibliography{../common/bibliography}} + + +\theendnotes + +\end{document} + + + + + + +