From 9980caea5dc9af6db978b0a1176deb1226ff2c7e Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Tue, 25 Aug 2020 14:40:15 -0700 Subject: [PATCH] update for changes in new scoutfs search xattrs --- c_defs_linux.go | 8 ++++--- scoutfs.go | 57 ++++++++++++++++++++++++++++++------------------- scoutfsdefs.go | 23 ++++++++++++-------- 3 files changed, 54 insertions(+), 34 deletions(-) diff --git a/c_defs_linux.go b/c_defs_linux.go index 748d1c5..1d9156a 100644 --- a/c_defs_linux.go +++ b/c_defs_linux.go @@ -48,7 +48,7 @@ package scoutfs // typedef struct scoutfs_ioctl_data_wait_err scoutfs_ioctl_data_wait_err_t; // typedef struct scoutfs_ioctl_setattr_more scoutfs_ioctl_setattr_more_t; // typedef struct scoutfs_ioctl_listxattr_hidden scoutfs_ioctl_listxattr_hidden_t; -// typedef struct scoutfs_ioctl_find_xattrs scoutfs_ioctl_find_xattrs_t; +// typedef struct scoutfs_ioctl_search_xattrs scoutfs_ioctl_search_xattrs_t; // typedef struct scoutfs_ioctl_statfs_more scoutfs_ioctl_statfs_more_t; import "C" @@ -60,7 +60,7 @@ const IOCSTATMORE = C.SCOUTFS_IOC_STAT_MORE const IOCDATAWAITING = C.SCOUTFS_IOC_DATA_WAITING const IOCSETATTRMORE = C.SCOUTFS_IOC_SETATTR_MORE const IOCLISTXATTRHIDDEN = C.SCOUTFS_IOC_LISTXATTR_HIDDEN -const IOCFINDXATTRS = C.SCOUTFS_IOC_FIND_XATTRS +const IOCSEARCHXATTRS = C.SCOUTFS_IOC_SEARCH_XATTRS const IOCSTATFSMORE = C.SCOUTFS_IOC_STATFS_MORE const IOCDATAWAITERR = C.SCOUTFS_IOC_DATA_WAIT_ERR @@ -71,6 +71,8 @@ const DATAWAITOPREAD = C.SCOUTFS_IOC_DWO_READ const DATAWAITOPWRITE = C.SCOUTFS_IOC_DWO_WRITE const DATAWAITOPCHANGESIZE = C.SCOUTFS_IOC_DWO_CHANGE_SIZE +const SEARCHXATTRSOFLAGEND = C.SCOUTFS_SEARCH_XATTRS_OFLAG_END + type InodesEntry C.scoutfs_ioctl_walk_inodes_entry_t type queryInodes C.scoutfs_ioctl_walk_inodes_t type inoPath C.scoutfs_ioctl_ino_path_t @@ -82,7 +84,7 @@ type dataWaiting C.scoutfs_ioctl_data_waiting_t type dataWaitErr C.scoutfs_ioctl_data_wait_err_t type setattrMore C.scoutfs_ioctl_setattr_more_t type listXattrHidden C.scoutfs_ioctl_listxattr_hidden_t -type findXattrs C.scoutfs_ioctl_find_xattrs_t +type searchXattrs C.scoutfs_ioctl_search_xattrs_t type statfsMore C.scoutfs_ioctl_statfs_more_t const sizeofstatfsMore = C.sizeof_scoutfs_ioctl_statfs_more_t diff --git a/scoutfs.go b/scoutfs.go index 1f3366d..25297bd 100644 --- a/scoutfs.go +++ b/scoutfs.go @@ -116,7 +116,7 @@ func (q *Query) Next() ([]InodesEntry, error) { } rbuf := bytes.NewReader(q.buf) - var inodes []InodesEntry + inodes := make([]InodesEntry, n) var e InodesEntry for i := 0; i < n; i++ { @@ -125,7 +125,7 @@ func (q *Query) Next() ([]InodesEntry, error) { return nil, err } - inodes = append(inodes, e) + inodes[i] = e } q.first = e @@ -375,7 +375,7 @@ func (w *Waiters) Next() ([]DataWaitingEntry, error) { } rbuf := bytes.NewReader(w.buf) - var inodes []DataWaitingEntry + inodes := make([]DataWaitingEntry, n) var e DataWaitingEntry for i := 0; i < n; i++ { @@ -384,7 +384,7 @@ func (w *Waiters) Next() ([]DataWaitingEntry, error) { return nil, err } - inodes = append(inodes, e) + inodes[i] = e } w.ino = inodes[n-1].Ino @@ -423,10 +423,11 @@ func SendDataWaitErr(dirfd *os.File, ino, version, offset, op, count uint64, err // XattrQuery to keep track of in-process xattr query type XattrQuery struct { next uint64 - batch uint32 + batch uint64 key string fsfd *os.File buf []byte + done bool } // NewXattrQuery creates a new scoutfs Xattr Query @@ -436,8 +437,10 @@ type XattrQuery struct { // (usually just the base mount point directory) func NewXattrQuery(f *os.File, key string, opts ...XOption) *XattrQuery { q := &XattrQuery{ - //default batch size is 128 - batch: 128, + // default batch size is 131072 for a nice round 1MB allocation. + // making this too small risks multiple calls into Next() wich + // has significant overhead per call. + batch: (128 * 1024), key: key, fsfd: f, } @@ -455,7 +458,7 @@ func NewXattrQuery(f *os.File, key string, opts ...XOption) *XattrQuery { type XOption func(*XattrQuery) // WithXBatchSize sets the max number of inodes to be returned at a time -func WithXBatchSize(size uint32) XOption { +func WithXBatchSize(size uint64) XOption { return func(q *XattrQuery) { q.batch = size } @@ -471,25 +474,33 @@ func WithXStartIno(ino uint64) XOption { // Next gets the next batch of inodes func (q *XattrQuery) Next() ([]uint64, error) { name := []byte(q.key) - query := findXattrs{ + query := searchXattrs{ Next_ino: q.next, Name_ptr: uint64(uintptr(unsafe.Pointer(&name[0]))), Inodes_ptr: uint64(uintptr(unsafe.Pointer(&q.buf[0]))), Name_bytes: uint16(len(name)), - Nr_inodes: uint16(q.batch), + Nr_inodes: q.batch, } - n, err := scoutfsctl(q.fsfd, IOCFINDXATTRS, unsafe.Pointer(&query)) + if q.done { + return nil, nil + } + + n, err := scoutfsctl(q.fsfd, IOCSEARCHXATTRS, unsafe.Pointer(&query)) if err != nil { return nil, err } + if query.Output_flags&SEARCHXATTRSOFLAGEND != 0 { + q.done = true + } + if n == 0 { return nil, nil } rbuf := bytes.NewReader(q.buf) - var inodes []uint64 + inodes := make([]uint64, n) var e uint64 for i := 0; i < n; i++ { @@ -498,7 +509,7 @@ func (q *XattrQuery) Next() ([]uint64, error) { return nil, err } - inodes = append(inodes, e) + inodes[i] = e } q.next = e @@ -559,31 +570,33 @@ func bufToStrings(b []byte) []string { // FSID contains the statfs more info for mounted scoutfs filesystem type FSID struct { - FSID uint64 - RandomID uint64 - ShortID string + FSID uint64 + RandomID uint64 + ShortID string + CommittedSeq uint64 } // GetIDs gets the statfs more filesystem and random id from file handle within // scoutfs filesystem func GetIDs(f *os.File) (FSID, error) { - stfs := statfsMore{Bytes: sizeofstatfsMore} + stfs := statfsMore{Valid_bytes: sizeofstatfsMore} _, err := scoutfsctl(f, IOCSTATFSMORE, unsafe.Pointer(&stfs)) if err != nil { return FSID{}, err } - if stfs.Bytes != sizeofstatfsMore { - return FSID{}, fmt.Errorf("unexpected return size: %v", stfs.Bytes) + if stfs.Valid_bytes != sizeofstatfsMore { + return FSID{}, fmt.Errorf("unexpected return size: %v", stfs.Valid_bytes) } short := fmt.Sprintf("f.%v.r.%v", fmt.Sprintf("%016x", stfs.Fsid)[:][:6], fmt.Sprintf("%016x", stfs.Rid)[:][:6]) return FSID{ - FSID: stfs.Fsid, - RandomID: stfs.Rid, - ShortID: short, + FSID: stfs.Fsid, + RandomID: stfs.Rid, + ShortID: short, + CommittedSeq: stfs.Committed_seq, }, nil } diff --git a/scoutfsdefs.go b/scoutfsdefs.go index 32973d1..1bf2279 100644 --- a/scoutfsdefs.go +++ b/scoutfsdefs.go @@ -11,8 +11,8 @@ const IOCSTATMORE = 0x80307305 const IOCDATAWAITING = 0x80287306 const IOCSETATTRMORE = 0x40287307 const IOCLISTXATTRHIDDEN = 0x80187308 -const IOCFINDXATTRS = 0x80207309 -const IOCSTATFSMORE = 0x8018730a +const IOCSEARCHXATTRS = 0x80387309 +const IOCSTATFSMORE = 0x8020730a const IOCDATAWAITERR = 0x8030730b const QUERYINODESMETASEQ = 0x0 @@ -22,6 +22,8 @@ const DATAWAITOPREAD = 0x1 const DATAWAITOPWRITE = 0x2 const DATAWAITOPCHANGESIZE = 0x4 +const SEARCHXATTRSOFLAGEND = 0x1 + type InodesEntry struct { Major uint64 Ino uint64 @@ -100,18 +102,21 @@ type listXattrHidden struct { Buf_bytes uint32 Hash_pos uint32 } -type findXattrs struct { +type searchXattrs struct { Next_ino uint64 + Last_ino uint64 Name_ptr uint64 Inodes_ptr uint64 + Output_flags uint64 + Nr_inodes uint64 Name_bytes uint16 - Nr_inodes uint16 - X_pad [4]uint8 + X_pad [6]uint8 } type statfsMore struct { - Bytes uint64 - Fsid uint64 - Rid uint64 + Valid_bytes uint64 + Fsid uint64 + Rid uint64 + Committed_seq uint64 } -const sizeofstatfsMore = 0x18 +const sizeofstatfsMore = 0x20