mirrors/scoutfs-go
1
0
Fork 0
mirror of https://github.com/versity/scoutfs-go.git synced 2025-12-23 05:05:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

query by metaseq and dataseq interface

Browse Source
This commit is contained in:
Ben McClelland
2018-10-03 11:11:19 -07:00
parent 61f8ca8981
commit 145debbdf5
4 changed files with 479 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
LICENSE
View File

@@ -1,6 +1,6 @@
BSD 3-Clause License BSD 3-Clause License
Copyright (c) 2018, Copyright (c) 2018 Versity Software, Inc.
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without

82
examples/count-changes/main.go Normal file
View File

@@ -0,0 +1,82 @@
// Copyright (c) 2018 Versity Software, Inc.
//
// Use of this source code is governed by a BSD-3-Clause license
// that can be found in the LICENSE file in the root of the source
// tree.
package main
import (
"fmt"
"log"
"net/http"
"os"
"time"
scoutfs "github.com/versity/scoutfs-go"
)
const (
max32 = 0xffffffff
max64 = 0xffffffffffffffff
)
type server struct {
update <-chan int
lastcount int
}
func queryPopulation(basedir string, update chan<- int) error {
h, err := scoutfs.NewQuery(basedir,
scoutfs.ByMSeq(scoutfs.InodesEntry{},
scoutfs.InodesEntry{Major: max64, Minor: max32, Ino: max64}))
if err != nil {
return fmt.Errorf("scoutfs new handle: %v", err)
}
defer h.Close()
count := 0
for {
for {
qents, err := h.Next()
if err != nil {
return fmt.Errorf("scoutfs next: %v", err)
}
if len(qents) == 0 {
break
}
for _, qent := range qents {
fmt.Printf("%#v\n", qent)
count++
}
}
update <- count
time.Sleep(500 * time.Millisecond)
}
}
func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
select {
case x, ok := <-s.update:
if ok {
s.lastcount = x
} else {
fmt.Println("Channel closed!")
os.Exit(1)
}
default:
}
fmt.Fprintf(w, "%s %d", `
<HEAD><meta HTTP-EQUIV="REFRESH" content="1"></HEAD>
<h1>SCOUTFS</h1></p>
Files/Directories updated:`, s.lastcount)
}
func main() {
update := make(chan int, 1)
s := &server{update: update}
go queryPopulation(os.Args[1], update)
log.Fatal(http.ListenAndServe(":8080", s))
}

178
scoutfs.go Normal file
View File

@@ -0,0 +1,178 @@
// Copyright (c) 2018 Versity Software, Inc.
//
// Use of this source code is governed by a BSD-3-Clause license
// that can be found in the LICENSE file in the root of the source
// tree.
package scoutfs
import (
"bytes"
"encoding/binary"
"os"
"syscall"
"unsafe"
)
const (
max64 = 0xffffffffffffffff
max32 = 0xffffffff
)
// Query to keep track of in-process query
type Query struct {
first InodesEntry
last InodesEntry
index uint8
batch uint32
fsfd *os.File
}
// Time represents a time value in seconds and nanoseconds
type Time struct {
Sec uint64
Nsec uint32
}
// NewQuery creates a new scoutfs WalkHandle
// Specify query type with By*() option
// (only 1 allowed, last one wins)
// and specify batching with WithBatchSize()
func NewQuery(path string, opts ...Option) (*Query, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
q := &Query{
//default batch size is 128
batch: 128,
fsfd: f,
}
for _, opt := range opts {
opt(q)
}
return q, nil
}
// Option sets various options for NewWalkHandle
type Option func(*Query)
// ByMSeq gets inodes in range of metadata sequence from, to inclusive
func ByMSeq(from, to InodesEntry) Option {
return func(q *Query) {
q.first = from
q.last = to
q.index = QUERYINODESMETASEQ
}
}
// ByDSeq gets inodes in range of data sequence from, to inclusive
func ByDSeq(from, to InodesEntry) Option {
return func(q *Query) {
q.first = from
q.last = to
q.index = QUERYINODESDATASEQ
}
}
// WithBatchSize sets the max number of inodes to be returned at a time
func WithBatchSize(size uint32) Option {
return func(q *Query) {
q.batch = size
}
}
// Next gets the next batch of inodes
func (q *Query) Next() ([]InodesEntry, error) {
buf := make([]byte, int(unsafe.Sizeof(InodesEntry{}))*int(q.batch))
query := queryInodes{
first: q.first,
last: q.last,
entries: uintptr(unsafe.Pointer(&buf[0])),
count: q.batch,
index: q.index,
}
pack, err := query.pack()
if err != nil {
return []InodesEntry{}, err
}
n, err := scoutfsctl(q.fsfd.Fd(), IOCQUERYINODES, uintptr(unsafe.Pointer(&pack)))
if err != nil {
return []InodesEntry{}, err
}
if n == 0 {
return []InodesEntry{}, nil
}
rbuf := bytes.NewReader(buf)
var inodes []InodesEntry
var e InodesEntry
for i := 0; i < n; i++ {
//packed scoutfs_ioctl_walk_inodes_entry requires
//unpacking each member individually
err := binary.Read(rbuf, binary.LittleEndian, &e.Major)
if err != nil {
return []InodesEntry{}, err
}
err = binary.Read(rbuf, binary.LittleEndian, &e.Minor)
if err != nil {
return []InodesEntry{}, err
}
err = binary.Read(rbuf, binary.LittleEndian, &e.Ino)
if err != nil {
return []InodesEntry{}, err
}
inodes = append(inodes, e)
}
q.first = e
q.first.Ino++
q.first.Minor++
if q.first.Ino == 0 {
q.first.Minor++
if q.first.Minor == 0 {
q.first.Major++
}
}
return inodes, nil
}
// Close queryHandle and cleanup
func (q *Query) Close() {
q.fsfd.Close()
}
// StatMore returns scoutfs specific metadata for path
func StatMore(path string) (Stat, error) {
f, err := os.Open(path)
if err != nil {
return Stat{}, err
}
defer f.Close()
var s Stat
_, err = scoutfsctl(f.Fd(), IOCSTATMORE, uintptr(unsafe.Pointer(&s)))
if err != nil {
return Stat{}, err
}
return s, nil
}
func scoutfsctl(fd, cmd, ptr uintptr) (int, error) {
count, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, cmd, ptr)
if err != 0 {
return 0, err
}
return int(count), nil
}

218
scoutfsdefs.go Normal file
View File

@@ -0,0 +1,218 @@
// Copyright (c) 2018 Versity Software, Inc.
//
// Use of this source code is governed by a BSD-3-Clause license
// that can be found in the LICENSE file in the root of the source
// tree.
package scoutfs
import (
"bytes"
"encoding/binary"
"fmt"
)
const (
//IOCQUERYINODES scoutfs ioctl
IOCQUERYINODES = 0x40357301
//IOCINOPATH scoutfs ioctl
IOCINOPATH = 0x401c7302
//IOCDATAVERSION scoutfs ioctl
IOCDATAVERSION = 0x40087304
//IOCRELEASE scoutfs ioctl
IOCRELEASE = 0x40187305
//IOCSTAGE scoutfs ioctl
IOCSTAGE = 0x401c7306
//IOCSTATMORE scoutfs ioctl
IOCSTATMORE = 0x40207307
//QUERYINODESMETASEQ find inodes by metadata sequence
QUERYINODESMETASEQ = '\u0000'
//QUERYINODESDATASEQ find inodes by data sequence
QUERYINODESDATASEQ = '\u0001'
)
/* pahole for scoutfs_ioctl_walk_inodes_entry
struct scoutfs_ioctl_walk_inodes_entry {
__u64 major; // 0 8
__u32 minor; // 8 4
__u64 ino; // 12 8
// size: 20, cachelines: 1, members: 3
// last cacheline: 20 bytes
};
*/
//InodesEntry is scoutfs entry for inode iteration
type InodesEntry struct {
Major uint64
Minor uint32
Ino uint64
}
/* pahole for scoutfs_ioctl_walk_inodes
struct scoutfs_ioctl_walk_inodes {
struct scoutfs_ioctl_walk_inodes_entry first; // 0 20
struct scoutfs_ioctl_walk_inodes_entry last; // 20 20
__u64 entries_ptr; // 40 8
__u32 nr_entries; // 48 4
__u8 index; // 52 1
// size: 53, cachelines: 1, members: 5
// last cacheline: 53 bytes
};
*/
//queryInodes is scoutfs request structure for IOCQUERYINODES
type queryInodes struct {
first InodesEntry
last InodesEntry
entries uintptr
count uint32
index uint8
}
//packed scoutfs_ioctl_walk_inodes requires packing queryInodes byhand
//the 53 size comes from pahole output above
func (q queryInodes) pack() ([53]byte, error) {
var pack [53]byte
var pbuf = bytes.NewBuffer(make([]byte, 0, len(pack)))
if err := binary.Write(pbuf, binary.LittleEndian, q.first.Major); err != nil {
return [53]byte{}, fmt.Errorf("pack first.Major: %v", err)
}
if err := binary.Write(pbuf, binary.LittleEndian, q.first.Minor); err != nil {
return [53]byte{}, fmt.Errorf("pack first.Minor: %v", err)
}
if err := binary.Write(pbuf, binary.LittleEndian, q.first.Ino); err != nil {
return [53]byte{}, fmt.Errorf("pack first.Ino: %v", err)
}
if err := binary.Write(pbuf, binary.LittleEndian, q.last.Major); err != nil {
return [53]byte{}, fmt.Errorf("pack last.Major: %v", err)
}
if err := binary.Write(pbuf, binary.LittleEndian, q.last.Minor); err != nil {
return [53]byte{}, fmt.Errorf("pack last.Minor: %v", err)
}
if err := binary.Write(pbuf, binary.LittleEndian, q.last.Ino); err != nil {
return [53]byte{}, fmt.Errorf("pack last.Ino: %v", err)
}
if err := binary.Write(pbuf, binary.LittleEndian, uint64(q.entries)); err != nil {
return [53]byte{}, fmt.Errorf("pack entries: %v", err)
}
if err := binary.Write(pbuf, binary.LittleEndian, q.count); err != nil {
return [53]byte{}, fmt.Errorf("pack count: %v", err)
}
if err := binary.Write(pbuf, binary.LittleEndian, q.index); err != nil {
return [53]byte{}, fmt.Errorf("pack index: %v", err)
}
if err := binary.Read(pbuf, binary.LittleEndian, &pack); err != nil {
return [53]byte{}, fmt.Errorf("read packed: %v", err)
}
return pack, nil
}
/* pahole for scoutfs_ioctl_ino_path
struct scoutfs_ioctl_ino_path {
__u64 ino; // 0 8
__u64 dir_ino; // 8 8
__u64 dir_pos; // 16 8
__u64 result_ptr; // 24 8
__u16 result_bytes; // 32 2
// size: 34, cachelines: 1, members: 5
// last cacheline: 34 bytes
};
*/
// InoPath ioctl struct
type InoPath struct {
Ino uint64
DirIno uint64
DirPos uint64
ResultPtr uint64
ResultBytes uint64
}
/* pahole for scoutfs_ioctl_ino_path_result
struct scoutfs_ioctl_ino_path_result {
__u64 dir_ino; // 0 8
__u64 dir_pos; // 8 8
__u16 path_bytes; // 16 2
__u8 path[0]; // 18 0
// size: 18, cachelines: 1, members: 4
// last cacheline: 18 bytes
};
*/
// InoPathResult ioctl struct
type InoPathResult struct {
DirIno uint64
DirPos uint64
PathBytes uint16
ResultBytes uint64
}
/* pahole for scoutfs_ioctl_release
struct scoutfs_ioctl_release {
__u64 block; // 0 8
__u64 count; // 8 8
__u64 data_version; // 16 8
// size: 24, cachelines: 1, members: 3
// last cacheline: 24 bytes
};
*/
// IocRelease ioctl struct
type IocRelease struct {
Block uint64
Count uint64
DataVersion uint64
}
/* pahole for scoutfs_ioctl_stage
struct scoutfs_ioctl_stage {
__u64 data_version; // 0 8
__u64 buf_ptr; // 8 8
__u64 offset; // 16 8
__s32 count; // 24 4
// size: 28, cachelines: 1, members: 4
// last cacheline: 28 bytes
};
*/
// IocStage ioctl struct
type IocStage struct {
DataVersion uint64
BufPtr uint64
Offset uint64
count int32
}
/* pahole for scoutfs_ioctl_stat_more
struct scoutfs_ioctl_stat_more {
__u64 valid_bytes; // 0 8
__u64 meta_seq; // 8 8
__u64 data_seq; // 16 8
__u64 data_version; // 24 8
__u64 online_blocks; // 32 8
__u64 offline_blocks; // 40 8
// size: 48, cachelines: 1, members: 6
// last cacheline: 48 bytes
};
*/
//Stat holds scoutfs specific per file metadata
type Stat struct {
ValidBytes uint64
MetaSeq uint64
DataSeq uint64
DataVersion uint64
OnlineBlocks uint64
OfflineBlocks uint64
}