mirror of
https://github.com/versity/scoutfs-go.git
synced 2025-12-23 05:05:17 +00:00
query by metaseq and dataseq interface
This commit is contained in:
Ben McClelland
2
LICENSE
2
LICENSE
@@ -1,6 +1,6 @@
|
||||
BSD 3-Clause License
|
||||
|
||||
Copyright (c) 2018,
|
||||
Copyright (c) 2018 Versity Software, Inc.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
|
||||
82
examples/count-changes/main.go
Normal file
82
examples/count-changes/main.go
Normal file
@@ -0,0 +1,82 @@
|
||||
// Copyright (c) 2018 Versity Software, Inc.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-3-Clause license
|
||||
// that can be found in the LICENSE file in the root of the source
|
||||
// tree.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
scoutfs "github.com/versity/scoutfs-go"
|
||||
)
|
||||
|
||||
const (
|
||||
max32 = 0xffffffff
|
||||
max64 = 0xffffffffffffffff
|
||||
)
|
||||
|
||||
type server struct {
|
||||
update <-chan int
|
||||
lastcount int
|
||||
}
|
||||
|
||||
func queryPopulation(basedir string, update chan<- int) error {
|
||||
h, err := scoutfs.NewQuery(basedir,
|
||||
scoutfs.ByMSeq(scoutfs.InodesEntry{},
|
||||
scoutfs.InodesEntry{Major: max64, Minor: max32, Ino: max64}))
|
||||
if err != nil {
|
||||
return fmt.Errorf("scoutfs new handle: %v", err)
|
||||
}
|
||||
defer h.Close()
|
||||
|
||||
count := 0
|
||||
for {
|
||||
for {
|
||||
qents, err := h.Next()
|
||||
if err != nil {
|
||||
return fmt.Errorf("scoutfs next: %v", err)
|
||||
}
|
||||
if len(qents) == 0 {
|
||||
break
|
||||
}
|
||||
for _, qent := range qents {
|
||||
fmt.Printf("%#v\n", qent)
|
||||
count++
|
||||
}
|
||||
}
|
||||
|
||||
update <- count
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
select {
|
||||
case x, ok := <-s.update:
|
||||
if ok {
|
||||
s.lastcount = x
|
||||
} else {
|
||||
fmt.Println("Channel closed!")
|
||||
os.Exit(1)
|
||||
}
|
||||
default:
|
||||
}
|
||||
|
||||
fmt.Fprintf(w, "%s %d", `
|
||||
<HEAD><meta HTTP-EQUIV="REFRESH" content="1"></HEAD>
|
||||
<h1>SCOUTFS</h1></p>
|
||||
Files/Directories updated:`, s.lastcount)
|
||||
}
|
||||
|
||||
func main() {
|
||||
update := make(chan int, 1)
|
||||
s := &server{update: update}
|
||||
go queryPopulation(os.Args[1], update)
|
||||
log.Fatal(http.ListenAndServe(":8080", s))
|
||||
}
|
||||
178
scoutfs.go
Normal file
178
scoutfs.go
Normal file
@@ -0,0 +1,178 @@
|
||||
// Copyright (c) 2018 Versity Software, Inc.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-3-Clause license
|
||||
// that can be found in the LICENSE file in the root of the source
|
||||
// tree.
|
||||
|
||||
package scoutfs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"os"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
const (
|
||||
max64 = 0xffffffffffffffff
|
||||
max32 = 0xffffffff
|
||||
)
|
||||
|
||||
// Query to keep track of in-process query
|
||||
type Query struct {
|
||||
first InodesEntry
|
||||
last InodesEntry
|
||||
index uint8
|
||||
batch uint32
|
||||
fsfd *os.File
|
||||
}
|
||||
|
||||
// Time represents a time value in seconds and nanoseconds
|
||||
type Time struct {
|
||||
Sec uint64
|
||||
Nsec uint32
|
||||
}
|
||||
|
||||
// NewQuery creates a new scoutfs WalkHandle
|
||||
// Specify query type with By*() option
|
||||
// (only 1 allowed, last one wins)
|
||||
// and specify batching with WithBatchSize()
|
||||
func NewQuery(path string, opts ...Option) (*Query, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
q := &Query{
|
||||
//default batch size is 128
|
||||
batch: 128,
|
||||
fsfd: f,
|
||||
}
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(q)
|
||||
}
|
||||
|
||||
return q, nil
|
||||
}
|
||||
|
||||
// Option sets various options for NewWalkHandle
|
||||
type Option func(*Query)
|
||||
|
||||
// ByMSeq gets inodes in range of metadata sequence from, to inclusive
|
||||
func ByMSeq(from, to InodesEntry) Option {
|
||||
return func(q *Query) {
|
||||
q.first = from
|
||||
q.last = to
|
||||
q.index = QUERYINODESMETASEQ
|
||||
}
|
||||
}
|
||||
|
||||
// ByDSeq gets inodes in range of data sequence from, to inclusive
|
||||
func ByDSeq(from, to InodesEntry) Option {
|
||||
return func(q *Query) {
|
||||
q.first = from
|
||||
q.last = to
|
||||
q.index = QUERYINODESDATASEQ
|
||||
}
|
||||
}
|
||||
|
||||
// WithBatchSize sets the max number of inodes to be returned at a time
|
||||
func WithBatchSize(size uint32) Option {
|
||||
return func(q *Query) {
|
||||
q.batch = size
|
||||
}
|
||||
}
|
||||
|
||||
// Next gets the next batch of inodes
|
||||
func (q *Query) Next() ([]InodesEntry, error) {
|
||||
buf := make([]byte, int(unsafe.Sizeof(InodesEntry{}))*int(q.batch))
|
||||
query := queryInodes{
|
||||
first: q.first,
|
||||
last: q.last,
|
||||
entries: uintptr(unsafe.Pointer(&buf[0])),
|
||||
count: q.batch,
|
||||
index: q.index,
|
||||
}
|
||||
|
||||
pack, err := query.pack()
|
||||
if err != nil {
|
||||
return []InodesEntry{}, err
|
||||
}
|
||||
|
||||
n, err := scoutfsctl(q.fsfd.Fd(), IOCQUERYINODES, uintptr(unsafe.Pointer(&pack)))
|
||||
if err != nil {
|
||||
return []InodesEntry{}, err
|
||||
}
|
||||
|
||||
if n == 0 {
|
||||
return []InodesEntry{}, nil
|
||||
}
|
||||
|
||||
rbuf := bytes.NewReader(buf)
|
||||
var inodes []InodesEntry
|
||||
|
||||
var e InodesEntry
|
||||
for i := 0; i < n; i++ {
|
||||
//packed scoutfs_ioctl_walk_inodes_entry requires
|
||||
//unpacking each member individually
|
||||
err := binary.Read(rbuf, binary.LittleEndian, &e.Major)
|
||||
if err != nil {
|
||||
return []InodesEntry{}, err
|
||||
}
|
||||
err = binary.Read(rbuf, binary.LittleEndian, &e.Minor)
|
||||
if err != nil {
|
||||
return []InodesEntry{}, err
|
||||
}
|
||||
err = binary.Read(rbuf, binary.LittleEndian, &e.Ino)
|
||||
if err != nil {
|
||||
return []InodesEntry{}, err
|
||||
}
|
||||
|
||||
inodes = append(inodes, e)
|
||||
}
|
||||
|
||||
q.first = e
|
||||
q.first.Ino++
|
||||
q.first.Minor++
|
||||
if q.first.Ino == 0 {
|
||||
q.first.Minor++
|
||||
if q.first.Minor == 0 {
|
||||
q.first.Major++
|
||||
}
|
||||
}
|
||||
|
||||
return inodes, nil
|
||||
}
|
||||
|
||||
// Close queryHandle and cleanup
|
||||
func (q *Query) Close() {
|
||||
q.fsfd.Close()
|
||||
}
|
||||
|
||||
// StatMore returns scoutfs specific metadata for path
|
||||
func StatMore(path string) (Stat, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return Stat{}, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
var s Stat
|
||||
|
||||
_, err = scoutfsctl(f.Fd(), IOCSTATMORE, uintptr(unsafe.Pointer(&s)))
|
||||
if err != nil {
|
||||
return Stat{}, err
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func scoutfsctl(fd, cmd, ptr uintptr) (int, error) {
|
||||
count, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, cmd, ptr)
|
||||
if err != 0 {
|
||||
return 0, err
|
||||
}
|
||||
return int(count), nil
|
||||
}
|
||||
218
scoutfsdefs.go
Normal file
218
scoutfsdefs.go
Normal file
@@ -0,0 +1,218 @@
|
||||
// Copyright (c) 2018 Versity Software, Inc.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-3-Clause license
|
||||
// that can be found in the LICENSE file in the root of the source
|
||||
// tree.
|
||||
|
||||
package scoutfs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
const (
|
||||
//IOCQUERYINODES scoutfs ioctl
|
||||
IOCQUERYINODES = 0x40357301
|
||||
//IOCINOPATH scoutfs ioctl
|
||||
IOCINOPATH = 0x401c7302
|
||||
//IOCDATAVERSION scoutfs ioctl
|
||||
IOCDATAVERSION = 0x40087304
|
||||
//IOCRELEASE scoutfs ioctl
|
||||
IOCRELEASE = 0x40187305
|
||||
//IOCSTAGE scoutfs ioctl
|
||||
IOCSTAGE = 0x401c7306
|
||||
//IOCSTATMORE scoutfs ioctl
|
||||
IOCSTATMORE = 0x40207307
|
||||
|
||||
//QUERYINODESMETASEQ find inodes by metadata sequence
|
||||
QUERYINODESMETASEQ = '\u0000'
|
||||
//QUERYINODESDATASEQ find inodes by data sequence
|
||||
QUERYINODESDATASEQ = '\u0001'
|
||||
)
|
||||
|
||||
/* pahole for scoutfs_ioctl_walk_inodes_entry
|
||||
struct scoutfs_ioctl_walk_inodes_entry {
|
||||
__u64 major; // 0 8
|
||||
__u32 minor; // 8 4
|
||||
__u64 ino; // 12 8
|
||||
|
||||
// size: 20, cachelines: 1, members: 3
|
||||
// last cacheline: 20 bytes
|
||||
};
|
||||
*/
|
||||
|
||||
//InodesEntry is scoutfs entry for inode iteration
|
||||
type InodesEntry struct {
|
||||
Major uint64
|
||||
Minor uint32
|
||||
Ino uint64
|
||||
}
|
||||
|
||||
/* pahole for scoutfs_ioctl_walk_inodes
|
||||
struct scoutfs_ioctl_walk_inodes {
|
||||
struct scoutfs_ioctl_walk_inodes_entry first; // 0 20
|
||||
struct scoutfs_ioctl_walk_inodes_entry last; // 20 20
|
||||
__u64 entries_ptr; // 40 8
|
||||
__u32 nr_entries; // 48 4
|
||||
__u8 index; // 52 1
|
||||
|
||||
// size: 53, cachelines: 1, members: 5
|
||||
// last cacheline: 53 bytes
|
||||
};
|
||||
*/
|
||||
|
||||
//queryInodes is scoutfs request structure for IOCQUERYINODES
|
||||
type queryInodes struct {
|
||||
first InodesEntry
|
||||
last InodesEntry
|
||||
entries uintptr
|
||||
count uint32
|
||||
index uint8
|
||||
}
|
||||
|
||||
//packed scoutfs_ioctl_walk_inodes requires packing queryInodes byhand
|
||||
//the 53 size comes from pahole output above
|
||||
func (q queryInodes) pack() ([53]byte, error) {
|
||||
var pack [53]byte
|
||||
var pbuf = bytes.NewBuffer(make([]byte, 0, len(pack)))
|
||||
|
||||
if err := binary.Write(pbuf, binary.LittleEndian, q.first.Major); err != nil {
|
||||
return [53]byte{}, fmt.Errorf("pack first.Major: %v", err)
|
||||
}
|
||||
if err := binary.Write(pbuf, binary.LittleEndian, q.first.Minor); err != nil {
|
||||
return [53]byte{}, fmt.Errorf("pack first.Minor: %v", err)
|
||||
}
|
||||
if err := binary.Write(pbuf, binary.LittleEndian, q.first.Ino); err != nil {
|
||||
return [53]byte{}, fmt.Errorf("pack first.Ino: %v", err)
|
||||
}
|
||||
if err := binary.Write(pbuf, binary.LittleEndian, q.last.Major); err != nil {
|
||||
return [53]byte{}, fmt.Errorf("pack last.Major: %v", err)
|
||||
}
|
||||
if err := binary.Write(pbuf, binary.LittleEndian, q.last.Minor); err != nil {
|
||||
return [53]byte{}, fmt.Errorf("pack last.Minor: %v", err)
|
||||
}
|
||||
if err := binary.Write(pbuf, binary.LittleEndian, q.last.Ino); err != nil {
|
||||
return [53]byte{}, fmt.Errorf("pack last.Ino: %v", err)
|
||||
}
|
||||
if err := binary.Write(pbuf, binary.LittleEndian, uint64(q.entries)); err != nil {
|
||||
return [53]byte{}, fmt.Errorf("pack entries: %v", err)
|
||||
}
|
||||
if err := binary.Write(pbuf, binary.LittleEndian, q.count); err != nil {
|
||||
return [53]byte{}, fmt.Errorf("pack count: %v", err)
|
||||
}
|
||||
if err := binary.Write(pbuf, binary.LittleEndian, q.index); err != nil {
|
||||
return [53]byte{}, fmt.Errorf("pack index: %v", err)
|
||||
}
|
||||
|
||||
if err := binary.Read(pbuf, binary.LittleEndian, &pack); err != nil {
|
||||
return [53]byte{}, fmt.Errorf("read packed: %v", err)
|
||||
}
|
||||
|
||||
return pack, nil
|
||||
}
|
||||
|
||||
/* pahole for scoutfs_ioctl_ino_path
|
||||
struct scoutfs_ioctl_ino_path {
|
||||
__u64 ino; // 0 8
|
||||
__u64 dir_ino; // 8 8
|
||||
__u64 dir_pos; // 16 8
|
||||
__u64 result_ptr; // 24 8
|
||||
__u16 result_bytes; // 32 2
|
||||
|
||||
// size: 34, cachelines: 1, members: 5
|
||||
// last cacheline: 34 bytes
|
||||
};
|
||||
*/
|
||||
|
||||
// InoPath ioctl struct
|
||||
type InoPath struct {
|
||||
Ino uint64
|
||||
DirIno uint64
|
||||
DirPos uint64
|
||||
ResultPtr uint64
|
||||
ResultBytes uint64
|
||||
}
|
||||
|
||||
/* pahole for scoutfs_ioctl_ino_path_result
|
||||
struct scoutfs_ioctl_ino_path_result {
|
||||
__u64 dir_ino; // 0 8
|
||||
__u64 dir_pos; // 8 8
|
||||
__u16 path_bytes; // 16 2
|
||||
__u8 path[0]; // 18 0
|
||||
|
||||
// size: 18, cachelines: 1, members: 4
|
||||
// last cacheline: 18 bytes
|
||||
};
|
||||
*/
|
||||
|
||||
// InoPathResult ioctl struct
|
||||
type InoPathResult struct {
|
||||
DirIno uint64
|
||||
DirPos uint64
|
||||
PathBytes uint16
|
||||
ResultBytes uint64
|
||||
}
|
||||
|
||||
/* pahole for scoutfs_ioctl_release
|
||||
struct scoutfs_ioctl_release {
|
||||
__u64 block; // 0 8
|
||||
__u64 count; // 8 8
|
||||
__u64 data_version; // 16 8
|
||||
|
||||
// size: 24, cachelines: 1, members: 3
|
||||
// last cacheline: 24 bytes
|
||||
};
|
||||
*/
|
||||
|
||||
// IocRelease ioctl struct
|
||||
type IocRelease struct {
|
||||
Block uint64
|
||||
Count uint64
|
||||
DataVersion uint64
|
||||
}
|
||||
|
||||
/* pahole for scoutfs_ioctl_stage
|
||||
struct scoutfs_ioctl_stage {
|
||||
__u64 data_version; // 0 8
|
||||
__u64 buf_ptr; // 8 8
|
||||
__u64 offset; // 16 8
|
||||
__s32 count; // 24 4
|
||||
|
||||
// size: 28, cachelines: 1, members: 4
|
||||
// last cacheline: 28 bytes
|
||||
};
|
||||
*/
|
||||
|
||||
// IocStage ioctl struct
|
||||
type IocStage struct {
|
||||
DataVersion uint64
|
||||
BufPtr uint64
|
||||
Offset uint64
|
||||
count int32
|
||||
}
|
||||
|
||||
/* pahole for scoutfs_ioctl_stat_more
|
||||
struct scoutfs_ioctl_stat_more {
|
||||
__u64 valid_bytes; // 0 8
|
||||
__u64 meta_seq; // 8 8
|
||||
__u64 data_seq; // 16 8
|
||||
__u64 data_version; // 24 8
|
||||
__u64 online_blocks; // 32 8
|
||||
__u64 offline_blocks; // 40 8
|
||||
|
||||
// size: 48, cachelines: 1, members: 6
|
||||
// last cacheline: 48 bytes
|
||||
};
|
||||
*/
|
||||
|
||||
//Stat holds scoutfs specific per file metadata
|
||||
type Stat struct {
|
||||
ValidBytes uint64
|
||||
MetaSeq uint64
|
||||
DataSeq uint64
|
||||
DataVersion uint64
|
||||
OnlineBlocks uint64
|
||||
OfflineBlocks uint64
|
||||
}
|
||||
Reference in New Issue
Block a user