Files
seaweedfs/weed/command/filer_meta_scan_test.go
T
Chris LuandGitHub 9d11278d95 filer: add filer.meta.scan to audit one directory's change history (#10645)
* filer: drain pending log chunk refs when the metadata stream ends

In metadata chunks mode the server sends log file refs in responses of their
own, and the client can only read them once it knows the run of refs is over.
That was inferred solely from the arrival of a normal event, so refs still
pending when the stream ended were dropped: the subscription returned no
events and no error.

A follower never noticed, because it runs forever and a live event always
arrives to close the run. A bounded subscription — StopTsNs set, range already
in the past — can receive nothing but refs and then EOF, and silently reports
that nothing happened. For anything auditing a path that is the worst possible
answer, since an empty result is indistinguishable from a quiet period.

Drain on EOF as well as at the transition point.

* filer: add filer.meta.scan to audit one directory's change history

Reconstructing what happened to a path means replaying the metadata log, and
filer.meta.tail is built for watching rather than auditing: it follows forever
unless given a stop, prints multi-line JSON, and takes ranges only as durations
before now, so an incident timestamp has to be converted by hand.

Its -pattern also cannot find a versioned object. A versioned key is stored as
<key>.versions/v_<id>, so the events carry the names "<key>.versions" and
"v_<id>" and a pattern of the object's own name matches neither — the search
comes back empty while the object is being written continuously.

filer.meta.scan prints one line per change, stops at the end of the range,
accepts absolute -since/-until with an explicit -tz, and reports versioned
writes against the object key with the version id alongside, so -name matches
the key a client would ask for. Delete markers are labelled as such rather than
appearing as zero-length writes, and pointer flips on the .versions container
are distinguished from writes of object data.

* filer.meta.scan: read persisted log chunks from the volume servers

Reading a range through the filer makes it decode every log entry in that
range and filter each one, so the cost lands on the filer and does not shrink
when the prefix is narrow — only the bytes on the wire do. On a cluster whose
metadata log is dense that is the expensive part of a scan, and it is charged
to the process least able to spare it.

Enable metadata chunks mode: the filer hands out log chunk ids and the scan
reads them from the volume servers itself. ReadLogFileRefs re-applies the same
path filter client-side, so the output is unchanged — verified identical to
the filer-read path over the same range, including after a restart drops the
in-memory buffer and the data must come off disk.

Direct read needs a route to the volume servers that the filer does not, so a
failure before anything has been printed retries through the filer; retrying
after partial output would duplicate lines. -directRead=false forces it.

* filer.meta.scan: confirm an empty direct-read result through the filer

An audit that returns nothing is read as "nothing happened here", so it is the
one answer that must not be produced by a bug. Direct read has more ways to
come back empty than the filer path does — it needs a route to the volume
servers, and it depends on the ref-drain contract holding.

When direct read yields no changes, re-run through the filer before reporting
it, and warn if the two disagree. Re-running is safe only because nothing was
printed; after partial output a replay would duplicate lines instead, so that
case reports the error rather than retrying.
2026-08-08 09:24:58 -07:00

178 lines
5.1 KiB
Go

package command
import (
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)
func TestLogicalPathResolvesVersionedLayout(t *testing.T) {
raw := false
scanRaw = &raw
tests := []struct {
name string
dir, entry string
wantPath string
wantVersion string
wantKind string
}{
{
// The reason this exists: a client asks for summary.xml, but every
// write lands on a v_<id> inside a sibling directory, so a filter on
// the stored name never matches what the client is looking for.
name: "version file reports the object key",
dir: "/buckets/b/rp/summary.xml.versions", entry: "v_abc123",
wantPath: "/buckets/b/rp/summary.xml", wantVersion: "abc123",
},
{
name: "versions directory reports the object key",
dir: "/buckets/b/rp", entry: "summary.xml.versions",
wantPath: "/buckets/b/rp/summary.xml", wantKind: "versions-container",
},
{
name: "unversioned object is unchanged",
dir: "/buckets/b/rp", entry: "summary.xml",
wantPath: "/buckets/b/rp/summary.xml",
},
{
// A v_ prefix only means a version inside a .versions directory.
name: "v_ prefix outside a versions directory is a normal name",
dir: "/buckets/b/rp", entry: "v_notaversion",
wantPath: "/buckets/b/rp/v_notaversion",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
path, version, kind := logicalPath(tt.dir, tt.entry)
if path != tt.wantPath || version != tt.wantVersion || kind != tt.wantKind {
t.Errorf("logicalPath(%q,%q) = (%q,%q,%q), want (%q,%q,%q)",
tt.dir, tt.entry, path, version, kind, tt.wantPath, tt.wantVersion, tt.wantKind)
}
})
}
}
func TestLogicalPathRawKeepsStoredPath(t *testing.T) {
raw := true
scanRaw = &raw
path, version, kind := logicalPath("/buckets/b/rp/summary.xml.versions", "v_abc123")
if path != "/buckets/b/rp/summary.xml.versions/v_abc123" || version != "" || kind != "" {
t.Errorf("raw mode must not rewrite the path, got (%q,%q,%q)", path, version, kind)
}
}
func TestToScanEventClassifiesOperations(t *testing.T) {
raw := false
scanRaw = &raw
entry := func(name string) *filer_pb.Entry {
return &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{FileSize: 11}}
}
tests := []struct {
name string
resp *filer_pb.SubscribeMetadataResponse
wantOp string
wantPath string
}{
{
name: "create",
resp: &filer_pb.SubscribeMetadataResponse{
EventNotification: &filer_pb.EventNotification{
NewParentPath: "/b/rp", NewEntry: entry("obj"),
},
},
wantOp: "CREATE", wantPath: "/b/rp/obj",
},
{
name: "delete",
resp: &filer_pb.SubscribeMetadataResponse{
Directory: "/b/rp",
EventNotification: &filer_pb.EventNotification{OldEntry: entry("obj")},
},
wantOp: "DELETE", wantPath: "/b/rp/obj",
},
{
name: "update in place",
resp: &filer_pb.SubscribeMetadataResponse{
Directory: "/b/rp",
EventNotification: &filer_pb.EventNotification{
OldEntry: entry("obj"), NewParentPath: "/b/rp", NewEntry: entry("obj"),
},
},
wantOp: "UPDATE", wantPath: "/b/rp/obj",
},
{
name: "rename",
resp: &filer_pb.SubscribeMetadataResponse{
Directory: "/b/rp",
EventNotification: &filer_pb.EventNotification{
OldEntry: entry("obj"), NewParentPath: "/b/rp2", NewEntry: entry("obj2"),
},
},
wantOp: "RENAME", wantPath: "/b/rp2/obj2",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := toScanEvent(tt.resp)
if got == nil {
t.Fatal("expected an event")
}
if got.op != tt.wantOp || got.path != tt.wantPath {
t.Errorf("got (%s,%s), want (%s,%s)", got.op, got.path, tt.wantOp, tt.wantPath)
}
})
}
}
// A retraction is a zero-length version carrying a flag; reporting it as a plain
// write would read as the opposite of what happened.
func TestDescribeEntryMarksDeleteMarkers(t *testing.T) {
marker := &filer_pb.Entry{
Name: "v_abc",
Attributes: &filer_pb.FuseAttributes{},
Extended: map[string][]byte{s3_constants.ExtDeleteMarkerKey: []byte("true")},
}
got := describeEntry(marker, "abc", "")
if want := "version=abc delete-marker"; got != want {
t.Errorf("describeEntry = %q, want %q", got, want)
}
}
func TestParseScanTimeHonoursTimezone(t *testing.T) {
saoPaulo, err := time.LoadLocation("America/Sao_Paulo")
if err != nil {
t.Skipf("tzdata unavailable: %v", err)
}
// The same wall-clock string is a different instant per zone, which is the
// difference between finding a window and missing it by the UTC offset.
inSP, err := parseScanTime("2026-08-03 01:16:42", saoPaulo)
if err != nil {
t.Fatal(err)
}
inUTC, err := parseScanTime("2026-08-03 01:16:42", time.UTC)
if err != nil {
t.Fatal(err)
}
if delta := inSP.Sub(inUTC); delta != 3*time.Hour {
t.Errorf("Sao Paulo should be 3h behind UTC, got %v", delta)
}
// An explicit zone in the string wins over loc.
explicit, err := parseScanTime("2026-08-03T01:16:42Z", saoPaulo)
if err != nil {
t.Fatal(err)
}
if !explicit.Equal(inUTC) {
t.Errorf("RFC3339 zone must win, got %s want %s", explicit, inUTC)
}
}