mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-08-31 13:17:13 +00:00
Chris LuandGitHub
627b5e9d59
* worker: move the collection filter parser into weed/util/wildcard
The parser sits beside the volume-list filtering it was written for, in
weed/plugin/worker, which imports weed/shell — so the shell commands that
parse the same filter three other ways can never call it. Move it down to
weed/util/wildcard, next to the comma-separated wildcard helper it already
replaced, leaving the behavior unchanged.
* shell: parse every collection filter the same way
The shell parsed a collection filter three ways: compileCollectionPattern
compiled one regex for ec.encode, ec.decode, volume.balance and the tier
commands; volume.list and volume.deleteEmpty matched a single wildcard; and
volume.tier.move, volume.fix.replication and volume.configure.replication
called filepath.Match on their own. None of them took a list, so
"ec.encode -collection=a,b" selected nothing, the same way the admin UI did.
They all go through the shared matcher now: a comma-separated list of names,
"*" and "?" wildcards, "_default" for the collection with no name, and regex
entries. The one thing that stays per-command is what an empty value means -
every collection for -collectionPattern, the unnamed collection for the ec
and tier -collection flag - so compileCollectionPattern keeps that mapping.
The matchers are compiled once per command instead of once per volume, and a
regex entry now has to match the whole name unless it anchors itself, so
-collection=bucket no longer picks up mybucket2.
* shell: keep dots in collection names, and commas inside a regex
A dot no longer marks an entry as a regex, so a collection named "my.bucket"
matches itself and not "my-bucket" - the difference decides which volumes
volume.deleteEmpty and volume.tier.move touch. A dot still counts when it is
quantified, so "bucket.*" stays a prefix regex.
The comma split also leaves alone the commas inside a character class or a
repetition count, so "bucket[0-9]{1,3}" stays one entry instead of becoming
two broken fragments.
* shell: let a regex entry match its own spelling
A collection named after regex syntax, say "logs(2024)", was unreachable:
the entry compiled to a pattern that matches "logs2024" instead. Match the
entry verbatim as well, so naming a collection always selects it, whatever
characters it holds.
* shell: reject a collection filter that names no collection
A value of "," parsed to no entries and then matched every collection, so a
typo widened ec.encode or volume.deleteEmpty to the whole cluster. Only a
genuinely empty filter means "all collections"; anything else has to name one.
* shell: keep commas inside a regex group out of the entry split
The split already left alone the commas inside a character class or a
repetition count, but not the ones inside a group, so "bucket(foo,bar)"
was cut into two fragments that no longer compile.
* shell: cover escaping a collection name that is not a regex
A name like "logs(2024" does not parse as a regex on its own; escaping it,
"logs\(2024", reaches it. Pin that so the escape hatch does not regress.
* shell: split entries only on commas inside a closed regex construct
An unmatched "{" or "[" made the splitter swallow every comma after it, so
"foo{bar,videos" became one entry that matches neither collection - the
silent no-op this filter work exists to remove. A construct now has to close
before its commas stop separating entries.
* shell: skip character classes while scanning a regex group
A ")" inside a class is a literal, so "(a[)],b)" ended its group early and
split into two fragments that no longer compile.
* shell: cover escaping a comma inside a collection name
A comma separates entries, so a name holding one is reached by escaping it.
* shell: follow the regexp parser when scanning a character class
A "]" leading a class is a member of it, and a POSIX class such as
"[:alpha:]" carries its own "]", so stopping at the first one cut a valid
filter like "(a[]),],b)" into fragments and rejected it.
131 lines
4.0 KiB
Go
131 lines
4.0 KiB
Go
package pluginworker
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
|
|
)
|
|
|
|
func makeTestVolumeListResponse(volumes ...*master_pb.VolumeInformationMessage) *master_pb.VolumeListResponse {
|
|
return &master_pb.VolumeListResponse{
|
|
VolumeSizeLimitMb: 30000,
|
|
TopologyInfo: &master_pb.TopologyInfo{
|
|
DataCenterInfos: []*master_pb.DataCenterInfo{
|
|
{
|
|
Id: "dc1",
|
|
RackInfos: []*master_pb.RackInfo{
|
|
{
|
|
Id: "rack1",
|
|
DataNodeInfos: []*master_pb.DataNodeInfo{
|
|
{
|
|
Id: "server1:8080",
|
|
DiskInfos: map[string]*master_pb.DiskInfo{
|
|
"hdd": {
|
|
VolumeInfos: volumes,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestBuildVolumeMetricsEmptyFilter(t *testing.T) {
|
|
resp := makeTestVolumeListResponse(
|
|
&master_pb.VolumeInformationMessage{Id: 1, Collection: "photos", Size: 100},
|
|
&master_pb.VolumeInformationMessage{Id: 2, Collection: "videos", Size: 200},
|
|
)
|
|
metrics, _, _, err := buildVolumeMetrics(resp, "")
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if len(metrics) != 2 {
|
|
t.Fatalf("expected 2 metrics, got %d", len(metrics))
|
|
}
|
|
}
|
|
|
|
func TestBuildVolumeMetricsAllCollections(t *testing.T) {
|
|
resp := makeTestVolumeListResponse(
|
|
&master_pb.VolumeInformationMessage{Id: 1, Collection: "photos", Size: 100},
|
|
&master_pb.VolumeInformationMessage{Id: 2, Collection: "videos", Size: 200},
|
|
)
|
|
metrics, _, _, err := buildVolumeMetrics(resp, string(wildcard.CollectionFilterAll))
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if len(metrics) != 2 {
|
|
t.Fatalf("expected 2 metrics, got %d", len(metrics))
|
|
}
|
|
}
|
|
|
|
func TestBuildVolumeMetricsEachCollection(t *testing.T) {
|
|
resp := makeTestVolumeListResponse(
|
|
&master_pb.VolumeInformationMessage{Id: 1, Collection: "photos", Size: 100},
|
|
&master_pb.VolumeInformationMessage{Id: 2, Collection: "videos", Size: 200},
|
|
)
|
|
// EACH_COLLECTION passes all volumes through; filtering happens in the handler
|
|
metrics, _, _, err := buildVolumeMetrics(resp, string(wildcard.CollectionFilterEach))
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if len(metrics) != 2 {
|
|
t.Fatalf("expected 2 metrics, got %d", len(metrics))
|
|
}
|
|
}
|
|
|
|
func TestBuildVolumeMetricsRegexFilter(t *testing.T) {
|
|
resp := makeTestVolumeListResponse(
|
|
&master_pb.VolumeInformationMessage{Id: 1, Collection: "photos", Size: 100},
|
|
&master_pb.VolumeInformationMessage{Id: 2, Collection: "videos", Size: 200},
|
|
&master_pb.VolumeInformationMessage{Id: 3, Collection: "photos-backup", Size: 300},
|
|
)
|
|
metrics, _, _, err := buildVolumeMetrics(resp, "^photos$")
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if len(metrics) != 1 {
|
|
t.Fatalf("expected 1 metric, got %d", len(metrics))
|
|
}
|
|
if metrics[0].Collection != "photos" {
|
|
t.Fatalf("expected collection 'photos', got %q", metrics[0].Collection)
|
|
}
|
|
}
|
|
|
|
func TestBuildVolumeMetricsCollectionList(t *testing.T) {
|
|
resp := makeTestVolumeListResponse(
|
|
&master_pb.VolumeInformationMessage{Id: 1, Collection: "photos", Size: 100},
|
|
&master_pb.VolumeInformationMessage{Id: 2, Collection: "videos", Size: 200},
|
|
&master_pb.VolumeInformationMessage{Id: 3, Collection: "clips", Size: 300},
|
|
)
|
|
metrics, _, _, err := buildVolumeMetrics(resp, "photos,videos")
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if len(metrics) != 2 {
|
|
t.Fatalf("expected 2 metrics, got %d", len(metrics))
|
|
}
|
|
for _, metric := range metrics {
|
|
if metric.Collection != "photos" && metric.Collection != "videos" {
|
|
t.Fatalf("unexpected collection %q", metric.Collection)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBuildVolumeMetricsInvalidRegex(t *testing.T) {
|
|
resp := makeTestVolumeListResponse(
|
|
&master_pb.VolumeInformationMessage{Id: 1, Collection: "photos", Size: 100},
|
|
)
|
|
_, _, _, err := buildVolumeMetrics(resp, "[invalid")
|
|
if err == nil {
|
|
t.Fatal("expected error for invalid regex")
|
|
}
|
|
if !isConfigError(err) {
|
|
t.Fatalf("expected config error for invalid regex, got: %v", err)
|
|
}
|
|
}
|