mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-08-16 04:06:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(filer.meta.tail): include extended metadata in Elasticsearch docs (#9200)

Browse Source 
* fix(filer.meta.tail): include extended metadata in Elasticsearch docs

The -es sink flattened only the FUSE attributes, so xattrs (including S3
user metadata like X-Amz-Meta-*) never reached Elasticsearch. Add an
Extended field and convert map[string][]byte to map[string]string so the
values index as text; non-UTF-8 values fall back to base64.

Addresses #9190 follow-up.

* fix(filer.meta.tail): prefix base64-encoded extended values with "base64:"

Addresses review feedback: a plain UTF-8 xattr and a base64 fallback are otherwise indistinguishable to a consumer reading the ES doc.
This commit is contained in:
Chris Lu
2026-04-23 11:54:08 -07:00
committed by GitHub
parent 036191c78a
commit 749430dceb
1 changed files with 34 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
weed/command/filer_meta_tail_elastic.go
+34 -10
View File
@@ -4,7 +4,9 @@ package command
import (
"context"
"encoding/base64"
"strings"
"unicode/utf8"
jsoniter "github.com/json-iterator/go"
elastic "github.com/olivere/elastic/v7"
@@ -13,16 +15,17 @@ import (
)
type EsDocument struct {
Dir string `json:"dir,omitempty"`
Name string `json:"name,omitempty"`
IsDirectory bool `json:"isDir,omitempty"`
Size uint64 `json:"size,omitempty"`
Uid uint32 `json:"uid,omitempty"`
Gid uint32 `json:"gid,omitempty"`
UserName string `json:"userName,omitempty"`
Crtime int64 `json:"crtime,omitempty"`
Mtime int64 `json:"mtime,omitempty"`
Mime string `json:"mime,omitempty"`
Dir string `json:"dir,omitempty"`
Name string `json:"name,omitempty"`
IsDirectory bool `json:"isDir,omitempty"`
Size uint64 `json:"size,omitempty"`
Uid uint32 `json:"uid,omitempty"`
Gid uint32 `json:"gid,omitempty"`
UserName string `json:"userName,omitempty"`
Crtime int64 `json:"crtime,omitempty"`
Mtime int64 `json:"mtime,omitempty"`
Mime string `json:"mime,omitempty"`
Extended map[string]string `json:"extended,omitempty"`
}
func toEsEntry(event *filer_pb.EventNotification) (*EsDocument, string) {
@@ -40,10 +43,31 @@ func toEsEntry(event *filer_pb.EventNotification) (*EsDocument, string) {
Crtime: entry.Attributes.Crtime,
Mtime: entry.Attributes.Mtime,
Mime: entry.Attributes.Mime,
Extended: toExtendedStrings(entry.Extended),
}
return esEntry, id
}
// toExtendedStrings converts the xattr map (e.g. S3 user metadata like
// X-Amz-Meta-*) to string values so Elasticsearch indexes them as text
// instead of base64-encoding the raw bytes. Non-UTF-8 values are prefixed
// with "base64:" so consumers can distinguish encoded bytes from plain
// strings that happen to look like base64.
func toExtendedStrings(extended map[string][]byte) map[string]string {
if len(extended) == 0 {
return nil
}
result := make(map[string]string, len(extended))
for k, v := range extended {
if utf8.Valid(v) {
result[k] = string(v)
} else {
result[k] = "base64:" + base64.StdEncoding.EncodeToString(v)
}
}
return result
}
func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) {
options := []elastic.ClientOptionFunc{}
options = append(options, elastic.SetURL(strings.Split(servers, ",")...))