mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-08-01 04:46:33 +00:00
Chris LuandGitHub
749430dceb
* fix(filer.meta.tail): include extended metadata in Elasticsearch docs The -es sink flattened only the FUSE attributes, so xattrs (including S3 user metadata like X-Amz-Meta-*) never reached Elasticsearch. Add an Extended field and convert map[string][]byte to map[string]string so the values index as text; non-UTF-8 values fall back to base64. Addresses #9190 follow-up. * fix(filer.meta.tail): prefix base64-encoded extended values with "base64:" Addresses review feedback: a plain UTF-8 xattr and a base64 fallback are otherwise indistinguishable to a consumer reading the ES doc.
105 lines
3.5 KiB
Go
105 lines
3.5 KiB
Go
//go:build elastic
|
|
|
|
package command
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"strings"
|
|
"unicode/utf8"
|
|
|
|
jsoniter "github.com/json-iterator/go"
|
|
elastic "github.com/olivere/elastic/v7"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
)
|
|
|
|
type EsDocument struct {
|
|
Dir string `json:"dir,omitempty"`
|
|
Name string `json:"name,omitempty"`
|
|
IsDirectory bool `json:"isDir,omitempty"`
|
|
Size uint64 `json:"size,omitempty"`
|
|
Uid uint32 `json:"uid,omitempty"`
|
|
Gid uint32 `json:"gid,omitempty"`
|
|
UserName string `json:"userName,omitempty"`
|
|
Crtime int64 `json:"crtime,omitempty"`
|
|
Mtime int64 `json:"mtime,omitempty"`
|
|
Mime string `json:"mime,omitempty"`
|
|
Extended map[string]string `json:"extended,omitempty"`
|
|
}
|
|
|
|
func toEsEntry(event *filer_pb.EventNotification) (*EsDocument, string) {
|
|
entry := event.NewEntry
|
|
dir, name := event.NewParentPath, entry.Name
|
|
id := util.Md5String([]byte(util.NewFullPath(dir, name)))
|
|
esEntry := &EsDocument{
|
|
Dir: dir,
|
|
Name: name,
|
|
IsDirectory: entry.IsDirectory,
|
|
Size: entry.Attributes.FileSize,
|
|
Uid: entry.Attributes.Uid,
|
|
Gid: entry.Attributes.Gid,
|
|
UserName: entry.Attributes.UserName,
|
|
Crtime: entry.Attributes.Crtime,
|
|
Mtime: entry.Attributes.Mtime,
|
|
Mime: entry.Attributes.Mime,
|
|
Extended: toExtendedStrings(entry.Extended),
|
|
}
|
|
return esEntry, id
|
|
}
|
|
|
|
// toExtendedStrings converts the xattr map (e.g. S3 user metadata like
|
|
// X-Amz-Meta-*) to string values so Elasticsearch indexes them as text
|
|
// instead of base64-encoding the raw bytes. Non-UTF-8 values are prefixed
|
|
// with "base64:" so consumers can distinguish encoded bytes from plain
|
|
// strings that happen to look like base64.
|
|
func toExtendedStrings(extended map[string][]byte) map[string]string {
|
|
if len(extended) == 0 {
|
|
return nil
|
|
}
|
|
result := make(map[string]string, len(extended))
|
|
for k, v := range extended {
|
|
if utf8.Valid(v) {
|
|
result[k] = string(v)
|
|
} else {
|
|
result[k] = "base64:" + base64.StdEncoding.EncodeToString(v)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) {
|
|
options := []elastic.ClientOptionFunc{}
|
|
options = append(options, elastic.SetURL(strings.Split(servers, ",")...))
|
|
options = append(options, elastic.SetSniff(false))
|
|
options = append(options, elastic.SetHealthcheck(false))
|
|
client, err := elastic.NewClient(options...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return func(resp *filer_pb.SubscribeMetadataResponse) error {
|
|
event := resp.EventNotification
|
|
if event.OldEntry != nil &&
|
|
(event.NewEntry == nil || resp.Directory != event.NewParentPath || event.OldEntry.Name != event.NewEntry.Name) {
|
|
// delete or not update the same file
|
|
dir, name := resp.Directory, event.OldEntry.Name
|
|
id := util.Md5String([]byte(util.NewFullPath(dir, name)))
|
|
println("delete", id)
|
|
_, err := client.Delete().Index(esIndex).Id(id).Do(context.Background())
|
|
return err
|
|
}
|
|
if event.NewEntry != nil {
|
|
// add a new file or update the same file
|
|
esEntry, id := toEsEntry(event)
|
|
value, err := jsoniter.Marshal(esEntry)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
println(string(value))
|
|
_, err = client.Index().Index(esIndex).Id(id).BodyJson(string(value)).Do(context.Background())
|
|
return err
|
|
}
|
|
return nil
|
|
}, nil
|
|
}
|