From 749430dcebf17dbd4dd6a942c28c5e995debaab0 Mon Sep 17 00:00:00 2001
From: Chris Lu <chrislusf@users.noreply.github.com>
Date: Thu, 23 Apr 2026 11:54:08 -0700
Subject: [PATCH] fix(filer.meta.tail): include extended metadata in
 Elasticsearch docs (#9200)

* fix(filer.meta.tail): include extended metadata in Elasticsearch docs

The -es sink flattened only the FUSE attributes, so xattrs (including S3
user metadata like X-Amz-Meta-*) never reached Elasticsearch. Add an
Extended field and convert map[string][]byte to map[string]string so the
values index as text; non-UTF-8 values fall back to base64.

Addresses #9190 follow-up.

* fix(filer.meta.tail): prefix base64-encoded extended values with "base64:"

Addresses review feedback: a plain UTF-8 xattr and a base64 fallback are otherwise indistinguishable to a consumer reading the ES doc.
---
 weed/command/filer_meta_tail_elastic.go | 44 +++++++++++++++++++------
 1 file changed, 34 insertions(+), 10 deletions(-)

diff --git a/weed/command/filer_meta_tail_elastic.go b/weed/command/filer_meta_tail_elastic.go
index 308520612..f11bc5098 100644
--- a/weed/command/filer_meta_tail_elastic.go
+++ b/weed/command/filer_meta_tail_elastic.go
@@ -4,7 +4,9 @@ package command
 
 import (
 	"context"
+	"encoding/base64"
 	"strings"
+	"unicode/utf8"
 
 	jsoniter "github.com/json-iterator/go"
 	elastic "github.com/olivere/elastic/v7"
@@ -13,16 +15,17 @@ import (
 )
 
 type EsDocument struct {
-	Dir         string `json:"dir,omitempty"`
-	Name        string `json:"name,omitempty"`
-	IsDirectory bool   `json:"isDir,omitempty"`
-	Size        uint64 `json:"size,omitempty"`
-	Uid         uint32 `json:"uid,omitempty"`
-	Gid         uint32 `json:"gid,omitempty"`
-	UserName    string `json:"userName,omitempty"`
-	Crtime      int64  `json:"crtime,omitempty"`
-	Mtime       int64  `json:"mtime,omitempty"`
-	Mime        string `json:"mime,omitempty"`
+	Dir         string            `json:"dir,omitempty"`
+	Name        string            `json:"name,omitempty"`
+	IsDirectory bool              `json:"isDir,omitempty"`
+	Size        uint64            `json:"size,omitempty"`
+	Uid         uint32            `json:"uid,omitempty"`
+	Gid         uint32            `json:"gid,omitempty"`
+	UserName    string            `json:"userName,omitempty"`
+	Crtime      int64             `json:"crtime,omitempty"`
+	Mtime       int64             `json:"mtime,omitempty"`
+	Mime        string            `json:"mime,omitempty"`
+	Extended    map[string]string `json:"extended,omitempty"`
 }
 
 func toEsEntry(event *filer_pb.EventNotification) (*EsDocument, string) {
@@ -40,10 +43,31 @@ func toEsEntry(event *filer_pb.EventNotification) (*EsDocument, string) {
 		Crtime:      entry.Attributes.Crtime,
 		Mtime:       entry.Attributes.Mtime,
 		Mime:        entry.Attributes.Mime,
+		Extended:    toExtendedStrings(entry.Extended),
 	}
 	return esEntry, id
 }
 
+// toExtendedStrings converts the xattr map (e.g. S3 user metadata like
+// X-Amz-Meta-*) to string values so Elasticsearch indexes them as text
+// instead of base64-encoding the raw bytes. Non-UTF-8 values are prefixed
+// with "base64:" so consumers can distinguish encoded bytes from plain
+// strings that happen to look like base64.
+func toExtendedStrings(extended map[string][]byte) map[string]string {
+	if len(extended) == 0 {
+		return nil
+	}
+	result := make(map[string]string, len(extended))
+	for k, v := range extended {
+		if utf8.Valid(v) {
+			result[k] = string(v)
+		} else {
+			result[k] = "base64:" + base64.StdEncoding.EncodeToString(v)
+		}
+	}
+	return result
+}
+
 func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) {
 	options := []elastic.ClientOptionFunc{}
 	options = append(options, elastic.SetURL(strings.Split(servers, ",")...))