mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-09-20 23:14:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

generate vtproto marshalers for filer_pb and use them on the metadata log path (#10337)

Browse Source 
* generate vtproto marshalers for filer_pb and use them on the metadata log path

Reflection-based proto.Unmarshal allocates a fresh message tree through
reflect.New on every call. On the metadata subscription fan-out the same
event is decoded once per subscriber, so reflect.New tops the decode
churn under many mounts.

Generate MarshalVT/UnmarshalVT/SizeVT for filer.proto (a separate
filer_vtproto.pb.go, filer.pb.go untouched) and call them on the log
entry marshal and the subscribe/replay decode paths. UnmarshalVT
allocates message structs directly and copies byte and string fields, so
it stays wire-compatible with proto.Unmarshal and preserves the
non-aliasing the persisted-log cache depends on.

For SubscribeMetadataResponse this cuts decode allocations 69 -> 50 and
~4.5us -> ~2.1us per event; the win scales with subscriber overlap.

* marshal log entries directly into the buffer

SizeVT is allocation-free and MarshalToSizedBufferVT writes into a
pre-sized slice, so the log entry can be marshaled straight into
logBuffer.buf. This drops the per-entry MarshalVT allocation and the
follow-up copy on the write path.

* expand vtproto benchmarks: marshal, decode, and marshal-into-buffer by chunk count

Parametrize by nested-message count (chunks per event) and add encode +
zero-alloc marshal-into-buffer benchmarks alongside the decode one, so
the write-path win from MarshalToSizedBufferVT is measurable too.

* keep proto.Unmarshal for metadata events to preserve UTF-8 validation

UnmarshalVT skips proto3's UTF-8 validation of string fields, so a
SubscribeMetadataResponse with an invalid-UTF-8 string (e.g. Directory
"\xff") that proto.Unmarshal rejects would decode and reach path
filtering and subscribers. Decode events with proto.Unmarshal again;
UnmarshalVT stays on the log entry paths, whose only variable-length
fields are bytes and so carry no UTF-8 constraint.

Tests cover the codec difference and that a malformed event is skipped
before delivery.
This commit is contained in:
Chris Lu
2026-07-15 02:32:05 -07:00
committed by GitHub
parent 0ad83d5061
commit c015cc3939
11 changed files with 24628 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
weed/util/log_buffer/log_buffer.go
+22 -20
View File
@@ -9,8 +9,6 @@ import (
"sync/atomic"
"time"
"google.golang.org/protobuf/proto"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
@@ -302,13 +300,9 @@ func (logBuffer *LogBuffer) AddLogEntryToBuffer(logEntry *filer_pb.LogEntry) err
logBuffer.LastTsNs.Store(processingTsNs)
}
logEntryData, err := proto.Marshal(logEntry)
if err != nil {
marshalErr = fmt.Errorf("failed to marshal LogEntry: %w", err)
glog.Errorf("%v", marshalErr)
return marshalErr
}
size := len(logEntryData)
// Size is computed without allocating; the entry is marshaled straight into
// the buffer below via MarshalToSizedBufferVT.
size := logEntry.SizeVT()
if logBuffer.pos == 0 {
logBuffer.startTime = ts
@@ -352,10 +346,17 @@ func (logBuffer *LogBuffer) AddLogEntryToBuffer(logEntry *filer_pb.LogEntry) err
}
logBuffer.stopTime = ts
// Marshal directly into the buffer, avoiding an intermediate slice and copy.
// On the (practically impossible) error the entry is dropped before idx/pos
// are advanced, leaving the buffer consistent.
if _, err := logEntry.MarshalToSizedBufferVT(logBuffer.buf[logBuffer.pos+4 : logBuffer.pos+4+size]); err != nil {
marshalErr = fmt.Errorf("failed to marshal LogEntry: %w", err)
glog.Errorf("%v", marshalErr)
return marshalErr
}
logBuffer.idx = append(logBuffer.idx, logBuffer.pos)
util.Uint32toBytes(logBuffer.sizeBuf, uint32(size))
copy(logBuffer.buf[logBuffer.pos:logBuffer.pos+4], logBuffer.sizeBuf)
copy(logBuffer.buf[logBuffer.pos+4:logBuffer.pos+4+size], logEntryData)
logBuffer.pos += size + 4
logBuffer.offset++
@@ -416,15 +417,9 @@ func (logBuffer *LogBuffer) AddDataToBuffer(partitionKey, data []byte, processin
// Note: This also enables AddToBuffer to work correctly with Kafka-style offset-based reads
logEntry.Offset = logBuffer.offset
// Marshal with correct timestamp and offset
logEntryData, err := proto.Marshal(logEntry)
if err != nil {
marshalErr = fmt.Errorf("failed to marshal LogEntry: %w", err)
glog.Errorf("%v", marshalErr)
return marshalErr
}
size := len(logEntryData)
// Size is computed without allocating; the entry is marshaled straight into
// the buffer below via MarshalToSizedBufferVT.
size := logEntry.SizeVT()
if logBuffer.pos == 0 {
logBuffer.startTime = ts
@@ -466,10 +461,17 @@ func (logBuffer *LogBuffer) AddDataToBuffer(partitionKey, data []byte, processin
}
logBuffer.stopTime = ts
// Marshal directly into the buffer, avoiding an intermediate slice and copy.
// On the (practically impossible) error the entry is dropped before idx/pos
// are advanced, leaving the buffer consistent.
if _, err := logEntry.MarshalToSizedBufferVT(logBuffer.buf[logBuffer.pos+4 : logBuffer.pos+4+size]); err != nil {
marshalErr = fmt.Errorf("failed to marshal LogEntry: %w", err)
glog.Errorf("%v", marshalErr)
return marshalErr
}
logBuffer.idx = append(logBuffer.idx, logBuffer.pos)
util.Uint32toBytes(logBuffer.sizeBuf, uint32(size))
copy(logBuffer.buf[logBuffer.pos:logBuffer.pos+4], logBuffer.sizeBuf)
copy(logBuffer.buf[logBuffer.pos+4:logBuffer.pos+4+size], logEntryData)
logBuffer.pos += size + 4
logBuffer.offset++