diff --git a/weed/command/fix.go b/weed/command/fix.go
index 7edbee27e..e92f67a47 100644
--- a/weed/command/fix.go
+++ b/weed/command/fix.go
@@ -37,14 +37,15 @@ var cmdFix = &Command{
}
var (
- fixVolumeCollection = cmdFix.Flag.String("collection", "", "an optional volume collection name, if specified only it will be processed")
- fixVolumeId = cmdFix.Flag.Int64("volumeId", 0, "an optional volume id, if not 0 (default) only it will be processed")
- fixIncludeDeleted = cmdFix.Flag.Bool("includeDeleted", true, "include deleted entries in the index file")
- fixIgnoreError = cmdFix.Flag.Bool("ignoreError", false, "an optional, if true will be processed despite errors")
- fixRemoteFile = cmdFix.Flag.Bool("remoteFile", false, "an optional, if true will not try to load the local .dat file, but only the remote file")
- fixGenerateEcx = cmdFix.Flag.Bool("ecx", false, "regenerate a lost EC index (.ecx) — and the .vif when missing — from the local .ec## shards (missing shards are reconstructed from parity when enough survive). Run with the volume server stopped.")
- fixEcDataShards = cmdFix.Flag.Int("ecDataShards", 0, "EC data shard count for -ecx (0 = read from .vif, otherwise default 10)")
- fixEcParityShards = cmdFix.Flag.Int("ecParityShards", 0, "EC parity shard count for -ecx (0 = read from .vif, infer from shard count, otherwise default 4)")
+ fixVolumeCollection = cmdFix.Flag.String("collection", "", "an optional volume collection name, if specified only it will be processed")
+ fixVolumeId = cmdFix.Flag.Int64("volumeId", 0, "an optional volume id, if not 0 (default) only it will be processed")
+ fixIncludeDeleted = cmdFix.Flag.Bool("includeDeleted", true, "include deleted entries in the index file")
+ fixIgnoreError = cmdFix.Flag.Bool("ignoreError", false, "an optional, if true will be processed despite errors")
+ fixRemoteFile = cmdFix.Flag.Bool("remoteFile", false, "an optional, if true will not try to load the local .dat file, but only the remote file")
+ fixGenerateEcx = cmdFix.Flag.Bool("ecx", false, "regenerate a lost EC index (.ecx) — and the .vif when missing — from the local .ec## shards (missing shards are reconstructed from parity when enough survive). Run with the volume server stopped.")
+ fixEcDataShards = cmdFix.Flag.Int("ecDataShards", 0, "EC data shard count for -ecx (0 = read from .vif, otherwise default 10)")
+ fixEcParityShards = cmdFix.Flag.Int("ecParityShards", 0, "EC parity shard count for -ecx (0 = read from .vif, infer from shard count, otherwise default 4)")
+ fixEcUnsafeIgnoreSidecar = cmdFix.Flag.Bool("ecUnsafeIgnoreSidecar", false, "for -ecx: proceed even when the EC bitrot checksum sidecar (.ecsum) is malformed or stale, instead of failing closed; the reconstructed shards are not verified against it")
)
type VolumeFileScanner4Fix struct {
@@ -381,7 +382,7 @@ func doFixEcxFromShards(basePath, baseFileName, collection string, volumeId int6
if !dataComplete {
ctx := &erasure_coding.ECContext{DataShards: dataShards, ParityShards: parityShards}
glog.Infof("volume %d: %d/%d shards present; reconstructing missing shards (%s) before index rebuild", volumeId, presentCount, dataShards+parityShards, ctx.String())
- if _, err := erasure_coding.RebuildEcFilesWithContext(base, ctx); err != nil {
+ if _, err := erasure_coding.RebuildEcFiles(base, ctx, *fixEcUnsafeIgnoreSidecar); err != nil {
fail(fmt.Errorf("volume %d: reconstruct missing shards from %d survivors: %w", volumeId, presentCount, err))
return
}
diff --git a/weed/command/fix_ecx_test.go b/weed/command/fix_ecx_test.go
index d47254d31..d99f82694 100644
--- a/weed/command/fix_ecx_test.go
+++ b/weed/command/fix_ecx_test.go
@@ -92,7 +92,7 @@ func buildAndEncodeTestEcVolume(t *testing.T, dir, baseName string) (base string
idxFile.Close()
nm.Close()
- if err := erasure_coding.WriteEcFiles(base); err != nil {
+ if _, err := erasure_coding.WriteEcFiles(base, erasure_coding.BackgroundECContext()); err != nil {
t.Fatalf("WriteEcFiles: %v", err)
}
if err := erasure_coding.WriteSortedFileFromIdx(base, ".ecx"); err != nil {
diff --git a/weed/command/volume.go b/weed/command/volume.go
index b7c9113f3..c2969c9c8 100644
--- a/weed/command/volume.go
+++ b/weed/command/volume.go
@@ -23,6 +23,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/server/constants"
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage"
+ "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/grace"
@@ -139,6 +140,9 @@ var (
volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
minFreeSpacePercent = cmdVolume.Flag.String("minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly (deprecated, use minFreeSpace instead).")
minFreeSpace = cmdVolume.Flag.String("minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.")
+
+ ecBitrotChecksum = cmdVolume.Flag.Bool("ec.bitrotChecksum", true, "write a bitrot checksum sidecar (.ecsum) when generating EC shards, enabling silent-corruption detection on scrub/rebuild")
+ ecBitrotBlockSizeMB = cmdVolume.Flag.Int("ec.bitrotBlockSizeMB", 16, "EC bitrot checksum block granularity in MiB; must be a power of two and at least 1")
)
func runVolume(cmd *Command, args []string) bool {
@@ -171,6 +175,26 @@ func runVolume(cmd *Command, args []string) bool {
*v.mastersString = *v.mserverString
}
+ // Apply EC bitrot checksum settings.
+ erasure_coding.BitrotProtectionEnabled = *ecBitrotChecksum
+ // Validate the block size before multiplying so an absurd MiB value cannot
+ // overflow int64 and slip a bogus size past the power-of-two check. The
+ // block size also becomes the per-shard scratch buffer the scrub/backfill
+ // path allocates, so the upper bound caps that allocation (64 MiB per
+ // concurrent scrub worker) and keeps a typo from taking the server down.
+ const maxBitrotBlockSizeMB = 64
+ if mb := *ecBitrotBlockSizeMB; mb >= 1 && mb <= maxBitrotBlockSizeMB {
+ if blockSize := int64(mb) * 1024 * 1024; blockSize&(blockSize-1) == 0 {
+ erasure_coding.BitrotBlockSize = blockSize
+ } else if mb != 16 {
+ glog.Warningf("ignoring invalid -ec.bitrotBlockSizeMB=%d (must be a power of two); using %d MiB",
+ mb, erasure_coding.BitrotBlockSize/(1024*1024))
+ }
+ } else if *ecBitrotBlockSizeMB != 16 {
+ glog.Warningf("ignoring out-of-range -ec.bitrotBlockSizeMB=%d (must be a power of two in [1, %d] MiB); using %d MiB",
+ *ecBitrotBlockSizeMB, maxBitrotBlockSizeMB, erasure_coding.BitrotBlockSize/(1024*1024))
+ }
+
minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent)
v.masters = pb.ServerAddresses(*v.mastersString).ToAddresses()
v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, minFreeSpaces)
diff --git a/weed/pb/volume_server.proto b/weed/pb/volume_server.proto
index c63177cb4..4f71016d4 100644
--- a/weed/pb/volume_server.proto
+++ b/weed/pb/volume_server.proto
@@ -428,6 +428,7 @@ message VolumeEcShardsGenerateResponse {
message VolumeEcShardsRebuildRequest {
uint32 volume_id = 1;
string collection = 2;
+ bool unsafe_ignore_sidecar = 3; // bypass the bitrot-sidecar fail-closed guard (operator override; distinct from ec.rebuild -force)
}
message VolumeEcShardsRebuildResponse {
repeated uint32 rebuilt_shard_ids = 1;
@@ -442,6 +443,7 @@ message VolumeEcShardsCopyRequest {
bool copy_ecj_file = 6;
bool copy_vif_file = 7;
uint32 disk_id = 8; // Target disk ID for storing EC shards
+ bool copy_ecsum_file = 9; // copy the bitrot checksum sidecar (.ecsum) when present; tolerant of a missing source (no-op), since this non-2PC path has no Prepare backstop
}
message VolumeEcShardsCopyResponse {
}
@@ -578,6 +580,30 @@ message EcShardConfig {
uint32 data_shards = 1; // Number of data shards (e.g., 10)
uint32 parity_shards = 2; // Number of parity shards (e.g., 4)
}
+
+// EcBitrotProtection is the entire content of a bitrot checksum sidecar
+// (.ecsum for the legacy generation, .ecsum.v for vacuum
+// generation N). On disk it is wrapped in a fixed header carrying a CRC32C
+// over this serialized payload (see weed/storage/erasure_coding/ec_bitrot.go).
+message EcBitrotProtection {
+ ChecksumAlgorithm algorithm = 1; // CRC32C (Castagnoli)
+ uint32 block_size = 2; // bytes per checksum block; default 16777216 (16 MiB), a power-of-two multiple of 1 MiB
+ uint32 generation = 3; // EC vacuum generation these checksums describe (0 = legacy/fresh); must match the sidecar filename version
+ EcShardConfig ec_shard_config = 4; // data/parity shard counts at encode time
+ repeated EcShardChecksums shards = 5; // one entry per shard id in the active layout
+ bytes encode_uuid = 6; // random per-encode identity, for stale-sidecar detection across in-place re-encodes
+}
+
+message EcShardChecksums {
+ uint32 shard_id = 1; // 0..MaxShardCount-1 (custom EC ratios go up to 32)
+ int64 covered_size = 2; // shard byte length these checksums cover (must equal the on-disk shard length)
+ bytes block_crc32c = 3; // packed little-endian uint32[] = ceil(covered_size/block_size) entries
+}
+
+enum ChecksumAlgorithm {
+ CHECKSUM_NONE = 0;
+ CHECKSUM_CRC32C = 1;
+}
message OldVersionVolumeInfo {
repeated RemoteFile files = 1;
uint32 version = 2;
@@ -655,6 +681,7 @@ enum VolumeScrubMode {
INDEX = 1;
FULL = 2;
LOCAL = 3;
+ CHECKSUM = 4; // EC only: verify each local shard's raw bytes against the bitrot checksum sidecar
}
message ScrubVolumeRequest {
diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go
index 4d2d37df5..c05774b06 100644
--- a/weed/pb/volume_server_pb/volume_server.pb.go
+++ b/weed/pb/volume_server_pb/volume_server.pb.go
@@ -22,13 +22,60 @@ const (
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
+type ChecksumAlgorithm int32
+
+const (
+ ChecksumAlgorithm_CHECKSUM_NONE ChecksumAlgorithm = 0
+ ChecksumAlgorithm_CHECKSUM_CRC32C ChecksumAlgorithm = 1
+)
+
+// Enum value maps for ChecksumAlgorithm.
+var (
+ ChecksumAlgorithm_name = map[int32]string{
+ 0: "CHECKSUM_NONE",
+ 1: "CHECKSUM_CRC32C",
+ }
+ ChecksumAlgorithm_value = map[string]int32{
+ "CHECKSUM_NONE": 0,
+ "CHECKSUM_CRC32C": 1,
+ }
+)
+
+func (x ChecksumAlgorithm) Enum() *ChecksumAlgorithm {
+ p := new(ChecksumAlgorithm)
+ *p = x
+ return p
+}
+
+func (x ChecksumAlgorithm) String() string {
+ return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (ChecksumAlgorithm) Descriptor() protoreflect.EnumDescriptor {
+ return file_volume_server_proto_enumTypes[0].Descriptor()
+}
+
+func (ChecksumAlgorithm) Type() protoreflect.EnumType {
+ return &file_volume_server_proto_enumTypes[0]
+}
+
+func (x ChecksumAlgorithm) Number() protoreflect.EnumNumber {
+ return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use ChecksumAlgorithm.Descriptor instead.
+func (ChecksumAlgorithm) EnumDescriptor() ([]byte, []int) {
+ return file_volume_server_proto_rawDescGZIP(), []int{0}
+}
+
type VolumeScrubMode int32
const (
- VolumeScrubMode_UNKNOWN VolumeScrubMode = 0
- VolumeScrubMode_INDEX VolumeScrubMode = 1
- VolumeScrubMode_FULL VolumeScrubMode = 2
- VolumeScrubMode_LOCAL VolumeScrubMode = 3
+ VolumeScrubMode_UNKNOWN VolumeScrubMode = 0
+ VolumeScrubMode_INDEX VolumeScrubMode = 1
+ VolumeScrubMode_FULL VolumeScrubMode = 2
+ VolumeScrubMode_LOCAL VolumeScrubMode = 3
+ VolumeScrubMode_CHECKSUM VolumeScrubMode = 4 // EC only: verify each local shard's raw bytes against the bitrot checksum sidecar
)
// Enum value maps for VolumeScrubMode.
@@ -38,12 +85,14 @@ var (
1: "INDEX",
2: "FULL",
3: "LOCAL",
+ 4: "CHECKSUM",
}
VolumeScrubMode_value = map[string]int32{
- "UNKNOWN": 0,
- "INDEX": 1,
- "FULL": 2,
- "LOCAL": 3,
+ "UNKNOWN": 0,
+ "INDEX": 1,
+ "FULL": 2,
+ "LOCAL": 3,
+ "CHECKSUM": 4,
}
)
@@ -58,11 +107,11 @@ func (x VolumeScrubMode) String() string {
}
func (VolumeScrubMode) Descriptor() protoreflect.EnumDescriptor {
- return file_volume_server_proto_enumTypes[0].Descriptor()
+ return file_volume_server_proto_enumTypes[1].Descriptor()
}
func (VolumeScrubMode) Type() protoreflect.EnumType {
- return &file_volume_server_proto_enumTypes[0]
+ return &file_volume_server_proto_enumTypes[1]
}
func (x VolumeScrubMode) Number() protoreflect.EnumNumber {
@@ -71,7 +120,7 @@ func (x VolumeScrubMode) Number() protoreflect.EnumNumber {
// Deprecated: Use VolumeScrubMode.Descriptor instead.
func (VolumeScrubMode) EnumDescriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{0}
+ return file_volume_server_proto_rawDescGZIP(), []int{1}
}
// Persistent state for volume servers.
@@ -3304,11 +3353,12 @@ func (*VolumeEcShardsGenerateResponse) Descriptor() ([]byte, []int) {
}
type VolumeEcShardsRebuildRequest struct {
- state protoimpl.MessageState `protogen:"open.v1"`
- VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"`
- Collection string `protobuf:"bytes,2,opt,name=collection,proto3" json:"collection,omitempty"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ state protoimpl.MessageState `protogen:"open.v1"`
+ VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"`
+ Collection string `protobuf:"bytes,2,opt,name=collection,proto3" json:"collection,omitempty"`
+ UnsafeIgnoreSidecar bool `protobuf:"varint,3,opt,name=unsafe_ignore_sidecar,json=unsafeIgnoreSidecar,proto3" json:"unsafe_ignore_sidecar,omitempty"` // bypass the bitrot-sidecar fail-closed guard (operator override; distinct from ec.rebuild -force)
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
}
func (x *VolumeEcShardsRebuildRequest) Reset() {
@@ -3355,6 +3405,13 @@ func (x *VolumeEcShardsRebuildRequest) GetCollection() string {
return ""
}
+func (x *VolumeEcShardsRebuildRequest) GetUnsafeIgnoreSidecar() bool {
+ if x != nil {
+ return x.UnsafeIgnoreSidecar
+ }
+ return false
+}
+
type VolumeEcShardsRebuildResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
RebuiltShardIds []uint32 `protobuf:"varint,1,rep,packed,name=rebuilt_shard_ids,json=rebuiltShardIds,proto3" json:"rebuilt_shard_ids,omitempty"`
@@ -3408,7 +3465,8 @@ type VolumeEcShardsCopyRequest struct {
SourceDataNode string `protobuf:"bytes,5,opt,name=source_data_node,json=sourceDataNode,proto3" json:"source_data_node,omitempty"`
CopyEcjFile bool `protobuf:"varint,6,opt,name=copy_ecj_file,json=copyEcjFile,proto3" json:"copy_ecj_file,omitempty"`
CopyVifFile bool `protobuf:"varint,7,opt,name=copy_vif_file,json=copyVifFile,proto3" json:"copy_vif_file,omitempty"`
- DiskId uint32 `protobuf:"varint,8,opt,name=disk_id,json=diskId,proto3" json:"disk_id,omitempty"` // Target disk ID for storing EC shards
+ DiskId uint32 `protobuf:"varint,8,opt,name=disk_id,json=diskId,proto3" json:"disk_id,omitempty"` // Target disk ID for storing EC shards
+ CopyEcsumFile bool `protobuf:"varint,9,opt,name=copy_ecsum_file,json=copyEcsumFile,proto3" json:"copy_ecsum_file,omitempty"` // copy the bitrot checksum sidecar (.ecsum) when present; tolerant of a missing source (no-op), since this non-2PC path has no Prepare backstop
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -3499,6 +3557,13 @@ func (x *VolumeEcShardsCopyRequest) GetDiskId() uint32 {
return 0
}
+func (x *VolumeEcShardsCopyRequest) GetCopyEcsumFile() bool {
+ if x != nil {
+ return x.CopyEcsumFile
+ }
+ return false
+}
+
type VolumeEcShardsCopyResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
unknownFields protoimpl.UnknownFields
@@ -4921,6 +4986,154 @@ func (x *EcShardConfig) GetParityShards() uint32 {
return 0
}
+// EcBitrotProtection is the entire content of a bitrot checksum sidecar
+// (.ecsum for the legacy generation, .ecsum.v for vacuum
+// generation N). On disk it is wrapped in a fixed header carrying a CRC32C
+// over this serialized payload (see weed/storage/erasure_coding/ec_bitrot.go).
+type EcBitrotProtection struct {
+ state protoimpl.MessageState `protogen:"open.v1"`
+ Algorithm ChecksumAlgorithm `protobuf:"varint,1,opt,name=algorithm,proto3,enum=volume_server_pb.ChecksumAlgorithm" json:"algorithm,omitempty"` // CRC32C (Castagnoli)
+ BlockSize uint32 `protobuf:"varint,2,opt,name=block_size,json=blockSize,proto3" json:"block_size,omitempty"` // bytes per checksum block; default 16777216 (16 MiB), a power-of-two multiple of 1 MiB
+ Generation uint32 `protobuf:"varint,3,opt,name=generation,proto3" json:"generation,omitempty"` // EC vacuum generation these checksums describe (0 = legacy/fresh); must match the sidecar filename version
+ EcShardConfig *EcShardConfig `protobuf:"bytes,4,opt,name=ec_shard_config,json=ecShardConfig,proto3" json:"ec_shard_config,omitempty"` // data/parity shard counts at encode time
+ Shards []*EcShardChecksums `protobuf:"bytes,5,rep,name=shards,proto3" json:"shards,omitempty"` // one entry per shard id in the active layout
+ EncodeUuid []byte `protobuf:"bytes,6,opt,name=encode_uuid,json=encodeUuid,proto3" json:"encode_uuid,omitempty"` // random per-encode identity, for stale-sidecar detection across in-place re-encodes
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
+}
+
+func (x *EcBitrotProtection) Reset() {
+ *x = EcBitrotProtection{}
+ mi := &file_volume_server_proto_msgTypes[86]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+}
+
+func (x *EcBitrotProtection) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*EcBitrotProtection) ProtoMessage() {}
+
+func (x *EcBitrotProtection) ProtoReflect() protoreflect.Message {
+ mi := &file_volume_server_proto_msgTypes[86]
+ if x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use EcBitrotProtection.ProtoReflect.Descriptor instead.
+func (*EcBitrotProtection) Descriptor() ([]byte, []int) {
+ return file_volume_server_proto_rawDescGZIP(), []int{86}
+}
+
+func (x *EcBitrotProtection) GetAlgorithm() ChecksumAlgorithm {
+ if x != nil {
+ return x.Algorithm
+ }
+ return ChecksumAlgorithm_CHECKSUM_NONE
+}
+
+func (x *EcBitrotProtection) GetBlockSize() uint32 {
+ if x != nil {
+ return x.BlockSize
+ }
+ return 0
+}
+
+func (x *EcBitrotProtection) GetGeneration() uint32 {
+ if x != nil {
+ return x.Generation
+ }
+ return 0
+}
+
+func (x *EcBitrotProtection) GetEcShardConfig() *EcShardConfig {
+ if x != nil {
+ return x.EcShardConfig
+ }
+ return nil
+}
+
+func (x *EcBitrotProtection) GetShards() []*EcShardChecksums {
+ if x != nil {
+ return x.Shards
+ }
+ return nil
+}
+
+func (x *EcBitrotProtection) GetEncodeUuid() []byte {
+ if x != nil {
+ return x.EncodeUuid
+ }
+ return nil
+}
+
+type EcShardChecksums struct {
+ state protoimpl.MessageState `protogen:"open.v1"`
+ ShardId uint32 `protobuf:"varint,1,opt,name=shard_id,json=shardId,proto3" json:"shard_id,omitempty"` // 0..MaxShardCount-1 (custom EC ratios go up to 32)
+ CoveredSize int64 `protobuf:"varint,2,opt,name=covered_size,json=coveredSize,proto3" json:"covered_size,omitempty"` // shard byte length these checksums cover (must equal the on-disk shard length)
+ BlockCrc32C []byte `protobuf:"bytes,3,opt,name=block_crc32c,json=blockCrc32c,proto3" json:"block_crc32c,omitempty"` // packed little-endian uint32[] = ceil(covered_size/block_size) entries
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
+}
+
+func (x *EcShardChecksums) Reset() {
+ *x = EcShardChecksums{}
+ mi := &file_volume_server_proto_msgTypes[87]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+}
+
+func (x *EcShardChecksums) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*EcShardChecksums) ProtoMessage() {}
+
+func (x *EcShardChecksums) ProtoReflect() protoreflect.Message {
+ mi := &file_volume_server_proto_msgTypes[87]
+ if x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use EcShardChecksums.ProtoReflect.Descriptor instead.
+func (*EcShardChecksums) Descriptor() ([]byte, []int) {
+ return file_volume_server_proto_rawDescGZIP(), []int{87}
+}
+
+func (x *EcShardChecksums) GetShardId() uint32 {
+ if x != nil {
+ return x.ShardId
+ }
+ return 0
+}
+
+func (x *EcShardChecksums) GetCoveredSize() int64 {
+ if x != nil {
+ return x.CoveredSize
+ }
+ return 0
+}
+
+func (x *EcShardChecksums) GetBlockCrc32C() []byte {
+ if x != nil {
+ return x.BlockCrc32C
+ }
+ return nil
+}
+
type OldVersionVolumeInfo struct {
state protoimpl.MessageState `protogen:"open.v1"`
Files []*RemoteFile `protobuf:"bytes,1,rep,name=files,proto3" json:"files,omitempty"`
@@ -4936,7 +5149,7 @@ type OldVersionVolumeInfo struct {
func (x *OldVersionVolumeInfo) Reset() {
*x = OldVersionVolumeInfo{}
- mi := &file_volume_server_proto_msgTypes[86]
+ mi := &file_volume_server_proto_msgTypes[88]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -4948,7 +5161,7 @@ func (x *OldVersionVolumeInfo) String() string {
func (*OldVersionVolumeInfo) ProtoMessage() {}
func (x *OldVersionVolumeInfo) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[86]
+ mi := &file_volume_server_proto_msgTypes[88]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -4961,7 +5174,7 @@ func (x *OldVersionVolumeInfo) ProtoReflect() protoreflect.Message {
// Deprecated: Use OldVersionVolumeInfo.ProtoReflect.Descriptor instead.
func (*OldVersionVolumeInfo) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{86}
+ return file_volume_server_proto_rawDescGZIP(), []int{88}
}
func (x *OldVersionVolumeInfo) GetFiles() []*RemoteFile {
@@ -5026,7 +5239,7 @@ type VolumeTierMoveDatToRemoteRequest struct {
func (x *VolumeTierMoveDatToRemoteRequest) Reset() {
*x = VolumeTierMoveDatToRemoteRequest{}
- mi := &file_volume_server_proto_msgTypes[87]
+ mi := &file_volume_server_proto_msgTypes[89]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -5038,7 +5251,7 @@ func (x *VolumeTierMoveDatToRemoteRequest) String() string {
func (*VolumeTierMoveDatToRemoteRequest) ProtoMessage() {}
func (x *VolumeTierMoveDatToRemoteRequest) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[87]
+ mi := &file_volume_server_proto_msgTypes[89]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -5051,7 +5264,7 @@ func (x *VolumeTierMoveDatToRemoteRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use VolumeTierMoveDatToRemoteRequest.ProtoReflect.Descriptor instead.
func (*VolumeTierMoveDatToRemoteRequest) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{87}
+ return file_volume_server_proto_rawDescGZIP(), []int{89}
}
func (x *VolumeTierMoveDatToRemoteRequest) GetVolumeId() uint32 {
@@ -5092,7 +5305,7 @@ type VolumeTierMoveDatToRemoteResponse struct {
func (x *VolumeTierMoveDatToRemoteResponse) Reset() {
*x = VolumeTierMoveDatToRemoteResponse{}
- mi := &file_volume_server_proto_msgTypes[88]
+ mi := &file_volume_server_proto_msgTypes[90]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -5104,7 +5317,7 @@ func (x *VolumeTierMoveDatToRemoteResponse) String() string {
func (*VolumeTierMoveDatToRemoteResponse) ProtoMessage() {}
func (x *VolumeTierMoveDatToRemoteResponse) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[88]
+ mi := &file_volume_server_proto_msgTypes[90]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -5117,7 +5330,7 @@ func (x *VolumeTierMoveDatToRemoteResponse) ProtoReflect() protoreflect.Message
// Deprecated: Use VolumeTierMoveDatToRemoteResponse.ProtoReflect.Descriptor instead.
func (*VolumeTierMoveDatToRemoteResponse) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{88}
+ return file_volume_server_proto_rawDescGZIP(), []int{90}
}
func (x *VolumeTierMoveDatToRemoteResponse) GetProcessed() int64 {
@@ -5145,7 +5358,7 @@ type VolumeTierMoveDatFromRemoteRequest struct {
func (x *VolumeTierMoveDatFromRemoteRequest) Reset() {
*x = VolumeTierMoveDatFromRemoteRequest{}
- mi := &file_volume_server_proto_msgTypes[89]
+ mi := &file_volume_server_proto_msgTypes[91]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -5157,7 +5370,7 @@ func (x *VolumeTierMoveDatFromRemoteRequest) String() string {
func (*VolumeTierMoveDatFromRemoteRequest) ProtoMessage() {}
func (x *VolumeTierMoveDatFromRemoteRequest) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[89]
+ mi := &file_volume_server_proto_msgTypes[91]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -5170,7 +5383,7 @@ func (x *VolumeTierMoveDatFromRemoteRequest) ProtoReflect() protoreflect.Message
// Deprecated: Use VolumeTierMoveDatFromRemoteRequest.ProtoReflect.Descriptor instead.
func (*VolumeTierMoveDatFromRemoteRequest) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{89}
+ return file_volume_server_proto_rawDescGZIP(), []int{91}
}
func (x *VolumeTierMoveDatFromRemoteRequest) GetVolumeId() uint32 {
@@ -5204,7 +5417,7 @@ type VolumeTierMoveDatFromRemoteResponse struct {
func (x *VolumeTierMoveDatFromRemoteResponse) Reset() {
*x = VolumeTierMoveDatFromRemoteResponse{}
- mi := &file_volume_server_proto_msgTypes[90]
+ mi := &file_volume_server_proto_msgTypes[92]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -5216,7 +5429,7 @@ func (x *VolumeTierMoveDatFromRemoteResponse) String() string {
func (*VolumeTierMoveDatFromRemoteResponse) ProtoMessage() {}
func (x *VolumeTierMoveDatFromRemoteResponse) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[90]
+ mi := &file_volume_server_proto_msgTypes[92]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -5229,7 +5442,7 @@ func (x *VolumeTierMoveDatFromRemoteResponse) ProtoReflect() protoreflect.Messag
// Deprecated: Use VolumeTierMoveDatFromRemoteResponse.ProtoReflect.Descriptor instead.
func (*VolumeTierMoveDatFromRemoteResponse) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{90}
+ return file_volume_server_proto_rawDescGZIP(), []int{92}
}
func (x *VolumeTierMoveDatFromRemoteResponse) GetProcessed() int64 {
@@ -5254,7 +5467,7 @@ type VolumeServerStatusRequest struct {
func (x *VolumeServerStatusRequest) Reset() {
*x = VolumeServerStatusRequest{}
- mi := &file_volume_server_proto_msgTypes[91]
+ mi := &file_volume_server_proto_msgTypes[93]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -5266,7 +5479,7 @@ func (x *VolumeServerStatusRequest) String() string {
func (*VolumeServerStatusRequest) ProtoMessage() {}
func (x *VolumeServerStatusRequest) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[91]
+ mi := &file_volume_server_proto_msgTypes[93]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -5279,7 +5492,7 @@ func (x *VolumeServerStatusRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use VolumeServerStatusRequest.ProtoReflect.Descriptor instead.
func (*VolumeServerStatusRequest) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{91}
+ return file_volume_server_proto_rawDescGZIP(), []int{93}
}
type VolumeServerStatusResponse struct {
@@ -5296,7 +5509,7 @@ type VolumeServerStatusResponse struct {
func (x *VolumeServerStatusResponse) Reset() {
*x = VolumeServerStatusResponse{}
- mi := &file_volume_server_proto_msgTypes[92]
+ mi := &file_volume_server_proto_msgTypes[94]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -5308,7 +5521,7 @@ func (x *VolumeServerStatusResponse) String() string {
func (*VolumeServerStatusResponse) ProtoMessage() {}
func (x *VolumeServerStatusResponse) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[92]
+ mi := &file_volume_server_proto_msgTypes[94]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -5321,7 +5534,7 @@ func (x *VolumeServerStatusResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use VolumeServerStatusResponse.ProtoReflect.Descriptor instead.
func (*VolumeServerStatusResponse) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{92}
+ return file_volume_server_proto_rawDescGZIP(), []int{94}
}
func (x *VolumeServerStatusResponse) GetDiskStatuses() []*DiskStatus {
@@ -5374,7 +5587,7 @@ type VolumeServerLeaveRequest struct {
func (x *VolumeServerLeaveRequest) Reset() {
*x = VolumeServerLeaveRequest{}
- mi := &file_volume_server_proto_msgTypes[93]
+ mi := &file_volume_server_proto_msgTypes[95]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -5386,7 +5599,7 @@ func (x *VolumeServerLeaveRequest) String() string {
func (*VolumeServerLeaveRequest) ProtoMessage() {}
func (x *VolumeServerLeaveRequest) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[93]
+ mi := &file_volume_server_proto_msgTypes[95]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -5399,7 +5612,7 @@ func (x *VolumeServerLeaveRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use VolumeServerLeaveRequest.ProtoReflect.Descriptor instead.
func (*VolumeServerLeaveRequest) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{93}
+ return file_volume_server_proto_rawDescGZIP(), []int{95}
}
type VolumeServerLeaveResponse struct {
@@ -5410,7 +5623,7 @@ type VolumeServerLeaveResponse struct {
func (x *VolumeServerLeaveResponse) Reset() {
*x = VolumeServerLeaveResponse{}
- mi := &file_volume_server_proto_msgTypes[94]
+ mi := &file_volume_server_proto_msgTypes[96]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -5422,7 +5635,7 @@ func (x *VolumeServerLeaveResponse) String() string {
func (*VolumeServerLeaveResponse) ProtoMessage() {}
func (x *VolumeServerLeaveResponse) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[94]
+ mi := &file_volume_server_proto_msgTypes[96]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -5435,7 +5648,7 @@ func (x *VolumeServerLeaveResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use VolumeServerLeaveResponse.ProtoReflect.Descriptor instead.
func (*VolumeServerLeaveResponse) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{94}
+ return file_volume_server_proto_rawDescGZIP(), []int{96}
}
// remote storage
@@ -5458,7 +5671,7 @@ type FetchAndWriteNeedleRequest struct {
func (x *FetchAndWriteNeedleRequest) Reset() {
*x = FetchAndWriteNeedleRequest{}
- mi := &file_volume_server_proto_msgTypes[95]
+ mi := &file_volume_server_proto_msgTypes[97]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -5470,7 +5683,7 @@ func (x *FetchAndWriteNeedleRequest) String() string {
func (*FetchAndWriteNeedleRequest) ProtoMessage() {}
func (x *FetchAndWriteNeedleRequest) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[95]
+ mi := &file_volume_server_proto_msgTypes[97]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -5483,7 +5696,7 @@ func (x *FetchAndWriteNeedleRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use FetchAndWriteNeedleRequest.ProtoReflect.Descriptor instead.
func (*FetchAndWriteNeedleRequest) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{95}
+ return file_volume_server_proto_rawDescGZIP(), []int{97}
}
func (x *FetchAndWriteNeedleRequest) GetVolumeId() uint32 {
@@ -5565,7 +5778,7 @@ type FetchAndWriteNeedleResponse struct {
func (x *FetchAndWriteNeedleResponse) Reset() {
*x = FetchAndWriteNeedleResponse{}
- mi := &file_volume_server_proto_msgTypes[96]
+ mi := &file_volume_server_proto_msgTypes[98]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -5577,7 +5790,7 @@ func (x *FetchAndWriteNeedleResponse) String() string {
func (*FetchAndWriteNeedleResponse) ProtoMessage() {}
func (x *FetchAndWriteNeedleResponse) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[96]
+ mi := &file_volume_server_proto_msgTypes[98]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -5590,7 +5803,7 @@ func (x *FetchAndWriteNeedleResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use FetchAndWriteNeedleResponse.ProtoReflect.Descriptor instead.
func (*FetchAndWriteNeedleResponse) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{96}
+ return file_volume_server_proto_rawDescGZIP(), []int{98}
}
func (x *FetchAndWriteNeedleResponse) GetETag() string {
@@ -5612,7 +5825,7 @@ type ScrubVolumeRequest struct {
func (x *ScrubVolumeRequest) Reset() {
*x = ScrubVolumeRequest{}
- mi := &file_volume_server_proto_msgTypes[97]
+ mi := &file_volume_server_proto_msgTypes[99]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -5624,7 +5837,7 @@ func (x *ScrubVolumeRequest) String() string {
func (*ScrubVolumeRequest) ProtoMessage() {}
func (x *ScrubVolumeRequest) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[97]
+ mi := &file_volume_server_proto_msgTypes[99]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -5637,7 +5850,7 @@ func (x *ScrubVolumeRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use ScrubVolumeRequest.ProtoReflect.Descriptor instead.
func (*ScrubVolumeRequest) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{97}
+ return file_volume_server_proto_rawDescGZIP(), []int{99}
}
func (x *ScrubVolumeRequest) GetMode() VolumeScrubMode {
@@ -5673,7 +5886,7 @@ type ScrubVolumeResponse struct {
func (x *ScrubVolumeResponse) Reset() {
*x = ScrubVolumeResponse{}
- mi := &file_volume_server_proto_msgTypes[98]
+ mi := &file_volume_server_proto_msgTypes[100]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -5685,7 +5898,7 @@ func (x *ScrubVolumeResponse) String() string {
func (*ScrubVolumeResponse) ProtoMessage() {}
func (x *ScrubVolumeResponse) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[98]
+ mi := &file_volume_server_proto_msgTypes[100]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -5698,7 +5911,7 @@ func (x *ScrubVolumeResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use ScrubVolumeResponse.ProtoReflect.Descriptor instead.
func (*ScrubVolumeResponse) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{98}
+ return file_volume_server_proto_rawDescGZIP(), []int{100}
}
func (x *ScrubVolumeResponse) GetTotalVolumes() uint64 {
@@ -5740,7 +5953,7 @@ type ScrubEcVolumeRequest struct {
func (x *ScrubEcVolumeRequest) Reset() {
*x = ScrubEcVolumeRequest{}
- mi := &file_volume_server_proto_msgTypes[99]
+ mi := &file_volume_server_proto_msgTypes[101]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -5752,7 +5965,7 @@ func (x *ScrubEcVolumeRequest) String() string {
func (*ScrubEcVolumeRequest) ProtoMessage() {}
func (x *ScrubEcVolumeRequest) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[99]
+ mi := &file_volume_server_proto_msgTypes[101]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -5765,7 +5978,7 @@ func (x *ScrubEcVolumeRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use ScrubEcVolumeRequest.ProtoReflect.Descriptor instead.
func (*ScrubEcVolumeRequest) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{99}
+ return file_volume_server_proto_rawDescGZIP(), []int{101}
}
func (x *ScrubEcVolumeRequest) GetMode() VolumeScrubMode {
@@ -5795,7 +6008,7 @@ type ScrubEcVolumeResponse struct {
func (x *ScrubEcVolumeResponse) Reset() {
*x = ScrubEcVolumeResponse{}
- mi := &file_volume_server_proto_msgTypes[100]
+ mi := &file_volume_server_proto_msgTypes[102]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -5807,7 +6020,7 @@ func (x *ScrubEcVolumeResponse) String() string {
func (*ScrubEcVolumeResponse) ProtoMessage() {}
func (x *ScrubEcVolumeResponse) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[100]
+ mi := &file_volume_server_proto_msgTypes[102]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -5820,7 +6033,7 @@ func (x *ScrubEcVolumeResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use ScrubEcVolumeResponse.ProtoReflect.Descriptor instead.
func (*ScrubEcVolumeResponse) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{100}
+ return file_volume_server_proto_rawDescGZIP(), []int{102}
}
func (x *ScrubEcVolumeResponse) GetTotalVolumes() uint64 {
@@ -5872,7 +6085,7 @@ type QueryRequest struct {
func (x *QueryRequest) Reset() {
*x = QueryRequest{}
- mi := &file_volume_server_proto_msgTypes[101]
+ mi := &file_volume_server_proto_msgTypes[103]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -5884,7 +6097,7 @@ func (x *QueryRequest) String() string {
func (*QueryRequest) ProtoMessage() {}
func (x *QueryRequest) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[101]
+ mi := &file_volume_server_proto_msgTypes[103]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -5897,7 +6110,7 @@ func (x *QueryRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use QueryRequest.ProtoReflect.Descriptor instead.
func (*QueryRequest) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{101}
+ return file_volume_server_proto_rawDescGZIP(), []int{103}
}
func (x *QueryRequest) GetSelections() []string {
@@ -5944,7 +6157,7 @@ type QueriedStripe struct {
func (x *QueriedStripe) Reset() {
*x = QueriedStripe{}
- mi := &file_volume_server_proto_msgTypes[102]
+ mi := &file_volume_server_proto_msgTypes[104]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -5956,7 +6169,7 @@ func (x *QueriedStripe) String() string {
func (*QueriedStripe) ProtoMessage() {}
func (x *QueriedStripe) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[102]
+ mi := &file_volume_server_proto_msgTypes[104]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -5969,7 +6182,7 @@ func (x *QueriedStripe) ProtoReflect() protoreflect.Message {
// Deprecated: Use QueriedStripe.ProtoReflect.Descriptor instead.
func (*QueriedStripe) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{102}
+ return file_volume_server_proto_rawDescGZIP(), []int{104}
}
func (x *QueriedStripe) GetRecords() []byte {
@@ -5989,7 +6202,7 @@ type VolumeNeedleStatusRequest struct {
func (x *VolumeNeedleStatusRequest) Reset() {
*x = VolumeNeedleStatusRequest{}
- mi := &file_volume_server_proto_msgTypes[103]
+ mi := &file_volume_server_proto_msgTypes[105]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6001,7 +6214,7 @@ func (x *VolumeNeedleStatusRequest) String() string {
func (*VolumeNeedleStatusRequest) ProtoMessage() {}
func (x *VolumeNeedleStatusRequest) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[103]
+ mi := &file_volume_server_proto_msgTypes[105]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6014,7 +6227,7 @@ func (x *VolumeNeedleStatusRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use VolumeNeedleStatusRequest.ProtoReflect.Descriptor instead.
func (*VolumeNeedleStatusRequest) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{103}
+ return file_volume_server_proto_rawDescGZIP(), []int{105}
}
func (x *VolumeNeedleStatusRequest) GetVolumeId() uint32 {
@@ -6045,7 +6258,7 @@ type VolumeNeedleStatusResponse struct {
func (x *VolumeNeedleStatusResponse) Reset() {
*x = VolumeNeedleStatusResponse{}
- mi := &file_volume_server_proto_msgTypes[104]
+ mi := &file_volume_server_proto_msgTypes[106]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6057,7 +6270,7 @@ func (x *VolumeNeedleStatusResponse) String() string {
func (*VolumeNeedleStatusResponse) ProtoMessage() {}
func (x *VolumeNeedleStatusResponse) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[104]
+ mi := &file_volume_server_proto_msgTypes[106]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6070,7 +6283,7 @@ func (x *VolumeNeedleStatusResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use VolumeNeedleStatusResponse.ProtoReflect.Descriptor instead.
func (*VolumeNeedleStatusResponse) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{104}
+ return file_volume_server_proto_rawDescGZIP(), []int{106}
}
func (x *VolumeNeedleStatusResponse) GetNeedleId() uint64 {
@@ -6125,7 +6338,7 @@ type PingRequest struct {
func (x *PingRequest) Reset() {
*x = PingRequest{}
- mi := &file_volume_server_proto_msgTypes[105]
+ mi := &file_volume_server_proto_msgTypes[107]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6137,7 +6350,7 @@ func (x *PingRequest) String() string {
func (*PingRequest) ProtoMessage() {}
func (x *PingRequest) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[105]
+ mi := &file_volume_server_proto_msgTypes[107]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6150,7 +6363,7 @@ func (x *PingRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use PingRequest.ProtoReflect.Descriptor instead.
func (*PingRequest) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{105}
+ return file_volume_server_proto_rawDescGZIP(), []int{107}
}
func (x *PingRequest) GetTarget() string {
@@ -6178,7 +6391,7 @@ type PingResponse struct {
func (x *PingResponse) Reset() {
*x = PingResponse{}
- mi := &file_volume_server_proto_msgTypes[106]
+ mi := &file_volume_server_proto_msgTypes[108]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6190,7 +6403,7 @@ func (x *PingResponse) String() string {
func (*PingResponse) ProtoMessage() {}
func (x *PingResponse) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[106]
+ mi := &file_volume_server_proto_msgTypes[108]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6203,7 +6416,7 @@ func (x *PingResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use PingResponse.ProtoReflect.Descriptor instead.
func (*PingResponse) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{106}
+ return file_volume_server_proto_rawDescGZIP(), []int{108}
}
func (x *PingResponse) GetStartTimeNs() int64 {
@@ -6238,7 +6451,7 @@ type FetchAndWriteNeedleRequest_Replica struct {
func (x *FetchAndWriteNeedleRequest_Replica) Reset() {
*x = FetchAndWriteNeedleRequest_Replica{}
- mi := &file_volume_server_proto_msgTypes[107]
+ mi := &file_volume_server_proto_msgTypes[109]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6250,7 +6463,7 @@ func (x *FetchAndWriteNeedleRequest_Replica) String() string {
func (*FetchAndWriteNeedleRequest_Replica) ProtoMessage() {}
func (x *FetchAndWriteNeedleRequest_Replica) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[107]
+ mi := &file_volume_server_proto_msgTypes[109]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6263,7 +6476,7 @@ func (x *FetchAndWriteNeedleRequest_Replica) ProtoReflect() protoreflect.Message
// Deprecated: Use FetchAndWriteNeedleRequest_Replica.ProtoReflect.Descriptor instead.
func (*FetchAndWriteNeedleRequest_Replica) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{95, 0}
+ return file_volume_server_proto_rawDescGZIP(), []int{97, 0}
}
func (x *FetchAndWriteNeedleRequest_Replica) GetUrl() string {
@@ -6298,7 +6511,7 @@ type QueryRequest_Filter struct {
func (x *QueryRequest_Filter) Reset() {
*x = QueryRequest_Filter{}
- mi := &file_volume_server_proto_msgTypes[108]
+ mi := &file_volume_server_proto_msgTypes[110]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6310,7 +6523,7 @@ func (x *QueryRequest_Filter) String() string {
func (*QueryRequest_Filter) ProtoMessage() {}
func (x *QueryRequest_Filter) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[108]
+ mi := &file_volume_server_proto_msgTypes[110]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6323,7 +6536,7 @@ func (x *QueryRequest_Filter) ProtoReflect() protoreflect.Message {
// Deprecated: Use QueryRequest_Filter.ProtoReflect.Descriptor instead.
func (*QueryRequest_Filter) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{101, 0}
+ return file_volume_server_proto_rawDescGZIP(), []int{103, 0}
}
func (x *QueryRequest_Filter) GetField() string {
@@ -6360,7 +6573,7 @@ type QueryRequest_InputSerialization struct {
func (x *QueryRequest_InputSerialization) Reset() {
*x = QueryRequest_InputSerialization{}
- mi := &file_volume_server_proto_msgTypes[109]
+ mi := &file_volume_server_proto_msgTypes[111]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6372,7 +6585,7 @@ func (x *QueryRequest_InputSerialization) String() string {
func (*QueryRequest_InputSerialization) ProtoMessage() {}
func (x *QueryRequest_InputSerialization) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[109]
+ mi := &file_volume_server_proto_msgTypes[111]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6385,7 +6598,7 @@ func (x *QueryRequest_InputSerialization) ProtoReflect() protoreflect.Message {
// Deprecated: Use QueryRequest_InputSerialization.ProtoReflect.Descriptor instead.
func (*QueryRequest_InputSerialization) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{101, 1}
+ return file_volume_server_proto_rawDescGZIP(), []int{103, 1}
}
func (x *QueryRequest_InputSerialization) GetCompressionType() string {
@@ -6426,7 +6639,7 @@ type QueryRequest_OutputSerialization struct {
func (x *QueryRequest_OutputSerialization) Reset() {
*x = QueryRequest_OutputSerialization{}
- mi := &file_volume_server_proto_msgTypes[110]
+ mi := &file_volume_server_proto_msgTypes[112]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6438,7 +6651,7 @@ func (x *QueryRequest_OutputSerialization) String() string {
func (*QueryRequest_OutputSerialization) ProtoMessage() {}
func (x *QueryRequest_OutputSerialization) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[110]
+ mi := &file_volume_server_proto_msgTypes[112]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6451,7 +6664,7 @@ func (x *QueryRequest_OutputSerialization) ProtoReflect() protoreflect.Message {
// Deprecated: Use QueryRequest_OutputSerialization.ProtoReflect.Descriptor instead.
func (*QueryRequest_OutputSerialization) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{101, 2}
+ return file_volume_server_proto_rawDescGZIP(), []int{103, 2}
}
func (x *QueryRequest_OutputSerialization) GetCsvOutput() *QueryRequest_OutputSerialization_CSVOutput {
@@ -6484,7 +6697,7 @@ type QueryRequest_InputSerialization_CSVInput struct {
func (x *QueryRequest_InputSerialization_CSVInput) Reset() {
*x = QueryRequest_InputSerialization_CSVInput{}
- mi := &file_volume_server_proto_msgTypes[111]
+ mi := &file_volume_server_proto_msgTypes[113]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6496,7 +6709,7 @@ func (x *QueryRequest_InputSerialization_CSVInput) String() string {
func (*QueryRequest_InputSerialization_CSVInput) ProtoMessage() {}
func (x *QueryRequest_InputSerialization_CSVInput) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[111]
+ mi := &file_volume_server_proto_msgTypes[113]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6509,7 +6722,7 @@ func (x *QueryRequest_InputSerialization_CSVInput) ProtoReflect() protoreflect.M
// Deprecated: Use QueryRequest_InputSerialization_CSVInput.ProtoReflect.Descriptor instead.
func (*QueryRequest_InputSerialization_CSVInput) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{101, 1, 0}
+ return file_volume_server_proto_rawDescGZIP(), []int{103, 1, 0}
}
func (x *QueryRequest_InputSerialization_CSVInput) GetFileHeaderInfo() string {
@@ -6570,7 +6783,7 @@ type QueryRequest_InputSerialization_JSONInput struct {
func (x *QueryRequest_InputSerialization_JSONInput) Reset() {
*x = QueryRequest_InputSerialization_JSONInput{}
- mi := &file_volume_server_proto_msgTypes[112]
+ mi := &file_volume_server_proto_msgTypes[114]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6582,7 +6795,7 @@ func (x *QueryRequest_InputSerialization_JSONInput) String() string {
func (*QueryRequest_InputSerialization_JSONInput) ProtoMessage() {}
func (x *QueryRequest_InputSerialization_JSONInput) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[112]
+ mi := &file_volume_server_proto_msgTypes[114]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6595,7 +6808,7 @@ func (x *QueryRequest_InputSerialization_JSONInput) ProtoReflect() protoreflect.
// Deprecated: Use QueryRequest_InputSerialization_JSONInput.ProtoReflect.Descriptor instead.
func (*QueryRequest_InputSerialization_JSONInput) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{101, 1, 1}
+ return file_volume_server_proto_rawDescGZIP(), []int{103, 1, 1}
}
func (x *QueryRequest_InputSerialization_JSONInput) GetType() string {
@@ -6613,7 +6826,7 @@ type QueryRequest_InputSerialization_ParquetInput struct {
func (x *QueryRequest_InputSerialization_ParquetInput) Reset() {
*x = QueryRequest_InputSerialization_ParquetInput{}
- mi := &file_volume_server_proto_msgTypes[113]
+ mi := &file_volume_server_proto_msgTypes[115]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6625,7 +6838,7 @@ func (x *QueryRequest_InputSerialization_ParquetInput) String() string {
func (*QueryRequest_InputSerialization_ParquetInput) ProtoMessage() {}
func (x *QueryRequest_InputSerialization_ParquetInput) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[113]
+ mi := &file_volume_server_proto_msgTypes[115]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6638,7 +6851,7 @@ func (x *QueryRequest_InputSerialization_ParquetInput) ProtoReflect() protorefle
// Deprecated: Use QueryRequest_InputSerialization_ParquetInput.ProtoReflect.Descriptor instead.
func (*QueryRequest_InputSerialization_ParquetInput) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{101, 1, 2}
+ return file_volume_server_proto_rawDescGZIP(), []int{103, 1, 2}
}
type QueryRequest_OutputSerialization_CSVOutput struct {
@@ -6654,7 +6867,7 @@ type QueryRequest_OutputSerialization_CSVOutput struct {
func (x *QueryRequest_OutputSerialization_CSVOutput) Reset() {
*x = QueryRequest_OutputSerialization_CSVOutput{}
- mi := &file_volume_server_proto_msgTypes[114]
+ mi := &file_volume_server_proto_msgTypes[116]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6666,7 +6879,7 @@ func (x *QueryRequest_OutputSerialization_CSVOutput) String() string {
func (*QueryRequest_OutputSerialization_CSVOutput) ProtoMessage() {}
func (x *QueryRequest_OutputSerialization_CSVOutput) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[114]
+ mi := &file_volume_server_proto_msgTypes[116]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6679,7 +6892,7 @@ func (x *QueryRequest_OutputSerialization_CSVOutput) ProtoReflect() protoreflect
// Deprecated: Use QueryRequest_OutputSerialization_CSVOutput.ProtoReflect.Descriptor instead.
func (*QueryRequest_OutputSerialization_CSVOutput) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{101, 2, 0}
+ return file_volume_server_proto_rawDescGZIP(), []int{103, 2, 0}
}
func (x *QueryRequest_OutputSerialization_CSVOutput) GetQuoteFields() string {
@@ -6726,7 +6939,7 @@ type QueryRequest_OutputSerialization_JSONOutput struct {
func (x *QueryRequest_OutputSerialization_JSONOutput) Reset() {
*x = QueryRequest_OutputSerialization_JSONOutput{}
- mi := &file_volume_server_proto_msgTypes[115]
+ mi := &file_volume_server_proto_msgTypes[117]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6738,7 +6951,7 @@ func (x *QueryRequest_OutputSerialization_JSONOutput) String() string {
func (*QueryRequest_OutputSerialization_JSONOutput) ProtoMessage() {}
func (x *QueryRequest_OutputSerialization_JSONOutput) ProtoReflect() protoreflect.Message {
- mi := &file_volume_server_proto_msgTypes[115]
+ mi := &file_volume_server_proto_msgTypes[117]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6751,7 +6964,7 @@ func (x *QueryRequest_OutputSerialization_JSONOutput) ProtoReflect() protoreflec
// Deprecated: Use QueryRequest_OutputSerialization_JSONOutput.ProtoReflect.Descriptor instead.
func (*QueryRequest_OutputSerialization_JSONOutput) Descriptor() ([]byte, []int) {
- return file_volume_server_proto_rawDescGZIP(), []int{101, 2, 1}
+ return file_volume_server_proto_rawDescGZIP(), []int{103, 2, 1}
}
func (x *QueryRequest_OutputSerialization_JSONOutput) GetRecordDelimiter() string {
@@ -6986,14 +7199,15 @@ const file_volume_server_proto_rawDesc = "" +
"\n" +
"collection\x18\x02 \x01(\tR\n" +
"collection\" \n" +
- "\x1eVolumeEcShardsGenerateResponse\"[\n" +
+ "\x1eVolumeEcShardsGenerateResponse\"\x8f\x01\n" +
"\x1cVolumeEcShardsRebuildRequest\x12\x1b\n" +
"\tvolume_id\x18\x01 \x01(\rR\bvolumeId\x12\x1e\n" +
"\n" +
"collection\x18\x02 \x01(\tR\n" +
- "collection\"K\n" +
+ "collection\x122\n" +
+ "\x15unsafe_ignore_sidecar\x18\x03 \x01(\bR\x13unsafeIgnoreSidecar\"K\n" +
"\x1dVolumeEcShardsRebuildResponse\x12*\n" +
- "\x11rebuilt_shard_ids\x18\x01 \x03(\rR\x0frebuiltShardIds\"\xa4\x02\n" +
+ "\x11rebuilt_shard_ids\x18\x01 \x03(\rR\x0frebuiltShardIds\"\xcc\x02\n" +
"\x19VolumeEcShardsCopyRequest\x12\x1b\n" +
"\tvolume_id\x18\x01 \x01(\rR\bvolumeId\x12\x1e\n" +
"\n" +
@@ -7004,7 +7218,8 @@ const file_volume_server_proto_rawDesc = "" +
"\x10source_data_node\x18\x05 \x01(\tR\x0esourceDataNode\x12\"\n" +
"\rcopy_ecj_file\x18\x06 \x01(\bR\vcopyEcjFile\x12\"\n" +
"\rcopy_vif_file\x18\a \x01(\bR\vcopyVifFile\x12\x17\n" +
- "\adisk_id\x18\b \x01(\rR\x06diskId\"\x1c\n" +
+ "\adisk_id\x18\b \x01(\rR\x06diskId\x12&\n" +
+ "\x0fcopy_ecsum_file\x18\t \x01(\bR\rcopyEcsumFile\"\x1c\n" +
"\x1aVolumeEcShardsCopyResponse\"w\n" +
"\x1bVolumeEcShardsDeleteRequest\x12\x1b\n" +
"\tvolume_id\x18\x01 \x01(\rR\bvolumeId\x12\x1e\n" +
@@ -7126,7 +7341,22 @@ const file_volume_server_proto_rawDesc = "" +
"\rEcShardConfig\x12\x1f\n" +
"\vdata_shards\x18\x01 \x01(\rR\n" +
"dataShards\x12#\n" +
- "\rparity_shards\x18\x02 \x01(\rR\fparityShards\"\x8b\x02\n" +
+ "\rparity_shards\x18\x02 \x01(\rR\fparityShards\"\xbc\x02\n" +
+ "\x12EcBitrotProtection\x12A\n" +
+ "\talgorithm\x18\x01 \x01(\x0e2#.volume_server_pb.ChecksumAlgorithmR\talgorithm\x12\x1d\n" +
+ "\n" +
+ "block_size\x18\x02 \x01(\rR\tblockSize\x12\x1e\n" +
+ "\n" +
+ "generation\x18\x03 \x01(\rR\n" +
+ "generation\x12G\n" +
+ "\x0fec_shard_config\x18\x04 \x01(\v2\x1f.volume_server_pb.EcShardConfigR\recShardConfig\x12:\n" +
+ "\x06shards\x18\x05 \x03(\v2\".volume_server_pb.EcShardChecksumsR\x06shards\x12\x1f\n" +
+ "\vencode_uuid\x18\x06 \x01(\fR\n" +
+ "encodeUuid\"s\n" +
+ "\x10EcShardChecksums\x12\x19\n" +
+ "\bshard_id\x18\x01 \x01(\rR\ashardId\x12!\n" +
+ "\fcovered_size\x18\x02 \x01(\x03R\vcoveredSize\x12!\n" +
+ "\fblock_crc32c\x18\x03 \x01(\fR\vblockCrc32c\"\x8b\x02\n" +
"\x14OldVersionVolumeInfo\x122\n" +
"\x05files\x18\x01 \x03(\v2\x1c.volume_server_pb.RemoteFileR\x05files\x12\x18\n" +
"\aversion\x18\x02 \x01(\rR\aversion\x12 \n" +
@@ -7269,12 +7499,16 @@ const file_volume_server_proto_rawDesc = "" +
"\rstart_time_ns\x18\x01 \x01(\x03R\vstartTimeNs\x12$\n" +
"\x0eremote_time_ns\x18\x02 \x01(\x03R\fremoteTimeNs\x12 \n" +
"\fstop_time_ns\x18\x03 \x01(\x03R\n" +
- "stopTimeNs*>\n" +
+ "stopTimeNs*;\n" +
+ "\x11ChecksumAlgorithm\x12\x11\n" +
+ "\rCHECKSUM_NONE\x10\x00\x12\x13\n" +
+ "\x0fCHECKSUM_CRC32C\x10\x01*L\n" +
"\x0fVolumeScrubMode\x12\v\n" +
"\aUNKNOWN\x10\x00\x12\t\n" +
"\x05INDEX\x10\x01\x12\b\n" +
"\x04FULL\x10\x02\x12\t\n" +
- "\x05LOCAL\x10\x032\xfb(\n" +
+ "\x05LOCAL\x10\x03\x12\f\n" +
+ "\bCHECKSUM\x10\x042\xfb(\n" +
"\fVolumeServer\x12\\\n" +
"\vBatchDelete\x12$.volume_server_pb.BatchDeleteRequest\x1a%.volume_server_pb.BatchDeleteResponse\"\x00\x12n\n" +
"\x11VacuumVolumeCheck\x12*.volume_server_pb.VacuumVolumeCheckRequest\x1a+.volume_server_pb.VacuumVolumeCheckResponse\"\x00\x12v\n" +
@@ -7338,258 +7572,264 @@ func file_volume_server_proto_rawDescGZIP() []byte {
return file_volume_server_proto_rawDescData
}
-var file_volume_server_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
-var file_volume_server_proto_msgTypes = make([]protoimpl.MessageInfo, 116)
+var file_volume_server_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
+var file_volume_server_proto_msgTypes = make([]protoimpl.MessageInfo, 118)
var file_volume_server_proto_goTypes = []any{
- (VolumeScrubMode)(0), // 0: volume_server_pb.VolumeScrubMode
- (*VolumeServerState)(nil), // 1: volume_server_pb.VolumeServerState
- (*BatchDeleteRequest)(nil), // 2: volume_server_pb.BatchDeleteRequest
- (*BatchDeleteResponse)(nil), // 3: volume_server_pb.BatchDeleteResponse
- (*DeleteResult)(nil), // 4: volume_server_pb.DeleteResult
- (*Empty)(nil), // 5: volume_server_pb.Empty
- (*VacuumVolumeCheckRequest)(nil), // 6: volume_server_pb.VacuumVolumeCheckRequest
- (*VacuumVolumeCheckResponse)(nil), // 7: volume_server_pb.VacuumVolumeCheckResponse
- (*VacuumVolumeCompactRequest)(nil), // 8: volume_server_pb.VacuumVolumeCompactRequest
- (*VacuumVolumeCompactResponse)(nil), // 9: volume_server_pb.VacuumVolumeCompactResponse
- (*VacuumVolumeCommitRequest)(nil), // 10: volume_server_pb.VacuumVolumeCommitRequest
- (*VacuumVolumeCommitResponse)(nil), // 11: volume_server_pb.VacuumVolumeCommitResponse
- (*VacuumVolumeCleanupRequest)(nil), // 12: volume_server_pb.VacuumVolumeCleanupRequest
- (*VacuumVolumeCleanupResponse)(nil), // 13: volume_server_pb.VacuumVolumeCleanupResponse
- (*DeleteCollectionRequest)(nil), // 14: volume_server_pb.DeleteCollectionRequest
- (*DeleteCollectionResponse)(nil), // 15: volume_server_pb.DeleteCollectionResponse
- (*AllocateVolumeRequest)(nil), // 16: volume_server_pb.AllocateVolumeRequest
- (*AllocateVolumeResponse)(nil), // 17: volume_server_pb.AllocateVolumeResponse
- (*VolumeSyncStatusRequest)(nil), // 18: volume_server_pb.VolumeSyncStatusRequest
- (*VolumeSyncStatusResponse)(nil), // 19: volume_server_pb.VolumeSyncStatusResponse
- (*VolumeIncrementalCopyRequest)(nil), // 20: volume_server_pb.VolumeIncrementalCopyRequest
- (*VolumeIncrementalCopyResponse)(nil), // 21: volume_server_pb.VolumeIncrementalCopyResponse
- (*VolumeMountRequest)(nil), // 22: volume_server_pb.VolumeMountRequest
- (*VolumeMountResponse)(nil), // 23: volume_server_pb.VolumeMountResponse
- (*VolumeUnmountRequest)(nil), // 24: volume_server_pb.VolumeUnmountRequest
- (*VolumeUnmountResponse)(nil), // 25: volume_server_pb.VolumeUnmountResponse
- (*VolumeDeleteRequest)(nil), // 26: volume_server_pb.VolumeDeleteRequest
- (*VolumeDeleteResponse)(nil), // 27: volume_server_pb.VolumeDeleteResponse
- (*VolumeMarkReadonlyRequest)(nil), // 28: volume_server_pb.VolumeMarkReadonlyRequest
- (*VolumeMarkReadonlyResponse)(nil), // 29: volume_server_pb.VolumeMarkReadonlyResponse
- (*VolumeMarkWritableRequest)(nil), // 30: volume_server_pb.VolumeMarkWritableRequest
- (*VolumeMarkWritableResponse)(nil), // 31: volume_server_pb.VolumeMarkWritableResponse
- (*VolumeConfigureRequest)(nil), // 32: volume_server_pb.VolumeConfigureRequest
- (*VolumeConfigureResponse)(nil), // 33: volume_server_pb.VolumeConfigureResponse
- (*VolumeStatusRequest)(nil), // 34: volume_server_pb.VolumeStatusRequest
- (*VolumeStatusResponse)(nil), // 35: volume_server_pb.VolumeStatusResponse
- (*GetStateRequest)(nil), // 36: volume_server_pb.GetStateRequest
- (*GetStateResponse)(nil), // 37: volume_server_pb.GetStateResponse
- (*SetStateRequest)(nil), // 38: volume_server_pb.SetStateRequest
- (*SetStateResponse)(nil), // 39: volume_server_pb.SetStateResponse
- (*VolumeCopyRequest)(nil), // 40: volume_server_pb.VolumeCopyRequest
- (*VolumeCopyResponse)(nil), // 41: volume_server_pb.VolumeCopyResponse
- (*CopyFileRequest)(nil), // 42: volume_server_pb.CopyFileRequest
- (*CopyFileResponse)(nil), // 43: volume_server_pb.CopyFileResponse
- (*ReceiveFileRequest)(nil), // 44: volume_server_pb.ReceiveFileRequest
- (*ReceiveFileInfo)(nil), // 45: volume_server_pb.ReceiveFileInfo
- (*ReceiveFileResponse)(nil), // 46: volume_server_pb.ReceiveFileResponse
- (*ReadNeedleBlobRequest)(nil), // 47: volume_server_pb.ReadNeedleBlobRequest
- (*ReadNeedleBlobResponse)(nil), // 48: volume_server_pb.ReadNeedleBlobResponse
- (*ReadNeedleMetaRequest)(nil), // 49: volume_server_pb.ReadNeedleMetaRequest
- (*ReadNeedleMetaResponse)(nil), // 50: volume_server_pb.ReadNeedleMetaResponse
- (*WriteNeedleBlobRequest)(nil), // 51: volume_server_pb.WriteNeedleBlobRequest
- (*WriteNeedleBlobResponse)(nil), // 52: volume_server_pb.WriteNeedleBlobResponse
- (*ReadAllNeedlesRequest)(nil), // 53: volume_server_pb.ReadAllNeedlesRequest
- (*ReadAllNeedlesResponse)(nil), // 54: volume_server_pb.ReadAllNeedlesResponse
- (*VolumeTailSenderRequest)(nil), // 55: volume_server_pb.VolumeTailSenderRequest
- (*VolumeTailSenderResponse)(nil), // 56: volume_server_pb.VolumeTailSenderResponse
- (*VolumeTailReceiverRequest)(nil), // 57: volume_server_pb.VolumeTailReceiverRequest
- (*VolumeTailReceiverResponse)(nil), // 58: volume_server_pb.VolumeTailReceiverResponse
- (*VolumeEcShardsGenerateRequest)(nil), // 59: volume_server_pb.VolumeEcShardsGenerateRequest
- (*VolumeEcShardsGenerateResponse)(nil), // 60: volume_server_pb.VolumeEcShardsGenerateResponse
- (*VolumeEcShardsRebuildRequest)(nil), // 61: volume_server_pb.VolumeEcShardsRebuildRequest
- (*VolumeEcShardsRebuildResponse)(nil), // 62: volume_server_pb.VolumeEcShardsRebuildResponse
- (*VolumeEcShardsCopyRequest)(nil), // 63: volume_server_pb.VolumeEcShardsCopyRequest
- (*VolumeEcShardsCopyResponse)(nil), // 64: volume_server_pb.VolumeEcShardsCopyResponse
- (*VolumeEcShardsDeleteRequest)(nil), // 65: volume_server_pb.VolumeEcShardsDeleteRequest
- (*VolumeEcShardsDeleteResponse)(nil), // 66: volume_server_pb.VolumeEcShardsDeleteResponse
- (*VolumeEcShardsMountRequest)(nil), // 67: volume_server_pb.VolumeEcShardsMountRequest
- (*VolumeEcShardsMountResponse)(nil), // 68: volume_server_pb.VolumeEcShardsMountResponse
- (*VolumeEcShardsUnmountRequest)(nil), // 69: volume_server_pb.VolumeEcShardsUnmountRequest
- (*VolumeEcShardsUnmountResponse)(nil), // 70: volume_server_pb.VolumeEcShardsUnmountResponse
- (*VolumeEcShardReadRequest)(nil), // 71: volume_server_pb.VolumeEcShardReadRequest
- (*VolumeEcShardReadResponse)(nil), // 72: volume_server_pb.VolumeEcShardReadResponse
- (*VolumeEcBlobDeleteRequest)(nil), // 73: volume_server_pb.VolumeEcBlobDeleteRequest
- (*VolumeEcBlobDeleteResponse)(nil), // 74: volume_server_pb.VolumeEcBlobDeleteResponse
- (*VolumeEcShardsToVolumeRequest)(nil), // 75: volume_server_pb.VolumeEcShardsToVolumeRequest
- (*VolumeEcShardsToVolumeResponse)(nil), // 76: volume_server_pb.VolumeEcShardsToVolumeResponse
- (*VolumeEcShardsInfoRequest)(nil), // 77: volume_server_pb.VolumeEcShardsInfoRequest
- (*VolumeEcShardsInfoResponse)(nil), // 78: volume_server_pb.VolumeEcShardsInfoResponse
- (*EcShardInfo)(nil), // 79: volume_server_pb.EcShardInfo
- (*ReadVolumeFileStatusRequest)(nil), // 80: volume_server_pb.ReadVolumeFileStatusRequest
- (*ReadVolumeFileStatusResponse)(nil), // 81: volume_server_pb.ReadVolumeFileStatusResponse
- (*DiskStatus)(nil), // 82: volume_server_pb.DiskStatus
- (*MemStatus)(nil), // 83: volume_server_pb.MemStatus
- (*RemoteFile)(nil), // 84: volume_server_pb.RemoteFile
- (*VolumeInfo)(nil), // 85: volume_server_pb.VolumeInfo
- (*EcShardConfig)(nil), // 86: volume_server_pb.EcShardConfig
- (*OldVersionVolumeInfo)(nil), // 87: volume_server_pb.OldVersionVolumeInfo
- (*VolumeTierMoveDatToRemoteRequest)(nil), // 88: volume_server_pb.VolumeTierMoveDatToRemoteRequest
- (*VolumeTierMoveDatToRemoteResponse)(nil), // 89: volume_server_pb.VolumeTierMoveDatToRemoteResponse
- (*VolumeTierMoveDatFromRemoteRequest)(nil), // 90: volume_server_pb.VolumeTierMoveDatFromRemoteRequest
- (*VolumeTierMoveDatFromRemoteResponse)(nil), // 91: volume_server_pb.VolumeTierMoveDatFromRemoteResponse
- (*VolumeServerStatusRequest)(nil), // 92: volume_server_pb.VolumeServerStatusRequest
- (*VolumeServerStatusResponse)(nil), // 93: volume_server_pb.VolumeServerStatusResponse
- (*VolumeServerLeaveRequest)(nil), // 94: volume_server_pb.VolumeServerLeaveRequest
- (*VolumeServerLeaveResponse)(nil), // 95: volume_server_pb.VolumeServerLeaveResponse
- (*FetchAndWriteNeedleRequest)(nil), // 96: volume_server_pb.FetchAndWriteNeedleRequest
- (*FetchAndWriteNeedleResponse)(nil), // 97: volume_server_pb.FetchAndWriteNeedleResponse
- (*ScrubVolumeRequest)(nil), // 98: volume_server_pb.ScrubVolumeRequest
- (*ScrubVolumeResponse)(nil), // 99: volume_server_pb.ScrubVolumeResponse
- (*ScrubEcVolumeRequest)(nil), // 100: volume_server_pb.ScrubEcVolumeRequest
- (*ScrubEcVolumeResponse)(nil), // 101: volume_server_pb.ScrubEcVolumeResponse
- (*QueryRequest)(nil), // 102: volume_server_pb.QueryRequest
- (*QueriedStripe)(nil), // 103: volume_server_pb.QueriedStripe
- (*VolumeNeedleStatusRequest)(nil), // 104: volume_server_pb.VolumeNeedleStatusRequest
- (*VolumeNeedleStatusResponse)(nil), // 105: volume_server_pb.VolumeNeedleStatusResponse
- (*PingRequest)(nil), // 106: volume_server_pb.PingRequest
- (*PingResponse)(nil), // 107: volume_server_pb.PingResponse
- (*FetchAndWriteNeedleRequest_Replica)(nil), // 108: volume_server_pb.FetchAndWriteNeedleRequest.Replica
- (*QueryRequest_Filter)(nil), // 109: volume_server_pb.QueryRequest.Filter
- (*QueryRequest_InputSerialization)(nil), // 110: volume_server_pb.QueryRequest.InputSerialization
- (*QueryRequest_OutputSerialization)(nil), // 111: volume_server_pb.QueryRequest.OutputSerialization
- (*QueryRequest_InputSerialization_CSVInput)(nil), // 112: volume_server_pb.QueryRequest.InputSerialization.CSVInput
- (*QueryRequest_InputSerialization_JSONInput)(nil), // 113: volume_server_pb.QueryRequest.InputSerialization.JSONInput
- (*QueryRequest_InputSerialization_ParquetInput)(nil), // 114: volume_server_pb.QueryRequest.InputSerialization.ParquetInput
- (*QueryRequest_OutputSerialization_CSVOutput)(nil), // 115: volume_server_pb.QueryRequest.OutputSerialization.CSVOutput
- (*QueryRequest_OutputSerialization_JSONOutput)(nil), // 116: volume_server_pb.QueryRequest.OutputSerialization.JSONOutput
- (*remote_pb.RemoteConf)(nil), // 117: remote_pb.RemoteConf
- (*remote_pb.RemoteStorageLocation)(nil), // 118: remote_pb.RemoteStorageLocation
+ (ChecksumAlgorithm)(0), // 0: volume_server_pb.ChecksumAlgorithm
+ (VolumeScrubMode)(0), // 1: volume_server_pb.VolumeScrubMode
+ (*VolumeServerState)(nil), // 2: volume_server_pb.VolumeServerState
+ (*BatchDeleteRequest)(nil), // 3: volume_server_pb.BatchDeleteRequest
+ (*BatchDeleteResponse)(nil), // 4: volume_server_pb.BatchDeleteResponse
+ (*DeleteResult)(nil), // 5: volume_server_pb.DeleteResult
+ (*Empty)(nil), // 6: volume_server_pb.Empty
+ (*VacuumVolumeCheckRequest)(nil), // 7: volume_server_pb.VacuumVolumeCheckRequest
+ (*VacuumVolumeCheckResponse)(nil), // 8: volume_server_pb.VacuumVolumeCheckResponse
+ (*VacuumVolumeCompactRequest)(nil), // 9: volume_server_pb.VacuumVolumeCompactRequest
+ (*VacuumVolumeCompactResponse)(nil), // 10: volume_server_pb.VacuumVolumeCompactResponse
+ (*VacuumVolumeCommitRequest)(nil), // 11: volume_server_pb.VacuumVolumeCommitRequest
+ (*VacuumVolumeCommitResponse)(nil), // 12: volume_server_pb.VacuumVolumeCommitResponse
+ (*VacuumVolumeCleanupRequest)(nil), // 13: volume_server_pb.VacuumVolumeCleanupRequest
+ (*VacuumVolumeCleanupResponse)(nil), // 14: volume_server_pb.VacuumVolumeCleanupResponse
+ (*DeleteCollectionRequest)(nil), // 15: volume_server_pb.DeleteCollectionRequest
+ (*DeleteCollectionResponse)(nil), // 16: volume_server_pb.DeleteCollectionResponse
+ (*AllocateVolumeRequest)(nil), // 17: volume_server_pb.AllocateVolumeRequest
+ (*AllocateVolumeResponse)(nil), // 18: volume_server_pb.AllocateVolumeResponse
+ (*VolumeSyncStatusRequest)(nil), // 19: volume_server_pb.VolumeSyncStatusRequest
+ (*VolumeSyncStatusResponse)(nil), // 20: volume_server_pb.VolumeSyncStatusResponse
+ (*VolumeIncrementalCopyRequest)(nil), // 21: volume_server_pb.VolumeIncrementalCopyRequest
+ (*VolumeIncrementalCopyResponse)(nil), // 22: volume_server_pb.VolumeIncrementalCopyResponse
+ (*VolumeMountRequest)(nil), // 23: volume_server_pb.VolumeMountRequest
+ (*VolumeMountResponse)(nil), // 24: volume_server_pb.VolumeMountResponse
+ (*VolumeUnmountRequest)(nil), // 25: volume_server_pb.VolumeUnmountRequest
+ (*VolumeUnmountResponse)(nil), // 26: volume_server_pb.VolumeUnmountResponse
+ (*VolumeDeleteRequest)(nil), // 27: volume_server_pb.VolumeDeleteRequest
+ (*VolumeDeleteResponse)(nil), // 28: volume_server_pb.VolumeDeleteResponse
+ (*VolumeMarkReadonlyRequest)(nil), // 29: volume_server_pb.VolumeMarkReadonlyRequest
+ (*VolumeMarkReadonlyResponse)(nil), // 30: volume_server_pb.VolumeMarkReadonlyResponse
+ (*VolumeMarkWritableRequest)(nil), // 31: volume_server_pb.VolumeMarkWritableRequest
+ (*VolumeMarkWritableResponse)(nil), // 32: volume_server_pb.VolumeMarkWritableResponse
+ (*VolumeConfigureRequest)(nil), // 33: volume_server_pb.VolumeConfigureRequest
+ (*VolumeConfigureResponse)(nil), // 34: volume_server_pb.VolumeConfigureResponse
+ (*VolumeStatusRequest)(nil), // 35: volume_server_pb.VolumeStatusRequest
+ (*VolumeStatusResponse)(nil), // 36: volume_server_pb.VolumeStatusResponse
+ (*GetStateRequest)(nil), // 37: volume_server_pb.GetStateRequest
+ (*GetStateResponse)(nil), // 38: volume_server_pb.GetStateResponse
+ (*SetStateRequest)(nil), // 39: volume_server_pb.SetStateRequest
+ (*SetStateResponse)(nil), // 40: volume_server_pb.SetStateResponse
+ (*VolumeCopyRequest)(nil), // 41: volume_server_pb.VolumeCopyRequest
+ (*VolumeCopyResponse)(nil), // 42: volume_server_pb.VolumeCopyResponse
+ (*CopyFileRequest)(nil), // 43: volume_server_pb.CopyFileRequest
+ (*CopyFileResponse)(nil), // 44: volume_server_pb.CopyFileResponse
+ (*ReceiveFileRequest)(nil), // 45: volume_server_pb.ReceiveFileRequest
+ (*ReceiveFileInfo)(nil), // 46: volume_server_pb.ReceiveFileInfo
+ (*ReceiveFileResponse)(nil), // 47: volume_server_pb.ReceiveFileResponse
+ (*ReadNeedleBlobRequest)(nil), // 48: volume_server_pb.ReadNeedleBlobRequest
+ (*ReadNeedleBlobResponse)(nil), // 49: volume_server_pb.ReadNeedleBlobResponse
+ (*ReadNeedleMetaRequest)(nil), // 50: volume_server_pb.ReadNeedleMetaRequest
+ (*ReadNeedleMetaResponse)(nil), // 51: volume_server_pb.ReadNeedleMetaResponse
+ (*WriteNeedleBlobRequest)(nil), // 52: volume_server_pb.WriteNeedleBlobRequest
+ (*WriteNeedleBlobResponse)(nil), // 53: volume_server_pb.WriteNeedleBlobResponse
+ (*ReadAllNeedlesRequest)(nil), // 54: volume_server_pb.ReadAllNeedlesRequest
+ (*ReadAllNeedlesResponse)(nil), // 55: volume_server_pb.ReadAllNeedlesResponse
+ (*VolumeTailSenderRequest)(nil), // 56: volume_server_pb.VolumeTailSenderRequest
+ (*VolumeTailSenderResponse)(nil), // 57: volume_server_pb.VolumeTailSenderResponse
+ (*VolumeTailReceiverRequest)(nil), // 58: volume_server_pb.VolumeTailReceiverRequest
+ (*VolumeTailReceiverResponse)(nil), // 59: volume_server_pb.VolumeTailReceiverResponse
+ (*VolumeEcShardsGenerateRequest)(nil), // 60: volume_server_pb.VolumeEcShardsGenerateRequest
+ (*VolumeEcShardsGenerateResponse)(nil), // 61: volume_server_pb.VolumeEcShardsGenerateResponse
+ (*VolumeEcShardsRebuildRequest)(nil), // 62: volume_server_pb.VolumeEcShardsRebuildRequest
+ (*VolumeEcShardsRebuildResponse)(nil), // 63: volume_server_pb.VolumeEcShardsRebuildResponse
+ (*VolumeEcShardsCopyRequest)(nil), // 64: volume_server_pb.VolumeEcShardsCopyRequest
+ (*VolumeEcShardsCopyResponse)(nil), // 65: volume_server_pb.VolumeEcShardsCopyResponse
+ (*VolumeEcShardsDeleteRequest)(nil), // 66: volume_server_pb.VolumeEcShardsDeleteRequest
+ (*VolumeEcShardsDeleteResponse)(nil), // 67: volume_server_pb.VolumeEcShardsDeleteResponse
+ (*VolumeEcShardsMountRequest)(nil), // 68: volume_server_pb.VolumeEcShardsMountRequest
+ (*VolumeEcShardsMountResponse)(nil), // 69: volume_server_pb.VolumeEcShardsMountResponse
+ (*VolumeEcShardsUnmountRequest)(nil), // 70: volume_server_pb.VolumeEcShardsUnmountRequest
+ (*VolumeEcShardsUnmountResponse)(nil), // 71: volume_server_pb.VolumeEcShardsUnmountResponse
+ (*VolumeEcShardReadRequest)(nil), // 72: volume_server_pb.VolumeEcShardReadRequest
+ (*VolumeEcShardReadResponse)(nil), // 73: volume_server_pb.VolumeEcShardReadResponse
+ (*VolumeEcBlobDeleteRequest)(nil), // 74: volume_server_pb.VolumeEcBlobDeleteRequest
+ (*VolumeEcBlobDeleteResponse)(nil), // 75: volume_server_pb.VolumeEcBlobDeleteResponse
+ (*VolumeEcShardsToVolumeRequest)(nil), // 76: volume_server_pb.VolumeEcShardsToVolumeRequest
+ (*VolumeEcShardsToVolumeResponse)(nil), // 77: volume_server_pb.VolumeEcShardsToVolumeResponse
+ (*VolumeEcShardsInfoRequest)(nil), // 78: volume_server_pb.VolumeEcShardsInfoRequest
+ (*VolumeEcShardsInfoResponse)(nil), // 79: volume_server_pb.VolumeEcShardsInfoResponse
+ (*EcShardInfo)(nil), // 80: volume_server_pb.EcShardInfo
+ (*ReadVolumeFileStatusRequest)(nil), // 81: volume_server_pb.ReadVolumeFileStatusRequest
+ (*ReadVolumeFileStatusResponse)(nil), // 82: volume_server_pb.ReadVolumeFileStatusResponse
+ (*DiskStatus)(nil), // 83: volume_server_pb.DiskStatus
+ (*MemStatus)(nil), // 84: volume_server_pb.MemStatus
+ (*RemoteFile)(nil), // 85: volume_server_pb.RemoteFile
+ (*VolumeInfo)(nil), // 86: volume_server_pb.VolumeInfo
+ (*EcShardConfig)(nil), // 87: volume_server_pb.EcShardConfig
+ (*EcBitrotProtection)(nil), // 88: volume_server_pb.EcBitrotProtection
+ (*EcShardChecksums)(nil), // 89: volume_server_pb.EcShardChecksums
+ (*OldVersionVolumeInfo)(nil), // 90: volume_server_pb.OldVersionVolumeInfo
+ (*VolumeTierMoveDatToRemoteRequest)(nil), // 91: volume_server_pb.VolumeTierMoveDatToRemoteRequest
+ (*VolumeTierMoveDatToRemoteResponse)(nil), // 92: volume_server_pb.VolumeTierMoveDatToRemoteResponse
+ (*VolumeTierMoveDatFromRemoteRequest)(nil), // 93: volume_server_pb.VolumeTierMoveDatFromRemoteRequest
+ (*VolumeTierMoveDatFromRemoteResponse)(nil), // 94: volume_server_pb.VolumeTierMoveDatFromRemoteResponse
+ (*VolumeServerStatusRequest)(nil), // 95: volume_server_pb.VolumeServerStatusRequest
+ (*VolumeServerStatusResponse)(nil), // 96: volume_server_pb.VolumeServerStatusResponse
+ (*VolumeServerLeaveRequest)(nil), // 97: volume_server_pb.VolumeServerLeaveRequest
+ (*VolumeServerLeaveResponse)(nil), // 98: volume_server_pb.VolumeServerLeaveResponse
+ (*FetchAndWriteNeedleRequest)(nil), // 99: volume_server_pb.FetchAndWriteNeedleRequest
+ (*FetchAndWriteNeedleResponse)(nil), // 100: volume_server_pb.FetchAndWriteNeedleResponse
+ (*ScrubVolumeRequest)(nil), // 101: volume_server_pb.ScrubVolumeRequest
+ (*ScrubVolumeResponse)(nil), // 102: volume_server_pb.ScrubVolumeResponse
+ (*ScrubEcVolumeRequest)(nil), // 103: volume_server_pb.ScrubEcVolumeRequest
+ (*ScrubEcVolumeResponse)(nil), // 104: volume_server_pb.ScrubEcVolumeResponse
+ (*QueryRequest)(nil), // 105: volume_server_pb.QueryRequest
+ (*QueriedStripe)(nil), // 106: volume_server_pb.QueriedStripe
+ (*VolumeNeedleStatusRequest)(nil), // 107: volume_server_pb.VolumeNeedleStatusRequest
+ (*VolumeNeedleStatusResponse)(nil), // 108: volume_server_pb.VolumeNeedleStatusResponse
+ (*PingRequest)(nil), // 109: volume_server_pb.PingRequest
+ (*PingResponse)(nil), // 110: volume_server_pb.PingResponse
+ (*FetchAndWriteNeedleRequest_Replica)(nil), // 111: volume_server_pb.FetchAndWriteNeedleRequest.Replica
+ (*QueryRequest_Filter)(nil), // 112: volume_server_pb.QueryRequest.Filter
+ (*QueryRequest_InputSerialization)(nil), // 113: volume_server_pb.QueryRequest.InputSerialization
+ (*QueryRequest_OutputSerialization)(nil), // 114: volume_server_pb.QueryRequest.OutputSerialization
+ (*QueryRequest_InputSerialization_CSVInput)(nil), // 115: volume_server_pb.QueryRequest.InputSerialization.CSVInput
+ (*QueryRequest_InputSerialization_JSONInput)(nil), // 116: volume_server_pb.QueryRequest.InputSerialization.JSONInput
+ (*QueryRequest_InputSerialization_ParquetInput)(nil), // 117: volume_server_pb.QueryRequest.InputSerialization.ParquetInput
+ (*QueryRequest_OutputSerialization_CSVOutput)(nil), // 118: volume_server_pb.QueryRequest.OutputSerialization.CSVOutput
+ (*QueryRequest_OutputSerialization_JSONOutput)(nil), // 119: volume_server_pb.QueryRequest.OutputSerialization.JSONOutput
+ (*remote_pb.RemoteConf)(nil), // 120: remote_pb.RemoteConf
+ (*remote_pb.RemoteStorageLocation)(nil), // 121: remote_pb.RemoteStorageLocation
}
var file_volume_server_proto_depIdxs = []int32{
- 4, // 0: volume_server_pb.BatchDeleteResponse.results:type_name -> volume_server_pb.DeleteResult
- 1, // 1: volume_server_pb.GetStateResponse.state:type_name -> volume_server_pb.VolumeServerState
- 1, // 2: volume_server_pb.SetStateRequest.state:type_name -> volume_server_pb.VolumeServerState
- 1, // 3: volume_server_pb.SetStateResponse.state:type_name -> volume_server_pb.VolumeServerState
- 45, // 4: volume_server_pb.ReceiveFileRequest.info:type_name -> volume_server_pb.ReceiveFileInfo
- 79, // 5: volume_server_pb.VolumeEcShardsInfoResponse.ec_shard_infos:type_name -> volume_server_pb.EcShardInfo
- 85, // 6: volume_server_pb.ReadVolumeFileStatusResponse.volume_info:type_name -> volume_server_pb.VolumeInfo
- 84, // 7: volume_server_pb.VolumeInfo.files:type_name -> volume_server_pb.RemoteFile
- 86, // 8: volume_server_pb.VolumeInfo.ec_shard_config:type_name -> volume_server_pb.EcShardConfig
- 84, // 9: volume_server_pb.OldVersionVolumeInfo.files:type_name -> volume_server_pb.RemoteFile
- 82, // 10: volume_server_pb.VolumeServerStatusResponse.disk_statuses:type_name -> volume_server_pb.DiskStatus
- 83, // 11: volume_server_pb.VolumeServerStatusResponse.memory_status:type_name -> volume_server_pb.MemStatus
- 1, // 12: volume_server_pb.VolumeServerStatusResponse.state:type_name -> volume_server_pb.VolumeServerState
- 108, // 13: volume_server_pb.FetchAndWriteNeedleRequest.replicas:type_name -> volume_server_pb.FetchAndWriteNeedleRequest.Replica
- 117, // 14: volume_server_pb.FetchAndWriteNeedleRequest.remote_conf:type_name -> remote_pb.RemoteConf
- 118, // 15: volume_server_pb.FetchAndWriteNeedleRequest.remote_location:type_name -> remote_pb.RemoteStorageLocation
- 0, // 16: volume_server_pb.ScrubVolumeRequest.mode:type_name -> volume_server_pb.VolumeScrubMode
- 0, // 17: volume_server_pb.ScrubEcVolumeRequest.mode:type_name -> volume_server_pb.VolumeScrubMode
- 79, // 18: volume_server_pb.ScrubEcVolumeResponse.broken_shard_infos:type_name -> volume_server_pb.EcShardInfo
- 109, // 19: volume_server_pb.QueryRequest.filter:type_name -> volume_server_pb.QueryRequest.Filter
- 110, // 20: volume_server_pb.QueryRequest.input_serialization:type_name -> volume_server_pb.QueryRequest.InputSerialization
- 111, // 21: volume_server_pb.QueryRequest.output_serialization:type_name -> volume_server_pb.QueryRequest.OutputSerialization
- 112, // 22: volume_server_pb.QueryRequest.InputSerialization.csv_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.CSVInput
- 113, // 23: volume_server_pb.QueryRequest.InputSerialization.json_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.JSONInput
- 114, // 24: volume_server_pb.QueryRequest.InputSerialization.parquet_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.ParquetInput
- 115, // 25: volume_server_pb.QueryRequest.OutputSerialization.csv_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.CSVOutput
- 116, // 26: volume_server_pb.QueryRequest.OutputSerialization.json_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.JSONOutput
- 2, // 27: volume_server_pb.VolumeServer.BatchDelete:input_type -> volume_server_pb.BatchDeleteRequest
- 6, // 28: volume_server_pb.VolumeServer.VacuumVolumeCheck:input_type -> volume_server_pb.VacuumVolumeCheckRequest
- 8, // 29: volume_server_pb.VolumeServer.VacuumVolumeCompact:input_type -> volume_server_pb.VacuumVolumeCompactRequest
- 10, // 30: volume_server_pb.VolumeServer.VacuumVolumeCommit:input_type -> volume_server_pb.VacuumVolumeCommitRequest
- 12, // 31: volume_server_pb.VolumeServer.VacuumVolumeCleanup:input_type -> volume_server_pb.VacuumVolumeCleanupRequest
- 14, // 32: volume_server_pb.VolumeServer.DeleteCollection:input_type -> volume_server_pb.DeleteCollectionRequest
- 16, // 33: volume_server_pb.VolumeServer.AllocateVolume:input_type -> volume_server_pb.AllocateVolumeRequest
- 18, // 34: volume_server_pb.VolumeServer.VolumeSyncStatus:input_type -> volume_server_pb.VolumeSyncStatusRequest
- 20, // 35: volume_server_pb.VolumeServer.VolumeIncrementalCopy:input_type -> volume_server_pb.VolumeIncrementalCopyRequest
- 22, // 36: volume_server_pb.VolumeServer.VolumeMount:input_type -> volume_server_pb.VolumeMountRequest
- 24, // 37: volume_server_pb.VolumeServer.VolumeUnmount:input_type -> volume_server_pb.VolumeUnmountRequest
- 26, // 38: volume_server_pb.VolumeServer.VolumeDelete:input_type -> volume_server_pb.VolumeDeleteRequest
- 28, // 39: volume_server_pb.VolumeServer.VolumeMarkReadonly:input_type -> volume_server_pb.VolumeMarkReadonlyRequest
- 30, // 40: volume_server_pb.VolumeServer.VolumeMarkWritable:input_type -> volume_server_pb.VolumeMarkWritableRequest
- 32, // 41: volume_server_pb.VolumeServer.VolumeConfigure:input_type -> volume_server_pb.VolumeConfigureRequest
- 34, // 42: volume_server_pb.VolumeServer.VolumeStatus:input_type -> volume_server_pb.VolumeStatusRequest
- 36, // 43: volume_server_pb.VolumeServer.GetState:input_type -> volume_server_pb.GetStateRequest
- 38, // 44: volume_server_pb.VolumeServer.SetState:input_type -> volume_server_pb.SetStateRequest
- 40, // 45: volume_server_pb.VolumeServer.VolumeCopy:input_type -> volume_server_pb.VolumeCopyRequest
- 80, // 46: volume_server_pb.VolumeServer.ReadVolumeFileStatus:input_type -> volume_server_pb.ReadVolumeFileStatusRequest
- 42, // 47: volume_server_pb.VolumeServer.CopyFile:input_type -> volume_server_pb.CopyFileRequest
- 44, // 48: volume_server_pb.VolumeServer.ReceiveFile:input_type -> volume_server_pb.ReceiveFileRequest
- 47, // 49: volume_server_pb.VolumeServer.ReadNeedleBlob:input_type -> volume_server_pb.ReadNeedleBlobRequest
- 49, // 50: volume_server_pb.VolumeServer.ReadNeedleMeta:input_type -> volume_server_pb.ReadNeedleMetaRequest
- 51, // 51: volume_server_pb.VolumeServer.WriteNeedleBlob:input_type -> volume_server_pb.WriteNeedleBlobRequest
- 53, // 52: volume_server_pb.VolumeServer.ReadAllNeedles:input_type -> volume_server_pb.ReadAllNeedlesRequest
- 55, // 53: volume_server_pb.VolumeServer.VolumeTailSender:input_type -> volume_server_pb.VolumeTailSenderRequest
- 57, // 54: volume_server_pb.VolumeServer.VolumeTailReceiver:input_type -> volume_server_pb.VolumeTailReceiverRequest
- 59, // 55: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:input_type -> volume_server_pb.VolumeEcShardsGenerateRequest
- 61, // 56: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:input_type -> volume_server_pb.VolumeEcShardsRebuildRequest
- 63, // 57: volume_server_pb.VolumeServer.VolumeEcShardsCopy:input_type -> volume_server_pb.VolumeEcShardsCopyRequest
- 65, // 58: volume_server_pb.VolumeServer.VolumeEcShardsDelete:input_type -> volume_server_pb.VolumeEcShardsDeleteRequest
- 67, // 59: volume_server_pb.VolumeServer.VolumeEcShardsMount:input_type -> volume_server_pb.VolumeEcShardsMountRequest
- 69, // 60: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:input_type -> volume_server_pb.VolumeEcShardsUnmountRequest
- 71, // 61: volume_server_pb.VolumeServer.VolumeEcShardRead:input_type -> volume_server_pb.VolumeEcShardReadRequest
- 73, // 62: volume_server_pb.VolumeServer.VolumeEcBlobDelete:input_type -> volume_server_pb.VolumeEcBlobDeleteRequest
- 75, // 63: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:input_type -> volume_server_pb.VolumeEcShardsToVolumeRequest
- 77, // 64: volume_server_pb.VolumeServer.VolumeEcShardsInfo:input_type -> volume_server_pb.VolumeEcShardsInfoRequest
- 88, // 65: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:input_type -> volume_server_pb.VolumeTierMoveDatToRemoteRequest
- 90, // 66: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:input_type -> volume_server_pb.VolumeTierMoveDatFromRemoteRequest
- 92, // 67: volume_server_pb.VolumeServer.VolumeServerStatus:input_type -> volume_server_pb.VolumeServerStatusRequest
- 94, // 68: volume_server_pb.VolumeServer.VolumeServerLeave:input_type -> volume_server_pb.VolumeServerLeaveRequest
- 96, // 69: volume_server_pb.VolumeServer.FetchAndWriteNeedle:input_type -> volume_server_pb.FetchAndWriteNeedleRequest
- 98, // 70: volume_server_pb.VolumeServer.ScrubVolume:input_type -> volume_server_pb.ScrubVolumeRequest
- 100, // 71: volume_server_pb.VolumeServer.ScrubEcVolume:input_type -> volume_server_pb.ScrubEcVolumeRequest
- 102, // 72: volume_server_pb.VolumeServer.Query:input_type -> volume_server_pb.QueryRequest
- 104, // 73: volume_server_pb.VolumeServer.VolumeNeedleStatus:input_type -> volume_server_pb.VolumeNeedleStatusRequest
- 106, // 74: volume_server_pb.VolumeServer.Ping:input_type -> volume_server_pb.PingRequest
- 3, // 75: volume_server_pb.VolumeServer.BatchDelete:output_type -> volume_server_pb.BatchDeleteResponse
- 7, // 76: volume_server_pb.VolumeServer.VacuumVolumeCheck:output_type -> volume_server_pb.VacuumVolumeCheckResponse
- 9, // 77: volume_server_pb.VolumeServer.VacuumVolumeCompact:output_type -> volume_server_pb.VacuumVolumeCompactResponse
- 11, // 78: volume_server_pb.VolumeServer.VacuumVolumeCommit:output_type -> volume_server_pb.VacuumVolumeCommitResponse
- 13, // 79: volume_server_pb.VolumeServer.VacuumVolumeCleanup:output_type -> volume_server_pb.VacuumVolumeCleanupResponse
- 15, // 80: volume_server_pb.VolumeServer.DeleteCollection:output_type -> volume_server_pb.DeleteCollectionResponse
- 17, // 81: volume_server_pb.VolumeServer.AllocateVolume:output_type -> volume_server_pb.AllocateVolumeResponse
- 19, // 82: volume_server_pb.VolumeServer.VolumeSyncStatus:output_type -> volume_server_pb.VolumeSyncStatusResponse
- 21, // 83: volume_server_pb.VolumeServer.VolumeIncrementalCopy:output_type -> volume_server_pb.VolumeIncrementalCopyResponse
- 23, // 84: volume_server_pb.VolumeServer.VolumeMount:output_type -> volume_server_pb.VolumeMountResponse
- 25, // 85: volume_server_pb.VolumeServer.VolumeUnmount:output_type -> volume_server_pb.VolumeUnmountResponse
- 27, // 86: volume_server_pb.VolumeServer.VolumeDelete:output_type -> volume_server_pb.VolumeDeleteResponse
- 29, // 87: volume_server_pb.VolumeServer.VolumeMarkReadonly:output_type -> volume_server_pb.VolumeMarkReadonlyResponse
- 31, // 88: volume_server_pb.VolumeServer.VolumeMarkWritable:output_type -> volume_server_pb.VolumeMarkWritableResponse
- 33, // 89: volume_server_pb.VolumeServer.VolumeConfigure:output_type -> volume_server_pb.VolumeConfigureResponse
- 35, // 90: volume_server_pb.VolumeServer.VolumeStatus:output_type -> volume_server_pb.VolumeStatusResponse
- 37, // 91: volume_server_pb.VolumeServer.GetState:output_type -> volume_server_pb.GetStateResponse
- 39, // 92: volume_server_pb.VolumeServer.SetState:output_type -> volume_server_pb.SetStateResponse
- 41, // 93: volume_server_pb.VolumeServer.VolumeCopy:output_type -> volume_server_pb.VolumeCopyResponse
- 81, // 94: volume_server_pb.VolumeServer.ReadVolumeFileStatus:output_type -> volume_server_pb.ReadVolumeFileStatusResponse
- 43, // 95: volume_server_pb.VolumeServer.CopyFile:output_type -> volume_server_pb.CopyFileResponse
- 46, // 96: volume_server_pb.VolumeServer.ReceiveFile:output_type -> volume_server_pb.ReceiveFileResponse
- 48, // 97: volume_server_pb.VolumeServer.ReadNeedleBlob:output_type -> volume_server_pb.ReadNeedleBlobResponse
- 50, // 98: volume_server_pb.VolumeServer.ReadNeedleMeta:output_type -> volume_server_pb.ReadNeedleMetaResponse
- 52, // 99: volume_server_pb.VolumeServer.WriteNeedleBlob:output_type -> volume_server_pb.WriteNeedleBlobResponse
- 54, // 100: volume_server_pb.VolumeServer.ReadAllNeedles:output_type -> volume_server_pb.ReadAllNeedlesResponse
- 56, // 101: volume_server_pb.VolumeServer.VolumeTailSender:output_type -> volume_server_pb.VolumeTailSenderResponse
- 58, // 102: volume_server_pb.VolumeServer.VolumeTailReceiver:output_type -> volume_server_pb.VolumeTailReceiverResponse
- 60, // 103: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:output_type -> volume_server_pb.VolumeEcShardsGenerateResponse
- 62, // 104: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:output_type -> volume_server_pb.VolumeEcShardsRebuildResponse
- 64, // 105: volume_server_pb.VolumeServer.VolumeEcShardsCopy:output_type -> volume_server_pb.VolumeEcShardsCopyResponse
- 66, // 106: volume_server_pb.VolumeServer.VolumeEcShardsDelete:output_type -> volume_server_pb.VolumeEcShardsDeleteResponse
- 68, // 107: volume_server_pb.VolumeServer.VolumeEcShardsMount:output_type -> volume_server_pb.VolumeEcShardsMountResponse
- 70, // 108: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:output_type -> volume_server_pb.VolumeEcShardsUnmountResponse
- 72, // 109: volume_server_pb.VolumeServer.VolumeEcShardRead:output_type -> volume_server_pb.VolumeEcShardReadResponse
- 74, // 110: volume_server_pb.VolumeServer.VolumeEcBlobDelete:output_type -> volume_server_pb.VolumeEcBlobDeleteResponse
- 76, // 111: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:output_type -> volume_server_pb.VolumeEcShardsToVolumeResponse
- 78, // 112: volume_server_pb.VolumeServer.VolumeEcShardsInfo:output_type -> volume_server_pb.VolumeEcShardsInfoResponse
- 89, // 113: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:output_type -> volume_server_pb.VolumeTierMoveDatToRemoteResponse
- 91, // 114: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:output_type -> volume_server_pb.VolumeTierMoveDatFromRemoteResponse
- 93, // 115: volume_server_pb.VolumeServer.VolumeServerStatus:output_type -> volume_server_pb.VolumeServerStatusResponse
- 95, // 116: volume_server_pb.VolumeServer.VolumeServerLeave:output_type -> volume_server_pb.VolumeServerLeaveResponse
- 97, // 117: volume_server_pb.VolumeServer.FetchAndWriteNeedle:output_type -> volume_server_pb.FetchAndWriteNeedleResponse
- 99, // 118: volume_server_pb.VolumeServer.ScrubVolume:output_type -> volume_server_pb.ScrubVolumeResponse
- 101, // 119: volume_server_pb.VolumeServer.ScrubEcVolume:output_type -> volume_server_pb.ScrubEcVolumeResponse
- 103, // 120: volume_server_pb.VolumeServer.Query:output_type -> volume_server_pb.QueriedStripe
- 105, // 121: volume_server_pb.VolumeServer.VolumeNeedleStatus:output_type -> volume_server_pb.VolumeNeedleStatusResponse
- 107, // 122: volume_server_pb.VolumeServer.Ping:output_type -> volume_server_pb.PingResponse
- 75, // [75:123] is the sub-list for method output_type
- 27, // [27:75] is the sub-list for method input_type
- 27, // [27:27] is the sub-list for extension type_name
- 27, // [27:27] is the sub-list for extension extendee
- 0, // [0:27] is the sub-list for field type_name
+ 5, // 0: volume_server_pb.BatchDeleteResponse.results:type_name -> volume_server_pb.DeleteResult
+ 2, // 1: volume_server_pb.GetStateResponse.state:type_name -> volume_server_pb.VolumeServerState
+ 2, // 2: volume_server_pb.SetStateRequest.state:type_name -> volume_server_pb.VolumeServerState
+ 2, // 3: volume_server_pb.SetStateResponse.state:type_name -> volume_server_pb.VolumeServerState
+ 46, // 4: volume_server_pb.ReceiveFileRequest.info:type_name -> volume_server_pb.ReceiveFileInfo
+ 80, // 5: volume_server_pb.VolumeEcShardsInfoResponse.ec_shard_infos:type_name -> volume_server_pb.EcShardInfo
+ 86, // 6: volume_server_pb.ReadVolumeFileStatusResponse.volume_info:type_name -> volume_server_pb.VolumeInfo
+ 85, // 7: volume_server_pb.VolumeInfo.files:type_name -> volume_server_pb.RemoteFile
+ 87, // 8: volume_server_pb.VolumeInfo.ec_shard_config:type_name -> volume_server_pb.EcShardConfig
+ 0, // 9: volume_server_pb.EcBitrotProtection.algorithm:type_name -> volume_server_pb.ChecksumAlgorithm
+ 87, // 10: volume_server_pb.EcBitrotProtection.ec_shard_config:type_name -> volume_server_pb.EcShardConfig
+ 89, // 11: volume_server_pb.EcBitrotProtection.shards:type_name -> volume_server_pb.EcShardChecksums
+ 85, // 12: volume_server_pb.OldVersionVolumeInfo.files:type_name -> volume_server_pb.RemoteFile
+ 83, // 13: volume_server_pb.VolumeServerStatusResponse.disk_statuses:type_name -> volume_server_pb.DiskStatus
+ 84, // 14: volume_server_pb.VolumeServerStatusResponse.memory_status:type_name -> volume_server_pb.MemStatus
+ 2, // 15: volume_server_pb.VolumeServerStatusResponse.state:type_name -> volume_server_pb.VolumeServerState
+ 111, // 16: volume_server_pb.FetchAndWriteNeedleRequest.replicas:type_name -> volume_server_pb.FetchAndWriteNeedleRequest.Replica
+ 120, // 17: volume_server_pb.FetchAndWriteNeedleRequest.remote_conf:type_name -> remote_pb.RemoteConf
+ 121, // 18: volume_server_pb.FetchAndWriteNeedleRequest.remote_location:type_name -> remote_pb.RemoteStorageLocation
+ 1, // 19: volume_server_pb.ScrubVolumeRequest.mode:type_name -> volume_server_pb.VolumeScrubMode
+ 1, // 20: volume_server_pb.ScrubEcVolumeRequest.mode:type_name -> volume_server_pb.VolumeScrubMode
+ 80, // 21: volume_server_pb.ScrubEcVolumeResponse.broken_shard_infos:type_name -> volume_server_pb.EcShardInfo
+ 112, // 22: volume_server_pb.QueryRequest.filter:type_name -> volume_server_pb.QueryRequest.Filter
+ 113, // 23: volume_server_pb.QueryRequest.input_serialization:type_name -> volume_server_pb.QueryRequest.InputSerialization
+ 114, // 24: volume_server_pb.QueryRequest.output_serialization:type_name -> volume_server_pb.QueryRequest.OutputSerialization
+ 115, // 25: volume_server_pb.QueryRequest.InputSerialization.csv_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.CSVInput
+ 116, // 26: volume_server_pb.QueryRequest.InputSerialization.json_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.JSONInput
+ 117, // 27: volume_server_pb.QueryRequest.InputSerialization.parquet_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.ParquetInput
+ 118, // 28: volume_server_pb.QueryRequest.OutputSerialization.csv_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.CSVOutput
+ 119, // 29: volume_server_pb.QueryRequest.OutputSerialization.json_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.JSONOutput
+ 3, // 30: volume_server_pb.VolumeServer.BatchDelete:input_type -> volume_server_pb.BatchDeleteRequest
+ 7, // 31: volume_server_pb.VolumeServer.VacuumVolumeCheck:input_type -> volume_server_pb.VacuumVolumeCheckRequest
+ 9, // 32: volume_server_pb.VolumeServer.VacuumVolumeCompact:input_type -> volume_server_pb.VacuumVolumeCompactRequest
+ 11, // 33: volume_server_pb.VolumeServer.VacuumVolumeCommit:input_type -> volume_server_pb.VacuumVolumeCommitRequest
+ 13, // 34: volume_server_pb.VolumeServer.VacuumVolumeCleanup:input_type -> volume_server_pb.VacuumVolumeCleanupRequest
+ 15, // 35: volume_server_pb.VolumeServer.DeleteCollection:input_type -> volume_server_pb.DeleteCollectionRequest
+ 17, // 36: volume_server_pb.VolumeServer.AllocateVolume:input_type -> volume_server_pb.AllocateVolumeRequest
+ 19, // 37: volume_server_pb.VolumeServer.VolumeSyncStatus:input_type -> volume_server_pb.VolumeSyncStatusRequest
+ 21, // 38: volume_server_pb.VolumeServer.VolumeIncrementalCopy:input_type -> volume_server_pb.VolumeIncrementalCopyRequest
+ 23, // 39: volume_server_pb.VolumeServer.VolumeMount:input_type -> volume_server_pb.VolumeMountRequest
+ 25, // 40: volume_server_pb.VolumeServer.VolumeUnmount:input_type -> volume_server_pb.VolumeUnmountRequest
+ 27, // 41: volume_server_pb.VolumeServer.VolumeDelete:input_type -> volume_server_pb.VolumeDeleteRequest
+ 29, // 42: volume_server_pb.VolumeServer.VolumeMarkReadonly:input_type -> volume_server_pb.VolumeMarkReadonlyRequest
+ 31, // 43: volume_server_pb.VolumeServer.VolumeMarkWritable:input_type -> volume_server_pb.VolumeMarkWritableRequest
+ 33, // 44: volume_server_pb.VolumeServer.VolumeConfigure:input_type -> volume_server_pb.VolumeConfigureRequest
+ 35, // 45: volume_server_pb.VolumeServer.VolumeStatus:input_type -> volume_server_pb.VolumeStatusRequest
+ 37, // 46: volume_server_pb.VolumeServer.GetState:input_type -> volume_server_pb.GetStateRequest
+ 39, // 47: volume_server_pb.VolumeServer.SetState:input_type -> volume_server_pb.SetStateRequest
+ 41, // 48: volume_server_pb.VolumeServer.VolumeCopy:input_type -> volume_server_pb.VolumeCopyRequest
+ 81, // 49: volume_server_pb.VolumeServer.ReadVolumeFileStatus:input_type -> volume_server_pb.ReadVolumeFileStatusRequest
+ 43, // 50: volume_server_pb.VolumeServer.CopyFile:input_type -> volume_server_pb.CopyFileRequest
+ 45, // 51: volume_server_pb.VolumeServer.ReceiveFile:input_type -> volume_server_pb.ReceiveFileRequest
+ 48, // 52: volume_server_pb.VolumeServer.ReadNeedleBlob:input_type -> volume_server_pb.ReadNeedleBlobRequest
+ 50, // 53: volume_server_pb.VolumeServer.ReadNeedleMeta:input_type -> volume_server_pb.ReadNeedleMetaRequest
+ 52, // 54: volume_server_pb.VolumeServer.WriteNeedleBlob:input_type -> volume_server_pb.WriteNeedleBlobRequest
+ 54, // 55: volume_server_pb.VolumeServer.ReadAllNeedles:input_type -> volume_server_pb.ReadAllNeedlesRequest
+ 56, // 56: volume_server_pb.VolumeServer.VolumeTailSender:input_type -> volume_server_pb.VolumeTailSenderRequest
+ 58, // 57: volume_server_pb.VolumeServer.VolumeTailReceiver:input_type -> volume_server_pb.VolumeTailReceiverRequest
+ 60, // 58: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:input_type -> volume_server_pb.VolumeEcShardsGenerateRequest
+ 62, // 59: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:input_type -> volume_server_pb.VolumeEcShardsRebuildRequest
+ 64, // 60: volume_server_pb.VolumeServer.VolumeEcShardsCopy:input_type -> volume_server_pb.VolumeEcShardsCopyRequest
+ 66, // 61: volume_server_pb.VolumeServer.VolumeEcShardsDelete:input_type -> volume_server_pb.VolumeEcShardsDeleteRequest
+ 68, // 62: volume_server_pb.VolumeServer.VolumeEcShardsMount:input_type -> volume_server_pb.VolumeEcShardsMountRequest
+ 70, // 63: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:input_type -> volume_server_pb.VolumeEcShardsUnmountRequest
+ 72, // 64: volume_server_pb.VolumeServer.VolumeEcShardRead:input_type -> volume_server_pb.VolumeEcShardReadRequest
+ 74, // 65: volume_server_pb.VolumeServer.VolumeEcBlobDelete:input_type -> volume_server_pb.VolumeEcBlobDeleteRequest
+ 76, // 66: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:input_type -> volume_server_pb.VolumeEcShardsToVolumeRequest
+ 78, // 67: volume_server_pb.VolumeServer.VolumeEcShardsInfo:input_type -> volume_server_pb.VolumeEcShardsInfoRequest
+ 91, // 68: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:input_type -> volume_server_pb.VolumeTierMoveDatToRemoteRequest
+ 93, // 69: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:input_type -> volume_server_pb.VolumeTierMoveDatFromRemoteRequest
+ 95, // 70: volume_server_pb.VolumeServer.VolumeServerStatus:input_type -> volume_server_pb.VolumeServerStatusRequest
+ 97, // 71: volume_server_pb.VolumeServer.VolumeServerLeave:input_type -> volume_server_pb.VolumeServerLeaveRequest
+ 99, // 72: volume_server_pb.VolumeServer.FetchAndWriteNeedle:input_type -> volume_server_pb.FetchAndWriteNeedleRequest
+ 101, // 73: volume_server_pb.VolumeServer.ScrubVolume:input_type -> volume_server_pb.ScrubVolumeRequest
+ 103, // 74: volume_server_pb.VolumeServer.ScrubEcVolume:input_type -> volume_server_pb.ScrubEcVolumeRequest
+ 105, // 75: volume_server_pb.VolumeServer.Query:input_type -> volume_server_pb.QueryRequest
+ 107, // 76: volume_server_pb.VolumeServer.VolumeNeedleStatus:input_type -> volume_server_pb.VolumeNeedleStatusRequest
+ 109, // 77: volume_server_pb.VolumeServer.Ping:input_type -> volume_server_pb.PingRequest
+ 4, // 78: volume_server_pb.VolumeServer.BatchDelete:output_type -> volume_server_pb.BatchDeleteResponse
+ 8, // 79: volume_server_pb.VolumeServer.VacuumVolumeCheck:output_type -> volume_server_pb.VacuumVolumeCheckResponse
+ 10, // 80: volume_server_pb.VolumeServer.VacuumVolumeCompact:output_type -> volume_server_pb.VacuumVolumeCompactResponse
+ 12, // 81: volume_server_pb.VolumeServer.VacuumVolumeCommit:output_type -> volume_server_pb.VacuumVolumeCommitResponse
+ 14, // 82: volume_server_pb.VolumeServer.VacuumVolumeCleanup:output_type -> volume_server_pb.VacuumVolumeCleanupResponse
+ 16, // 83: volume_server_pb.VolumeServer.DeleteCollection:output_type -> volume_server_pb.DeleteCollectionResponse
+ 18, // 84: volume_server_pb.VolumeServer.AllocateVolume:output_type -> volume_server_pb.AllocateVolumeResponse
+ 20, // 85: volume_server_pb.VolumeServer.VolumeSyncStatus:output_type -> volume_server_pb.VolumeSyncStatusResponse
+ 22, // 86: volume_server_pb.VolumeServer.VolumeIncrementalCopy:output_type -> volume_server_pb.VolumeIncrementalCopyResponse
+ 24, // 87: volume_server_pb.VolumeServer.VolumeMount:output_type -> volume_server_pb.VolumeMountResponse
+ 26, // 88: volume_server_pb.VolumeServer.VolumeUnmount:output_type -> volume_server_pb.VolumeUnmountResponse
+ 28, // 89: volume_server_pb.VolumeServer.VolumeDelete:output_type -> volume_server_pb.VolumeDeleteResponse
+ 30, // 90: volume_server_pb.VolumeServer.VolumeMarkReadonly:output_type -> volume_server_pb.VolumeMarkReadonlyResponse
+ 32, // 91: volume_server_pb.VolumeServer.VolumeMarkWritable:output_type -> volume_server_pb.VolumeMarkWritableResponse
+ 34, // 92: volume_server_pb.VolumeServer.VolumeConfigure:output_type -> volume_server_pb.VolumeConfigureResponse
+ 36, // 93: volume_server_pb.VolumeServer.VolumeStatus:output_type -> volume_server_pb.VolumeStatusResponse
+ 38, // 94: volume_server_pb.VolumeServer.GetState:output_type -> volume_server_pb.GetStateResponse
+ 40, // 95: volume_server_pb.VolumeServer.SetState:output_type -> volume_server_pb.SetStateResponse
+ 42, // 96: volume_server_pb.VolumeServer.VolumeCopy:output_type -> volume_server_pb.VolumeCopyResponse
+ 82, // 97: volume_server_pb.VolumeServer.ReadVolumeFileStatus:output_type -> volume_server_pb.ReadVolumeFileStatusResponse
+ 44, // 98: volume_server_pb.VolumeServer.CopyFile:output_type -> volume_server_pb.CopyFileResponse
+ 47, // 99: volume_server_pb.VolumeServer.ReceiveFile:output_type -> volume_server_pb.ReceiveFileResponse
+ 49, // 100: volume_server_pb.VolumeServer.ReadNeedleBlob:output_type -> volume_server_pb.ReadNeedleBlobResponse
+ 51, // 101: volume_server_pb.VolumeServer.ReadNeedleMeta:output_type -> volume_server_pb.ReadNeedleMetaResponse
+ 53, // 102: volume_server_pb.VolumeServer.WriteNeedleBlob:output_type -> volume_server_pb.WriteNeedleBlobResponse
+ 55, // 103: volume_server_pb.VolumeServer.ReadAllNeedles:output_type -> volume_server_pb.ReadAllNeedlesResponse
+ 57, // 104: volume_server_pb.VolumeServer.VolumeTailSender:output_type -> volume_server_pb.VolumeTailSenderResponse
+ 59, // 105: volume_server_pb.VolumeServer.VolumeTailReceiver:output_type -> volume_server_pb.VolumeTailReceiverResponse
+ 61, // 106: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:output_type -> volume_server_pb.VolumeEcShardsGenerateResponse
+ 63, // 107: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:output_type -> volume_server_pb.VolumeEcShardsRebuildResponse
+ 65, // 108: volume_server_pb.VolumeServer.VolumeEcShardsCopy:output_type -> volume_server_pb.VolumeEcShardsCopyResponse
+ 67, // 109: volume_server_pb.VolumeServer.VolumeEcShardsDelete:output_type -> volume_server_pb.VolumeEcShardsDeleteResponse
+ 69, // 110: volume_server_pb.VolumeServer.VolumeEcShardsMount:output_type -> volume_server_pb.VolumeEcShardsMountResponse
+ 71, // 111: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:output_type -> volume_server_pb.VolumeEcShardsUnmountResponse
+ 73, // 112: volume_server_pb.VolumeServer.VolumeEcShardRead:output_type -> volume_server_pb.VolumeEcShardReadResponse
+ 75, // 113: volume_server_pb.VolumeServer.VolumeEcBlobDelete:output_type -> volume_server_pb.VolumeEcBlobDeleteResponse
+ 77, // 114: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:output_type -> volume_server_pb.VolumeEcShardsToVolumeResponse
+ 79, // 115: volume_server_pb.VolumeServer.VolumeEcShardsInfo:output_type -> volume_server_pb.VolumeEcShardsInfoResponse
+ 92, // 116: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:output_type -> volume_server_pb.VolumeTierMoveDatToRemoteResponse
+ 94, // 117: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:output_type -> volume_server_pb.VolumeTierMoveDatFromRemoteResponse
+ 96, // 118: volume_server_pb.VolumeServer.VolumeServerStatus:output_type -> volume_server_pb.VolumeServerStatusResponse
+ 98, // 119: volume_server_pb.VolumeServer.VolumeServerLeave:output_type -> volume_server_pb.VolumeServerLeaveResponse
+ 100, // 120: volume_server_pb.VolumeServer.FetchAndWriteNeedle:output_type -> volume_server_pb.FetchAndWriteNeedleResponse
+ 102, // 121: volume_server_pb.VolumeServer.ScrubVolume:output_type -> volume_server_pb.ScrubVolumeResponse
+ 104, // 122: volume_server_pb.VolumeServer.ScrubEcVolume:output_type -> volume_server_pb.ScrubEcVolumeResponse
+ 106, // 123: volume_server_pb.VolumeServer.Query:output_type -> volume_server_pb.QueriedStripe
+ 108, // 124: volume_server_pb.VolumeServer.VolumeNeedleStatus:output_type -> volume_server_pb.VolumeNeedleStatusResponse
+ 110, // 125: volume_server_pb.VolumeServer.Ping:output_type -> volume_server_pb.PingResponse
+ 78, // [78:126] is the sub-list for method output_type
+ 30, // [30:78] is the sub-list for method input_type
+ 30, // [30:30] is the sub-list for extension type_name
+ 30, // [30:30] is the sub-list for extension extendee
+ 0, // [0:30] is the sub-list for field type_name
}
func init() { file_volume_server_proto_init() }
@@ -7606,8 +7846,8 @@ func file_volume_server_proto_init() {
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_volume_server_proto_rawDesc), len(file_volume_server_proto_rawDesc)),
- NumEnums: 1,
- NumMessages: 116,
+ NumEnums: 2,
+ NumMessages: 118,
NumExtensions: 0,
NumServices: 1,
},
diff --git a/weed/server/volume_grpc_erasure_coding.go b/weed/server/volume_grpc_erasure_coding.go
index 4882101db..b0b1ebc05 100644
--- a/weed/server/volume_grpc_erasure_coding.go
+++ b/weed/server/volume_grpc_erasure_coding.go
@@ -7,6 +7,7 @@ import (
"math"
"os"
"path"
+ "path/filepath"
"strconv"
"strings"
"time"
@@ -84,6 +85,7 @@ func (vs *VolumeServer) VolumeEcShardsGenerate(ctx context.Context, req *volume_
os.Remove(baseFileName + ecCtx.ToExt(i))
}
os.Remove(v.IndexFileName() + ".ecx")
+ os.Remove(erasure_coding.BitrotSidecarPath(baseFileName, 0))
}()
// IMPORTANT: Generate .ecx BEFORE EC shards to prevent a race condition.
@@ -106,8 +108,19 @@ func (vs *VolumeServer) VolumeEcShardsGenerate(ctx context.Context, req *volume_
datSize, _, _ := v.FileStat()
// write .ec00 ~ .ec[TotalShards-1] files using context
- if err := erasure_coding.WriteEcFilesWithContext(baseFileName, ecCtx); err != nil {
- return nil, fmt.Errorf("WriteEcFilesWithContext %s: %v", baseFileName, err)
+ ecBitrot, err := erasure_coding.WriteEcFiles(baseFileName, ecCtx)
+ if err != nil {
+ return nil, fmt.Errorf("WriteEcFiles %s: %v", baseFileName, err)
+ }
+ // Persist the generation-0 bitrot checksum sidecar (.ecsum) alongside
+ // the shards so it travels with them during distribution (copy_ecsum_file).
+ // The source loading its own canonical sidecar is correct — it holds all
+ // shards and a complete manifest. Best-effort: a failed sidecar write leaves
+ // the generation unprotected rather than failing the encode.
+ if erasure_coding.BitrotProtectionEnabled && ecBitrot != nil {
+ if serr := erasure_coding.SaveBitrotSidecar(erasure_coding.BitrotSidecarPath(baseFileName, 0), ecBitrot); serr != nil {
+ glog.Warningf("failed to write EC bitrot sidecar for volume %d: %v", req.VolumeId, serr)
+ }
}
// write .vif files
@@ -203,9 +216,11 @@ func (vs *VolumeServer) VolumeEcShardsRebuild(ctx context.Context, req *volume_s
additionalDirs = append(additionalDirs, otherLocation.Directory)
}
- // Rebuild missing EC files, searching all disk locations for input shards
+ // Rebuild missing EC files, searching all disk locations for input shards.
+ // Present input shards are verified against the bitrot sidecar (when present)
+ // and corrupt ones are regenerated; unsafe_ignore_sidecar bypasses the guard.
dataBaseFileName := path.Join(rebuildDataDir, baseFileName)
- if generatedShardIds, err := erasure_coding.RebuildEcFiles(dataBaseFileName, additionalDirs...); err != nil {
+ if generatedShardIds, err := erasure_coding.RebuildEcFiles(dataBaseFileName, erasure_coding.BackgroundECContext(), req.UnsafeIgnoreSidecar, additionalDirs...); err != nil {
return nil, fmt.Errorf("RebuildEcFiles %s: %v", dataBaseFileName, err)
} else {
rebuiltShardIds = generatedShardIds
@@ -219,6 +234,30 @@ func (vs *VolumeServer) VolumeEcShardsRebuild(ctx context.Context, req *volume_s
return nil, fmt.Errorf("RebuildEcxFile %s: %v", indexBaseFileName, err)
}
+ // Opportunistic bitrot backfill: if protection is enabled, no sidecar exists
+ // yet (a volume encoded before this feature), and this rebuilder can reach
+ // every shard, compute and write a generation-0 sidecar. The TOFU baseline
+ // blesses current bytes; ComputeProtectionFromShards refuses a partial
+ // manifest, so a multi-server rebuild that cannot reach all shards just skips.
+ if erasure_coding.BitrotProtectionEnabled {
+ sidecarPath := erasure_coding.BitrotSidecarPath(dataBaseFileName, 0)
+ if _, statErr := os.Stat(sidecarPath); os.IsNotExist(statErr) {
+ ctx := erasure_coding.NewDefaultECContext("", 0)
+ if vi, _, found, _ := volume_info.MaybeLoadVolumeInfo(dataBaseFileName + ".vif"); found && vi.EcShardConfig != nil {
+ if ds, ps := int(vi.EcShardConfig.DataShards), int(vi.EcShardConfig.ParityShards); ds > 0 && ps > 0 && ds+ps <= erasure_coding.MaxShardCount {
+ ctx = &erasure_coding.ECContext{DataShards: ds, ParityShards: ps}
+ }
+ }
+ if prot, berr := erasure_coding.ComputeProtectionFromShards(dataBaseFileName, ctx, 0, additionalDirs); berr != nil {
+ glog.V(2).Infof("bitrot backfill skipped for %s: %v", dataBaseFileName, berr)
+ } else if werr := erasure_coding.SaveBitrotSidecar(sidecarPath, prot); werr != nil {
+ glog.Warningf("bitrot backfill: write sidecar for %s: %v", dataBaseFileName, werr)
+ } else {
+ glog.V(0).Infof("bitrot backfill: wrote sidecar for %s after rebuild", dataBaseFileName)
+ }
+ }
+ }
+
return &volume_server_pb.VolumeEcShardsRebuildResponse{
RebuiltShardIds: rebuiltShardIds,
}, nil
@@ -330,6 +369,17 @@ func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_serv
return err
}
}
+
+ if req.CopyEcsumFile {
+ // Propagate the generation-0 bitrot checksum sidecar when the source
+ // has one. This non-2PC copy path (balance / fresh-encode / rebuild
+ // distribution) has no Prepare backstop, and fresh-encode sidecar
+ // writes are best-effort, so a missing source sidecar is a no-op
+ // (ignore-not-found): the holder is simply unprotected.
+ if _, err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, dataBaseFileName, erasure_coding.BitrotSidecarExt, false, true, nil); err != nil {
+ return fmt.Errorf("VolumeEcShardsCopy volume %d: copy %s sidecar: %w", req.VolumeId, erasure_coding.BitrotSidecarExt, err)
+ }
+ }
return nil
})
if err != nil {
@@ -393,27 +443,54 @@ func deleteEcShardIdsForEachLocation(bName string, location *storage.DiskLocatio
return err
}
- if hasEcxFile && existingShardCount == 0 {
- // Remove .ecx/.ecj from both idx and data directories
- // since they may be in either location depending on when -dir.idx was configured
- if err := os.Remove(indexBaseFilename + ".ecx"); err != nil && !os.IsNotExist(err) {
- return err
- }
- os.Remove(indexBaseFilename + ".ecj")
+ if existingShardCount == 0 {
+ // The whole EC generation is gone on this disk.
+
+ // Remove the bitrot checksum sidecar(s) (.ecsum and any .ecsum.v)
+ // from both dirs. This is gated on the shards being gone, NOT on a
+ // stray .ecx still being present: a sidecar whose shards have all been
+ // deleted is orphaned and must go even when no .ecx remains, or it
+ // leaks. The per-shard-id delete that ec.rebuild uses for
+ // copied-survivor cleanup leaves shards behind, so this guard does not
+ // fire there.
+ removeBitrotSidecars(dataBaseFilename)
if location.IdxDirectory != location.Directory {
- os.Remove(dataBaseFilename + ".ecx")
- os.Remove(dataBaseFilename + ".ecj")
+ removeBitrotSidecars(indexBaseFilename)
}
- if !hasIdxFile {
- // .vif is used for ec volumes and normal volumes
- os.Remove(dataBaseFilename + ".vif")
+ if hasEcxFile {
+ // Remove .ecx/.ecj from both idx and data directories
+ // since they may be in either location depending on when -dir.idx was configured
+ if err := os.Remove(indexBaseFilename + ".ecx"); err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ os.Remove(indexBaseFilename + ".ecj")
+ if location.IdxDirectory != location.Directory {
+ os.Remove(dataBaseFilename + ".ecx")
+ os.Remove(dataBaseFilename + ".ecj")
+ }
+
+ if !hasIdxFile {
+ // .vif is used for ec volumes and normal volumes
+ os.Remove(dataBaseFilename + ".vif")
+ }
}
}
return nil
}
+// removeBitrotSidecars removes the legacy .ecsum and any versioned
+// .ecsum.v sidecars. Best-effort; logs nothing on absence.
+func removeBitrotSidecars(baseFilename string) {
+ os.Remove(baseFilename + erasure_coding.BitrotSidecarExt)
+ if matches, _ := filepath.Glob(baseFilename + erasure_coding.BitrotSidecarExt + ".v*"); matches != nil {
+ for _, m := range matches {
+ os.Remove(m)
+ }
+ }
+}
+
func checkEcVolumeStatus(bName string, location *storage.DiskLocation) (hasEcxFile bool, hasIdxFile bool, existingShardCount int, err error) {
// check whether to delete the .ecx and .ecj file also
fileInfos, err := os.ReadDir(location.Directory)
@@ -681,6 +758,13 @@ func (vs *VolumeServer) VolumeEcShardsToVolume(ctx context.Context, req *volume_
return nil, fmt.Errorf("WriteIdxFileFromEcIndex %s: %v", v.IndexBaseFileName(), err)
}
+ // The EC generation is gone; drop its bitrot sidecar(s) so a later EC
+ // re-encode cannot mistake a stale .ecsum for protection.
+ removeBitrotSidecars(dataBaseFileName)
+ if indexBaseFileName != dataBaseFileName {
+ removeBitrotSidecars(indexBaseFileName)
+ }
+
var volumeLocation *storage.DiskLocation
for _, location := range vs.store.Locations {
if candidate, found := location.FindEcVolume(needle.VolumeId(req.VolumeId)); found && candidate == v {
diff --git a/weed/server/volume_grpc_scrub.go b/weed/server/volume_grpc_scrub.go
index 27afe2dac..c306944ae 100644
--- a/weed/server/volume_grpc_scrub.go
+++ b/weed/server/volume_grpc_scrub.go
@@ -125,6 +125,12 @@ func (vs *VolumeServer) ScrubEcVolume(ctx context.Context, req *volume_server_pb
files, shardInfos, serrs = v.ScrubLocal()
case volume_server_pb.VolumeScrubMode_FULL:
files, shardInfos, serrs = vs.store.ScrubEcVolume(v.VolumeId)
+ case volume_server_pb.VolumeScrubMode_CHECKSUM:
+ // Verify each local shard's raw bytes against the bitrot sidecar,
+ // exercising cold parity shards. Read-only. ChecksumScrub's first
+ // return is blocks scanned, not files — discard it so TotalFiles
+ // (a needle/file count) isn't inflated by the block count.
+ _, shardInfos, serrs = v.ChecksumScrub()
default:
return nil, fmt.Errorf("unsupported EC volume scrub mode %d", m)
}
diff --git a/weed/shell/command_ec_common.go b/weed/shell/command_ec_common.go
index 1690e9801..199641442 100644
--- a/weed/shell/command_ec_common.go
+++ b/weed/shell/command_ec_common.go
@@ -350,6 +350,7 @@ func oneServerCopyAndMountEcShardsFromSource(grpcDialOption grpc.DialOption,
CopyEcxFile: true,
CopyEcjFile: true,
CopyVifFile: true,
+ CopyEcsumFile: true, // propagate the bitrot sidecar with the shards (no-op if the source has none)
SourceDataNode: string(existingLocation),
DiskId: destDiskId,
})
diff --git a/weed/shell/command_ec_scrub.go b/weed/shell/command_ec_scrub.go
index 24117464b..ee3235972 100644
--- a/weed/shell/command_ec_scrub.go
+++ b/weed/shell/command_ec_scrub.go
@@ -49,7 +49,7 @@ func (c *commandEcVolumeScrub) Do(args []string, commandEnv *CommandEnv, writer
volScrubCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
nodesStr := volScrubCommand.String("node", "", "comma-separated list of volume server : (optional)")
volumeIDsStr := volScrubCommand.String("volumeId", "", "comma-separated EC volume IDs to process (optional)")
- mode := volScrubCommand.String("mode", "local", "scrubbing mode (index/local/full)")
+ mode := volScrubCommand.String("mode", "local", "scrubbing mode (index/local/full/checksum)")
maxParallelization := volScrubCommand.Int("maxParallelization", DefaultMaxParallelization, "run up to X tasks in parallel, whenever possible")
if err = volScrubCommand.Parse(args); err != nil {
@@ -96,6 +96,8 @@ func (c *commandEcVolumeScrub) Do(args []string, commandEnv *CommandEnv, writer
c.mode = volume_server_pb.VolumeScrubMode_LOCAL
case "FULL":
c.mode = volume_server_pb.VolumeScrubMode_FULL
+ case "CHECKSUM":
+ c.mode = volume_server_pb.VolumeScrubMode_CHECKSUM
default:
return fmt.Errorf("unsupported scrubbing mode %q", *mode)
}
diff --git a/weed/storage/disk_location_ec_realworld_test.go b/weed/storage/disk_location_ec_realworld_test.go
index ea92d3f00..42ce29bd7 100644
--- a/weed/storage/disk_location_ec_realworld_test.go
+++ b/weed/storage/disk_location_ec_realworld_test.go
@@ -86,7 +86,7 @@ func TestCalculateExpectedShardSizeWithRealEncoding(t *testing.T) {
expectedShardSize := calculateExpectedShardSize(tt.datFileSize, erasure_coding.DataShardsCount)
// Run actual EC encoding
- err = erasure_coding.WriteEcFiles(baseFileName)
+ _, err = erasure_coding.WriteEcFiles(baseFileName, erasure_coding.BackgroundECContext())
if err != nil {
t.Fatalf("Failed to encode EC files: %v", err)
}
@@ -167,7 +167,7 @@ func TestCalculateExpectedShardSizeEdgeCases(t *testing.T) {
expectedShardSize := calculateExpectedShardSize(tt.datFileSize, erasure_coding.DataShardsCount)
// Run actual EC encoding
- err = erasure_coding.WriteEcFiles(baseFileName)
+ _, err = erasure_coding.WriteEcFiles(baseFileName, erasure_coding.BackgroundECContext())
if err != nil {
t.Fatalf("Failed to encode EC files: %v", err)
}
diff --git a/weed/storage/erasure_coding/ec_bitrot.go b/weed/storage/erasure_coding/ec_bitrot.go
new file mode 100644
index 000000000..e4f820fe4
--- /dev/null
+++ b/weed/storage/erasure_coding/ec_bitrot.go
@@ -0,0 +1,554 @@
+package erasure_coding
+
+// EC bitrot detection — checksum sidecar.
+//
+// A per-volume sidecar file stores a CRC32C (Castagnoli) checksum for every
+// fixed-size block of every EC shard, so a scrub (and the reconstruction path)
+// can detect silent disk corruption in any shard — including cold parity shards
+// that are never read during normal serving.
+//
+// The sidecar is OPTIONAL: an absent or generation-mismatched sidecar simply
+// means "feature off" for that generation, so old binaries, JSON-only Rust
+// nodes, and rollback deployments ignore it and degrade gracefully. See
+// EC_BITROT_DETECTION_DESIGN.md.
+//
+// On-disk layout of .ecsum (legacy/generation 0) and .ecsum.v:
+//
+// [ magic(4) | format_version(2) | payload_len(4) | payload_crc32c(4) ] [ proto payload ]
+//
+// The header's payload_crc32c lets a loader detect corruption of the sidecar
+// itself BEFORE trusting any contents, so a rotted sidecar can never be
+// mistaken for shard corruption.
+
+import (
+ "crypto/rand"
+ "encoding/binary"
+ "fmt"
+ "io"
+ "math/bits"
+ "os"
+ "path/filepath"
+
+ "github.com/seaweedfs/seaweedfs/weed/glog"
+ "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+ "github.com/seaweedfs/seaweedfs/weed/storage/needle"
+ "google.golang.org/protobuf/proto"
+)
+
+const (
+ // BitrotSidecarExt is the canonical extension for the checksum sidecar.
+ // Generation 0 (legacy/fresh encode) uses ".ecsum"; vacuum generation
+ // N uses ".ecsum.v", mirroring the .vif/.ecx versioned convention.
+ BitrotSidecarExt = ".ecsum"
+
+ // DefaultBitrotBlockSize is the default checksum granularity (16 MiB). It is
+ // a power-of-two multiple of ErasureCodingSmallBlockSize (1 MiB) and keeps
+ // the sidecar tiny (~11 KB for a 30 GB volume) while localizing corruption
+ // to a 16 MiB region.
+ DefaultBitrotBlockSize = 16 * 1024 * 1024
+
+ bitrotMagic uint32 = 0x45435355 // "ECSU"
+ bitrotFormatVersion uint16 = 1
+ bitrotHeaderSize = 14 // magic(4)+version(2)+payload_len(4)+payload_crc32c(4)
+)
+
+// Config knobs (wired to volume-server flags). Defaults: protection on for new
+// encodes, 16 MiB block granularity.
+var (
+ // BitrotProtectionEnabled controls whether encode/vacuum write a sidecar.
+ // When false, generations are produced unprotected (no sidecar), which is a
+ // legitimate "off" state. Detection on already-protected generations is
+ // unaffected.
+ BitrotProtectionEnabled = true
+ // BitrotBlockSize is the checksum block granularity used for new sidecars.
+ // Must remain a power-of-two multiple of 1 MiB.
+ BitrotBlockSize int64 = DefaultBitrotBlockSize
+)
+
+// BitrotStatus is the resolved protection state of an EC volume's active
+// generation after loading and validating its sidecar.
+type BitrotStatus int
+
+const (
+ // BitrotOff: no sidecar, or a sidecar that does not describe the active
+ // generation. The generation is unprotected; this is NOT corruption.
+ BitrotOff BitrotStatus = iota
+ // BitrotOn: a complete, well-formed, generation-matching sidecar is loaded.
+ BitrotOn
+ // BitrotInvalid: a generation-matching sidecar that is malformed, incomplete,
+ // or self-integrity-failed. The generation is unprotected pending repair of
+ // the sidecar, and an integrity alarm should fire. The rebuild path treats
+ // this as fail-closed (see ec_encoder rebuild verify).
+ BitrotInvalid
+)
+
+func (s BitrotStatus) String() string {
+ switch s {
+ case BitrotOn:
+ return "on"
+ case BitrotInvalid:
+ return "invalid"
+ default:
+ return "off"
+ }
+}
+
+// BitrotSidecarPath returns the sidecar path for a base file name and EC
+// generation. Generation 0 is the un-suffixed legacy path; generation N>0 is
+// the versioned path, consistent with how .vif/.ecx are versioned by the 2PC
+// switch.
+func BitrotSidecarPath(baseFileName string, generation uint32) string {
+ if generation == 0 {
+ return baseFileName + BitrotSidecarExt
+ }
+ return fmt.Sprintf("%s%s.v%d", baseFileName, BitrotSidecarExt, generation)
+}
+
+// NewEncodeUUID returns a fresh random per-encode identity used to detect a
+// stale sidecar left behind by an in-place re-encode.
+func NewEncodeUUID() []byte {
+ b := make([]byte, 16)
+ if _, err := rand.Read(b); err != nil {
+ // rand.Read essentially never fails; fall back to a zero UUID rather
+ // than aborting an encode over entropy. Stale detection still has the
+ // other defenses (wholesale-mismatch cap + RS arbiter).
+ glog.Warningf("ec bitrot: failed to read encode uuid entropy: %v", err)
+ }
+ return b
+}
+
+// isPow2MultipleOf1MiB reports whether block_size is a power-of-two multiple of
+// 1 MiB (i.e. a power of two that is at least 1 MiB).
+func isPow2MultipleOf1MiB(blockSize uint32) bool {
+ return blockSize >= (1<<20) && bits.OnesCount32(blockSize) == 1
+}
+
+// shardChecksumBuilder accumulates the per-block CRC32C of a single shard's byte
+// stream as it is written. It tolerates arbitrary chunk sizes that cross block
+// boundaries, so it works for both the 256 KiB encode buffers and the 1 MiB
+// rebuild buffers.
+type shardChecksumBuilder struct {
+ blockSize int64
+ cur needle.CRC
+ curLen int64
+ total int64
+ blocks []uint32
+}
+
+func newShardChecksumBuilder(blockSize int64) *shardChecksumBuilder {
+ return &shardChecksumBuilder{blockSize: blockSize}
+}
+
+func (b *shardChecksumBuilder) write(p []byte) {
+ for len(p) > 0 {
+ room := b.blockSize - b.curLen
+ n := int64(len(p))
+ if n > room {
+ n = room
+ }
+ b.cur = b.cur.Update(p[:n])
+ b.curLen += n
+ b.total += n
+ p = p[n:]
+ if b.curLen == b.blockSize {
+ b.blocks = append(b.blocks, uint32(b.cur))
+ b.cur = 0
+ b.curLen = 0
+ }
+ }
+}
+
+// finalize flushes any partial last block and returns the covered size and the
+// packed little-endian uint32 CRC array.
+func (b *shardChecksumBuilder) finalize() (coveredSize int64, packed []byte) {
+ if b.curLen > 0 {
+ b.blocks = append(b.blocks, uint32(b.cur))
+ b.cur = 0
+ b.curLen = 0
+ }
+ return b.total, packUint32LE(b.blocks)
+}
+
+// buildProtectionFromBuilders assembles a generation-0 EcBitrotProtection from
+// the per-shard checksum builders produced during an encode pass. Callers that
+// produce a versioned generation (vacuum) set prot.Generation afterwards. A
+// fresh encode_uuid is minted on every call so an in-place re-encode is
+// distinguishable from a stale sidecar.
+func buildProtectionFromBuilders(ctx *ECContext, builders []*shardChecksumBuilder, blockSize int64) *volume_server_pb.EcBitrotProtection {
+ shards := make([]*volume_server_pb.EcShardChecksums, 0, len(builders))
+ for id, b := range builders {
+ covered, packed := b.finalize()
+ shards = append(shards, &volume_server_pb.EcShardChecksums{
+ ShardId: uint32(id),
+ CoveredSize: covered,
+ BlockCrc32C: packed,
+ })
+ }
+ return &volume_server_pb.EcBitrotProtection{
+ Algorithm: volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+ BlockSize: uint32(blockSize),
+ Generation: 0,
+ EcShardConfig: &volume_server_pb.EcShardConfig{
+ DataShards: uint32(ctx.DataShards),
+ ParityShards: uint32(ctx.ParityShards),
+ },
+ Shards: shards,
+ EncodeUuid: NewEncodeUUID(),
+ }
+}
+
+func packUint32LE(vals []uint32) []byte {
+ out := make([]byte, len(vals)*4)
+ for i, v := range vals {
+ binary.LittleEndian.PutUint32(out[i*4:], v)
+ }
+ return out
+}
+
+func unpackUint32LE(b []byte) []uint32 {
+ out := make([]uint32, len(b)/4)
+ for i := range out {
+ out[i] = binary.LittleEndian.Uint32(b[i*4:])
+ }
+ return out
+}
+
+// expectedBlockCount returns ceil(coveredSize/blockSize).
+func expectedBlockCount(coveredSize, blockSize int64) int {
+ if blockSize <= 0 {
+ return 0
+ }
+ return int((coveredSize + blockSize - 1) / blockSize)
+}
+
+// SaveBitrotSidecar atomically writes prot to path, wrapped in the on-disk
+// header with a CRC32C over the serialized payload.
+func SaveBitrotSidecar(path string, prot *volume_server_pb.EcBitrotProtection) error {
+ payload, err := proto.Marshal(prot)
+ if err != nil {
+ return fmt.Errorf("marshal bitrot sidecar: %w", err)
+ }
+ // The header records payload_len as a uint32 and the allocation below adds
+ // it to a constant. Bound the payload well under any overflow (a real
+ // manifest is a few KB) so neither the length field nor make() can wrap.
+ const maxBitrotPayloadSize = 1 << 30 // 1 GiB, vastly above any real sidecar
+ if len(payload) > maxBitrotPayloadSize {
+ return fmt.Errorf("bitrot sidecar payload too large: %d bytes", len(payload))
+ }
+ buf := make([]byte, bitrotHeaderSize+len(payload))
+ binary.BigEndian.PutUint32(buf[0:4], bitrotMagic)
+ binary.BigEndian.PutUint16(buf[4:6], bitrotFormatVersion)
+ binary.BigEndian.PutUint32(buf[6:10], uint32(len(payload)))
+ binary.BigEndian.PutUint32(buf[10:14], uint32(needle.NewCRC(payload)))
+ copy(buf[bitrotHeaderSize:], payload)
+
+ tmp := path + ".tmp"
+ if err := os.WriteFile(tmp, buf, 0644); err != nil {
+ return fmt.Errorf("write bitrot sidecar tmp %s: %w", tmp, err)
+ }
+ if err := os.Rename(tmp, path); err != nil {
+ os.Remove(tmp)
+ return fmt.Errorf("rename bitrot sidecar %s: %w", path, err)
+ }
+ return nil
+}
+
+// LoadBitrotSidecar reads and self-integrity-checks a sidecar file. It returns
+// the parsed message, or an error if the file is missing, truncated, has a bad
+// magic/version, or fails the payload CRC. A self-integrity failure is a
+// sidecar-integrity problem (caller maps it to BitrotInvalid), never a shard
+// corruption signal.
+func LoadBitrotSidecar(path string) (*volume_server_pb.EcBitrotProtection, error) {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return nil, err
+ }
+ if len(data) < bitrotHeaderSize {
+ return nil, fmt.Errorf("bitrot sidecar %s too short (%d bytes)", path, len(data))
+ }
+ if magic := binary.BigEndian.Uint32(data[0:4]); magic != bitrotMagic {
+ return nil, fmt.Errorf("bitrot sidecar %s bad magic %#x", path, magic)
+ }
+ if ver := binary.BigEndian.Uint16(data[4:6]); ver != bitrotFormatVersion {
+ return nil, fmt.Errorf("bitrot sidecar %s unsupported format version %d", path, ver)
+ }
+ payloadLen := binary.BigEndian.Uint32(data[6:10])
+ wantCRC := binary.BigEndian.Uint32(data[10:14])
+ payload := data[bitrotHeaderSize:]
+ if int(payloadLen) != len(payload) {
+ return nil, fmt.Errorf("bitrot sidecar %s length mismatch: header %d, actual %d", path, payloadLen, len(payload))
+ }
+ if got := uint32(needle.NewCRC(payload)); got != wantCRC {
+ return nil, fmt.Errorf("bitrot sidecar %s self-integrity CRC mismatch: header %#x, computed %#x", path, wantCRC, got)
+ }
+ prot := &volume_server_pb.EcBitrotProtection{}
+ if err := proto.Unmarshal(payload, prot); err != nil {
+ return nil, fmt.Errorf("unmarshal bitrot sidecar %s: %w", path, err)
+ }
+ return prot, nil
+}
+
+// ValidateBitrotManifest performs the disk-free manifest/syntax checks that
+// every loader runs: supported algorithm, valid block size, exactly one entry
+// per shard id in the active layout (no duplicates, no out-of-range ids),
+// positive covered_size, and a packed-CRC count consistent with covered_size.
+// It does NOT compare covered_size against on-disk shard lengths — that is a
+// per-node physical check done only for locally-held shards.
+func ValidateBitrotManifest(prot *volume_server_pb.EcBitrotProtection, dataShards, parityShards int) error {
+ if prot.Algorithm != volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C {
+ return fmt.Errorf("unsupported checksum algorithm %v", prot.Algorithm)
+ }
+ bs := int64(prot.BlockSize)
+ if !isPow2MultipleOf1MiB(prot.BlockSize) {
+ return fmt.Errorf("invalid block_size %d (must be a power-of-two multiple of 1 MiB)", prot.BlockSize)
+ }
+ total := dataShards + parityShards
+ if total <= 0 || total > MaxShardCount {
+ return fmt.Errorf("invalid active layout: data=%d parity=%d", dataShards, parityShards)
+ }
+ if len(prot.Shards) != total {
+ return fmt.Errorf("incomplete manifest: %d shard entries, expected %d", len(prot.Shards), total)
+ }
+ seen := make([]bool, MaxShardCount)
+ for _, s := range prot.Shards {
+ if s.ShardId >= uint32(total) {
+ return fmt.Errorf("shard id %d out of range [0,%d)", s.ShardId, total)
+ }
+ if seen[s.ShardId] {
+ return fmt.Errorf("duplicate shard id %d", s.ShardId)
+ }
+ seen[s.ShardId] = true
+ if s.CoveredSize <= 0 {
+ return fmt.Errorf("shard %d has non-positive covered_size %d", s.ShardId, s.CoveredSize)
+ }
+ wantCount := expectedBlockCount(s.CoveredSize, bs)
+ if len(s.BlockCrc32C) != wantCount*4 {
+ return fmt.Errorf("shard %d crc count mismatch: %d bytes, expected %d (covered_size=%d block_size=%d)",
+ s.ShardId, len(s.BlockCrc32C), wantCount*4, s.CoveredSize, prot.BlockSize)
+ }
+ }
+ return nil
+}
+
+// shardChecksums returns the EcShardChecksums entry for a shard id, or nil.
+func shardChecksums(prot *volume_server_pb.EcBitrotProtection, shardId uint32) *volume_server_pb.EcShardChecksums {
+ for _, s := range prot.Shards {
+ if s.ShardId == shardId {
+ return s
+ }
+ }
+ return nil
+}
+
+// verifyShardFileBlocks reads a shard file at path in block_size chunks and
+// compares each block's CRC32C against the manifest entry. It returns the list
+// of mismatching block indices (empty == clean) and a fatal error only for I/O
+// problems or a covered_size/length mismatch (which is itself corruption of the
+// shard). It does not interpret the result — the caller (scrub / rebuild)
+// arbitrates shard-vs-sidecar via Reed-Solomon before acting.
+func verifyShardFileBlocks(path string, entry *volume_server_pb.EcShardChecksums, blockSize int64) (mismatched []int, err error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ fi, err := f.Stat()
+ if err != nil {
+ return nil, err
+ }
+ if fi.Size() != entry.CoveredSize {
+ // Length drift (truncation or unexpected trailing bytes) is shard
+ // corruption: report every block as mismatched so the caller treats the
+ // shard as bad.
+ want := unpackUint32LE(entry.BlockCrc32C)
+ all := make([]int, len(want))
+ for i := range all {
+ all[i] = i
+ }
+ return all, nil
+ }
+ want := unpackUint32LE(entry.BlockCrc32C)
+ buf := make([]byte, blockSize)
+ var offset int64
+ for i := 0; i < len(want); i++ {
+ toRead := blockSize
+ if rem := entry.CoveredSize - offset; rem < toRead {
+ toRead = rem
+ }
+ // os.File.ReadAt fills the slice or returns an error; a full read that
+ // lands exactly on EOF may still report io.EOF, which is not corruption.
+ // A short read (n < toRead) means the shard is truncated and the EOF is
+ // propagated as an error.
+ n, rerr := f.ReadAt(buf[:toRead], offset)
+ if rerr == io.EOF && int64(n) == toRead {
+ rerr = nil
+ }
+ if rerr != nil {
+ return nil, rerr
+ }
+ if uint32(needle.NewCRC(buf[:n])) != want[i] {
+ mismatched = append(mismatched, i)
+ }
+ offset += int64(n)
+ }
+ return mismatched, nil
+}
+
+// ComputeProtectionFromShards builds a complete EcBitrotProtection by reading
+// the on-disk bytes of every shard for a generation — the trust-on-first-use
+// backfill primitive for EC volumes that were encoded before this feature. It
+// requires EVERY shard to be reachable (locally or in additionalDirs) and
+// returns an error otherwise, so a partial sidecar is never produced. This is
+// the per-holder building block a coordinator assembles across servers; on a
+// server that holds (or can reach) all shards it yields a complete manifest
+// directly. Caveat: it blesses whatever bytes exist now and cannot detect
+// pre-baseline corruption.
+func ComputeProtectionFromShards(baseFileName string, ctx *ECContext, generation uint32, additionalDirs []string) (*volume_server_pb.EcBitrotProtection, error) {
+ shards := make([]*volume_server_pb.EcShardChecksums, 0, ctx.Total())
+ for id := 0; id < ctx.Total(); id++ {
+ path := findShardFile(baseFileName, ctx.ToExt(id), additionalDirs)
+ if path == "" {
+ return nil, fmt.Errorf("bitrot backfill: shard %d missing for %s; refusing to write a partial sidecar", id, baseFileName)
+ }
+ covered, packed, err := computeShardFileCRCs(path, BitrotBlockSize)
+ if err != nil {
+ return nil, fmt.Errorf("bitrot backfill: read shard %d (%s): %w", id, path, err)
+ }
+ shards = append(shards, &volume_server_pb.EcShardChecksums{
+ ShardId: uint32(id),
+ CoveredSize: covered,
+ BlockCrc32C: packed,
+ })
+ }
+ return &volume_server_pb.EcBitrotProtection{
+ Algorithm: volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+ BlockSize: uint32(BitrotBlockSize),
+ Generation: generation,
+ EcShardConfig: &volume_server_pb.EcShardConfig{
+ DataShards: uint32(ctx.DataShards),
+ ParityShards: uint32(ctx.ParityShards),
+ },
+ Shards: shards,
+ EncodeUuid: NewEncodeUUID(),
+ }, nil
+}
+
+// computeShardFileCRCs reads a shard file sequentially and returns its size and
+// packed per-block CRC32C array.
+func computeShardFileCRCs(path string, blockSize int64) (coveredSize int64, packed []byte, err error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return 0, nil, err
+ }
+ defer f.Close()
+ b := newShardChecksumBuilder(blockSize)
+ buf := make([]byte, blockSize)
+ for {
+ n, rerr := f.Read(buf)
+ if n > 0 {
+ b.write(buf[:n])
+ }
+ if rerr == io.EOF {
+ break
+ }
+ if rerr != nil {
+ return 0, nil, rerr
+ }
+ if n == 0 {
+ break
+ }
+ }
+ coveredSize, packed = b.finalize()
+ return coveredSize, packed, nil
+}
+
+// BitrotProtection returns the active-generation bitrot protection and its
+// status. The returned message is read-only; callers must not mutate it.
+func (ev *EcVolume) BitrotProtection() (*volume_server_pb.EcBitrotProtection, BitrotStatus) {
+ ev.bitrotLock.RLock()
+ defer ev.bitrotLock.RUnlock()
+ return ev.bitrot, ev.bitrotStatus
+}
+
+// loadActiveBitrotSidecar loads the generation-0 checksum sidecar into the
+// volume. Best-effort: any failure leaves protection off/invalid without
+// failing the mount. OSS only produces generation-0 (fresh-encode) sidecars.
+func (ev *EcVolume) loadActiveBitrotSidecar() {
+ ev.loadBitrotForGeneration(0)
+}
+
+// loadBitrotForGeneration loads and validates the sidecar describing generation
+// `generation`, setting ev.bitrot/ev.bitrotStatus. Called at mount. Absent or
+// generation/config-mismatched => BitrotOff; self-integrity/manifest failure =>
+// BitrotInvalid; usable => BitrotOn.
+func (ev *EcVolume) loadBitrotForGeneration(generation uint32) {
+ ev.bitrotLock.Lock()
+ defer ev.bitrotLock.Unlock()
+ ev.bitrot = nil
+ ev.bitrotStatus = BitrotOff
+
+ if ev.ECContext == nil {
+ return
+ }
+ path := findBitrotSidecar(generation, ev.DataBaseFileName(), ev.IndexBaseFileName())
+ if path == "" {
+ return
+ }
+ prot, err := LoadBitrotSidecar(path)
+ if err != nil {
+ glog.Warningf("ec volume %d: bitrot sidecar %s self-integrity failed: %v", ev.VolumeId, path, err)
+ ev.bitrotStatus = BitrotInvalid
+ return
+ }
+ if prot.Generation != generation {
+ return // not for this generation -> off, not corruption
+ }
+ if prot.EcShardConfig == nil ||
+ int(prot.EcShardConfig.DataShards) != ev.ECContext.DataShards ||
+ int(prot.EcShardConfig.ParityShards) != ev.ECContext.ParityShards {
+ return
+ }
+ if err := ValidateBitrotManifest(prot, ev.ECContext.DataShards, ev.ECContext.ParityShards); err != nil {
+ glog.Warningf("ec volume %d: bitrot sidecar %s manifest invalid: %v", ev.VolumeId, path, err)
+ ev.bitrotStatus = BitrotInvalid
+ return
+ }
+ ev.bitrot = prot
+ ev.bitrotStatus = BitrotOn
+ glog.V(1).Infof("ec volume %d: loaded bitrot protection generation %d (%d shards, block_size %d)",
+ ev.VolumeId, generation, len(prot.Shards), prot.BlockSize)
+}
+
+// RemoveBitrotSidecars removes the legacy .ecsum and any versioned
+// .ecsum.v sidecars for a base file name. Best-effort.
+func RemoveBitrotSidecars(base string) {
+ os.Remove(base + BitrotSidecarExt)
+ if matches, _ := filepath.Glob(base + BitrotSidecarExt + ".v*"); matches != nil {
+ for _, m := range matches {
+ os.Remove(m)
+ }
+ }
+}
+
+// findBitrotSidecar resolves the sidecar path for a generation, searching the
+// data base, the index base, and any additional directories — mirroring how
+// shard/.vif lookups handle split data/idx layouts and per-disk mirrors.
+func findBitrotSidecar(generation uint32, dataBase, indexBase string, additionalDirs ...string) string {
+ candidates := []string{
+ BitrotSidecarPath(dataBase, generation),
+ BitrotSidecarPath(indexBase, generation),
+ }
+ base := filepath.Base(dataBase)
+ for _, dir := range additionalDirs {
+ candidates = append(candidates, BitrotSidecarPath(filepath.Join(dir, base), generation))
+ }
+ for _, c := range candidates {
+ if c == "" {
+ continue
+ }
+ if _, err := os.Stat(c); err == nil {
+ return c
+ }
+ }
+ return ""
+}
diff --git a/weed/storage/erasure_coding/ec_bitrot_test.go b/weed/storage/erasure_coding/ec_bitrot_test.go
new file mode 100644
index 000000000..62b6504d7
--- /dev/null
+++ b/weed/storage/erasure_coding/ec_bitrot_test.go
@@ -0,0 +1,362 @@
+package erasure_coding
+
+import (
+ "math/rand"
+ "os"
+ "path/filepath"
+ "slices"
+ "testing"
+
+ "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+ "github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+func TestBitrotSidecarRoundTripAndSelfIntegrity(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "vol.ecsum")
+ prot := &volume_server_pb.EcBitrotProtection{
+ Algorithm: volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+ BlockSize: DefaultBitrotBlockSize,
+ Generation: 0,
+ EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 10, ParityShards: 4},
+ Shards: []*volume_server_pb.EcShardChecksums{
+ {ShardId: 0, CoveredSize: 5, BlockCrc32C: packUint32LE([]uint32{123})},
+ },
+ EncodeUuid: NewEncodeUUID(),
+ }
+ if err := SaveBitrotSidecar(path, prot); err != nil {
+ t.Fatalf("save: %v", err)
+ }
+ got, err := LoadBitrotSidecar(path)
+ if err != nil {
+ t.Fatalf("load: %v", err)
+ }
+ if got.BlockSize != prot.BlockSize || len(got.Shards) != 1 || got.Shards[0].CoveredSize != 5 {
+ t.Fatalf("roundtrip mismatch: %+v", got)
+ }
+
+ // Corrupt a payload byte and confirm the self-integrity check trips.
+ data, _ := os.ReadFile(path)
+ data[bitrotHeaderSize+2] ^= 0xff
+ if err := os.WriteFile(path, data, 0644); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := LoadBitrotSidecar(path); err == nil {
+ t.Fatalf("expected self-integrity failure after payload corruption")
+ }
+}
+
+func TestShardChecksumBuilderCrossesBoundaries(t *testing.T) {
+ blockSize := int64(1 << 20)
+ data := make([]byte, 0, 3*blockSize+12345)
+ r := rand.New(rand.NewSource(1))
+ data = data[:cap(data)]
+ r.Read(data)
+
+ // Feed in odd-sized chunks that cross block boundaries.
+ b := newShardChecksumBuilder(blockSize)
+ for off := 0; off < len(data); off += 7777 {
+ end := off + 7777
+ if end > len(data) {
+ end = len(data)
+ }
+ b.write(data[off:end])
+ }
+ covered, packed := b.finalize()
+ if covered != int64(len(data)) {
+ t.Fatalf("covered=%d want=%d", covered, len(data))
+ }
+ got := unpackUint32LE(packed)
+
+ // Compare to a direct per-block CRC.
+ want := []uint32{}
+ for off := int64(0); off < int64(len(data)); off += blockSize {
+ end := off + blockSize
+ if end > int64(len(data)) {
+ end = int64(len(data))
+ }
+ want = append(want, uint32(needle.NewCRC(data[off:end])))
+ }
+ if len(got) != len(want) {
+ t.Fatalf("block count got=%d want=%d", len(got), len(want))
+ }
+ for i := range want {
+ if got[i] != want[i] {
+ t.Fatalf("block %d crc mismatch got=%x want=%x", i, got[i], want[i])
+ }
+ }
+}
+
+func TestValidateBitrotManifest(t *testing.T) {
+ mk := func(mut func(*volume_server_pb.EcBitrotProtection)) *volume_server_pb.EcBitrotProtection {
+ p := &volume_server_pb.EcBitrotProtection{
+ Algorithm: volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+ BlockSize: DefaultBitrotBlockSize,
+ EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 2, ParityShards: 1},
+ }
+ for id := 0; id < 3; id++ {
+ p.Shards = append(p.Shards, &volume_server_pb.EcShardChecksums{
+ ShardId: uint32(id), CoveredSize: 5, BlockCrc32C: packUint32LE([]uint32{1}),
+ })
+ }
+ if mut != nil {
+ mut(p)
+ }
+ return p
+ }
+ if err := ValidateBitrotManifest(mk(nil), 2, 1); err != nil {
+ t.Fatalf("valid manifest rejected: %v", err)
+ }
+ bad := map[string]func(*volume_server_pb.EcBitrotProtection){
+ "missing shard": func(p *volume_server_pb.EcBitrotProtection) { p.Shards = p.Shards[:2] },
+ "out of range id": func(p *volume_server_pb.EcBitrotProtection) { p.Shards[2].ShardId = 9 },
+ "duplicate id": func(p *volume_server_pb.EcBitrotProtection) { p.Shards[2].ShardId = 0 },
+ "zero covered": func(p *volume_server_pb.EcBitrotProtection) { p.Shards[0].CoveredSize = 0 },
+ "bad crc count": func(p *volume_server_pb.EcBitrotProtection) { p.Shards[0].BlockCrc32C = []byte{1, 2, 3} },
+ "bad block size": func(p *volume_server_pb.EcBitrotProtection) { p.BlockSize = 3 << 20 }, // not power of two
+ "bad algorithm": func(p *volume_server_pb.EcBitrotProtection) {
+ p.Algorithm = volume_server_pb.ChecksumAlgorithm_CHECKSUM_NONE
+ },
+ }
+ for name, mut := range bad {
+ if err := ValidateBitrotManifest(mk(mut), 2, 1); err == nil {
+ t.Errorf("%s: expected validation error, got nil", name)
+ }
+ }
+}
+
+// writeRandomDat creates a .dat of n bytes for encoding.
+func writeRandomDat(t *testing.T, base string, n int) {
+ t.Helper()
+ data := make([]byte, n)
+ rand.New(rand.NewSource(42)).Read(data)
+ if err := os.WriteFile(base+".dat", data, 0644); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestEncodeProducesVerifiableSidecar(t *testing.T) {
+ old := BitrotBlockSize
+ BitrotBlockSize = 1 << 20
+ t.Cleanup(func() { BitrotBlockSize = old })
+
+ dir := t.TempDir()
+ base := filepath.Join(dir, "7")
+ writeRandomDat(t, base, 6*1024*1024) // ~6 MiB
+
+ ctx := &ECContext{DataShards: 10, ParityShards: 4}
+ prot, err := WriteEcFiles(base, ctx)
+ if err != nil {
+ t.Fatalf("encode: %v", err)
+ }
+ if err := SaveBitrotSidecar(BitrotSidecarPath(base, 0), prot); err != nil {
+ t.Fatalf("save sidecar: %v", err)
+ }
+ if err := ValidateBitrotManifest(prot, 10, 4); err != nil {
+ t.Fatalf("encode produced invalid manifest: %v", err)
+ }
+
+ // Every shard file verifies clean against the sidecar right after encode.
+ for id := 0; id < ctx.Total(); id++ {
+ entry := shardChecksums(prot, uint32(id))
+ mism, verr := verifyShardFileBlocks(base+ctx.ToExt(id), entry, int64(prot.BlockSize))
+ if verr != nil {
+ t.Fatalf("verify shard %d: %v", id, verr)
+ }
+ if len(mism) != 0 {
+ t.Fatalf("shard %d unexpectedly mismatched at encode time: %v", id, mism)
+ }
+ }
+
+ // Corrupt one byte in shard 12 (a parity shard) and confirm detection.
+ corruptOneByte(t, base+ctx.ToExt(12))
+ entry := shardChecksums(prot, 12)
+ mism, verr := verifyShardFileBlocks(base+ctx.ToExt(12), entry, int64(prot.BlockSize))
+ if verr != nil {
+ t.Fatalf("verify corrupt shard: %v", verr)
+ }
+ if len(mism) == 0 {
+ t.Fatalf("expected to detect corruption in parity shard 12")
+ }
+}
+
+func TestRebuildExcludesCorruptPresentShard(t *testing.T) {
+ old := BitrotBlockSize
+ BitrotBlockSize = 1 << 20
+ t.Cleanup(func() { BitrotBlockSize = old })
+
+ dir := t.TempDir()
+ base := filepath.Join(dir, "9")
+ writeRandomDat(t, base, 6*1024*1024)
+
+ ctx := &ECContext{DataShards: 10, ParityShards: 4}
+ prot, err := WriteEcFiles(base, ctx)
+ if err != nil {
+ t.Fatalf("encode: %v", err)
+ }
+ if err := SaveBitrotSidecar(BitrotSidecarPath(base, 0), prot); err != nil {
+ t.Fatalf("save sidecar: %v", err)
+ }
+
+ // Snapshot a good copy of shard 3, then delete shard 0 (missing) and corrupt
+ // shard 3 (present-but-corrupt). Rebuild must regenerate both correctly.
+ good3, _ := os.ReadFile(base + ctx.ToExt(3))
+ if err := os.Remove(base + ctx.ToExt(0)); err != nil {
+ t.Fatal(err)
+ }
+ corruptOneByte(t, base+ctx.ToExt(3))
+
+ generated, err := RebuildEcFiles(base, ctx, false)
+ if err != nil {
+ t.Fatalf("rebuild: %v", err)
+ }
+ if !slices.Contains(generated, 0) || !slices.Contains(generated, 3) {
+ t.Fatalf("expected shards 0 and 3 regenerated, got %v", generated)
+ }
+
+ // Shard 3 should now byte-match its original good content, and every shard
+ // verifies clean against the sidecar.
+ now3, _ := os.ReadFile(base + ctx.ToExt(3))
+ if string(now3) != string(good3) {
+ t.Fatalf("rebuilt shard 3 does not match original good content")
+ }
+ for id := 0; id < ctx.Total(); id++ {
+ entry := shardChecksums(prot, uint32(id))
+ mism, verr := verifyShardFileBlocks(base+ctx.ToExt(id), entry, int64(prot.BlockSize))
+ if verr != nil || len(mism) != 0 {
+ t.Fatalf("post-rebuild shard %d not clean: mism=%v err=%v", id, mism, verr)
+ }
+ }
+}
+
+func TestComputeProtectionFromShardsBackfill(t *testing.T) {
+ old := BitrotBlockSize
+ BitrotBlockSize = 1 << 20
+ t.Cleanup(func() { BitrotBlockSize = old })
+
+ dir := t.TempDir()
+ base := filepath.Join(dir, "11")
+ writeRandomDat(t, base, 6*1024*1024)
+ ctx := &ECContext{DataShards: 10, ParityShards: 4}
+ if _, err := WriteEcFiles(base, ctx); err != nil {
+ t.Fatalf("encode: %v", err)
+ }
+
+ // Backfill from the on-disk shards (no encode pass) yields a complete,
+ // valid manifest that every shard verifies clean against.
+ prot, err := ComputeProtectionFromShards(base, ctx, 0, nil)
+ if err != nil {
+ t.Fatalf("backfill: %v", err)
+ }
+ if err := ValidateBitrotManifest(prot, 10, 4); err != nil {
+ t.Fatalf("backfill manifest invalid: %v", err)
+ }
+ for id := 0; id < ctx.Total(); id++ {
+ entry := shardChecksums(prot, uint32(id))
+ mism, verr := verifyShardFileBlocks(base+ctx.ToExt(id), entry, int64(prot.BlockSize))
+ if verr != nil || len(mism) != 0 {
+ t.Fatalf("backfilled shard %d not clean: mism=%v err=%v", id, mism, verr)
+ }
+ }
+
+ // A missing shard must abort backfill (never write a partial manifest).
+ if err := os.Remove(base + ctx.ToExt(5)); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := ComputeProtectionFromShards(base, ctx, 0, nil); err == nil {
+ t.Fatalf("expected backfill error with a missing shard")
+ }
+}
+
+func TestRebuildFailClosedCleansUpGeneratedShard(t *testing.T) {
+ old := BitrotBlockSize
+ BitrotBlockSize = 1 << 20
+ t.Cleanup(func() { BitrotBlockSize = old })
+
+ dir := t.TempDir()
+ base := filepath.Join(dir, "13")
+ writeRandomDat(t, base, 6*1024*1024)
+ ctx := &ECContext{DataShards: 10, ParityShards: 4}
+ prot, err := WriteEcFiles(base, ctx)
+ if err != nil {
+ t.Fatalf("encode: %v", err)
+ }
+
+ // Tamper one block CRC of shard 0 but re-save so the header self-integrity and
+ // manifest stay valid: a well-formed but wrong ("stale") sidecar that loads as ON.
+ for _, s := range prot.Shards {
+ if s.ShardId == 0 && len(s.BlockCrc32C) >= 1 {
+ s.BlockCrc32C[0] ^= 0xff
+ }
+ }
+ if err := SaveBitrotSidecar(BitrotSidecarPath(base, 0), prot); err != nil {
+ t.Fatalf("save sidecar: %v", err)
+ }
+
+ // Delete shard 0 so the rebuild regenerates it; the regenerated (correct)
+ // bytes will not match the tampered sidecar entry -> post-verify fails closed.
+ shard0 := base + ctx.ToExt(0)
+ if err := os.Remove(shard0); err != nil {
+ t.Fatal(err)
+ }
+
+ if _, rerr := RebuildEcFiles(base, ctx, false); rerr == nil {
+ t.Fatalf("expected rebuild to fail closed on the stale sidecar")
+ }
+ // The genuinely-missing shard 0 must NOT be published: fail-closed cleanup
+ // returns it to missing.
+ if _, err := os.Stat(shard0); !os.IsNotExist(err) {
+ t.Fatalf("shard 0 should have been cleaned up after a failed rebuild, err=%v", err)
+ }
+
+ // With the unsafe override, the rebuild proceeds and regenerates shard 0.
+ if _, rerr := RebuildEcFiles(base, ctx, true); rerr != nil {
+ t.Fatalf("rebuild with unsafeIgnoreSidecar should succeed: %v", rerr)
+ }
+ if _, err := os.Stat(shard0); err != nil {
+ t.Fatalf("shard 0 should exist after unsafe rebuild: %v", err)
+ }
+}
+
+func TestRemoveBitrotSidecars(t *testing.T) {
+ dir := t.TempDir()
+ base := filepath.Join(dir, "5")
+ // Legacy + two versioned sidecars, plus an unrelated file that must survive.
+ for _, p := range []string{base + ".ecsum", base + ".ecsum.v1", base + ".ecsum.v7"} {
+ if err := os.WriteFile(p, []byte("x"), 0644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ keep := base + ".ec00"
+ if err := os.WriteFile(keep, []byte("x"), 0644); err != nil {
+ t.Fatal(err)
+ }
+
+ RemoveBitrotSidecars(base)
+
+ for _, p := range []string{base + ".ecsum", base + ".ecsum.v1", base + ".ecsum.v7"} {
+ if _, err := os.Stat(p); !os.IsNotExist(err) {
+ t.Errorf("expected %s removed, err=%v", p, err)
+ }
+ }
+ if _, err := os.Stat(keep); err != nil {
+ t.Errorf("unrelated file %s should survive: %v", keep, err)
+ }
+}
+
+func corruptOneByte(t *testing.T, path string) {
+ t.Helper()
+ f, err := os.OpenFile(path, os.O_RDWR, 0644)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+ var b [1]byte
+ if _, err := f.ReadAt(b[:], 0); err != nil {
+ t.Fatal(err)
+ }
+ b[0] ^= 0xff
+ if _, err := f.WriteAt(b[:], 0); err != nil {
+ t.Fatal(err)
+ }
+}
diff --git a/weed/storage/erasure_coding/ec_consistency_test.go b/weed/storage/erasure_coding/ec_consistency_test.go
index ec6e7d632..da1a26cc7 100644
--- a/weed/storage/erasure_coding/ec_consistency_test.go
+++ b/weed/storage/erasure_coding/ec_consistency_test.go
@@ -18,7 +18,7 @@ import (
// existed in VolumeEcShardsGenerate before the fix in this PR.
//
// Previously, the order was:
-// 1. WriteEcFilesWithContext(baseFileName, ecCtx) — EC shards from .dat
+// 1. WriteEcFiles(baseFileName, ecCtx) — EC shards from .dat
// 2. WriteSortedFileFromIdx(v.IndexFileName(), ".ecx") — .ecx from .idx
//
// If a write appended data to .dat/.idx between steps 1 and 2, the .ecx would
@@ -48,7 +48,7 @@ func TestEcConsistency_WritesBetweenEncodeAndEcx(t *testing.T) {
})
// Phase 2: EC encode — generates .ec00-.ec13 from current .dat
- err = generateEcFiles(baseFileName, int(smallBlockSize), largeBlockSize, smallBlockSize, ctx)
+ _, err = generateEcFiles(baseFileName, int(smallBlockSize), largeBlockSize, smallBlockSize, ctx)
require.NoError(t, err, "EC encoding")
// Phase 3: SIMULATE a write between EC encoding and .ecx generation
@@ -145,7 +145,7 @@ func TestEcConsistency_ExactLargeRowEncoding(t *testing.T) {
require.NoError(t, err)
// EC encode
- err = generateEcFiles(baseFileName, int(smallBlockSize), largeBlockSize, smallBlockSize, ctx)
+ _, err = generateEcFiles(baseFileName, int(smallBlockSize), largeBlockSize, smallBlockSize, ctx)
require.NoError(t, err)
// Check shard sizes — each shard should be exactly largeBlockSize
diff --git a/weed/storage/erasure_coding/ec_context.go b/weed/storage/erasure_coding/ec_context.go
index 770fe41af..9fc352d83 100644
--- a/weed/storage/erasure_coding/ec_context.go
+++ b/weed/storage/erasure_coding/ec_context.go
@@ -30,6 +30,17 @@ func NewDefaultECContext(collection string, volumeId needle.VolumeId) *ECContext
}
}
+// BackgroundECContext returns a non-nil placeholder EC context, analogous to
+// context.Background(): pass it to WriteEcFiles / RebuildEcFiles when the caller
+// has no specific layout, rather than a nil context. Its zero Total() is the
+// "unset" signal — WriteEcFiles resolves it to the default ratio and
+// RebuildEcFiles resolves it from the volume's .vif (falling back to default),
+// so the placeholder itself never reaches the encoder. A fresh value is returned
+// each call so callers cannot mutate a shared default.
+func BackgroundECContext() *ECContext {
+ return &ECContext{}
+}
+
// CreateEncoder creates a Reed-Solomon encoder for this context
func (ctx *ECContext) CreateEncoder() (reedsolomon.Encoder, error) {
return reedsolomon.New(ctx.DataShards, ctx.ParityShards)
diff --git a/weed/storage/erasure_coding/ec_encoder.go b/weed/storage/erasure_coding/ec_encoder.go
index cd4d95fd8..b0e689eb4 100644
--- a/weed/storage/erasure_coding/ec_encoder.go
+++ b/weed/storage/erasure_coding/ec_encoder.go
@@ -9,6 +9,7 @@ import (
"github.com/klauspost/reedsolomon"
"github.com/seaweedfs/seaweedfs/weed/glog"
+ "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/idx"
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
@@ -57,74 +58,89 @@ func WriteSortedFileFromIdx(baseFileName string, ext string) (e error) {
return nil
}
-// WriteEcFiles generates .ec00 ~ .ec13 files using default EC context
-func WriteEcFiles(baseFileName string) error {
- ctx := NewDefaultECContext("", 0)
- return WriteEcFilesWithContext(baseFileName, ctx)
-}
-
-// WriteEcFilesWithContext generates EC files using the provided context
-func WriteEcFilesWithContext(baseFileName string, ctx *ECContext) error {
+// WriteEcFiles generates .ec00 ~ .ec13 files from baseFileName.dat. Pass
+// BackgroundECContext for the default ratio, or an explicit ctx for a configured
+// (e.g. custom-ratio) layout. It returns the bitrot protection (per-shard block
+// CRC32C) computed during the single encode pass; the caller persists it as a
+// .ecsum sidecar.
+func WriteEcFiles(baseFileName string, ctx *ECContext) (*volume_server_pb.EcBitrotProtection, error) {
+ if ctx == nil || ctx.Total() == 0 {
+ ctx = NewDefaultECContext("", 0)
+ }
return generateEcFiles(baseFileName, 256*1024, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, ctx)
}
-// RebuildEcFiles rebuilds missing EC shard files.
+// RebuildEcFiles rebuilds missing EC shard files. Pass BackgroundECContext to
+// resolve the layout from the volume's .vif (falling back to the default ratio),
+// or an explicit ctx when the caller already knows the shard layout.
// additionalDirs are extra directories to search for existing shard files,
// which handles multi-disk servers where shards may be spread across disks.
-func RebuildEcFiles(baseFileName string, additionalDirs ...string) ([]uint32, error) {
- // Attempt to load EC config from .vif file to preserve original configuration
- var ctx *ECContext
- if volumeInfo, _, found, _ := volume_info.MaybeLoadVolumeInfo(baseFileName + ".vif"); found && volumeInfo.EcShardConfig != nil {
- ds := int(volumeInfo.EcShardConfig.DataShards)
- ps := int(volumeInfo.EcShardConfig.ParityShards)
+// When a bitrot checksum sidecar is present for the (generation-0) volume,
+// present input shards are verified against it and corrupt ones are excluded
+// from Reed-Solomon and regenerated; unsafeIgnoreSidecar bypasses that guard.
+func RebuildEcFiles(baseFileName string, ctx *ECContext, unsafeIgnoreSidecar bool, additionalDirs ...string) ([]uint32, error) {
+ if ctx == nil || ctx.Total() == 0 {
+ // Resolve the layout from the .vif to preserve the original configuration.
+ volumeInfo, _, foundVif, vifErr := volume_info.MaybeLoadVolumeInfo(baseFileName + ".vif")
+ if vifErr != nil {
+ // The .vif exists but cannot be read or parsed. Fail closed rather
+ // than silently falling back to the default ratio, which would
+ // rebuild a custom-ratio volume with the wrong layout. Pass an
+ // explicit ctx to override.
+ return nil, fmt.Errorf("RebuildEcFiles %s: cannot load .vif: %w", baseFileName, vifErr)
+ }
+ if foundVif && volumeInfo.EcShardConfig != nil {
+ ds := int(volumeInfo.EcShardConfig.DataShards)
+ ps := int(volumeInfo.EcShardConfig.ParityShards)
- // Validate EC config before using it
- if ds > 0 && ps > 0 && ds+ps <= MaxShardCount {
- ctx = &ECContext{
- DataShards: ds,
- ParityShards: ps,
+ // Validate EC config before using it
+ if ds > 0 && ps > 0 && ds+ps <= MaxShardCount {
+ ctx = &ECContext{
+ DataShards: ds,
+ ParityShards: ps,
+ }
+ glog.V(0).Infof("Rebuilding EC files for %s with config from .vif: %s", baseFileName, ctx.String())
+ } else {
+ glog.Warningf("Invalid EC config in .vif for %s (data=%d, parity=%d), using default", baseFileName, ds, ps)
+ ctx = NewDefaultECContext("", 0)
}
- glog.V(0).Infof("Rebuilding EC files for %s with config from .vif: %s", baseFileName, ctx.String())
} else {
- glog.Warningf("Invalid EC config in .vif for %s (data=%d, parity=%d), using default", baseFileName, ds, ps)
+ glog.V(0).Infof("Rebuilding EC files for %s with default config", baseFileName)
ctx = NewDefaultECContext("", 0)
}
- } else {
- glog.V(0).Infof("Rebuilding EC files for %s with default config", baseFileName)
- ctx = NewDefaultECContext("", 0)
}
- return RebuildEcFilesWithContext(baseFileName, ctx, additionalDirs...)
-}
-
-// RebuildEcFilesWithContext rebuilds missing EC files using the provided context.
-// additionalDirs are extra directories to search for existing shard files.
-func RebuildEcFilesWithContext(baseFileName string, ctx *ECContext, additionalDirs ...string) ([]uint32, error) {
- return generateMissingEcFiles(baseFileName, 256*1024, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, ctx, additionalDirs)
+ return generateMissingEcFiles(baseFileName, 256*1024, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, ctx, unsafeIgnoreSidecar, additionalDirs)
}
func ToExt(ecIndex int) string {
return fmt.Sprintf(".ec%02d", ecIndex)
}
-func generateEcFiles(baseFileName string, bufferSize int, largeBlockSize int64, smallBlockSize int64, ctx *ECContext) error {
+func generateEcFiles(baseFileName string, bufferSize int, largeBlockSize int64, smallBlockSize int64, ctx *ECContext) (*volume_server_pb.EcBitrotProtection, error) {
file, err := os.OpenFile(baseFileName+".dat", os.O_RDONLY, 0)
if err != nil {
- return fmt.Errorf("failed to open dat file: %w", err)
+ return nil, fmt.Errorf("failed to open dat file: %w", err)
}
defer file.Close()
fi, err := file.Stat()
if err != nil {
- return fmt.Errorf("failed to stat dat file: %w", err)
+ return nil, fmt.Errorf("failed to stat dat file: %w", err)
+ }
+
+ // One rolling-CRC builder per shard; fed as each shard's bytes are written.
+ builders := make([]*shardChecksumBuilder, ctx.Total())
+ for i := range builders {
+ builders[i] = newShardChecksumBuilder(BitrotBlockSize)
}
glog.V(0).Infof("encodeDatFile %s.dat size:%d with EC context %s", baseFileName, fi.Size(), ctx.String())
- err = encodeDatFile(fi.Size(), baseFileName, bufferSize, largeBlockSize, file, smallBlockSize, ctx)
+ err = encodeDatFile(fi.Size(), baseFileName, bufferSize, largeBlockSize, file, smallBlockSize, ctx, builders)
if err != nil {
- return fmt.Errorf("encodeDatFile: %w", err)
+ return nil, fmt.Errorf("encodeDatFile: %w", err)
}
- return nil
+ return buildProtectionFromBuilders(ctx, builders, BitrotBlockSize), nil
}
// findShardFile looks for a shard file at baseFileName+ext, then in additionalDirs.
@@ -143,12 +159,12 @@ func findShardFile(baseFileName string, ext string, additionalDirs []string) str
return ""
}
-func generateMissingEcFiles(baseFileName string, bufferSize int, largeBlockSize int64, smallBlockSize int64, ctx *ECContext, additionalDirs []string) (generatedShardIds []uint32, err error) {
+func generateMissingEcFiles(baseFileName string, bufferSize int, largeBlockSize int64, smallBlockSize int64, ctx *ECContext, unsafeIgnoreSidecar bool, additionalDirs []string) (generatedShardIds []uint32, err error) {
// Pass 1: discover which shards exist and which are missing,
// opening input files but NOT creating output files yet.
shardHasData := make([]bool, ctx.Total())
- shardPaths := make([]string, ctx.Total()) // non-empty for present shards
+ shardPaths := make([]string, ctx.Total()) // non-empty for present shards (also the in-place output for a reclassified-corrupt shard)
inputFiles := make([]*os.File, ctx.Total())
presentCount := 0
for shardId := 0; shardId < ctx.Total(); shardId++ {
@@ -168,6 +184,67 @@ func generateMissingEcFiles(baseFileName string, bufferSize int, largeBlockSize
}
}
+ // Bitrot verify-and-exclude: when a generation-0 checksum sidecar is present
+ // and valid, verify each present input shard against it and reclassify
+ // corrupt ones as missing so Reed-Solomon regenerates them instead of
+ // silently consuming corrupt bytes. corruptOwned marks shards whose
+ // (corrupt) original file must be replaced in place at its discovered path.
+ corruptOwned := make([]bool, ctx.Total())
+ prot, status := loadRebuildSidecar(baseFileName, ctx, additionalDirs)
+ switch status {
+ case BitrotInvalid:
+ if !unsafeIgnoreSidecar {
+ return nil, fmt.Errorf("bitrot sidecar for %s is malformed/unverifiable; refusing to rebuild (pass unsafeIgnoreSidecar to override)", baseFileName)
+ }
+ glog.Warningf("bitrot sidecar for %s is malformed/unverifiable; proceeding because unsafeIgnoreSidecar is set", baseFileName)
+ case BitrotOn:
+ corrupt := make([]int, 0, ctx.Total())
+ for shardId := 0; shardId < ctx.Total(); shardId++ {
+ if !shardHasData[shardId] {
+ continue
+ }
+ entry := shardChecksums(prot, uint32(shardId))
+ if entry == nil {
+ continue
+ }
+ mismatched, verr := verifyShardFileBlocks(shardPaths[shardId], entry, int64(prot.BlockSize))
+ if verr != nil {
+ // A read error means we cannot trust this shard as a Reed-Solomon
+ // input. Exclude it (treat as corrupt) rather than silently
+ // feeding possibly-corrupt bytes into reconstruction.
+ glog.Warningf("bitrot: failed to verify present shard %d for %s: %v; excluding it", shardId, baseFileName, verr)
+ corrupt = append(corrupt, shardId)
+ continue
+ }
+ if len(mismatched) > 0 {
+ corrupt = append(corrupt, shardId)
+ }
+ }
+ if len(corrupt) > 0 {
+ // Wholesale-mismatch guard (RS-arbiter conservative form): localized
+ // bitrot touches a few shards; a stale/wrong sidecar mismatches more
+ // than parity_shards. In that case refuse rather than excluding good
+ // shards en masse.
+ if len(corrupt) > ctx.ParityShards && !unsafeIgnoreSidecar {
+ return nil, fmt.Errorf("bitrot sidecar suspect for %s: %d/%d present shards mismatch (> parity %d); refusing to rebuild (pass unsafeIgnoreSidecar to override)",
+ baseFileName, len(corrupt), presentCount, ctx.ParityShards)
+ }
+ if presentCount-len(corrupt) < ctx.DataShards && !unsafeIgnoreSidecar {
+ return nil, fmt.Errorf("bitrot: only %d verified-good shards for %s, need %d data shards; sidecar may be stale (pass unsafeIgnoreSidecar to override)",
+ presentCount-len(corrupt), baseFileName, ctx.DataShards)
+ }
+ if !unsafeIgnoreSidecar {
+ for _, shardId := range corrupt {
+ glog.Warningf("bitrot: present shard %d for %s fails checksum; excluding from rebuild inputs and regenerating", shardId, baseFileName)
+ shardHasData[shardId] = false
+ corruptOwned[shardId] = true
+ generatedShardIds = append(generatedShardIds, uint32(shardId))
+ presentCount--
+ }
+ }
+ }
+ }
+
// Pre-check: bail out before creating any output files.
if presentCount < ctx.DataShards {
return nil, fmt.Errorf("not enough shards to rebuild %s: found %d shards, need at least %d (data shards), missing shards: %v",
@@ -177,29 +254,131 @@ func generateMissingEcFiles(baseFileName string, bufferSize int, largeBlockSize
glog.V(0).Infof("rebuilding %s: %d shards present, %d missing %v, config %s",
baseFileName, presentCount, len(generatedShardIds), generatedShardIds, ctx.String())
- // Pass 2: create output files for missing shards now that we know
- // reconstruction is possible.
+ // Pass 2: create output files for missing shards. A genuinely-absent shard
+ // is written at baseFileName+ext; a reclassified-corrupt shard is written to
+ // a temp file beside its discovered location and atomically renamed over the
+ // corrupt original after the rebuild (and checksum) succeed, so we never
+ // leave a duplicate shard id or a half-written file.
outputFiles := make([]*os.File, ctx.Total())
+ writePaths := make([]string, ctx.Total())
+ finalPaths := make([]string, ctx.Total())
for shardId := 0; shardId < ctx.Total(); shardId++ {
if shardHasData[shardId] {
continue
}
- outputFileName := baseFileName + ctx.ToExt(shardId)
- outputFiles[shardId], err = os.OpenFile(outputFileName, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0644)
+ finalPath := baseFileName + ctx.ToExt(shardId)
+ writePath := finalPath
+ if corruptOwned[shardId] && shardPaths[shardId] != "" {
+ finalPath = shardPaths[shardId]
+ writePath = shardPaths[shardId] + ".rebuilding"
+ }
+ outputFiles[shardId], err = os.OpenFile(writePath, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return nil, err
}
defer outputFiles[shardId].Close()
+ writePaths[shardId] = writePath
+ finalPaths[shardId] = finalPath
}
- err = rebuildEcFiles(shardHasData, inputFiles, outputFiles, ctx)
- if err != nil {
+ if err = rebuildEcFiles(shardHasData, inputFiles, outputFiles, ctx); err != nil {
return nil, fmt.Errorf("rebuildEcFiles: %w", err)
}
+
+ // Verify regenerated shards against the sidecar. Reed-Solomon is
+ // deterministic, so a regenerated shard that does NOT match the sidecar
+ // means the sidecar is wrong/stale (not the shard) — fail closed rather than
+ // publishing bytes we cannot trust. On ANY verification failure (sync, read
+ // error, or mismatch) remove every generated output so the rebuild publishes
+ // nothing: a genuinely-missing shard returns to missing; a reclassified-
+ // corrupt shard keeps its untouched original.
+ if status == BitrotOn && !unsafeIgnoreSidecar {
+ for shardId := 0; shardId < ctx.Total(); shardId++ {
+ if writePaths[shardId] == "" {
+ continue
+ }
+ entry := shardChecksums(prot, uint32(shardId))
+ if entry == nil {
+ continue
+ }
+ if err = outputFiles[shardId].Sync(); err != nil {
+ cleanupRebuildOutputs(outputFiles, writePaths)
+ return nil, fmt.Errorf("sync regenerated shard %d: %w", shardId, err)
+ }
+ mismatched, verr := verifyShardFileBlocks(writePaths[shardId], entry, int64(prot.BlockSize))
+ if verr != nil {
+ cleanupRebuildOutputs(outputFiles, writePaths)
+ return nil, fmt.Errorf("bitrot: verify regenerated shard %d for %s: %w", shardId, baseFileName, verr)
+ }
+ if len(mismatched) > 0 {
+ cleanupRebuildOutputs(outputFiles, writePaths)
+ return nil, fmt.Errorf("bitrot: regenerated shard %d for %s does not match sidecar (%d blocks differ); sidecar likely stale — aborting (pass unsafeIgnoreSidecar to override)",
+ shardId, baseFileName, len(mismatched))
+ }
+ }
+ }
+
+ // Atomically move reclassified-corrupt rebuilds over their originals.
+ for shardId := 0; shardId < ctx.Total(); shardId++ {
+ if writePaths[shardId] != "" && writePaths[shardId] != finalPaths[shardId] {
+ outputFiles[shardId].Close()
+ if rerr := os.Rename(writePaths[shardId], finalPaths[shardId]); rerr != nil {
+ return nil, fmt.Errorf("bitrot: replace corrupt shard %d (%s -> %s): %w", shardId, writePaths[shardId], finalPaths[shardId], rerr)
+ }
+ }
+ }
return
}
-func encodeData(file *os.File, enc reedsolomon.Encoder, startOffset, blockSize int64, buffers [][]byte, outputs []*os.File, ctx *ECContext) error {
+// cleanupRebuildOutputs removes every generated output on a failed fail-closed
+// rebuild: temp replacements AND genuinely-missing shards written directly at
+// their final path, so no unverified bytes are published. A reclassified-corrupt
+// shard's untouched original (at finalPath, distinct from its temp writePath) is
+// left in place; a genuinely-missing shard (writePath == finalPath) returns to
+// missing.
+func cleanupRebuildOutputs(outputFiles []*os.File, writePaths []string) {
+ for i := range writePaths {
+ if writePaths[i] == "" {
+ continue
+ }
+ if outputFiles[i] != nil {
+ outputFiles[i].Close()
+ }
+ os.Remove(writePaths[i])
+ }
+}
+
+// loadRebuildSidecar loads and validates the generation-0 checksum sidecar for a
+// rebuild. RebuildEcFiles operates on the un-suffixed (generation 0) shard
+// names, so only the legacy sidecar is relevant here. Returns BitrotOff when
+// absent or describing a different generation/config, BitrotInvalid on a
+// self-integrity/manifest failure, BitrotOn when usable.
+func loadRebuildSidecar(baseFileName string, ctx *ECContext, additionalDirs []string) (*volume_server_pb.EcBitrotProtection, BitrotStatus) {
+ path := findBitrotSidecar(0, baseFileName, baseFileName, additionalDirs...)
+ if path == "" {
+ return nil, BitrotOff
+ }
+ prot, err := LoadBitrotSidecar(path)
+ if err != nil {
+ glog.Warningf("bitrot: sidecar %s self-integrity failed: %v", path, err)
+ return nil, BitrotInvalid
+ }
+ if prot.Generation != 0 {
+ return nil, BitrotOff
+ }
+ if prot.EcShardConfig == nil ||
+ int(prot.EcShardConfig.DataShards) != ctx.DataShards ||
+ int(prot.EcShardConfig.ParityShards) != ctx.ParityShards {
+ return nil, BitrotOff
+ }
+ if err := ValidateBitrotManifest(prot, ctx.DataShards, ctx.ParityShards); err != nil {
+ glog.Warningf("bitrot: sidecar %s manifest invalid: %v", path, err)
+ return nil, BitrotInvalid
+ }
+ return prot, BitrotOn
+}
+
+func encodeData(file *os.File, enc reedsolomon.Encoder, startOffset, blockSize int64, buffers [][]byte, outputs []*os.File, ctx *ECContext, builders []*shardChecksumBuilder) error {
bufferSize := int64(len(buffers[0]))
if bufferSize == 0 {
@@ -212,7 +391,7 @@ func encodeData(file *os.File, enc reedsolomon.Encoder, startOffset, blockSize i
}
for b := int64(0); b < batchCount; b++ {
- err := encodeDataOneBatch(file, enc, startOffset+b*bufferSize, blockSize, buffers, outputs, ctx)
+ err := encodeDataOneBatch(file, enc, startOffset+b*bufferSize, blockSize, buffers, outputs, ctx, builders)
if err != nil {
return err
}
@@ -245,7 +424,7 @@ func closeEcFiles(files []*os.File) {
}
}
-func encodeDataOneBatch(file *os.File, enc reedsolomon.Encoder, startOffset, blockSize int64, buffers [][]byte, outputs []*os.File, ctx *ECContext) error {
+func encodeDataOneBatch(file *os.File, enc reedsolomon.Encoder, startOffset, blockSize int64, buffers [][]byte, outputs []*os.File, ctx *ECContext, builders []*shardChecksumBuilder) error {
// read data into buffers
for i := 0; i < ctx.DataShards; i++ {
@@ -272,12 +451,16 @@ func encodeDataOneBatch(file *os.File, enc reedsolomon.Encoder, startOffset, blo
if err != nil {
return err
}
+ // Accumulate this shard's block CRC over exactly the bytes written.
+ if builders != nil && builders[i] != nil {
+ builders[i].write(buffers[i])
+ }
}
return nil
}
-func encodeDatFile(remainingSize int64, baseFileName string, bufferSize int, largeBlockSize int64, file *os.File, smallBlockSize int64, ctx *ECContext) error {
+func encodeDatFile(remainingSize int64, baseFileName string, bufferSize int, largeBlockSize int64, file *os.File, smallBlockSize int64, ctx *ECContext, builders []*shardChecksumBuilder) error {
var processedSize int64
@@ -302,7 +485,7 @@ func encodeDatFile(remainingSize int64, baseFileName string, bufferSize int, lar
smallRowSize := smallBlockSize * int64(ctx.DataShards)
for remainingSize >= largeRowSize {
- err = encodeData(file, enc, processedSize, largeBlockSize, buffers, outputs, ctx)
+ err = encodeData(file, enc, processedSize, largeBlockSize, buffers, outputs, ctx, builders)
if err != nil {
return fmt.Errorf("failed to encode large chunk data: %w", err)
}
@@ -310,7 +493,7 @@ func encodeDatFile(remainingSize int64, baseFileName string, bufferSize int, lar
processedSize += largeRowSize
}
for remainingSize > 0 {
- err = encodeData(file, enc, processedSize, smallBlockSize, buffers, outputs, ctx)
+ err = encodeData(file, enc, processedSize, smallBlockSize, buffers, outputs, ctx, builders)
if err != nil {
return fmt.Errorf("failed to encode small chunk data: %w", err)
}
diff --git a/weed/storage/erasure_coding/ec_roundtrip_test.go b/weed/storage/erasure_coding/ec_roundtrip_test.go
index 38fdfff16..0ba3fc990 100644
--- a/weed/storage/erasure_coding/ec_roundtrip_test.go
+++ b/weed/storage/erasure_coding/ec_roundtrip_test.go
@@ -81,7 +81,7 @@ func testEcRead(t *testing.T, large, small, datSize int64) {
ctx := NewDefaultECContext("", 0)
// 2. EC encode with test block sizes
- err = generateEcFiles(baseFileName, int(small), large, small, ctx)
+ _, err = generateEcFiles(baseFileName, int(small), large, small, ctx)
require.NoError(t, err, "EC encoding")
// 3. Open EC shard files for reading
@@ -213,7 +213,7 @@ func TestEcOffByOneBug_Issue8947(t *testing.T) {
require.NoError(t, err)
ctx := NewDefaultECContext("", 0)
- err = generateEcFiles(baseFileName, int(small), large, small, ctx)
+ _, err = generateEcFiles(baseFileName, int(small), large, small, ctx)
require.NoError(t, err, "EC encoding")
ecFiles, err := openEcFiles(baseFileName, true, ctx)
@@ -316,7 +316,7 @@ func testDecodeDat(t *testing.T, datSize int64) {
ctx := NewDefaultECContext("", 0)
// 2. EC encode with PRODUCTION block sizes
- err = generateEcFiles(baseFileName, 256*1024, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, ctx)
+ _, err = generateEcFiles(baseFileName, 256*1024, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, ctx)
require.NoError(t, err, "EC encoding")
// 3. Decode via WriteDatFile
diff --git a/weed/storage/erasure_coding/ec_test.go b/weed/storage/erasure_coding/ec_test.go
index 3894d75d3..01189bf5e 100644
--- a/weed/storage/erasure_coding/ec_test.go
+++ b/weed/storage/erasure_coding/ec_test.go
@@ -27,7 +27,7 @@ func TestEncodingDecoding(t *testing.T) {
// Create default EC context for testing
ctx := NewDefaultECContext("", 0)
- err := generateEcFiles(baseFileName, bufferSize, largeBlockSize, smallBlockSize, ctx)
+ _, err := generateEcFiles(baseFileName, bufferSize, largeBlockSize, smallBlockSize, ctx)
if err != nil {
t.Logf("generateEcFiles: %v", err)
}
@@ -194,6 +194,7 @@ func removeGeneratedFiles(baseFileName string, ctx *ECContext) {
os.Remove(fname)
}
os.Remove(baseFileName + ".ecx")
+ RemoveBitrotSidecars(baseFileName)
}
func TestLocateData(t *testing.T) {
diff --git a/weed/storage/erasure_coding/ec_volume.go b/weed/storage/erasure_coding/ec_volume.go
index fd5e5b5db..4860ae2ce 100644
--- a/weed/storage/erasure_coding/ec_volume.go
+++ b/weed/storage/erasure_coding/ec_volume.go
@@ -59,6 +59,13 @@ type EcVolume struct {
// Seeded from .ecj in NewEcVolume and updated under deletedNeedlesLock.
deletedNeedlesLock sync.RWMutex
deletedNeedles map[types.NeedleId]struct{}
+
+ // Bitrot checksum sidecar for the active generation (optional). bitrot is
+ // nil unless bitrotStatus == BitrotOn, and is loaded at mount. Guarded by
+ // bitrotLock.
+ bitrotLock sync.RWMutex
+ bitrot *volume_server_pb.EcBitrotProtection
+ bitrotStatus BitrotStatus
}
func NewEcVolume(diskType types.DiskType, dir string, dirIdx string, collection string, vid needle.VolumeId) (ev *EcVolume, err error) {
@@ -168,6 +175,9 @@ func NewEcVolume(diskType types.DiskType, dir string, dirIdx string, collection
ev.ShardLocations = make(map[ShardId][]pb.ServerAddress)
+ // Load the active-generation bitrot checksum sidecar (optional).
+ ev.loadActiveBitrotSidecar()
+
return
}
@@ -257,6 +267,12 @@ func (ev *EcVolume) Destroy() {
os.Remove(ev.FileName(".ecx"))
os.Remove(ev.FileName(".ecj"))
os.Remove(ev.FileName(".vif"))
+ // Remove the bitrot checksum sidecar(s) so a later volume reuse cannot load
+ // stale protection. Search both the data and index bases.
+ RemoveBitrotSidecars(ev.DataBaseFileName())
+ if ev.IndexBaseFileName() != ev.DataBaseFileName() {
+ RemoveBitrotSidecars(ev.IndexBaseFileName())
+ }
}
// DiskType returns the disk type the EC volume currently reports under.
diff --git a/weed/storage/erasure_coding/ec_volume_scrub.go b/weed/storage/erasure_coding/ec_volume_scrub.go
index a563875f7..aa759058b 100644
--- a/weed/storage/erasure_coding/ec_volume_scrub.go
+++ b/weed/storage/erasure_coding/ec_volume_scrub.go
@@ -1,6 +1,7 @@
package erasure_coding
import (
+ "bytes"
"fmt"
"slices"
@@ -22,6 +23,190 @@ func (ev *EcVolume) ScrubIndex() (int64, []error) {
return idx.CheckIndexFile(ev.ecxFile, ev.ecxFileSize, ev.Version)
}
+// ChecksumScrub verifies every locally-held EC shard's raw bytes against the
+// active-generation bitrot checksum sidecar. It is read-only: it detects and
+// reports corruption but never mutates or quarantines anything (the apply path
+// drives quarantine + rebuild). It is the only path that exercises cold parity
+// shards, which are never read during normal serving.
+//
+// Returns the number of blocks scanned, the broken shards (sidecar mismatch or
+// length drift), and errors. A wholesale mismatch (more than parity_shards
+// shards differing) is reported as a suspect/stale sidecar — an integrity error
+// — rather than flagging the shards as corrupt, so a bad sidecar can never be
+// mistaken for mass bitrot.
+func (ecv *EcVolume) ChecksumScrub() (blocksScanned int64, brokenShards []*volume_server_pb.EcShardInfo, errs []error) {
+ prot, status := ecv.BitrotProtection()
+ switch status {
+ case BitrotOff:
+ // Unprotected generation: nothing to verify. Not an error.
+ return 0, nil, nil
+ case BitrotInvalid:
+ return 0, nil, []error{fmt.Errorf("ec volume %d: bitrot sidecar is malformed/unverifiable (sidecar integrity)", ecv.VolumeId)}
+ }
+
+ blockSize := int64(prot.BlockSize)
+ var broken []*EcVolumeShard
+ for _, shard := range ecv.Shards {
+ entry := shardChecksums(prot, uint32(shard.ShardId))
+ if entry == nil {
+ errs = append(errs, fmt.Errorf("ec volume %d: no checksum entry for local shard %d", ecv.VolumeId, shard.ShardId))
+ continue
+ }
+ if shard.Size() != entry.CoveredSize {
+ errs = append(errs, fmt.Errorf("ec volume %d shard %d: size %d != covered_size %d (truncated or extended)",
+ ecv.VolumeId, shard.ShardId, shard.Size(), entry.CoveredSize))
+ broken = append(broken, shard)
+ continue
+ }
+ want := unpackUint32LE(entry.BlockCrc32C)
+ buf := make([]byte, blockSize)
+ var offset int64
+ shardBad := false
+ for i := 0; i < len(want); i++ {
+ toRead := blockSize
+ if rem := entry.CoveredSize - offset; rem < toRead {
+ toRead = rem
+ }
+ n, rerr := shard.ReadAt(buf[:toRead], offset)
+ if rerr != nil || int64(n) != toRead {
+ errs = append(errs, fmt.Errorf("ec volume %d shard %d: read block %d at %d: %v (got %d/%d)",
+ ecv.VolumeId, shard.ShardId, i, offset, rerr, n, toRead))
+ shardBad = true
+ break
+ }
+ if uint32(needle.NewCRC(buf[:n])) != want[i] {
+ errs = append(errs, fmt.Errorf("ec volume %d shard %d: checksum mismatch at block %d (offset %d)",
+ ecv.VolumeId, shard.ShardId, i, offset))
+ shardBad = true
+ }
+ offset += int64(n)
+ blocksScanned++
+ }
+ if shardBad {
+ broken = append(broken, shard)
+ }
+ }
+
+ // Wholesale-mismatch guard: localized bitrot touches a few shards; a
+ // stale/wrong sidecar mismatches more than parity_shards. Classify the
+ // latter as sidecar integrity, never as mass shard corruption.
+ if ecv.ECContext != nil && len(broken) > ecv.ECContext.ParityShards {
+ return blocksScanned, nil, []error{fmt.Errorf("ec volume %d: %d/%d local shards mismatch (> parity %d): suspect stale sidecar, not flagging shard corruption",
+ ecv.VolumeId, len(broken), len(ecv.Shards), ecv.ECContext.ParityShards)}
+ }
+
+ // Reed-Solomon arbitration: a sidecar mismatch could mean the shard rotted
+ // OR the sidecar block is stale. When enough OTHER shards are local to
+ // reconstruct, reconstruct each flagged shard from the verified-clean shards
+ // and compare to its on-disk bytes. Only shards RS *also* disagrees with are
+ // truly corrupt; the rest are stale-sidecar false positives we must not
+ // quarantine. This is the arbiter that makes auto-repair safe.
+ if ecv.ECContext != nil && len(broken) > 0 {
+ brokenSet := make(map[ShardId]bool, len(broken))
+ for _, s := range broken {
+ brokenSet[s.ShardId] = true
+ }
+ cleanCount := 0
+ for _, s := range ecv.Shards {
+ if !brokenSet[s.ShardId] {
+ cleanCount++
+ }
+ }
+ if cleanCount >= ecv.ECContext.DataShards {
+ confirmed := make([]*EcVolumeShard, 0, len(broken))
+ for _, s := range broken {
+ corrupt, aerr := ecv.rsConfirmsShardCorrupt(s.ShardId, brokenSet, blockSize)
+ if aerr != nil {
+ // Could not arbitrate this shard; stay conservative and keep it.
+ errs = append(errs, fmt.Errorf("ec volume %d shard %d: RS arbitration failed: %v", ecv.VolumeId, s.ShardId, aerr))
+ confirmed = append(confirmed, s)
+ continue
+ }
+ if corrupt {
+ confirmed = append(confirmed, s)
+ } else {
+ errs = append(errs, fmt.Errorf("ec volume %d shard %d: sidecar mismatch but Reed-Solomon confirms the bytes are correct: stale sidecar, not flagging shard",
+ ecv.VolumeId, s.ShardId))
+ }
+ }
+ broken = confirmed
+ }
+ }
+
+ for _, s := range broken {
+ brokenShards = append(brokenShards, s.ToEcShardInfo())
+ }
+ slices.SortFunc(brokenShards, func(a, b *volume_server_pb.EcShardInfo) int {
+ return int(a.ShardId) - int(b.ShardId)
+ })
+ return blocksScanned, brokenShards, errs
+}
+
+// rsConfirmsShardCorrupt reconstructs targetId from the verified-clean local
+// shards (all local shards except those in brokenSet) and compares the result
+// to targetId's on-disk bytes block-by-block. It returns true only if
+// Reed-Solomon disagrees with the disk (the shard is genuinely corrupt); a full
+// match means the shard is fine and its sidecar block is stale. The caller
+// guarantees at least DataShards clean shards are local before invoking this.
+func (ecv *EcVolume) rsConfirmsShardCorrupt(targetId ShardId, brokenSet map[ShardId]bool, blockSize int64) (bool, error) {
+ enc, err := ecv.ECContext.CreateEncoder()
+ if err != nil {
+ return false, err
+ }
+ total := ecv.ECContext.Total()
+ local := make(map[ShardId]*EcVolumeShard, len(ecv.Shards))
+ for _, s := range ecv.Shards {
+ local[s.ShardId] = s
+ }
+ target := local[targetId]
+ if target == nil {
+ return false, fmt.Errorf("target shard not local")
+ }
+ size := target.Size()
+
+ buffers := make([][]byte, total)
+ diskBuf := make([]byte, blockSize)
+ var offset int64
+ for offset < size {
+ n := blockSize
+ if rem := size - offset; rem < n {
+ n = rem
+ }
+ // Fill clean local shards; nil for the target and any broken/absent shard
+ // so Reconstruct regenerates the target from the trusted inputs.
+ for id := 0; id < total; id++ {
+ sid := ShardId(id)
+ if sid == targetId || brokenSet[sid] {
+ buffers[id] = nil
+ continue
+ }
+ s := local[sid]
+ if s == nil {
+ buffers[id] = nil
+ continue
+ }
+ buf := make([]byte, n)
+ got, rerr := s.ReadAt(buf, offset)
+ if rerr != nil || int64(got) != n {
+ return false, fmt.Errorf("read clean shard %d at %d: %v (got %d/%d)", sid, offset, rerr, got, n)
+ }
+ buffers[id] = buf
+ }
+ if err := enc.Reconstruct(buffers); err != nil {
+ return false, fmt.Errorf("reconstruct: %w", err)
+ }
+ got, rerr := target.ReadAt(diskBuf[:n], offset)
+ if rerr != nil || int64(got) != n {
+ return false, fmt.Errorf("read target shard %d at %d: %v", targetId, offset, rerr)
+ }
+ if !bytes.Equal(buffers[targetId][:n], diskBuf[:n]) {
+ return true, nil // Reed-Solomon disagrees with disk: genuinely corrupt.
+ }
+ offset += n
+ }
+ return false, nil // every block matches: shard is fine, sidecar block is stale.
+}
+
// ScrubLocal checks the integrity of local shards for a EC volume. Notably, it cannot verify CRC on needles.
// Returns a count of processed file entries, slice of found broken shards, and slice of found errors.
func (ecv *EcVolume) ScrubLocal() (int64, []*volume_server_pb.EcShardInfo, []error) {
diff --git a/weed/storage/erasure_coding/shard_distribution.go b/weed/storage/erasure_coding/shard_distribution.go
index 591c60354..34ea64fbf 100644
--- a/weed/storage/erasure_coding/shard_distribution.go
+++ b/weed/storage/erasure_coding/shard_distribution.go
@@ -135,6 +135,9 @@ func DistributeEcShards(volumeID uint32, collection string, targets []*worker_pb
if _, hasVif := shardFiles["vif"]; hasVif {
assignedShards = append(assignedShards, "vif")
}
+ if _, hasEcsum := shardFiles["ecsum"]; hasEcsum {
+ assignedShards = append(assignedShards, "ecsum")
+ }
}
if shardDisks[target.Node] == nil {
@@ -322,6 +325,9 @@ func sendShardFileToDestination(volumeID uint32, collection string, dialOption g
} else if shardType == "vif" {
ext = ".vif"
shardId = 0
+ } else if shardType == "ecsum" {
+ ext = BitrotSidecarExt
+ shardId = 0
} else if strings.HasPrefix(shardType, "ec") && len(shardType) == 4 {
ext = "." + shardType
fmt.Sscanf(shardType[2:], "%d", &shardId)
diff --git a/weed/storage/remote_tier_integration_test.go b/weed/storage/remote_tier_integration_test.go
index 6326fc66e..11cbe2f91 100644
--- a/weed/storage/remote_tier_integration_test.go
+++ b/weed/storage/remote_tier_integration_test.go
@@ -299,7 +299,7 @@ func TestRemoteTier_ECEncode_RequiresLocalDat(t *testing.T) {
tierUpVolume(t, dir, vid, b)
baseFileName := filepath.Join(dir, fmt.Sprintf("%d", uint32(vid)))
- err := erasure_coding.WriteEcFiles(baseFileName)
+ _, err := erasure_coding.WriteEcFiles(baseFileName, erasure_coding.BackgroundECContext())
require.Error(t, err, "EC encoder must not run with .dat missing — caller is expected to download first")
require.Contains(t, err.Error(), ".dat")
}
@@ -321,7 +321,8 @@ func TestRemoteTier_ECEncodeDecode_AfterDownload(t *testing.T) {
require.NoError(t, err)
require.NoError(t, erasure_coding.WriteSortedFileFromIdx(baseFileName, ".ecx"))
- require.NoError(t, erasure_coding.WriteEcFiles(baseFileName))
+ _, ecErr := erasure_coding.WriteEcFiles(baseFileName, erasure_coding.BackgroundECContext())
+ require.NoError(t, ecErr)
for i := 0; i < erasure_coding.TotalShardsCount; i++ {
shardPath := fmt.Sprintf("%s.ec%02d", baseFileName, i)
@@ -335,7 +336,7 @@ func TestRemoteTier_ECEncodeDecode_AfterDownload(t *testing.T) {
shardPath := fmt.Sprintf("%s.ec%02d", baseFileName, i)
require.NoError(t, os.Remove(shardPath))
}
- rebuilt, err := erasure_coding.RebuildEcFiles(baseFileName)
+ rebuilt, err := erasure_coding.RebuildEcFiles(baseFileName, erasure_coding.BackgroundECContext(), false)
require.NoError(t, err)
require.NotEmpty(t, rebuilt, "rebuild should report which parity shards were regenerated")
for i := erasure_coding.DataShardsCount; i < erasure_coding.TotalShardsCount; i++ {
diff --git a/weed/worker/tasks/erasure_coding/ec_task.go b/weed/worker/tasks/erasure_coding/ec_task.go
index bcc7bb880..935883119 100644
--- a/weed/worker/tasks/erasure_coding/ec_task.go
+++ b/weed/worker/tasks/erasure_coding/ec_task.go
@@ -558,9 +558,18 @@ func (t *ErasureCodingTask) generateEcShardsLocally(localFiles map[string]string
}
// Generate EC shard files (.ec00 ~ .ec13)
- if err := erasure_coding.WriteEcFiles(baseName); err != nil {
+ ecBitrot, err := erasure_coding.WriteEcFiles(baseName, erasure_coding.BackgroundECContext())
+ if err != nil {
return nil, fmt.Errorf("failed to generate EC shard files: %v", err)
}
+ // Persist the bitrot checksum sidecar (generation 0) alongside the shards so
+ // it travels with them during distribution. Best-effort: a failed sidecar
+ // write leaves the generation unprotected rather than failing the encode.
+ if erasure_coding.BitrotProtectionEnabled && ecBitrot != nil {
+ if serr := erasure_coding.SaveBitrotSidecar(erasure_coding.BitrotSidecarPath(baseName, 0), ecBitrot); serr != nil {
+ glog.Warningf("failed to write EC bitrot sidecar for %s: %v", baseName, serr)
+ }
+ }
// Collect generated shard file paths and log details
var generatedShards []string
@@ -625,6 +634,20 @@ func (t *ErasureCodingTask) generateEcShardsLocally(localFiles map[string]string
}
}
+ // Add the generation-0 bitrot checksum sidecar so it is distributed with
+ // the shards (DistributeEcShards only ships files present in shardFiles).
+ // Best-effort like the sidecar write above: if it is absent the holders
+ // are simply unprotected rather than failing the encode.
+ ecsumFile := erasure_coding.BitrotSidecarPath(baseName, 0)
+ if info, err := os.Stat(ecsumFile); err == nil {
+ shardFiles["ecsum"] = ecsumFile
+ t.GetLogger().WithFields(map[string]interface{}{
+ "file_type": "ecsum",
+ "file_path": ecsumFile,
+ "size_bytes": info.Size(),
+ }).Info("EC bitrot checksum sidecar generated")
+ }
+
// Log summary of generation
t.GetLogger().WithFields(map[string]interface{}{
"total_files": len(shardFiles),