mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-07-23 16:43:23 +00:00
Chris LuandGitHub
c1a1e3c1e3
* volume: copying a remote-backed volume only needs space for the index VolumeCopy sized its target-location check by the source .dat even when that .dat lives in a cloud tier and only .idx/.vif land locally, so re-replicating a tiered volume demanded the full remote size in free disk. Require the index size instead. * shell: volume.tier.upload keeps volume replicas Tiering a replicated volume deleted every replica but the upload source, leaving one server holding the only .idx and the only .vif that knows the remote object key — losing that server orphaned the volume even though its data sat intact in the cloud. Replicate the uploaded .idx/.vif onto the other replica servers instead (VolumeCopy skips the .dat for remote-backed volumes), so all replicas serve reads from the same remote object and the volume keeps its replica count. An already-tiered replica is preferred as the upload source, so a rerun after a partial failure reuses the existing remote object instead of uploading a second copy under a new key. * shell: group tier upload locations instead of re-prepending * rust volume: copying a remote-backed volume only needs space for the index Mirror the Go VolumeCopy change: size the free-location check by the source .idx when the .dat lives in a cloud tier, since only .idx/.vif land locally.
57 lines
1.9 KiB
Go
57 lines
1.9 KiB
Go
package shell
|
|
|
|
import (
|
|
"io"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
)
|
|
|
|
func tierUploadTopo(nodes map[string]*master_pb.VolumeInformationMessage) *master_pb.TopologyInfo {
|
|
var dns []*master_pb.DataNodeInfo
|
|
for _, id := range []string{"n1", "n2", "n3"} {
|
|
vi, found := nodes[id]
|
|
if !found {
|
|
continue
|
|
}
|
|
dns = append(dns, &master_pb.DataNodeInfo{
|
|
Id: id,
|
|
DiskInfos: map[string]*master_pb.DiskInfo{"": {VolumeInfos: []*master_pb.VolumeInformationMessage{vi}}},
|
|
})
|
|
}
|
|
return &master_pb.TopologyInfo{
|
|
DataCenterInfos: []*master_pb.DataCenterInfo{{
|
|
RackInfos: []*master_pb.RackInfo{{DataNodeInfos: dns}},
|
|
}},
|
|
}
|
|
}
|
|
|
|
// A replica that is already tiered must become the upload source, so a rerun
|
|
// after a partial failure reuses its remote object instead of uploading a
|
|
// second copy under a new key.
|
|
func TestCollectVolumeTierUploadLocationsPrefersTieredReplica(t *testing.T) {
|
|
topo := tierUploadTopo(map[string]*master_pb.VolumeInformationMessage{
|
|
"n1": {Id: 1, Collection: "c"},
|
|
"n2": {Id: 1, Collection: "c", RemoteStorageName: "s3.default", RemoteStorageKey: "some-key"},
|
|
"n3": {Id: 1, Collection: "c"},
|
|
})
|
|
locations := collectVolumeTierUploadLocations(topo, needle.VolumeId(1), "c", io.Discard)
|
|
if len(locations) != 3 {
|
|
t.Fatalf("want 3 locations, got %d", len(locations))
|
|
}
|
|
if locations[0].Url != "n2" {
|
|
t.Fatalf("want tiered replica n2 first, got %s", locations[0].Url)
|
|
}
|
|
}
|
|
|
|
func TestCollectVolumeTierUploadLocationsFiltersCollection(t *testing.T) {
|
|
topo := tierUploadTopo(map[string]*master_pb.VolumeInformationMessage{
|
|
"n1": {Id: 1, Collection: "c"},
|
|
"n2": {Id: 1, Collection: "other"},
|
|
})
|
|
if locations := collectVolumeTierUploadLocations(topo, needle.VolumeId(1), "c", io.Discard); len(locations) != 1 || locations[0].Url != "n1" {
|
|
t.Fatalf("want only n1, got %+v", locations)
|
|
}
|
|
}
|