seaweedfs/weed/shell/command_ec_check_replication_test.go

package shell

import (
	"bytes"
	"fmt"
	"reflect"
	"testing"

	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)

// TestEcCheckReplication exercises the ec.check.replication classification and reporting over a
// fake topology. EC volume 1 is fully replicated, EC volume 2 is
// under-replicated, and EC volume 3 is over-replicated.
func TestEcCheckReplication(t *testing.T) {
	nodes := []*master_pb.DataNodeInfo{
		{
			Address: "localhost:1111",
			DiskInfos: map[string]*master_pb.DiskInfo{
				"d1": {
					EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
						{
							Id:          1,
							EcIndexBits: 0b0000000011111,
						},
						{
							Id:          2,
							EcIndexBits: 0b0000000011111,
						},
						{
							Id:          3,
							EcIndexBits: 0b11111111111111,
						},
					},
				},
			},
		},
		{
			Address: "localhost:2222",
			DiskInfos: map[string]*master_pb.DiskInfo{
				"d2": {
					EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
						{
							Id:          1,
							EcIndexBits: 0b11111111100000,
						},
						{
							Id:          2,
							EcIndexBits: 0b11111000000000,
						},
						{
							Id:          3,
							EcIndexBits: 0b00000000001111,
						},
					},
				},
			},
		},
	}

	testCases := []struct {
		name        string
		showDetails bool
		volumeIDMap map[uint32]bool
		want        string
		wantErr     error
	}{
		{
			name:        "default",
			showDetails: false,
			want: `Found 1/3 under-replicated EC volumes: [2]
Found 1/3 over-replicated EC volumes: [3]
`,
		},
		{
			name:        "single healthy volume",
			showDetails: false,
			volumeIDMap: map[uint32]bool{1: true},
			want: `EC volumes are healthy.
`,
		},
		{
			name:        "filtered by volume ID",
			showDetails: false,
			volumeIDMap: map[uint32]bool{1: true, 2: true, 5: true},
			want: `Found 1/2 under-replicated EC volumes: [2]
`,
		},
		{
			name:        "no valid volume IDs",
			showDetails: false,
			volumeIDMap: map[uint32]bool{5: true, 6: true},
			want:        "",
			wantErr:     fmt.Errorf("no EC volumes found"),
		},
		{
			name:        "with details",
			showDetails: true,
			want: `Found 1/3 under-replicated EC volumes: [2]
Found 1/3 over-replicated EC volumes: [3]

Shards map for under-replicated EC volume 2 (10/14 shards):
	00 => [localhost:1111]
	01 => [localhost:1111]
	02 => [localhost:1111]
	03 => [localhost:1111]
	04 => [localhost:1111]
	05 is missing
	06 is missing
	07 is missing
	08 is missing
	09 => [localhost:2222]
	10 (parity) => [localhost:2222]
	11 (parity) => [localhost:2222]
	12 (parity) => [localhost:2222]
	13 (parity) => [localhost:2222]

Shards map for over-replicated EC volume 3 (18/14 shards):
	00 => [localhost:1111 localhost:2222]
	01 => [localhost:1111 localhost:2222]
	02 => [localhost:1111 localhost:2222]
	03 => [localhost:1111 localhost:2222]
	04 => [localhost:1111]
	05 => [localhost:1111]
	06 => [localhost:1111]
	07 => [localhost:1111]
	08 => [localhost:1111]
	09 => [localhost:1111]
	10 (parity) => [localhost:1111]
	11 (parity) => [localhost:1111]
	12 (parity) => [localhost:1111]
	13 (parity) => [localhost:1111]
`,
		},
	}

	for _, tc := range testCases {
		var buf bytes.Buffer

		runner := &ecCheckReplicationRunner{
			writer:      &buf,
			dataNodes:   nodes,
			volumeIDMap: tc.volumeIDMap,
		}
		gotErr := runner.checkEcVolumes(tc.showDetails)
		got := buf.String()

		if got != tc.want {
			t.Errorf("%s: got %v, want %v", tc.name, got, tc.want)
		}
		if !reflect.DeepEqual(gotErr, tc.wantErr) {
			t.Errorf("%s: got error %v, want %v", tc.name, gotErr, tc.wantErr)
		}
	}
}

// TestEcCheckReplicationNoVolumes checks the empty-topology behavior: an unfiltered
// run over a cluster with no EC volumes is healthy (informational message, no
// error), while a run filtered to volume IDs that are not EC volumes errors.
func TestEcCheckReplicationNoVolumes(t *testing.T) {
	// a data node reporting a normal volume but no EC shards
	nodes := []*master_pb.DataNodeInfo{
		{
			Address:   "localhost:1111",
			DiskInfos: map[string]*master_pb.DiskInfo{"d1": {}},
		},
	}

	t.Run("unfiltered", func(t *testing.T) {
		var buf bytes.Buffer
		runner := &ecCheckReplicationRunner{writer: &buf, dataNodes: nodes}
		if err := runner.checkEcVolumes(false); err != nil {
			t.Errorf("got error %v, want nil", err)
		}
		if got, want := buf.String(), "No EC volumes found.\n"; got != want {
			t.Errorf("got %q, want %q", got, want)
		}
	})

	t.Run("filtered no match", func(t *testing.T) {
		var buf bytes.Buffer
		runner := &ecCheckReplicationRunner{writer: &buf, dataNodes: nodes, volumeIDMap: map[uint32]bool{5: true}}
		if err := runner.checkEcVolumes(false); err == nil {
			t.Errorf("got nil error, want %q", "no EC volumes found")
		}
		if got := buf.String(); got != "" {
			t.Errorf("got output %q, want none", got)
		}
	})
}

// TestEcCheckReplicationUnexpectedShard verifies that a shard id beyond a volume's
// expected data+parity range is flagged as over-replication and surfaced in the
// details map, rather than being silently ignored. Volume 20 is a healthy 10+4
// (shards 0..13 on nodeA) with a stray shard 14 on nodeB.
func TestEcCheckReplicationUnexpectedShard(t *testing.T) {
	nodes := []*master_pb.DataNodeInfo{
		{
			Address: "nodeA",
			DiskInfos: map[string]*master_pb.DiskInfo{
				"d1": {
					EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
						{Id: 20, EcIndexBits: 0b11111111111111}, // shards 0..13
					},
				},
			},
		},
		{
			Address: "nodeB",
			DiskInfos: map[string]*master_pb.DiskInfo{
				"d2": {
					EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
						{Id: 20, EcIndexBits: 0b100000000000000}, // stray shard 14
					},
				},
			},
		},
	}

	want := `Found 1/1 over-replicated EC volumes: [20]

Shards map for over-replicated EC volume 20 (15/14 shards):
	00 => [nodeA]
	01 => [nodeA]
	02 => [nodeA]
	03 => [nodeA]
	04 => [nodeA]
	05 => [nodeA]
	06 => [nodeA]
	07 => [nodeA]
	08 => [nodeA]
	09 => [nodeA]
	10 (parity) => [nodeA]
	11 (parity) => [nodeA]
	12 (parity) => [nodeA]
	13 (parity) => [nodeA]
	14 (unexpected) => [nodeB]
`

	var buf bytes.Buffer
	runner := &ecCheckReplicationRunner{writer: &buf, dataNodes: nodes}
	if err := runner.checkEcVolumes(true); err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if got := buf.String(); got != want {
		t.Errorf("unexpected shard:\n got: %q\nwant: %q", got, want)
	}
}

// TestEcCheckReplicationMixedAnomalies verifies that a volume with both a missing
// shard and a duplicated shard is reported in both lists. Volume 30 has shard 5
// missing everywhere (under-replicated) and shard 0 on two nodes (over-replicated).
func TestEcCheckReplicationMixedAnomalies(t *testing.T) {
	nodes := []*master_pb.DataNodeInfo{
		{
			Address: "nodeA",
			DiskInfos: map[string]*master_pb.DiskInfo{
				"d1": {
					EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
						{Id: 30, EcIndexBits: 0b11111111011111}, // shards 0..13 except 5
					},
				},
			},
		},
		{
			Address: "nodeB",
			DiskInfos: map[string]*master_pb.DiskInfo{
				"d2": {
					EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
						{Id: 30, EcIndexBits: 0b00000000000001}, // duplicate of shard 0
					},
				},
			},
		},
	}

	want := `Found 1/1 under-replicated EC volumes: [30]
Found 1/1 over-replicated EC volumes: [30]
`

	var buf bytes.Buffer
	runner := &ecCheckReplicationRunner{writer: &buf, dataNodes: nodes}
	if err := runner.checkEcVolumes(false); err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if got := buf.String(); got != want {
		t.Errorf("mixed anomalies:\n got: %q\nwant: %q", got, want)
	}
}