seaweedfs/weed/shell/command_volume_list_test.go

package shell

import (
	"bytes"
	"flag"
	"fmt"

	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
	"github.com/seaweedfs/seaweedfs/weed/storage/types"
	"github.com/stretchr/testify/assert"

	//"google.golang.org/protobuf/proto"
	"strconv"
	"strings"
	"testing"

	"github.com/golang/protobuf/proto"

	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)

func TestParsing(t *testing.T) {
	topo := parseOutput(topoData)

	assert.Equal(t, 5, len(topo.DataCenterInfos))

	topo = parseOutput(topoData2)

	dataNodes := topo.DataCenterInfos[0].RackInfos[0].DataNodeInfos
	assert.Equal(t, 14, len(dataNodes))
	diskInfo := dataNodes[0].DiskInfos[""]
	assert.Equal(t, 1559, len(diskInfo.VolumeInfos))
	assert.Equal(t, 6740, len(diskInfo.EcShardInfos))

}

// TODO: actually parsing all fields would be nice...
func parseOutput(output string) *master_pb.TopologyInfo {
	lines := strings.Split(output, "\n")
	var topo *master_pb.TopologyInfo
	var dc *master_pb.DataCenterInfo
	var rack *master_pb.RackInfo
	var dn *master_pb.DataNodeInfo
	var disk *master_pb.DiskInfo
	for _, line := range lines {
		line = strings.TrimSpace(line)
		parts := strings.Split(line, " ")
		switch parts[0] {
		case "Topology":
			if topo == nil {
				topo = &master_pb.TopologyInfo{
					Id: parts[1],
				}
			}
		case "DataCenter":
			if dc == nil {
				dc = &master_pb.DataCenterInfo{
					Id: parts[1],
				}
				topo.DataCenterInfos = append(topo.DataCenterInfos, dc)
			} else {
				dc = nil
			}
		case "Rack":
			if rack == nil {
				rack = &master_pb.RackInfo{
					Id: parts[1],
				}
				dc.RackInfos = append(dc.RackInfos, rack)
			} else {
				rack = nil
			}
		case "DataNode":
			if dn == nil {
				dn = &master_pb.DataNodeInfo{
					Id:        parts[1],
					DiskInfos: make(map[string]*master_pb.DiskInfo),
				}
				rack.DataNodeInfos = append(rack.DataNodeInfos, dn)
			} else {
				dn = nil
			}
		case "Disk":
			if disk == nil {
				diskType := parts[1][:strings.Index(parts[1], "(")]
				volumeCountStr := parts[1][strings.Index(parts[1], ":")+1 : strings.Index(parts[1], "/")]
				maxVolumeCountStr := parts[1][strings.Index(parts[1], "/")+1:]
				maxVolumeCount, _ := strconv.Atoi(maxVolumeCountStr)
				volumeCount, _ := strconv.Atoi(volumeCountStr)
				disk = &master_pb.DiskInfo{
					Type:           diskType,
					MaxVolumeCount: int64(maxVolumeCount),
					VolumeCount:    int64(volumeCount),
				}
				dn.DiskInfos[types.ToDiskType(diskType).String()] = disk
			} else {
				disk = nil
			}
		case "volume":
			volumeLine := line[len("volume "):]
			volume := &master_pb.VolumeInformationMessage{}
			proto.UnmarshalText(volumeLine, volume)
			disk.VolumeInfos = append(disk.VolumeInfos, volume)
		case "ec":
			ecVolumeLine := line[len("ec volume "):]
			ecShard := &master_pb.VolumeEcShardInformationMessage{}
			for _, part := range strings.Split(ecVolumeLine, " ") {
				if strings.HasPrefix(part, "id:") {
					id, _ := strconv.ParseInt(part[len("id:"):], 10, 64)
					ecShard.Id = uint32(id)
				}
				if strings.HasPrefix(part, "collection:") {
					ecShard.Collection = part[len("collection:"):]
				}
				// TODO: we need to parse EC shard sizes as well
				if strings.HasPrefix(part, "shards:") {
					shards := part[len("shards:["):]
					shards = strings.TrimRight(shards, "]")
					shardsInfo := erasure_coding.NewShardsInfo()
					for _, shardId := range strings.Split(shards, ",") {
						sid, _ := strconv.Atoi(shardId)
						shardsInfo.Set(erasure_coding.NewShardInfo(erasure_coding.ShardId(sid), 0))
					}
					ecShard.EcIndexBits = shardsInfo.Bitmap()
					ecShard.ShardSizes = shardsInfo.SizesInt64()
				}
			}
			disk.EcShardInfos = append(disk.EcShardInfos, ecShard)
		}
	}

	return topo
}

// TestWriteDataNodeInfo_SplitsCollapsedDisksByPhysicalDiskId verifies that
// the verbose Disk block in volume.list shows one entry per physical disk
// when the master collapsed several same-type disks under a single
// DiskInfos["hdd"] map entry. Before the split, six physical disks would
// appear as "Disk hdd ... id:0" with all volumes stacked on it.
func TestWriteDataNodeInfo_SplitsCollapsedDisksByPhysicalDiskId(t *testing.T) {
	dn := &master_pb.DataNodeInfo{
		Id: "node1:8081",
		DiskInfos: map[string]*master_pb.DiskInfo{
			"hdd": {
				Type:           "hdd",
				MaxVolumeCount: 60,
				VolumeInfos: []*master_pb.VolumeInformationMessage{
					{Id: 1, DiskId: 0, Collection: "c"},
					{Id: 2, DiskId: 1, Collection: "c"},
					{Id: 3, DiskId: 2, Collection: "c"},
				},
				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
					{Id: 100, DiskId: 3, Collection: "c", EcIndexBits: 1},
					{Id: 101, DiskId: 4, Collection: "c", EcIndexBits: 1},
					{Id: 102, DiskId: 5, Collection: "c", EcIndexBits: 1},
				},
			},
		},
	}

	c := &commandVolumeList{}
	fs := flag.NewFlagSet("volume.list", flag.ContinueOnError)
	c.collectionPattern = fs.String("collection", "", "")
	c.dataCenter = fs.String("dataCenter", "", "")
	c.rack = fs.String("rack", "", "")
	c.dataNode = fs.String("dataNode", "", "")
	c.readonly = fs.Bool("readonly", false, "")
	c.writable = fs.Bool("writable", false, "")
	c.volumeId = fs.Uint64("volumeId", 0, "")

	var buf bytes.Buffer
	verbosity := 5
	rackInvocations := 0
	c.writeDataNodeInfo(&buf, dn, verbosity, func() { rackInvocations++ })

	out := buf.String()
	// Match the "id:N\n" line ending so substring checks don't accept
	// unrelated tokens like "ec volume id:101" as a match for id:1.
	for diskID := 0; diskID < 6; diskID++ {
		needle := fmt.Sprintf("id:%d\n", diskID)
		if !strings.Contains(out, needle) {
			t.Errorf("output missing %q; got:\n%s", needle, out)
		}
	}

	// The parent's outRackInfo callback rides the same code path as the
	// DataNode header — it fires from the inner writeDiskInfo callback,
	// once per disk before the fix. After the fix the header guard runs
	// outRackInfo at most once per DataNode. This proxy lets us pin the
	// "header printed once" invariant without depending on the exact
	// format of the rendered DataNode line.
	if rackInvocations != 1 {
		t.Errorf("DataNode header callback ran %d times; want 1 (regression: header printed once per split disk)", rackInvocations)
	}
}