master: align default volume size with EC rows (#10761)

Co-authored-by: joe <joe@gmail.com>

weed/command/master.go

@@ -88,7 +88,7 @@ func init() {
 	m.ipBind = cmdMaster.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
 	m.metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
 	m.peers = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095; use 'none' for single-master mode")
-	m.volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
+	m.volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", util.DefaultVolumeSizeLimitMB, "Master stops directing writes to oversized volumes.")
 	m.volumePreallocate = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.")
 	m.maxParallelVacuumPerServer = cmdMaster.Flag.Int("maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel per volume server")
 	// m.pulseSeconds = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
@@ -160,8 +160,8 @@ func runMaster(cmd *Command, args []string) bool {
 	}
 
 	masterWhiteList := util.StringSplit(*m.whiteList, ",")
-	if *m.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
-		glog.Fatalf("volumeSizeLimitMB should be smaller than 30000")
+	if *m.volumeSizeLimitMB > util.MaxVolumeSizeLimitMB {
+		glog.Fatalf("volumeSizeLimitMB should not exceed %d", util.MaxVolumeSizeLimitMB)
 	}
 
 	switch {

weed/command/master_test.go

@@ -4,8 +4,25 @@ import (
 	"testing"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
+func TestDefaultVolumeSizeLimitAlignsWithErasureCodingRows(t *testing.T) {
+	if got := *m.volumeSizeLimitMB; got != util.DefaultVolumeSizeLimitMB {
+		t.Fatalf("master volume size default = %d MiB, want %d MiB", got, util.DefaultVolumeSizeLimitMB)
+	}
+	if got := *masterOptions.volumeSizeLimitMB; got != util.DefaultVolumeSizeLimitMB {
+		t.Fatalf("server master volume size default = %d MiB, want %d MiB", got, util.DefaultVolumeSizeLimitMB)
+	}
+
+	defaultSize := int64(util.DefaultVolumeSizeLimitMB) * util.MiByte
+	largeRowSize := int64(erasure_coding.DataShardsCount) * erasure_coding.ErasureCodingLargeBlockSize
+	if remainder := defaultSize % largeRowSize; remainder != 0 {
+		t.Fatalf("default volume size leaves %d bytes outside complete EC large-block rows", remainder)
+	}
+}
+
 func TestPeerIndexIgnoresGrpcPort(t *testing.T) {
 	self := pb.ServerAddress("127.0.0.1:9000.19000")
 	peers := []pb.ServerAddress{

weed/command/mini.go

@@ -1265,8 +1265,8 @@ func runMini(cmd *Command, args []string) bool {
 
 	go stats_collect.StartMetricsServer(*miniMetricsHttpIp, *miniMetricsHttpPort)
 
-	if *miniMasterOptions.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
-		glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
+	if *miniMasterOptions.volumeSizeLimitMB > util.MaxVolumeSizeLimitMB {
+		glog.Fatalf("master.volumeSizeLimitMB should not exceed %d", util.MaxVolumeSizeLimitMB)
 	}
 
 	if *miniMasterOptions.metaFolder == "" {

weed/command/server.go

@@ -95,7 +95,7 @@ func init() {
 	masterOptions.portGrpc = cmdServer.Flag.Int("master.port.grpc", 0, "master server grpc listen port")
 	masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
 	masterOptions.peers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
-	masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
+	masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", util.DefaultVolumeSizeLimitMB, "Master stops directing writes to oversized volumes.")
 	masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
 	masterOptions.maxParallelVacuumPerServer = cmdServer.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server")
 	masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "", "Default replication type if not specified.")
@@ -340,8 +340,8 @@ func runServer(cmd *Command, args []string) bool {
 	*volumeDataFolders = util.ResolveCommaSeparatedPaths(*volumeDataFolders)
 	folders := strings.Split(*volumeDataFolders, ",")
 
-	if *masterOptions.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
-		glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
+	if *masterOptions.volumeSizeLimitMB > util.MaxVolumeSizeLimitMB {
+		glog.Fatalf("master.volumeSizeLimitMB should not exceed %d", util.MaxVolumeSizeLimitMB)
 	}
 
 	if *masterOptions.metaFolder == "" {

weed/util/volume_size.go

@@ -0,0 +1,11 @@
+package util
+
+const (
+	// DefaultVolumeSizeLimitMB is 30 GiB expressed in the MiB units used by
+	// volumeSizeLimitMB. This also aligns the default volume size with three
+	// complete 10 GiB erasure-coding data rows.
+	DefaultVolumeSizeLimitMB = 30 * KiByte
+	// MaxVolumeSizeLimitMB expresses VolumeSizeLimitGB in MiB: 30 GiB for
+	// 4-byte offsets and 8,000 GiB for 5-byte offsets.
+	MaxVolumeSizeLimitMB = VolumeSizeLimitGB * KiByte
+)