Add large bucket support for erasure coded backend (#5160)

This PR implements an object layer which
combines input erasure sets of XL layers
into a unified namespace.

This object layer extends the existing
erasure coded implementation, it is assumed
in this design that providing > 16 disks is
a static configuration as well i.e if you started
the setup with 32 disks with 4 sets 8 disks per
pack then you would need to provide 4 sets always.

Some design details and restrictions:

- Objects are distributed using consistent ordering
  to a unique erasure coded layer.
- Each pack has its own dsync so locks are synchronized
  properly at pack (erasure layer).
- Each pack still has a maximum of 16 disks
  requirement, you can start with multiple
  such sets statically.
- Static sets set of disks and cannot be
  changed, there is no elastic expansion allowed.
- Static sets set of disks and cannot be
  changed, there is no elastic removal allowed.
- ListObjects() across sets can be noticeably
  slower since List happens on all servers,
  and is merged at this sets layer.

Fixes #5465
Fixes #5464
Fixes #5461
Fixes #5460
Fixes #5459
Fixes #5458
Fixes #5460
Fixes #5488
Fixes #5489
Fixes #5497
Fixes #5496

cmd/server-startup-msg_test.go

@@ -31,17 +31,12 @@ import (
 
 // Tests if we generate storage info.
 func TestStorageInfoMsg(t *testing.T) {
-	infoStorage := StorageInfo{
-		Total: 10 * humanize.GiByte,
-		Free:  2 * humanize.GiByte,
-		Backend: struct {
-			Type             BackendType
-			OnlineDisks      int
-			OfflineDisks     int
-			StandardSCParity int
-			RRSCParity       int
-		}{Erasure, 7, 1, 4, 5},
-	}
+	infoStorage := StorageInfo{}
+	infoStorage.Total = 10 * humanize.GiByte
+	infoStorage.Free = 2 * humanize.GiByte
+	infoStorage.Backend.Type = Erasure
+	infoStorage.Backend.OnlineDisks = 7
+	infoStorage.Backend.OfflineDisks = 1
 
 	if msg := getStorageInfoMsg(infoStorage); !strings.Contains(msg, "2.0 GiB Free, 10 GiB Total") || !strings.Contains(msg, "7 Online, 1 Offline") {
 		t.Fatal("Unexpected storage info message, found:", msg)
@@ -155,97 +150,3 @@ func TestPrintStartupMessage(t *testing.T) {
 	apiEndpoints := []string{"http://127.0.0.1:9000"}
 	printStartupMessage(apiEndpoints)
 }
-
-func TestGetStandardStorageClassInfoMsg(t *testing.T) {
-	tests := []struct {
-		name string
-		args StorageInfo
-		want string
-	}{
-		{"1", StorageInfo{
-			Total: 20 * humanize.GiByte,
-			Free:  2 * humanize.GiByte,
-			Backend: struct {
-				Type             BackendType
-				OnlineDisks      int
-				OfflineDisks     int
-				StandardSCParity int
-				RRSCParity       int
-			}{Erasure, 15, 1, 5, 3},
-		}, "Objects with " + standardStorageClass + " class can withstand [4] drive failure(s).\n"},
-		{"2", StorageInfo{
-			Total: 30 * humanize.GiByte,
-			Free:  3 * humanize.GiByte,
-			Backend: struct {
-				Type             BackendType
-				OnlineDisks      int
-				OfflineDisks     int
-				StandardSCParity int
-				RRSCParity       int
-			}{Erasure, 10, 0, 5, 3},
-		}, "Objects with " + standardStorageClass + " class can withstand [5] drive failure(s).\n"},
-		{"3", StorageInfo{
-			Total: 15 * humanize.GiByte,
-			Free:  2 * humanize.GiByte,
-			Backend: struct {
-				Type             BackendType
-				OnlineDisks      int
-				OfflineDisks     int
-				StandardSCParity int
-				RRSCParity       int
-			}{Erasure, 12, 3, 6, 2},
-		}, "Objects with " + standardStorageClass + " class can withstand [3] drive failure(s).\n"},
-	}
-	for _, tt := range tests {
-		if got := getStandardStorageClassInfoMsg(tt.args); got != tt.want {
-			t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got)
-		}
-	}
-}
-
-func TestGetRRSStorageClassInfoMsg(t *testing.T) {
-	tests := []struct {
-		name string
-		args StorageInfo
-		want string
-	}{
-		{"1", StorageInfo{
-			Total: 20 * humanize.GiByte,
-			Free:  2 * humanize.GiByte,
-			Backend: struct {
-				Type             BackendType
-				OnlineDisks      int
-				OfflineDisks     int
-				StandardSCParity int
-				RRSCParity       int
-			}{Erasure, 15, 1, 5, 3},
-		}, "Objects with " + reducedRedundancyStorageClass + " class can withstand [2] drive failure(s).\n"},
-		{"2", StorageInfo{
-			Total: 30 * humanize.GiByte,
-			Free:  3 * humanize.GiByte,
-			Backend: struct {
-				Type             BackendType
-				OnlineDisks      int
-				OfflineDisks     int
-				StandardSCParity int
-				RRSCParity       int
-			}{Erasure, 16, 0, 5, 3},
-		}, "Objects with " + reducedRedundancyStorageClass + " class can withstand [3] drive failure(s).\n"},
-		{"3", StorageInfo{
-			Total: 15 * humanize.GiByte,
-			Free:  2 * humanize.GiByte,
-			Backend: struct {
-				Type             BackendType
-				OnlineDisks      int
-				OfflineDisks     int
-				StandardSCParity int
-				RRSCParity       int
-			}{Erasure, 12, 3, 6, 5},
-		}, "Objects with " + reducedRedundancyStorageClass + " class can withstand [2] drive failure(s).\n"},
-	}
-	for _, tt := range tests {
-		if got := getRRSStorageClassInfoMsg(tt.args); got != tt.want {
-			t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got)
-		}
-	}
-}