From e5f7870f5ead6430a6b46abffa1c29432136baf5 Mon Sep 17 00:00:00 2001 From: Daniel Valdivia Date: Tue, 22 Sep 2020 11:15:21 -0700 Subject: [PATCH] Parity API (#280) --- models/parity_response.go | 37 +++ pkg/utils/parity.go | 219 ++++++++++++++ pkg/utils/parity_test.go | 281 ++++++++++++++++++ restapi/admin_parity.go | 63 ++++ restapi/admin_parity_test.go | 79 +++++ restapi/configure_console.go | 2 + restapi/embedded_spec.go | 90 ++++++ restapi/operations/admin_api/get_parity.go | 90 ++++++ .../admin_api/get_parity_parameters.go | 154 ++++++++++ .../admin_api/get_parity_responses.go | 136 +++++++++ .../admin_api/get_parity_urlbuilder.go | 126 ++++++++ restapi/operations/console_api.go | 12 + swagger.yml | 32 ++ 13 files changed, 1321 insertions(+) create mode 100644 models/parity_response.go create mode 100644 pkg/utils/parity.go create mode 100644 pkg/utils/parity_test.go create mode 100644 restapi/admin_parity.go create mode 100644 restapi/admin_parity_test.go create mode 100644 restapi/operations/admin_api/get_parity.go create mode 100644 restapi/operations/admin_api/get_parity_parameters.go create mode 100644 restapi/operations/admin_api/get_parity_responses.go create mode 100644 restapi/operations/admin_api/get_parity_urlbuilder.go diff --git a/models/parity_response.go b/models/parity_response.go new file mode 100644 index 000000000..730420c68 --- /dev/null +++ b/models/parity_response.go @@ -0,0 +1,37 @@ +// Code generated by go-swagger; DO NOT EDIT. + +// This file is part of MinIO Console Server +// Copyright (c) 2020 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +// + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "github.com/go-openapi/strfmt" +) + +// ParityResponse parity response +// +// swagger:model parityResponse +type ParityResponse []string + +// Validate validates this parity response +func (m ParityResponse) Validate(formats strfmt.Registry) error { + return nil +} diff --git a/pkg/utils/parity.go b/pkg/utils/parity.go new file mode 100644 index 000000000..528e5f3d5 --- /dev/null +++ b/pkg/utils/parity.go @@ -0,0 +1,219 @@ +// This file is part of MinIO Console Server +// Copyright (c) 2020 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package utils + +import ( + "errors" + "fmt" + "sort" + + "github.com/minio/minio/pkg/ellipses" +) + +// This file implements and supports ellipses pattern for +// `minio server` command line arguments. + +// Supported set sizes this is used to find the optimal +// single set size. +var setSizes = []uint64{4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16} + +// getDivisibleSize - returns a greatest common divisor of +// all the ellipses sizes. +func getDivisibleSize(totalSizes []uint64) (result uint64) { + gcd := func(x, y uint64) uint64 { + for y != 0 { + x, y = y, x%y + } + return x + } + result = totalSizes[0] + for i := 1; i < len(totalSizes); i++ { + result = gcd(result, totalSizes[i]) + } + return result +} + +// isValidSetSize - checks whether given count is a valid set size for erasure coding. +var isValidSetSize = func(count uint64) bool { + return (count >= setSizes[0] && count <= setSizes[len(setSizes)-1]) +} + +// possibleSetCountsWithSymmetry returns symmetrical setCounts based on the +// input argument patterns, the symmetry calculation is to ensure that +// we also use uniform number of drives common across all ellipses patterns. +func possibleSetCountsWithSymmetry(setCounts []uint64, argPatterns []ellipses.ArgPattern) []uint64 { + var newSetCounts = make(map[uint64]struct{}) + for _, ss := range setCounts { + var symmetry bool + for _, argPattern := range argPatterns { + for _, p := range argPattern { + if uint64(len(p.Seq)) > ss { + symmetry = uint64(len(p.Seq))%ss == 0 + } else { + symmetry = ss%uint64(len(p.Seq)) == 0 + } + } + } + // With no arg patterns, it is expected that user knows + // the right symmetry, so either ellipses patterns are + // provided (recommended) or no ellipses patterns. + if _, ok := newSetCounts[ss]; !ok && (symmetry || argPatterns == nil) { + newSetCounts[ss] = struct{}{} + } + } + + setCounts = []uint64{} + for setCount := range newSetCounts { + setCounts = append(setCounts, setCount) + } + + // Not necessarily needed but it ensures to the readers + // eyes that we prefer a sorted setCount slice for the + // subsequent function to figure out the right common + // divisor, it avoids loops. + sort.Slice(setCounts, func(i, j int) bool { + return setCounts[i] < setCounts[j] + }) + + return setCounts +} + +func commonSetDriveCount(divisibleSize uint64, setCounts []uint64) (setSize uint64) { + // prefers setCounts to be sorted for optimal behavior. + if divisibleSize < setCounts[len(setCounts)-1] { + return divisibleSize + } + + // Figure out largest value of total_drives_in_erasure_set which results + // in least number of total_drives/total_drives_erasure_set ratio. + prevD := divisibleSize / setCounts[0] + for _, cnt := range setCounts { + if divisibleSize%cnt == 0 { + d := divisibleSize / cnt + if d <= prevD { + prevD = d + setSize = cnt + } + } + } + return setSize +} + +// getSetIndexes returns list of indexes which provides the set size +// on each index, this function also determines the final set size +// The final set size has the affinity towards choosing smaller +// indexes (total sets) +func getSetIndexes(args []string, totalSizes []uint64, argPatterns []ellipses.ArgPattern) (setIndexes [][]uint64, err error) { + if len(totalSizes) == 0 || len(args) == 0 { + return nil, errors.New("invalid argument") + } + + setIndexes = make([][]uint64, len(totalSizes)) + for _, totalSize := range totalSizes { + // Check if totalSize has minimum range upto setSize + if totalSize < setSizes[0] { + return nil, fmt.Errorf("incorrect number of endpoints provided %s", args) + } + } + + commonSize := getDivisibleSize(totalSizes) + possibleSetCounts := func(setSize uint64) (ss []uint64) { + for _, s := range setSizes { + if setSize%s == 0 { + ss = append(ss, s) + } + } + return ss + } + + setCounts := possibleSetCounts(commonSize) + if len(setCounts) == 0 { + err = fmt.Errorf("incorrect number of endpoints provided %s, number of disks %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes) + return nil, err + } + + // Returns possible set counts with symmetry. + setCounts = possibleSetCountsWithSymmetry(setCounts, argPatterns) + if len(setCounts) == 0 { + err = fmt.Errorf("no symmetric distribution detected with input endpoints provided %s, disks %d cannot be spread symmetrically by any supported erasure set sizes %d", args, commonSize, setSizes) + return nil, err + } + + // Final set size with all the symmetry accounted for. + setSize := commonSetDriveCount(commonSize, setCounts) + + // Check whether setSize is with the supported range. + if !isValidSetSize(setSize) { + err = fmt.Errorf("incorrect number of endpoints provided %s, number of disks %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes) + return nil, err + } + + for i := range totalSizes { + for j := uint64(0); j < totalSizes[i]/setSize; j++ { + setIndexes[i] = append(setIndexes[i], setSize) + } + } + + return setIndexes, nil +} + +// Return the total size for each argument patterns. +func getTotalSizes(argPatterns []ellipses.ArgPattern) []uint64 { + var totalSizes []uint64 + for _, argPattern := range argPatterns { + var totalSize uint64 = 1 + for _, p := range argPattern { + totalSize = totalSize * uint64(len(p.Seq)) + } + totalSizes = append(totalSizes, totalSize) + } + return totalSizes +} + +// PossibleParityValues returns possible parities for input args, +// parties are calculated in uniform manner for one zone or +// multiple zones, ensuring that parities returned are common +// and applicable across all zones. +func PossibleParityValues(args ...string) ([]string, error) { + setIndexes, err := parseEndpointSet(args...) + if err != nil { + return nil, err + } + maximumParity := setIndexes[0][0] / 2 + var parities []string + for maximumParity >= 2 { + parities = append(parities, fmt.Sprintf("EC:%d", maximumParity)) + maximumParity-- + } + return parities, nil +} + +// Parses all arguments and returns an endpointSet which is a collection +// of endpoints following the ellipses pattern, this is what is used +// by the object layer for initializing itself. +func parseEndpointSet(args ...string) (setIndexes [][]uint64, err error) { + var argPatterns = make([]ellipses.ArgPattern, len(args)) + for i, arg := range args { + patterns, err := ellipses.FindEllipsesPatterns(arg) + if err != nil { + return nil, err + } + argPatterns[i] = patterns + } + + return getSetIndexes(args, getTotalSizes(argPatterns), argPatterns) +} diff --git a/pkg/utils/parity_test.go b/pkg/utils/parity_test.go new file mode 100644 index 000000000..352b47857 --- /dev/null +++ b/pkg/utils/parity_test.go @@ -0,0 +1,281 @@ +// This file is part of MinIO Console Server +// Copyright (c) 2020 MinIO, Inc. +// +// All rights reserved + +package utils + +import ( + "reflect" + "testing" + + "github.com/minio/minio/pkg/ellipses" +) + +func TestGetDivisibleSize(t *testing.T) { + testCases := []struct { + totalSizes []uint64 + result uint64 + }{{[]uint64{24, 32, 16}, 8}, + {[]uint64{32, 8, 4}, 4}, + {[]uint64{8, 8, 8}, 8}, + {[]uint64{24}, 24}, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run("", func(t *testing.T) { + gotGCD := getDivisibleSize(testCase.totalSizes) + if testCase.result != gotGCD { + t.Errorf("Expected %v, got %v", testCase.result, gotGCD) + } + }) + } +} + +// Test tests calculating set indexes. +func TestGetSetIndexes(t *testing.T) { + testCases := []struct { + args []string + totalSizes []uint64 + indexes [][]uint64 + success bool + }{ + // Invalid inputs. + { + []string{"data{1...3}"}, + []uint64{3}, + nil, + false, + }, + { + []string{"data/controller1/export{1...2}, data/controller2/export{1...4}, data/controller3/export{1...8}"}, + []uint64{2, 4, 8}, + nil, + false, + }, + { + []string{"data{1...17}/export{1...52}"}, + []uint64{14144}, + nil, + false, + }, + // Valid inputs. + { + []string{"data{1...27}"}, + []uint64{27}, + [][]uint64{{9, 9, 9}}, + true, + }, + { + []string{"http://host{1...3}/data{1...180}"}, + []uint64{540}, + [][]uint64{{15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}}, + true, + }, + { + []string{"http://host{1...2}.rack{1...4}/data{1...180}"}, + []uint64{1440}, + [][]uint64{{16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16}}, + true, + }, + { + []string{"http://host{1...2}/data{1...180}"}, + []uint64{360}, + [][]uint64{{12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12}}, + true, + }, + { + []string{"data/controller1/export{1...4}, data/controller2/export{1...8}, data/controller3/export{1...12}"}, + []uint64{4, 8, 12}, + [][]uint64{{4}, {4, 4}, {4, 4, 4}}, + true, + }, + { + []string{"data{1...64}"}, + []uint64{64}, + [][]uint64{{16, 16, 16, 16}}, + true, + }, + { + []string{"data{1...24}"}, + []uint64{24}, + [][]uint64{{12, 12}}, + true, + }, + { + []string{"data/controller{1...11}/export{1...8}"}, + []uint64{88}, + [][]uint64{{11, 11, 11, 11, 11, 11, 11, 11}}, + true, + }, + { + []string{"data{1...4}"}, + []uint64{4}, + [][]uint64{{4}}, + true, + }, + { + []string{"data/controller1/export{1...10}, data/controller2/export{1...10}, data/controller3/export{1...10}"}, + []uint64{10, 10, 10}, + [][]uint64{{10}, {10}, {10}}, + true, + }, + { + []string{"data{1...16}/export{1...52}"}, + []uint64{832}, + [][]uint64{{16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16}}, + true, + }, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run("", func(t *testing.T) { + var argPatterns = make([]ellipses.ArgPattern, len(testCase.args)) + for i, arg := range testCase.args { + patterns, err := ellipses.FindEllipsesPatterns(arg) + if err != nil { + t.Fatalf("Unexpected failure %s", err) + } + argPatterns[i] = patterns + } + gotIndexes, err := getSetIndexes(testCase.args, testCase.totalSizes, argPatterns) + if err != nil && testCase.success { + t.Errorf("Expected success but failed instead %s", err) + } + if err == nil && !testCase.success { + t.Errorf("Expected failure but passed instead") + } + if !reflect.DeepEqual(testCase.indexes, gotIndexes) { + t.Errorf("Expected %v, got %v", testCase.indexes, gotIndexes) + } + }) + } +} + +// Test tests possible parities returned for any input args +func TestPossibleParities(t *testing.T) { + testCases := []struct { + arg string + parities []string + success bool + }{ + // Tests invalid inputs. + { + "...", + nil, + false, + }, + // No range specified. + { + "{...}", + nil, + false, + }, + // Invalid range. + { + "http://minio{2...3}/export/set{1...0}", + nil, + false, + }, + // Range cannot be smaller than 4 minimum. + { + "/export{1..2}", + nil, + false, + }, + // Unsupported characters. + { + "/export/test{1...2O}", + nil, + false, + }, + // Tests valid inputs. + { + "{1...27}", + []string{"EC:4", "EC:3", "EC:2"}, + true, + }, + { + "/export/set{1...64}", + []string{"EC:8", "EC:7", "EC:6", "EC:5", "EC:4", "EC:3", "EC:2"}, + true, + }, + // Valid input for distributed setup. + { + "http://minio{2...3}/export/set{1...64}", + []string{"EC:8", "EC:7", "EC:6", "EC:5", "EC:4", "EC:3", "EC:2"}, + true, + }, + // Supporting some advanced cases. + { + "http://minio{1...64}.mydomain.net/data", + []string{"EC:8", "EC:7", "EC:6", "EC:5", "EC:4", "EC:3", "EC:2"}, + true, + }, + { + "http://rack{1...4}.mydomain.minio{1...16}/data", + []string{"EC:8", "EC:7", "EC:6", "EC:5", "EC:4", "EC:3", "EC:2"}, + true, + }, + // Supporting kubernetes cases. + { + "http://minio{0...15}.mydomain.net/data{0...1}", + []string{"EC:8", "EC:7", "EC:6", "EC:5", "EC:4", "EC:3", "EC:2"}, + true, + }, + // No host regex, just disks. + { + "http://server1/data{1...32}", + []string{"EC:8", "EC:7", "EC:6", "EC:5", "EC:4", "EC:3", "EC:2"}, + true, + }, + // No host regex, just disks with two position numerics. + { + "http://server1/data{01...32}", + []string{"EC:8", "EC:7", "EC:6", "EC:5", "EC:4", "EC:3", "EC:2"}, + true, + }, + // More than 2 ellipses are supported as well. + { + "http://minio{2...3}/export/set{1...64}/test{1...2}", + []string{"EC:8", "EC:7", "EC:6", "EC:5", "EC:4", "EC:3", "EC:2"}, + true, + }, + // More than 1 ellipses per argument for standalone setup. + { + "/export{1...10}/disk{1...10}", + []string{"EC:5", "EC:4", "EC:3", "EC:2"}, + true, + }, + // IPv6 ellipses with hexadecimal expansion + { + "http://[2001:3984:3989::{1...a}]/disk{1...10}", + []string{"EC:5", "EC:4", "EC:3", "EC:2"}, + true, + }, + // IPv6 ellipses with hexadecimal expansion with 3 position numerics. + { + "http://[2001:3984:3989::{001...00a}]/disk{1...10}", + []string{"EC:5", "EC:4", "EC:3", "EC:2"}, + true, + }, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run("", func(t *testing.T) { + gotPs, err := PossibleParityValues(testCase.arg) + if err != nil && testCase.success { + t.Errorf("Expected success but failed instead %s", err) + } + if err == nil && !testCase.success { + t.Errorf("Expected failure but passed instead") + } + if !reflect.DeepEqual(testCase.parities, gotPs) { + t.Errorf("Expected %v, got %v", testCase.parities, gotPs) + } + }) + } +} diff --git a/restapi/admin_parity.go b/restapi/admin_parity.go new file mode 100644 index 000000000..5d8a3eb41 --- /dev/null +++ b/restapi/admin_parity.go @@ -0,0 +1,63 @@ +// This file is part of MinIO Console Server +// Copyright (c) 2020 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package restapi + +import ( + "fmt" + "log" + + "github.com/minio/console/pkg/utils" + + "github.com/go-openapi/runtime/middleware" + "github.com/minio/console/models" + "github.com/minio/console/restapi/operations" + "github.com/minio/console/restapi/operations/admin_api" +) + +func registerParityHandlers(api *operations.ConsoleAPI) { + api.AdminAPIGetParityHandler = admin_api.GetParityHandlerFunc(func(params admin_api.GetParityParams, principal *models.Principal) middleware.Responder { + resp, err := getParityResponse(params) + if err != nil { + return admin_api.NewGetParityDefault(int(err.Code)).WithPayload(err) + } + return admin_api.NewGetParityOK().WithPayload(resp) + }) +} + +func GetParityInfo(nodes int64, disksPerNode int64) (models.ParityResponse, error) { + parityVals, err := utils.PossibleParityValues(fmt.Sprintf(`http://minio{1...%d}/export/set{1...%d}`, nodes, disksPerNode)) + + if err != nil { + return nil, err + } + + return parityVals, nil +} + +func getParityResponse(params admin_api.GetParityParams) (models.ParityResponse, *models.Error) { + nodes := params.Nodes + disksPerNode := params.DisksPerNode + + parityValues, err := GetParityInfo(nodes, disksPerNode) + + if err != nil { + log.Println("error getting parity info:", err) + return nil, prepareError(err) + } + + return parityValues, nil +} diff --git a/restapi/admin_parity_test.go b/restapi/admin_parity_test.go new file mode 100644 index 000000000..12eaaaa87 --- /dev/null +++ b/restapi/admin_parity_test.go @@ -0,0 +1,79 @@ +// This file is part of MinIO Console Server +// Copyright (c) 2020 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package restapi + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/minio/console/models" +) + +func Test_getParityInfo(t *testing.T) { + tests := []struct { + description string + wantErr bool + nodes int64 + disksPerNode int64 + expectedResp models.ParityResponse + }{ + { + description: "Incorrect Number of endpoints provided", + wantErr: true, + nodes: 1, + disksPerNode: 1, + expectedResp: nil, + }, + { + description: "Number of endpoints is valid", + wantErr: false, + nodes: 4, + disksPerNode: 10, + expectedResp: models.ParityResponse{"EC:4", "EC:3", "EC:2"}, + }, + { + description: "More nodes than disks", + wantErr: false, + nodes: 4, + disksPerNode: 1, + expectedResp: models.ParityResponse{"EC:2"}, + }, + { + description: "More disks than nodes", + wantErr: false, + nodes: 2, + disksPerNode: 50, + expectedResp: models.ParityResponse{"EC:5", "EC:4", "EC:3", "EC:2"}, + }, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + parity, err := GetParityInfo(tt.nodes, tt.disksPerNode) + if (err != nil) != tt.wantErr { + t.Errorf("GetParityInfo() error = %v, wantErr %v", err, tt.wantErr) + } + + if !reflect.DeepEqual(parity, tt.expectedResp) { + ji, _ := json.Marshal(parity) + vi, _ := json.Marshal(tt.expectedResp) + t.Errorf("\ngot: %s \nwant: %s", ji, vi) + } + }) + } +} diff --git a/restapi/configure_console.go b/restapi/configure_console.go index 9ad5365a0..1f67b3d28 100644 --- a/restapi/configure_console.go +++ b/restapi/configure_console.go @@ -114,6 +114,8 @@ func configureAPI(api *operations.ConsoleAPI) http.Handler { registerResourceQuotaHandlers(api) // Register Nodes' handlers registerNodesHandlers(api) + // Register Parity' handlers + registerParityHandlers(api) api.PreServerShutdown = func() {} diff --git a/restapi/embedded_spec.go b/restapi/embedded_spec.go index 0e09c6812..e3950ec3f 100644 --- a/restapi/embedded_spec.go +++ b/restapi/embedded_spec.go @@ -572,6 +572,45 @@ func init() { } } }, + "/get-parity/{nodes}/{disksPerNode}": { + "get": { + "tags": [ + "AdminAPI" + ], + "summary": "Gets parity by sending number of nodes \u0026 number of disks", + "operationId": "GetParity", + "parameters": [ + { + "minimum": 2, + "type": "integer", + "name": "nodes", + "in": "path", + "required": true + }, + { + "minimum": 1, + "type": "integer", + "name": "disksPerNode", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "A successful response.", + "schema": { + "$ref": "#/definitions/parityResponse" + } + }, + "default": { + "description": "Generic error response.", + "schema": { + "$ref": "#/definitions/error" + } + } + } + } + }, "/groups": { "get": { "tags": [ @@ -2881,6 +2920,12 @@ func init() { "get" ] }, + "parityResponse": { + "type": "array", + "items": { + "type": "string" + } + }, "podAffinityTerm": { "description": "Required. A pod affinity term, associated with the corresponding weight.", "type": "object", @@ -4223,6 +4268,45 @@ func init() { } } }, + "/get-parity/{nodes}/{disksPerNode}": { + "get": { + "tags": [ + "AdminAPI" + ], + "summary": "Gets parity by sending number of nodes \u0026 number of disks", + "operationId": "GetParity", + "parameters": [ + { + "minimum": 2, + "type": "integer", + "name": "nodes", + "in": "path", + "required": true + }, + { + "minimum": 1, + "type": "integer", + "name": "disksPerNode", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "A successful response.", + "schema": { + "$ref": "#/definitions/parityResponse" + } + }, + "default": { + "description": "Generic error response.", + "schema": { + "$ref": "#/definitions/error" + } + } + } + } + }, "/groups": { "get": { "tags": [ @@ -7011,6 +7095,12 @@ func init() { "get" ] }, + "parityResponse": { + "type": "array", + "items": { + "type": "string" + } + }, "podAffinityTerm": { "description": "Required. A pod affinity term, associated with the corresponding weight.", "type": "object", diff --git a/restapi/operations/admin_api/get_parity.go b/restapi/operations/admin_api/get_parity.go new file mode 100644 index 000000000..bbaf9958c --- /dev/null +++ b/restapi/operations/admin_api/get_parity.go @@ -0,0 +1,90 @@ +// Code generated by go-swagger; DO NOT EDIT. + +// This file is part of MinIO Console Server +// Copyright (c) 2020 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +// + +package admin_api + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the generate command + +import ( + "net/http" + + "github.com/go-openapi/runtime/middleware" + + "github.com/minio/console/models" +) + +// GetParityHandlerFunc turns a function with the right signature into a get parity handler +type GetParityHandlerFunc func(GetParityParams, *models.Principal) middleware.Responder + +// Handle executing the request and returning a response +func (fn GetParityHandlerFunc) Handle(params GetParityParams, principal *models.Principal) middleware.Responder { + return fn(params, principal) +} + +// GetParityHandler interface for that can handle valid get parity params +type GetParityHandler interface { + Handle(GetParityParams, *models.Principal) middleware.Responder +} + +// NewGetParity creates a new http.Handler for the get parity operation +func NewGetParity(ctx *middleware.Context, handler GetParityHandler) *GetParity { + return &GetParity{Context: ctx, Handler: handler} +} + +/*GetParity swagger:route GET /get-parity/{nodes}/{disksPerNode} AdminAPI getParity + +Gets parity by sending number of nodes & number of disks + +*/ +type GetParity struct { + Context *middleware.Context + Handler GetParityHandler +} + +func (o *GetParity) ServeHTTP(rw http.ResponseWriter, r *http.Request) { + route, rCtx, _ := o.Context.RouteInfo(r) + if rCtx != nil { + r = rCtx + } + var Params = NewGetParityParams() + + uprinc, aCtx, err := o.Context.Authorize(r, route) + if err != nil { + o.Context.Respond(rw, r, route.Produces, route, err) + return + } + if aCtx != nil { + r = aCtx + } + var principal *models.Principal + if uprinc != nil { + principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise + } + + if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params + o.Context.Respond(rw, r, route.Produces, route, err) + return + } + + res := o.Handler.Handle(Params, principal) // actually handle the request + + o.Context.Respond(rw, r, route.Produces, route, res) + +} diff --git a/restapi/operations/admin_api/get_parity_parameters.go b/restapi/operations/admin_api/get_parity_parameters.go new file mode 100644 index 000000000..08ba32850 --- /dev/null +++ b/restapi/operations/admin_api/get_parity_parameters.go @@ -0,0 +1,154 @@ +// Code generated by go-swagger; DO NOT EDIT. + +// This file is part of MinIO Console Server +// Copyright (c) 2020 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +// + +package admin_api + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "net/http" + + "github.com/go-openapi/errors" + "github.com/go-openapi/runtime/middleware" + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" + "github.com/go-openapi/validate" +) + +// NewGetParityParams creates a new GetParityParams object +// no default values defined in spec. +func NewGetParityParams() GetParityParams { + + return GetParityParams{} +} + +// GetParityParams contains all the bound params for the get parity operation +// typically these are obtained from a http.Request +// +// swagger:parameters GetParity +type GetParityParams struct { + + // HTTP Request Object + HTTPRequest *http.Request `json:"-"` + + /* + Required: true + Minimum: 1 + In: path + */ + DisksPerNode int64 + /* + Required: true + Minimum: 2 + In: path + */ + Nodes int64 +} + +// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface +// for simple values it will use straight method calls. +// +// To ensure default values, the struct must have been initialized with NewGetParityParams() beforehand. +func (o *GetParityParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { + var res []error + + o.HTTPRequest = r + + rDisksPerNode, rhkDisksPerNode, _ := route.Params.GetOK("disksPerNode") + if err := o.bindDisksPerNode(rDisksPerNode, rhkDisksPerNode, route.Formats); err != nil { + res = append(res, err) + } + + rNodes, rhkNodes, _ := route.Params.GetOK("nodes") + if err := o.bindNodes(rNodes, rhkNodes, route.Formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +// bindDisksPerNode binds and validates parameter DisksPerNode from path. +func (o *GetParityParams) bindDisksPerNode(rawData []string, hasKey bool, formats strfmt.Registry) error { + var raw string + if len(rawData) > 0 { + raw = rawData[len(rawData)-1] + } + + // Required: true + // Parameter is provided by construction from the route + + value, err := swag.ConvertInt64(raw) + if err != nil { + return errors.InvalidType("disksPerNode", "path", "int64", raw) + } + o.DisksPerNode = value + + if err := o.validateDisksPerNode(formats); err != nil { + return err + } + + return nil +} + +// validateDisksPerNode carries on validations for parameter DisksPerNode +func (o *GetParityParams) validateDisksPerNode(formats strfmt.Registry) error { + + if err := validate.MinimumInt("disksPerNode", "path", int64(o.DisksPerNode), 1, false); err != nil { + return err + } + + return nil +} + +// bindNodes binds and validates parameter Nodes from path. +func (o *GetParityParams) bindNodes(rawData []string, hasKey bool, formats strfmt.Registry) error { + var raw string + if len(rawData) > 0 { + raw = rawData[len(rawData)-1] + } + + // Required: true + // Parameter is provided by construction from the route + + value, err := swag.ConvertInt64(raw) + if err != nil { + return errors.InvalidType("nodes", "path", "int64", raw) + } + o.Nodes = value + + if err := o.validateNodes(formats); err != nil { + return err + } + + return nil +} + +// validateNodes carries on validations for parameter Nodes +func (o *GetParityParams) validateNodes(formats strfmt.Registry) error { + + if err := validate.MinimumInt("nodes", "path", int64(o.Nodes), 2, false); err != nil { + return err + } + + return nil +} diff --git a/restapi/operations/admin_api/get_parity_responses.go b/restapi/operations/admin_api/get_parity_responses.go new file mode 100644 index 000000000..4165d2a9d --- /dev/null +++ b/restapi/operations/admin_api/get_parity_responses.go @@ -0,0 +1,136 @@ +// Code generated by go-swagger; DO NOT EDIT. + +// This file is part of MinIO Console Server +// Copyright (c) 2020 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +// + +package admin_api + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "net/http" + + "github.com/go-openapi/runtime" + + "github.com/minio/console/models" +) + +// GetParityOKCode is the HTTP code returned for type GetParityOK +const GetParityOKCode int = 200 + +/*GetParityOK A successful response. + +swagger:response getParityOK +*/ +type GetParityOK struct { + + /* + In: Body + */ + Payload models.ParityResponse `json:"body,omitempty"` +} + +// NewGetParityOK creates GetParityOK with default headers values +func NewGetParityOK() *GetParityOK { + + return &GetParityOK{} +} + +// WithPayload adds the payload to the get parity o k response +func (o *GetParityOK) WithPayload(payload models.ParityResponse) *GetParityOK { + o.Payload = payload + return o +} + +// SetPayload sets the payload to the get parity o k response +func (o *GetParityOK) SetPayload(payload models.ParityResponse) { + o.Payload = payload +} + +// WriteResponse to the client +func (o *GetParityOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.WriteHeader(200) + payload := o.Payload + if payload == nil { + // return empty array + payload = models.ParityResponse{} + } + + if err := producer.Produce(rw, payload); err != nil { + panic(err) // let the recovery middleware deal with this + } +} + +/*GetParityDefault Generic error response. + +swagger:response getParityDefault +*/ +type GetParityDefault struct { + _statusCode int + + /* + In: Body + */ + Payload *models.Error `json:"body,omitempty"` +} + +// NewGetParityDefault creates GetParityDefault with default headers values +func NewGetParityDefault(code int) *GetParityDefault { + if code <= 0 { + code = 500 + } + + return &GetParityDefault{ + _statusCode: code, + } +} + +// WithStatusCode adds the status to the get parity default response +func (o *GetParityDefault) WithStatusCode(code int) *GetParityDefault { + o._statusCode = code + return o +} + +// SetStatusCode sets the status to the get parity default response +func (o *GetParityDefault) SetStatusCode(code int) { + o._statusCode = code +} + +// WithPayload adds the payload to the get parity default response +func (o *GetParityDefault) WithPayload(payload *models.Error) *GetParityDefault { + o.Payload = payload + return o +} + +// SetPayload sets the payload to the get parity default response +func (o *GetParityDefault) SetPayload(payload *models.Error) { + o.Payload = payload +} + +// WriteResponse to the client +func (o *GetParityDefault) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.WriteHeader(o._statusCode) + if o.Payload != nil { + payload := o.Payload + if err := producer.Produce(rw, payload); err != nil { + panic(err) // let the recovery middleware deal with this + } + } +} diff --git a/restapi/operations/admin_api/get_parity_urlbuilder.go b/restapi/operations/admin_api/get_parity_urlbuilder.go new file mode 100644 index 000000000..9b242a71b --- /dev/null +++ b/restapi/operations/admin_api/get_parity_urlbuilder.go @@ -0,0 +1,126 @@ +// Code generated by go-swagger; DO NOT EDIT. + +// This file is part of MinIO Console Server +// Copyright (c) 2020 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +// + +package admin_api + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the generate command + +import ( + "errors" + "net/url" + golangswaggerpaths "path" + "strings" + + "github.com/go-openapi/swag" +) + +// GetParityURL generates an URL for the get parity operation +type GetParityURL struct { + DisksPerNode int64 + Nodes int64 + + _basePath string + // avoid unkeyed usage + _ struct{} +} + +// WithBasePath sets the base path for this url builder, only required when it's different from the +// base path specified in the swagger spec. +// When the value of the base path is an empty string +func (o *GetParityURL) WithBasePath(bp string) *GetParityURL { + o.SetBasePath(bp) + return o +} + +// SetBasePath sets the base path for this url builder, only required when it's different from the +// base path specified in the swagger spec. +// When the value of the base path is an empty string +func (o *GetParityURL) SetBasePath(bp string) { + o._basePath = bp +} + +// Build a url path and query string +func (o *GetParityURL) Build() (*url.URL, error) { + var _result url.URL + + var _path = "/get-parity/{nodes}/{disksPerNode}" + + disksPerNode := swag.FormatInt64(o.DisksPerNode) + if disksPerNode != "" { + _path = strings.Replace(_path, "{disksPerNode}", disksPerNode, -1) + } else { + return nil, errors.New("disksPerNode is required on GetParityURL") + } + + nodes := swag.FormatInt64(o.Nodes) + if nodes != "" { + _path = strings.Replace(_path, "{nodes}", nodes, -1) + } else { + return nil, errors.New("nodes is required on GetParityURL") + } + + _basePath := o._basePath + if _basePath == "" { + _basePath = "/api/v1" + } + _result.Path = golangswaggerpaths.Join(_basePath, _path) + + return &_result, nil +} + +// Must is a helper function to panic when the url builder returns an error +func (o *GetParityURL) Must(u *url.URL, err error) *url.URL { + if err != nil { + panic(err) + } + if u == nil { + panic("url can't be nil") + } + return u +} + +// String returns the string representation of the path with query string +func (o *GetParityURL) String() string { + return o.Must(o.Build()).String() +} + +// BuildFull builds a full url with scheme, host, path and query string +func (o *GetParityURL) BuildFull(scheme, host string) (*url.URL, error) { + if scheme == "" { + return nil, errors.New("scheme is required for a full url on GetParityURL") + } + if host == "" { + return nil, errors.New("host is required for a full url on GetParityURL") + } + + base, err := o.Build() + if err != nil { + return nil, err + } + + base.Scheme = scheme + base.Host = host + return base, nil +} + +// StringFull returns the string representation of a complete url +func (o *GetParityURL) StringFull(scheme, host string) string { + return o.Must(o.BuildFull(scheme, host)).String() +} diff --git a/restapi/operations/console_api.go b/restapi/operations/console_api.go index 06799b113..213a05c93 100644 --- a/restapi/operations/console_api.go +++ b/restapi/operations/console_api.go @@ -117,6 +117,9 @@ func NewConsoleAPI(spec *loads.Document) *ConsoleAPI { AdminAPIGetMaxAllocatableMemHandler: admin_api.GetMaxAllocatableMemHandlerFunc(func(params admin_api.GetMaxAllocatableMemParams, principal *models.Principal) middleware.Responder { return middleware.NotImplemented("operation admin_api.GetMaxAllocatableMem has not yet been implemented") }), + AdminAPIGetParityHandler: admin_api.GetParityHandlerFunc(func(params admin_api.GetParityParams, principal *models.Principal) middleware.Responder { + return middleware.NotImplemented("operation admin_api.GetParity has not yet been implemented") + }), AdminAPIGetResourceQuotaHandler: admin_api.GetResourceQuotaHandlerFunc(func(params admin_api.GetResourceQuotaParams, principal *models.Principal) middleware.Responder { return middleware.NotImplemented("operation admin_api.GetResourceQuota has not yet been implemented") }), @@ -319,6 +322,8 @@ type ConsoleAPI struct { AdminAPIDeleteTenantHandler admin_api.DeleteTenantHandler // AdminAPIGetMaxAllocatableMemHandler sets the operation handler for the get max allocatable mem operation AdminAPIGetMaxAllocatableMemHandler admin_api.GetMaxAllocatableMemHandler + // AdminAPIGetParityHandler sets the operation handler for the get parity operation + AdminAPIGetParityHandler admin_api.GetParityHandler // AdminAPIGetResourceQuotaHandler sets the operation handler for the get resource quota operation AdminAPIGetResourceQuotaHandler admin_api.GetResourceQuotaHandler // AdminAPIGetTenantUsageHandler sets the operation handler for the get tenant usage operation @@ -524,6 +529,9 @@ func (o *ConsoleAPI) Validate() error { if o.AdminAPIGetMaxAllocatableMemHandler == nil { unregistered = append(unregistered, "admin_api.GetMaxAllocatableMemHandler") } + if o.AdminAPIGetParityHandler == nil { + unregistered = append(unregistered, "admin_api.GetParityHandler") + } if o.AdminAPIGetResourceQuotaHandler == nil { unregistered = append(unregistered, "admin_api.GetResourceQuotaHandler") } @@ -816,6 +824,10 @@ func (o *ConsoleAPI) initHandlerCache() { if o.handlers["GET"] == nil { o.handlers["GET"] = make(map[string]http.Handler) } + o.handlers["GET"]["/get-parity/{nodes}/{disksPerNode}"] = admin_api.NewGetParity(o.context, o.AdminAPIGetParityHandler) + if o.handlers["GET"] == nil { + o.handlers["GET"] = make(map[string]http.Handler) + } o.handlers["GET"]["/namespaces/{namespace}/resourcequotas/{resource-quota-name}"] = admin_api.NewGetResourceQuota(o.context, o.AdminAPIGetResourceQuotaHandler) if o.handlers["GET"] == nil { o.handlers["GET"] = make(map[string]http.Handler) diff --git a/swagger.yml b/swagger.yml index 475ce1699..eabf5492b 100644 --- a/swagger.yml +++ b/swagger.yml @@ -1306,6 +1306,33 @@ paths: tags: - AdminAPI + /get-parity/{nodes}/{disksPerNode}: + get: + summary: Gets parity by sending number of nodes & number of disks + operationId: GetParity + parameters: + - name: nodes + in: path + required: true + type: integer + minimum: 2 + - name: disksPerNode + in: path + required: true + type: integer + minimum: 1 + responses: + 200: + description: A successful response. + schema: + $ref: "#/definitions/parityResponse" + default: + description: Generic error response. + schema: + $ref: "#/definitions/error" + tags: + - AdminAPI + definitions: bucketAccess: type: string @@ -2677,3 +2704,8 @@ definitions: max_memory: type: integer format: int64 + + parityResponse: + type: array + items: + type: string