Added EC Parity correct values to add tenant modal (#517)

Co-authored-by: Benjamin Perez <benjamin@bexsoft.net>
Co-authored-by: Daniel Valdivia <hola@danielvaldivia.com>

portal-ui/src/common/__tests__/utils.test.ts

@@ -14,7 +14,12 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import { getBytes, niceBytes, setMemoryResource } from "../utils";
+import {
+  erasureCodeCalc,
+  getBytes,
+  niceBytes,
+  setMemoryResource,
+} from "../utils";
 
 test("A variety of formatting results", () => {
   expect(niceBytes("1024")).toBe("1.0 KiB");
@@ -52,3 +57,29 @@ test("Determine the amount of memory to use", () => {
     request: 2147483648,
   });
 });
+
+test("Determine the correct values for EC Parity calculation", () => {
+  expect(erasureCodeCalc([], 50, 5000, 4)).toStrictEqual({
+    error: 1,
+    defaultEC: "",
+    erasureCodeSet: 0,
+    maxEC: "",
+    rawCapacity: "0",
+    storageFactors: [],
+  });
+  expect(erasureCodeCalc(["EC:2"], 4, 26843545600, 4)).toStrictEqual({
+    error: 0,
+    storageFactors: [
+      {
+        erasureCode: "EC:2",
+        storageFactor: 2,
+        maxCapacity: "53687091200",
+        maxFailureTolerations: 2,
+      },
+    ],
+    maxEC: "EC:2",
+    rawCapacity: "107374182400",
+    erasureCodeSet: 4,
+    defaultEC: "EC:2",
+  });
+});

portal-ui/src/common/types.ts

@@ -250,19 +250,11 @@ export interface IStorageDistribution {
   pvSize: number;
 }
 
-export interface IErasureCodeCalc {
-  error: number;
-  maxEC: string;
-  erasureCodeSet: number;
-  rawCapacity: string;
-  storageFactors: IStorageFactors[];
-  defaultEC: string;
-}
-
 export interface IStorageFactors {
   erasureCode: string;
   storageFactor: number;
   maxCapacity: string;
+  maxFailureTolerations: number;
 }
 
 export interface ITenantHealthInList {
@@ -338,3 +330,12 @@ export interface ICapacity {
   value: string;
   unit: string;
 }
+
+export interface IErasureCodeCalc {
+  error: number;
+  maxEC: string;
+  erasureCodeSet: number;
+  rawCapacity: string;
+  defaultEC: string;
+  storageFactors: IStorageFactors[];
+}

portal-ui/src/common/utils.ts

@@ -15,7 +15,12 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import storage from "local-storage-fallback";
-import { ICapacity, IPoolModel } from "./types";
+import {
+  ICapacity,
+  IErasureCodeCalc,
+  IPoolModel,
+  IStorageFactors,
+} from "./types";
 
 const minStReq = 1073741824; // Minimal Space required for MinIO
 const minMemReq = 2147483648; // Minimal Memory required for MinIO in bytes
@@ -193,7 +198,8 @@ export const setMemoryResource = (
 export const calculateDistribution = (
   capacityToUse: ICapacity,
   forcedNodes: number = 0,
-  limitSize: number = 0
+  limitSize: number = 0,
+  drivesPerServer: number = 0
 ) => {
   let numberOfNodes = {};
   const requestedSizeBytes = getBytes(
@@ -222,7 +228,22 @@ export const calculateDistribution = (
     };
   }
 
-  numberOfNodes = calculateStorage(requestedSizeBytes, forcedNodes, limitSize);
+  if (drivesPerServer <= 0) {
+    return {
+      error: "Number of drives must be at least 1",
+      nodes: 0,
+      persistentVolumes: 0,
+      disks: 0,
+      pvSize: 0,
+    };
+  }
+
+  numberOfNodes = calculateStorage(
+    requestedSizeBytes,
+    forcedNodes,
+    limitSize,
+    drivesPerServer
+  );
 
   return numberOfNodes;
 };
@@ -230,14 +251,21 @@ export const calculateDistribution = (
 const calculateStorage = (
   requestedBytes: string,
   forcedNodes: number,
-  limitSize: number
+  limitSize: number,
+  drivesPerServer: number
 ) => {
   // Size validation
   const intReqBytes = parseInt(requestedBytes, 10);
   const maxDiskSize = minStReq * 256; // 256 GiB
 
   // We get the distribution
-  return structureCalc(forcedNodes, intReqBytes, maxDiskSize, limitSize);
+  return structureCalc(
+    forcedNodes,
+    intReqBytes,
+    maxDiskSize,
+    limitSize,
+    drivesPerServer
+  );
 };
 
 const structureCalc = (
@@ -324,6 +352,67 @@ const structureCalc = (
   };
 };
 
+// Erasure Code Parity Calc
+export const erasureCodeCalc = (
+  parityValidValues: string[],
+  totalDisks: number,
+  pvSize: number,
+  totalNodes: number
+): IErasureCodeCalc => {
+  // Parity Values is empty
+  if (parityValidValues.length < 1) {
+    return {
+      error: 1,
+      defaultEC: "",
+      erasureCodeSet: 0,
+      maxEC: "",
+      rawCapacity: "0",
+      storageFactors: [],
+    };
+  }
+
+  const totalStorage = totalDisks * pvSize;
+  const maxEC = parityValidValues[0];
+  const maxParityNumber = parseInt(maxEC.split(":")[1], 10);
+
+  const erasureStripeSet = maxParityNumber * 2; // ESS is calculated by multiplying maximum parity by two.
+
+  const storageFactors: IStorageFactors[] = parityValidValues.map(
+    (currentParity) => {
+      const parityNumber = parseInt(currentParity.split(":")[1], 10);
+      const storageFactor =
+        erasureStripeSet / (erasureStripeSet - parityNumber);
+
+      const maxCapacity = Math.floor(totalStorage / storageFactor);
+      const maxTolerations =
+        totalDisks - Math.floor(totalDisks / storageFactor);
+      return {
+        erasureCode: currentParity,
+        storageFactor,
+        maxCapacity: maxCapacity.toString(10),
+        maxFailureTolerations: maxTolerations,
+      };
+    }
+  );
+
+  let defaultEC = maxEC;
+
+  const fourVar = parityValidValues.find((element) => element === "EC:4");
+
+  if (totalDisks >= 8 && totalNodes > 16 && fourVar) {
+    defaultEC = "EC:4";
+  }
+
+  return {
+    error: 0,
+    storageFactors,
+    maxEC,
+    rawCapacity: totalStorage.toString(10),
+    erasureCodeSet: erasureStripeSet,
+    defaultEC,
+  };
+};
+
 // Pool Name Generator
 export const generatePoolName = (pools: IPoolModel[]) => {
   const poolCounter = pools.length;

portal-ui/src/screens/Console/Account/Account.tsx

@@ -103,10 +103,9 @@ const Account = ({ classes }: IServiceAccountsProps) => {
     newServiceAccount,
     setNewServiceAccount,
   ] = useState<NewServiceAccount | null>(null);
-  const [
-    changePasswordModalOpen,
-    setChangePasswordModalOpen,
-  ] = useState<boolean>(false);
+  const [changePasswordModalOpen, setChangePasswordModalOpen] = useState<
+    boolean
+  >(false);
 
   useEffect(() => {
     fetchRecords();

portal-ui/src/screens/Console/Tenants/ListTenants/AddTenant.tsx

@@ -32,6 +32,7 @@ import FormSwitchWrapper from "../../Common/FormComponents/FormSwitchWrapper/For
 import SelectWrapper from "../../Common/FormComponents/SelectWrapper/SelectWrapper";
 import {
   calculateDistribution,
+  erasureCodeCalc,
   generatePoolName,
   getBytes,
   k8sfactorForDropdown,
@@ -46,12 +47,14 @@ import { IWizardElement } from "../../Common/GenericWizard/types";
 import { NewServiceAccount } from "../../Common/CredentialsPrompt/types";
 import RadioGroupSelector from "../../Common/FormComponents/RadioGroupSelector/RadioGroupSelector";
 import FileSelector from "../../Common/FormComponents/FileSelector/FileSelector";
+import ErrorBlock from "../../../shared/ErrorBlock";
 import {
   IAffinityModel,
   ICapacity,
+  IErasureCodeCalc,
   ITenantCreator,
 } from "../../../../common/types";
-import ErrorBlock from "../../../shared/ErrorBlock";
+import { ecListTransform, Opts } from "./utils";
 
 interface IAddTenantProps {
   open: boolean;
@@ -107,11 +110,6 @@ const styles = (theme: Theme) =>
     ...modalBasic,
   });
 
-interface Opts {
-  label: string;
-  value: string;
-}
-
 const AddTenant = ({
   open,
   closeModalAndRefresh,
@@ -165,7 +163,9 @@ const AddTenant = ({
   const [vaultRetry, setVaultRetry] = useState<string>("0");
   const [vaultPing, setVaultPing] = useState<string>("0");
   const [ecParityChoices, setECParityChoices] = useState<Opts[]>([]);
+  const [cleanECChoices, setCleanECChoices] = useState<string[]>([]);
   const [nodes, setNodes] = useState<string>("4");
+  const [drivesPerServer, setDrivesPerServer] = useState<string>("1");
   const [memoryNode, setMemoryNode] = useState<string>("2");
   const [ecParity, setECParity] = useState<string>("");
   const [distribution, setDistribution] = useState<any>({
@@ -175,6 +175,14 @@ const AddTenant = ({
     disks: 0,
     volumePerDisk: 0,
   });
+  const [ecParityCalc, setEcParityCalc] = useState<IErasureCodeCalc>({
+    error: 0,
+    defaultEC: "",
+    erasureCodeSet: 0,
+    maxEC: "",
+    rawCapacity: "0",
+    storageFactors: [],
+  });
 
   // Forms Validation
   const [nameTenantValid, setNameTenantValid] = useState<boolean>(false);
@@ -271,34 +279,66 @@ const AddTenant = ({
       return debounceNamespace.cancel;
     }
   }, [namespace, debounceNamespace]);
+
+  useEffect(() => {
+    if (ecParityChoices.length > 0 && distribution.error === "") {
+      const ecCodeValidated = erasureCodeCalc(
+        cleanECChoices,
+        distribution.persistentVolumes,
+        distribution.pvSize,
+        distribution.nodes
+      );
+
+      setEcParityCalc(ecCodeValidated);
+      setECParity(ecCodeValidated.defaultEC);
+    }
+  }, [ecParityChoices.length, distribution, cleanECChoices]);
   /*End debounce functions*/
 
   /*Calculate Allocation*/
   useEffect(() => {
     validateClusterSize();
-    setECParityChoices([]);
+    getECValue();
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [nodes, volumeSize, sizeFactor]);
+  }, [nodes, volumeSize, sizeFactor, drivesPerServer]);
 
   const validateClusterSize = () => {
     const size = volumeSize;
     const factor = sizeFactor;
     const limitSize = getBytes("12", "Ti", true);
 
-    const capacityElement: ICapacity = {
+    const clusterCapacity: ICapacity = {
       unit: factor,
       value: size.toString(),
     };
 
     const distrCalculate = calculateDistribution(
-      capacityElement,
+      clusterCapacity,
       parseInt(nodes),
-      parseInt(limitSize)
+      parseInt(limitSize),
+      parseInt(drivesPerServer)
     );
 
     setDistribution(distrCalculate);
   };
 
+  const getECValue = () => {
+    setECParity("");
+
+    if (nodes.trim() !== "" && drivesPerServer.trim() !== "") {
+      api
+        .invoke("GET", `/api/v1/get-parity/${nodes}/${drivesPerServer}`)
+        .then((ecList: string[]) => {
+          setECParityChoices(ecListTransform(ecList));
+          setCleanECChoices(ecList);
+        })
+        .catch((err: any) => {
+          setECParityChoices([]);
+          setConfigValid(false);
+          setECParity("");
+        });
+    }
+  };
   /*Calculate Allocation End*/
 
   /* Validations of pages */
@@ -355,17 +395,34 @@ const AddTenant = ({
         customValidation: parseInt(memoryNode) < 2,
         customValidationMessage: "Memory size must be greater than 2Gi",
       },
+      {
+        fieldKey: "drivesps",
+        required: true,
+        value: drivesPerServer,
+        customValidation: parseInt(drivesPerServer) < 1,
+        customValidationMessage: "There must be at least one drive",
+      },
     ]);
 
     setConfigValid(
       !("nodes" in commonValidation) &&
         !("volume_size" in commonValidation) &&
         !("memory_per_node" in commonValidation) &&
-        distribution.error === ""
+        !("drivesps" in commonValidation) &&
+        distribution.error === "" &&
+        ecParityCalc.error === 0
     );
 
     setValidationErrors(commonValidation);
-  }, [nodes, volumeSize, sizeFactor, memoryNode, distribution]);
+  }, [
+    nodes,
+    volumeSize,
+    sizeFactor,
+    memoryNode,
+    distribution,
+    drivesPerServer,
+    ecParityCalc,
+  ]);
 
   useEffect(() => {
     let customAccountValidation: IValidation[] = [];
@@ -760,9 +817,7 @@ const AddTenant = ({
         },
       };
 
-      const ecLimit = "EC:0";
-
-      const erasureCode = ecLimit.split(":")[1];
+      const erasureCode = ecParity.split(":")[1];
 
       let dataSend: ITenantCreator = {
         name: tenantName,
@@ -991,6 +1046,10 @@ const AddTenant = ({
     },
   };
 
+  const usableInformation = ecParityCalc.storageFactors.find(
+    (element) => element.erasureCode === ecParity
+  );
+
   const wizardSteps: IWizardElement[] = [
     {
       label: "Name Tenant",
@@ -2004,13 +2063,29 @@ const AddTenant = ({
                 setNodes(e.target.value);
                 clearValidationError("nodes");
               }}
-              label="Number of Nodes"
+              label="Number of Servers"
               value={nodes}
               min="4"
               required
               error={validationErrors["nodes"] || ""}
             />
           </Grid>
+          <Grid item xs={12}>
+            <InputBoxWrapper
+              id="drivesps"
+              name="drivesps"
+              type="number"
+              onChange={(e: React.ChangeEvent<HTMLInputElement>) => {
+                setDrivesPerServer(e.target.value);
+                clearValidationError("drivesps");
+              }}
+              label="Number of Drives per Server"
+              value={drivesPerServer}
+              min="1"
+              required
+              error={validationErrors["drivesps"] || ""}
+            />
+          </Grid>
           <Grid item xs={12}>
             <div className={classes.multiContainer}>
               <div>
@@ -2077,12 +2152,20 @@ const AddTenant = ({
               </span>
             </Grid>
           )}
-          <h5>Resource Allocation</h5>
+          <h4>Resource Allocation</h4>
           <Table className={classes.table} aria-label="simple table">
             <TableBody>
               <TableRow>
                 <TableCell component="th" scope="row">
-                  Volumes per Node
+                  Number of Servers
+                </TableCell>
+                <TableCell align="right">
+                  {parseInt(nodes) > 0 ? nodes : "-"}
+                </TableCell>
+              </TableRow>
+              <TableRow>
+                <TableCell component="th" scope="row">
+                  Drives per Server
                 </TableCell>
                 <TableCell align="right">
                   {distribution ? distribution.disks : "-"}
@@ -2090,7 +2173,7 @@ const AddTenant = ({
               </TableRow>
               <TableRow>
                 <TableCell component="th" scope="row">
-                  Disk Size
+                  Drive Capacity
                 </TableCell>
                 <TableCell align="right">
                   {distribution ? niceBytes(distribution.pvSize) : "-"}
@@ -2106,6 +2189,52 @@ const AddTenant = ({
               </TableRow>
             </TableBody>
           </Table>
+          {ecParityCalc.error === 0 && usableInformation && (
+            <React.Fragment>
+              <h4>Erasure Code Configuration</h4>
+              <Table className={classes.table} aria-label="simple table">
+                <TableBody>
+                  <TableRow>
+                    <TableCell component="th" scope="row">
+                      EC Parity
+                    </TableCell>
+                    <TableCell align="right">
+                      {ecParity !== "" ? ecParity : "-"}
+                    </TableCell>
+                  </TableRow>
+                  <TableRow>
+                    <TableCell component="th" scope="row">
+                      Raw Capacity
+                    </TableCell>
+                    <TableCell align="right">
+                      {niceBytes(ecParityCalc.rawCapacity)}
+                    </TableCell>
+                  </TableRow>
+                  <TableRow>
+                    <TableCell component="th" scope="row">
+                      Usable Capacity
+                    </TableCell>
+                    <TableCell align="right">
+                      {niceBytes(usableInformation.maxCapacity)}
+                    </TableCell>
+                  </TableRow>
+                  <TableRow>
+                    <TableCell component="th" scope="row">
+                      Number of server failures to tolerate
+                    </TableCell>
+                    <TableCell align="right">
+                      {distribution
+                        ? Math.floor(
+                            usableInformation.maxFailureTolerations /
+                              distribution.disks
+                          )
+                        : "-"}
+                    </TableCell>
+                  </TableRow>
+                </TableBody>
+              </Table>
+            </React.Fragment>
+          )}
         </React.Fragment>
       ),
       buttons: [

portal-ui/src/screens/Console/Tenants/ListTenants/utils.ts

@@ -0,0 +1,26 @@
+// This file is part of MinIO Console Server
+// Copyright (c) 2020 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+export interface Opts {
+  label: string;
+  value: string;
+}
+
+export const ecListTransform = (ecList: string[]): Opts[] => {
+  return ecList.map((value) => {
+    return { label: value, value };
+  });
+};

swagger.yml

@@ -3675,4 +3675,4 @@ definitions:
         $ref: "#/definitions/objectRetentionUnit"
       validity:
         type: integer
-        format: int32
+        format: int32