To support multiple CSRs per tenant (#1997)

.github/workflows/deploy-tenant.sh

@@ -20,17 +20,17 @@ export SCRIPT_DIR
 
 source "${SCRIPT_DIR}/common.sh"
 
+function install_tenants() {
+	echo "Installing tenants"
 
+	# Install lite & kes tenants
+	try kubectl apply -k "${SCRIPT_DIR}/../../portal-ui/tests/scripts/tenant-lite"
+	try kubectl apply -k "${SCRIPT_DIR}/../../portal-ui/tests/scripts/tenant-kes-encryption"
 
-function install_tenant() {
-    echo "Installing lite tenant"
-
-	try kubectl apply -k "${SCRIPT_DIR}/../../portal-ui/tests/scripts/tenant"
-
-    echo "Waiting for the tenant statefulset, this indicates the tenant is being fulfilled"
-    waitdone=0
-    totalwait=0
-    while true; do
+	echo "Waiting for the tenant statefulset, this indicates the tenant is being fulfilled"
+	waitdone=0
+	totalwait=0
+	while true; do
 	waitdone=$(kubectl -n tenant-lite get pods -l v1.min.io/tenant=storage-lite --no-headers | wc -l)
 	if [ "$waitdone" -ne 0 ]; then
 	    echo "Found $waitdone pods"
@@ -39,41 +39,34 @@ function install_tenant() {
 	sleep 5
 	totalwait=$((totalwait + 5))
 	if [ "$totalwait" -gt 300 ]; then
-	    echo "Tenant never created statefulset after 5 minutes"
-	    try false
+		echo "Tenant never created statefulset after 5 minutes"
+		try false
 	fi
-    done
+	done
 
-    echo "Waiting for tenant pods to come online (5m timeout)"
-    try kubectl wait --namespace tenant-lite \
+	echo "Waiting for tenant pods to come online (5m timeout)"
+	try kubectl wait --namespace tenant-lite \
 	--for=condition=ready pod \
 	--selector="v1.min.io/tenant=storage-lite" \
 	--timeout=300s
 
-    echo "Build passes basic tenant creation"
+	echo "Build passes basic tenant creation"
 }
 
 
 function main() {
-    destroy_kind
-
-    setup_kind
-
-    install_operator
-
-    install_tenant
-
-    check_tenant_status tenant-lite storage-lite
-
-    kubectl proxy &
-
-    # Beginning  Kubernetes 1.24 ----> Service Account Token Secrets are not 
-    # automatically generated, to generate them manually, users must manually
-    # create the secret, for our examples where we lead people to get the JWT
-    # from the console-sa service account, they additionally need to manually
-    # generate the secret via
-    kubectl apply -f "${SCRIPT_DIR}/console-sa-secret.yaml"
-
+	destroy_kind
+	setup_kind
+	install_operator
+	install_tenants
+	check_tenant_status tenant-lite storage-lite
+	kubectl proxy &
+	# Beginning  Kubernetes 1.24 ----> Service Account Token Secrets are not 
+	# automatically generated, to generate them manually, users must manually
+	# create the secret, for our examples where we lead people to get the JWT
+	# from the console-sa service account, they additionally need to manually
+	# generate the secret via
+	kubectl apply -f "${SCRIPT_DIR}/console-sa-secret.yaml"
 }
 
 main "$@"

models/csr_elements.go

@@ -0,0 +1,133 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+// This file is part of MinIO Console Server
+// Copyright (c) 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+//
+
+package models
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"context"
+	"strconv"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/strfmt"
+	"github.com/go-openapi/swag"
+)
+
+// CsrElements csr elements
+//
+// swagger:model csrElements
+type CsrElements struct {
+
+	// csr element
+	CsrElement []*CsrElement `json:"csrElement"`
+}
+
+// Validate validates this csr elements
+func (m *CsrElements) Validate(formats strfmt.Registry) error {
+	var res []error
+
+	if err := m.validateCsrElement(formats); err != nil {
+		res = append(res, err)
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}
+
+func (m *CsrElements) validateCsrElement(formats strfmt.Registry) error {
+	if swag.IsZero(m.CsrElement) { // not required
+		return nil
+	}
+
+	for i := 0; i < len(m.CsrElement); i++ {
+		if swag.IsZero(m.CsrElement[i]) { // not required
+			continue
+		}
+
+		if m.CsrElement[i] != nil {
+			if err := m.CsrElement[i].Validate(formats); err != nil {
+				if ve, ok := err.(*errors.Validation); ok {
+					return ve.ValidateName("csrElement" + "." + strconv.Itoa(i))
+				} else if ce, ok := err.(*errors.CompositeError); ok {
+					return ce.ValidateName("csrElement" + "." + strconv.Itoa(i))
+				}
+				return err
+			}
+		}
+
+	}
+
+	return nil
+}
+
+// ContextValidate validate this csr elements based on the context it is used
+func (m *CsrElements) ContextValidate(ctx context.Context, formats strfmt.Registry) error {
+	var res []error
+
+	if err := m.contextValidateCsrElement(ctx, formats); err != nil {
+		res = append(res, err)
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}
+
+func (m *CsrElements) contextValidateCsrElement(ctx context.Context, formats strfmt.Registry) error {
+
+	for i := 0; i < len(m.CsrElement); i++ {
+
+		if m.CsrElement[i] != nil {
+			if err := m.CsrElement[i].ContextValidate(ctx, formats); err != nil {
+				if ve, ok := err.(*errors.Validation); ok {
+					return ve.ValidateName("csrElement" + "." + strconv.Itoa(i))
+				} else if ce, ok := err.(*errors.CompositeError); ok {
+					return ce.ValidateName("csrElement" + "." + strconv.Itoa(i))
+				}
+				return err
+			}
+		}
+
+	}
+
+	return nil
+}
+
+// MarshalBinary interface implementation
+func (m *CsrElements) MarshalBinary() ([]byte, error) {
+	if m == nil {
+		return nil, nil
+	}
+	return swag.WriteJSON(m)
+}
+
+// UnmarshalBinary interface implementation
+func (m *CsrElements) UnmarshalBinary(b []byte) error {
+	var res CsrElements
+	if err := swag.ReadJSON(b, &res); err != nil {
+		return err
+	}
+	*m = res
+	return nil
+}

operator-integration/tenant_test.go

@@ -260,7 +260,7 @@ func TestListTenants(t *testing.T) {
 	}
 	TenantName := &result.Tenants[0].Name // The array has to be empty, no index accessible
 	fmt.Println(*TenantName)
-	assert.Equal("storage-lite", *TenantName, *TenantName)
+	assert.Equal("storage-kms-encrypted", *TenantName, *TenantName)
 	printEndFunc("TestListTenants")
 }
 
@@ -637,5 +637,39 @@ func TestGetCSR(t *testing.T) {
 		assert.Equal(
 			200, resp.StatusCode, finalResponse)
 	}
-	assert.Equal(strings.Contains(finalResponse, "Automatically approved by MinIO Operator"), true)
+	assert.Equal(strings.Contains(finalResponse, "Automatically approved by MinIO Operator"), true, finalResponse)
+}
+
+func TestGetMultipleCSRs(t *testing.T) {
+	/*
+		We can have multiple CSRs per tenant, the idea is to support them in our API and test them here, making sure we
+		can retrieve them all, as an example I found this tenant:
+		storage-kms-encrypted  -client  -tenant-kms-encrypted-csr
+		storage-kms-encrypted  -kes     -tenant-kms-encrypted-csr
+		storage-kms-encrypted           -tenant-kms-encrypted-csr
+		Notice the nomenclature of it:
+		<tenant-name>-<*>-<namespace>-csr
+		where * is anything either nothing or something, anything.
+	*/
+	assert := assert.New(t)
+	namespace := "tenant-kms-encrypted"
+	tenant := "storage-kms-encrypted"
+	resp, err := GetCSR(namespace, tenant)
+	assert.Nil(err)
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	finalResponse := inspectHTTPResponse(resp)
+	if resp != nil {
+		assert.Equal(
+			200, resp.StatusCode, finalResponse)
+	}
+	var expectedMessages [3]string
+	expectedMessages[0] = "storage-kms-encrypted-tenant-kms-encrypted-csr"
+	expectedMessages[1] = "storage-kms-encrypted-kes-tenant-kms-encrypted-csr"
+	expectedMessages[2] = "Automatically approved by MinIO Operator"
+	for _, element := range expectedMessages {
+		assert.Equal(strings.Contains(finalResponse, element), true)
+	}
 }

operatorapi/embedded_spec.go

@@ -608,7 +608,7 @@ func init() {
           "200": {
             "description": "A successful response.",
             "schema": {
-              "$ref": "#/definitions/csrElement"
+              "$ref": "#/definitions/csrElements"
             }
           },
           "default": {
@@ -2387,6 +2387,17 @@ func init() {
         }
       }
     },
+    "csrElements": {
+      "type": "object",
+      "properties": {
+        "csrElement": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/csrElement"
+          }
+        }
+      }
+    },
     "deleteTenantRequest": {
       "type": "object",
       "properties": {
@@ -5201,7 +5212,7 @@ func init() {
           "200": {
             "description": "A successful response.",
             "schema": {
-              "$ref": "#/definitions/csrElement"
+              "$ref": "#/definitions/csrElements"
             }
           },
           "default": {
@@ -7823,6 +7834,17 @@ func init() {
         }
       }
     },
+    "csrElements": {
+      "type": "object",
+      "properties": {
+        "csrElement": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/csrElement"
+          }
+        }
+      }
+    },
     "deleteTenantRequest": {
       "type": "object",
       "properties": {

operatorapi/operations/operator_api/list_tenant_certificate_signing_request_responses.go

@@ -42,7 +42,7 @@ type ListTenantCertificateSigningRequestOK struct {
 	/*
 	  In: Body
 	*/
-	Payload *models.CsrElement `json:"body,omitempty"`
+	Payload *models.CsrElements `json:"body,omitempty"`
 }
 
 // NewListTenantCertificateSigningRequestOK creates ListTenantCertificateSigningRequestOK with default headers values
@@ -52,13 +52,13 @@ func NewListTenantCertificateSigningRequestOK() *ListTenantCertificateSigningReq
 }
 
 // WithPayload adds the payload to the list tenant certificate signing request o k response
-func (o *ListTenantCertificateSigningRequestOK) WithPayload(payload *models.CsrElement) *ListTenantCertificateSigningRequestOK {
+func (o *ListTenantCertificateSigningRequestOK) WithPayload(payload *models.CsrElements) *ListTenantCertificateSigningRequestOK {
 	o.Payload = payload
 	return o
 }
 
 // SetPayload sets the payload to the list tenant certificate signing request o k response
-func (o *ListTenantCertificateSigningRequestOK) SetPayload(payload *models.CsrElement) {
+func (o *ListTenantCertificateSigningRequestOK) SetPayload(payload *models.CsrElements) {
 	o.Payload = payload
 }
 

operatorapi/volumes.go

@@ -30,6 +30,7 @@ import (
 	"github.com/minio/console/models"
 	"github.com/minio/console/operatorapi/operations"
 	"github.com/minio/console/operatorapi/operations/operator_api"
+	v1 "k8s.io/api/certificates/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -230,43 +231,67 @@ func getPVCEventsResponse(session *models.Principal, params operator_api.GetPVCE
 	return retval, nil
 }
 
-func getTenantCSResponse(session *models.Principal, params operator_api.ListTenantCertificateSigningRequestParams) (*models.CsrElement, *models.Error) {
+func getTenantCSResponse(session *models.Principal, params operator_api.ListTenantCertificateSigningRequestParams) (*models.CsrElements, *models.Error) {
 	ctx, cancel := context.WithCancel(params.HTTPRequest.Context())
 	defer cancel()
 	clientset, err := cluster.K8sClient(session.STSSessionToken)
 	if err != nil {
 		return nil, errors.ErrorWithContext(ctx, err)
 	}
-	csrName := params.Tenant + "-" + params.Namespace + "-csr"
-	csrResult, csrError := clientset.CertificatesV1().CertificateSigningRequests().Get(ctx, csrName, metav1.GetOptions{})
-	if csrError != nil {
-		return nil, errors.ErrorWithContext(ctx, err)
+
+	// Get CSRs by Label "v1.min.io/tenant=" + params.Tenant
+	listByTenantLabel := metav1.ListOptions{LabelSelector: "v1.min.io/tenant=" + params.Tenant}
+	listResult, listError := clientset.CertificatesV1().CertificateSigningRequests().List(ctx, listByTenantLabel)
+	if listError != nil {
+		return nil, errors.ErrorWithContext(ctx, listError)
 	}
-	annotations := []*models.Annotation{}
-	for k, v := range csrResult.ObjectMeta.Annotations {
-		annotations = append(annotations, &models.Annotation{Key: k, Value: v})
+
+	// Get CSR by label "v1.min.io/kes=" + params.Tenant + "-kes"
+	listByKESLabel := metav1.ListOptions{LabelSelector: "v1.min.io/kes=" + params.Tenant + "-kes"}
+	listKESResult, listKESError := clientset.CertificatesV1().CertificateSigningRequests().List(ctx, listByKESLabel)
+	if listKESError != nil {
+		return nil, errors.ErrorWithContext(ctx, listKESError)
 	}
-	var DeletionGracePeriodSeconds int64
-	DeletionGracePeriodSeconds = 0
-	if csrResult.ObjectMeta.DeletionGracePeriodSeconds != nil {
-		DeletionGracePeriodSeconds = *csrResult.ObjectMeta.DeletionGracePeriodSeconds
+
+	var listOfCSRs []v1.CertificateSigningRequest
+	for index := 0; index < len(listResult.Items); index++ {
+		listOfCSRs = append(listOfCSRs, listResult.Items[index])
 	}
-	messages := ""
-	// A CSR.Status can contain multiple Conditions
-	for i := 0; i < len(csrResult.Status.Conditions); i++ {
-		messages = messages + " " + csrResult.Status.Conditions[i].Message
+	for index := 0; index < len(listKESResult.Items); index++ {
+		listOfCSRs = append(listOfCSRs, listKESResult.Items[index])
 	}
-	retval := &models.CsrElement{
-		Name:                       csrResult.ObjectMeta.Name,
-		Annotations:                annotations,
-		DeletionGracePeriodSeconds: DeletionGracePeriodSeconds,
-		GenerateName:               csrResult.ObjectMeta.GenerateName,
-		Generation:                 csrResult.ObjectMeta.Generation,
-		Namespace:                  csrResult.ObjectMeta.Namespace,
-		ResourceVersion:            csrResult.ObjectMeta.ResourceVersion,
-		Status:                     messages,
+
+	var arrayElements []*models.CsrElement
+	for index := 0; index < len(listOfCSRs); index++ {
+		csrResult := listOfCSRs[index]
+		annotations := []*models.Annotation{}
+		for k, v := range csrResult.ObjectMeta.Annotations {
+			annotations = append(annotations, &models.Annotation{Key: k, Value: v})
+		}
+		var DeletionGracePeriodSeconds int64
+		DeletionGracePeriodSeconds = 0
+		if csrResult.ObjectMeta.DeletionGracePeriodSeconds != nil {
+			DeletionGracePeriodSeconds = *csrResult.ObjectMeta.DeletionGracePeriodSeconds
+		}
+		messages := ""
+		// A CSR.Status can contain multiple Conditions
+		for i := 0; i < len(csrResult.Status.Conditions); i++ {
+			messages = messages + " " + csrResult.Status.Conditions[i].Message
+		}
+		retval := &models.CsrElement{
+			Name:                       csrResult.ObjectMeta.Name,
+			Annotations:                annotations,
+			DeletionGracePeriodSeconds: DeletionGracePeriodSeconds,
+			GenerateName:               csrResult.ObjectMeta.GenerateName,
+			Generation:                 csrResult.ObjectMeta.Generation,
+			Namespace:                  csrResult.ObjectMeta.Namespace,
+			ResourceVersion:            csrResult.ObjectMeta.ResourceVersion,
+			Status:                     messages,
+		}
+		arrayElements = append(arrayElements, retval)
 	}
-	return retval, nil
+	result := &models.CsrElements{CsrElement: arrayElements}
+	return result, nil
 }
 
 func getPVCDescribeResponse(session *models.Principal, params operator_api.GetPVCDescribeParams) (*models.DescribePVCWrapper, *models.Error) {

portal-ui/tests/scripts/operator.sh

@@ -151,7 +151,7 @@ function install_tenant() {
 	value=storage-lite
 	echo "Installing lite tenant"
 
-	try kubectl apply -k "${SCRIPT_DIR}/tenant"
+	try kubectl apply -k "${SCRIPT_DIR}/tenant-lite"
 
 	echo "Waiting for the tenant statefulset, this indicates the tenant is being fulfilled"
 	echo $namespace

portal-ui/tests/scripts/tenant-kes-encryption/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: tenant-kms-encrypted
+
+images:
+  - name: minio/operator
+
+resources:
+  - github.com/minio/operator/examples/kustomization/tenant-kes-encryption

swagger-operator.yml

@@ -327,7 +327,7 @@ paths:
         200:
           description: A successful response.
           schema:
-            $ref: "#/definitions/csrElement"
+            $ref: "#/definitions/csrElements"
         default:
           description: Generic error response.
           schema:
@@ -1634,6 +1634,14 @@ definitions:
       password:
         type: string
 
+  csrElements:
+    type: object
+    properties:
+      csrElement:
+        type: array
+        items:
+          $ref: "#/definitions/csrElement"
+
   csrElement:
     type: object
     properties: