update to v0.3.16

Co-authored-by: Daniel Valdivia <hola@danielvaldivia.com>

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/minio/minio-go/v7 v7.0.5-0.20200807085956-d7db33ea7618
 	github.com/minio/operator v0.0.0-20200806194125-c2ff646f4af1
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
+	github.com/secure-io/sio-go v0.3.1
 	github.com/stretchr/testify v1.6.1
 	github.com/unrolled/secure v1.0.7
 	golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de

k8s/console/base/console-deployment.yaml

@@ -15,7 +15,7 @@ spec:
       serviceAccountName: console-sa
       containers:
         - name: console
-          image: minio/console:v0.3.13
+          image: minio/console:v0.3.16
           imagePullPolicy: "IfNotPresent"
           args:
             - server

k8s/console/base/kustomization.yaml

@@ -8,4 +8,4 @@ resources:
   - console-configmap.yaml
   - console-service.yaml
   - console-deployment.yaml
-  - minio-operator.yaml
+  - https://github.com/minio/operator/?ref=v3.0.10

k8s/operator-console/base/console-cluster-role.yaml

@@ -14,6 +14,7 @@ rules:
       - list
       - patch
       - update
+      - deletecollection
   - apiGroups:
       - ""
     resources:
@@ -22,12 +23,21 @@ rules:
       - services
       - events
       - resourcequotas
+      - nodes
     verbs:
       - get
       - watch
       - create
       - list
       - patch
+  - apiGroups:
+      - ""
+    resources:
+      - persistentvolumeclaims
+    verbs:
+      - deletecollection
+      - list
+      - get
   - apiGroups:
       - "storage.k8s.io"
     resources:

k8s/operator-console/base/console-deployment.yaml

@@ -15,7 +15,7 @@ spec:
       serviceAccountName: console-sa
       containers:
         - name: console
-          image: minio/console:v0.3.13
+          image: minio/console:v0.3.16
           imagePullPolicy: "IfNotPresent"
           env:
             - name: CONSOLE_OPERATOR_MODE

k8s/operator-console/base/kustomization.yaml

@@ -8,4 +8,4 @@ resources:
   - console-configmap.yaml
   - console-service.yaml
   - console-deployment.yaml
-  - minio-operator.yaml
+  - https://github.com/minio/operator/?ref=v3.0.10

models/delete_tenant_request.go

@@ -0,0 +1,60 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+// This file is part of MinIO Console Server
+// Copyright (c) 2020 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+//
+
+package models
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"github.com/go-openapi/strfmt"
+	"github.com/go-openapi/swag"
+)
+
+// DeleteTenantRequest delete tenant request
+//
+// swagger:model deleteTenantRequest
+type DeleteTenantRequest struct {
+
+	// delete pvcs
+	DeletePvcs bool `json:"delete_pvcs,omitempty"`
+}
+
+// Validate validates this delete tenant request
+func (m *DeleteTenantRequest) Validate(formats strfmt.Registry) error {
+	return nil
+}
+
+// MarshalBinary interface implementation
+func (m *DeleteTenantRequest) MarshalBinary() ([]byte, error) {
+	if m == nil {
+		return nil, nil
+	}
+	return swag.WriteJSON(m)
+}
+
+// UnmarshalBinary interface implementation
+func (m *DeleteTenantRequest) UnmarshalBinary(b []byte) error {
+	var res DeleteTenantRequest
+	if err := swag.ReadJSON(b, &res); err != nil {
+		return err
+	}
+	*m = res
+	return nil
+}

models/zone.go

@@ -207,6 +207,9 @@ func (m *Zone) UnmarshalBinary(b []byte) error {
 // swagger:model ZoneVolumeConfiguration
 type ZoneVolumeConfiguration struct {
 
+	// labels
+	Labels map[string]string `json:"labels,omitempty"`
+
 	// size
 	// Required: true
 	Size *int64 `json:"size"`

pkg/auth/token.go

@@ -17,14 +17,17 @@
 package auth
 
 import (
+	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
-	"crypto/rand"
+	"crypto/hmac"
 	"crypto/sha1"
+	"crypto/sha256"
 	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"strings"
@@ -33,6 +36,9 @@ import (
 	"github.com/minio/console/models"
 	"github.com/minio/console/pkg/auth/token"
 	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/secure-io/sio-go/sioutil"
+	"golang.org/x/crypto/chacha20"
+	"golang.org/x/crypto/chacha20poly1305"
 	"golang.org/x/crypto/pbkdf2"
 )
 
@@ -40,6 +46,7 @@ var (
 	errNoAuthToken  = errors.New("session token missing")
 	errReadingToken = errors.New("session token internal data is malformed")
 	errClaimsFormat = errors.New("encrypted session token claims not in the right format")
+	errorGeneric    = errors.New("an error has occurred")
 )
 
 // derivedKey is the key used to encrypt the session token claims, its derived using pbkdf on CONSOLE_PBKDF_PASSPHRASE with CONSOLE_PBKDF_SALT
@@ -102,9 +109,10 @@ func NewEncryptedTokenForClient(credentials *credentials.Value, actions []string
 // returns a base64 encoded ciphertext
 func encryptClaims(accessKeyID, secretAccessKey, sessionToken string, actions []string) (string, error) {
 	payload := []byte(fmt.Sprintf("%s#%s#%s#%s", accessKeyID, secretAccessKey, sessionToken, strings.Join(actions, ",")))
-	ciphertext, err := encrypt(payload)
+	ciphertext, err := encrypt(payload, []byte{})
 	if err != nil {
-		return "", err
+		log.Println(err)
+		return "", errorGeneric
 	}
 	return base64.StdEncoding.EncodeToString(ciphertext), nil
 }
@@ -116,7 +124,7 @@ func decryptClaims(ciphertext string) (*DecryptedClaims, error) {
 		log.Println(err)
 		return nil, errClaimsFormat
 	}
-	plaintext, err := decrypt(decoded)
+	plaintext, err := decrypt(decoded, []byte{})
 	if err != nil {
 		log.Println(err)
 		return nil, errClaimsFormat
@@ -136,37 +144,137 @@ func decryptClaims(ciphertext string) (*DecryptedClaims, error) {
 	}, nil
 }
 
-// Encrypt a blob of data using AEAD (AES-GCM) with a pbkdf2 derived key
+const (
-func encrypt(plaintext []byte) ([]byte, error) {
+	aesGcm   = 0x00
-	block, _ := aes.NewCipher(derivedKey)
+	c20p1305 = 0x01
-	gcm, err := cipher.NewGCM(block)
+)
+
+// Encrypt a blob of data using AEAD scheme, AES-GCM if the executing CPU
+// provides AES hardware support, otherwise will use ChaCha20-Poly1305
+// with a pbkdf2 derived key, this function should be used to encrypt a session
+// or data key provided as plaintext.
+//
+// The returned ciphertext data consists of:
+//    iv | AEAD ID | nonce | encrypted data
+//     32      1        12      ~ len(data)
+func encrypt(plaintext, associatedData []byte) ([]byte, error) {
+	iv, err := sioutil.Random(32) // 32 bit IV
 	if err != nil {
 		return nil, err
 	}
-	nonce := make([]byte, gcm.NonceSize())
+	var algorithm byte
-	if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
+	if sioutil.NativeAES() {
+		algorithm = aesGcm
+	} else {
+		algorithm = c20p1305
+	}
+	var aead cipher.AEAD
+	switch algorithm {
+	case aesGcm:
+		mac := hmac.New(sha256.New, derivedKey)
+		mac.Write(iv)
+		sealingKey := mac.Sum(nil)
+
+		var block cipher.Block
+		block, err = aes.NewCipher(sealingKey)
+		if err != nil {
+			return nil, err
+		}
+		aead, err = cipher.NewGCM(block)
+		if err != nil {
+			return nil, err
+		}
+	case c20p1305:
+		var sealingKey []byte
+		sealingKey, err = chacha20.HChaCha20(derivedKey, iv)
+		if err != nil {
+			return nil, err
+		}
+		aead, err = chacha20poly1305.New(sealingKey)
+		if err != nil {
+			return nil, err
+		}
+	}
+	nonce, err := sioutil.Random(aead.NonceSize())
+	if err != nil {
 		return nil, err
 	}
-	cipherText := gcm.Seal(nonce, nonce, plaintext, nil)
+
-	return cipherText, nil
+	sealedBytes := aead.Seal(nil, nonce, plaintext, associatedData)
+
+	// ciphertext = iv | AEAD ID | nonce | sealed bytes
+
+	var buf bytes.Buffer
+	buf.Write(iv)
+	buf.WriteByte(algorithm)
+	buf.Write(nonce)
+	buf.Write(sealedBytes)
+
+	return buf.Bytes(), nil
 }
 
-// Decrypts a blob of data using AEAD (AES-GCM) with a pbkdf2 derived key
+// Decrypts a blob of data using AEAD scheme AES-GCM if the executing CPU
-func decrypt(data []byte) ([]byte, error) {
+// provides AES hardware support, otherwise will use ChaCha20-Poly1305with
-	block, err := aes.NewCipher(derivedKey)
+// and a pbkdf2 derived key
+func decrypt(ciphertext []byte, associatedData []byte) ([]byte, error) {
+	var (
+		iv        [32]byte
+		algorithm [1]byte
+		nonce     [12]byte // This depends on the AEAD but both used ciphers have the same nonce length.
+	)
+
+	r := bytes.NewReader(ciphertext)
+	if _, err := io.ReadFull(r, iv[:]); err != nil {
+		return nil, err
+	}
+	if _, err := io.ReadFull(r, algorithm[:]); err != nil {
+		return nil, err
+	}
+	if _, err := io.ReadFull(r, nonce[:]); err != nil {
+		return nil, err
+	}
+
+	var aead cipher.AEAD
+	switch algorithm[0] {
+	case aesGcm:
+		mac := hmac.New(sha256.New, derivedKey)
+		mac.Write(iv[:])
+		sealingKey := mac.Sum(nil)
+		block, err := aes.NewCipher(sealingKey[:])
+		if err != nil {
+			return nil, err
+		}
+		aead, err = cipher.NewGCM(block)
+		if err != nil {
+			return nil, err
+		}
+	case c20p1305:
+		sealingKey, err := chacha20.HChaCha20(derivedKey, iv[:])
+		if err != nil {
+			return nil, err
+		}
+		aead, err = chacha20poly1305.New(sealingKey)
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("invalid algorithm: %v", algorithm)
+	}
+
+	if len(nonce) != aead.NonceSize() {
+		return nil, fmt.Errorf("invalid nonce size %d, expected %d", len(nonce), aead.NonceSize())
+	}
+
+	sealedBytes, err := ioutil.ReadAll(r)
 	if err != nil {
 		return nil, err
 	}
-	gcm, err := cipher.NewGCM(block)
+
-	if err != nil {
+	plaintext, err := aead.Open(nil, nonce[:], sealedBytes, associatedData)
-		return nil, err
-	}
-	nonceSize := gcm.NonceSize()
-	nonce, cipherText := data[:nonceSize], data[nonceSize:]
-	plaintext, err := gcm.Open(nil, nonce, cipherText, nil)
 	if err != nil {
 		return nil, err
 	}
+
 	return plaintext, nil
 }
 

pkg/auth/token_test.go

@@ -36,12 +36,12 @@ func TestNewJWTWithClaimsForClient(t *testing.T) {
 	funcAssert := assert.New(t)
 	// Test-1 : NewEncryptedTokenForClient() is generated correctly without errors
 	function := "NewEncryptedTokenForClient()"
-	jwt, err := NewEncryptedTokenForClient(creds, []string{""})
+	token, err := NewEncryptedTokenForClient(creds, []string{""})
-	if err != nil || jwt == "" {
+	if err != nil || token == "" {
 		t.Errorf("Failed on %s:, error occurred: %s", function, err)
 	}
-	// saving jwt for future tests
+	// saving token for future tests
-	goodToken = jwt
+	goodToken = token
 	// Test-2 : NewEncryptedTokenForClient() throws error because of empty credentials
 	if _, err = NewEncryptedTokenForClient(nil, []string{""}); err != nil {
 		funcAssert.Equal("provided credentials are empty", err.Error())

restapi/admin_tenants.go

@@ -110,7 +110,7 @@ func registerTenantHandlers(api *operations.ConsoleAPI) {
 		err := getDeleteTenantResponse(session, params)
 		if err != nil {
 			log.Println(err)
-			return admin_api.NewTenantInfoDefault(500).WithPayload(&models.Error{Code: 500, Message: swag.String("Unable to delete tenant")})
+			return admin_api.NewTenantInfoDefault(500).WithPayload(&models.Error{Code: 500, Message: swag.String(err.Error())})
 		}
 		return admin_api.NewTenantInfoOK()
 
@@ -145,25 +145,58 @@ func registerTenantHandlers(api *operations.ConsoleAPI) {
 	})
 }
 
-// deleteTenantAction performs the actions of deleting a tenant
-func deleteTenantAction(ctx context.Context, operatorClient OperatorClient, nameSpace, instanceName string) error {
-	err := operatorClient.TenantDelete(ctx, nameSpace, instanceName, metav1.DeleteOptions{})
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
 // getDeleteTenantResponse gets the output of deleting a minio instance
 func getDeleteTenantResponse(session *models.Principal, params admin_api.DeleteTenantParams) error {
 	opClientClientSet, err := cluster.OperatorClient(session.SessionToken)
 	if err != nil {
 		return err
 	}
+	// get Kubernetes Client
+	clientset, err := cluster.K8sClient(session.SessionToken)
+	if err != nil {
+		return err
+	}
 	opClient := &operatorClient{
 		client: opClientClientSet,
 	}
-	return deleteTenantAction(context.Background(), opClient, params.Namespace, params.Tenant)
+	deleteTenantPVCs := false
+	if params.Body != nil {
+		deleteTenantPVCs = params.Body.DeletePvcs
+	}
+	return deleteTenantAction(context.Background(), opClient, clientset.CoreV1(), params.Namespace, params.Tenant, deleteTenantPVCs)
+}
+
+// deleteTenantAction performs the actions of deleting a tenant
+//
+// It also adds the option of deleting the tenant's underlying pvcs if deletePvcs set
+func deleteTenantAction(
+	ctx context.Context,
+	operatorClient OperatorClient,
+	clientset v1.CoreV1Interface,
+	namespace, tenantName string,
+	deletePvcs bool) error {
+
+	err := operatorClient.TenantDelete(ctx, namespace, tenantName, metav1.DeleteOptions{})
+	if err != nil {
+		// try to delete pvc even if the tenant doesn't exist anymore but only if deletePvcs is set to true,
+		// else, we return the error
+		if (deletePvcs && !k8sErrors.IsNotFound(err)) || !deletePvcs {
+			return err
+		}
+	}
+
+	if deletePvcs {
+		opts := metav1.ListOptions{
+			LabelSelector: fmt.Sprintf("%s=%s", operator.TenantLabel, tenantName),
+		}
+		err = clientset.PersistentVolumeClaims(namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, opts)
+		if err != nil {
+			return err
+		}
+		// delete all tenant's secrets only if deletePvcs = true
+		return clientset.Secrets(namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, opts)
+	}
+	return nil
 }
 
 func getTenantScheme(mi *operator.Tenant) string {
@@ -365,12 +398,16 @@ func getTenantCreatedResponse(session *models.Principal, params admin_api.Create
 		secretKey = tenantReq.SecretKey
 	}
 
-	secretName := fmt.Sprintf("%s-secret", *tenantReq.Name)
+	tenantName := *tenantReq.Name
+	secretName := fmt.Sprintf("%s-secret", tenantName)
 	imm := true
 
 	instanceSecret := corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: secretName,
+			Labels: map[string]string{
+				operator.TenantLabel: tenantName,
+			},
 		},
 		Immutable: &imm,
 		Data: map[string][]byte{
@@ -399,7 +436,7 @@ func getTenantCreatedResponse(session *models.Principal, params admin_api.Create
 	//Construct a MinIO Instance with everything we are getting from parameters
 	minInst := operator.Tenant{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: *tenantReq.Name,
+			Name: tenantName,
 		},
 		Spec: operator.TenantSpec{
 			Image:     minioImage,
@@ -489,6 +526,9 @@ func getTenantCreatedResponse(session *models.Principal, params admin_api.Create
 		externalTLSCertificateSecret := corev1.Secret{
 			ObjectMeta: metav1.ObjectMeta{
 				Name: externalTLSCertificateSecretName,
+				Labels: map[string]string{
+					operator.TenantLabel: tenantName,
+				},
 			},
 			Type:      corev1.SecretTypeTLS,
 			Immutable: &imm,
@@ -516,13 +556,13 @@ func getTenantCreatedResponse(session *models.Principal, params admin_api.Create
 		})
 		// KES client mTLSCertificates used by MinIO instance, only if autoCert is not enabled
 		if !minInst.Spec.RequestAutoCert {
-			minInst.Spec.ExternalClientCertSecret, err = getTenantExternalClientCertificates(ctx, clientset, ns, tenantReq.Encryption, secretName)
+			minInst.Spec.ExternalClientCertSecret, err = getTenantExternalClientCertificates(ctx, clientset, ns, tenantReq.Encryption, secretName, tenantName)
 			if err != nil {
 				return nil, err
 			}
 		}
 		// KES configuration for Tenant instance
-		minInst.Spec.KES, err = getKESConfiguration(ctx, clientset, ns, tenantReq.Encryption, secretName, minInst.Spec.RequestAutoCert)
+		minInst.Spec.KES, err = getKESConfiguration(ctx, clientset, ns, tenantReq.Encryption, secretName, tenantName, minInst.Spec.RequestAutoCert)
 		if err != nil {
 			return nil, err
 		}
@@ -538,7 +578,7 @@ func getTenantCreatedResponse(session *models.Principal, params admin_api.Create
 	}
 
 	if enableConsole {
-		consoleSelector := fmt.Sprintf("%s-console", *tenantReq.Name)
+		consoleSelector := fmt.Sprintf("%s-console", tenantName)
 		consoleSecretName := fmt.Sprintf("%s-secret", consoleSelector)
 		consoleAccess = RandomCharString(16)
 		consoleSecret = RandomCharString(32)
@@ -546,6 +586,9 @@ func getTenantCreatedResponse(session *models.Principal, params admin_api.Create
 		instanceSecret := corev1.Secret{
 			ObjectMeta: metav1.ObjectMeta{
 				Name: consoleSecretName,
+				Labels: map[string]string{
+					operator.TenantLabel: tenantName,
+				},
 			},
 			Immutable: &imm,
 			Data: map[string][]byte{
@@ -583,7 +626,7 @@ func getTenantCreatedResponse(session *models.Principal, params admin_api.Create
 			return nil, err
 		}
 
-		const consoleVersion = "minio/console:v0.3.13"
+		const consoleVersion = "minio/console:v0.3.16"
 		minInst.Spec.Console = &operator.ConsoleConfiguration{
 			Replicas:      1,
 			Image:         consoleVersion,
@@ -608,6 +651,9 @@ func getTenantCreatedResponse(session *models.Principal, params admin_api.Create
 			consoleExternalTLSCertificateSecret := corev1.Secret{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: consoleExternalTLSCertificateSecretName,
+					Labels: map[string]string{
+						operator.TenantLabel: tenantName,
+					},
 				},
 				Type:      corev1.SecretTypeTLS,
 				Immutable: &imm,
@@ -661,7 +707,7 @@ func getTenantCreatedResponse(session *models.Principal, params admin_api.Create
 
 	if tenantReq.ImagePullSecret != "" {
 		imagePullSecret = tenantReq.ImagePullSecret
-	} else if imagePullSecret, err = setImageRegistry(ctx, *tenantReq.Name, tenantReq.ImageRegistry, clientset.CoreV1(), ns); err != nil {
+	} else if imagePullSecret, err = setImageRegistry(ctx, tenantName, tenantReq.ImageRegistry, clientset.CoreV1(), ns); err != nil {
 		log.Println("error setting image registry secret:", err)
 		return nil, err
 	}
@@ -689,7 +735,7 @@ func getTenantCreatedResponse(session *models.Principal, params admin_api.Create
 
 	// Integratrions
 	if os.Getenv("GKE_INTEGRATION") != "" {
-		err := gkeIntegration(clientset, *tenantReq.Name, ns, session.SessionToken)
+		err := gkeIntegration(clientset, tenantName, ns, session.SessionToken)
 		if err != nil {
 			return nil, err
 		}
@@ -738,6 +784,9 @@ func setImageRegistry(ctx context.Context, tenantName string, req *models.ImageR
 	instanceSecret := corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: pullSecretName,
+			Labels: map[string]string{
+				operator.TenantLabel: tenantName,
+			},
 		},
 		Data: map[string][]byte{
 			corev1.DockerConfigJsonKey: []byte(string(imRegistryJSON)),
@@ -1092,7 +1141,8 @@ func parseTenantZoneRequest(zoneParams *models.Zone, annotations map[string]stri
 	// Pass annotations to the volume
 	vct := &corev1.PersistentVolumeClaim{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: "data",
+			Name:   "data",
+			Labels: zoneParams.VolumeConfiguration.Labels,
 		},
 		Spec: volTemp,
 	}
@@ -1355,7 +1405,7 @@ func parseNodeSelectorTerm(term *corev1.NodeSelectorTerm) *models.NodeSelectorTe
 	return &t
 }
 
-func getTenantExternalClientCertificates(ctx context.Context, clientSet *kubernetes.Clientset, ns string, encryptionCfg *models.EncryptionConfiguration, secretName string) (clientCertificates *operator.LocalCertificateReference, err error) {
+func getTenantExternalClientCertificates(ctx context.Context, clientSet *kubernetes.Clientset, ns string, encryptionCfg *models.EncryptionConfiguration, secretName, tenantName string) (clientCertificates *operator.LocalCertificateReference, err error) {
 	instanceExternalClientCertificateSecretName := fmt.Sprintf("%s-instance-external-client-mtls-certificates", secretName)
 	// If there's an error during this process we delete all KES configuration secrets
 	defer func() {
@@ -1380,6 +1430,9 @@ func getTenantExternalClientCertificates(ctx context.Context, clientSet *kuberne
 	instanceExternalClientCertificateSecret := corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: instanceExternalClientCertificateSecretName,
+			Labels: map[string]string{
+				operator.TenantLabel: tenantName,
+			},
 		},
 		Type:      corev1.SecretTypeTLS,
 		Immutable: &imm,
@@ -1400,7 +1453,7 @@ func getTenantExternalClientCertificates(ctx context.Context, clientSet *kuberne
 	return clientCertificates, nil
 }
 
-func getKESConfiguration(ctx context.Context, clientSet *kubernetes.Clientset, ns string, encryptionCfg *models.EncryptionConfiguration, secretName string, autoCert bool) (kesConfiguration *operator.KESConfig, err error) {
+func getKESConfiguration(ctx context.Context, clientSet *kubernetes.Clientset, ns string, encryptionCfg *models.EncryptionConfiguration, secretName, tenantName string, autoCert bool) (kesConfiguration *operator.KESConfig, err error) {
 	// secrets used by the KES configuration
 	instanceExternalClientCertificateSecretName := fmt.Sprintf("%s-instance-external-client-mtls-certificates", secretName)
 	kesExternalCertificateSecretName := fmt.Sprintf("%s-kes-external-mtls-certificates", secretName)
@@ -1456,6 +1509,9 @@ func getKESConfiguration(ctx context.Context, clientSet *kubernetes.Clientset, n
 		kesExternalCertificateSecret := corev1.Secret{
 			ObjectMeta: metav1.ObjectMeta{
 				Name: kesExternalCertificateSecretName,
+				Labels: map[string]string{
+					operator.TenantLabel: tenantName,
+				},
 			},
 			Type:      corev1.SecretTypeTLS,
 			Immutable: &imm,
@@ -1641,6 +1697,9 @@ func getKESConfiguration(ctx context.Context, clientSet *kubernetes.Clientset, n
 		kesClientCertSecret := corev1.Secret{
 			ObjectMeta: metav1.ObjectMeta{
 				Name: kesClientCertSecretName,
+				Labels: map[string]string{
+					operator.TenantLabel: tenantName,
+				},
 			},
 			Immutable: &imm,
 			Data:      mTLSCertificates,
@@ -1664,6 +1723,9 @@ func getKESConfiguration(ctx context.Context, clientSet *kubernetes.Clientset, n
 	kesConfigurationSecret := corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: kesConfigurationSecretName,
+			Labels: map[string]string{
+				operator.TenantLabel: tenantName,
+			},
 		},
 		Immutable: &imm,
 		Data: map[string][]byte{

restapi/admin_tenants_test.go

@@ -33,9 +33,11 @@ import (
 	operator "github.com/minio/operator/pkg/apis/minio.min.io/v1"
 	v1 "github.com/minio/operator/pkg/apis/minio.min.io/v1"
 	corev1 "k8s.io/api/core/v1"
+	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	types "k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/fake"
 )
@@ -335,12 +337,13 @@ func Test_TenantInfo(t *testing.T) {
 
 func Test_deleteTenantAction(t *testing.T) {
 	opClient := opClientMock{}
-
 	type args struct {
 		ctx              context.Context
 		operatorClient   OperatorClient
 		nameSpace        string
 		tenantName       string
+		deletePvcs       bool
+		objs             []runtime.Object
 		mockTenantDelete func(ctx context.Context, namespace string, tenantName string, options metav1.DeleteOptions) error
 	}
 	tests := []struct {
@@ -355,6 +358,7 @@ func Test_deleteTenantAction(t *testing.T) {
 				operatorClient: opClient,
 				nameSpace:      "default",
 				tenantName:     "minio-tenant",
+				deletePvcs:     false,
 				mockTenantDelete: func(ctx context.Context, namespace string, tenantName string, options metav1.DeleteOptions) error {
 					return nil
 				},
@@ -368,17 +372,155 @@ func Test_deleteTenantAction(t *testing.T) {
 				operatorClient: opClient,
 				nameSpace:      "default",
 				tenantName:     "minio-tenant",
+				deletePvcs:     false,
 				mockTenantDelete: func(ctx context.Context, namespace string, tenantName string, options metav1.DeleteOptions) error {
 					return errors.New("something happened")
 				},
 			},
 			wantErr: true,
 		},
+		{
+			// Delete only PVCs of the defined tenant on the specific namespace
+			name: "Delete PVCs on Tenant Deletion",
+			args: args{
+				ctx:            context.Background(),
+				operatorClient: opClient,
+				nameSpace:      "minio-tenant",
+				tenantName:     "tenant1",
+				deletePvcs:     true,
+				objs: []runtime.Object{
+					&corev1.PersistentVolumeClaim{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "PVC1",
+							Namespace: "minio-tenant",
+							Labels: map[string]string{
+								operator.TenantLabel: "tenant1",
+								operator.ZoneLabel:   "zone-1",
+							},
+						},
+					},
+				},
+				mockTenantDelete: func(ctx context.Context, namespace string, tenantName string, options metav1.DeleteOptions) error {
+					return nil
+				},
+			},
+			wantErr: false,
+		},
+		{
+			// Do not delete underlying pvcs
+			name: "Don't Delete PVCs on Tenant Deletion",
+			args: args{
+				ctx:            context.Background(),
+				operatorClient: opClient,
+				nameSpace:      "minio-tenant",
+				tenantName:     "tenant1",
+				deletePvcs:     false,
+				objs: []runtime.Object{
+					&corev1.PersistentVolumeClaim{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "PVC1",
+							Namespace: "minio-tenant",
+							Labels: map[string]string{
+								operator.TenantLabel: "tenant1",
+								operator.ZoneLabel:   "zone-1",
+							},
+						},
+					},
+				},
+				mockTenantDelete: func(ctx context.Context, namespace string, tenantName string, options metav1.DeleteOptions) error {
+					return nil
+				},
+			},
+			wantErr: false,
+		},
+		{
+			// If error is different than NotFound, PVC deletion should not continue
+			name: "Don't delete pvcs if error Deleting Tenant, return",
+			args: args{
+				ctx:            context.Background(),
+				operatorClient: opClient,
+				nameSpace:      "minio-tenant",
+				tenantName:     "tenant1",
+				deletePvcs:     true,
+				objs: []runtime.Object{
+					&corev1.PersistentVolumeClaim{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "PVC1",
+							Namespace: "minio-tenant",
+							Labels: map[string]string{
+								operator.TenantLabel: "tenant1",
+								operator.ZoneLabel:   "zone-1",
+							},
+						},
+					},
+				},
+				mockTenantDelete: func(ctx context.Context, namespace string, tenantName string, options metav1.DeleteOptions) error {
+					return errors.New("error returned")
+				},
+			},
+			wantErr: true,
+		},
+		{
+			// If error is NotFound while trying to Delete Tenant, PVC deletion should continue
+			name: "Delete pvcs if tenant not found",
+			args: args{
+				ctx:            context.Background(),
+				operatorClient: opClient,
+				nameSpace:      "minio-tenant",
+				tenantName:     "tenant1",
+				deletePvcs:     true,
+				objs: []runtime.Object{
+					&corev1.PersistentVolumeClaim{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "PVC1",
+							Namespace: "minio-tenant",
+							Labels: map[string]string{
+								operator.TenantLabel: "tenant1",
+								operator.ZoneLabel:   "zone-1",
+							},
+						},
+					},
+				},
+				mockTenantDelete: func(ctx context.Context, namespace string, tenantName string, options metav1.DeleteOptions) error {
+					return k8sErrors.NewNotFound(schema.GroupResource{}, "tenant1")
+				},
+			},
+			wantErr: false,
+		},
+		{
+			// If error is NotFound while trying to Delete Tenant and pvcdeletion=false,
+			// error should be returned
+			name: "Don't delete pvcs and return error if tenant not found",
+			args: args{
+				ctx:            context.Background(),
+				operatorClient: opClient,
+				nameSpace:      "minio-tenant",
+				tenantName:     "tenant1",
+				deletePvcs:     false,
+				objs: []runtime.Object{
+					&corev1.PersistentVolumeClaim{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "PVC1",
+							Namespace: "minio-tenant",
+							Labels: map[string]string{
+								operator.TenantLabel: "tenant1",
+								operator.ZoneLabel:   "zone-1",
+							},
+						},
+					},
+				},
+				mockTenantDelete: func(ctx context.Context, namespace string, tenantName string, options metav1.DeleteOptions) error {
+					return k8sErrors.NewNotFound(schema.GroupResource{}, "tenant1")
+				},
+			},
+			wantErr: true,
+		},
 	}
 	for _, tt := range tests {
 		opClientTenantDeleteMock = tt.args.mockTenantDelete
+		kubeClient := fake.NewSimpleClientset(tt.args.objs...)
 		t.Run(tt.name, func(t *testing.T) {
-			if err := deleteTenantAction(tt.args.ctx, tt.args.operatorClient, tt.args.nameSpace, tt.args.tenantName); (err != nil) != tt.wantErr {
+			if err := deleteTenantAction(tt.args.ctx, tt.args.operatorClient, kubeClient.CoreV1(), tt.args.nameSpace, tt.args.tenantName, tt.args.deletePvcs); (err != nil) != tt.wantErr {
 				t.Errorf("deleteTenantAction() error = %v, wantErr %v", err, tt.wantErr)
 			}
 		})
@@ -729,7 +871,7 @@ func Test_UpdateTenantAction(t *testing.T) {
 				},
 				params: admin_api.UpdateTenantParams{
 					Body: &models.UpdateTenantRequest{
-						ConsoleImage: "minio/console:v0.3.13",
+						ConsoleImage: "minio/console:v0.3.16",
 					},
 				},
 			},
@@ -768,7 +910,7 @@ func Test_UpdateTenantAction(t *testing.T) {
 		cnsClient := fake.NewSimpleClientset(tt.objs...)
 		t.Run(tt.name, func(t *testing.T) {
 			if err := updateTenantAction(tt.args.ctx, tt.args.operatorClient, cnsClient.CoreV1(), tt.args.httpCl, tt.args.nameSpace, tt.args.params); (err != nil) != tt.wantErr {
-				t.Errorf("deleteTenantAction() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("updateTenantAction() error = %v, wantErr %v", err, tt.wantErr)
 			}
 		})
 	}

restapi/embedded_spec.go

@@ -1007,7 +1007,7 @@ func init() {
         "tags": [
           "AdminAPI"
         ],
-        "summary": "Delete Tenant",
+        "summary": "Delete tenant and underlying pvcs",
         "operationId": "DeleteTenant",
         "parameters": [
           {
@@ -1021,6 +1021,13 @@ func init() {
             "name": "tenant",
             "in": "path",
             "required": true
+          },
+          {
+            "name": "body",
+            "in": "body",
+            "schema": {
+              "$ref": "#/definitions/deleteTenantRequest"
+            }
           }
         ],
         "responses": {
@@ -2105,6 +2112,14 @@ func init() {
         }
       }
     },
+    "deleteTenantRequest": {
+      "type": "object",
+      "properties": {
+        "delete_pvcs": {
+          "type": "boolean"
+        }
+      }
+    },
     "encryptionConfiguration": {
       "type": "object",
       "properties": {
@@ -3235,6 +3250,12 @@ func init() {
             "size"
           ],
           "properties": {
+            "labels": {
+              "type": "object",
+              "additionalProperties": {
+                "type": "string"
+              }
+            },
             "size": {
               "type": "integer"
             },
@@ -4423,7 +4444,7 @@ func init() {
         "tags": [
           "AdminAPI"
         ],
-        "summary": "Delete Tenant",
+        "summary": "Delete tenant and underlying pvcs",
         "operationId": "DeleteTenant",
         "parameters": [
           {
@@ -4437,6 +4458,13 @@ func init() {
             "name": "tenant",
             "in": "path",
             "required": true
+          },
+          {
+            "name": "body",
+            "in": "body",
+            "schema": {
+              "$ref": "#/definitions/deleteTenantRequest"
+            }
           }
         ],
         "responses": {
@@ -5699,6 +5727,12 @@ func init() {
         "size"
       ],
       "properties": {
+        "labels": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "string"
+          }
+        },
         "size": {
           "type": "integer"
         },
@@ -6032,6 +6066,14 @@ func init() {
         }
       }
     },
+    "deleteTenantRequest": {
+      "type": "object",
+      "properties": {
+        "delete_pvcs": {
+          "type": "boolean"
+        }
+      }
+    },
     "encryptionConfiguration": {
       "type": "object",
       "properties": {
@@ -7096,6 +7138,12 @@ func init() {
             "size"
           ],
           "properties": {
+            "labels": {
+              "type": "object",
+              "additionalProperties": {
+                "type": "string"
+              }
+            },
             "size": {
               "type": "integer"
             },

restapi/operations/admin_api/delete_tenant.go

@@ -50,7 +50,7 @@ func NewDeleteTenant(ctx *middleware.Context, handler DeleteTenantHandler) *Dele
 
 /*DeleteTenant swagger:route DELETE /namespaces/{namespace}/tenants/{tenant} AdminAPI deleteTenant
 
-Delete Tenant
+Delete tenant and underlying pvcs
 
 */
 type DeleteTenant struct {

restapi/operations/admin_api/delete_tenant_parameters.go

@@ -26,8 +26,11 @@ import (
 	"net/http"
 
 	"github.com/go-openapi/errors"
+	"github.com/go-openapi/runtime"
 	"github.com/go-openapi/runtime/middleware"
 	"github.com/go-openapi/strfmt"
+
+	"github.com/minio/console/models"
 )
 
 // NewDeleteTenantParams creates a new DeleteTenantParams object
@@ -46,6 +49,10 @@ type DeleteTenantParams struct {
 	// HTTP Request Object
 	HTTPRequest *http.Request `json:"-"`
 
+	/*
+	  In: body
+	*/
+	Body *models.DeleteTenantRequest
 	/*
 	  Required: true
 	  In: path
@@ -67,6 +74,22 @@ func (o *DeleteTenantParams) BindRequest(r *http.Request, route *middleware.Matc
 
 	o.HTTPRequest = r
 
+	if runtime.HasBody(r) {
+		defer r.Body.Close()
+		var body models.DeleteTenantRequest
+		if err := route.Consumer.Consume(r.Body, &body); err != nil {
+			res = append(res, errors.NewParseError("body", "body", "", err))
+		} else {
+			// validate body object
+			if err := body.Validate(route.Formats); err != nil {
+				res = append(res, err)
+			}
+
+			if len(res) == 0 {
+				o.Body = &body
+			}
+		}
+	}
 	rNamespace, rhkNamespace, _ := route.Params.GetOK("namespace")
 	if err := o.bindNamespace(rNamespace, rhkNamespace, route.Formats); err != nil {
 		res = append(res, err)

swagger.yml

@@ -1069,7 +1069,7 @@ paths:
       tags:
         - AdminAPI
     delete:
-      summary: Delete Tenant
+      summary: Delete tenant and underlying pvcs
       operationId: DeleteTenant
       parameters:
         - name: namespace
@@ -1080,6 +1080,11 @@ paths:
           in: path
           required: true
           type: string
+        - name: body
+          in: body
+          required: false
+          schema:
+            $ref: "#/definitions/deleteTenantRequest"
       responses:
         204:
           description: A successful response.
@@ -2083,6 +2088,10 @@ definitions:
             type: integer
           storage_class_name:
             type: string
+          labels:
+            type: object
+            additionalProperties:
+              type: string
       resources:
         $ref: "#/definitions/zoneResources"
       node_selector:
@@ -2516,3 +2525,9 @@ definitions:
       used:
         type: integer
         format: int64
+  
+  deleteTenantRequest:
+    type: object
+    properties:
+      delete_pvcs:
+        type: boolean