Add Operator test (#1591)

portal-ui/tests/scripts/kind-config.yaml

@@ -0,0 +1,9 @@
+# four node (two workers) cluster config
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+  - role: worker
+  - role: worker
+  - role: worker
+  - role: worker

portal-ui/tests/scripts/operator.sh

@@ -0,0 +1,194 @@
+# This file is part of MinIO Console Server
+# Copyright (c) 2022 MinIO, Inc.
+# # This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# # This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+# # You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+SCRIPT_DIR=$(dirname "$0")
+export SCRIPT_DIR
+source "${SCRIPT_DIR}/common.sh"
+
+## this enables :dev tag for minio/operator container image.
+CI="true"
+export CI
+
+## Make sure to install things if not present already
+sudo curl -#L "https://dl.k8s.io/release/v1.23.1/bin/linux/amd64/kubectl" -o /usr/local/bin/kubectl
+sudo chmod +x /usr/local/bin/kubectl
+
+sudo curl -#L "https://dl.min.io/client/mc/release/linux-amd64/mc" -o /usr/local/bin/mc
+sudo chmod +x /usr/local/bin/mc
+
+yell() { echo "$0: $*" >&2; }
+
+die() {
+	yell "$*"
+	(kind delete cluster || true ) && exit 111
+}
+
+try() { "$@" || die "cannot $*"; }
+
+function setup_kind() {
+	try kind create cluster --config "${SCRIPT_DIR}/kind-config.yaml"
+	echo "Kind is ready"
+	try kubectl get nodes
+}
+
+function install_operator() {
+
+	echo "Installing Current Operator"
+	try kubectl apply -k "${SCRIPT_DIR}/resources"
+
+	echo "key, value for pod selector in kustomize test"
+	key=name
+	value=minio-operator
+
+	# Reusing the wait for both, Kustomize and Helm
+	echo "Waiting for k8s api"
+	sleep 10
+
+	echo "Waiting for Operator Pods to come online (2m timeout)"
+	try kubectl wait --namespace minio-operator \
+	--for=condition=ready pod \
+	--selector $key=$value \
+	--timeout=120s
+
+	echo "start - get data to verify proper image is being used"
+	kubectl get pods --namespace minio-operator
+	kubectl describe pods -n minio-operator | grep Image
+	echo "end - get data to verify proper image is being used"
+}
+
+function destroy_kind() {
+	kind delete cluster
+}
+
+function wait_for_resource() {
+	waitdone=0
+	totalwait=0
+	echo "command to wait on:"
+	command_to_wait="kubectl -n $1 get pods -l $3=$2 --no-headers"
+	echo $command_to_wait
+
+	while true; do
+	waitdone=$($command_to_wait | wc -l)
+	if [ "$waitdone" -ne 0 ]; then
+		echo "Found $waitdone pods"
+			break
+	fi
+	sleep 5
+	totalwait=$((totalwait + 5))
+	if [ "$totalwait" -gt 305 ]; then
+			echo "Unable to get resource after 5 minutes, exiting."
+			try false
+	fi
+	done
+}
+
+function check_tenant_status() {
+	# Check MinIO is accessible
+	key=v1.min.io/tenant
+	if [ $# -ge 3 ]; then
+		echo "Third argument provided, then set key value"
+		key=$3
+	else
+		echo "No third argument provided, using default key"
+	fi
+
+	wait_for_resource $1 $2 $key
+
+	echo "Waiting for pods to be ready. (5m timeout)"
+
+	if [ $# -ge 4 ]; then
+		echo "Fourth argument provided, then get secrets from helm"
+		USER=$(kubectl get secret minio1-secret -o jsonpath="{.data.accesskey}" | base64 --decode)
+		PASSWORD=$(kubectl get secret minio1-secret -o jsonpath="{.data.secretkey}" | base64 --decode)
+	else
+		echo "No fourth argument provided, using default USER and PASSWORD"
+		USER=$(kubectl -n $1 get secrets $2-env-configuration -o go-template='{{index .data "config.env"|base64decode }}' | grep 'export MINIO_ROOT_USER="' | sed -e 's/export MINIO_ROOT_USER="//g' | sed -e 's/"//g')
+		PASSWORD=$(kubectl -n $1 get secrets $2-env-configuration -o go-template='{{index .data "config.env"|base64decode }}' | grep 'export MINIO_ROOT_PASSWORD="' | sed -e 's/export MINIO_ROOT_PASSWORD="//g' | sed -e 's/"//g')
+	fi
+
+	try kubectl wait --namespace $1 \
+		--for=condition=ready pod \
+		--selector=$key=$2 \
+		--timeout=300s
+
+	echo "Tenant is created successfully, proceeding to validate 'mc admin info minio/'"
+
+	if [ "$4" = "helm" ]; then
+		# File: operator/helm/tenant/values.yaml
+		# Content: s3.bucketDNS: false
+		echo "In helm values by default bucketDNS.s3 is disabled, skipping mc validation on helm test"
+	else
+		kubectl run admin-mc -i --tty --image minio/mc --command -- bash -c "until (mc alias set minio/ https://minio.$1.svc.cluster.local $USER $PASSWORD); do echo \"...waiting... for 5secs\" && sleep 5; done; mc admin info minio/;"
+	fi
+
+	echo "Done."
+}
+
+# Install tenant function is being used by deploy-tenant and check-prometheus
+function install_tenant() {
+
+	echo "Check if helm will install the Tenant"
+	if [ "$1" = "helm" ]; then
+		namespace=default
+		key=app
+		value=minio
+		helm install --namespace tenant-ns \
+			 --create-namespace tenant minio/tenant
+	else
+		namespace=tenant-lite
+		key=v1.min.io/tenant
+		value=storage-lite
+		echo "Installing lite tenant"
+
+		try kubectl apply -k "${SCRIPT_DIR}/../examples/kustomization/tenant-lite"
+	fi
+
+	echo "Waiting for the tenant statefulset, this indicates the tenant is being fulfilled"
+	echo $namespace
+	echo $value
+	echo $key
+	wait_for_resource $namespace $value $key
+
+	echo "Waiting for tenant pods to come online (5m timeout)"
+	try kubectl wait --namespace $namespace \
+	--for=condition=ready pod \
+	--selector $key=$value \
+	--timeout=300s
+
+	echo "Build passes basic tenant creation"
+
+}
+
+__init__() {
+  export TIMESTAMP=$(date "+%s")
+  echo $TIMESTAMP > portal-ui/tests/constants/timestamp.txt
+  export GOPATH=/tmp/gopath
+  export PATH=${PATH}:${GOPATH}/bin
+
+  destroy_kind
+
+  setup_kind
+
+  install_operator
+
+  install_tenant
+
+  kubectl proxy
+
+}
+
+main() {
+  (yarn start &> /dev/null) & (./console operator &> /dev/null) & (testcafe "chrome:headless" portal-ui/tests/operator/ -q --skip-js-errors -c 3)
+}
+
+( __init__ "$@" && main "$@" )

portal-ui/tests/scripts/resources/assets.go

@@ -0,0 +1,27 @@
+// This file is part of MinIO Console Server
+// Copyright (c) 2021 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import "embed"
+
+//go:embed *
+var fs embed.FS
+
+// GetStaticResources returns the fs with the embedded assets
+func GetStaticResources() embed.FS {
+	return fs
+}

portal-ui/tests/scripts/resources/base/cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: minio-operator-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: minio-operator-role
+subjects:
+  - kind: ServiceAccount
+    name: minio-operator
+    namespace: default

portal-ui/tests/scripts/resources/base/cluster-role.yaml

@@ -0,0 +1,130 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: minio-operator-role
+rules:
+  - apiGroups:
+      - "apiextensions.k8s.io"
+    resources:
+      - customresourcedefinitions
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - persistentvolumeclaims
+    verbs:
+      - get
+      - update
+      - list
+  - apiGroups:
+      - ""
+    resources:
+      - namespaces
+    verbs:
+      - get
+      - watch
+      - list
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - services
+      - events
+      - configmaps
+    verbs:
+      - get
+      - watch
+      - create
+      - list
+      - delete
+      - deletecollection
+      - update
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - watch
+      - create
+      - update
+      - list
+      - delete
+      - deletecollection
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+      - deployments
+      - deployments/finalizers
+    verbs:
+      - get
+      - create
+      - list
+      - patch
+      - watch
+      - update
+      - delete
+  - apiGroups:
+      - batch
+    resources:
+      - jobs
+    verbs:
+      - get
+      - create
+      - list
+      - patch
+      - watch
+      - update
+      - delete
+  - apiGroups:
+      - "certificates.k8s.io"
+    resources:
+      - "certificatesigningrequests"
+      - "certificatesigningrequests/approval"
+      - "certificatesigningrequests/status"
+    verbs:
+      - update
+      - create
+      - get
+      - delete
+  - apiGroups:
+      - certificates.k8s.io
+    resourceNames:
+      - kubernetes.io/legacy-unknown
+      - kubernetes.io/kube-apiserver-client
+      - kubernetes.io/kubelet-serving
+    resources:
+      - signers
+    verbs:
+      - approve
+      - sign
+  - apiGroups:
+      - minio.min.io
+    resources:
+      - "*"
+    verbs:
+      - "*"
+  - apiGroups:
+      - min.io
+    resources:
+      - "*"
+    verbs:
+      - "*"
+  - apiGroups:
+      - monitoring.coreos.com
+    resources:
+      - prometheuses
+    verbs:
+      - '*'
+  - apiGroups:
+      - "coordination.k8s.io"
+    resources:
+      - leases
+    verbs:
+      - get
+      - update
+      - create

portal-ui/tests/scripts/resources/base/console-ui.yaml

@@ -0,0 +1,310 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: console-sa
+  namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: console-sa-role
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - watch
+      - create
+      - list
+      - patch
+      - update
+      - deletecollection
+  - apiGroups:
+      - ""
+    resources:
+      - namespaces
+      - services
+      - events
+      - resourcequotas
+      - nodes
+    verbs:
+      - get
+      - watch
+      - create
+      - list
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - watch
+      - create
+      - list
+      - patch
+      - delete
+      - deletecollection
+  - apiGroups:
+      - ""
+    resources:
+      - persistentvolumeclaims
+    verbs:
+      - deletecollection
+      - list
+      - get
+      - watch
+      - update
+  - apiGroups:
+      - storage.k8s.io
+    resources:
+      - storageclasses
+    verbs:
+      - get
+      - watch
+      - create
+      - list
+      - patch
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+      - deployments
+    verbs:
+      - get
+      - create
+      - list
+      - patch
+      - watch
+      - update
+      - delete
+  - apiGroups:
+      - batch
+    resources:
+      - jobs
+    verbs:
+      - get
+      - create
+      - list
+      - patch
+      - watch
+      - update
+      - delete
+  - apiGroups:
+      - certificates.k8s.io
+    resources:
+      - certificatesigningrequests
+      - certificatesigningrequests/approval
+      - certificatesigningrequests/status
+    verbs:
+      - update
+      - create
+      - get
+  - apiGroups:
+      - minio.min.io
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - min.io
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - persistentvolumes
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - delete
+  - apiGroups:
+      - ""
+    resources:
+      - persistentvolumeclaims
+    verbs:
+      - get
+      - list
+      - watch
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - events
+    verbs:
+      - create
+      - list
+      - watch
+      - update
+      - patch
+  - apiGroups:
+      - snapshot.storage.k8s.io
+    resources:
+      - volumesnapshots
+    verbs:
+      - get
+      - list
+  - apiGroups:
+      - snapshot.storage.k8s.io
+    resources:
+      - volumesnapshotcontents
+    verbs:
+      - get
+      - list
+  - apiGroups:
+      - storage.k8s.io
+    resources:
+      - csinodes
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - storage.k8s.io
+    resources:
+      - volumeattachments
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - delete
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - delete
+  - apiGroups:
+      - direct.csi.min.io
+    resources:
+      - volumes
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - delete
+  - apiGroups:
+      - apiextensions.k8s.io
+    resources:
+      - customresourcedefinitions
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - delete
+  - apiGroups:
+      - direct.csi.min.io
+    resources:
+      - directcsidrives
+      - directcsivolumes
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - delete
+  - apiGroups:
+      - ""
+    resources:
+      - pod
+      - pods/log
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: console-sa-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: console-sa-role
+subjects:
+  - kind: ServiceAccount
+    name: console-sa
+    namespace: default
+---
+apiVersion: v1
+data:
+  CONSOLE_PORT: "9090"
+  CONSOLE_TLS_PORT: "9443"
+kind: ConfigMap
+metadata:
+  name: console-env
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    name: console
+  name: console
+spec:
+  ports:
+    - name: http
+      port: 9090
+    - name: https
+      port: 9443
+  selector:
+    app: console
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: console
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: console
+  template:
+    metadata:
+      labels:
+        app: console
+    spec:
+      containers:
+        - args:
+            - server
+          env:
+            - name: CONSOLE_OPERATOR_MODE
+              value: "on"
+          image: minio/console:v0.14.2
+          imagePullPolicy: IfNotPresent
+          name: console
+          securityContext:
+            runAsUser: 1000
+            runAsGroup: 1000
+            runAsNonRoot: true
+          ports:
+            - containerPort: 9090
+              name: http
+            - containerPort: 9433
+              name: https
+      serviceAccountName: console-sa

portal-ui/tests/scripts/resources/base/deployment.yaml

@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: minio-operator
+  namespace: minio-operator
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      name: minio-operator
+  template:
+    metadata:
+      labels:
+        name: minio-operator
+    spec:
+      serviceAccountName: minio-operator
+      containers:
+        - name: minio-operator
+          image: minio/operator:v4.4.6
+          imagePullPolicy: IfNotPresent
+          resources:
+            requests:
+              cpu: 200m
+              memory: 256Mi
+              ephemeral-storage: 500Mi
+          securityContext:
+            runAsUser: 1000
+            runAsGroup: 1000
+            runAsNonRoot: true
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: name
+                    operator: In
+                    values:
+                      - minio-operator
+              topologyKey: kubernetes.io/hostname

portal-ui/tests/scripts/resources/base/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+
+resources:
+  - crds/minio.min.io_tenants.yaml

portal-ui/tests/scripts/resources/base/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: minio-operator

portal-ui/tests/scripts/resources/base/service-account.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: minio-operator
+  namespace: default

portal-ui/tests/scripts/resources/base/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: operator # Please do not change this value
+  labels:
+    name: minio-operator
+  namespace: minio-operator
+spec:
+  type: ClusterIP
+  ports:
+    - port: 4222
+      name: https
+  selector:
+    name: minio-operator
+    operator: leader

portal-ui/tests/scripts/resources/kustomization.yaml

@@ -0,0 +1,17 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: minio-operator
+commonAnnotations:
+  operator.min.io/authors: "MinIO, Inc."
+  operator.min.io/license: "AGPLv3"
+  operator.min.io/support: "https://subnet.min.io"
+
+resources:
+  - base/namespace.yaml
+  - base/service-account.yaml
+  - base/cluster-role.yaml
+  - base/cluster-role-binding.yaml
+  - base/crds/minio.min.io_tenants.yaml
+  - base/service.yaml
+  - base/deployment.yaml
+  - base/console-ui.yaml

portal-ui/tests/scripts/resources/patch-crd/crd-conversion.yaml

@@ -0,0 +1,16 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: tenants.minio.min.io
+spec:
+  preserveUnknownFields: false
+  conversion:
+    strategy: Webhook
+    webhook:
+      conversionReviewVersions: [ "v1", "v2" ]
+      clientConfig:
+        service:
+          name: operator
+          namespace: minio-operator
+          port: 4222
+          path: /webhook/v1/crd-conversion

portal-ui/tests/scripts/resources/patch-crd/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+
+resources:
+  - ../base
+
+patchesStrategicMerge:
+  - crd-conversion.yaml

portal-ui/tests/scripts/tenant-lite/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../base
+namespace: tenant-lite
+patchesStrategicMerge:
+  - tenant.yaml
+patchesJson6902:
+  - target:
+      group: minio.min.io
+      version: v2
+      kind: Tenant
+      name: storage
+    path: tenantNamePatch.yaml

portal-ui/tests/scripts/tenant-lite/tenant.yaml

@@ -0,0 +1,24 @@
+apiVersion: minio.min.io/v2
+kind: Tenant
+metadata:
+  name: storage
+  namespace: minio-tenant
+spec:
+  ## Specification for MinIO Pool(s) in this Tenant.
+  pools:
+    ## Servers specifies the number of MinIO Tenant Pods / Servers in this pool.
+    ## For standalone mode, supply 1. For distributed mode, supply 4 or more.
+    ## Note that the operator does not support upgrading from standalone to distributed mode.
+    - servers: 4
+      ## volumesPerServer specifies the number of volumes attached per MinIO Tenant Pod / Server.
+      volumesPerServer: 2
+      ## This VolumeClaimTemplate is used across all the volumes provisioned for MinIO Tenant in this Pool.
+      volumeClaimTemplate:
+        metadata:
+          name: data
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 2Gi

portal-ui/tests/scripts/tenant-lite/tenantNamePatch.yaml

@@ -0,0 +1,3 @@
+- op: replace
+  path: /metadata/name
+  value: storage-lite