Add Operator test (#1591)

This commit is contained in:
Cesar Celis Hernandez
2022-02-17 11:54:16 -05:00
committed by GitHub
parent 9c19c639dd
commit 9f521bbfb4
21 changed files with 9176 additions and 0 deletions

67
.github/workflows/operator.yaml vendored Normal file
View File

@@ -0,0 +1,67 @@
name: "Operator UI"
on:
pull_request:
branches:
- master
push:
branches:
- master
jobs:
permissions:
name: Operator Tests
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [ 1.17.x ]
os: [ ubuntu-latest ]
steps:
- name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }}
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
id: go
- uses: actions/setup-node@v2
with:
node-version: '17'
- name: Check out code into the Go module directory
uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "::set-output name=dir::$(yarn cache dir)"
- uses: actions/cache@v2
id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Make assets
run: |
make assets
- name: Build Console on ${{ matrix.os }}
env:
GO111MODULE: on
GOOS: linux
run: |
make console
- name: Start Console, front-end app
run: |
(./console operator) & (make initialize-operator)
- name: Run TestCafe Tests
uses: DevExpress/testcafe-action@latest
with:
args: '"chrome:headless" portal-ui/tests/operator/ --skip-js-errors -c 3'

View File

@@ -74,6 +74,10 @@ test-integration:
@(GO111MODULE=on go test -race -v github.com/minio/console/integration/...) @(GO111MODULE=on go test -race -v github.com/minio/console/integration/...)
@(docker stop minio) @(docker stop minio)
test-operator:
@(env bash $(PWD)/portal-ui/tests/scripts/operator.sh)
@(docker stop minio)
test-permissions: test-permissions:
@(docker run -v /data1 -v /data2 -v /data3 -v /data4 -d --name minio --rm -p 9000:9000 quay.io/minio/minio:latest server /data{1...4}) @(docker run -v /data1 -v /data2 -v /data3 -v /data4 -d --name minio --rm -p 9000:9000 quay.io/minio/minio:latest server /data{1...4})
@(env bash $(PWD)/portal-ui/tests/scripts/permissions.sh) @(env bash $(PWD)/portal-ui/tests/scripts/permissions.sh)
@@ -85,6 +89,9 @@ test-apply-permissions:
test-start-docker-minio: test-start-docker-minio:
@(docker run -v /data1 -v /data2 -v /data3 -v /data4 -d --name minio --rm -p 9000:9000 quay.io/minio/minio:latest server /data{1...4}) @(docker run -v /data1 -v /data2 -v /data3 -v /data4 -d --name minio --rm -p 9000:9000 quay.io/minio/minio:latest server /data{1...4})
initialize-operator:
@echo "Done initializing operator test"
initialize-permissions: test-start-docker-minio test-apply-permissions initialize-permissions: test-start-docker-minio test-apply-permissions
@echo "Done initializing permissions test" @echo "Done initializing permissions test"

View File

@@ -0,0 +1,28 @@
// This file is part of MinIO Console Server
// Copyright (c) 2022 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
import * as roles from "../utils/roles";
import * as elements from "../utils/elements";
import { diagnosticsElement, supportElement } from "../utils/elements-menu";
fixture("For user with default permissions").page("http://localhost:9090");
test("Login to Operator Web Page", async (t) => {
await t
.navigateTo("http://localhost:9090/login")
.typeText("#jwt","anyrandompasswordwillwork")
.click("button.MuiButton-root")
});

View File

@@ -0,0 +1,9 @@
# four node (two workers) cluster config
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
- role: worker
- role: worker
- role: worker
- role: worker

View File

@@ -0,0 +1,194 @@
# This file is part of MinIO Console Server
# Copyright (c) 2022 MinIO, Inc.
# # This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# # This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
SCRIPT_DIR=$(dirname "$0")
export SCRIPT_DIR
source "${SCRIPT_DIR}/common.sh"
## this enables :dev tag for minio/operator container image.
CI="true"
export CI
## Make sure to install things if not present already
sudo curl -#L "https://dl.k8s.io/release/v1.23.1/bin/linux/amd64/kubectl" -o /usr/local/bin/kubectl
sudo chmod +x /usr/local/bin/kubectl
sudo curl -#L "https://dl.min.io/client/mc/release/linux-amd64/mc" -o /usr/local/bin/mc
sudo chmod +x /usr/local/bin/mc
yell() { echo "$0: $*" >&2; }
die() {
yell "$*"
(kind delete cluster || true ) && exit 111
}
try() { "$@" || die "cannot $*"; }
function setup_kind() {
try kind create cluster --config "${SCRIPT_DIR}/kind-config.yaml"
echo "Kind is ready"
try kubectl get nodes
}
function install_operator() {
echo "Installing Current Operator"
try kubectl apply -k "${SCRIPT_DIR}/resources"
echo "key, value for pod selector in kustomize test"
key=name
value=minio-operator
# Reusing the wait for both, Kustomize and Helm
echo "Waiting for k8s api"
sleep 10
echo "Waiting for Operator Pods to come online (2m timeout)"
try kubectl wait --namespace minio-operator \
--for=condition=ready pod \
--selector $key=$value \
--timeout=120s
echo "start - get data to verify proper image is being used"
kubectl get pods --namespace minio-operator
kubectl describe pods -n minio-operator | grep Image
echo "end - get data to verify proper image is being used"
}
function destroy_kind() {
kind delete cluster
}
function wait_for_resource() {
waitdone=0
totalwait=0
echo "command to wait on:"
command_to_wait="kubectl -n $1 get pods -l $3=$2 --no-headers"
echo $command_to_wait
while true; do
waitdone=$($command_to_wait | wc -l)
if [ "$waitdone" -ne 0 ]; then
echo "Found $waitdone pods"
break
fi
sleep 5
totalwait=$((totalwait + 5))
if [ "$totalwait" -gt 305 ]; then
echo "Unable to get resource after 5 minutes, exiting."
try false
fi
done
}
function check_tenant_status() {
# Check MinIO is accessible
key=v1.min.io/tenant
if [ $# -ge 3 ]; then
echo "Third argument provided, then set key value"
key=$3
else
echo "No third argument provided, using default key"
fi
wait_for_resource $1 $2 $key
echo "Waiting for pods to be ready. (5m timeout)"
if [ $# -ge 4 ]; then
echo "Fourth argument provided, then get secrets from helm"
USER=$(kubectl get secret minio1-secret -o jsonpath="{.data.accesskey}" | base64 --decode)
PASSWORD=$(kubectl get secret minio1-secret -o jsonpath="{.data.secretkey}" | base64 --decode)
else
echo "No fourth argument provided, using default USER and PASSWORD"
USER=$(kubectl -n $1 get secrets $2-env-configuration -o go-template='{{index .data "config.env"|base64decode }}' | grep 'export MINIO_ROOT_USER="' | sed -e 's/export MINIO_ROOT_USER="//g' | sed -e 's/"//g')
PASSWORD=$(kubectl -n $1 get secrets $2-env-configuration -o go-template='{{index .data "config.env"|base64decode }}' | grep 'export MINIO_ROOT_PASSWORD="' | sed -e 's/export MINIO_ROOT_PASSWORD="//g' | sed -e 's/"//g')
fi
try kubectl wait --namespace $1 \
--for=condition=ready pod \
--selector=$key=$2 \
--timeout=300s
echo "Tenant is created successfully, proceeding to validate 'mc admin info minio/'"
if [ "$4" = "helm" ]; then
# File: operator/helm/tenant/values.yaml
# Content: s3.bucketDNS: false
echo "In helm values by default bucketDNS.s3 is disabled, skipping mc validation on helm test"
else
kubectl run admin-mc -i --tty --image minio/mc --command -- bash -c "until (mc alias set minio/ https://minio.$1.svc.cluster.local $USER $PASSWORD); do echo \"...waiting... for 5secs\" && sleep 5; done; mc admin info minio/;"
fi
echo "Done."
}
# Install tenant function is being used by deploy-tenant and check-prometheus
function install_tenant() {
echo "Check if helm will install the Tenant"
if [ "$1" = "helm" ]; then
namespace=default
key=app
value=minio
helm install --namespace tenant-ns \
--create-namespace tenant minio/tenant
else
namespace=tenant-lite
key=v1.min.io/tenant
value=storage-lite
echo "Installing lite tenant"
try kubectl apply -k "${SCRIPT_DIR}/../examples/kustomization/tenant-lite"
fi
echo "Waiting for the tenant statefulset, this indicates the tenant is being fulfilled"
echo $namespace
echo $value
echo $key
wait_for_resource $namespace $value $key
echo "Waiting for tenant pods to come online (5m timeout)"
try kubectl wait --namespace $namespace \
--for=condition=ready pod \
--selector $key=$value \
--timeout=300s
echo "Build passes basic tenant creation"
}
__init__() {
export TIMESTAMP=$(date "+%s")
echo $TIMESTAMP > portal-ui/tests/constants/timestamp.txt
export GOPATH=/tmp/gopath
export PATH=${PATH}:${GOPATH}/bin
destroy_kind
setup_kind
install_operator
install_tenant
kubectl proxy
}
main() {
(yarn start &> /dev/null) & (./console operator &> /dev/null) & (testcafe "chrome:headless" portal-ui/tests/operator/ -q --skip-js-errors -c 3)
}
( __init__ "$@" && main "$@" )

View File

@@ -0,0 +1,27 @@
// This file is part of MinIO Console Server
// Copyright (c) 2021 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package resources
import "embed"
//go:embed *
var fs embed.FS
// GetStaticResources returns the fs with the embedded assets
func GetStaticResources() embed.FS {
return fs
}

View File

@@ -0,0 +1,12 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: minio-operator-binding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: minio-operator-role
subjects:
- kind: ServiceAccount
name: minio-operator
namespace: default

View File

@@ -0,0 +1,130 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: minio-operator-role
rules:
- apiGroups:
- "apiextensions.k8s.io"
resources:
- customresourcedefinitions
verbs:
- get
- update
- apiGroups:
- ""
resources:
- persistentvolumeclaims
verbs:
- get
- update
- list
- apiGroups:
- ""
resources:
- namespaces
verbs:
- get
- watch
- list
- apiGroups:
- ""
resources:
- pods
- services
- events
- configmaps
verbs:
- get
- watch
- create
- list
- delete
- deletecollection
- update
- patch
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- watch
- create
- update
- list
- delete
- deletecollection
- apiGroups:
- apps
resources:
- statefulsets
- deployments
- deployments/finalizers
verbs:
- get
- create
- list
- patch
- watch
- update
- delete
- apiGroups:
- batch
resources:
- jobs
verbs:
- get
- create
- list
- patch
- watch
- update
- delete
- apiGroups:
- "certificates.k8s.io"
resources:
- "certificatesigningrequests"
- "certificatesigningrequests/approval"
- "certificatesigningrequests/status"
verbs:
- update
- create
- get
- delete
- apiGroups:
- certificates.k8s.io
resourceNames:
- kubernetes.io/legacy-unknown
- kubernetes.io/kube-apiserver-client
- kubernetes.io/kubelet-serving
resources:
- signers
verbs:
- approve
- sign
- apiGroups:
- minio.min.io
resources:
- "*"
verbs:
- "*"
- apiGroups:
- min.io
resources:
- "*"
verbs:
- "*"
- apiGroups:
- monitoring.coreos.com
resources:
- prometheuses
verbs:
- '*'
- apiGroups:
- "coordination.k8s.io"
resources:
- leases
verbs:
- get
- update
- create

View File

@@ -0,0 +1,310 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: console-sa
namespace: default
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: console-sa-role
rules:
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- watch
- create
- list
- patch
- update
- deletecollection
- apiGroups:
- ""
resources:
- namespaces
- services
- events
- resourcequotas
- nodes
verbs:
- get
- watch
- create
- list
- patch
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- watch
- create
- list
- patch
- delete
- deletecollection
- apiGroups:
- ""
resources:
- persistentvolumeclaims
verbs:
- deletecollection
- list
- get
- watch
- update
- apiGroups:
- storage.k8s.io
resources:
- storageclasses
verbs:
- get
- watch
- create
- list
- patch
- apiGroups:
- apps
resources:
- statefulsets
- deployments
verbs:
- get
- create
- list
- patch
- watch
- update
- delete
- apiGroups:
- batch
resources:
- jobs
verbs:
- get
- create
- list
- patch
- watch
- update
- delete
- apiGroups:
- certificates.k8s.io
resources:
- certificatesigningrequests
- certificatesigningrequests/approval
- certificatesigningrequests/status
verbs:
- update
- create
- get
- apiGroups:
- minio.min.io
resources:
- '*'
verbs:
- '*'
- apiGroups:
- min.io
resources:
- '*'
verbs:
- '*'
- apiGroups:
- ""
resources:
- persistentvolumes
verbs:
- get
- list
- watch
- create
- delete
- apiGroups:
- ""
resources:
- persistentvolumeclaims
verbs:
- get
- list
- watch
- update
- apiGroups:
- ""
resources:
- events
verbs:
- create
- list
- watch
- update
- patch
- apiGroups:
- snapshot.storage.k8s.io
resources:
- volumesnapshots
verbs:
- get
- list
- apiGroups:
- snapshot.storage.k8s.io
resources:
- volumesnapshotcontents
verbs:
- get
- list
- apiGroups:
- storage.k8s.io
resources:
- csinodes
verbs:
- get
- list
- watch
- apiGroups:
- storage.k8s.io
resources:
- volumeattachments
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- endpoints
verbs:
- get
- list
- watch
- create
- update
- delete
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- get
- list
- watch
- create
- update
- delete
- apiGroups:
- direct.csi.min.io
resources:
- volumes
verbs:
- get
- list
- watch
- create
- update
- delete
- apiGroups:
- apiextensions.k8s.io
resources:
- customresourcedefinitions
verbs:
- get
- list
- watch
- create
- update
- delete
- apiGroups:
- direct.csi.min.io
resources:
- directcsidrives
- directcsivolumes
verbs:
- get
- list
- watch
- create
- update
- delete
- apiGroups:
- ""
resources:
- pod
- pods/log
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: console-sa-binding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: console-sa-role
subjects:
- kind: ServiceAccount
name: console-sa
namespace: default
---
apiVersion: v1
data:
CONSOLE_PORT: "9090"
CONSOLE_TLS_PORT: "9443"
kind: ConfigMap
metadata:
name: console-env
---
apiVersion: v1
kind: Service
metadata:
labels:
name: console
name: console
spec:
ports:
- name: http
port: 9090
- name: https
port: 9443
selector:
app: console
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: console
spec:
replicas: 1
selector:
matchLabels:
app: console
template:
metadata:
labels:
app: console
spec:
containers:
- args:
- server
env:
- name: CONSOLE_OPERATOR_MODE
value: "on"
image: minio/console:v0.14.2
imagePullPolicy: IfNotPresent
name: console
securityContext:
runAsUser: 1000
runAsGroup: 1000
runAsNonRoot: true
ports:
- containerPort: 9090
name: http
- containerPort: 9433
name: https
serviceAccountName: console-sa

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,39 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: minio-operator
namespace: minio-operator
spec:
replicas: 2
selector:
matchLabels:
name: minio-operator
template:
metadata:
labels:
name: minio-operator
spec:
serviceAccountName: minio-operator
containers:
- name: minio-operator
image: minio/operator:v4.4.6
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 200m
memory: 256Mi
ephemeral-storage: 500Mi
securityContext:
runAsUser: 1000
runAsGroup: 1000
runAsNonRoot: true
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: name
operator: In
values:
- minio-operator
topologyKey: kubernetes.io/hostname

View File

@@ -0,0 +1,6 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- crds/minio.min.io_tenants.yaml

View File

@@ -0,0 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: minio-operator

View File

@@ -0,0 +1,5 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: minio-operator
namespace: default

View File

@@ -0,0 +1,15 @@
apiVersion: v1
kind: Service
metadata:
name: operator # Please do not change this value
labels:
name: minio-operator
namespace: minio-operator
spec:
type: ClusterIP
ports:
- port: 4222
name: https
selector:
name: minio-operator
operator: leader

View File

@@ -0,0 +1,17 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: minio-operator
commonAnnotations:
operator.min.io/authors: "MinIO, Inc."
operator.min.io/license: "AGPLv3"
operator.min.io/support: "https://subnet.min.io"
resources:
- base/namespace.yaml
- base/service-account.yaml
- base/cluster-role.yaml
- base/cluster-role-binding.yaml
- base/crds/minio.min.io_tenants.yaml
- base/service.yaml
- base/deployment.yaml
- base/console-ui.yaml

View File

@@ -0,0 +1,16 @@
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: tenants.minio.min.io
spec:
preserveUnknownFields: false
conversion:
strategy: Webhook
webhook:
conversionReviewVersions: [ "v1", "v2" ]
clientConfig:
service:
name: operator
namespace: minio-operator
port: 4222
path: /webhook/v1/crd-conversion

View File

@@ -0,0 +1,9 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../base
patchesStrategicMerge:
- crd-conversion.yaml

View File

@@ -0,0 +1,14 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../base
namespace: tenant-lite
patchesStrategicMerge:
- tenant.yaml
patchesJson6902:
- target:
group: minio.min.io
version: v2
kind: Tenant
name: storage
path: tenantNamePatch.yaml

View File

@@ -0,0 +1,24 @@
apiVersion: minio.min.io/v2
kind: Tenant
metadata:
name: storage
namespace: minio-tenant
spec:
## Specification for MinIO Pool(s) in this Tenant.
pools:
## Servers specifies the number of MinIO Tenant Pods / Servers in this pool.
## For standalone mode, supply 1. For distributed mode, supply 4 or more.
## Note that the operator does not support upgrading from standalone to distributed mode.
- servers: 4
## volumesPerServer specifies the number of volumes attached per MinIO Tenant Pod / Server.
volumesPerServer: 2
## This VolumeClaimTemplate is used across all the volumes provisioned for MinIO Tenant in this Pool.
volumeClaimTemplate:
metadata:
name: data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2Gi

View File

@@ -0,0 +1,3 @@
- op: replace
path: /metadata/name
value: storage-lite