Add list of tenants integration test (#1722)

.github/workflows/common.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+# Copyright (C) 2022, MinIO, Inc.
+#
+# This code is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3,
+# as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License, version 3,
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+
+yell() { echo "$0: $*" >&2; }
+
+die() {
+  yell "$*"
+  (kind delete cluster || true ) && exit 111
+}
+
+try() { "$@" || die "cannot $*"; }
+
+function setup_kind() {
+	# TODO once feature is added: https://github.com/kubernetes-sigs/kind/issues/1300
+	echo "kind: Cluster" > kind-config.yaml
+	echo "apiVersion: kind.x-k8s.io/v1alpha4" >> kind-config.yaml
+	echo "nodes:" >> kind-config.yaml
+	echo "  - role: control-plane" >> kind-config.yaml
+	echo "  - role: worker" >> kind-config.yaml
+	echo "  - role: worker" >> kind-config.yaml
+	echo "  - role: worker" >> kind-config.yaml
+	echo "  - role: worker" >> kind-config.yaml
+	try kind create cluster --config kind-config.yaml
+	echo "Kind is ready"
+	try kubectl get nodes
+}
+
+function install_operator() {
+    echo "Installing Current Operator"
+
+    # TODO: Compile the current branch and create an overlay to use that image version
+    try kubectl apply -k "${SCRIPT_DIR}/../../portal-ui/tests/scripts/resources"
+
+    echo "Waiting for k8s api"
+    sleep 10
+    echo "Waiting for Operator Pods to come online (2m timeout)"
+
+    try kubectl wait --namespace minio-operator \
+	--for=condition=ready pod \
+	--selector=name=minio-operator \
+	--timeout=120s
+}
+
+function destroy_kind() {
+    kind delete cluster
+}
+
+function check_tenant_status() {
+    # Check MinIO is accessible
+
+    waitdone=0
+    totalwait=0
+    while true; do
+	waitdone=$(kubectl -n $1 get pods -l v1.min.io/tenant=$2 --no-headers | wc -l)
+	if [ "$waitdone" -ne 0 ]; then
+	    echo "Found $waitdone pods"
+	    break
+	fi
+	sleep 5
+	totalwait=$((totalwait + 5))
+	if [ "$totalwait" -gt 305 ]; then
+	    echo "Unable to create tenant after 5 minutes, exiting."
+	    try false
+	fi
+    done
+
+    echo "Waiting for pods to be ready. (5m timeout)"
+
+    USER=$(kubectl -n $1 get secrets $2-env-configuration -o go-template='{{index .data "config.env"|base64decode }}' | grep 'export MINIO_ROOT_USER="' | sed -e 's/export MINIO_ROOT_USER="//g' | sed -e 's/"//g')
+    PASSWORD=$(kubectl -n $1 get secrets $2-env-configuration -o go-template='{{index .data "config.env"|base64decode }}' | grep 'export MINIO_ROOT_PASSWORD="' | sed -e 's/export MINIO_ROOT_PASSWORD="//g' | sed -e 's/"//g')
+
+    try kubectl wait --namespace $1 \
+        --for=condition=ready pod \
+        --selector=v1.min.io/tenant=$2 \
+        --timeout=300s
+
+    echo "Tenant is created successfully, proceeding to validate 'mc admin info minio/'"
+
+    kubectl run admin-mc -i --tty --image minio/mc --command -- bash -c "until (mc alias set minio/ https://minio.$1.svc.cluster.local $USER $PASSWORD); do echo \"...waiting... for 5secs\" && sleep 5; done; mc admin info minio/;"
+
+    echo "Done."
+}

.github/workflows/deploy-tenant.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# Copyright (C) 2022, MinIO, Inc.
+#
+# This code is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3,
+# as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License, version 3,
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+
+# This script requires: kubectl, kind
+
+SCRIPT_DIR=$(dirname "$0")
+export SCRIPT_DIR
+
+source "${SCRIPT_DIR}/common.sh"
+
+
+
+function install_tenant() {
+    echo "Installing lite tenant"
+
+	try kubectl apply -k "${SCRIPT_DIR}/../../portal-ui/tests/scripts/tenant"
+
+    echo "Waiting for the tenant statefulset, this indicates the tenant is being fulfilled"
+    waitdone=0
+    totalwait=0
+    while true; do
+	waitdone=$(kubectl -n tenant-lite get pods -l v1.min.io/tenant=storage-lite --no-headers | wc -l)
+	if [ "$waitdone" -ne 0 ]; then
+	    echo "Found $waitdone pods"
+	    break
+	fi
+	sleep 5
+	totalwait=$((totalwait + 5))
+	if [ "$totalwait" -gt 300 ]; then
+	    echo "Tenant never created statefulset after 5 minutes"
+	    try false
+	fi
+    done
+
+    echo "Waiting for tenant pods to come online (5m timeout)"
+    try kubectl wait --namespace tenant-lite \
+	--for=condition=ready pod \
+	--selector="v1.min.io/tenant=storage-lite" \
+	--timeout=300s
+
+    echo "Build passes basic tenant creation"
+}
+
+
+function main() {
+    destroy_kind
+
+    setup_kind
+
+    install_operator
+
+    install_tenant
+
+    check_tenant_status tenant-lite storage-lite
+
+    kubectl -n minio-operator port-forward svc/console 9090 &
+}
+
+main "$@"

.github/workflows/jobs.yaml

@@ -15,6 +15,56 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+
+  operator-api-tests:
+
+    name: Operator API Tests
+    needs:
+      - lint-job
+      - no-warnings-and-make-assets
+      - reuse-golang-dependencies
+      - vulnerable-dependencies-checks
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        go-version: [ 1.17.x ]
+
+    steps:
+
+      - name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }}
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+        id: go
+
+      - uses: actions/checkout@v2
+
+      - uses: actions/cache@v2
+        name: Go Mod Cache
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ github.run_id }}
+
+      - name: Operator API Tests
+        run: |
+          curl -sLO "https://dl.k8s.io/release/v1.23.1/bin/linux/amd64/kubectl" -o kubectl
+          chmod +x kubectl
+          mv kubectl /usr/local/bin
+          "${GITHUB_WORKSPACE}/.github/workflows/deploy-tenant.sh"
+          echo "start ---> make test-operator-integration";
+          make test-operator-integration;
+
+      - uses: actions/cache@v2
+        id: coverage-cache-operator
+        name: Coverage Cache Operator
+        with:
+          path: |
+            ./operator-integration/coverage/
+          key: ${{ runner.os }}-coverage-2-operator-${{ github.run_id }}
+
   lint-job:
     name: Checking Lint
     runs-on: ${{ matrix.os }}
@@ -702,6 +752,8 @@ jobs:
     needs:
       - integration-tests
       - test-restapi-on-go
+      - operator-api-tests
+      - test-pkg-on-go
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -736,6 +788,14 @@ jobs:
             ./integration/coverage/
           key: ${{ runner.os }}-coverage-2-${{ github.run_id }}
 
+      - uses: actions/cache@v2
+        id: coverage-cache-operator
+        name: Coverage Cache Operator
+        with:
+          path: |
+            ./operator-integration/coverage/
+          key: ${{ runner.os }}-coverage-2-operator-${{ github.run_id }}
+
       - uses: actions/cache@v2
         id: coverage-cache-restapi
         name: Coverage Cache RestAPI
@@ -763,14 +823,14 @@ jobs:
           echo "go build gocoverage.go"
           go build gocovmerge.go
           echo "put together the outs for final coverage resolution"
-          ./gocovmerge ../integration/coverage/system.out ../restapi/coverage/coverage.out ../pkg/coverage/coverage-pkg.out > all.out
+          ./gocovmerge ../integration/coverage/system.out ../restapi/coverage/coverage.out ../pkg/coverage/coverage-pkg.out ../operator-integration/coverage/operator-api.out > all.out
           echo "grep to obtain the result"
           go tool cover -func=all.out | grep total > tmp2
           result=`cat tmp2 | awk 'END {print $3}'`
           result=${result%\%}
           echo "result:"
           echo $result
-          threshold=50.4
+          threshold=50.5
           if (( $(echo "$result >= $threshold" |bc -l) )); then
             echo "It is equal or greater than threshold, passed!"
           else

Makefile

@@ -80,6 +80,11 @@ test-integration:
 	@(docker stop minio)
 	@(docker network rm mynet123)
 
+test-operator-integration:
+	@(echo "Start cd operator-integration && go test:")
+	@(pwd)
+	@(cd operator-integration && go test -coverpkg=../restapi -c -tags testrunmain . && mkdir -p coverage && ./operator-integration.test -test.run "^Test*" -test.coverprofile=coverage/operator-api.out)
+
 test-operator:
 	@(env bash $(PWD)/portal-ui/tests/scripts/operator.sh)
 	@(docker stop minio)

operator-integration/tenant_test.go

@@ -0,0 +1,269 @@
+// This file is part of MinIO Console Server
+// Copyright (c) 2021 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package operatorintegration
+
+import (
+	"bytes"
+	b64 "encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/go-openapi/loads"
+	"github.com/minio/console/models"
+	"github.com/minio/console/restapi"
+	"github.com/minio/console/restapi/operations"
+	"github.com/stretchr/testify/assert"
+)
+
+var token string
+
+func decodeBase64(value string) string {
+	/*
+		Helper function to decode in base64
+	*/
+	result, err := b64.StdEncoding.DecodeString(value)
+	if err != nil {
+		log.Fatal("error:", err)
+	}
+	return string(result)
+}
+
+func printMessage(message string) {
+	/*
+		Helper function to print HTTP response.
+	*/
+	fmt.Println(message)
+}
+
+func printLoggingMessage(message string, functionName string) {
+	/*
+		Helper function to have standard output across the tests.
+	*/
+	finalString := "......................." + functionName + "(): " + message
+	printMessage(finalString)
+}
+
+func printStartFunc(functionName string) {
+	/*
+		Common function for all tests to tell that test has started
+	*/
+	printMessage("")
+	printLoggingMessage("started", functionName)
+}
+
+func printEndFunc(functionName string) {
+	/*
+		Helper function for all tests to tell that test has ended, is completed
+	*/
+	printLoggingMessage("completed", functionName)
+	printMessage("")
+}
+
+func initConsoleServer() (*restapi.Server, error) {
+
+	//os.Setenv("CONSOLE_MINIO_SERVER", "localhost:9000")
+
+	swaggerSpec, err := loads.Embedded(restapi.SwaggerJSON, restapi.FlatSwaggerJSON)
+	if err != nil {
+		return nil, err
+	}
+
+	noLog := func(string, ...interface{}) {
+		// nothing to log
+	}
+
+	// Initialize MinIO loggers
+	restapi.LogInfo = noLog
+	restapi.LogError = noLog
+
+	api := operations.NewConsoleAPI(swaggerSpec)
+	api.Logger = noLog
+
+	server := restapi.NewServer(api)
+	// register all APIs
+	server.ConfigureAPI()
+
+	//restapi.GlobalRootCAs, restapi.GlobalPublicCerts, restapi.GlobalTLSCertsManager = globalRootCAs, globalPublicCerts, globalTLSCerts
+
+	consolePort, _ := strconv.Atoi("9090")
+
+	server.Host = "0.0.0.0"
+	server.Port = consolePort
+	restapi.Port = "9090"
+	restapi.Hostname = "0.0.0.0"
+
+	return server, nil
+}
+
+func TestMain(m *testing.M) {
+	printStartFunc("TestMain")
+	// start console server
+	go func() {
+		fmt.Println("start server")
+		srv, err := initConsoleServer()
+		if err != nil {
+			log.Println(err)
+			log.Println("init fail")
+			return
+		}
+		srv.Serve()
+
+	}()
+
+	fmt.Println("sleeping")
+	time.Sleep(2 * time.Second)
+
+	client := &http.Client{
+		Timeout: 2 * time.Second,
+	}
+
+	// kubectl to get token
+	app := "kubectl"
+	arg0 := "get"
+	arg1 := "serviceaccount"
+	arg2 := "console-sa"
+	arg3 := "--namespace"
+	arg4 := "minio-operator"
+	arg5 := "-o"
+	arg6 := "jsonpath=\"{.secrets[0].name}\""
+	cmd := exec.Command(app, arg0, arg1, arg2, arg3, arg4, arg5, arg6)
+	var out bytes.Buffer
+	var stderr bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = &stderr
+	err := cmd.Run()
+	if err != nil {
+		fmt.Println(fmt.Sprint(err) + ": " + stderr.String())
+		return
+	}
+	secret := out.String() // "console-sa-token-kxdw2" <-- secret
+	app2 := "kubectl"
+	argu0 := "--namespace"
+	argu1 := "minio-operator"
+	argu2 := "get"
+	argu3 := "secret"
+	argu4 := secret[1 : len(secret)-1]
+	argu5 := "-o"
+	argu6 := "jsonpath=\"{.data.token}\""
+	cmd2 := exec.Command(app2, argu0, argu1, argu2, argu3, argu4, argu5, argu6)
+	var out2 bytes.Buffer
+	var stderr2 bytes.Buffer
+	cmd2.Stdout = &out2
+	cmd2.Stderr = &stderr2
+	err2 := cmd2.Run()
+	if err2 != nil {
+		fmt.Println(fmt.Sprint(err2) + ": -> " + stderr2.String())
+		return
+	}
+	secret2 := out2.String()
+	secret3 := decodeBase64(secret2[1 : len(secret2)-1])
+	requestData := map[string]string{
+		"jwt": secret3,
+	}
+
+	requestDataJSON, _ := json.Marshal(requestData)
+
+	requestDataBody := bytes.NewReader(requestDataJSON)
+
+	request, err := http.NewRequest("POST", "http://localhost:9090/api/v1/login/operator", requestDataBody)
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
+	request.Header.Add("Content-Type", "application/json")
+
+	response, err := client.Do(request)
+
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
+	if response != nil {
+		for _, cookie := range response.Cookies() {
+			if cookie.Name == "token" {
+				token = cookie.Value
+				break
+			}
+		}
+	}
+
+	if token == "" {
+		log.Println("authentication token not found in cookies response")
+		return
+	}
+
+	code := m.Run()
+	printEndFunc("TestMain")
+	os.Exit(code)
+}
+
+func ListTenants() (*http.Response, error) {
+	/*
+		Helper function to list buckets
+		HTTP Verb: GET
+		URL: http://localhost:9090/api/v1/tenants
+	*/
+	request, err := http.NewRequest(
+		"GET", "http://localhost:9090/api/v1/tenants", nil)
+	if err != nil {
+		log.Println(err)
+	}
+	request.Header.Add("Cookie", fmt.Sprintf("token=%s", token))
+	request.Header.Add("Content-Type", "application/json")
+	client := &http.Client{
+		Timeout: 2 * time.Second,
+	}
+	response, err := client.Do(request)
+	return response, err
+}
+
+func TestListTenants(t *testing.T) {
+	// Tenants can be listed via API: https://github.com/miniohq/engineering/issues/591
+	printStartFunc("TestListTenants")
+	assert := assert.New(t)
+	resp, err := ListTenants()
+	assert.Nil(err)
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	if resp != nil {
+		assert.Equal(
+			200, resp.StatusCode, "Status Code is incorrect")
+	}
+	bodyBytes, _ := ioutil.ReadAll(resp.Body)
+	result := models.ListTenantsResponse{}
+	err = json.Unmarshal(bodyBytes, &result)
+	if err != nil {
+		log.Println(err)
+		assert.Nil(err)
+	}
+	TenantName := &result.Tenants[0].Name // The array has to be empty, no index accessible
+	fmt.Println(*TenantName)
+	assert.Equal("storage-lite", *TenantName, *TenantName)
+	printEndFunc("TestListTenants")
+}