From cc43b3c743035d9b0b0830d58604e23604b308b9 Mon Sep 17 00:00:00 2001 From: Cesar Celis Hernandez Date: Tue, 15 Mar 2022 23:13:33 -0400 Subject: [PATCH] Add list of tenants integration test (#1722) --- .github/workflows/common.sh | 94 ++++++++++ .github/workflows/deploy-tenant.sh | 71 ++++++++ .github/workflows/jobs.yaml | 64 ++++++- Makefile | 5 + operator-integration/tenant_test.go | 269 ++++++++++++++++++++++++++++ 5 files changed, 501 insertions(+), 2 deletions(-) create mode 100755 .github/workflows/common.sh create mode 100755 .github/workflows/deploy-tenant.sh create mode 100644 operator-integration/tenant_test.go diff --git a/.github/workflows/common.sh b/.github/workflows/common.sh new file mode 100755 index 000000000..045ebc21f --- /dev/null +++ b/.github/workflows/common.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Copyright (C) 2022, MinIO, Inc. +# +# This code is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3, +# as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License, version 3, +# along with this program. If not, see + +yell() { echo "$0: $*" >&2; } + +die() { + yell "$*" + (kind delete cluster || true ) && exit 111 +} + +try() { "$@" || die "cannot $*"; } + +function setup_kind() { + # TODO once feature is added: https://github.com/kubernetes-sigs/kind/issues/1300 + echo "kind: Cluster" > kind-config.yaml + echo "apiVersion: kind.x-k8s.io/v1alpha4" >> kind-config.yaml + echo "nodes:" >> kind-config.yaml + echo " - role: control-plane" >> kind-config.yaml + echo " - role: worker" >> kind-config.yaml + echo " - role: worker" >> kind-config.yaml + echo " - role: worker" >> kind-config.yaml + echo " - role: worker" >> kind-config.yaml + try kind create cluster --config kind-config.yaml + echo "Kind is ready" + try kubectl get nodes +} + +function install_operator() { + echo "Installing Current Operator" + + # TODO: Compile the current branch and create an overlay to use that image version + try kubectl apply -k "${SCRIPT_DIR}/../../portal-ui/tests/scripts/resources" + + echo "Waiting for k8s api" + sleep 10 + echo "Waiting for Operator Pods to come online (2m timeout)" + + try kubectl wait --namespace minio-operator \ + --for=condition=ready pod \ + --selector=name=minio-operator \ + --timeout=120s +} + +function destroy_kind() { + kind delete cluster +} + +function check_tenant_status() { + # Check MinIO is accessible + + waitdone=0 + totalwait=0 + while true; do + waitdone=$(kubectl -n $1 get pods -l v1.min.io/tenant=$2 --no-headers | wc -l) + if [ "$waitdone" -ne 0 ]; then + echo "Found $waitdone pods" + break + fi + sleep 5 + totalwait=$((totalwait + 5)) + if [ "$totalwait" -gt 305 ]; then + echo "Unable to create tenant after 5 minutes, exiting." + try false + fi + done + + echo "Waiting for pods to be ready. (5m timeout)" + + USER=$(kubectl -n $1 get secrets $2-env-configuration -o go-template='{{index .data "config.env"|base64decode }}' | grep 'export MINIO_ROOT_USER="' | sed -e 's/export MINIO_ROOT_USER="//g' | sed -e 's/"//g') + PASSWORD=$(kubectl -n $1 get secrets $2-env-configuration -o go-template='{{index .data "config.env"|base64decode }}' | grep 'export MINIO_ROOT_PASSWORD="' | sed -e 's/export MINIO_ROOT_PASSWORD="//g' | sed -e 's/"//g') + + try kubectl wait --namespace $1 \ + --for=condition=ready pod \ + --selector=v1.min.io/tenant=$2 \ + --timeout=300s + + echo "Tenant is created successfully, proceeding to validate 'mc admin info minio/'" + + kubectl run admin-mc -i --tty --image minio/mc --command -- bash -c "until (mc alias set minio/ https://minio.$1.svc.cluster.local $USER $PASSWORD); do echo \"...waiting... for 5secs\" && sleep 5; done; mc admin info minio/;" + + echo "Done." +} \ No newline at end of file diff --git a/.github/workflows/deploy-tenant.sh b/.github/workflows/deploy-tenant.sh new file mode 100755 index 000000000..c6da2618d --- /dev/null +++ b/.github/workflows/deploy-tenant.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Copyright (C) 2022, MinIO, Inc. +# +# This code is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3, +# as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License, version 3, +# along with this program. If not, see + +# This script requires: kubectl, kind + +SCRIPT_DIR=$(dirname "$0") +export SCRIPT_DIR + +source "${SCRIPT_DIR}/common.sh" + + + +function install_tenant() { + echo "Installing lite tenant" + + try kubectl apply -k "${SCRIPT_DIR}/../../portal-ui/tests/scripts/tenant" + + echo "Waiting for the tenant statefulset, this indicates the tenant is being fulfilled" + waitdone=0 + totalwait=0 + while true; do + waitdone=$(kubectl -n tenant-lite get pods -l v1.min.io/tenant=storage-lite --no-headers | wc -l) + if [ "$waitdone" -ne 0 ]; then + echo "Found $waitdone pods" + break + fi + sleep 5 + totalwait=$((totalwait + 5)) + if [ "$totalwait" -gt 300 ]; then + echo "Tenant never created statefulset after 5 minutes" + try false + fi + done + + echo "Waiting for tenant pods to come online (5m timeout)" + try kubectl wait --namespace tenant-lite \ + --for=condition=ready pod \ + --selector="v1.min.io/tenant=storage-lite" \ + --timeout=300s + + echo "Build passes basic tenant creation" +} + + +function main() { + destroy_kind + + setup_kind + + install_operator + + install_tenant + + check_tenant_status tenant-lite storage-lite + + kubectl -n minio-operator port-forward svc/console 9090 & +} + +main "$@" diff --git a/.github/workflows/jobs.yaml b/.github/workflows/jobs.yaml index 1a9b7a3c2..7ca77bef2 100644 --- a/.github/workflows/jobs.yaml +++ b/.github/workflows/jobs.yaml @@ -15,6 +15,56 @@ concurrency: cancel-in-progress: true jobs: + + operator-api-tests: + + name: Operator API Tests + needs: + - lint-job + - no-warnings-and-make-assets + - reuse-golang-dependencies + - vulnerable-dependencies-checks + runs-on: ubuntu-latest + + strategy: + matrix: + go-version: [ 1.17.x ] + + steps: + + - name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }} + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + id: go + + - uses: actions/checkout@v2 + + - uses: actions/cache@v2 + name: Go Mod Cache + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ github.run_id }} + + - name: Operator API Tests + run: | + curl -sLO "https://dl.k8s.io/release/v1.23.1/bin/linux/amd64/kubectl" -o kubectl + chmod +x kubectl + mv kubectl /usr/local/bin + "${GITHUB_WORKSPACE}/.github/workflows/deploy-tenant.sh" + echo "start ---> make test-operator-integration"; + make test-operator-integration; + + - uses: actions/cache@v2 + id: coverage-cache-operator + name: Coverage Cache Operator + with: + path: | + ./operator-integration/coverage/ + key: ${{ runner.os }}-coverage-2-operator-${{ github.run_id }} + lint-job: name: Checking Lint runs-on: ${{ matrix.os }} @@ -702,6 +752,8 @@ jobs: needs: - integration-tests - test-restapi-on-go + - operator-api-tests + - test-pkg-on-go runs-on: ${{ matrix.os }} strategy: matrix: @@ -736,6 +788,14 @@ jobs: ./integration/coverage/ key: ${{ runner.os }}-coverage-2-${{ github.run_id }} + - uses: actions/cache@v2 + id: coverage-cache-operator + name: Coverage Cache Operator + with: + path: | + ./operator-integration/coverage/ + key: ${{ runner.os }}-coverage-2-operator-${{ github.run_id }} + - uses: actions/cache@v2 id: coverage-cache-restapi name: Coverage Cache RestAPI @@ -763,14 +823,14 @@ jobs: echo "go build gocoverage.go" go build gocovmerge.go echo "put together the outs for final coverage resolution" - ./gocovmerge ../integration/coverage/system.out ../restapi/coverage/coverage.out ../pkg/coverage/coverage-pkg.out > all.out + ./gocovmerge ../integration/coverage/system.out ../restapi/coverage/coverage.out ../pkg/coverage/coverage-pkg.out ../operator-integration/coverage/operator-api.out > all.out echo "grep to obtain the result" go tool cover -func=all.out | grep total > tmp2 result=`cat tmp2 | awk 'END {print $3}'` result=${result%\%} echo "result:" echo $result - threshold=50.4 + threshold=50.5 if (( $(echo "$result >= $threshold" |bc -l) )); then echo "It is equal or greater than threshold, passed!" else diff --git a/Makefile b/Makefile index c1aa31b73..b3eb62fb2 100644 --- a/Makefile +++ b/Makefile @@ -80,6 +80,11 @@ test-integration: @(docker stop minio) @(docker network rm mynet123) +test-operator-integration: + @(echo "Start cd operator-integration && go test:") + @(pwd) + @(cd operator-integration && go test -coverpkg=../restapi -c -tags testrunmain . && mkdir -p coverage && ./operator-integration.test -test.run "^Test*" -test.coverprofile=coverage/operator-api.out) + test-operator: @(env bash $(PWD)/portal-ui/tests/scripts/operator.sh) @(docker stop minio) diff --git a/operator-integration/tenant_test.go b/operator-integration/tenant_test.go new file mode 100644 index 000000000..6312fa959 --- /dev/null +++ b/operator-integration/tenant_test.go @@ -0,0 +1,269 @@ +// This file is part of MinIO Console Server +// Copyright (c) 2021 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package operatorintegration + +import ( + "bytes" + b64 "encoding/base64" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "os/exec" + "strconv" + "testing" + "time" + + "github.com/go-openapi/loads" + "github.com/minio/console/models" + "github.com/minio/console/restapi" + "github.com/minio/console/restapi/operations" + "github.com/stretchr/testify/assert" +) + +var token string + +func decodeBase64(value string) string { + /* + Helper function to decode in base64 + */ + result, err := b64.StdEncoding.DecodeString(value) + if err != nil { + log.Fatal("error:", err) + } + return string(result) +} + +func printMessage(message string) { + /* + Helper function to print HTTP response. + */ + fmt.Println(message) +} + +func printLoggingMessage(message string, functionName string) { + /* + Helper function to have standard output across the tests. + */ + finalString := "......................." + functionName + "(): " + message + printMessage(finalString) +} + +func printStartFunc(functionName string) { + /* + Common function for all tests to tell that test has started + */ + printMessage("") + printLoggingMessage("started", functionName) +} + +func printEndFunc(functionName string) { + /* + Helper function for all tests to tell that test has ended, is completed + */ + printLoggingMessage("completed", functionName) + printMessage("") +} + +func initConsoleServer() (*restapi.Server, error) { + + //os.Setenv("CONSOLE_MINIO_SERVER", "localhost:9000") + + swaggerSpec, err := loads.Embedded(restapi.SwaggerJSON, restapi.FlatSwaggerJSON) + if err != nil { + return nil, err + } + + noLog := func(string, ...interface{}) { + // nothing to log + } + + // Initialize MinIO loggers + restapi.LogInfo = noLog + restapi.LogError = noLog + + api := operations.NewConsoleAPI(swaggerSpec) + api.Logger = noLog + + server := restapi.NewServer(api) + // register all APIs + server.ConfigureAPI() + + //restapi.GlobalRootCAs, restapi.GlobalPublicCerts, restapi.GlobalTLSCertsManager = globalRootCAs, globalPublicCerts, globalTLSCerts + + consolePort, _ := strconv.Atoi("9090") + + server.Host = "0.0.0.0" + server.Port = consolePort + restapi.Port = "9090" + restapi.Hostname = "0.0.0.0" + + return server, nil +} + +func TestMain(m *testing.M) { + printStartFunc("TestMain") + // start console server + go func() { + fmt.Println("start server") + srv, err := initConsoleServer() + if err != nil { + log.Println(err) + log.Println("init fail") + return + } + srv.Serve() + + }() + + fmt.Println("sleeping") + time.Sleep(2 * time.Second) + + client := &http.Client{ + Timeout: 2 * time.Second, + } + + // kubectl to get token + app := "kubectl" + arg0 := "get" + arg1 := "serviceaccount" + arg2 := "console-sa" + arg3 := "--namespace" + arg4 := "minio-operator" + arg5 := "-o" + arg6 := "jsonpath=\"{.secrets[0].name}\"" + cmd := exec.Command(app, arg0, arg1, arg2, arg3, arg4, arg5, arg6) + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + err := cmd.Run() + if err != nil { + fmt.Println(fmt.Sprint(err) + ": " + stderr.String()) + return + } + secret := out.String() // "console-sa-token-kxdw2" <-- secret + app2 := "kubectl" + argu0 := "--namespace" + argu1 := "minio-operator" + argu2 := "get" + argu3 := "secret" + argu4 := secret[1 : len(secret)-1] + argu5 := "-o" + argu6 := "jsonpath=\"{.data.token}\"" + cmd2 := exec.Command(app2, argu0, argu1, argu2, argu3, argu4, argu5, argu6) + var out2 bytes.Buffer + var stderr2 bytes.Buffer + cmd2.Stdout = &out2 + cmd2.Stderr = &stderr2 + err2 := cmd2.Run() + if err2 != nil { + fmt.Println(fmt.Sprint(err2) + ": -> " + stderr2.String()) + return + } + secret2 := out2.String() + secret3 := decodeBase64(secret2[1 : len(secret2)-1]) + requestData := map[string]string{ + "jwt": secret3, + } + + requestDataJSON, _ := json.Marshal(requestData) + + requestDataBody := bytes.NewReader(requestDataJSON) + + request, err := http.NewRequest("POST", "http://localhost:9090/api/v1/login/operator", requestDataBody) + if err != nil { + log.Println(err) + return + } + + request.Header.Add("Content-Type", "application/json") + + response, err := client.Do(request) + + if err != nil { + log.Println(err) + return + } + + if response != nil { + for _, cookie := range response.Cookies() { + if cookie.Name == "token" { + token = cookie.Value + break + } + } + } + + if token == "" { + log.Println("authentication token not found in cookies response") + return + } + + code := m.Run() + printEndFunc("TestMain") + os.Exit(code) +} + +func ListTenants() (*http.Response, error) { + /* + Helper function to list buckets + HTTP Verb: GET + URL: http://localhost:9090/api/v1/tenants + */ + request, err := http.NewRequest( + "GET", "http://localhost:9090/api/v1/tenants", nil) + if err != nil { + log.Println(err) + } + request.Header.Add("Cookie", fmt.Sprintf("token=%s", token)) + request.Header.Add("Content-Type", "application/json") + client := &http.Client{ + Timeout: 2 * time.Second, + } + response, err := client.Do(request) + return response, err +} + +func TestListTenants(t *testing.T) { + // Tenants can be listed via API: https://github.com/miniohq/engineering/issues/591 + printStartFunc("TestListTenants") + assert := assert.New(t) + resp, err := ListTenants() + assert.Nil(err) + if err != nil { + log.Println(err) + return + } + if resp != nil { + assert.Equal( + 200, resp.StatusCode, "Status Code is incorrect") + } + bodyBytes, _ := ioutil.ReadAll(resp.Body) + result := models.ListTenantsResponse{} + err = json.Unmarshal(bodyBytes, &result) + if err != nil { + log.Println(err) + assert.Nil(err) + } + TenantName := &result.Tenants[0].Name // The array has to be empty, no index accessible + fmt.Println(*TenantName) + assert.Equal("storage-lite", *TenantName, *TenantName) + printEndFunc("TestListTenants") +}