versitygw/cmd/versitygw/test.go

// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package main

import (
	"fmt"
	"strings"

	"github.com/urfave/cli/v2"
	"github.com/versity/versitygw/tests/integration"
)

var (
	awsID             string
	awsSecret         string
	endpoint          string
	iamEndpoint       string
	websiteSchemeTest string
	websiteDomainTest string
	websitePortTest   string
	prefix            string
	dstBucket         string
	partSize          int64
	objSize           int64
	concurrency       int
	files             int
	totalReqs         int
	upload            bool
	download          bool
	hostStyle         bool
	checksumDisable   bool
	versioningEnabled bool
	azureTests        bool
	testDebug         bool
	tlsStatus         bool
	parallel          bool
	windowsTests      bool
	sidecarTests      bool
)

func testCommand() *cli.Command {
	return &cli.Command{
		Name:  "test",
		Usage: "Client side testing command for the gateway",
		Description: `The testing CLI is used to test group of versitygw actions.
		It also includes some performance and stress testing`,
		Subcommands: initTestCommands(),
		Flags:       initTestFlags(),
	}
}

func initTestFlags() []cli.Flag {
	return []cli.Flag{
		&cli.StringFlag{
			Name:        "access",
			Usage:       "aws user access key",
			EnvVars:     []string{"AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY"},
			Aliases:     []string{"a"},
			Destination: &awsID,
		},
		&cli.StringFlag{
			Name:        "secret",
			Usage:       "aws user secret access key",
			EnvVars:     []string{"AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY"},
			Aliases:     []string{"s"},
			Destination: &awsSecret,
		},
		&cli.StringFlag{
			Name:        "endpoint",
			Usage:       "s3 server endpoint",
			Destination: &endpoint,
			Aliases:     []string{"e"},
		},
		&cli.StringFlag{
			Name:        "iam-endpoint",
			Usage:       "standalone IAM/STS service endpoint, when it is a separate process from the s3 endpoint (defaults to --endpoint)",
			Destination: &iamEndpoint,
			Aliases:     []string{"ie"},
		},
		&cli.BoolFlag{
			Name:        "host-style",
			Usage:       "Use host-style bucket addressing",
			Value:       false,
			Destination: &hostStyle,
		},
		&cli.BoolFlag{
			Name:        "debug",
			Usage:       "enable debug mode",
			Aliases:     []string{"d"},
			Destination: &testDebug,
		},
		&cli.BoolFlag{
			Name:        "allow-insecure",
			Usage:       "skip tls verification",
			Aliases:     []string{"ai"},
			Destination: &tlsStatus,
		},
	}
}

func initTestCommands() []*cli.Command {
	return append([]*cli.Command{
		{
			Name:        "full-flow",
			Usage:       "Tests the full flow of gateway.",
			Description: `Runs all the available tests to test the full flow of the gateway.`,
			Action:      getAction(integration.TestFullFlow),
			Flags: []cli.Flag{
				&cli.BoolFlag{
					Name:        "versioning-enabled",
					Usage:       "Test the bucket object versioning, if the versioning is enabled",
					Destination: &versioningEnabled,
					Aliases:     []string{"vs"},
				},
				&cli.BoolFlag{
					Name:        "azure-test-mode",
					Usage:       "Skips tests that are not supported by Azure",
					Destination: &azureTests,
					Aliases:     []string{"azure"},
				},
				&cli.BoolFlag{
					Name:        "windows-test-mode",
					Usage:       "Skips tests that are not supported on Windows",
					Destination: &windowsTests,
					Aliases:     []string{"windows"},
				},
				&cli.BoolFlag{
					Name:        "sidecar-test-mode",
					Usage:       "Skips tests that are not supported by Sidecar",
					Destination: &sidecarTests,
					Aliases:     []string{"sidecar"},
				},
				&cli.BoolFlag{
					Name:        "parallel",
					Usage:       "executes the tests concurrently",
					Destination: &parallel,
					Aliases:     []string{"p"},
				},
			},
		},
		{
			Name:        "website-hosting",
			Usage:       "Tests static website hosting endpoint.",
			Description: `Runs the static website hosting integration tests against a dedicated website endpoint.`,
			Action:      websiteHostingAction,
			Flags: []cli.Flag{
				&cli.StringFlag{
					Name:        "scheme",
					Usage:       "website endpoint scheme: http or https",
					EnvVars:     []string{"VGW_TEST_WEBSITE_SCHEME"},
					Destination: &websiteSchemeTest,
					Aliases:     []string{"website-scheme", "protocol"},
					Value:       "http",
				},
				&cli.StringFlag{
					Name:        "domain",
					Usage:       "website endpoint base domain used for virtual-host routing",
					EnvVars:     []string{"VGW_TEST_WEBSITE_DOMAIN"},
					Destination: &websiteDomainTest,
					Aliases:     []string{"website-domain"},
				},
				&cli.StringFlag{
					Name:        "port",
					Usage:       "website endpoint port",
					EnvVars:     []string{"VGW_TEST_WEBSITE_PORT"},
					Destination: &websitePortTest,
					Aliases:     []string{"website-port"},
				},
			},
		},
		{
			Name:   "posix",
			Usage:  "Tests posix specific features",
			Action: getAction(integration.TestPosix),
			Flags: []cli.Flag{
				&cli.BoolFlag{
					Name:        "versioning-enabled",
					Usage:       "Test posix when versioning is enabled",
					Destination: &versioningEnabled,
					Aliases:     []string{"vs"},
				},
				&cli.BoolFlag{
					Name:        "windows-test-mode",
					Usage:       "Skips tests that are not supported on Windows",
					Destination: &windowsTests,
					Aliases:     []string{"windows"},
				},
			},
		},
		{
			Name:   "scoutfs",
			Usage:  "Tests scoutfs full flow",
			Action: getAction(integration.TestScoutfs),
		},
		{
			Name:   "gw-iam",
			Usage:  "Tests gateway IAM service integration",
			Action: getAction(integration.TestGatewayIAM),
		},
		{
			Name:   "iam",
			Usage:  "Tests standalone IAM API integration",
			Action: getAction(integration.TestIAM),
		},
		{
			Name:   "access-control",
			Usage:  "Tests gateway access control with bucket ACLs and Policies",
			Action: getAction(integration.TestAccessControl),
		},
		{
			Name:  "s3-iam",
			Usage: "Tests s3 gateway access control backed by the standalone IAM service",
			Description: `Runs the access-control tests for an s3 gateway configured with --iam-standalone-endpoint:
			IAM user identity policies, their interaction with bucket policies, governance-retention
			bypass, and bucket creation. Requires --iam-endpoint pointing at the IAM service's
			control-plane API, since the tests create the users and policies they then exercise.`,
			Action: getAction(integration.TestS3IAMAccessControl),
		},
		{
			Name:  "s3-iam-session",
			Usage: "Tests s3 gateway access control for assumed-role session credentials",
			Description: `Runs the role/session access-control tests against an s3 gateway backed by the
			standalone IAM service. Every test mints real temporary credentials via
			AssumeRoleWithWebIdentity against GitHub Actions' OIDC issuer, so the whole group skips
			itself outside a GitHub Actions job holding id-token: write permission.`,
			Action: getAction(integration.TestS3IAMSessionAccessControl),
		},
		{
			Name:   "noacl",
			Usage:  "Tests gateway in ACL-disabled mode",
			Action: getAction(integration.TestNoAclMode),
		},
		{
			Name:   "data-integrity-etag",
			Usage:  "Tests checksum-derived ETag behavior",
			Action: getAction(integration.TestDataIntegrityETag),
		},
		{
			Name:  "bench",
			Usage: "Runs download/upload performance test on the gateway",
			Description: `Uploads/downloads some number(specified by flags) of files with some capacity(bytes).
			Logs the results to the console`,
			Flags: []cli.Flag{
				&cli.IntFlag{
					Name:        "files",
					Usage:       "Number of objects to read/write",
					Value:       1,
					Destination: &files,
				},
				&cli.Int64Flag{
					Name:        "objsize",
					Usage:       "Uploading object size",
					Value:       0,
					Destination: &objSize,
				},
				&cli.StringFlag{
					Name:        "prefix",
					Usage:       "Object name prefix",
					Destination: &prefix,
				},
				&cli.BoolFlag{
					Name:        "upload",
					Usage:       "Upload data to the gateway",
					Value:       false,
					Destination: &upload,
				},
				&cli.BoolFlag{
					Name:        "download",
					Usage:       "Download data to the gateway",
					Value:       false,
					Destination: &download,
				},
				&cli.StringFlag{
					Name:        "bucket",
					Usage:       "Destination bucket name to read/write data",
					Destination: &dstBucket,
				},
				&cli.Int64Flag{
					Name:        "partSize",
					Usage:       "Upload/download size per thread",
					Value:       64 * 1024 * 1024,
					Destination: &partSize,
				},
				&cli.IntFlag{
					Name:        "concurrency",
					Usage:       "Upload/download threads per object",
					Value:       1,
					Destination: &concurrency,
				},
				&cli.BoolFlag{
					Name:        "checksumDis",
					Usage:       "Disable server checksum",
					Value:       false,
					Destination: &checksumDisable,
				},
			},
			Action: func(ctx *cli.Context) error {
				if upload && download {
					return fmt.Errorf("must only specify one of upload or download")
				}
				if !upload && !download {
					return fmt.Errorf("must specify one of upload or download")
				}

				if dstBucket == "" {
					return fmt.Errorf("must specify bucket")
				}

				opts := []integration.Option{
					integration.WithAccess(awsID),
					integration.WithSecret(awsSecret),
					integration.WithRegion(region),
					integration.WithEndpoint(endpoint),
					integration.WithConcurrency(concurrency),
					integration.WithPartSize(partSize),
					integration.WithTLSStatus(tlsStatus),
				}
				if testDebug {
					opts = append(opts, integration.WithDebug())
				}
				if hostStyle {
					opts = append(opts, integration.WithHostStyle())
				}
				if checksumDisable {
					opts = append(opts, integration.WithDisableChecksum())
				}

				s3conf := integration.NewS3Conf(opts...)

				if upload {
					return integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
				} else {
					return integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
				}
			},
		},
		{
			Name:        "throughput",
			Usage:       "Runs throughput performance test on the gateway",
			Description: `Calls HeadBucket action the number of times and concurrency level specified with flags by measuring gateway throughput.`,
			Flags: []cli.Flag{
				&cli.IntFlag{
					Name:        "reqs",
					Usage:       "Total number of requests to send.",
					Value:       1000,
					Destination: &totalReqs,
				},
				&cli.StringFlag{
					Name:        "bucket",
					Usage:       "Destination bucket name to make the requests",
					Destination: &dstBucket,
				},
				&cli.IntFlag{
					Name:        "concurrency",
					Usage:       "threads per request",
					Value:       1,
					Destination: &concurrency,
				},
				&cli.BoolFlag{
					Name:        "checksumDis",
					Usage:       "Disable server checksum",
					Value:       false,
					Destination: &checksumDisable,
				},
			},
			Action: func(ctx *cli.Context) error {
				if dstBucket == "" {
					return fmt.Errorf("must specify the destination bucket")
				}

				opts := []integration.Option{
					integration.WithAccess(awsID),
					integration.WithSecret(awsSecret),
					integration.WithRegion(region),
					integration.WithEndpoint(endpoint),
					integration.WithConcurrency(concurrency),
					integration.WithTLSStatus(tlsStatus),
				}
				if testDebug {
					opts = append(opts, integration.WithDebug())
				}
				if checksumDisable {
					opts = append(opts, integration.WithDisableChecksum())
				}
				if hostStyle {
					opts = append(opts, integration.WithHostStyle())
				}

				s3conf := integration.NewS3Conf(opts...)

				return integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
			},
		},
	}, extractIntTests()...)
}

type testFunc func(*integration.TestState)

func websiteHostingAction(ctx *cli.Context) error {
	websiteSchemeTest = strings.ToLower(strings.TrimSpace(websiteSchemeTest))
	if websiteSchemeTest != "http" && websiteSchemeTest != "https" {
		return fmt.Errorf("website scheme must be http or https")
	}
	if websiteDomainTest == "" {
		return fmt.Errorf("must specify website domain")
	}
	if websitePortTest == "" {
		return fmt.Errorf("must specify website port")
	}

	opts := []integration.Option{
		integration.WithAccess(awsID),
		integration.WithSecret(awsSecret),
		integration.WithRegion(region),
		integration.WithEndpoint(endpoint),
	}
	if websiteSchemeTest != "" {
		opts = append(opts, integration.WithWebsiteScheme(websiteSchemeTest))
	}
	if websiteDomainTest != "" {
		opts = append(opts, integration.WithWebsiteDomain(websiteDomainTest))
	}
	if websitePortTest != "" {
		opts = append(opts, integration.WithWebsitePort(websitePortTest))
	}
	if testDebug {
		opts = append(opts, integration.WithDebug())
	}

	s := integration.NewS3Conf(opts...)
	ts := integration.NewTestState(ctx.Context, s, false)
	integration.TestWebsiteHosting(ts)
	ts.Wait()

	fmt.Println()
	fmt.Println("RAN:", integration.RunCount.Load(), "PASS:", integration.PassCount.Load(), "FAIL:", integration.FailCount.Load(), "SKIP:", integration.SkipCount.Load())
	if integration.FailCount.Load() > 0 {
		return fmt.Errorf("test failed with %v errors", integration.FailCount.Load())
	}
	return nil
}

func getAction(tf testFunc) func(ctx *cli.Context) error {
	return func(ctx *cli.Context) error {
		opts := []integration.Option{
			integration.WithAccess(awsID),
			integration.WithSecret(awsSecret),
			integration.WithRegion(region),
			integration.WithEndpoint(endpoint),
			integration.WithIAMEndpoint(iamEndpoint),
			integration.WithTLSStatus(tlsStatus),
		}
		if testDebug {
			opts = append(opts, integration.WithDebug())
		}
		if versioningEnabled {
			opts = append(opts, integration.WithVersioningEnabled())
		}
		if azureTests {
			opts = append(opts, integration.WithAzureMode())
		}
		if windowsTests {
			opts = append(opts, integration.WithWindowsMode())
			opts = append(opts, integration.WithSidecarMode())
		}
		if sidecarTests {
			opts = append(opts, integration.WithSidecarMode())
		}
		if hostStyle {
			opts = append(opts, integration.WithHostStyle())
		}

		s := integration.NewS3Conf(opts...)
		ts := integration.NewTestState(ctx.Context, s, parallel)
		tf(ts)
		ts.Wait()

		fmt.Println()
		fmt.Println("RAN:", integration.RunCount.Load(), "PASS:", integration.PassCount.Load(), "FAIL:", integration.FailCount.Load(), "SKIP:", integration.SkipCount.Load())
		if integration.FailCount.Load() > 0 {
			return fmt.Errorf("test failed with %v errors", integration.FailCount.Load())
		}
		return nil
	}
}

func extractIntTests() (commands []*cli.Command) {
	tests := integration.GetIntTests()
	for key, val := range tests {
		k := key
		testFunc := val
		commands = append(commands, &cli.Command{
			Name:  k,
			Usage: fmt.Sprintf("Runs %v integration test", key),
			Action: func(ctx *cli.Context) error {
				opts := []integration.Option{
					integration.WithAccess(awsID),
					integration.WithSecret(awsSecret),
					integration.WithRegion(region),
					integration.WithEndpoint(endpoint),
					integration.WithIAMEndpoint(iamEndpoint),
					integration.WithTLSStatus(tlsStatus),
				}
				if testDebug {
					opts = append(opts, integration.WithDebug())
				}
				if versioningEnabled {
					opts = append(opts, integration.WithVersioningEnabled())
				}
				if hostStyle {
					opts = append(opts, integration.WithHostStyle())
				}
				if azureTests {
					opts = append(opts, integration.WithAzureMode())
				}
				if windowsTests {
					opts = append(opts, integration.WithWindowsMode())
					opts = append(opts, integration.WithSidecarMode())
				}
				if sidecarTests {
					opts = append(opts, integration.WithSidecarMode())
				}

				s := integration.NewS3Conf(opts...)
				err := testFunc(s)
				return err
			},
			Flags: []cli.Flag{
				&cli.BoolFlag{
					Name:        "versioning-enabled",
					Usage:       "Test the bucket object versioning, if the versioning is enabled",
					Destination: &versioningEnabled,
					Aliases:     []string{"vs"},
				},
				&cli.BoolFlag{
					Name:        "azure-test-mode",
					Usage:       "Skips tests that are not supported by Azure",
					Destination: &azureTests,
					Aliases:     []string{"azure"},
				},
				&cli.BoolFlag{
					Name:        "sidecar-test-mode",
					Usage:       "Skips tests that are not supported by Sidecar",
					Destination: &sidecarTests,
					Aliases:     []string{"sidecar"},
				},
				&cli.BoolFlag{
					Name:        "windows-test-mode",
					Usage:       "Skips tests that are not supported on Windows",
					Destination: &windowsTests,
					Aliases:     []string{"windows"},
				},
			},
		})
	}
	return
}