versitygw/integration/utils.go

package integration

import (
	"bytes"
	"context"
	"crypto/rand"
	"crypto/sha256"
	"encoding/hex"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	rnd "math/rand"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go-v2/aws"
	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
	"github.com/aws/aws-sdk-go-v2/service/s3"
	"github.com/aws/aws-sdk-go-v2/service/s3/types"
	"github.com/aws/smithy-go"
	"github.com/versity/versitygw/s3err"
	"github.com/versity/versitygw/s3response"
)

var (
	bcktCount  = 0
	succUsrCrt = "The user has been created successfully"
	failUsrCrt = "failed to create a user: update iam data: account already exists"
)

func getBucketName() string {
	bcktCount++
	return fmt.Sprintf("test-bucket-%v", bcktCount)
}

func setup(s *S3Conf, bucket string) error {
	s3client := s3.NewFromConfig(s.Config())

	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
	_, err := s3client.CreateBucket(ctx, &s3.CreateBucketInput{
		Bucket: &bucket,
	})
	cancel()
	return err
}

func teardown(s *S3Conf, bucket string) error {
	s3client := s3.NewFromConfig(s.Config())

	deleteObject := func(bucket, key, versionId *string) error {
		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
		_, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{
			Bucket:    bucket,
			Key:       key,
			VersionId: versionId,
		})
		cancel()
		if err != nil {
			return fmt.Errorf("failed to delete object %v: %v", *key, err)
		}
		return nil
	}

	in := &s3.ListObjectsV2Input{Bucket: &bucket}
	for {
		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
		out, err := s3client.ListObjectsV2(ctx, in)
		cancel()
		if err != nil {
			return fmt.Errorf("failed to list objects: %v", err)
		}

		for _, item := range out.Contents {
			err = deleteObject(&bucket, item.Key, nil)
			if err != nil {
				return err
			}
		}

		if out.IsTruncated {
			in.ContinuationToken = out.ContinuationToken
		} else {
			break
		}
	}

	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
	_, err := s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{
		Bucket: &bucket,
	})
	cancel()
	return err
}

func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) {
	runF(testName)
	bucketName := getBucketName()
	err := setup(s, bucketName)
	if err != nil {
		failF("%v: failed to create a bucket: %v", testName, err.Error())
		return
	}
	client := s3.NewFromConfig(s.Config())
	handlerErr := handler(client, bucketName)
	if handlerErr != nil {
		failF("%v: %v", testName, handlerErr.Error())
	}

	err = teardown(s, bucketName)
	if err != nil {
		if handlerErr == nil {
			failF("%v: failed to delete the bucket: %v", testName, err.Error())
		} else {
			fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err.Error())
		}
	}
	if handlerErr == nil {
		passF(testName)
	}
}

type authConfig struct {
	testName string
	path     string
	method   string
	body     []byte
	service  string
	date     time.Time
}

func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) {
	runF(cfg.testName)
	req, err := createSignedReq(cfg.method, s.endpoint, cfg.path, s.awsID, s.awsSecret, cfg.service, s.awsRegion, cfg.body, cfg.date)
	if err != nil {
		failF("%v: %v", cfg.testName, err.Error())
		return
	}

	err = handler(req)
	if err != nil {
		failF("%v: %v", cfg.testName, err.Error())
		return
	}
	passF(cfg.testName)
}

func createSignedReq(method, endpoint, path, access, secret, service, region string, body []byte, date time.Time) (*http.Request, error) {
	req, err := http.NewRequest(method, fmt.Sprintf("%v/%v", endpoint, path), bytes.NewReader(body))
	if err != nil {
		return nil, fmt.Errorf("failed to send the request: %w", err)
	}

	signer := v4.NewSigner()

	hashedPayload := sha256.Sum256(body)
	hexPayload := hex.EncodeToString(hashedPayload[:])

	req.Header.Set("X-Amz-Content-Sha256", hexPayload)

	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: access, SecretAccessKey: secret}, req, hexPayload, service, region, date)
	if signErr != nil {
		return nil, fmt.Errorf("failed to sign the request: %w", signErr)
	}

	return req, nil
}

func checkAuthErr(resp *http.Response, apiErr s3err.APIError) error {
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	var errResp s3err.APIErrorResponse
	err = xml.Unmarshal(body, &errResp)
	if err != nil {
		return err
	}

	if resp.StatusCode != apiErr.HTTPStatusCode {
		return fmt.Errorf("expected response status code to be %v, instead got %v", apiErr.HTTPStatusCode, resp.StatusCode)
	}
	if errResp.Code != apiErr.Code {
		return fmt.Errorf("expected error code to be %v, instead got %v", apiErr.Code, errResp.Code)
	}
	if errResp.Message != apiErr.Description {
		return fmt.Errorf("expected error message to be %v, instead got %v", apiErr.Description, errResp.Message)
	}

	return nil
}

func checkApiErr(err error, apiErr s3err.APIError) error {
	if err == nil {
		return fmt.Errorf("expected %v, instead got nil", apiErr.Code)
	}
	var ae smithy.APIError
	if errors.As(err, &ae) {
		if ae.ErrorCode() == apiErr.Code && ae.ErrorMessage() == apiErr.Description {
			return nil
		}

		return fmt.Errorf("expected %v, instead got %v", apiErr.Code, ae.ErrorCode())
	} else {
		return fmt.Errorf("expected aws api error, instead got: %v", err.Error())
	}
}

func checkSdkApiErr(err error, code string) error {
	var ae smithy.APIError
	if errors.As(err, &ae) {
		if ae.ErrorCode() != code {
			return fmt.Errorf("expected %v, instead got %v", ae.ErrorCode(), code)
		}
		return nil
	}
	return err
}

func putObjects(client *s3.Client, objs []string, bucket string) error {
	for _, key := range objs {
		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
		_, err := client.PutObject(ctx, &s3.PutObjectInput{
			Key:    &key,
			Bucket: &bucket,
		})
		cancel()
		if err != nil {
			return err
		}
	}
	return nil
}

func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (csum [32]byte, data []byte, err error) {
	data = make([]byte, lgth)
	rand.Read(data)
	csum = sha256.Sum256(data)
	r := bytes.NewReader(data)
	input.Body = r

	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
	_, err = client.PutObject(ctx, input)
	cancel()

	return
}

func createMp(s3client *s3.Client, bucket, key string) (*s3.CreateMultipartUploadOutput, error) {
	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
	out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
		Bucket: &bucket,
		Key:    &key,
	})
	cancel()
	return out, err
}

func isEqual(a, b []byte) bool {
	if len(a) != len(b) {
		return false
	}

	for i, d := range a {
		if d != b[i] {
			return false
		}
	}

	return true
}

func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool {
	if len(list1) != len(list2) {
		return false
	}
	for i, item := range list1 {
		if *item.Key != *list2[i].Key || *item.UploadId != *list2[i].UploadId {
			return false
		}
	}

	return true
}

func compareParts(parts1, parts2 []types.Part) bool {
	if len(parts1) != len(parts2) {
		return false
	}

	for i, prt := range parts1 {
		if prt.PartNumber != parts2[i].PartNumber {
			return false
		}
		if *prt.ETag != *parts2[i].ETag {
			return false
		}
	}
	return true
}

func areTagsSame(tags1, tags2 []types.Tag) bool {
	if len(tags1) != len(tags2) {
		return false
	}

	for _, tag := range tags1 {
		if !containsTag(tag, tags2) {
			return false
		}
	}
	return true
}

func containsTag(tag types.Tag, list []types.Tag) bool {
	for _, item := range list {
		if *item.Key == *tag.Key && *item.Value == *tag.Value {
			return true
		}
	}
	return false
}

func compareGrants(grts1, grts2 []types.Grant) bool {
	if len(grts1) != len(grts2) {
		return false
	}

	for i, grt := range grts1 {
		if grt.Permission != grts2[i].Permission {
			return false
		}
		if *grt.Grantee.ID != *grts2[i].Grantee.ID {
			return false
		}
	}
	return true
}

func execCommand(args ...string) ([]byte, error) {
	cmd := exec.Command("./versitygw", args...)

	return cmd.CombinedOutput()
}

func getString(str *string) string {
	if str == nil {
		return ""
	}
	return *str
}

func getPtr(str string) *string {
	return &str
}

func areMapsSame(mp1, mp2 map[string]string) bool {
	if len(mp1) != len(mp2) {
		return false
	}
	for key, val := range mp1 {
		if mp2[key] != val {
			return false
		}
	}
	return true
}

func compareBuckets(list1 []types.Bucket, list2 []s3response.ListAllMyBucketsEntry) bool {
	if len(list1) != len(list2) {
		return false
	}

	elementMap := make(map[string]bool)

	for _, elem := range list1 {
		elementMap[*elem.Name] = true
	}

	for _, elem := range list2 {
		if _, found := elementMap[elem.Name]; !found {
			return false
		}
	}

	return true
}

func compareObjects(list1 []string, list2 []types.Object) bool {
	if len(list1) != len(list2) {
		return false
	}

	elementMap := make(map[string]bool)

	for _, elem := range list1 {
		elementMap[elem] = true
	}

	for _, elem := range list2 {
		if _, found := elementMap[*elem.Key]; !found {
			return false
		}
	}

	return true
}

func comparePrefixes(list1 []string, list2 []types.CommonPrefix) bool {
	if len(list1) != len(list2) {
		return false
	}

	elementMap := make(map[string]bool)

	for _, elem := range list1 {
		elementMap[elem] = true
	}

	for _, elem := range list2 {
		if _, found := elementMap[*elem.Prefix]; !found {
			return false
		}
	}

	return true
}

func compareDelObjects(list1 []string, list2 []types.DeletedObject) bool {
	if len(list1) != len(list2) {
		return false
	}

	elementMap := make(map[string]bool)

	for _, elem := range list1 {
		elementMap[elem] = true
	}

	for _, elem := range list2 {
		if _, found := elementMap[*elem.Key]; !found {
			return false
		}
	}

	return true
}

func uploadParts(client *s3.Client, size, partCount int, bucket, key, uploadId string) (parts []types.Part, err error) {
	dr := NewDataReader(size, size)
	datafile := "rand.data"
	w, err := os.Create(datafile)
	if err != nil {
		return parts, err
	}
	defer w.Close()

	_, err = io.Copy(w, dr)
	if err != nil {
		return parts, err
	}

	fileInfo, err := w.Stat()
	if err != nil {
		return parts, err
	}

	partSize := fileInfo.Size() / int64(partCount)
	var offset int64

	for partNumber := int64(1); partNumber <= int64(partCount); partNumber++ {
		partStart := (partNumber - 1) * partSize
		partEnd := partStart + partSize - 1
		if partEnd > fileInfo.Size()-1 {
			partEnd = fileInfo.Size() - 1
		}
		partBuffer := make([]byte, partEnd-partStart+1)
		_, err := w.ReadAt(partBuffer, partStart)
		if err != nil {
			return parts, err
		}
		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
		out, err := client.UploadPart(ctx, &s3.UploadPartInput{
			Bucket:     &bucket,
			Key:        &key,
			UploadId:   &uploadId,
			Body:       bytes.NewReader(partBuffer),
			PartNumber: int32(partNumber),
		})
		cancel()
		if err != nil {
			return parts, err
		} else {
			parts = append(parts, types.Part{ETag: out.ETag, PartNumber: int32(partNumber)})
			offset += partSize
		}
	}

	return parts, err
}

type user struct {
	access string
	secret string
	role   string
}

func createUsers(s *S3Conf, users []user) error {
	for _, usr := range users {
		out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "create-user", "-a", usr.access, "-s", usr.secret, "-r", usr.role)
		if err != nil {
			return err
		}
		if !strings.Contains(string(out), succUsrCrt) && !strings.Contains(string(out), failUsrCrt) {
			return fmt.Errorf("failed to create a user account")
		}
	}
	return nil
}

func changeBucketsOwner(s *S3Conf, buckets []string, owner string) error {
	for _, bucket := range buckets {
		out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "change-bucket-owner", "-b", bucket, "-o", owner)
		if err != nil {
			return err
		}
		if !strings.Contains(string(out), "Bucket owner has been updated successfully") {
			return fmt.Errorf(string(out))
		}
	}

	return nil
}

const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"

func genRandString(length int) string {
	source := rnd.NewSource(time.Now().UnixNano())
	random := rnd.New(source)
	result := make([]byte, length)
	for i := range result {
		result[i] = charset[random.Intn(len(charset))]
	}
	return string(result)
}