fix: azure tests and add azure ci test

.github/workflows/azurite.yml

@@ -0,0 +1,37 @@
+name: azurite functional tests
+
+on: pull_request
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+              
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 'stable'
+        id: go
+
+      - name: Set up Docker Compose
+        run: |
+          docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . up -d azurite azuritegw
+
+      - name: Wait for Azurite to be ready
+        run: sleep 40
+
+      - name: Get Dependencies
+        run: |
+          go mod download
+  
+      - name: Build and Run
+        run: |
+          make
+          ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow --azure
+  
+      - name: Shut down services
+        run: |
+          docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . down azurite azuritegw

.github/workflows/functional.yml

@@ -1,7 +1,8 @@
 name: functional tests
-on: pull_request
-jobs:
 
+on: pull_request
+
+jobs:
   build:
     name: RunTests
     runs-on: ubuntu-latest
@@ -18,7 +19,7 @@ jobs:
 
     - name: Get Dependencies
       run: |
-        go get -v -t -d ./...
+        go mod download
 
     - name: Build and Run
       run: |

backend/azure/azure.go

@@ -183,12 +183,14 @@ func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
 		var acl auth.ACL
 		if len(aclBytes) > 0 {
 			if err := json.Unmarshal(aclBytes, &acl); err != nil {
-				return fmt.Errorf("unmarshal bucket acl: %w", err)
+				return fmt.Errorf("unmarshal acl: %w", err)
 			}
 		}
+
 		if acl.Owner == acct.Access {
 			return s3err.GetAPIError(s3err.ErrBucketAlreadyOwnedByYou)
 		}
+		return s3err.GetAPIError(s3err.ErrBucketAlreadyExists)
 	}
 	return azureErrToS3Err(err)
 }

cmd/versitygw/test.go

@@ -37,6 +37,7 @@ var (
 	pathStyle         bool
 	checksumDisable   bool
 	versioningEnabled bool
+	azureTests        bool
 )
 
 func testCommand() *cli.Command {
@@ -95,6 +96,12 @@ func initTestCommands() []*cli.Command {
 					Destination: &versioningEnabled,
 					Aliases:     []string{"vs"},
 				},
+				&cli.BoolFlag{
+					Name:        "azure-test-mode",
+					Usage:       "Skips tests that are not supported by Azure",
+					Destination: &azureTests,
+					Aliases:     []string{"azure"},
+				},
 			},
 		},
 		{
@@ -288,6 +295,9 @@ func getAction(tf testFunc) func(*cli.Context) error {
 		if versioningEnabled {
 			opts = append(opts, integration.WithVersioningEnabled())
 		}
+		if azureTests {
+			opts = append(opts, integration.WithAzureMode())
+		}
 
 		s := integration.NewS3Conf(opts...)
 		tf(s)

tests/docker-compose.yml

@@ -2,7 +2,7 @@ services:
   posix:
     build: 
       context: .
-      dockerfile: ./Dockerfile.dev
+      dockerfile: tests/Dockerfile.dev
       args:
         - IAM_DIR=${IAM_DIR}
         - SETUP_DIR=${SETUP_DIR}
@@ -14,7 +14,7 @@ services:
   proxy:
     build: 
       context: .
-      dockerfile: ./Dockerfile.dev
+      dockerfile: tests/Dockerfile.dev
     volumes:
       - ./:/app
     ports:
@@ -34,9 +34,9 @@ services:
   azuritegw:
     build:
       context: .
-      dockerfile: ./Dockerfile.dev
+      dockerfile: tests/Dockerfile.dev
     volumes:
       - ./:/app
     ports:
       - 7070:7070
-    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]
+    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -buildvcs=false -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]

tests/integration/group-tests.go

@@ -456,7 +456,9 @@ func TestFullFlow(s *S3Conf) {
 	TestDeleteObjectTagging(s)
 	TestCreateMultipartUpload(s)
 	TestUploadPart(s)
-	TestUploadPartCopy(s)
+	if !s.azureTests {
+		TestUploadPartCopy(s)
+	}
 	TestListParts(s)
 	TestListMultipartUploads(s)
 	TestAbortMultipartUpload(s)

tests/integration/s3conf.go

@@ -41,6 +41,7 @@ type S3Conf struct {
 	Concurrency       int
 	debug             bool
 	versioningEnabled bool
+	azureTests        bool
 }
 
 func NewS3Conf(opts ...Option) *S3Conf {
@@ -84,6 +85,9 @@ func WithDebug() Option {
 func WithVersioningEnabled() Option {
 	return func(s *S3Conf) { s.versioningEnabled = true }
 }
+func WithAzureMode() Option {
+	return func(s *S3Conf) { s.azureTests = true }
+}
 
 func (c *S3Conf) getCreds() credentials.StaticCredentialsProvider {
 	// TODO support token/IAM

tests/integration/tests.go

@@ -2691,13 +2691,20 @@ func PutObject_non_existing_bucket(s *S3Conf) error {
 
 func PutObject_special_chars(s *S3Conf) error {
 	testName := "PutObject_special_chars"
+
+	objnames := []string{
+		"my!key", "my-key", "my_key", "my.key", "my'key", "my(key", "my)key",
+		"my&key", "my@key", "my=key", "my;key", "my:key", "my key", "my,key",
+		"my?key", "my^key", "my{}key", "my%key", "my`key",
+		"my[]key", "my~key", "my<>key", "my|key", "my#key",
+	}
+	if !s.azureTests {
+		// azure currently can't handle backslashes in object names
+		objnames = append(objnames, "my\\key")
+	}
+
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
-		objs, err := putObjects(s3client, []string{
-			"my!key", "my-key", "my_key", "my.key", "my'key", "my(key", "my)key",
-			"my&key", "my@key", "my=key", "my;key", "my:key", "my key", "my,key",
-			"my?key", "my\\key", "my^key", "my{}key", "my%key", "my`key",
-			"my[]key", "my~key", "my<>key", "my|key", "my#key",
-		}, bucket)
+		objs, err := putObjects(s3client, objnames, bucket)
 		if err != nil {
 			return err
 		}
@@ -2712,7 +2719,8 @@ func PutObject_special_chars(s *S3Conf) error {
 		}
 
 		if !compareObjects(res.Contents, objs) {
-			return fmt.Errorf("expected the objects to be %v, instead got %v", objs, res.Contents)
+			return fmt.Errorf("expected the objects to be %v, instead got %v",
+				objStrings(objs), objStrings(res.Contents))
 		}
 
 		return nil

tests/integration/utils.go

@@ -109,32 +109,57 @@ func teardown(s *S3Conf, bucket string) error {
 		return nil
 	}
 
-	in := &s3.ListObjectVersionsInput{Bucket: &bucket}
-	for {
-		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
-		out, err := s3client.ListObjectVersions(ctx, in)
-		cancel()
-		if err != nil {
-			return fmt.Errorf("failed to list objects: %w", err)
-		}
-
-		for _, item := range out.Versions {
-			err = deleteObject(&bucket, item.Key, item.VersionId)
+	if s.versioningEnabled {
+		in := &s3.ListObjectVersionsInput{Bucket: &bucket}
+		for {
+			ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+			out, err := s3client.ListObjectVersions(ctx, in)
+			cancel()
 			if err != nil {
-				return err
+				return fmt.Errorf("failed to list objects: %w", err)
+			}
+
+			for _, item := range out.Versions {
+				err = deleteObject(&bucket, item.Key, item.VersionId)
+				if err != nil {
+					return err
+				}
+			}
+			for _, item := range out.DeleteMarkers {
+				err = deleteObject(&bucket, item.Key, item.VersionId)
+				if err != nil {
+					return err
+				}
+			}
+
+			if out.IsTruncated != nil && *out.IsTruncated {
+				in.KeyMarker = out.KeyMarker
+				in.VersionIdMarker = out.NextVersionIdMarker
+			} else {
+				break
 			}
 		}
-		for _, item := range out.DeleteMarkers {
-			err = deleteObject(&bucket, item.Key, item.VersionId)
+	} else {
+		for {
+			ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+			out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
+				Bucket: &bucket,
+			})
+			cancel()
 			if err != nil {
-				return err
+				return fmt.Errorf("failed to list objects: %w", err)
 			}
-		}
 
-		if out.IsTruncated != nil && *out.IsTruncated {
-			in.KeyMarker = out.KeyMarker
-			in.VersionIdMarker = out.NextVersionIdMarker
-		} else {
+			for _, item := range out.Contents {
+				err = deleteObject(&bucket, item.Key, nil)
+				if err != nil {
+					return err
+				}
+			}
+
+			if out.IsTruncated != nil && *out.IsTruncated {
+				continue
+			}
 			break
 		}
 	}
@@ -594,20 +619,28 @@ func compareBuckets(list1 []types.Bucket, list2 []s3response.ListAllMyBucketsEnt
 
 func compareObjects(list1, list2 []types.Object) bool {
 	if len(list1) != len(list2) {
+		fmt.Println("list lengths are not equal")
 		return false
 	}
 
 	for i, obj := range list1 {
 		if *obj.Key != *list2[i].Key {
+			fmt.Printf("keys are not equal: %q != %q\n", *obj.Key, *list2[i].Key)
 			return false
 		}
 		if *obj.ETag != *list2[i].ETag {
+			fmt.Printf("etags are not equal: (%q %q)  %q != %q\n",
+				*obj.Key, *list2[i].Key, *obj.ETag, *list2[i].ETag)
 			return false
 		}
 		if *obj.Size != *list2[i].Size {
+			fmt.Printf("sizes are not equal: (%q %q)  %v != %v\n",
+				*obj.Key, *list2[i].Key, *obj.Size, *list2[i].Size)
 			return false
 		}
 		if obj.StorageClass != list2[i].StorageClass {
+			fmt.Printf("storage classes are not equal: (%q %q)  %v != %v\n",
+				*obj.Key, *list2[i].Key, obj.StorageClass, list2[i].StorageClass)
 			return false
 		}
 	}