test(s3-retention): force-drop collection after deleteBucket to free volumes

COMPLIANCE-mode retention leaves objects that BypassGovernanceRetention cannot
clear, so the test's DeleteBucket keeps returning BucketNotEmpty and the
underlying SeaweedFS collection (with its 7 reserved volumes) leaks. After a
few leaks on the single-node `weed mini` server, the master logs "Not enough
data nodes found" and every subsequent PutObject 500s, timing the suite out.

Call the master's /col/delete admin endpoint from deleteBucket so the
collection's volumes are reclaimed even when S3-level cleanup is blocked.

test/s3/retention/s3_retention_test.go

@@ -3,6 +3,9 @@ package retention
 import (
 	"context"
 	"fmt"
+	"io"
+	"net/http"
+	"net/url"
 	"strings"
 	"testing"
 	"time"
@@ -18,24 +21,26 @@ import (
 
 // S3TestConfig holds configuration for S3 tests
 type S3TestConfig struct {
-	Endpoint      string
+	Endpoint       string
-	AccessKey     string
+	MasterEndpoint string
-	SecretKey     string
+	AccessKey      string
-	Region        string
+	SecretKey      string
-	BucketPrefix  string
+	Region         string
-	UseSSL        bool
+	BucketPrefix   string
-	SkipVerifySSL bool
+	UseSSL         bool
+	SkipVerifySSL  bool
 }
 
 // Default test configuration - should match test_config.json
 var defaultConfig = &S3TestConfig{
-	Endpoint:      "http://localhost:8333", // Default SeaweedFS S3 port
+	Endpoint:       "http://localhost:8333", // Default SeaweedFS S3 port
-	AccessKey:     "some_access_key1",
+	MasterEndpoint: "http://localhost:9333", // Default SeaweedFS master HTTP port
-	SecretKey:     "some_secret_key1",
+	AccessKey:      "some_access_key1",
-	Region:        "us-east-1",
+	SecretKey:      "some_secret_key1",
-	BucketPrefix:  "test-retention-",
+	Region:         "us-east-1",
-	UseSSL:        false,
+	BucketPrefix:   "test-retention-",
-	SkipVerifySSL: true,
+	UseSSL:         false,
+	SkipVerifySSL:  true,
 }
 
 // getS3Client creates an AWS S3 client for testing
@@ -105,7 +110,13 @@ func deleteBucket(t *testing.T, client *s3.Client, bucketName string) {
 	// Wait a bit for eventual consistency
 	time.Sleep(100 * time.Millisecond)
 
-	// Try to delete the bucket multiple times in case of eventual consistency issues
+	// Try to delete the bucket multiple times in case of eventual consistency issues.
+	// Always force-drop the underlying collection at the master afterwards: COMPLIANCE-mode
+	// retention can leave undeletable objects, so the S3 DeleteBucket may keep failing with
+	// BucketNotEmpty and leak the collection's volumes. Without this, running enough tests
+	// on a single `weed mini` server exhausts the data node's volume slots and every
+	// subsequent PutObject 500s with "Not enough data nodes found".
+	defer forceDeleteCollection(t, bucketName)
 	for retries := 0; retries < 3; retries++ {
 		_, err = client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{
 			Bucket: aws.String(bucketName),
@@ -122,6 +133,39 @@ func deleteBucket(t *testing.T, client *s3.Client, bucketName string) {
 	}
 }
 
+// forceDeleteCollection drops the SeaweedFS collection backing a test bucket via the master's
+// /col/delete admin endpoint. The S3 layer normally drops the collection on DeleteBucket, but
+// when retention/legal-hold blocks the bucket cleanup, the collection (and its reserved
+// volumes) leaks. Best-effort: a 400 from the master means the collection was already gone,
+// which is the success path and not an error.
+func forceDeleteCollection(t *testing.T, bucketName string) {
+	if defaultConfig.MasterEndpoint == "" {
+		return
+	}
+	endpoint := strings.TrimRight(defaultConfig.MasterEndpoint, "/") + "/col/delete?collection=" + url.QueryEscape(bucketName)
+	req, err := http.NewRequest(http.MethodGet, endpoint, nil)
+	if err != nil {
+		t.Logf("Note: building collection delete request for %s failed: %v", bucketName, err)
+		return
+	}
+	httpClient := &http.Client{Timeout: 5 * time.Second}
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		t.Logf("Note: force-delete collection %s failed: %v", bucketName, err)
+		return
+	}
+	defer resp.Body.Close()
+	io.Copy(io.Discard, resp.Body)
+	switch resp.StatusCode {
+	case http.StatusNoContent:
+		t.Logf("Force-deleted collection %s", bucketName)
+	case http.StatusBadRequest:
+		// Collection already gone - normal path when DeleteBucket succeeded.
+	default:
+		t.Logf("Note: force-delete collection %s returned HTTP %d", bucketName, resp.StatusCode)
+	}
+}
+
 // deleteAllObjectVersions deletes all object versions in a bucket
 func deleteAllObjectVersions(t *testing.T, client *s3.Client, bucketName string) error {
 	// List all object versions