feat: run functional tests on FreeBSD in CI

GitHub has no FreeBSD runners, so the new `functional-freebsd.yml` workflow runs the `runtests.sh` inside a FreeBSD 15.1 VM. The VM is booted under KVM on `ubuntu-latest` by `vmactions/freebsd-vm`, the most widely used action for this.

The `prepare` step installs `bash`, `git` and `gmake`, downloads the latest stable Go from `go.dev` to match `setup-go`'s `stable`, and creates a `tester` user that owns the synced workspace. `git` is required because the synced `.git` directory makes `go build` stamp VCS info. The `run` step then runs `gmake testbin`, `runtests.sh` and the coverage report as `tester`, matching the unprivileged Linux job.

FreeBSD enables ASLR by default (`kern.elf64.aslr.enable=1`), and the race-enabled `versitygw` built by `testbin` refuses to start under it with "This sanitizer is not compatible with enabled ASLR", so `prepare` turns ASLR off with `sysctl`.

The nested VM freezes for up to ~13 seconds at a time under the parallel suite, longer than the few seconds of margin some object-lock tests gave their retain until dates, so a date could expire in transit (the gateway then rejects it with `InvalidArgument`) or a COMPLIANCE lock could expire mid-test. A new `retryOnPastRetainDate` helper resends a request whose seconds-out date expired in transit, `complianceTestRetention` grows from 10s to 30s, and the tests that locked for twice that window, or extended by a whole one, now stay within a single window, which keeps the added suite time to about 10 seconds per run.

.github/workflows/functional-freebsd.yml

@@ -0,0 +1,37 @@
+name: functional tests (freebsd)
+permissions: {}
+on: pull_request
+
+jobs:
+  build:
+    name: RunTestsFreeBSD
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Checkout
+      uses: actions/checkout@v7
+
+    # GitHub has no FreeBSD runners, so the tests run in a FreeBSD VM booted
+    # under KVM on the Ubuntu runner. The workspace is synced into the VM, and
+    # prepare/run execute there as root under sh with set -eu.
+    - name: Build and Run
+      uses: vmactions/freebsd-vm@v1
+      with:
+        release: '15.1'
+        copyback: false
+        prepare: |
+          pkg install -y bash git gmake
+          GO_VERSION=$(fetch -qo - 'https://go.dev/VERSION?m=text' | head -n 1)
+          fetch -qo /tmp/go.tar.gz "https://go.dev/dl/${GO_VERSION}.freebsd-$(uname -m).tar.gz"
+          tar -C /usr/local -xzf /tmp/go.tar.gz
+          ln -s /usr/local/go/bin/go /usr/local/bin/go
+          pw useradd tester -m
+          chown -R tester .
+          # FreeBSD enables ASLR by default, and binaries built with -race
+          # refuse to start under it.
+          sysctl kern.elf64.aslr.enable=0
+        # Run as a regular user, like the Linux job does.
+        run: |
+          su tester -c 'gmake testbin'
+          su tester -c 'bash ./runtests.sh'
+          su tester -c 'go tool covdata percent -i=/tmp/covdata'

tests/integration/PutObjectRetention.go

@@ -399,7 +399,7 @@ func PutObjectRetention_shorten_governance_without_bypass(s *S3Conf) error {
 			return err
 		}
 
-		shorter := time.Now().Add(complianceTestRetention / 2)
+		shorter := date.Add(-time.Minute)
 
 		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
 		_, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{
@@ -456,7 +456,7 @@ func PutObjectRetention_shorten_governance_with_bypass(s *S3Conf) error {
 			return err
 		}
 
-		shorter := time.Now().Add(complianceTestRetention / 2)
+		shorter := date.Add(-time.Minute)
 
 		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
 		_, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{
@@ -487,7 +487,6 @@ func PutObjectRetention_shorten_compliance_denied(s *S3Conf) error {
 		}
 
 		policy := genPolicyDoc("Allow", fmt.Sprintf(`"%v"`, s.awsID), `["s3:BypassGovernanceRetention"]`, fmt.Sprintf(`"arn:aws:s3:::%v/*"`, bucket))
-		bypass := true
 
 		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
 		_, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
@@ -499,65 +498,51 @@ func PutObjectRetention_shorten_compliance_denied(s *S3Conf) error {
 			return err
 		}
 
-		// The COMPLIANCE lock goes on last, after the slow setup above rather
-		// than before it, so the whole complianceTestRetention budget is left
-		// for the two attempts below. Started any earlier, a slow object upload
-		// can consume the entire window and leave an already expired retention
-		// with nothing to shorten.
-		date := time.Now().Add(complianceTestRetention)
+		// The server rejects a RetainUntilDate that has already passed
+		// (InvalidArgument) before it ever gets to the object-lock comparison
+		// this test is exercising (ErrObjectLocked), and a stalled host can
+		// hold a request past a complianceTestRetention window. So each
+		// attempt locks the object for a fresh complianceTestRetention first,
+		// which only ever extends the lock and is always allowed, then asks
+		// for a date just short of it, the widest margin a shortening can
+		// have. An attempt that still loses its date to a stall is retried.
+		for _, bypass := range []bool{false, true} {
+			err := retryOnPastRetainDate(func() error {
+				date := time.Now().Add(complianceTestRetention)
 
-		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
-		_, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{
-			Bucket: &bucket,
-			Key:    &obj,
-			Retention: &types.ObjectLockRetention{
-				Mode:            types.ObjectLockRetentionModeCompliance,
-				RetainUntilDate: &date,
-			},
-		})
-		cancel()
-		if err != nil {
-			return err
-		}
+				ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+				_, err := s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{
+					Bucket: &bucket,
+					Key:    &obj,
+					Retention: &types.ObjectLockRetention{
+						Mode:            types.ObjectLockRetentionModeCompliance,
+						RetainUntilDate: &date,
+					},
+				})
+				cancel()
+				if err != nil {
+					return err
+				}
 
-		// Recomputed fresh right before each request below, rather than once
-		// up front: the server rejects a RetainUntilDate that has already
-		// passed (InvalidArgument) before it ever gets to the object-lock
-		// comparison this test is exercising (ErrObjectLocked)
-		newShorterDate := func() time.Time {
-			return time.Now().Add(time.Until(date) / 2)
-		}
+				shorter := date.Add(-time.Second)
+				ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+				_, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{
+					Bucket: &bucket,
+					Key:    &obj,
+					Retention: &types.ObjectLockRetention{
+						Mode:            types.ObjectLockRetentionModeCompliance,
+						RetainUntilDate: &shorter,
+					},
+					BypassGovernanceRetention: &bypass,
+				})
+				cancel()
+				return err
+			})
 
-		// Neither asking to bypass nor holding the permission helps.
-		shorter := newShorterDate()
-		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
-		_, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{
-			Bucket: &bucket,
-			Key:    &obj,
-			Retention: &types.ObjectLockRetention{
-				Mode:            types.ObjectLockRetentionModeCompliance,
-				RetainUntilDate: &shorter,
-			},
-		})
-		cancel()
-		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil {
-			return err
-		}
-
-		shorter = newShorterDate()
-		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
-		_, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{
-			Bucket: &bucket,
-			Key:    &obj,
-			Retention: &types.ObjectLockRetention{
-				Mode:            types.ObjectLockRetentionModeCompliance,
-				RetainUntilDate: &shorter,
-			},
-			BypassGovernanceRetention: &bypass,
-		})
-		cancel()
-		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil {
-			return err
+			// Neither asking to bypass nor holding the permission helps.
+			if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil {
+				return fmt.Errorf("bypass %v: %w", bypass, err)
+			}
 		}
 
 		return cleanupLockedObjects(s3client, bucket, []objToDelete{{key: obj, isCompliance: true}})

tests/integration/WORM_protection.go

@@ -1319,14 +1319,18 @@ func WORMProtection_legal_hold_outlives_expired_retention(s *S3Conf) error {
 	testName := "WORMProtection_legal_hold_outlives_expired_retention"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
 		obj := "my-obj"
-		retainUntil := time.Now().Add(lockWaitTime)
-		_, err := putObjectWithData(10, &s3.PutObjectInput{
-			Bucket:                    &bucket,
-			Key:                       &obj,
-			ObjectLockMode:            types.ObjectLockModeGovernance,
-			ObjectLockRetainUntilDate: &retainUntil,
-			ObjectLockLegalHoldStatus: types.ObjectLockLegalHoldStatusOn,
-		}, s3client, withPutObjectChecksumAlgo(types.ChecksumAlgorithmCrc32))
+		var retainUntil time.Time
+		err := retryOnPastRetainDate(func() error {
+			retainUntil = time.Now().Add(lockWaitTime)
+			_, err := putObjectWithData(10, &s3.PutObjectInput{
+				Bucket:                    &bucket,
+				Key:                       &obj,
+				ObjectLockMode:            types.ObjectLockModeGovernance,
+				ObjectLockRetainUntilDate: &retainUntil,
+				ObjectLockLegalHoldStatus: types.ObjectLockLegalHoldStatusOn,
+			}, s3client, withPutObjectChecksumAlgo(types.ChecksumAlgorithmCrc32))
+			return err
+		})
 		if err != nil {
 			return err
 		}

tests/integration/utils.go

@@ -3172,21 +3172,24 @@ func cleanupLockedObjects(client *s3.Client, bucket string, objs []objToDelete)
 			// A GOVERNANCE retention can be weakened with the bypass
 			// permission and header, so shorten it to a few seconds out
 			// rather than waiting for the original date. The margin absorbs
-			// network delay.
-			retDate := time.Now().Add(lockWaitTime)
+			// network delay, and a stall longer than that is retried.
+			err := retryOnPastRetainDate(func() error {
+				retDate := time.Now().Add(lockWaitTime)
 
-			ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
-			_, err := client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{
-				Bucket:                    &bucket,
-				Key:                       &obj.key,
-				VersionId:                 getNonEmptyPtr(obj.versionId),
-				BypassGovernanceRetention: getBoolPtr(true),
-				Retention: &types.ObjectLockRetention{
-					Mode:            types.ObjectLockRetentionModeGovernance,
-					RetainUntilDate: &retDate,
-				},
+				ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+				_, err := client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{
+					Bucket:                    &bucket,
+					Key:                       &obj.key,
+					VersionId:                 getNonEmptyPtr(obj.versionId),
+					BypassGovernanceRetention: getBoolPtr(true),
+					Retention: &types.ObjectLockRetention{
+						Mode:            types.ObjectLockRetentionModeGovernance,
+						RetainUntilDate: &retDate,
+					},
+				})
+				cancel()
+				return err
 			})
-			cancel()
 
 			// If object was already deleted, ignore the error
 			if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
@@ -3253,6 +3256,26 @@ func waitOutComplianceRetention(client *s3.Client, bucket string, obj objToDelet
 	return nil
 }
 
+// retryOnPastRetainDate calls req, a request carrying a retain until date
+// only seconds out, again for as long as the gateway rejects that date as
+// already passed, up to maxRetryAttempts times. A loaded host can stall
+// a request for longer than such a margin, which expires the date in transit,
+// and the gateway then answers InvalidArgument instead of whatever the request
+// was meant to exercise. req must compute its date from time.Now() on
+// every call for a retry to help.
+func retryOnPastRetainDate(req func() error) error {
+	pastDate := s3err.GetInvalidArgumentErr(s3err.InvalidArgPastObjectLockRetainDate, "")
+
+	var err error
+	for range maxRetryAttempts {
+		err = req()
+		if checkApiErr(err, pastDate) != nil {
+			return err
+		}
+	}
+	return err
+}
+
 type objectLockMode string
 
 const (