diff --git a/.github/workflows/functional-freebsd.yml b/.github/workflows/functional-freebsd.yml new file mode 100644 index 00000000..94cde21f --- /dev/null +++ b/.github/workflows/functional-freebsd.yml @@ -0,0 +1,37 @@ +name: functional tests (freebsd) +permissions: {} +on: pull_request + +jobs: + build: + name: RunTestsFreeBSD + runs-on: ubuntu-latest + steps: + + - name: Checkout + uses: actions/checkout@v7 + + # GitHub has no FreeBSD runners, so the tests run in a FreeBSD VM booted + # under KVM on the Ubuntu runner. The workspace is synced into the VM, and + # prepare/run execute there as root under sh with set -eu. + - name: Build and Run + uses: vmactions/freebsd-vm@v1 + with: + release: '15.1' + copyback: false + prepare: | + pkg install -y bash git gmake + GO_VERSION=$(fetch -qo - 'https://go.dev/VERSION?m=text' | head -n 1) + fetch -qo /tmp/go.tar.gz "https://go.dev/dl/${GO_VERSION}.freebsd-$(uname -m).tar.gz" + tar -C /usr/local -xzf /tmp/go.tar.gz + ln -s /usr/local/go/bin/go /usr/local/bin/go + pw useradd tester -m + chown -R tester . + # FreeBSD enables ASLR by default, and binaries built with -race + # refuse to start under it. + sysctl kern.elf64.aslr.enable=0 + # Run as a regular user, like the Linux job does. + run: | + su tester -c 'gmake testbin' + su tester -c 'bash ./runtests.sh' + su tester -c 'go tool covdata percent -i=/tmp/covdata' diff --git a/tests/integration/PutObjectRetention.go b/tests/integration/PutObjectRetention.go index 21bfc91d..6e36c4aa 100644 --- a/tests/integration/PutObjectRetention.go +++ b/tests/integration/PutObjectRetention.go @@ -399,7 +399,7 @@ func PutObjectRetention_shorten_governance_without_bypass(s *S3Conf) error { return err } - shorter := time.Now().Add(complianceTestRetention / 2) + shorter := date.Add(-time.Minute) ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ @@ -456,7 +456,7 @@ func PutObjectRetention_shorten_governance_with_bypass(s *S3Conf) error { return err } - shorter := time.Now().Add(complianceTestRetention / 2) + shorter := date.Add(-time.Minute) ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ @@ -487,7 +487,6 @@ func PutObjectRetention_shorten_compliance_denied(s *S3Conf) error { } policy := genPolicyDoc("Allow", fmt.Sprintf(`"%v"`, s.awsID), `["s3:BypassGovernanceRetention"]`, fmt.Sprintf(`"arn:aws:s3:::%v/*"`, bucket)) - bypass := true ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ @@ -499,65 +498,51 @@ func PutObjectRetention_shorten_compliance_denied(s *S3Conf) error { return err } - // The COMPLIANCE lock goes on last, after the slow setup above rather - // than before it, so the whole complianceTestRetention budget is left - // for the two attempts below. Started any earlier, a slow object upload - // can consume the entire window and leave an already expired retention - // with nothing to shorten. - date := time.Now().Add(complianceTestRetention) + // The server rejects a RetainUntilDate that has already passed + // (InvalidArgument) before it ever gets to the object-lock comparison + // this test is exercising (ErrObjectLocked), and a stalled host can + // hold a request past a complianceTestRetention window. So each + // attempt locks the object for a fresh complianceTestRetention first, + // which only ever extends the lock and is always allowed, then asks + // for a date just short of it, the widest margin a shortening can + // have. An attempt that still loses its date to a stall is retried. + for _, bypass := range []bool{false, true} { + err := retryOnPastRetainDate(func() error { + date := time.Now().Add(complianceTestRetention) - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - Retention: &types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionModeCompliance, - RetainUntilDate: &date, - }, - }) - cancel() - if err != nil { - return err - } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + RetainUntilDate: &date, + }, + }) + cancel() + if err != nil { + return err + } - // Recomputed fresh right before each request below, rather than once - // up front: the server rejects a RetainUntilDate that has already - // passed (InvalidArgument) before it ever gets to the object-lock - // comparison this test is exercising (ErrObjectLocked) - newShorterDate := func() time.Time { - return time.Now().Add(time.Until(date) / 2) - } + shorter := date.Add(-time.Second) + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + RetainUntilDate: &shorter, + }, + BypassGovernanceRetention: &bypass, + }) + cancel() + return err + }) - // Neither asking to bypass nor holding the permission helps. - shorter := newShorterDate() - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - Retention: &types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionModeCompliance, - RetainUntilDate: &shorter, - }, - }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { - return err - } - - shorter = newShorterDate() - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ - Bucket: &bucket, - Key: &obj, - Retention: &types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionModeCompliance, - RetainUntilDate: &shorter, - }, - BypassGovernanceRetention: &bypass, - }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { - return err + // Neither asking to bypass nor holding the permission helps. + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return fmt.Errorf("bypass %v: %w", bypass, err) + } } return cleanupLockedObjects(s3client, bucket, []objToDelete{{key: obj, isCompliance: true}}) diff --git a/tests/integration/WORM_protection.go b/tests/integration/WORM_protection.go index 1efe6e33..2cac5a0f 100644 --- a/tests/integration/WORM_protection.go +++ b/tests/integration/WORM_protection.go @@ -1319,14 +1319,18 @@ func WORMProtection_legal_hold_outlives_expired_retention(s *S3Conf) error { testName := "WORMProtection_legal_hold_outlives_expired_retention" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" - retainUntil := time.Now().Add(lockWaitTime) - _, err := putObjectWithData(10, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - ObjectLockMode: types.ObjectLockModeGovernance, - ObjectLockRetainUntilDate: &retainUntil, - ObjectLockLegalHoldStatus: types.ObjectLockLegalHoldStatusOn, - }, s3client, withPutObjectChecksumAlgo(types.ChecksumAlgorithmCrc32)) + var retainUntil time.Time + err := retryOnPastRetainDate(func() error { + retainUntil = time.Now().Add(lockWaitTime) + _, err := putObjectWithData(10, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ObjectLockMode: types.ObjectLockModeGovernance, + ObjectLockRetainUntilDate: &retainUntil, + ObjectLockLegalHoldStatus: types.ObjectLockLegalHoldStatusOn, + }, s3client, withPutObjectChecksumAlgo(types.ChecksumAlgorithmCrc32)) + return err + }) if err != nil { return err } diff --git a/tests/integration/utils.go b/tests/integration/utils.go index e925c34a..003df299 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -3172,21 +3172,24 @@ func cleanupLockedObjects(client *s3.Client, bucket string, objs []objToDelete) // A GOVERNANCE retention can be weakened with the bypass // permission and header, so shorten it to a few seconds out // rather than waiting for the original date. The margin absorbs - // network delay. - retDate := time.Now().Add(lockWaitTime) + // network delay, and a stall longer than that is retried. + err := retryOnPastRetainDate(func() error { + retDate := time.Now().Add(lockWaitTime) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ - Bucket: &bucket, - Key: &obj.key, - VersionId: getNonEmptyPtr(obj.versionId), - BypassGovernanceRetention: getBoolPtr(true), - Retention: &types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionModeGovernance, - RetainUntilDate: &retDate, - }, + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj.key, + VersionId: getNonEmptyPtr(obj.versionId), + BypassGovernanceRetention: getBoolPtr(true), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: &retDate, + }, + }) + cancel() + return err }) - cancel() // If object was already deleted, ignore the error if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) { @@ -3253,6 +3256,26 @@ func waitOutComplianceRetention(client *s3.Client, bucket string, obj objToDelet return nil } +// retryOnPastRetainDate calls req, a request carrying a retain until date +// only seconds out, again for as long as the gateway rejects that date as +// already passed, up to maxRetryAttempts times. A loaded host can stall +// a request for longer than such a margin, which expires the date in transit, +// and the gateway then answers InvalidArgument instead of whatever the request +// was meant to exercise. req must compute its date from time.Now() on +// every call for a retry to help. +func retryOnPastRetainDate(req func() error) error { + pastDate := s3err.GetInvalidArgumentErr(s3err.InvalidArgPastObjectLockRetainDate, "") + + var err error + for range maxRetryAttempts { + err = req() + if checkApiErr(err, pastDate) != nil { + return err + } + } + return err +} + type objectLockMode string const (