diff --git a/.github/workflows/functional-iam-oidc.yml b/.github/workflows/functional-iam-oidc.yml index 0c7a11d9..fefb5eeb 100644 --- a/.github/workflows/functional-iam-oidc.yml +++ b/.github/workflows/functional-iam-oidc.yml @@ -46,43 +46,6 @@ jobs: run: | make testbin - - name: Run GitHub OIDC live web-identity test + - name: Run GitHub OIDC live web-identity and s3 session tests run: | - set -Eeuo pipefail - - IAM_PID="" - cleanup() { - local status=$? - trap - EXIT - if [[ -n "$IAM_PID" ]] && kill -0 "$IAM_PID" 2>/dev/null; then - kill "$IAM_PID" 2>/dev/null || true - fi - if [[ -n "$IAM_PID" ]]; then - wait "$IAM_PID" 2>/dev/null || true - fi - exit "$status" - } - trap cleanup EXIT - - mkdir -p /tmp/iam-oidc - ./versitygw --health /healthz -p :7078 -a user -s pass iam --dir /tmp/iam-oidc & - IAM_PID=$! - - ready="" - for _ in {1..50}; do - if curl --fail --silent --max-time 1 http://127.0.0.1:7078/healthz >/dev/null 2>&1; then - ready=1 - break - fi - if ! kill -0 "$IAM_PID" 2>/dev/null; then - echo "IAM API server stopped before becoming ready" >&2 - exit 1 - fi - sleep 0.2 - done - if [[ -z "$ready" ]]; then - echo "timed out waiting for IAM API server" >&2 - exit 1 - fi - - ./versitygw test -a user -s pass -e http://127.0.0.1:7078 IAMAssumeRoleWithWebIdentity_github_oidc_live + ./runoidctests.sh diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index a4d63798..6e626665 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -23,5 +23,5 @@ jobs: if [ "$rc" -ne 0 ]; then overall_rc="$rc" fi - done < <(find . \( -path './runiamtests.sh' -o -path './tests/*.sh' -o -path './tests/*/*.sh' \) -print0) + done < <(find . \( -path './runiamtests.sh' -o -path './genmtlscerts.sh' -o -path './runoidctests.sh' -o -path './tests/*.sh' -o -path './tests/*/*.sh' \) -print0) exit "$overall_rc" diff --git a/auth/access-control.go b/auth/access-control.go index 93d71743..1283b2f1 100644 --- a/auth/access-control.go +++ b/auth/access-control.go @@ -18,16 +18,28 @@ import ( "context" "encoding/json" "errors" + "fmt" "net/http" "net/url" "strings" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v3" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/s3err" ) -func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource string, opts AccessOptions) error { +func VerifyObjectCopyAccess(ctx fiber.Ctx, be backend.Backend, copySource string, opts AccessOptions) error { + // Verify destination bucket access first. VerifyAccess enforces the + // readonly gate before its own root/admin bypass, and that ordering + // must hold here too — readonly mode blocks writes for everyone, + // including root/admin, not just ordinary users. + if err := VerifyAccess(ctx, be, opts); err != nil { + return err + } + // Root/admin already cleared the destination check above; skip the + // source-bucket ACL lookup entirely for them, same as before. if opts.IsRoot { return nil } @@ -35,10 +47,6 @@ func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource return nil } - // Verify destination bucket access - if err := VerifyAccess(ctx, be, opts); err != nil { - return err - } // Verify source bucket access. // URL-decode the copy source before splitting so that clients which send // the bucket/key separator as "%2F" are handled correctly. @@ -53,7 +61,7 @@ func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource } // Get source bucket ACL - srcBucketACLBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: &srcBucket}) + srcBucketACLBytes, err := be.GetBucketAcl(ctx.RequestCtx(), &s3.GetBucketAclInput{Bucket: &srcBucket}) if err != nil { return err } @@ -71,6 +79,8 @@ func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource Bucket: srcBucket, Object: srcObject, Actions: []Action{GetObjectAction}, + Iam: opts.Iam, + DisableACL: opts.DisableACL, }); err != nil { return err } @@ -89,50 +99,404 @@ type AccessOptions struct { Readonly bool IsPublicRequest bool DisableACL bool + Iam IAMService } -func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error { +// VerifyAccess decides whether opts.Acc may perform opts.Actions against +// opts.Bucket/opts.Object, combining the bucket's own resource-based +// decision (policy, or ACL absent one) with an identity-based decision from +// opts.Iam when it implements PolicyEvaluator. An explicit Deny from either +// source denies the request outright, even when the other source would +// otherwise allow it; absent any explicit Deny, either source's Allow is +// independently sufficient; absent both, the request is denied. All three +// denial shapes are Code: AccessDenied, HTTP 403 — differing only in the +// dynamic Message text. +func VerifyAccess(ctx fiber.Ctx, be backend.Backend, opts AccessOptions) error { + if err := verifyAccessGates(opts); err != nil || !authorizationApplies(opts) { + return err + } + + errs, err := objectsAccessErrors(ctx.RequestCtx(), be, opts, []string{opts.Object}, requestConditionContext(ctx)) + if err != nil { + return err + } + return errs[0] +} + +// VerifyObjectsAccess authorizes a multi-object delete — the parsed contents +// of a DeleteObjects request body, passed straight through. It answers, for +// every object independently, whether policy allows deleting it and whether +// an object lock protects it, in a single pass. DeleteObjects supports +// partial success — unlike every other write path — so a denial on one +// object must not affect any other: the caller sends only the objects that +// pass through to the backend, and reports the rest as per-object errors +// straight from the returned slice. +// +// Both halves are deliberately here rather than split across the caller: the +// per-object work needs one loop, not one loop per concern at a different +// layer, and the expensive parts of each half — the bucket policy, the +// batched identity-policy round trip, the bucket's lock configuration — are +// resolved once up front for the whole request. +// +// Every key is authorized against its own object ARN, the way real AWS does +// it: a policy granting s3:DeleteObject on "arn:aws:s3:::bucket/*" and +// nothing else deletes successfully. An object named with a VersionId is +// authorized against s3:DeleteObjectVersion instead of s3:DeleteObject, the +// same split the single-object DELETE path already makes: a policy granting +// only s3:DeleteObject denies the versioned deletes in the same batch that +// its keyed deletes succeed under, and the batch's response reports that +// denial on just that object, the rest unaffected. +// +// The returned slice has one entry per object: nil where that object may +// proceed, an AWS-shaped denial otherwise. opts.Object and opts.Actions are +// both ignored in favor of objects. The second return is non-nil only for a +// failure that isn't about any one object — readonly mode, or an error +// resolving the bucket's policy or lock configuration — and fails the whole +// request, matching what a hard failure did before this returned per-object +// results at all. +func VerifyObjectsAccess(ctx fiber.Ctx, be backend.Backend, opts AccessOptions, objects []types.ObjectIdentifier, bypass BypassMode) ([]error, error) { + if err := verifyAccessGates(opts); err != nil { + return nil, err + } + if len(objects) == 0 { + return nil, nil + } + + rctx := ctx.RequestCtx() + condCtx := requestConditionContext(ctx) + + keys := make([]string, len(objects)) + for i, obj := range objects { + if obj.Key != nil { + keys[i] = *obj.Key + } + } + + errs := make([]error, len(objects)) + + // Authorization doesn't apply to root, admin, or a public-bucket + // request — errs stays all-nil from policy's perspective, and object + // locks still apply to them, so the loop below runs either way. + if authorizationApplies(opts) { + var plainIdx, versionedIdx []int + for i, obj := range objects { + if obj.VersionId != nil && *obj.VersionId != "" { + versionedIdx = append(versionedIdx, i) + } else { + plainIdx = append(plainIdx, i) + } + } + + if err := authorizeObjectSubset(rctx, be, opts, keys, plainIdx, DeleteObjectAction, errs, condCtx); err != nil { + return nil, err + } + if err := authorizeObjectSubset(rctx, be, opts, keys, versionedIdx, DeleteObjectVersionAction, errs, condCtx); err != nil { + return nil, err + } + } + + lockState, err := loadObjectLockState(rctx, be, opts.Bucket, false) + if err != nil { + return nil, err + } + if lockState.applies { + for i, obj := range objects { + if errs[i] != nil { + // Already denied by policy — no need to also resolve this + // object's lock state, and a lock error here would only + // overwrite the more specific policy denial. + continue + } + if err := lockState.checkObject(rctx, be, opts.Iam, opts.Acc, opts.Bucket, obj, bypass, opts.IsPublicRequest, condCtx); err != nil { + errs[i] = err + } + } + } + + return errs, nil +} + +// authorizeObjectSubset runs objectsAccessErrors for the objects at idx (a +// subset of keys, given by original index) against a single action, and +// scatters the results back into errs at their original positions. Splitting +// DeleteObjects' batch into one group per action this way keeps the +// round-trip count at one per distinct action in the batch — normally one or +// two — rather than one per object. +func authorizeObjectSubset(ctx context.Context, be backend.Backend, opts AccessOptions, keys []string, idx []int, action Action, errs []error, condCtx map[string][]string) error { + if len(idx) == 0 { + return nil + } + + subKeys := make([]string, len(idx)) + for i, origIdx := range idx { + subKeys[i] = keys[origIdx] + } + + subOpts := opts + subOpts.Actions = []Action{action} + subErrs, err := objectsAccessErrors(ctx, be, subOpts, subKeys, condCtx) + if err != nil { + return err + } + for i, origIdx := range idx { + errs[origIdx] = subErrs[i] + } + return nil +} + +// verifyAccessGates applies the checks that depend on gateway configuration +// rather than on the caller's policies. Readonly mode blocks writes for +// everyone, root and admin included, which is why it runs before any bypass. +func verifyAccessGates(opts AccessOptions) error { if opts.Readonly { if opts.AclPermission == PermissionWrite || opts.AclPermission == PermissionWriteAcp { return s3err.GetAPIError(s3err.ErrAccessDenied) } } - // Skip the access check for public bucket requests - if opts.IsPublicRequest { - return nil + return nil +} + +// authorizationApplies reports whether policy/ACL evaluation is meaningful +// for this caller at all. It is not for an anonymous request to a public +// bucket (already authorized by the public-access check) nor for root/admin +// (who bypass policy entirely — though not object locks). +func authorizationApplies(opts AccessOptions) bool { + return !opts.IsPublicRequest && !opts.IsRoot && opts.Acc.Role != RoleAdmin +} + +// objectsAccessErrors evaluates every key against the bucket's resource +// policy (or ACL) and the caller's identity policy, returning one result per +// key: nil where the key is authorized, and the AWS-shaped denial otherwise. +// The returned slice always has one entry per key. +// +// The keys are evaluated as one batch, not one VerifyAccess call each: the +// bucket policy is fetched once, and the identity policy is evaluated for +// every key in a single round trip to the IAM service. A per-key loop would +// cost a backend call and a network round trip per object, and DeleteObjects +// accepts up to 1000 of them. +func objectsAccessErrors(ctx context.Context, be backend.Backend, opts AccessOptions, keys []string, condCtx map[string][]string) ([]error, error) { + resourceDecisions, err := verifyResourceAccess(ctx, be, opts, keys, condCtx) + if err != nil { + return nil, err } - if opts.IsRoot { - return nil + + errs := make([]error, len(keys)) + + // An explicit deny from the bucket policy wins outright, whatever the + // IAM backend is, so a request carrying one needs no identity policy at + // all — which also saves the standalone IAM service round trip. Only the + // first denied key is recorded: the request fails there regardless of + // what the rest would have evaluated to. + for i, rd := range resourceDecisions { + if rd.Decision == policyDecisionDeny { + errs[i] = s3err.GetExplicitDenyAccessErr(opts.Acc.Access, string(rd.Action), objectPolicyArn(opts.Bucket, keys[i], be.NormalizeObjectKey), "a resource-based policy") + return errs, nil + } } - if opts.Acc.Role == RoleAdmin { - return nil + + pe, hasPolicyEvaluator := opts.Iam.(PolicyEvaluator) + if !hasPolicyEvaluator { + // No identity-policy layer exists for this backend at all: preserve + // today's exact behavior and generic message, unconditionally, for + // every internal/LDAP/Vault/IPA/S3-IAM deployment. + for i, rd := range resourceDecisions { + if rd.Decision != policyDecisionAllow { + errs[i] = s3err.GetAPIError(s3err.ErrAccessDenied) + } + } + return errs, nil } + identity, err := identityPolicyDecisions(pe, opts, keys, be.NormalizeObjectKey, condCtx) + if err != nil { + return nil, err + } + + principal := identity.PrincipalArn + if principal == "" { + principal = opts.Acc.Access + } + + for i := range keys { + resourceArn := objectPolicyArn(opts.Bucket, keys[i], be.NormalizeObjectKey) + + if identity.Decisions[i].Decision == policyDecisionDeny { + errs[i] = s3err.GetExplicitDenyAccessErr(principal, string(identity.Decisions[i].Action), resourceArn, "an identity-based policy") + continue + } + if identity.HasSessionPolicy && identity.SessionDecisions[i].Decision == policyDecisionDeny { + errs[i] = s3err.GetExplicitDenyAccessErr(principal, string(identity.SessionDecisions[i].Action), resourceArn, "an identity-based policy") + continue + } + + granted := resourceDecisions[i].Decision == policyDecisionAllow || + identity.Decisions[i].Decision == policyDecisionAllow + + // A session policy filters everything the session can do — including + // what the bucket policy granted it, not just what the role's own + // policies did. Confirmed against real AWS: a role with no identity + // policy at all, a bucket policy granting it both s3:GetObject and + // s3:PutObject, and a session policy allowing only s3:GetObject + // yields a successful Get and a denied Put. + if identity.HasSessionPolicy && identity.SessionDecisions[i].Decision != policyDecisionAllow { + granted = false + } + if granted { + continue + } + + blamedAction := resourceDecisions[i].Action + if blamedAction == "" { + blamedAction = identity.Decisions[i].Action + } + if blamedAction == "" && identity.HasSessionPolicy { + blamedAction = identity.SessionDecisions[i].Action + } + errs[i] = s3err.GetImplicitDenyAccessErr(principal, string(blamedAction), resourceArn) + } + + return errs, nil +} + +// decisionForResource is one resource's tri-state decision plus, for +// Deny/NoMatch, the specific action responsible — so the caller can build an +// AWS-shaped message naming it. +type decisionForResource struct { + Decision policyDecision + Action Action +} + +// verifyResourceAccess checks the bucket's own policy or, absent one, ACL, +// for each object key, returning one decision per key. The bucket policy is +// fetched once regardless of how many keys there are. ACL evaluation can +// only ever produce Allow/NoMatch — ACLs have no concept of an explicit +// deny — and applies to the whole bucket, so every key shares its verdict. +func verifyResourceAccess(ctx context.Context, be backend.Backend, opts AccessOptions, objects []string, condCtx map[string][]string) ([]decisionForResource, error) { + decisions := make([]decisionForResource, len(objects)) + policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket) if policyErr != nil { if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) { - return policyErr + return nil, policyErr } - } else { - return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, be.NormalizeObjectKey, opts.Actions...) + + decision := policyDecisionAllow + if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission, opts.DisableACL); err != nil { + decision = policyDecisionNoMatch + } + for i := range decisions { + decisions[i] = decisionForResource{Decision: decision} + } + return decisions, nil } - if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission, opts.DisableACL); err != nil { - return err + for i, object := range objects { + decision, action, err := verifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, object, condCtx, be.NormalizeObjectKey, opts.Actions...) + if err != nil { + return nil, err + } + decisions[i] = decisionForResource{Decision: decision, Action: action} + } + return decisions, nil +} + +// identityPolicyDecisions evaluates every action in opts.Actions against +// every object key, all in a single request, and aggregates each key's +// actions with the same precedence bucketPolicyDecision uses for a bucket +// policy: a Deny on any action wins immediately; otherwise Allow only if +// every action has a matching Allow; otherwise NoMatch, paired with the +// first action that lacked one. +// +// It returns one decision per key, plus the resolved principal ARN, which is +// shared across the whole batch since one call always evaluates a single +// identity. +func identityPolicyDecisions(pe PolicyEvaluator, opts AccessOptions, objects []string, normalizeObjectKey objectKeyNormalizer, condition map[string][]string) (identityDecisions, error) { + resources := make([]string, len(objects)) + for i, object := range objects { + resources[i] = objectPolicyArn(opts.Bucket, object, normalizeObjectKey) } - return nil + eval, err := pe.EvaluatePolicy(opts.Acc.Access, opts.Acc.SessionToken, opts.Actions, resources, condition) + if err != nil { + return identityDecisions{}, err + } + + decisions, err := aggregateActionDecisions(eval.Decisions, resources, opts.Actions) + if err != nil { + return identityDecisions{}, err + } + + result := identityDecisions{Decisions: decisions, PrincipalArn: eval.PrincipalArn} + if eval.HasSessionPolicy { + sessionDecisions, err := aggregateActionDecisions(eval.SessionDecisions, resources, opts.Actions) + if err != nil { + return identityDecisions{}, err + } + result.HasSessionPolicy = true + result.SessionDecisions = sessionDecisions + } + return result, nil +} + +// identityDecisions is identityPolicyDecisions' result: one aggregated +// decision per object from the caller's identity policies, the same from its +// session policy when it has one, and the resolved principal ARN. +type identityDecisions struct { + Decisions []decisionForResource + SessionDecisions []decisionForResource + HasSessionPolicy bool + PrincipalArn string +} + +// aggregateActionDecisions collapses each resource's per-action decisions +// into one, using the same precedence bucketPolicyDecision uses: a Deny on +// any action wins immediately; otherwise Allow only if every action has a +// matching Allow; otherwise NoMatch, paired with the first action that +// lacked one. +func aggregateActionDecisions(matrix [][]policyDecision, resources []string, actions []Action) ([]decisionForResource, error) { + if len(matrix) != len(resources) { + // A protocol mismatch between the gateway and IAM service builds — + // fail closed rather than authorizing a key nobody evaluated. + return nil, fmt.Errorf("evaluate policy returned %d resource decisions for %d resources", len(matrix), len(resources)) + } + + results := make([]decisionForResource, len(resources)) + for i, perAction := range matrix { + if len(perAction) != len(actions) { + return nil, fmt.Errorf("evaluate policy returned %d action decisions for %d actions", len(perAction), len(actions)) + } + + result := decisionForResource{Decision: policyDecisionAllow} + for j, decision := range perAction { + if decision == policyDecisionDeny { + result = decisionForResource{Decision: policyDecisionDeny, Action: actions[j]} + break + } + if decision == policyDecisionNoMatch && result.Decision != policyDecisionNoMatch { + result.Decision = policyDecisionNoMatch + result.Action = actions[j] + } + } + results[i] = result + } + return results, nil +} + +// objectPolicyArn builds the ARN a policy statement is matched against for +// one bucket/object pair — the bucket's own ARN when object is empty. +func objectPolicyArn(bucket, object string, normalizeObjectKey objectKeyNormalizer) string { + return ResourceArnPrefix + makePolicyResource(bucket, object, normalizeObjectKey) } // VerifyPublicAccess checks if the bucket is publically accessible by ACL or Policy -func VerifyPublicAccess(ctx context.Context, be backend.Backend, action Action, permission Permission, bucket, object string) error { +func VerifyPublicAccess(ctx fiber.Ctx, be backend.Backend, action Action, permission Permission, bucket, object string) error { // ACL disabled - policy, err := be.GetBucketPolicy(ctx, bucket) + policy, err := be.GetBucketPolicy(ctx.RequestCtx(), bucket) if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) { return err } if err == nil { - err = VerifyPublicBucketPolicy(policy, bucket, object, be.NormalizeObjectKey, action) + err = VerifyPublicBucketPolicy(policy, bucket, object, requestConditionContext(ctx), be.NormalizeObjectKey, action) if errors.Is(err, errExplicitDeny) { // Explicit public-policy Deny has higher precedence than any // public ACL grant, so do not continue to ACL fallback. @@ -160,7 +524,7 @@ func VerifyPublicAccess(ctx context.Context, be backend.Backend, action Action, return s3err.GetAPIError(s3err.ErrAccessDenied) } - err = VerifyPublicBucketACL(ctx, be, bucket, action, permission) + err = VerifyPublicBucketACL(ctx.RequestCtx(), be, bucket, action, permission) if err != nil { return s3err.GetAPIError(s3err.ErrAccessDenied) } @@ -168,6 +532,67 @@ func VerifyPublicAccess(ctx context.Context, be backend.Backend, action Action, return nil } +// VerifyCreateBucketAccess decides whether acc may create a bucket named +// bucket. Unlike VerifyAccess, the bucket doesn't exist yet at this point, +// so there is no bucket policy or ACL to consult — root/admin always +// bypass, and otherwise authorization comes from whichever mechanism the +// configured iam backend actually supports: for backends that implement +// PolicyEvaluator (currently only the standalone IAM service client), an +// identity-policy Allow for s3:CreateBucket grants access, exactly like any +// other IAM-policy-gated action; the legacy userplus-role bypass applies +// only to backends with no such policy layer (internal/LDAP/Vault/IPA/S3-IAM), +// since those have no other way to grant a plain "user" account this +// permission. +func VerifyCreateBucketAccess(ctx fiber.Ctx, iam IAMService, isRoot bool, acc Account, bucket string) error { + if isRoot || acc.Role == RoleAdmin { + return nil + } + + pe, hasPolicyEvaluator := iam.(PolicyEvaluator) + if !hasPolicyEvaluator { + if acc.Role == RoleUserPlus { + return nil + } + return s3err.GetAPIError(s3err.ErrAccessDenied) + } + + resourceArn := ResourceArnPrefix + bucket + identity, err := identityPolicyDecisions(pe, AccessOptions{ + Acc: acc, + Bucket: bucket, + Actions: []Action{CreateBucketAction}, + }, []string{""}, nil, requestConditionContext(ctx)) + if err != nil { + return err + } + + principal := identity.PrincipalArn + if principal == "" { + principal = acc.Access + } + + // A session policy narrows what the session may do; there is no resource + // policy for a bucket that does not exist yet, so the two decisions + // simply intersect here. + decision := identity.Decisions[0].Decision + if identity.HasSessionPolicy { + switch sd := identity.SessionDecisions[0].Decision; { + case sd == policyDecisionDeny: + decision = policyDecisionDeny + case sd != policyDecisionAllow && decision == policyDecisionAllow: + decision = policyDecisionNoMatch + } + } + + switch decision { + case policyDecisionDeny: + return s3err.GetExplicitDenyAccessErr(principal, string(CreateBucketAction), resourceArn, "an identity-based policy") + case policyDecisionAllow: + return nil + } + return s3err.GetImplicitDenyAccessErr(principal, string(CreateBucketAction), resourceArn) +} + func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error { // Owner check if acct.Access == acl.Owner { diff --git a/auth/access-control_test.go b/auth/access-control_test.go index 14de895b..248566ab 100644 --- a/auth/access-control_test.go +++ b/auth/access-control_test.go @@ -18,16 +18,32 @@ import ( "context" "encoding/json" "errors" + "net/http" "path/filepath" "testing" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v3" "github.com/stretchr/testify/assert" + "github.com/valyala/fasthttp" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/s3err" ) +// testFiberCtx returns a fiber.Ctx for tests to pass to functions that read +// request-derived data (e.g. the condition context) off it, released +// automatically when the test ends. +func testFiberCtx(t *testing.T) fiber.Ctx { + t.Helper() + app := fiber.New() + ctx := app.AcquireCtx(&fasthttp.RequestCtx{}) + t.Cleanup(func() { + app.ReleaseCtx(ctx) + }) + return ctx +} + // noBucketPolicyBackend is a test stub that returns ErrNoSuchBucketPolicy for // GetBucketPolicy and serves a configurable ACL for GetBucketAcl. type noBucketPolicyBackend struct { @@ -94,6 +110,264 @@ func publicReadACL() ACL { } } +// mockPolicyEvaluator implements IAMService (via the embedded +// IAMServiceSingle, whose methods are never exercised here) and +// PolicyEvaluator, recording every EvaluatePolicy call so tests can assert +// both the outcome and exactly what VerifyAccess asked it to evaluate. +type mockPolicyEvaluator struct { + IAMService + decision policyDecision + principalArn string + err error + calls []evaluatePolicyCall +} + +type evaluatePolicyCall struct { + access, sessionToken string + resources []string + actions []Action + condition map[string][]string +} + +func (m *mockPolicyEvaluator) EvaluatePolicy(access, sessionToken string, actions []Action, resources []string, condition map[string][]string) (PolicyEvaluation, error) { + m.calls = append(m.calls, evaluatePolicyCall{ + access: access, + sessionToken: sessionToken, + actions: actions, + resources: resources, + condition: condition, + }) + decisions := make([][]policyDecision, len(resources)) + for i := range resources { + decisions[i] = make([]policyDecision, len(actions)) + for j := range actions { + decisions[i][j] = m.decision + } + } + return PolicyEvaluation{Decisions: decisions, PrincipalArn: m.principalArn}, m.err +} + +func newMockPolicyEvaluator(decision policyDecision) *mockPolicyEvaluator { + return &mockPolicyEvaluator{IAMService: NewIAMServiceSingle(Account{}), decision: decision} +} + +// requireAccessDeniedAPIError asserts err is an s3err.APIError with the AWS +// AccessDenied shape (Code, HTTP 403) and returns it for the caller to +// inspect the dynamic Description text further. +func requireAccessDeniedAPIError(t *testing.T, err error) s3err.APIError { + t.Helper() + apiErr, ok := err.(s3err.APIError) + if !ok { + t.Fatalf("err = %#v (%T), want s3err.APIError", err, err) + } + assert.Equal(t, "AccessDenied", apiErr.Code) + assert.Equal(t, http.StatusForbidden, apiErr.HTTPStatusCode) + return apiErr +} + +// TestVerifyAccess_ResourceAllowStillChecksIdentityForExplicitDeny confirms +// the fix for the core bug: a bucket policy Allow used to short-circuit +// before the identity-policy layer was ever consulted, so an identity +// policy's explicit Deny was silently ignored whenever the bucket policy +// already allowed. Now the identity policy is always consulted too — here +// it has no opinion (NoMatch), so the bucket policy's Allow still stands, +// but EvaluatePolicy must actually have been called for that to be a real +// verdict rather than a skipped check. +func TestVerifyAccess_ResourceAllowStillChecksIdentityForExplicitDeny(t *testing.T) { + be := &publicBucketPolicyBackend{ + policy: []byte(`{ + "Statement": [{ + "Effect": "Allow", + "Principal": "testuser", + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::bucket/*" + }] + }`), + } + pe := newMockPolicyEvaluator(policyDecisionNoMatch) + + err := VerifyAccess(testFiberCtx(t), be, AccessOptions{ + Acc: Account{Access: "testuser", Role: RoleUser}, + Bucket: "bucket", + Object: "key.txt", + Actions: []Action{GetObjectAction}, + Iam: pe, + }) + + assert.NoError(t, err) + assert.Len(t, pe.calls, 1, "EvaluatePolicy must now be called even when the resource-level check already allows, so an explicit identity-policy Deny can still override it") +} + +// TestVerifyAccess_IdentityExplicitDenyOverridesResourceAllow is the +// explicit-deny-wins fix: a bucket policy Allow does not save a request the +// caller's own identity policy explicitly denies. The Message names the +// resolved principal ARN and calls out "an identity-based policy" — +// matching what real AWS returns for this case. +func TestVerifyAccess_IdentityExplicitDenyOverridesResourceAllow(t *testing.T) { + be := &publicBucketPolicyBackend{ + policy: []byte(`{ + "Statement": [{ + "Effect": "Allow", + "Principal": "testuser", + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::bucket/*" + }] + }`), + } + pe := newMockPolicyEvaluator(policyDecisionDeny) + pe.principalArn = "arn:aws:iam::000000000000:user/testuser" + + err := VerifyAccess(testFiberCtx(t), be, AccessOptions{ + Acc: Account{Access: "testuser", Role: RoleUser}, + Bucket: "bucket", + Object: "key.txt", + Actions: []Action{GetObjectAction}, + Iam: pe, + }) + + apiErr := requireAccessDeniedAPIError(t, err) + assert.Contains(t, apiErr.Description, "arn:aws:iam::000000000000:user/testuser") + assert.Contains(t, apiErr.Description, "s3:GetObject") + assert.Contains(t, apiErr.Description, "with an explicit deny in an identity-based policy") +} + +// TestVerifyAccess_ResourceExplicitDenyOverridesIdentityAllow is the +// reverse case: an identity policy Allow does not save a request the +// bucket policy explicitly denies. The resource-level Deny short-circuits +// before the identity policy is even consulted (it can't change the +// outcome, and it saves the standalone IAM service round trip), and the +// Message calls out "a resource-based policy". +func TestVerifyAccess_ResourceExplicitDenyOverridesIdentityAllow(t *testing.T) { + be := &publicBucketPolicyBackend{ + policy: []byte(`{ + "Statement": [{ + "Effect": "Deny", + "Principal": "testuser", + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::bucket/*" + }] + }`), + } + pe := newMockPolicyEvaluator(policyDecisionAllow) + + err := VerifyAccess(testFiberCtx(t), be, AccessOptions{ + Acc: Account{Access: "testuser", Role: RoleUser}, + Bucket: "bucket", + Object: "key.txt", + Actions: []Action{GetObjectAction}, + Iam: pe, + }) + + apiErr := requireAccessDeniedAPIError(t, err) + assert.Contains(t, apiErr.Description, "testuser") + assert.Contains(t, apiErr.Description, "with an explicit deny in a resource-based policy") + assert.Empty(t, pe.calls, "a resource-level explicit deny should short-circuit before consulting the identity policy") +} + +// TestVerifyAccess_IdentityPolicyAllowsWhenResourceDenies is the core +// same-account fix: a private bucket with no ACL grant and no bucket policy +// still allows access when the caller's IAM identity policy grants it — +// matching real AWS, where a bucket policy is only *required* for +// cross-account access; within the same account (this gateway is always +// single-account) an identity-based Allow alone is sufficient. +func TestVerifyAccess_IdentityPolicyAllowsWhenResourceDenies(t *testing.T) { + be := noBucketPolicyBackend{srcAcl: ACL{Owner: "someone-else"}} + pe := newMockPolicyEvaluator(policyDecisionAllow) + + err := VerifyAccess(testFiberCtx(t), be, AccessOptions{ + Acc: Account{Access: "testuser", Role: RoleUser}, + Bucket: "bucket", + Object: "key.txt", + Actions: []Action{GetObjectAction}, + AclPermission: PermissionRead, + Iam: pe, + }) + + assert.NoError(t, err) + assert.Len(t, pe.calls, 1) + assert.Equal(t, []string{"arn:aws:s3:::bucket/key.txt"}, pe.calls[0].resources) + assert.Equal(t, []Action{GetObjectAction}, pe.calls[0].actions) + assert.Equal(t, "testuser", pe.calls[0].access) +} + +// TestVerifyAccess_DeniedWhenNeitherResourceNorIdentityPolicyAllows confirms +// access is denied — with the AWS-shaped implicit-deny message, since a +// PolicyEvaluator is configured — when neither the resource-level check +// (ACL owned by someone else, no bucket policy) nor the identity policy has +// any opinion at all (NoMatch, not an explicit Deny from either side). It +// also pins that the message names the resolved principal ARN, not the +// access key — matching real AWS's implicit-deny message shape (previously +// this fell back to the access key even when the PolicyEvaluator resolved +// an ARN, since identityPolicyDecision only threaded PrincipalArn through +// on its Deny branch). +func TestVerifyAccess_DeniedWhenNeitherResourceNorIdentityPolicyAllows(t *testing.T) { + be := noBucketPolicyBackend{srcAcl: ACL{Owner: "someone-else"}} + pe := newMockPolicyEvaluator(policyDecisionNoMatch) + pe.principalArn = "arn:aws:iam::000000000000:user/testuser" + + err := VerifyAccess(testFiberCtx(t), be, AccessOptions{ + Acc: Account{Access: "testuser", Role: RoleUser}, + Bucket: "bucket", + Object: "key.txt", + Actions: []Action{GetObjectAction}, + AclPermission: PermissionRead, + Iam: pe, + }) + + apiErr := requireAccessDeniedAPIError(t, err) + assert.Contains(t, apiErr.Description, "arn:aws:iam::000000000000:user/testuser") + assert.Contains(t, apiErr.Description, "because no identity-based policy allows the s3:GetObject action") + assert.Len(t, pe.calls, 1) +} + +// TestVerifyAccess_NoPolicyEvaluatorIsANoOp confirms backends that don't +// implement PolicyEvaluator (every backend except the standalone IAM +// client) are entirely unaffected by this layer — backward compatibility +// via the type assertion, not a config flag. +func TestVerifyAccess_NoPolicyEvaluatorIsANoOp(t *testing.T) { + be := &publicBucketPolicyBackend{ + policy: []byte(`{ + "Statement": [{ + "Effect": "Allow", + "Principal": "testuser", + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::bucket/*" + }] + }`), + } + + err := VerifyAccess(testFiberCtx(t), be, AccessOptions{ + Acc: Account{Access: "testuser", Role: RoleUser}, + Bucket: "bucket", + Object: "key.txt", + Actions: []Action{GetObjectAction}, + Iam: NewIAMServiceSingle(Account{}), + }) + + assert.NoError(t, err) +} + +// TestVerifyAccess_NoPolicyEvaluatorDeniedKeepsGenericMessage pins that, +// with no PolicyEvaluator configured, a denied request's error stays +// byte-for-byte today's generic message — the dynamic AWS-shaped messages +// above only ever appear once a PolicyEvaluator is actually in play, so +// every internal/LDAP/Vault/IPA/S3-IAM deployment sees no message change +// from this fix at all. +func TestVerifyAccess_NoPolicyEvaluatorDeniedKeepsGenericMessage(t *testing.T) { + be := noBucketPolicyBackend{srcAcl: ACL{Owner: "someone-else"}} + + err := VerifyAccess(testFiberCtx(t), be, AccessOptions{ + Acc: Account{Access: "testuser", Role: RoleUser}, + Bucket: "bucket", + Object: "key.txt", + Actions: []Action{GetObjectAction}, + AclPermission: PermissionRead, + Iam: NewIAMServiceSingle(Account{}), + }) + + assert.Equal(t, s3err.GetAPIError(s3err.ErrAccessDenied), err) +} + func TestVerifyAccess_NormalizesObjectKeyBeforePolicyMatch(t *testing.T) { be := &publicBucketPolicyBackend{ normalizeFn: testNormalizeObjectKey, @@ -107,7 +381,7 @@ func TestVerifyAccess_NormalizesObjectKeyBeforePolicyMatch(t *testing.T) { }`), } - err := VerifyAccess(context.Background(), be, AccessOptions{ + err := VerifyAccess(testFiberCtx(t), be, AccessOptions{ Acc: Account{Access: "testuser", Role: RoleUser}, Bucket: "bucket", Object: "public/../private.txt", @@ -131,7 +405,7 @@ func TestVerifyAccess_NormalizesPolicyResourceBeforeMatch(t *testing.T) { }`), } - err := VerifyAccess(context.Background(), be, AccessOptions{ + err := VerifyAccess(testFiberCtx(t), be, AccessOptions{ Acc: Account{Access: "testuser", Role: RoleUser}, Bucket: "bucket", Object: "private.txt", @@ -154,7 +428,7 @@ func TestVerifyPublicAccess_PublicPolicyDenyStopsACLFallback(t *testing.T) { acl: publicReadACL(), } - err := VerifyPublicAccess(context.Background(), be, GetObjectAction, PermissionRead, "bucket", "private/secret.txt") + err := VerifyPublicAccess(testFiberCtx(t), be, GetObjectAction, PermissionRead, "bucket", "private/secret.txt") assert.Error(t, err) assert.True(t, errors.Is(err, s3err.GetAPIError(s3err.ErrAccessDenied))) @@ -174,7 +448,7 @@ func TestVerifyPublicAccess_PublicPolicyNoMatchFallsBackToACL(t *testing.T) { acl: publicReadACL(), } - err := VerifyPublicAccess(context.Background(), be, GetObjectAction, PermissionRead, "bucket", "public/object.txt") + err := VerifyPublicAccess(testFiberCtx(t), be, GetObjectAction, PermissionRead, "bucket", "public/object.txt") assert.NoError(t, err) assert.Equal(t, 1, be.aclCalls) @@ -194,7 +468,7 @@ func TestVerifyPublicAccess_NormalizedDenyStopsACLFallback(t *testing.T) { acl: publicReadACL(), } - err := VerifyPublicAccess(context.Background(), be, GetObjectAction, PermissionRead, "bucket", "public/../private/secret.txt") + err := VerifyPublicAccess(testFiberCtx(t), be, GetObjectAction, PermissionRead, "bucket", "public/../private/secret.txt") assert.Error(t, err) assert.True(t, errors.Is(err, s3err.GetAPIError(s3err.ErrAccessDenied))) @@ -204,21 +478,15 @@ func TestVerifyPublicAccess_NormalizedDenyStopsACLFallback(t *testing.T) { func TestVerifyObjectCopyAccess_URLEncodedSlashSeparator(t *testing.T) { const testUser = "testuser" - // Source bucket ACL: grants READ to testUser. - srcAcl := ACL{ - Owner: "owner", - Grantees: []Grantee{ - { - Access: testUser, - Permission: PermissionRead, - Type: types.TypeCanonicalUser, - }, - }, - } + // Source and destination bucket ACLs: testUser owns both. opts sets + // DisableACL, which now applies uniformly to the source-bucket check + // VerifyObjectCopyAccess performs internally as well as the + // destination's, collapsing both to an owner-only check — a grantee + // entry alone (without ownership) would no longer be sufficient. + srcAcl := ACL{Owner: testUser} be := noBucketPolicyBackend{srcAcl: srcAcl} - // Destination bucket ACL: testUser is the owner (DisableACL=true path). opts := AccessOptions{ Acl: ACL{Owner: testUser}, AclPermission: PermissionWrite, @@ -249,7 +517,7 @@ func TestVerifyObjectCopyAccess_URLEncodedSlashSeparator(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := VerifyObjectCopyAccess(context.Background(), be, tt.copySource, opts) + err := VerifyObjectCopyAccess(testFiberCtx(t), be, tt.copySource, opts) assert.NoError(t, err, "should accept %%2F as the bucket/key separator in x-amz-copy-source") }) @@ -259,16 +527,10 @@ func TestVerifyObjectCopyAccess_URLEncodedSlashSeparator(t *testing.T) { func TestVerifyObjectCopyAccess_LiteralSlashSeparator(t *testing.T) { const testUser = "testuser" - srcAcl := ACL{ - Owner: "owner", - Grantees: []Grantee{ - { - Access: testUser, - Permission: PermissionRead, - Type: types.TypeCanonicalUser, - }, - }, - } + // testUser owns both source and destination buckets — see the comment + // in TestVerifyObjectCopyAccess_URLEncodedSlashSeparator on why + // DisableACL requires ownership here rather than a grantee entry. + srcAcl := ACL{Owner: testUser} be := noBucketPolicyBackend{srcAcl: srcAcl} @@ -282,6 +544,180 @@ func TestVerifyObjectCopyAccess_LiteralSlashSeparator(t *testing.T) { DisableACL: true, } - err := VerifyObjectCopyAccess(context.Background(), be, "src-bucket/src-key", opts) + err := VerifyObjectCopyAccess(testFiberCtx(t), be, "src-bucket/src-key", opts) assert.NoError(t, err, "literal slash separator should work") } + +// TestVerifyCreateBucketAccess_RootAndAdminBypass confirms root and admin +// accounts may always create a bucket, with no iam backend consulted at +// all — CreateBucket has no existing bucket to check a policy or ACL +// against, so this bypass (unlike VerifyAccess's, which still runs the +// resource-policy check first) is the entire decision. +func TestVerifyCreateBucketAccess_RootAndAdminBypass(t *testing.T) { + err := VerifyCreateBucketAccess(testFiberCtx(t), NewIAMServiceSingle(Account{}), true, Account{Access: "testuser", Role: RoleUser}, "bucket") + assert.NoError(t, err) + + err = VerifyCreateBucketAccess(testFiberCtx(t), NewIAMServiceSingle(Account{}), false, Account{Access: "testuser", Role: RoleAdmin}, "bucket") + assert.NoError(t, err) +} + +// TestVerifyCreateBucketAccess_NoPolicyEvaluatorUsesLegacyRoleGate confirms +// that for every backend without an identity-policy layer (internal, LDAP, +// Vault, IPA, S3-IAM) bucket creation keeps working exactly as it always +// has: userplus is allowed, a plain user is denied with the generic +// AccessDenied error, and EvaluatePolicy is never a factor since these +// backends don't implement PolicyEvaluator at all. +func TestVerifyCreateBucketAccess_NoPolicyEvaluatorUsesLegacyRoleGate(t *testing.T) { + iam := NewIAMServiceSingle(Account{}) + + err := VerifyCreateBucketAccess(testFiberCtx(t), iam, false, Account{Access: "testuser", Role: RoleUserPlus}, "bucket") + assert.NoError(t, err) + + err = VerifyCreateBucketAccess(testFiberCtx(t), iam, false, Account{Access: "testuser", Role: RoleUser}, "bucket") + assert.Equal(t, s3err.GetAPIError(s3err.ErrAccessDenied), err) +} + +// TestVerifyCreateBucketAccess_PolicyEvaluatorAllow confirms the core fix: +// a standalone-IAM-service user, who is always Role RoleUser regardless of +// their attached IAM policy, can create a bucket when that policy grants +// s3:CreateBucket — the identity-policy Allow is what grants access, not +// the role. +func TestVerifyCreateBucketAccess_PolicyEvaluatorAllow(t *testing.T) { + pe := newMockPolicyEvaluator(policyDecisionAllow) + + err := VerifyCreateBucketAccess(testFiberCtx(t), pe, false, Account{Access: "testuser", Role: RoleUser}, "bucket") + + assert.NoError(t, err) + assert.Len(t, pe.calls, 1) + assert.Equal(t, "testuser", pe.calls[0].access) + assert.Equal(t, []string{"arn:aws:s3:::bucket"}, pe.calls[0].resources) + assert.Equal(t, []Action{CreateBucketAction}, pe.calls[0].actions) +} + +// TestVerifyCreateBucketAccess_PolicyEvaluatorNoMatchDenies confirms a +// standalone-IAM-service user with no policy granting s3:CreateBucket is +// denied — with the AWS-shaped implicit-deny message — even though the +// legacy role gate alone would have denied them anyway; this pins that the +// policy layer, not the role, is now what's actually being asked. +func TestVerifyCreateBucketAccess_PolicyEvaluatorNoMatchDenies(t *testing.T) { + pe := newMockPolicyEvaluator(policyDecisionNoMatch) + pe.principalArn = "arn:aws:iam::000000000000:user/testuser" + + err := VerifyCreateBucketAccess(testFiberCtx(t), pe, false, Account{Access: "testuser", Role: RoleUser}, "bucket") + + apiErr := requireAccessDeniedAPIError(t, err) + assert.Contains(t, apiErr.Description, "arn:aws:iam::000000000000:user/testuser") + assert.Contains(t, apiErr.Description, "s3:CreateBucket") + assert.Contains(t, apiErr.Description, "because no identity-based policy allows the s3:CreateBucket action") +} + +// TestVerifyCreateBucketAccess_PolicyEvaluatorExplicitDenyWins confirms an +// explicit Deny in the identity policy is reported with the AWS-shaped +// explicit-deny message, naming the resolved principal ARN when the +// PolicyEvaluator reports one. +func TestVerifyCreateBucketAccess_PolicyEvaluatorExplicitDenyWins(t *testing.T) { + pe := newMockPolicyEvaluator(policyDecisionDeny) + pe.principalArn = "arn:aws:iam::000000000000:user/testuser" + + err := VerifyCreateBucketAccess(testFiberCtx(t), pe, false, Account{Access: "testuser", Role: RoleUser}, "bucket") + + apiErr := requireAccessDeniedAPIError(t, err) + assert.Contains(t, apiErr.Description, "arn:aws:iam::000000000000:user/testuser") + assert.Contains(t, apiErr.Description, "s3:CreateBucket") + assert.Contains(t, apiErr.Description, "with an explicit deny in an identity-based policy") +} + +// TestVerifyCreateBucketAccess_PolicyEvaluatorIgnoresUserPlus confirms the +// legacy userplus bypass does not leak into the PolicyEvaluator path: once +// a backend implements identity-policy evaluation, that policy is the sole +// gate for non-admin accounts, matching the standalone IAM service's real +// behavior (its accounts are always Role RoleUser, never RoleUserPlus, so +// this also documents why the bypass would be a no-op there in practice). +func TestVerifyCreateBucketAccess_PolicyEvaluatorIgnoresUserPlus(t *testing.T) { + pe := newMockPolicyEvaluator(policyDecisionNoMatch) + + err := VerifyCreateBucketAccess(testFiberCtx(t), pe, false, Account{Access: "testuser", Role: RoleUserPlus}, "bucket") + + assert.Error(t, err) + assert.Len(t, pe.calls, 1, "EvaluatePolicy must be consulted even for a userplus account once a PolicyEvaluator is configured") +} + +// noObjectLockBackend answers "no lock configuration" for +// GetObjectLockConfiguration, so VerifyObjectsAccess's lock check is a no-op +// and only the policy/ACL half of the result is under test — matching what +// loadObjectLockState treats as "object lock was never configured on this +// bucket", not the BackendUnsupported stub's ErrNotImplemented, which would +// otherwise fail the whole request before either object was authorized. +type noObjectLockBackend struct { + noBucketPolicyBackend +} + +func (b noObjectLockBackend) GetObjectLockConfiguration(_ context.Context, _ string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound) +} + +// actionSplitPolicyEvaluator denies exactly one action and allows every +// other, recording each EvaluatePolicy call it receives — for asserting not +// just the outcome but that DeleteObjects' mixed batch was split into one +// call per action rather than evaluated as a single undifferentiated batch. +type actionSplitPolicyEvaluator struct { + IAMService + denyAction Action + calls []evaluatePolicyCall +} + +func (m *actionSplitPolicyEvaluator) EvaluatePolicy(access, sessionToken string, actions []Action, resources []string, condition map[string][]string) (PolicyEvaluation, error) { + m.calls = append(m.calls, evaluatePolicyCall{ + access: access, + sessionToken: sessionToken, + actions: actions, + resources: resources, + condition: condition, + }) + decisions := make([][]policyDecision, len(resources)) + for i := range resources { + decisions[i] = make([]policyDecision, len(actions)) + for j, a := range actions { + if a == m.denyAction { + decisions[i][j] = policyDecisionNoMatch + } else { + decisions[i][j] = policyDecisionAllow + } + } + } + return PolicyEvaluation{Decisions: decisions}, nil +} + +func TestVerifyObjectsAccess_VersionedDeleteNeedsSeparatePermission(t *testing.T) { + be := noObjectLockBackend{noBucketPolicyBackend{srcAcl: ACL{Owner: "someone-else"}}} + pe := &actionSplitPolicyEvaluator{denyAction: DeleteObjectVersionAction} + + objects := []types.ObjectIdentifier{ + {Key: strPtr("plain.txt")}, + {Key: strPtr("versioned.txt"), VersionId: strPtr("v1")}, + } + + errs, err := VerifyObjectsAccess(testFiberCtx(t), be, AccessOptions{ + Acc: Account{Access: "testuser", Role: RoleUser}, + Bucket: "bucket", + AclPermission: PermissionWrite, + Iam: pe, + }, objects, BypassNone) + + assert.NoError(t, err) + if assert.Len(t, errs, 2) { + assert.NoError(t, errs[0], "the keyed delete should be authorized against s3:DeleteObject, which is allowed") + apiErr := requireAccessDeniedAPIError(t, errs[1]) + assert.Contains(t, apiErr.Description, "s3:DeleteObjectVersion") + assert.Contains(t, apiErr.Description, "because no identity-based policy allows the s3:DeleteObjectVersion action") + } + + if assert.Len(t, pe.calls, 2, "the batch should split into one EvaluatePolicy call per distinct action") { + assert.Equal(t, []Action{DeleteObjectAction}, pe.calls[0].actions) + assert.Equal(t, []string{"arn:aws:s3:::bucket/plain.txt"}, pe.calls[0].resources) + assert.Equal(t, []Action{DeleteObjectVersionAction}, pe.calls[1].actions) + assert.Equal(t, []string{"arn:aws:s3:::bucket/versioned.txt"}, pe.calls[1].resources) + } +} + +func strPtr(s string) *string { return &s } diff --git a/auth/acl.go b/auth/acl.go index 807efbc7..60dfc6b3 100644 --- a/auth/acl.go +++ b/auth/acl.go @@ -18,7 +18,6 @@ import ( "context" "encoding/json" "encoding/xml" - "errors" "fmt" "strings" @@ -361,7 +360,7 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService) ([]byte, error } // Check if the specified accounts exist - accList, err := CheckIfAccountsExist(accs, iam) + accList, err := iam.ResolveAccounts(accs) if err != nil { return nil, err } @@ -380,25 +379,6 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService) ([]byte, error return result, nil } -func CheckIfAccountsExist(accs []string, iam IAMService) ([]string, error) { - result := []string{} - - for _, acc := range accs { - _, err := iam.GetUserAccount(acc) - if err != nil { - if err == ErrNoSuchUser || err == s3err.GetAPIError(s3err.ErrAdminUserNotFound) { - result = append(result, acc) - continue - } - if errors.Is(err, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)) { - return nil, err - } - return nil, fmt.Errorf("check user account: %w", err) - } - } - return result, nil -} - func splitUnique(s, divider string) []string { elements := strings.Split(s, divider) uniqueElements := make(map[string]bool) diff --git a/auth/bucket_policy.go b/auth/bucket_policy.go index 7c463ad5..5f0d2041 100644 --- a/auth/bucket_policy.go +++ b/auth/bucket_policy.go @@ -20,15 +20,19 @@ import ( "fmt" "net/http" + "github.com/versity/versitygw/internal/condition" "github.com/versity/versitygw/s3err" ) var errAccessDenied = errors.New("access denied") var errExplicitDeny = errors.New("explicit deny") -// policyDecision preserves the difference between "not allowed" and "denied". -// Public bucket authorization needs that distinction so no-match can fall back -// to ACLs while explicit Deny cannot. +// policyDecision preserves the difference between "not allowed" and +// "denied". Public bucket authorization needs that distinction so no-match +// can fall back to ACLs while explicit Deny cannot; VerifyAccess needs it to +// combine a bucket policy's decision with an identity policy's own — an +// explicit Deny from either source must override an Allow from the other, +// which a plain bool can't express. type policyDecision int const ( @@ -44,15 +48,18 @@ func (p policyErr) Error() string { } const ( - policyErrResourceMismatch = policyErr("Action does not apply to any resource(s) in statement") - policyErrInvalidResource = policyErr("Policy has invalid resource") - policyErrInvalidPrincipal = policyErr("Invalid principal in policy") - policyErrInvalidAction = policyErr("Policy has invalid action") - policyErrInvalidPolicy = policyErr("This policy contains invalid Json") - policyErrInvalidFirstChar = policyErr("Policies must be valid JSON and the first byte must be '{'") - policyErrEmptyStatement = policyErr("Could not parse the policy: Statement is empty!") - policyErrMissingStatmentField = policyErr("Missing required field Statement") - policyErrInvalidVersion = policyErr("The policy must contain a valid version string") + policyErrResourceMismatch = policyErr("Action does not apply to any resource(s) in statement") + policyErrInvalidResource = policyErr("Policy has invalid resource") + policyErrInvalidPrincipal = policyErr("Invalid principal in policy") + policyErrInvalidAction = policyErr("Policy has invalid action") + policyErrInvalidPolicy = policyErr("This policy contains invalid Json") + policyErrInvalidFirstChar = policyErr("Policies must be valid JSON and the first byte must be '{'") + policyErrEmptyStatement = policyErr("Could not parse the policy: Statement is empty!") + policyErrMissingStatmentField = policyErr("Missing required field Statement") + policyErrInvalidVersion = policyErr("The policy must contain a valid version string") + policyErrInvalidConditionKey = policyErr("Policy has an invalid condition key") + policyErrConditionActionMismatch = policyErr("Conditions do not apply to combination of actions and resources in statement") + policyErrInvalidIPCondition = policyErr("Invalid IP address in Conditions") ) type BucketPolicy struct { @@ -103,32 +110,58 @@ func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error { return nil } -func (bp *BucketPolicy) isAllowed(principal string, action Action, resource string, normalizeObjectKey objectKeyNormalizer) bool { +// decisionFor evaluates a single action against bp for principal/resource, +// returning the tri-state policyDecision. A statement whose principal/action/resource +// otherwise matches but whose Condition block can't be evaluated +// denies the whole decision immediately, regardless of that statement's own +// Effect — the same "can't rule out a hidden Deny" fail-closed contract +// iamapi/policy.EvaluateIdentityPolicies uses for identity policies, +// enforced per-statement here instead of per-document. In practice this +// branch is unreachable for any policy PutBucketPolicy accepted after +// Condition write-time validation existed — it only guards a document +// stored before that validation existed, or naming a future operator the +// gateway doesn't yet recognize. +func (bp *BucketPolicy) decisionFor(principal string, action Action, resource string, condCtx map[string][]string, normalizeObjectKey objectKeyNormalizer) policyDecision { var isAllowed bool for _, statement := range bp.Statement { - if statement.findMatch(principal, action, resource, normalizeObjectKey) { - switch statement.Effect { - case BucketPolicyAccessTypeAllow: - isAllowed = true - case BucketPolicyAccessTypeDeny: - return false - } + matched, evaluable := statement.findMatch(principal, action, resource, condCtx, bp.Version, normalizeObjectKey) + if !evaluable { + return policyDecisionDeny + } + if !matched { + continue + } + switch statement.Effect { + case BucketPolicyAccessTypeAllow: + isAllowed = true + case BucketPolicyAccessTypeDeny: + return policyDecisionDeny } } - return isAllowed + if isAllowed { + return policyDecisionAllow + } + return policyDecisionNoMatch } -func (bp *BucketPolicy) publicDecisionFor(resource string, action Action, normalizeObjectKey objectKeyNormalizer) policyDecision { +// publicDecisionFor mirrors decisionFor for the anonymous/public-bucket-access +// path +func (bp *BucketPolicy) publicDecisionFor(resource string, action Action, condCtx map[string][]string, normalizeObjectKey objectKeyNormalizer) policyDecision { var isAllowed bool for _, statement := range bp.Statement { - if statement.isPublicFor(resource, action, normalizeObjectKey) { - switch statement.Effect { - case BucketPolicyAccessTypeAllow: - isAllowed = true - case BucketPolicyAccessTypeDeny: - return policyDecisionDeny - } + matched, evaluable := statement.isPublicFor(resource, action, condCtx, bp.Version, normalizeObjectKey) + if !evaluable { + return policyDecisionDeny + } + if !matched { + continue + } + switch statement.Effect { + case BucketPolicyAccessTypeAllow: + isAllowed = true + case BucketPolicyAccessTypeDeny: + return policyDecisionDeny } } @@ -156,6 +189,7 @@ type BucketPolicyItem struct { Principals Principals `json:"Principal"` Actions Actions `json:"Action"` Resources Resources `json:"Resource"` + Condition json.RawMessage `json:"Condition,omitempty"` } func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error { @@ -169,6 +203,16 @@ func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error { return err } + // Condition applicability is checked before the action/resource-type + // pairing below: AWS reports a Condition key that doesn't apply to the + // statement's actions even when those actions also don't apply to the + // statement's resource type, e.g. s3:prefix with s3:ListBucketMultipartUploads + // against an object resource — reported as the Condition mismatch, not + // the resource-type one. + if err := validateBucketPolicyCondition(bpi.Condition, bpi.Actions); err != nil { + return err + } + containsObjectAction := bpi.Resources.ContainsObjectPattern() containsBucketAction := bpi.Resources.ContainsBucketPattern() @@ -188,18 +232,27 @@ func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error { return nil } -func (bpi *BucketPolicyItem) findMatch(principal string, action Action, resource string, normalizeObjectKey objectKeyNormalizer) bool { - if bpi.Principals.Contains(principal) && bpi.Actions.FindMatch(action) && bpi.Resources.FindMatch(resource, normalizeObjectKey) { - return true +// findMatch reports whether the statement's principal/action/resource cover +// this request, and — only when they do — whether its Condition block holds +// against condCtx. matched is only meaningful when evaluable is true; see +// condition.Evaluate and decisionFor's fail-closed handling of evaluable =false. +func (bpi *BucketPolicyItem) findMatch(principal string, action Action, resource string, condCtx map[string][]string, version PolicyVersion, normalizeObjectKey objectKeyNormalizer) (matched bool, evaluable bool) { + if !(bpi.Principals.Contains(principal) && bpi.Actions.FindMatch(action) && bpi.Resources.FindMatch(resource, normalizeObjectKey)) { + return false, true } - - return false + return condition.Evaluate(bpi.Condition, condCtx, string(version)) } // isPublicFor checks if the bucket policy statement grants public access -// for given resource and action -func (bpi *BucketPolicyItem) isPublicFor(resource string, action Action, normalizeObjectKey objectKeyNormalizer) bool { - return bpi.Principals.isPublic() && bpi.Actions.FindMatch(action) && bpi.Resources.FindMatch(resource, normalizeObjectKey) +// for given resource and action, and — only when it otherwise matches — +// whether its Condition block holds against condCtx. A public statement's +// Condition is evaluated with whatever request-derived keys condCtx carries; +// there is no caller identity to resolve for an anonymous request +func (bpi *BucketPolicyItem) isPublicFor(resource string, action Action, condCtx map[string][]string, version PolicyVersion, normalizeObjectKey objectKeyNormalizer) (matched bool, evaluable bool) { + if !(bpi.Principals.isPublic() && bpi.Actions.FindMatch(action) && bpi.Resources.FindMatch(resource, normalizeObjectKey)) { + return false, true + } + return condition.Evaluate(bpi.Condition, condCtx, string(version)) } // isPublic checks if the statement grants public access @@ -250,29 +303,44 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err return nil } -func VerifyBucketPolicy(policy []byte, access, bucket, object string, normalizeObjectKey objectKeyNormalizer, actions ...Action) error { +// verifyBucketPolicy parses policyBytes and evaluates it against every +// action, aggregating with the same precedence isAllowed uses for a single +// action: a Deny on any action wins immediately (returned along with that +// action, for building an AWS-shaped message); otherwise the decision is +// Allow only if every action has a matching Allow; otherwise NoMatch, +// paired with the first action that lacked one. Zero actions is +// conservatively NoMatch, not vacuously Allow. +func verifyBucketPolicy(policyBytes []byte, access, bucket, object string, condCtx map[string][]string, normalizeObjectKey objectKeyNormalizer, actions ...Action) (policyDecision, Action, error) { if len(actions) == 0 { - return s3err.GetAPIError(s3err.ErrAccessDenied) + return policyDecisionNoMatch, "", nil } - var bucketPolicy BucketPolicy - if err := json.Unmarshal(policy, &bucketPolicy); err != nil { - return fmt.Errorf("failed to parse the bucket policy: %w", err) + var bp BucketPolicy + if err := json.Unmarshal(policyBytes, &bp); err != nil { + return policyDecisionNoMatch, "", fmt.Errorf("failed to parse the bucket policy: %w", err) } resource := makePolicyResource(bucket, object, normalizeObjectKey) + result := policyDecisionAllow + var blamed Action for _, action := range actions { - if !bucketPolicy.isAllowed(access, action, resource, normalizeObjectKey) { - return s3err.GetAPIError(s3err.ErrAccessDenied) + switch d := bp.decisionFor(access, action, resource, condCtx, normalizeObjectKey); d { + case policyDecisionDeny: + return policyDecisionDeny, action, nil + case policyDecisionNoMatch: + if result != policyDecisionNoMatch { + result = policyDecisionNoMatch + blamed = action + } } } - return nil + return result, blamed, nil } // Checks if the bucket policy grants public access -func VerifyPublicBucketPolicy(policy []byte, bucket, object string, normalizeObjectKey objectKeyNormalizer, action Action) error { +func VerifyPublicBucketPolicy(policy []byte, bucket, object string, condCtx map[string][]string, normalizeObjectKey objectKeyNormalizer, action Action) error { var bucketPolicy BucketPolicy if err := json.Unmarshal(policy, &bucketPolicy); err != nil { return err @@ -280,7 +348,7 @@ func VerifyPublicBucketPolicy(policy []byte, bucket, object string, normalizeObj resource := makePolicyResource(bucket, object, normalizeObjectKey) - switch bucketPolicy.publicDecisionFor(resource, action, normalizeObjectKey) { + switch bucketPolicy.publicDecisionFor(resource, action, condCtx, normalizeObjectKey) { case policyDecisionAllow: return nil case policyDecisionDeny: diff --git a/auth/bucket_policy_condition.go b/auth/bucket_policy_condition.go new file mode 100644 index 00000000..70a2d00f --- /dev/null +++ b/auth/bucket_policy_condition.go @@ -0,0 +1,190 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/versity/versitygw/internal/condition" +) + +// conditionKeyRule is one condition key's write-time compatibility check: a +// PutBucketPolicy statement naming this key in its Condition block is only +// accepted when appliesTo holds for every (non-wildcard) action the +// statement names +type conditionKeyRule struct { + appliesTo func(Action) bool + // ipSemantic marks a key AWS validates as an IP address/CIDR at write + // time, independent of which operator wraps it. + ipSemantic bool +} + +func anyAction(Action) bool { return true } + +// isListAction is s3:prefix/s3:delimiter/s3:max-keys' applicable-action set: +// s3:ListBucket and s3:ListBucketVersions, not s3:GetObject and — notably — +// not s3:ListBucketMultipartUploads either, so this is deliberately not +// "every List-shaped action". +func isListAction(a Action) bool { + return a == ListBucketAction || a == ListBucketVersionsAction +} + +// isAclPutAction is s3:x-amz-acl's applicable-action set: s3:PutObject, +// s3:PutBucketAcl, and s3:PutObjectAcl. s3:CreateBucket is excluded — AWS +// rejects s3:CreateBucket in any bucket-policy statement at all, a +// pre-existing, Condition-unrelated validation gap, since bucket policies +// attach to a bucket that must already exist. +func isAclPutAction(a Action) bool { + switch a { + case PutObjectAction, PutBucketAclAction, PutObjectAclAction: + return true + default: + return false + } +} + +// isVersionedAction is s3:VersionId's applicable-action set: the *Version* +// action family. +func isVersionedAction(a Action) bool { + switch a { + case GetObjectVersionAction, DeleteObjectVersionAction, GetObjectVersionAttributesAction, + GetObjectVersionTaggingAction, PutObjectVersionTaggingAction, DeleteObjectVersionTaggingAction: + return true + default: + return false + } +} + +// bucketPolicyConditionKeys is the fixed catalogue of condition keys this +// gateway's S3 bucket-policy Condition support recognizes, each mapped to +// the actions it may be used with. Keys are looked up case-insensitively +// (AWS documents condition key *names*, unlike their values, as +// case-insensitive: "AWS:SourceIp" is accepted the same as "aws:SourceIp"), +// so every key here is stored lowercase. +// +// This deliberately does not cover AWS's full S3 condition-key catalogue — +// tag-based keys (s3:ExistingObjectTag/*, s3:RequestObjectTag/*, +// s3:RequestObjectTagKeys), object-lock keys, s3:x-amz-server-side-encryption +// (the gateway never reads that header, so enforcing it would be +// misleading), and aws:MultiFactorAuthAge (no MFA concept here) are out of +// scope. A Condition naming one of those is still accepted at write time — +// the key just never appears in the runtime context, so any Condition +// depending on it simply never matches, the same as any other key this +// package doesn't populate. +var bucketPolicyConditionKeys = map[string]conditionKeyRule{ + // Generic keys: AWS accepts these with any action. + "aws:sourceip": {appliesTo: anyAction, ipSemantic: true}, + "aws:currenttime": {appliesTo: anyAction}, + "aws:epochtime": {appliesTo: anyAction}, + "aws:securetransport": {appliesTo: anyAction}, + "aws:useragent": {appliesTo: anyAction}, + "aws:referer": {appliesTo: anyAction}, + "aws:principalarn": {appliesTo: anyAction}, + "aws:username": {appliesTo: anyAction}, + "aws:userid": {appliesTo: anyAction}, + "aws:multifactorauthage": {appliesTo: anyAction}, + + // S3-specific keys: only valid with a specific action subset. + "s3:prefix": {appliesTo: isListAction}, + "s3:delimiter": {appliesTo: isListAction}, + "s3:max-keys": {appliesTo: isListAction}, + "s3:x-amz-acl": {appliesTo: isAclPutAction}, + "s3:versionid": {appliesTo: isVersionedAction}, +} + +// lookupConditionKeyRule finds key's rule case-insensitively. +func lookupConditionKeyRule(key string) (conditionKeyRule, bool) { + rule, ok := bucketPolicyConditionKeys[strings.ToLower(key)] + return rule, ok +} + +// validateBucketPolicyCondition checks a bucket-policy statement's raw +// Condition block against the same write-time rules real AWS enforces for +// PutBucketPolicy: +// +// - an unrecognized operator name -> "Invalid Condition type : " +// - a key outside bucketPolicyConditionKeys -> policyErrInvalidConditionKey +// - a key whose rule doesn't apply to some (non-wildcard) action in +// actions -> policyErrConditionActionMismatch. For an explicit +// multi-action list, EVERY action must support the key (e.g. +// ["s3:GetObject","s3:PutObject"] with the PutObject-only s3:x-amz-acl +// is rejected even though PutObject alone would accept it); a wildcard +// action pattern (containing '*' or '?', e.g. "s3:*" or +// "s3:PutObject*") is exempt from this check entirely, so both accept +// s3:x-amz-acl even though s3:* covers many actions that don't support +// it. +// - an ipSemantic key (aws:SourceIp) with a value that doesn't parse as +// an IP address or CIDR range -> policyErrInvalidIPCondition, +// regardless of which operator wraps it. +func validateBucketPolicyCondition(raw json.RawMessage, actions Actions) error { + block, err := condition.Parse(raw) + if err != nil { + op, ok := unrecognizedConditionOperator(raw) + if ok { + //lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations + return fmt.Errorf("Invalid Condition type : %s", op) + } + return policyErrInvalidPolicy + } + + concreteActions := make([]Action, 0, len(actions)) + for action := range actions { + if strings.ContainsAny(string(action), "*?") { + continue + } + concreteActions = append(concreteActions, action) + } + + for _, kvs := range block { + for key, values := range kvs { + rule, ok := lookupConditionKeyRule(key) + if !ok { + return policyErrInvalidConditionKey + } + for _, action := range concreteActions { + if !rule.appliesTo(action) { + return policyErrConditionActionMismatch + } + } + if rule.ipSemantic { + for _, v := range values { + if !condition.ParseIPOrCIDR(v) { + return policyErrInvalidIPCondition + } + } + } + } + } + return nil +} + +// unrecognizedConditionOperator re-walks raw's top-level operator names to +// find the first one ParseOperatorName rejects, for building AWS's exact +// "Invalid Condition type : " message — condition.Parse itself only +// reports that parsing failed, not which operator caused it. +func unrecognizedConditionOperator(raw json.RawMessage) (string, bool) { + var top map[string]json.RawMessage + if err := json.Unmarshal(raw, &top); err != nil { + return "", false + } + for operator := range top { + if _, ok := condition.ParseOperatorName(operator); !ok { + return operator, true + } + } + return "", false +} diff --git a/auth/bucket_policy_condition_test.go b/auth/bucket_policy_condition_test.go new file mode 100644 index 00000000..58b716c6 --- /dev/null +++ b/auth/bucket_policy_condition_test.go @@ -0,0 +1,190 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func actionSet(actions ...Action) Actions { + a := make(Actions, len(actions)) + for _, act := range actions { + a[act] = struct{}{} + } + return a +} + +func TestValidateBucketPolicyCondition(t *testing.T) { + tests := []struct { + name string + raw string + actions Actions + // wantErr is compared by message text, not type: unrecognized-operator + // errors carry a dynamic operator name via fmt.Errorf rather than a + // policyErr constant. + wantErr error + }{ + { + name: "no condition is valid", + raw: ``, + actions: actionSet(GetObjectAction), + }, + { + name: "recognized generic key with any action", + raw: `{"StringEquals":{"aws:PrincipalArn":"arn:aws:iam::123456789012:role/foo"}}`, + actions: actionSet(GetObjectAction), + }, + { + // AWS: "AWS:SourceIp" (uppercase prefix) is accepted the same + // as "aws:SourceIp" - key names are case-insensitive. + name: "condition key recognized case-insensitively", + raw: `{"IpAddress":{"AWS:SourceIp":"10.0.0.0/8"}}`, + actions: actionSet(GetObjectAction), + }, + { + name: "unrecognized operator", + raw: `{"NotARealOperator":{"aws:SourceIp":"1.2.3.4/32"}}`, + actions: actionSet(GetObjectAction), + wantErr: policyErr("Invalid Condition type : NotARealOperator"), + }, + { + // AWS is case-sensitive about operator names specifically, + // unlike condition keys. + name: "operator name is case-sensitive", + raw: `{"stringequals":{"s3:prefix":"foo"}}`, + actions: actionSet(ListBucketAction), + wantErr: policyErr("Invalid Condition type : stringequals"), + }, + { + name: "unrecognized condition key", + raw: `{"StringEquals":{"s3:FakeKeyDoesNotExist":"foo"}}`, + actions: actionSet(GetObjectAction), + wantErr: policyErrInvalidConditionKey, + }, + { + name: "s3-specific key rejected for an unsupported action", + raw: `{"StringEquals":{"s3:prefix":"foo"}}`, + actions: actionSet(GetObjectAction), + wantErr: policyErrConditionActionMismatch, + }, + { + name: "s3-specific key accepted for its supported action", + raw: `{"StringEquals":{"s3:prefix":"foo"}}`, + actions: actionSet(ListBucketAction), + }, + { + // s3:prefix/delimiter/max-keys apply to + // ListBucket/ListBucketVersions only, NOT + // ListBucketMultipartUploads. + name: "s3:prefix rejected for ListBucketMultipartUploads", + raw: `{"StringEquals":{"s3:prefix":"foo"}}`, + actions: actionSet(ListBucketMultipartUploadsAction), + wantErr: policyErrConditionActionMismatch, + }, + { + name: "s3:x-amz-acl accepted for PutObject", + raw: `{"StringEquals":{"s3:x-amz-acl":"public-read"}}`, + actions: actionSet(PutObjectAction), + }, + { + name: "s3:x-amz-acl accepted for PutBucketAcl", + raw: `{"StringEquals":{"s3:x-amz-acl":"public-read"}}`, + actions: actionSet(PutBucketAclAction), + }, + { + name: "s3:x-amz-acl accepted for PutObjectAcl", + raw: `{"StringEquals":{"s3:x-amz-acl":"public-read"}}`, + actions: actionSet(PutObjectAclAction), + }, + { + name: "s3:VersionId accepted for GetObjectVersion", + raw: `{"StringEquals":{"s3:VersionId":"abc123"}}`, + actions: actionSet(GetObjectVersionAction), + }, + { + name: "s3:VersionId rejected for plain GetObject", + raw: `{"StringEquals":{"s3:VersionId":"abc123"}}`, + actions: actionSet(GetObjectAction), + wantErr: policyErrConditionActionMismatch, + }, + { + // Every action in an explicit multi-action list must support + // the key, even though PutObject alone would. + name: "multi-action statement requires every action to support the key", + raw: `{"StringEquals":{"s3:x-amz-acl":"public-read"}}`, + actions: actionSet(GetObjectAction, PutObjectAction), + wantErr: policyErrConditionActionMismatch, + }, + { + // A wildcard action ("s3:*", "s3:PutObject*", …) is exempt from + // the per-action applicability check entirely, even though it + // covers actions the key doesn't support. + name: "wildcard action is exempt from the action-applicability check", + raw: `{"StringEquals":{"s3:x-amz-acl":"public-read"}}`, + actions: actionSet(AllActions), + }, + { + name: "aws:SourceIp with a valid CIDR", + raw: `{"IpAddress":{"aws:SourceIp":"10.0.0.0/8"}}`, + actions: actionSet(GetObjectAction), + }, + { + name: "aws:SourceIp with a bare valid address", + raw: `{"IpAddress":{"aws:SourceIp":"203.0.113.5"}}`, + actions: actionSet(GetObjectAction), + }, + { + name: "aws:SourceIp with an invalid value", + raw: `{"IpAddress":{"aws:SourceIp":"not-an-ip"}}`, + actions: actionSet(GetObjectAction), + wantErr: policyErrInvalidIPCondition, + }, + { + // The IP-format check is keyed by condition-key identity, not + // by operator - it fires even under an operator that has + // nothing to do with IP semantics. + name: "aws:SourceIp invalid value rejected regardless of operator", + raw: `{"Null":{"aws:SourceIp":"true"}}`, + actions: actionSet(GetObjectAction), + wantErr: policyErrInvalidIPCondition, + }, + { + // A non-IP key under IpAddress is not itself validated as an + // IP - only recognized IP-semantic keys are. + name: "non-IP key under IpAddress operator is not IP-validated", + raw: `{"IpAddress":{"aws:Referer":"not-an-ip"}}`, + actions: actionSet(GetObjectAction), + }, + { + name: "malformed condition JSON", + raw: `not json`, + actions: actionSet(GetObjectAction), + wantErr: policyErrInvalidPolicy, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateBucketPolicyCondition([]byte(tt.raw), tt.actions) + if tt.wantErr == nil { + assert.NoError(t, err) + return + } + assert.EqualError(t, err, tt.wantErr.Error()) + }) + } +} diff --git a/auth/bucket_policy_principals.go b/auth/bucket_policy_principals.go index 3f17d85e..a4ba83f9 100644 --- a/auth/bucket_policy_principals.go +++ b/auth/bucket_policy_principals.go @@ -100,7 +100,7 @@ func (p Principals) Validate(iam IAMService) error { return policyErrInvalidPrincipal } - accs, err := CheckIfAccountsExist(p.ToSlice(), iam) + accs, err := iam.ResolveAccounts(p.ToSlice()) if err != nil { return err } diff --git a/auth/bucket_policy_test.go b/auth/bucket_policy_test.go new file mode 100644 index 00000000..28f9cd8e --- /dev/null +++ b/auth/bucket_policy_test.go @@ -0,0 +1,158 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBucketPolicyDecision_Condition(t *testing.T) { + tests := []struct { + name string + policy string + action Action + object string + condCtx map[string][]string + want policyDecision + }{ + { + name: "Allow with matching Condition grants access", + policy: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"}, + "Action":"s3:GetObject","Resource":"arn:aws:s3:::mybucket/*", + "Condition":{"StringEquals":{"aws:UserAgent":"good-agent"}}}]}`, + action: GetObjectAction, + object: "key", + condCtx: map[string][]string{"aws:UserAgent": {"good-agent"}}, + want: policyDecisionAllow, + }, + { + name: "Allow with non-matching Condition does not grant access", + policy: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"}, + "Action":"s3:GetObject","Resource":"arn:aws:s3:::mybucket/*", + "Condition":{"StringEquals":{"aws:UserAgent":"good-agent"}}}]}`, + action: GetObjectAction, + object: "key", + condCtx: map[string][]string{"aws:UserAgent": {"bad-agent"}}, + want: policyDecisionNoMatch, + }, + { + name: "Allow with no matching context key does not grant access", + policy: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"}, + "Action":"s3:GetObject","Resource":"arn:aws:s3:::mybucket/*", + "Condition":{"StringEquals":{"aws:UserAgent":"good-agent"}}}]}`, + action: GetObjectAction, + object: "key", + condCtx: nil, + want: policyDecisionNoMatch, + }, + { + name: "Deny with matching Condition wins over an unconditional Allow", + policy: `{"Version":"2012-10-17","Statement":[ + {"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:GetObject","Resource":"arn:aws:s3:::mybucket/*"}, + {"Effect":"Deny","Principal":{"AWS":"*"},"Action":"s3:GetObject","Resource":"arn:aws:s3:::mybucket/*", + "Condition":{"IpAddress":{"aws:SourceIp":"10.0.0.0/8"}}}]}`, + action: GetObjectAction, + object: "key", + condCtx: map[string][]string{"aws:SourceIp": {"10.1.2.3"}}, + want: policyDecisionDeny, + }, + { + name: "Deny with non-matching Condition leaves the unconditional Allow standing", + policy: `{"Version":"2012-10-17","Statement":[ + {"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:GetObject","Resource":"arn:aws:s3:::mybucket/*"}, + {"Effect":"Deny","Principal":{"AWS":"*"},"Action":"s3:GetObject","Resource":"arn:aws:s3:::mybucket/*", + "Condition":{"IpAddress":{"aws:SourceIp":"10.0.0.0/8"}}}]}`, + action: GetObjectAction, + object: "key", + condCtx: map[string][]string{"aws:SourceIp": {"203.0.113.5"}}, + want: policyDecisionAllow, + }, + { + // s3:prefix, wired up from the request's "prefix" query param by + // the S3 auth middleware, is exercised end to end against a + // ListBucket-shaped policy. + name: "s3:prefix condition key matches against ListBucket", + policy: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"}, + "Action":"s3:ListBucket","Resource":"arn:aws:s3:::mybucket", + "Condition":{"StringEquals":{"s3:prefix":"photos/"}}}]}`, + action: ListBucketAction, + object: "", + condCtx: map[string][]string{"s3:prefix": {"photos/"}}, + want: policyDecisionAllow, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + decision, _, err := verifyBucketPolicy([]byte(tt.policy), "someaccess", "mybucket", tt.object, tt.condCtx, nil, tt.action) + assert.NoError(t, err) + assert.Equal(t, tt.want, decision) + }) + } +} + +func TestBucketPolicyDecision_UnevaluableConditionFailsClosed(t *testing.T) { + // This shape (an unrecognized operator) can no longer be written via + // PutBucketPolicy once write-time validation rejects it - this test + // exercises the defense-in-depth fallback for a document that reached + // storage some other way (a legacy write, a migration, ...), the same + // scenario iamapi/policy.EvaluateIdentityPolicies guards against. + policy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"}, + "Action":"s3:GetObject","Resource":"arn:aws:s3:::mybucket/*", + "Condition":{"SomeFutureOperator":{"aws:UserAgent":"good-agent"}}}]}` + + decision, _, err := verifyBucketPolicy([]byte(policy), "someaccess", "mybucket", "key", nil, nil, GetObjectAction) + assert.NoError(t, err) + assert.Equal(t, policyDecisionDeny, decision) +} + +func TestVerifyPublicBucketPolicy_Condition(t *testing.T) { + tests := []struct { + name string + policy string + condCtx map[string][]string + wantErr error + }{ + { + name: "public Allow with matching Condition grants access", + policy: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*", + "Action":"s3:GetObject","Resource":"arn:aws:s3:::mybucket/*", + "Condition":{"StringEquals":{"aws:UserAgent":"good-agent"}}}]}`, + condCtx: map[string][]string{"aws:UserAgent": {"good-agent"}}, + wantErr: nil, + }, + { + name: "public Allow with non-matching Condition denies access", + policy: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*", + "Action":"s3:GetObject","Resource":"arn:aws:s3:::mybucket/*", + "Condition":{"StringEquals":{"aws:UserAgent":"good-agent"}}}]}`, + condCtx: map[string][]string{"aws:UserAgent": {"bad-agent"}}, + wantErr: errAccessDenied, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := VerifyPublicBucketPolicy([]byte(tt.policy), "mybucket", "key", tt.condCtx, nil, GetObjectAction) + if tt.wantErr == nil { + assert.NoError(t, err) + return + } + assert.Equal(t, tt.wantErr, err) + }) + } +} diff --git a/auth/condition_context.go b/auth/condition_context.go new file mode 100644 index 00000000..7d1e5cc5 --- /dev/null +++ b/auth/condition_context.go @@ -0,0 +1,68 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "strconv" + "time" + + "github.com/gofiber/fiber/v3" +) + +// requestConditionContext builds the IAM policy-condition keys describing +// this request — aws:SourceIp, aws:SecureTransport, aws:CurrentTime and +// friends — for identity-policy and bucket-policy Condition blocks to +// evaluate against. The identity-derived keys (aws:PrincipalArn, +// aws:username, aws:PrincipalTag/*, …) are deliberately absent: the S3 +// gateway has no way to know them, so the IAM service fills them in itself +// when it evaluates an identity policy. +func requestConditionContext(ctx fiber.Ctx) map[string][]string { + now := time.Now().UTC() + condCtx := map[string][]string{ + "aws:CurrentTime": {now.Format(time.RFC3339)}, + "aws:EpochTime": {strconv.FormatInt(now.Unix(), 10)}, + "aws:SecureTransport": {strconv.FormatBool(ctx.Secure())}, + } + // ctx.IP() is the real peer address: the gateway's fiber app configures + // neither ProxyHeader nor TrustProxy, so no client-supplied header can + // influence it. Adding either for logging would make aws:SourceIp + // client-controlled — revisit this if that ever changes. + if ip := ctx.IP(); ip != "" { + condCtx["aws:SourceIp"] = []string{ip} + } + if ua := ctx.Get("User-Agent"); ua != "" { + condCtx["aws:UserAgent"] = []string{ua} + } + if ref := ctx.Get("Referer"); ref != "" { + condCtx["aws:Referer"] = []string{ref} + } + if prefix := ctx.Query("prefix"); prefix != "" { + condCtx["s3:prefix"] = []string{prefix} + } + if delim := ctx.Query("delimiter"); delim != "" { + condCtx["s3:delimiter"] = []string{delim} + } + if maxKeys := ctx.Query("max-keys"); maxKeys != "" { + condCtx["s3:max-keys"] = []string{maxKeys} + } + if acl := ctx.Get("X-Amz-Acl"); acl != "" { + condCtx["s3:x-amz-acl"] = []string{acl} + } + if versionID := ctx.Query("versionId"); versionID != "" { + condCtx["s3:VersionId"] = []string{versionID} + } + + return condCtx +} diff --git a/auth/iam.go b/auth/iam.go index 13f47517..9d642e54 100644 --- a/auth/iam.go +++ b/auth/iam.go @@ -22,6 +22,27 @@ import ( "github.com/versity/versitygw/s3err" ) +// resolveAccountsByLookup implements ResolveAccounts for backends that have +// no batch endpoint, by calling getUserAccount once per access key and +// collecting the ones that don't exist. +func resolveAccountsByLookup(accessKeyIDs []string, getUserAccount func(string) (Account, error)) ([]string, error) { + missing := []string{} + for _, access := range accessKeyIDs { + _, err := getUserAccount(access) + if err != nil { + if err == ErrNoSuchUser || err == s3err.GetAPIError(s3err.ErrAdminUserNotFound) { + missing = append(missing, access) + continue + } + if errors.Is(err, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)) { + return nil, err + } + return nil, fmt.Errorf("check user account: %w", err) + } + } + return missing, nil +} + type Role string const ( @@ -51,6 +72,27 @@ type Account struct { UserID int `json:"userID"` GroupID int `json:"groupID"` ProjectID int `json:"projectID"` + + // SessionToken and IsSession describe a temporary credential minted by + // AssumeRoleWithWebIdentity, and are set only by the S3 auth + // middlewares for the duration of one request. They ride on Account + // rather than on auth.AccessOptions so the ~55 controller sites that + // already forward the request's Account into an authorization check + // carry them without a single edit. + // + // Both are json:"-": a session is request state, never persisted by an + // IAM backend nor echoed by the admin API. + SessionToken string `json:"-"` + IsSession bool `json:"-"` +} + +// String elides the two credential-bearing fields so an Account can't leak +// them into a log line through a %v/%+v verb. debuglogger redacts the +// X-Amz-Security-Token *header*, which does nothing for a struct printed +// after the token has been parsed out of it. +func (a Account) String() string { + return fmt.Sprintf("Account{Access:%s, Secret:REDACTED, Role:%s, UserID:%d, GroupID:%d, ProjectID:%d, SessionToken:REDACTED, IsSession:%t}", + a.Access, a.Role, a.UserID, a.GroupID, a.ProjectID, a.IsSession) } type ListUserAccountsResult struct { @@ -98,6 +140,7 @@ func updateAcc(acc *Account, props MutableProps) { type IAMService interface { CreateAccount(account Account) error GetUserAccount(access string) (Account, error) + ResolveAccounts(accessKeyIDs []string) ([]string, error) UpdateUserAccount(access string, props MutableProps) error DeleteUserAccount(access string) error ListUserAccounts() ([]Account, error) @@ -109,6 +152,13 @@ var ( ErrUserExists = errors.New("user already exists") // ErrNoSuchUser is returned when the user does not exist ErrNoSuchUser = errors.New("user not found") + // ErrInvalidSessionToken is returned when a request's + // X-Amz-Security-Token is missing for a temporary (ASIA…) access key, + // doesn't match the session that key belongs to, or is present + // alongside a permanent credential. Callers render it as S3's + // InvalidToken, distinct from the InvalidAccessKeyId that ErrNoSuchUser + // produces — matching real S3, which reports the two separately. + ErrInvalidSessionToken = errors.New("invalid session token") ) type Opts struct { @@ -153,6 +203,15 @@ type Opts struct { IpaUser string IpaPassword string IpaInsecure bool + StandaloneIAMEndpoint string + StandaloneIAMAccess string + StandaloneIAMSecret string + StandaloneClientCert string + StandaloneClientCertKey string + StandaloneServerCA string + StandaloneDefaultUserID int + StandaloneDefaultGroupID int + StandaloneDefaultProjectID int } func New(o *Opts) (IAMService, error) { @@ -160,6 +219,31 @@ func New(o *Opts) (IAMService, error) { var err error switch { + case o.StandaloneIAMEndpoint != "": + svc, err = NewIAMServiceStandalone(o.RootAccount, IAMServiceStandaloneConfig{ + Endpoint: o.StandaloneIAMEndpoint, + Access: o.StandaloneIAMAccess, + Secret: o.StandaloneIAMSecret, + ClientCert: o.StandaloneClientCert, + ClientCertKey: o.StandaloneClientCertKey, + ServerCA: o.StandaloneServerCA, + DefaultUserID: o.StandaloneDefaultUserID, + DefaultGroupID: o.StandaloneDefaultGroupID, + DefaultProjectID: o.StandaloneDefaultProjectID, + }) + fmt.Printf("initializing standalone IAM with %q\n", o.StandaloneIAMEndpoint) + if err != nil { + return nil, err + } + // Never cache-wrapped, unlike every other backend below: IAMCache + // only implements the base IAMService methods, so wrapping this + // backend in it would silently strip the SigningKeyProvider/ + // PolicyEvaluator interfaces signature verification and policy + // enforcement depend on — not just skip a performance + // optimization, but break both outright. + // + // TODO: Do we need to implement cache for this ? + return svc, nil case o.Dir != "": svc, err = NewInternal(o.RootAccount, o.Dir) fmt.Printf("initializing internal IAM with %q\n", o.Dir) diff --git a/auth/iam_cache.go b/auth/iam_cache.go index 2eea1ba0..35e67022 100644 --- a/auth/iam_cache.go +++ b/auth/iam_cache.go @@ -170,6 +170,13 @@ func (c *IAMCache) GetUserAccount(access string) (Account, error) { return a, nil } +// ResolveAccounts returns the subset of accessKeyIDs that do not exist. It +// loops over the cache's own GetUserAccount so lookups benefit from caching +// the same way a single-account check would. +func (c *IAMCache) ResolveAccounts(accessKeyIDs []string) ([]string, error) { + return resolveAccountsByLookup(accessKeyIDs, c.GetUserAccount) +} + // DeleteUserAccount deletes account from IAM service and cache func (c *IAMCache) DeleteUserAccount(access string) error { err := c.service.DeleteUserAccount(access) diff --git a/auth/iam_internal.go b/auth/iam_internal.go index f9901de2..4972ee9d 100644 --- a/auth/iam_internal.go +++ b/auth/iam_internal.go @@ -117,6 +117,11 @@ func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) { return acct, nil } +// ResolveAccounts returns the subset of accessKeyIDs that do not exist. +func (s *IAMServiceInternal) ResolveAccounts(accessKeyIDs []string) ([]string, error) { + return resolveAccountsByLookup(accessKeyIDs, s.GetUserAccount) +} + // UpdateUserAccount updates the specified user account fields. Returns // ErrNoSuchUser if the account does not exist. func (s *IAMServiceInternal) UpdateUserAccount(access string, props MutableProps) error { diff --git a/auth/iam_ipa.go b/auth/iam_ipa.go index e54e2e03..2cd5ea40 100644 --- a/auth/iam_ipa.go +++ b/auth/iam_ipa.go @@ -211,6 +211,11 @@ func (ipa *IpaIAMService) GetUserAccount(access string) (Account, error) { return account, nil } +// ResolveAccounts returns the subset of accessKeyIDs that do not exist. +func (ipa *IpaIAMService) ResolveAccounts(accessKeyIDs []string) ([]string, error) { + return resolveAccountsByLookup(accessKeyIDs, ipa.GetUserAccount) +} + func (ipa *IpaIAMService) UpdateUserAccount(access string, props MutableProps) error { return fmt.Errorf("not implemented") } diff --git a/auth/iam_ldap.go b/auth/iam_ldap.go index c7c7879d..c20c33f5 100644 --- a/auth/iam_ldap.go +++ b/auth/iam_ldap.go @@ -239,6 +239,11 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) { }, nil } +// ResolveAccounts returns the subset of accessKeyIDs that do not exist. +func (ld *LdapIAMService) ResolveAccounts(accessKeyIDs []string) ([]string, error) { + return resolveAccountsByLookup(accessKeyIDs, ld.GetUserAccount) +} + func (ld *LdapIAMService) UpdateUserAccount(access string, props MutableProps) error { req := ldap.NewModifyRequest(ld.buildUserDN(access), nil) if props.Secret != nil { diff --git a/auth/iam_s3_object.go b/auth/iam_s3_object.go index f8dafa09..d8ba2199 100644 --- a/auth/iam_s3_object.go +++ b/auth/iam_s3_object.go @@ -149,6 +149,11 @@ func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) { return acct, nil } +// ResolveAccounts returns the subset of accessKeyIDs that do not exist. +func (s *IAMServiceS3) ResolveAccounts(accessKeyIDs []string) ([]string, error) { + return resolveAccountsByLookup(accessKeyIDs, s.GetUserAccount) +} + func (s *IAMServiceS3) UpdateUserAccount(access string, props MutableProps) error { s.Lock() defer s.Unlock() diff --git a/auth/iam_single.go b/auth/iam_single.go index 9cf3e249..26e2c317 100644 --- a/auth/iam_single.go +++ b/auth/iam_single.go @@ -45,6 +45,11 @@ func (s IAMServiceSingle) GetUserAccount(access string) (Account, error) { return Account{}, s3err.GetAPIError(s3err.ErrAdminUserNotFound) } +// ResolveAccounts returns the subset of accessKeyIDs that do not exist. +func (s IAMServiceSingle) ResolveAccounts(accessKeyIDs []string) ([]string, error) { + return resolveAccountsByLookup(accessKeyIDs, s.GetUserAccount) +} + // UpdateUserAccount no accounts in single tenant mode func (IAMServiceSingle) UpdateUserAccount(access string, props MutableProps) error { return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported) diff --git a/auth/iam_standalone.go b/auth/iam_standalone.go new file mode 100644 index 00000000..76407917 --- /dev/null +++ b/auth/iam_standalone.go @@ -0,0 +1,505 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "time" + + "github.com/versity/versitygw/iamapi/private" + "github.com/versity/versitygw/internal/netutil" + "github.com/versity/versitygw/internal/sigv4auth" + "github.com/versity/versitygw/s3err" +) + +const ( + // standaloneSigningRegion/standaloneSigningService are the SigV4 + // envelope this client signs its own calls to the private endpoints with + standaloneSigningRegion = "us-east-1" + standaloneSigningService = sigv4auth.ServiceIAM + + standaloneRequestTimeout = 10 * time.Second +) + +// IAMServiceStandaloneConfig configures IAMServiceStandalone. +type IAMServiceStandaloneConfig struct { + // Endpoint is either a "host:port" TCP address (mTLS required - + // ClientCert/ClientCertKey/ServerCA) or a unix socket path, matching + // the standalone IAM service's own --private-ports address shape. + Endpoint string + // Access/Secret are this client's own SigV4 identity — the credential + // it signs its private requests with. Defaults both to the + // gateway's root account when unset. + Access string + Secret string + // ClientCert/ClientCertKey/ServerCA configure outbound mTLS. Required + // (all three) for a TCP Endpoint; unused for a unix socket Endpoint. + ClientCert string + ClientCertKey string + ServerCA string + // DefaultUserID/GroupID/ProjectID are assigned to every resolved + // (non-root) account. The standalone IAM service's user model + // (iamapi/types.User, mirroring real AWS IAM) has no POSIX uid/gid/ + // project-id concept, so there is no per-user value to fetch instead — + // every standalone-backed account shares one POSIX identity for + // backend file-ownership purposes. + DefaultUserID int + DefaultGroupID int + DefaultProjectID int +} + +// IAMServiceStandalone is the S3 gateway's client for a standalone IAM +// service's private endpoints. It never holds a plaintext secret for any account +// but its own signing identity and the locally-known root account — every +// other account's secret stays inside the IAM service process. +// CreateAccount/UpdateUserAccount/ DeleteUserAccount/ListUserAccounts +// are unsupported here for the same reason: mutating a user requires setting a secret, which must never +// flow into this process — manage users via the IAM service's own control-plane API instead. +type IAMServiceStandalone struct { + client *http.Client + baseURL string + access string + secret string + rootAcc Account + cfg IAMServiceStandaloneConfig +} + +var ( + _ IAMService = (*IAMServiceStandalone)(nil) + _ SigningKeyProvider = (*IAMServiceStandalone)(nil) + _ PolicyEvaluator = (*IAMServiceStandalone)(nil) +) + +// NewIAMServiceStandalone constructs the standalone IAM service client. +// rootAcc is the gateway's own root account — always resolved locally, +// never round-tripped through the IAM service. +func NewIAMServiceStandalone(rootAcc Account, cfg IAMServiceStandaloneConfig) (*IAMServiceStandalone, error) { + if cfg.Endpoint == "" { + return nil, fmt.Errorf("iam standalone: endpoint is required") + } + + access := cfg.Access + if access == "" { + access = rootAcc.Access + } + secret := cfg.Secret + if secret == "" { + secret = rootAcc.Secret + } + + client, baseURL, err := newStandaloneHTTPClient(cfg) + if err != nil { + return nil, err + } + + return &IAMServiceStandalone{ + client: client, + baseURL: baseURL, + access: access, + secret: secret, + rootAcc: rootAcc, + cfg: cfg, + }, nil +} + +func newStandaloneHTTPClient(cfg IAMServiceStandaloneConfig) (*http.Client, string, error) { + if netutil.IsUnixSocketPath(cfg.Endpoint) { + sock := cfg.Endpoint + transport := &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", sock) + }, + } + // The host in this URL is never actually resolved/dialed — the + // DialContext override above always connects to the unix socket + // regardless — it just needs to be a syntactically valid URL. + return &http.Client{Transport: transport, Timeout: standaloneRequestTimeout}, "http://unix", nil + } + + if cfg.ClientCert == "" || cfg.ClientCertKey == "" || cfg.ServerCA == "" { + return nil, "", fmt.Errorf("iam standalone: client-cert, client-cert-key, and server-ca are all required for a TCP endpoint (%q)", cfg.Endpoint) + } + + cert, err := netutil.LoadClientCert(cfg.ClientCert, cfg.ClientCertKey) + if err != nil { + return nil, "", fmt.Errorf("iam standalone: %w", err) + } + pool, err := netutil.LoadCACertPool(cfg.ServerCA) + if err != nil { + return nil, "", fmt.Errorf("iam standalone: %w", err) + } + + transport := &http.Transport{ + TLSClientConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{cert}, + RootCAs: pool, + }, + } + return &http.Client{Transport: transport, Timeout: standaloneRequestTimeout}, "https://" + cfg.Endpoint, nil +} + +// doPrivateRequest signs reqBody as this client's own identity (s.access/ +// s.secret, the one place in this file that touches a secret directly — +// signing an outbound request as itself, not verifying an inbound one) and +// POSTs it to path, unmarshaling the response into respBody. +// +// A 403 is dispatched on the error body's code: an unresolvable access key +// becomes ErrNoSuchUser (matching IAMService.GetUserAccount's contract), a +// rejected security token becomes ErrInvalidSessionToken, and anything +// else — most importantly this gateway's own IAM-client credential being +// rejected — stays a plain error, so a gateway misconfiguration surfaces as +// a server fault instead of telling the end user their access key doesn't +// exist. +func (s *IAMServiceStandalone) doPrivateRequest(path string, reqBody, respBody any) error { + bodyBytes, err := json.Marshal(reqBody) + if err != nil { + return fmt.Errorf("iam standalone: marshal request: %w", err) + } + + req, err := http.NewRequest(http.MethodPost, s.baseURL+path, bytes.NewReader(bodyBytes)) + if err != nil { + return fmt.Errorf("iam standalone: build request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + payloadHash := sigv4auth.PayloadSHA256Hex(bodyBytes) + req.Header.Set("X-Amz-Content-Sha256", payloadHash) + + signingTime := time.Now().UTC() + yyyymmdd := signingTime.Format(sigv4auth.YYYYMMDD) + derivedKey := sigv4auth.DeriveKey(s.secret, yyyymmdd, standaloneSigningRegion, standaloneSigningService) + in := sigv4auth.SigningInputFromRequest(req) + in.AccessKeyID = s.access + in.CredentialScope = sigv4auth.BuildCredentialScope(yyyymmdd, standaloneSigningRegion, standaloneSigningService) + in.PayloadHash = payloadHash + in.SigningTime = signingTime + in.DisableURIPathEscaping = true + result := sigv4auth.BuildAndSign(derivedKey, in) + req.Header.Set("X-Amz-Date", result.AmzDate) + req.Header.Set("Authorization", result.AuthorizationHeader) + + resp, err := s.client.Do(req) + if err != nil { + return fmt.Errorf("iam standalone: request to %s failed: %w", path, err) + } + defer resp.Body.Close() + + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("iam standalone: read response from %s: %w", path, err) + } + + if resp.StatusCode != http.StatusOK { + return standaloneResponseError(path, resp.StatusCode, respBytes) + } + + if respBody != nil { + if err := json.Unmarshal(respBytes, respBody); err != nil { + return fmt.Errorf("iam standalone: unmarshal response from %s: %w", path, err) + } + } + return nil +} + +// standaloneResponseError turns a non-200 private-endpoint response into +// the sentinel the S3 request pipeline dispatches on, using the JSON error +// body's machine-readable code rather than the status alone (403 covers +// several distinct failures, only two of which are about the *end user's* +// credential). +func standaloneResponseError(path string, status int, body []byte) error { + var errBody struct { + Error string `json:"error"` + Code string `json:"code"` + } + // A body that doesn't parse leaves Code empty, which falls through to + // the generic error below — the safe direction, since misreporting a + // server fault as "no such user" is what this dispatch exists to avoid. + _ = json.Unmarshal(body, &errBody) + + switch errBody.Code { + case private.CodeNoSuchIdentity: + return ErrNoSuchUser + case private.CodeInvalidToken: + return ErrInvalidSessionToken + } + + return fmt.Errorf("iam standalone: %s returned %d: %s", path, status, string(body)) +} + +// DeriveSigningKey implements SigningKeyProvider. Root is special-cased +// locally: its secret is already known to this process either way, so +// there's no reason to round-trip it through the IAM service. +func (s *IAMServiceStandalone) DeriveSigningKey(access, sessionToken, date, region, service string) ([]byte, Account, error) { + if access == s.rootAcc.Access { + if sessionToken != "" { + return nil, Account{}, ErrInvalidSessionToken + } + return sigv4auth.DeriveKey(s.rootAcc.Secret, date, region, service), s.rootAcc, nil + } + + var resp private.DeriveSigningKeyResponse + err := s.doPrivateRequest(private.DerivePath, private.DeriveSigningKeyRequest{ + AccessKeyID: access, + SessionToken: sessionToken, + Date: date, + Region: region, + Service: service, + }, &resp) + if err != nil { + return nil, Account{}, err + } + + return resp.DerivedKey, s.accountFor(access, sessionToken), nil +} + +// accountFor builds the Account metadata DeriveSigningKey/GetUserAccount +// return for a resolved non-root identity +func (s *IAMServiceStandalone) accountFor(access, sessionToken string) Account { + return Account{ + Access: access, + Role: RoleUser, + UserID: s.cfg.DefaultUserID, + GroupID: s.cfg.DefaultGroupID, + ProjectID: s.cfg.DefaultProjectID, + SessionToken: sessionToken, + IsSession: sigv4auth.IsTempAccessKeyID(access), + } +} + +// EvaluatePolicy implements PolicyEvaluator, evaluating every action in +// actions against resource in a single request rather than one round trip +// per action. +func (s *IAMServiceStandalone) EvaluatePolicy(access, sessionToken string, actions []Action, resources []string, condition map[string][]string) (PolicyEvaluation, error) { + actionStrs := make([]string, len(actions)) + for i, action := range actions { + actionStrs[i] = string(action) + } + + var resp private.EvaluatePolicyResponse + err := s.doPrivateRequest(private.EvaluatePath, private.EvaluatePolicyRequest{ + AccessKeyID: access, + SessionToken: sessionToken, + Actions: actionStrs, + Resources: resources, + Condition: condition, + }, &resp) + if err != nil { + return PolicyEvaluation{}, err + } + if len(resp.Decisions) != len(resources) { + // A protocol mismatch between the gateway and IAM service builds — + // fail closed rather than silently under- or over-evaluating the + // requested matrix. + return PolicyEvaluation{}, fmt.Errorf("iam standalone: evaluate-policy returned %d resource decisions for %d resources", len(resp.Decisions), len(resources)) + } + + decisions, err := decisionMatrixFromWire(resp.Decisions, len(actions)) + if err != nil { + return PolicyEvaluation{}, err + } + + eval := PolicyEvaluation{ + Decisions: decisions, + PrincipalArn: resp.PrincipalArn, + } + + if resp.HasSessionPolicy { + if len(resp.SessionDecisions) != len(resources) { + return PolicyEvaluation{}, fmt.Errorf("iam standalone: evaluate-policy returned %d session-decision rows for %d resources", len(resp.SessionDecisions), len(resources)) + } + sessionDecisions, err := decisionMatrixFromWire(resp.SessionDecisions, len(actions)) + if err != nil { + return PolicyEvaluation{}, err + } + eval.HasSessionPolicy = true + eval.SessionDecisions = sessionDecisions + } + + return eval, nil +} + +// decisionMatrixFromWire converts one wire decision matrix, checking every +// row is the expected width. A short row is a protocol mismatch between +// gateway and IAM service builds, and is failed closed rather than padded. +func decisionMatrixFromWire(rows [][]string, actionCount int) ([][]policyDecision, error) { + out := make([][]policyDecision, len(rows)) + for i, perAction := range rows { + if len(perAction) != actionCount { + return nil, fmt.Errorf("iam standalone: evaluate-policy returned %d action decisions for %d actions", len(perAction), actionCount) + } + out[i] = make([]policyDecision, len(perAction)) + for j, d := range perAction { + out[i][j] = decisionFromWireValue(d) + } + } + return out, nil +} + +// decisionFromWireValue translates the private endpoint's wire-format +// Decision string to the auth package's own policyDecision. An unrecognized +// value (a protocol mismatch between mismatched gateway/IAM-service builds) +// fails closed as Deny rather than silently granting access. +func decisionFromWireValue(v string) policyDecision { + switch v { + case private.DecisionAllow: + return policyDecisionAllow + case private.DecisionNoMatch: + return policyDecisionNoMatch + default: + return policyDecisionDeny + } +} + +// GetUserAccount resolves access via the resolve-identity endpoint, which +// answers existence and principal identity while returning no credential +// material at all. This is not blanket-unsupported like the mutating +// methods below: ResolveAccounts (bucket-policy Principal and ACL grantee +// validation) depends on GetUserAccount working to tell a nonexistent +// grantee (ErrNoSuchUser) apart from an unsupported one +// (ErrAdminMethodNotSupported, which it treats as fatal). +// +// Callers that need to validate several access keys at once should use +// ResolveAccounts instead — one round trip for the whole set rather than +// one per key. +func (s *IAMServiceStandalone) GetUserAccount(access string) (Account, error) { + if access == s.rootAcc.Access { + return s.rootAcc, nil + } + + accounts, err := s.resolveAccountDetails([]string{access}) + if err != nil { + return Account{}, err + } + if !accounts[0].Found { + return Account{}, ErrNoSuchUser + } + return accounts[0].Account, nil +} + +// resolvedAccount is one resolveAccountDetails result. The zero value means +// "no such access key". +type resolvedAccount struct { + Found bool + // IsSession distinguishes an ephemeral AssumeRoleWithWebIdentity + // session from a long-term user. Callers persisting a reference to a + // principal (a bucket policy Principal, an ACL grantee, a bucket owner) + // must refuse a session: the ASIA… key it is named by stops existing + // when the session expires, leaving a reference that can never match + // and, for a bucket owner, a bucket nobody but root can administer. + IsSession bool + Account Account +} + +// resolveAccountDetails resolves every access key in accesses in a single +// round trip, returning one positional result per input. Only the root +// account is answered locally; a root access key mixed into the batch still +// costs nothing, since it never reaches the IAM service. +func (s *IAMServiceStandalone) resolveAccountDetails(accesses []string) ([]resolvedAccount, error) { + out := make([]resolvedAccount, len(accesses)) + + // Root is known to this process, so it is answered here and left out of + // the request entirely — the IAM service has no record of it. + remote := make([]string, 0, len(accesses)) + remoteIdx := make([]int, 0, len(accesses)) + for i, access := range accesses { + if access == s.rootAcc.Access { + out[i] = resolvedAccount{Found: true, Account: s.rootAcc} + continue + } + remote = append(remote, access) + remoteIdx = append(remoteIdx, i) + } + if len(remote) == 0 { + return out, nil + } + + var resp private.ResolveIdentityResponse + err := s.doPrivateRequest(private.ResolveIdentityPath, private.ResolveIdentityRequest{ + AccessKeyIDs: remote, + }, &resp) + if err != nil { + return nil, err + } + if len(resp.Identities) != len(remote) { + // A protocol mismatch between the gateway and IAM service builds — + // fail closed rather than silently mis-attributing results to the + // wrong access keys. + return nil, fmt.Errorf("iam standalone: resolve-identity returned %d identities for %d access keys", len(resp.Identities), len(remote)) + } + + for i, identity := range resp.Identities { + if !identity.Found { + continue + } + out[remoteIdx[i]] = resolvedAccount{ + Found: true, + IsSession: identity.Kind == private.KindSession, + // No session token is known here, and none is needed: this + // Account answers "who is this" for validation, never + // authenticates a request. + Account: s.accountFor(remote[i], ""), + } + } + return out, nil +} + +// ResolveAccounts returns the subset of accessKeyIDs that do not exist, in +// a single round trip. A temporary (ASIA…) session access key counts as +// nonexistent even while its session is live — see resolvedAccount.IsSession. +func (s *IAMServiceStandalone) ResolveAccounts(accessKeyIDs []string) ([]string, error) { + resolved, err := s.resolveAccountDetails(accessKeyIDs) + if err != nil { + return nil, fmt.Errorf("check user account: %w", err) + } + missing := []string{} + for i, acc := range resolved { + if !acc.Found || acc.IsSession { + missing = append(missing, accessKeyIDs[i]) + } + } + return missing, nil +} + +// CreateAccount is not supported +func (s *IAMServiceStandalone) CreateAccount(Account) error { + return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported) +} + +// UpdateUserAccount is not supported +func (s *IAMServiceStandalone) UpdateUserAccount(string, MutableProps) error { + return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported) +} + +// DeleteUserAccount is not supported +func (s *IAMServiceStandalone) DeleteUserAccount(string) error { + return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported) +} + +// ListUserAccounts is not supported +func (s *IAMServiceStandalone) ListUserAccounts() ([]Account, error) { + return nil, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported) +} + +func (s *IAMServiceStandalone) Shutdown() error { + s.client.CloseIdleConnections() + return nil +} diff --git a/auth/iam_standalone_test.go b/auth/iam_standalone_test.go new file mode 100644 index 00000000..8f055888 --- /dev/null +++ b/auth/iam_standalone_test.go @@ -0,0 +1,356 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import ( + "context" + "errors" + "net" + "os" + "path/filepath" + "testing" + "time" + + "github.com/versity/versitygw/iamapi" + "github.com/versity/versitygw/iamapi/private" + "github.com/versity/versitygw/iamapi/storage" + "github.com/versity/versitygw/iamapi/types" + "github.com/versity/versitygw/internal/netutil" + "github.com/versity/versitygw/internal/sigv4auth" + "github.com/versity/versitygw/s3err" +) + +const standaloneTestRootAccess = "AKIDROOT" +const standaloneTestRootSecret = "ROOTSECRET" + +// standaloneTestServer starts a real private.PrivateAPI on a unix socket in +// t.TempDir(), backed by a real file storage.Storer — an actual server, not +// a hand-rolled mock — so IAMServiceStandalone is exercised against exactly +// the same code path the smoke-tested `versitygw iam` binary runs. +func standaloneTestServer(t *testing.T) (store storage.Storer, sockPath string) { + t.Helper() + + store, err := storage.New(storage.Config{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("storage.New: %v", err) + } + + p, err := private.New(store, iamapi.RootCredentials{ + Access: standaloneTestRootAccess, + Secret: standaloneTestRootSecret, + }) + if err != nil { + t.Fatalf("private.New: %v", err) + } + + // A unix socket path is limited to ~104 bytes on macOS (sockaddr_un), + // which t.TempDir() alone can exceed once it embeds this test's full + // name — os.MkdirTemp with a short, fixed prefix keeps it well under + // that regardless of the test name. + sockDir, err := os.MkdirTemp("", "vgw-priv") + if err != nil { + t.Fatalf("MkdirTemp: %v", err) + } + t.Cleanup(func() { os.RemoveAll(sockDir) }) + sockPath = filepath.Join(sockDir, "p.sock") + + errCh := make(chan error, 1) + go func() { + errCh <- p.ServeMultiPort([]string{sockPath}, netutil.TLSOptions{}) + }() + + waitForSocket(t, sockPath, errCh) + + t.Cleanup(func() { + if err := p.Shutdown(); err != nil { + t.Logf("shutdown private API: %v", err) + } + }) + + return store, sockPath +} + +func waitForSocket(t *testing.T, path string, errCh <-chan error) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + select { + case err := <-errCh: + t.Fatalf("ServeMultiPort exited early: %v", err) + default: + } + conn, err := net.Dial("unix", path) + if err == nil { + conn.Close() + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("private API socket %s never became ready", path) +} + +func createStandaloneTestUser(t *testing.T, store storage.Storer, userName, accessKeyID, secret, policyDocument string) { + t.Helper() + ctx := context.Background() + + if _, err := store.CreateUser(ctx, types.User{UserName: userName, Path: "/", CreateDate: time.Now().UTC()}); err != nil { + t.Fatalf("CreateUser: %v", err) + } + if _, err := store.CreateAccessKey(ctx, storage.CreateAccessKeyInput{ + UserName: userName, + AccessKeyID: accessKeyID, + SecretAccessKey: secret, + Status: "Active", + CreateDate: time.Now().UTC(), + }); err != nil { + t.Fatalf("CreateAccessKey: %v", err) + } + if policyDocument != "" { + if err := store.PutUserPolicy(ctx, storage.PutUserPolicyInput{ + UserName: userName, + PolicyName: "P", + PolicyDocument: policyDocument, + }); err != nil { + t.Fatalf("PutUserPolicy: %v", err) + } + } +} + +func TestIAMServiceStandaloneDeriveSigningKeyAndGetUserAccount(t *testing.T) { + store, sock := standaloneTestServer(t) + createStandaloneTestUser(t, store, "alice", "AKIAALICE", "alicesecret", "") + + rootAcc := Account{Access: standaloneTestRootAccess, Secret: standaloneTestRootSecret, Role: RoleAdmin} + client, err := NewIAMServiceStandalone(rootAcc, IAMServiceStandaloneConfig{Endpoint: sock}) + if err != nil { + t.Fatalf("NewIAMServiceStandalone: %v", err) + } + defer client.Shutdown() + + yyyymmdd := time.Now().UTC().Format(sigv4auth.YYYYMMDD) + derivedKey, account, err := client.DeriveSigningKey("AKIAALICE", "", yyyymmdd, "us-east-1", "s3") + if err != nil { + t.Fatalf("DeriveSigningKey: %v", err) + } + + want := sigv4auth.DeriveKey("alicesecret", yyyymmdd, "us-east-1", "s3") + if string(derivedKey) != string(want) { + t.Errorf("derived key = %x, want %x", derivedKey, want) + } + if account.Secret != "" { + t.Errorf("account.Secret should never be populated by the standalone client, got %q", account.Secret) + } + if account.Role != RoleUser { + t.Errorf("account.Role = %v, want %v", account.Role, RoleUser) + } + + // GetUserAccount resolves via the metadata-only endpoint and must agree. + got, err := client.GetUserAccount("AKIAALICE") + if err != nil { + t.Fatalf("GetUserAccount: %v", err) + } + if got.Access != "AKIAALICE" || got.Secret != "" { + t.Errorf("GetUserAccount() = %+v", got) + } +} + +func TestIAMServiceStandaloneGetUserAccountUnknownReturnsErrNoSuchUser(t *testing.T) { + store, sock := standaloneTestServer(t) + _ = store + + rootAcc := Account{Access: standaloneTestRootAccess, Secret: standaloneTestRootSecret, Role: RoleAdmin} + client, err := NewIAMServiceStandalone(rootAcc, IAMServiceStandaloneConfig{Endpoint: sock}) + if err != nil { + t.Fatalf("NewIAMServiceStandalone: %v", err) + } + defer client.Shutdown() + + _, err = client.GetUserAccount("AKIADOESNOTEXIST") + if !errors.Is(err, ErrNoSuchUser) { + t.Errorf("GetUserAccount() error = %v, want ErrNoSuchUser", err) + } +} + +func TestIAMServiceStandaloneGetUserAccountRoot(t *testing.T) { + _, sock := standaloneTestServer(t) + + rootAcc := Account{Access: standaloneTestRootAccess, Secret: standaloneTestRootSecret, Role: RoleAdmin} + client, err := NewIAMServiceStandalone(rootAcc, IAMServiceStandaloneConfig{Endpoint: sock}) + if err != nil { + t.Fatalf("NewIAMServiceStandalone: %v", err) + } + defer client.Shutdown() + + got, err := client.GetUserAccount(standaloneTestRootAccess) + if err != nil { + t.Fatalf("GetUserAccount(root): %v", err) + } + if got.Secret != standaloneTestRootSecret { + t.Errorf("root account should resolve locally with its real secret, got %+v", got) + } +} + +func TestIAMServiceStandaloneEvaluatePolicy(t *testing.T) { + store, sock := standaloneTestServer(t) + createStandaloneTestUser(t, store, "bob", "AKIABOB", "bobsecret", + `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"},{"Effect":"Deny","Action":"s3:DeleteObject","Resource":"*"}]}`) + + rootAcc := Account{Access: standaloneTestRootAccess, Secret: standaloneTestRootSecret, Role: RoleAdmin} + client, err := NewIAMServiceStandalone(rootAcc, IAMServiceStandaloneConfig{Endpoint: sock}) + if err != nil { + t.Fatalf("NewIAMServiceStandalone: %v", err) + } + defer client.Shutdown() + + tests := []struct { + name string + action Action + want policyDecision + }{ + {name: "allowed action", action: Action("s3:GetObject"), want: policyDecisionAllow}, + {name: "action with no matching statement", action: Action("s3:PutObject"), want: policyDecisionNoMatch}, + {name: "explicitly denied action", action: Action("s3:DeleteObject"), want: policyDecisionDeny}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + eval, err := client.EvaluatePolicy("AKIABOB", "", []Action{tt.action}, []string{"*"}, nil) + if err != nil { + t.Fatalf("EvaluatePolicy: %v", err) + } + if len(eval.Decisions) != 1 || len(eval.Decisions[0]) != 1 || eval.Decisions[0][0] != tt.want { + t.Errorf("Decisions = %v, want [[%v]]", eval.Decisions, tt.want) + } + }) + } +} + +// TestIAMServiceStandaloneEvaluatePolicyBatchesMultipleActions confirms +// several actions are evaluated in a single request, with Decisions +// returned in the same order as the requested actions — the fix for +// identityPolicyDecision previously issuing one round trip per action. +func TestIAMServiceStandaloneEvaluatePolicyBatchesMultipleActions(t *testing.T) { + store, sock := standaloneTestServer(t) + createStandaloneTestUser(t, store, "bob", "AKIABOB", "bobsecret", + `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"},{"Effect":"Deny","Action":"s3:DeleteObject","Resource":"*"}]}`) + + rootAcc := Account{Access: standaloneTestRootAccess, Secret: standaloneTestRootSecret, Role: RoleAdmin} + client, err := NewIAMServiceStandalone(rootAcc, IAMServiceStandaloneConfig{Endpoint: sock}) + if err != nil { + t.Fatalf("NewIAMServiceStandalone: %v", err) + } + defer client.Shutdown() + + eval, err := client.EvaluatePolicy("AKIABOB", "", []Action{"s3:GetObject", "s3:PutObject", "s3:DeleteObject"}, []string{"*"}, nil) + if err != nil { + t.Fatalf("EvaluatePolicy: %v", err) + } + want := []policyDecision{policyDecisionAllow, policyDecisionNoMatch, policyDecisionDeny} + if len(eval.Decisions) != 1 || len(eval.Decisions[0]) != len(want) { + t.Fatalf("Decisions = %v, want [%v]", eval.Decisions, want) + } + for i := range want { + if eval.Decisions[0][i] != want[i] { + t.Errorf("Decisions[0][%d] = %v, want %v", i, eval.Decisions[0][i], want[i]) + } + } +} + +func TestIAMServiceStandaloneMutatingMethodsNotSupported(t *testing.T) { + _, sock := standaloneTestServer(t) + + rootAcc := Account{Access: standaloneTestRootAccess, Secret: standaloneTestRootSecret, Role: RoleAdmin} + client, err := NewIAMServiceStandalone(rootAcc, IAMServiceStandaloneConfig{Endpoint: sock}) + if err != nil { + t.Fatalf("NewIAMServiceStandalone: %v", err) + } + defer client.Shutdown() + + notSupported := s3err.GetAPIError(s3err.ErrAdminMethodNotSupported) + + if err := client.CreateAccount(Account{}); !errors.Is(err, notSupported) { + t.Errorf("CreateAccount() error = %v, want %v", err, notSupported) + } + if err := client.UpdateUserAccount("x", MutableProps{}); !errors.Is(err, notSupported) { + t.Errorf("UpdateUserAccount() error = %v, want %v", err, notSupported) + } + if err := client.DeleteUserAccount("x"); !errors.Is(err, notSupported) { + t.Errorf("DeleteUserAccount() error = %v, want %v", err, notSupported) + } + if _, err := client.ListUserAccounts(); !errors.Is(err, notSupported) { + t.Errorf("ListUserAccounts() error = %v, want %v", err, notSupported) + } +} + +func TestNewIAMServiceStandaloneRequiresMTLSForTCPEndpoint(t *testing.T) { + rootAcc := Account{Access: standaloneTestRootAccess, Secret: standaloneTestRootSecret} + _, err := NewIAMServiceStandalone(rootAcc, IAMServiceStandaloneConfig{Endpoint: "127.0.0.1:9443"}) + if err == nil { + t.Fatal("expected an error constructing a TCP-endpoint client without mTLS configured") + } +} + +// TestNewIAMServiceStandaloneDefaultsToRootCredentials confirms this +// client's own signing identity (the credential it signs its private +// requests with) falls back to the gateway's root account when +// Access/Secret aren't explicitly configured — so a deployment doesn't need +// to mint a dedicated IAM identity just for the gateway to talk to its own +// standalone IAM service. +func TestNewIAMServiceStandaloneDefaultsToRootCredentials(t *testing.T) { + rootAcc := Account{Access: standaloneTestRootAccess, Secret: standaloneTestRootSecret} + _, sock := standaloneTestServer(t) + + client, err := NewIAMServiceStandalone(rootAcc, IAMServiceStandaloneConfig{Endpoint: sock}) + if err != nil { + t.Fatalf("NewIAMServiceStandalone: %v", err) + } + defer client.Shutdown() + + if client.access != rootAcc.Access { + t.Errorf("access = %q, want root access %q", client.access, rootAcc.Access) + } + if client.secret != rootAcc.Secret { + t.Errorf("secret = %q, want root secret %q", client.secret, rootAcc.Secret) + } + + // Also confirm the client actually works end-to-end when signing with + // the defaulted root identity, not just that the fields were set. + if _, err := client.GetUserAccount(standaloneTestRootAccess); err != nil { + t.Fatalf("GetUserAccount(root) with defaulted signing identity: %v", err) + } +} + +// TestNewIAMServiceStandaloneRespectsExplicitCredentials confirms an +// explicitly configured Access/Secret is used as-is, not overridden by the +// root account's credentials. +func TestNewIAMServiceStandaloneRespectsExplicitCredentials(t *testing.T) { + rootAcc := Account{Access: standaloneTestRootAccess, Secret: standaloneTestRootSecret} + _, sock := standaloneTestServer(t) + + client, err := NewIAMServiceStandalone(rootAcc, IAMServiceStandaloneConfig{ + Endpoint: sock, + Access: "AKIDCUSTOM", + Secret: "CUSTOMSECRET", + }) + if err != nil { + t.Fatalf("NewIAMServiceStandalone: %v", err) + } + defer client.Shutdown() + + if client.access != "AKIDCUSTOM" { + t.Errorf("access = %q, want %q", client.access, "AKIDCUSTOM") + } + if client.secret != "CUSTOMSECRET" { + t.Errorf("secret = %q, want %q", client.secret, "CUSTOMSECRET") + } +} diff --git a/auth/iam_vault.go b/auth/iam_vault.go index 6b3c8c08..23a2ae34 100644 --- a/auth/iam_vault.go +++ b/auth/iam_vault.go @@ -261,6 +261,11 @@ func (vt *VaultIAMService) GetUserAccount(access string) (Account, error) { return acc, nil } +// ResolveAccounts returns the subset of accessKeyIDs that do not exist. +func (vt *VaultIAMService) ResolveAccounts(accessKeyIDs []string) ([]string, error) { + return resolveAccountsByLookup(accessKeyIDs, vt.GetUserAccount) +} + func (vt *VaultIAMService) UpdateUserAccount(access string, props MutableProps) error { acc, err := vt.GetUserAccount(access) if err != nil { diff --git a/auth/object_lock.go b/auth/object_lock.go index ba57ebda..e6f72521 100644 --- a/auth/object_lock.go +++ b/auth/object_lock.go @@ -23,6 +23,7 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v3" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/debuglogger" "github.com/versity/versitygw/s3err" @@ -35,6 +36,48 @@ type BucketLockConfig struct { CreatedAt *time.Time } +// BypassMode says whether, and on whose authority, a request may override a +// GOVERNANCE-mode retention. It exists because the two ways that can happen +// are not equivalent, and collapsing them into one boolean previously let +// root overwrite locked objects it should not have been able to. +type BypassMode int + +const ( + // BypassNone is a request that has not asked to override anything: any + // unexpired retention blocks it outright. + BypassNone BypassMode = iota + + // BypassRequested is a request carrying x-amz-bypass-governance-retention + // — DeleteObject, DeleteObjects, or PutObjectRetention. Root and admin + // may always override a GOVERNANCE retention this way, matching real + // AWS, where the account root can bypass regardless of policy; everyone + // else needs s3:BypassGovernanceRetention. + BypassRequested + + // BypassOverwrite is the gateway's own extension: replacing an existing + // governance-locked object via PutObject, CopyObject or POST Object, + // none of which has a bypass header for a client to send. Because the + // caller never asked to override anything, the permission is required + // from everyone here — root included — and root's blanket bypass above + // deliberately does not apply. (Real S3 has no analogue: it only allows + // object lock on versioned buckets, where an overwrite creates a new + // version rather than replacing a locked one.) + BypassOverwrite +) + +// allowsGovernanceOverride reports whether this mode permits overriding a +// GOVERNANCE retention at all, given the permission to do so. +func (b BypassMode) allowsGovernanceOverride() bool { return b != BypassNone } + +// BypassModeForRequest maps the presence of the client's +// x-amz-bypass-governance-retention header onto a BypassMode. +func BypassModeForRequest(headerPresent bool) BypassMode { + if headerPresent { + return BypassRequested + } + return BypassNone +} + const ( maxObjectLockRetentionDays int32 = 36500 maxObjectLockRetentionYears int32 = 100 @@ -138,8 +181,8 @@ func ParseObjectLockRetentionInputToJSON(input *s3response.PutObjectRetentionInp // IsObjectLockRetentionPutAllowed checks if the object lock retention PUT request // is allowed against the current state of the object lock -func IsObjectLockRetentionPutAllowed(ctx context.Context, be backend.Backend, bucket, object, versionId, userAccess string, input *s3response.PutObjectRetentionInput, bypass bool) error { - ret, err := be.GetObjectRetention(ctx, bucket, object, versionId) +func IsObjectLockRetentionPutAllowed(ctx fiber.Ctx, be backend.Backend, iam IAMService, bucket, object, versionId string, acc Account, input *s3response.PutObjectRetentionInput, bypass bool) error { + ret, err := be.GetObjectRetention(ctx.RequestCtx(), bucket, object, versionId) if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) { // if object lock configuration is not set // allow the retention modification without any checks @@ -155,46 +198,164 @@ func IsObjectLockRetentionPutAllowed(ctx context.Context, be backend.Backend, bu return err } - if retention.Mode == input.Mode { - // if retention mode is the same - // the operation is allowed + // Pushing the date further out only ever strengthens the lock, so it + // needs nothing beyond s3:PutObjectRetention — in either mode. Anything + // that weakens it, an earlier date or a mode change, does not. + // + // A stored retention carrying no date can't be compared, so it counts as + // weakenable rather than being assumed an extension — the fail-closed + // direction. + isExtension := retention.Mode == input.Mode && + retention.RetainUntilDate != nil && + !input.RetainUntilDate.Time.Before(*retention.RetainUntilDate) + if isExtension { return nil } if retention.Mode == types.ObjectLockRetentionModeCompliance { - // COMPLIANCE mode is by definition not allowed to modify - debuglogger.Logf("object lock retention change request from 'COMPLIANCE' to 'GOVERNANCE' is not allowed") + // COMPLIANCE is absolute until it expires: it can be extended (above) + // but never shortened, and never downgraded to GOVERNANCE — by + // anyone, with any permission, including the account root. That + // immutability is the whole point of the mode, and real AWS rejects + // a shortening PutObjectRetention on a COMPLIANCE object even with + // the bypass header present. + debuglogger.Logf("weakening a 'COMPLIANCE' object lock retention is not allowed") return s3err.GetAPIError(s3err.ErrObjectLocked) } if !bypass { // if x-amz-bypass-governance-retention is not provided // return error: object is locked - debuglogger.Logf("object lock retention mode change is not allowed and bypass governence is not forced") + debuglogger.Logf("weakening a 'GOVERNANCE' object lock retention is not allowed without the bypass governance header") return s3err.GetAPIError(s3err.ErrObjectLocked) } - // the last case left, when user tries to chenge - // from 'GOVERNANCE' to 'COMPLIANCE' with - // 'x-amz-bypass-governance-retention' header - // first we need to check if user has 's3:BypassGovernanceRetention' - policy, err := be.GetBucketPolicy(ctx, bucket) - if err != nil { - // if it fails to get the policy, return object is locked - debuglogger.Logf("failed to get the bucket policy: %v", err) - return s3err.GetAPIError(s3err.ErrObjectLocked) - } - err = VerifyBucketPolicy(policy, userAccess, bucket, object, be.NormalizeObjectKey, BypassGovernanceRetentionAction) - if err != nil { - // if user doesn't have "s3:BypassGovernanceRetention" permission - // return object is locked - debuglogger.Logf("the user is missing 's3:BypassGovernanceRetention' permission") - return s3err.GetAPIError(s3err.ErrObjectLocked) + // What's left is weakening a GOVERNANCE retention — shortening its date, + // or switching it to COMPLIANCE — with the bypass header. That needs + // s3:BypassGovernanceRetention, via the bucket policy and/or (when + // configured) the IAM identity policy. + if err := verifyBypassGovernancePermission(ctx.RequestCtx(), be, iam, acc, bucket, object, BypassRequested, false, requestConditionContext(ctx)); err != nil { + debuglogger.Logf("the user is missing 's3:BypassGovernanceRetention' permission: %v", err) + return err } return nil } +// verifyBypassGovernancePermission decides whether acc may use +// x-amz-bypass-governance-retention to override a GOVERNANCE-mode lock on +// bucket/key. For a public (anonymous) request it consults only the +// bucket's public policy grant, wrapped in the generic ErrObjectLocked. For +// an authenticated request it combines the bucket policy decision with an +// identity-policy decision from iam when it implements PolicyEvaluator +// (currently only the standalone IAM service client), using the same +// explicit-deny-wins precedence as VerifyAccess. Unlike the "no header" +// case, a failed permission check here is reported as the specific +// AccessDenied error naming s3:BypassGovernanceRetention, not the generic +// "object protected by object lock" message — that message is reserved for +// when the bypass header itself is absent, or for backends with no +// identity-policy layer at all, where it preserves the existing behavior. +func verifyBypassGovernancePermission(ctx context.Context, be backend.Backend, iam IAMService, acc Account, bucket, key string, mode BypassMode, isBucketPublic bool, condCtx map[string][]string) error { + // Root and admin override a GOVERNANCE retention unconditionally when + // the client actually asked to — matching real AWS, where the account + // root can bypass whatever the policies say. + // + // This deliberately does not extend to BypassOverwrite: there the + // caller never requested a bypass (no S3 write API has a header for + // it), so there is nothing to grant root on their behalf, and letting + // it through would mean root silently replacing locked objects. See + // BypassMode. + if mode == BypassRequested && acc.Role == RoleAdmin { + return nil + } + + if isBucketPublic { + policy, err := be.GetBucketPolicy(ctx, bucket) + if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) { + return s3err.GetAPIError(s3err.ErrObjectLocked) + } + if err != nil { + return err + } + if err := VerifyPublicBucketPolicy(policy, bucket, key, condCtx, be.NormalizeObjectKey, BypassGovernanceRetentionAction); err != nil { + return s3err.GetAPIError(s3err.ErrObjectLocked) + } + return nil + } + + var resourceDecision policyDecision + policy, err := be.GetBucketPolicy(ctx, bucket) + switch { + case errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)): + resourceDecision = policyDecisionNoMatch + case err != nil: + return err + default: + resourceDecision, _, err = verifyBucketPolicy(policy, acc.Access, bucket, key, condCtx, be.NormalizeObjectKey, BypassGovernanceRetentionAction) + if err != nil { + return err + } + } + + resourceArn := objectPolicyArn(bucket, key, be.NormalizeObjectKey) + + if resourceDecision == policyDecisionDeny { + return s3err.GetExplicitDenyAccessErr(acc.Access, string(BypassGovernanceRetentionAction), resourceArn, "a resource-based policy") + } + + pe, hasPolicyEvaluator := iam.(PolicyEvaluator) + + // Only BypassOverwrite reaches here as root — BypassRequested already + // returned above. Root has no identity policy to evaluate: with the + // standalone IAM backend it is not an IAM user at all, so asking that + // service about it would fail with ErrNoSuchUser rather than return a + // decision. It therefore falls back to the bucket-policy decision alone, + // exactly as a backend with no identity-policy layer does, and so still + // needs an explicit grant to replace a locked object. + if !hasPolicyEvaluator || acc.Role == RoleAdmin { + // No identity-policy layer for this backend: preserve today's exact + // behavior for every internal/LDAP/Vault/IPA deployment. + if resourceDecision == policyDecisionAllow { + return nil + } + return s3err.GetAPIError(s3err.ErrObjectLocked) + } + + identity, err := identityPolicyDecisions(pe, AccessOptions{ + Acc: acc, + Bucket: bucket, + Object: key, + Actions: []Action{BypassGovernanceRetentionAction}, + }, []string{key}, be.NormalizeObjectKey, condCtx) + if err != nil { + return err + } + + identityDecision := identity.Decisions[0].Decision + sessionDenies := identity.HasSessionPolicy && identity.SessionDecisions[0].Decision == policyDecisionDeny + // A session policy filters this permission the same way it filters any + // other: it can only take away what the role or the bucket policy grants. + sessionWithholds := identity.HasSessionPolicy && identity.SessionDecisions[0].Decision != policyDecisionAllow + + if identityDecision == policyDecisionDeny || sessionDenies { + principal := identity.PrincipalArn + if principal == "" { + principal = acc.Access + } + return s3err.GetExplicitDenyAccessErr(principal, string(BypassGovernanceRetentionAction), resourceArn, "an identity-based policy") + } + if !sessionWithholds && + (resourceDecision == policyDecisionAllow || identityDecision == policyDecisionAllow) { + return nil + } + + principal := identity.PrincipalArn + if principal == "" { + principal = acc.Access + } + return s3err.GetImplicitDenyAccessErr(principal, string(BypassGovernanceRetentionAction), resourceArn) +} + func ParseObjectLockRetentionOutput(input []byte) (*types.ObjectLockRetention, error) { var retention types.ObjectLockRetention if err := json.Unmarshal(input, &retention); err != nil { @@ -221,35 +382,74 @@ func ParseObjectLegalHoldOutput(status *bool) *s3response.GetObjectLegalHoldResu } } -func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass, isBucketPublic bool, be backend.Backend, isOverwrite bool) error { +// CheckObjectAccess enforces the object locks protecting objects, for the +// single-object write paths. The multi-object delete path uses +// VerifyObjectsAccess instead, which folds this together with the +// authorization check into one pass. +func CheckObjectAccess(ctx fiber.Ctx, bucket string, acc Account, objects []types.ObjectIdentifier, bypass BypassMode, isBucketPublic bool, be backend.Backend, iam IAMService, isOverwrite bool) error { + rctx := ctx.RequestCtx() + state, err := loadObjectLockState(rctx, be, bucket, isOverwrite) + if err != nil || !state.applies { + return err + } + + condCtx := requestConditionContext(ctx) + for _, obj := range objects { + if err := state.checkObject(rctx, be, iam, acc, bucket, obj, bypass, isBucketPublic, condCtx); err != nil { + return err + } + } + + return nil +} + +// objectLockState is the bucket-level object-lock configuration a request is +// evaluated against, resolved once so a request naming many objects doesn't +// re-fetch it per key. +type objectLockState struct { + // applies is false when nothing about this bucket can block the request: + // object lock is off, unconfigured, or the write creates a new version + // rather than replacing anything. + applies bool + // defaultRetention is the bucket's default retention, only set when it + // is configured and still in force. + defaultRetention *types.DefaultRetention + // versioningEnabled makes a delete without a version id a new delete + // marker, which no retention protects against. + versioningEnabled bool +} + +func loadObjectLockState(ctx context.Context, be backend.Backend, bucket string, isOverwrite bool) (objectLockState, error) { + var state objectLockState + if isOverwrite { // if bucket versioning is enabled, any overwrite request // should be enabled, as it leads to a new object version // creation res, err := be.GetBucketVersioning(ctx, bucket) if err == nil && res.Status != nil && *res.Status == types.BucketVersioningStatusEnabled { - return nil + return state, nil } } + data, err := be.GetObjectLockConfiguration(ctx, bucket) if err != nil { if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) { - return nil + return state, nil } - return err + return state, err } var bucketLockConfig BucketLockConfig if err := json.Unmarshal(data, &bucketLockConfig); err != nil { - return fmt.Errorf("parse object lock config: %w", err) + return state, fmt.Errorf("parse object lock config: %w", err) } if !bucketLockConfig.Enabled { - return nil + return state, nil } - - checkDefaultRetention := false + state.applies = true if bucketLockConfig.DefaultRetention != nil && bucketLockConfig.CreatedAt != nil { expirationDate := *bucketLockConfig.CreatedAt @@ -261,130 +461,114 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [ } if expirationDate.After(time.Now()) { - checkDefaultRetention = true + state.defaultRetention = bucketLockConfig.DefaultRetention } } - var versioningEnabled bool vers, err := be.GetBucketVersioning(ctx, bucket) if err == nil && vers.Status != nil { - versioningEnabled = *vers.Status == types.BucketVersioningStatusEnabled + state.versioningEnabled = *vers.Status == types.BucketVersioningStatusEnabled } - for _, obj := range objects { - var key, versionId string - if obj.Key != nil { - key = *obj.Key - } - if obj.VersionId != nil { - versionId = *obj.VersionId - } - // if bucket versioning is enabled and versionId isn't provided - // no lock check is needed, as it leads to a new delete marker creation - if versioningEnabled && versionId == "" { - continue - } - checkRetention := true - retentionData, err := be.GetObjectRetention(ctx, bucket, key, versionId) - if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) { - continue - } - // the object is a delete marker, if a `MethodNotAllowed` error is returned - // no object lock check is needed - if errors.Is(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)) { - continue - } - if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) { - checkRetention = false - } - if err != nil && checkRetention { + return state, nil +} + +// checkObject reports whether one object's retention or legal hold blocks +// this request. A nil error means this object is writable; it says nothing +// about any other object in the same request. +func (s objectLockState) checkObject(ctx context.Context, be backend.Backend, iam IAMService, acc Account, bucket string, obj types.ObjectIdentifier, bypass BypassMode, isBucketPublic bool, condCtx map[string][]string) error { + var key, versionId string + if obj.Key != nil { + key = *obj.Key + } + if obj.VersionId != nil { + versionId = *obj.VersionId + } + + // if bucket versioning is enabled and versionId isn't provided + // no lock check is needed, as it leads to a new delete marker creation + if s.versioningEnabled && versionId == "" { + return nil + } + + checkRetention := true + retentionData, err := be.GetObjectRetention(ctx, bucket, key, versionId) + if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) { + return nil + } + // the object is a delete marker, if a `MethodNotAllowed` error is returned + // no object lock check is needed + if errors.Is(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)) { + return nil + } + if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) { + checkRetention = false + } + if err != nil && checkRetention { + return err + } + + if checkRetention { + retention, err := ParseObjectLockRetentionOutput(retentionData) + if err != nil { return err } - if checkRetention { - retention, err := ParseObjectLockRetentionOutput(retentionData) - if err != nil { - return err + if retention.Mode != "" && retention.RetainUntilDate != nil { + // An expired retention protects nothing, and an object's own + // retention supersedes the bucket default, so this object is + // past its lock. Note this also skips the legal-hold check + // below, preserving long-standing behavior; it returns for + // this object only, where the same statement previously + // short-circuited the caller's whole request and let every + // remaining object through unchecked. + if retention.RetainUntilDate.Before(time.Now()) { + return nil } - if retention.Mode != "" && retention.RetainUntilDate != nil { - if retention.RetainUntilDate.Before(time.Now()) { - // if the object retention is expired, the object - // is allowed for write operations(delete, modify) - return nil - } - - switch retention.Mode { - case types.ObjectLockRetentionModeGovernance: - if !bypass { - return s3err.GetAPIError(s3err.ErrObjectLocked) - } else { - policy, err := be.GetBucketPolicy(ctx, bucket) - if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) { - return s3err.GetAPIError(s3err.ErrObjectLocked) - } - if err != nil { - return err - } - if isBucketPublic { - err = VerifyPublicBucketPolicy(policy, bucket, key, be.NormalizeObjectKey, BypassGovernanceRetentionAction) - } else { - err = VerifyBucketPolicy(policy, userAccess, bucket, key, be.NormalizeObjectKey, BypassGovernanceRetentionAction) - } - if err != nil { - return s3err.GetAPIError(s3err.ErrObjectLocked) - } - } - case types.ObjectLockRetentionModeCompliance: - return s3err.GetAPIError(s3err.ErrObjectLocked) - } - } - } - - checkLegalHold := true - - status, err := be.GetObjectLegalHold(ctx, bucket, key, versionId) - if err != nil { - if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) { - continue - } - if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) { - checkLegalHold = false - } else { + if err := s.checkRetentionMode(ctx, be, iam, acc, bucket, key, retention.Mode, bypass, isBucketPublic, condCtx); err != nil { return err } } + } - if checkLegalHold && *status { - return s3err.GetAPIError(s3err.ErrObjectLocked) - } + checkLegalHold := true - if checkDefaultRetention { - switch bucketLockConfig.DefaultRetention.Mode { - case types.ObjectLockRetentionModeGovernance: - if !bypass { - return s3err.GetAPIError(s3err.ErrObjectLocked) - } else { - policy, err := be.GetBucketPolicy(ctx, bucket) - if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) { - return s3err.GetAPIError(s3err.ErrObjectLocked) - } - if err != nil { - return err - } - if isBucketPublic { - err = VerifyPublicBucketPolicy(policy, bucket, key, be.NormalizeObjectKey, BypassGovernanceRetentionAction) - } else { - err = VerifyBucketPolicy(policy, userAccess, bucket, key, be.NormalizeObjectKey, BypassGovernanceRetentionAction) - } - if err != nil { - return s3err.GetAPIError(s3err.ErrObjectLocked) - } - } - case types.ObjectLockRetentionModeCompliance: - return s3err.GetAPIError(s3err.ErrObjectLocked) - } + status, err := be.GetObjectLegalHold(ctx, bucket, key, versionId) + if err != nil { + if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) { + return nil } + if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) { + checkLegalHold = false + } else { + return err + } + } + + if checkLegalHold && *status { + return s3err.GetAPIError(s3err.ErrObjectLocked) + } + + if s.defaultRetention != nil { + return s.checkRetentionMode(ctx, be, iam, acc, bucket, key, s.defaultRetention.Mode, bypass, isBucketPublic, condCtx) + } + + return nil +} + +// checkRetentionMode applies one retention mode's rule: COMPLIANCE blocks +// unconditionally, GOVERNANCE blocks unless the request both asked to +// override it and is permitted to. +func (s objectLockState) checkRetentionMode(ctx context.Context, be backend.Backend, iam IAMService, acc Account, bucket, key string, mode types.ObjectLockRetentionMode, bypass BypassMode, isBucketPublic bool, condCtx map[string][]string) error { + switch mode { + case types.ObjectLockRetentionModeGovernance: + if !bypass.allowsGovernanceOverride() { + return s3err.GetAPIError(s3err.ErrObjectLocked) + } + return verifyBypassGovernancePermission(ctx, be, iam, acc, bucket, key, bypass, isBucketPublic, condCtx) + case types.ObjectLockRetentionModeCompliance: + return s3err.GetAPIError(s3err.ErrObjectLocked) } return nil diff --git a/auth/object_lock_test.go b/auth/object_lock_test.go new file mode 100644 index 00000000..65d54107 --- /dev/null +++ b/auth/object_lock_test.go @@ -0,0 +1,318 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +// TestVerifyBypassGovernancePermission_IdentityAllowNoBucketPolicy is the +// same-account fix this function exists for: an IAM identity policy Allow +// is sufficient to use x-amz-bypass-governance-retention even when the +// bucket has no policy at all. The old bucket-policy-only check treated "no +// bucket policy" as an immediate ErrObjectLocked, never even consulting the +// identity policy. +func TestVerifyBypassGovernancePermission_IdentityAllowNoBucketPolicy(t *testing.T) { + be := noBucketPolicyBackend{} + pe := newMockPolicyEvaluator(policyDecisionAllow) + + err := verifyBypassGovernancePermission(context.Background(), be, pe, Account{Access: "testuser"}, "bucket", "key.txt", BypassRequested, false, nil) + + assert.NoError(t, err) +} + +// TestVerifyBypassGovernancePermission_ResourceAllowIdentitySilent is the +// reverse: a bucket policy Allow is sufficient when the identity policy has +// no opinion on the action, but the identity policy must still be +// consulted (not skipped) so an explicit Deny there can override it. +func TestVerifyBypassGovernancePermission_ResourceAllowIdentitySilent(t *testing.T) { + be := &publicBucketPolicyBackend{ + policy: []byte(`{ + "Statement": [{ + "Effect": "Allow", + "Principal": "testuser", + "Action": "s3:BypassGovernanceRetention", + "Resource": "arn:aws:s3:::bucket/*" + }] + }`), + } + pe := newMockPolicyEvaluator(policyDecisionNoMatch) + + err := verifyBypassGovernancePermission(context.Background(), be, pe, Account{Access: "testuser"}, "bucket", "key.txt", BypassRequested, false, nil) + + assert.NoError(t, err) + assert.Len(t, pe.calls, 1, "identity policy must be consulted even though the bucket policy already allows") +} + +// TestVerifyBypassGovernancePermission_IdentityExplicitDenyOverridesResourceAllow +// is the explicit-deny-wins case: a bucket policy Allow does not save a +// bypass request the caller's own identity policy explicitly denies. AWS +// reports this as a specific AccessDenied naming +// s3:BypassGovernanceRetention, not the generic "object protected by object +// lock" message. +func TestVerifyBypassGovernancePermission_IdentityExplicitDenyOverridesResourceAllow(t *testing.T) { + be := &publicBucketPolicyBackend{ + policy: []byte(`{ + "Statement": [{ + "Effect": "Allow", + "Principal": "testuser", + "Action": "s3:BypassGovernanceRetention", + "Resource": "arn:aws:s3:::bucket/*" + }] + }`), + } + pe := newMockPolicyEvaluator(policyDecisionDeny) + pe.principalArn = "arn:aws:iam::000000000000:user/testuser" + + err := verifyBypassGovernancePermission(context.Background(), be, pe, Account{Access: "testuser"}, "bucket", "key.txt", BypassRequested, false, nil) + + apiErr := requireAccessDeniedAPIError(t, err) + assert.Contains(t, apiErr.Description, "arn:aws:iam::000000000000:user/testuser") + assert.Contains(t, apiErr.Description, "s3:BypassGovernanceRetention") + assert.Contains(t, apiErr.Description, "with an explicit deny in an identity-based policy") +} + +// TestVerifyBypassGovernancePermission_ResourceExplicitDenyOverridesIdentityAllow +// is the reverse: an identity policy Allow does not save a bypass request +// the bucket policy explicitly denies, and the resource-level Deny +// short-circuits before the identity policy is even consulted. +func TestVerifyBypassGovernancePermission_ResourceExplicitDenyOverridesIdentityAllow(t *testing.T) { + be := &publicBucketPolicyBackend{ + policy: []byte(`{ + "Statement": [{ + "Effect": "Deny", + "Principal": "testuser", + "Action": "s3:BypassGovernanceRetention", + "Resource": "arn:aws:s3:::bucket/*" + }] + }`), + } + pe := newMockPolicyEvaluator(policyDecisionAllow) + + err := verifyBypassGovernancePermission(context.Background(), be, pe, Account{Access: "testuser"}, "bucket", "key.txt", BypassRequested, false, nil) + + apiErr := requireAccessDeniedAPIError(t, err) + assert.Contains(t, apiErr.Description, "with an explicit deny in a resource-based policy") + assert.Empty(t, pe.calls, "a resource-level explicit deny should short-circuit before consulting the identity policy") +} + +// TestVerifyBypassGovernancePermission_ImplicitDenyWhenNeitherAllows: with +// no bucket policy and no identity-policy grant, AWS denies with "because +// no identity-based policy allows the s3:BypassGovernanceRetention action" +// — the same implicit-deny shape VerifyAccess uses for ordinary actions. +func TestVerifyBypassGovernancePermission_ImplicitDenyWhenNeitherAllows(t *testing.T) { + be := noBucketPolicyBackend{} + pe := newMockPolicyEvaluator(policyDecisionNoMatch) + pe.principalArn = "arn:aws:iam::000000000000:user/testuser" + + err := verifyBypassGovernancePermission(context.Background(), be, pe, Account{Access: "testuser"}, "bucket", "key.txt", BypassRequested, false, nil) + + apiErr := requireAccessDeniedAPIError(t, err) + assert.Contains(t, apiErr.Description, "arn:aws:iam::000000000000:user/testuser") + assert.Contains(t, apiErr.Description, "because no identity-based policy allows the s3:BypassGovernanceRetention action") +} + +// TestVerifyBypassGovernancePermission_NoPolicyEvaluatorPreservesGenericMessage +// confirms backends with no identity-policy layer (every backend except the +// standalone IAM service) are unaffected: the generic ErrObjectLocked stays +// exactly as before when there is no bucket policy to grant the bypass. +func TestVerifyBypassGovernancePermission_NoPolicyEvaluatorPreservesGenericMessage(t *testing.T) { + be := noBucketPolicyBackend{} + iam := NewIAMServiceSingle(Account{}) + + err := verifyBypassGovernancePermission(context.Background(), be, iam, Account{Access: "testuser"}, "bucket", "key.txt", BypassRequested, false, nil) + + assert.Equal(t, s3err.GetAPIError(s3err.ErrObjectLocked), err) +} + +// TestVerifyBypassGovernancePermission_NoPolicyEvaluatorBucketPolicyAllowStillWorks +// pins that, without a PolicyEvaluator, a bucket policy Allow alone is still +// sufficient — the pre-existing (bucket-policy-only) behavior. +func TestVerifyBypassGovernancePermission_NoPolicyEvaluatorBucketPolicyAllowStillWorks(t *testing.T) { + be := &publicBucketPolicyBackend{ + policy: []byte(`{ + "Statement": [{ + "Effect": "Allow", + "Principal": "testuser", + "Action": "s3:BypassGovernanceRetention", + "Resource": "arn:aws:s3:::bucket/*" + }] + }`), + } + iam := NewIAMServiceSingle(Account{}) + + err := verifyBypassGovernancePermission(context.Background(), be, iam, Account{Access: "testuser"}, "bucket", "key.txt", BypassRequested, false, nil) + + assert.NoError(t, err) +} + +// TestVerifyBypassGovernancePermission_PublicBucketAllowed and +// TestVerifyBypassGovernancePermission_PublicBucketDenied confirm the +// isBucketPublic branch (anonymous requests, evaluated only against the +// bucket's public policy grant, wrapped in the generic ErrObjectLocked) is +// unchanged by this refactor. +func TestVerifyBypassGovernancePermission_PublicBucketAllowed(t *testing.T) { + be := &publicBucketPolicyBackend{ + policy: []byte(`{ + "Statement": [{ + "Effect": "Allow", + "Principal": "*", + "Action": "s3:BypassGovernanceRetention", + "Resource": "arn:aws:s3:::bucket/*" + }] + }`), + } + + err := verifyBypassGovernancePermission(context.Background(), be, nil, Account{}, "bucket", "key.txt", BypassRequested, true, nil) + + assert.NoError(t, err) +} + +func TestVerifyBypassGovernancePermission_PublicBucketDenied(t *testing.T) { + be := &publicBucketPolicyBackend{ + policy: []byte(`{ + "Statement": [{ + "Effect": "Allow", + "Principal": "*", + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::bucket/*" + }] + }`), + } + + err := verifyBypassGovernancePermission(context.Background(), be, nil, Account{}, "bucket", "key.txt", BypassRequested, true, nil) + + assert.Equal(t, s3err.GetAPIError(s3err.ErrObjectLocked), err) +} + +// TestVerifyBypassGovernancePermission_RootBypassesOnlyWhenRequested pins the +// asymmetry between the two ways a governance retention can be overridden. +// +// Root bypasses unconditionally when the client actually sent +// x-amz-bypass-governance-retention, matching real AWS, where the account +// root can bypass regardless of policy. It does not get that on the +// overwrite path, where no client asked for anything and letting root +// through would mean silently replacing a locked object. +func TestVerifyBypassGovernancePermission_RootBypassesOnlyWhenRequested(t *testing.T) { + root := Account{Access: "root", Role: RoleAdmin} + + // No bucket policy and no identity policy: the only thing that could + // possibly permit this is root's own status. + be := &publicBucketPolicyBackend{policy: []byte(`{"Statement":[]}`)} + pe := newMockPolicyEvaluator(policyDecisionNoMatch) + + err := verifyBypassGovernancePermission(context.Background(), be, pe, root, "bucket", "key.txt", BypassRequested, false, nil) + assert.NoError(t, err, "root must bypass a governance retention it explicitly asked to bypass") + + err = verifyBypassGovernancePermission(context.Background(), be, pe, root, "bucket", "key.txt", BypassOverwrite, false, nil) + assert.Error(t, err, "root must not silently overwrite a governance-locked object: no bypass was requested") + + err = verifyBypassGovernancePermission(context.Background(), be, pe, root, "bucket", "key.txt", BypassNone, false, nil) + assert.Error(t, err, "root must not bypass when the request did not ask to") +} + +// TestVerifyBypassGovernancePermission_NonRootStillNeedsPermission confirms +// the root shortcut is exactly that, and does not leak to ordinary users. +func TestVerifyBypassGovernancePermission_NonRootStillNeedsPermission(t *testing.T) { + user := Account{Access: "testuser", Role: RoleUser} + be := &publicBucketPolicyBackend{policy: []byte(`{"Statement":[]}`)} + + err := verifyBypassGovernancePermission(context.Background(), be, + newMockPolicyEvaluator(policyDecisionNoMatch), user, "bucket", "key.txt", BypassRequested, false, nil) + assert.Error(t, err, "a plain user with no grant anywhere must not bypass") + + err = verifyBypassGovernancePermission(context.Background(), be, + newMockPolicyEvaluator(policyDecisionAllow), user, "bucket", "key.txt", BypassRequested, false, nil) + assert.NoError(t, err, "an identity-policy Allow grants the bypass") +} + +// TestIsObjectLockRetentionPutAllowed_WeakeningRules covers which retention +// rewrites need s3:BypassGovernanceRetention and which need nothing. +// +// Extending a GOVERNANCE or COMPLIANCE retention, or rewriting it with the +// identical date, succeeds with no bypass header, while shortening either +// one without the header fails with "Access Denied because object +// protected by object lock." A COMPLIANCE retention cannot be weakened at +// all, even with the header. +func TestIsObjectLockRetentionPutAllowed_WeakeningRules(t *testing.T) { + now := time.Now() + stored := now.Add(time.Hour) + + tests := []struct { + name string + mode types.ObjectLockRetentionMode + newMode types.ObjectLockRetentionMode + newDate time.Time + bypass bool + wantAllow bool + }{ + {name: "governance extended", mode: types.ObjectLockRetentionModeGovernance, newMode: types.ObjectLockRetentionModeGovernance, newDate: stored.Add(time.Hour), wantAllow: true}, + {name: "governance same date", mode: types.ObjectLockRetentionModeGovernance, newMode: types.ObjectLockRetentionModeGovernance, newDate: stored, wantAllow: true}, + {name: "governance shortened without bypass", mode: types.ObjectLockRetentionModeGovernance, newMode: types.ObjectLockRetentionModeGovernance, newDate: now.Add(time.Minute)}, + {name: "governance shortened with bypass", mode: types.ObjectLockRetentionModeGovernance, newMode: types.ObjectLockRetentionModeGovernance, newDate: now.Add(time.Minute), bypass: true, wantAllow: true}, + {name: "compliance extended", mode: types.ObjectLockRetentionModeCompliance, newMode: types.ObjectLockRetentionModeCompliance, newDate: stored.Add(time.Hour), wantAllow: true}, + {name: "compliance shortened without bypass", mode: types.ObjectLockRetentionModeCompliance, newMode: types.ObjectLockRetentionModeCompliance, newDate: now.Add(time.Minute)}, + {name: "compliance shortened with bypass", mode: types.ObjectLockRetentionModeCompliance, newMode: types.ObjectLockRetentionModeCompliance, newDate: now.Add(time.Minute), bypass: true}, + {name: "compliance downgraded to governance", mode: types.ObjectLockRetentionModeCompliance, newMode: types.ObjectLockRetentionModeGovernance, newDate: stored.Add(time.Hour), bypass: true}, + {name: "governance upgraded to compliance with bypass", mode: types.ObjectLockRetentionModeGovernance, newMode: types.ObjectLockRetentionModeCompliance, newDate: stored, bypass: true, wantAllow: true}, + {name: "governance upgraded to compliance without bypass", mode: types.ObjectLockRetentionModeGovernance, newMode: types.ObjectLockRetentionModeCompliance, newDate: stored}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + retention, err := json.Marshal(types.ObjectLockRetention{Mode: tt.mode, RetainUntilDate: &stored}) + assert.NoError(t, err) + + be := &objectRetentionBackend{retention: retention} + // A permissive evaluator, so any denial below is the retention + // rule talking rather than a missing permission. + pe := newMockPolicyEvaluator(policyDecisionAllow) + + err = IsObjectLockRetentionPutAllowed(testFiberCtx(t), be, pe, "bucket", "key.txt", "", + Account{Access: "testuser", Role: RoleUser}, + &s3response.PutObjectRetentionInput{Mode: tt.newMode, RetainUntilDate: s3response.AmzDate{Time: tt.newDate}}, + tt.bypass) + + if tt.wantAllow { + assert.NoError(t, err) + } else { + assert.Equal(t, s3err.GetAPIError(s3err.ErrObjectLocked), err) + } + }) + } +} + +// objectRetentionBackend serves one canned object retention. +type objectRetentionBackend struct { + backend.BackendUnsupported + retention []byte +} + +func (b *objectRetentionBackend) GetObjectRetention(_ context.Context, _, _, _ string) ([]byte, error) { + return b.retention, nil +} + +func (b *objectRetentionBackend) GetBucketPolicy(_ context.Context, _ string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy) +} diff --git a/auth/post_policy.go b/auth/post_policy.go index 3dc5ba86..747f4acf 100644 --- a/auth/post_policy.go +++ b/auth/post_policy.go @@ -346,6 +346,14 @@ func lookupField(in PostPolicyEvalInput, field string) (string, bool) { // isIgnoredCoverageField reports whether a submitted field is exempt from the // POST policy's field coverage requirement. +// +// x-amz-security-token is deliberately NOT exempt, despite being generated +// by the SDK rather than chosen by the form author. The POST signature +// covers only the base64 policy document, so an uncovered token field would +// be completely unbound — anyone could swap in another session's token. +// Requiring the policy to declare a condition for it is what binds it, and +// is what real AWS requires as well; the SDK's POST presigner emits the +// matching condition for exactly this reason. func isIgnoredCoverageField(field string) bool { return field == "file" || field == "policy" || diff --git a/auth/signing_key_provider.go b/auth/signing_key_provider.go new file mode 100644 index 00000000..aff059f7 --- /dev/null +++ b/auth/signing_key_provider.go @@ -0,0 +1,119 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import "github.com/versity/versitygw/internal/sigv4auth" + +// SigningKeyProvider is implemented by IAM backends that can compute a +// SigV4 derived signing key (kSigning) without ever revealing the account's +// underlying secret to this process — currently only the standalone IAM +// service client (IAMServiceStandalone). Callers that resolve an +// IAMService's derived key type-assert for this interface first and fall +// back to fetching the account's secret via GetUserAccount and deriving the +// key locally (sigv4auth.DeriveKey) when it isn't implemented, so every +// other backend (internal, LDAP, Vault, IPA, S3) is unaffected. +// +// date/region/service are the request's credential-scope components +// (yyyymmdd/region/service, matching sigv4auth.DeriveKey's parameters). +// sessionToken is the request's X-Amz-Security-Token, required when access +// is a temporary (ASIA…) key and empty otherwise. +// +// The returned Account never has Secret populated. Returns ErrNoSuchUser if +// access does not exist (matching IAMService.GetUserAccount), or +// ErrInvalidSessionToken if the token is missing, wrong, or paired with a +// permanent access key. +type SigningKeyProvider interface { + DeriveSigningKey(access, sessionToken, date, region, service string) ([]byte, Account, error) +} + +// PolicyEvaluation is what a PolicyEvaluator reports for a batch of +// resources and actions evaluated together in a single request. +// +// Decisions[i][j] is the tri-state decision for resources[i] and actions[j], +// in the order both were given to EvaluatePolicy. PrincipalArn is the +// (best-effort) resolved principal ARN, shared across the whole batch since +// one call always evaluates a single identity — it is used only to build an +// AWS-shaped Deny message, and is "" when it can't be resolved, in which +// case the caller falls back to the access key. +// +// SessionDecisions is the same matrix evaluated against the caller's session +// policy alone, meaningful only when HasSessionPolicy is set. A session +// policy filters everything the session can do, including what the bucket +// policy grants it, so it cannot be folded into Decisions — which describe +// only the identity (user or role) policies. +type PolicyEvaluation struct { + Decisions [][]policyDecision + SessionDecisions [][]policyDecision + HasSessionPolicy bool + PrincipalArn string +} + +// PolicyEvaluator is implemented by IAM backends that enforce IAM identity +// (user/role/session) policies against S3 requests — currently only the +// standalone IAM service client. VerifyAccess type-asserts for this +// interface and, when present, combines its tri-state decision with the +// bucket's own policy/ACL decision using AWS's real precedence: an explicit +// Deny from either source wins outright, otherwise either source's Allow is +// independently sufficient; backends without it are unaffected — there is +// no identity-policy layer for them. +// +// The full actions × resources matrix is evaluated in a single batched +// request rather than one round trip per cell. Both dimensions are really +// used: a copy checks several actions (its source's and destination's), and +// a batch delete checks several resources (one object ARN per key, up to +// 1000 of them). A round trip per cell would multiply request latency for +// no benefit, since one request can carry the whole matrix. +// +// condition carries only the condition keys the gateway itself can observe +// from the request (aws:SourceIp, aws:CurrentTime, aws:SecureTransport, …); +// the identity-derived keys are filled in by the implementation, which is +// the only side that knows who the access key belongs to. +type PolicyEvaluator interface { + EvaluatePolicy(access, sessionToken string, actions []Action, resources []string, condition map[string][]string) (PolicyEvaluation, error) +} + +// ResolveDerivedKey resolves access's SigV4 derived signing key (kSigning) +// and account metadata for date/region/service. root is special-cased +// locally — its secret is already known to this process either way, so +// there's no reason to round-trip it through iam. Otherwise, if iam +// implements SigningKeyProvider, the key is fetched from it directly and +// the account's secret never enters this process; otherwise iam's account +// is resolved via GetUserAccount and the key is derived locally from its +// secret, preserving today's behavior for every backend that doesn't +// implement SigningKeyProvider (internal, LDAP, Vault, IPA, S3). +// +// sessionToken is the request's X-Amz-Security-Token, or "" when it carries +// none. A temporary (ASIA…) access key, or a token paired with a permanent +// one, is only meaningful to a SigningKeyProvider backend: no other backend +// can mint a session, so for them either shape is rejected as +// ErrInvalidSessionToken rather than silently resolving to something else. +func ResolveDerivedKey(iam IAMService, root Account, access, sessionToken, date, region, service string) ([]byte, Account, error) { + if access == root.Access { + if sessionToken != "" { + return nil, Account{}, ErrInvalidSessionToken + } + return sigv4auth.DeriveKey(root.Secret, date, region, service), root, nil + } + if skp, ok := iam.(SigningKeyProvider); ok { + return skp.DeriveSigningKey(access, sessionToken, date, region, service) + } + if sessionToken != "" || sigv4auth.IsTempAccessKeyID(access) { + return nil, Account{}, ErrInvalidSessionToken + } + account, err := iam.GetUserAccount(access) + if err != nil { + return nil, Account{}, err + } + return sigv4auth.DeriveKey(account.Secret, date, region, service), account, nil +} diff --git a/aws/LICENSE.txt b/aws/LICENSE.txt deleted file mode 100644 index d6456956..00000000 --- a/aws/LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/aws/NOTICE.txt b/aws/NOTICE.txt deleted file mode 100644 index 5cc3afc1..00000000 --- a/aws/NOTICE.txt +++ /dev/null @@ -1,4 +0,0 @@ -AWS SDK for Go -Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. -Copyright 2014-2015 Stripe, Inc. -Copyright 2024 Versity Software diff --git a/aws/README.md b/aws/README.md deleted file mode 100644 index fa2d6980..00000000 --- a/aws/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# AWS SDK Go v2 - -This directory contains code from the [AWS SDK Go v2](https://github.com/aws/aws-sdk-go-v2) repository, modified in accordance with the Apache 2.0 License. - -## Description - -The AWS SDK Go v2 is a collection of libraries and tools that enable developers to build applications that integrate with various AWS services. This directory and below contains modified code from the original repository, tailored to suit versitygw specific requirements. - -## License - -The code in this directory is licensed under the Apache 2.0 License. Please refer to the [LICENSE](./LICENSE) file for more information. diff --git a/aws/internal/awstesting/unit/unit.go b/aws/internal/awstesting/unit/unit.go deleted file mode 100644 index 989be144..00000000 --- a/aws/internal/awstesting/unit/unit.go +++ /dev/null @@ -1,61 +0,0 @@ -// Package unit performs initialization and validation for unit tests -package unit - -import ( - "context" - "crypto/rsa" - "math/big" - - "github.com/aws/aws-sdk-go-v2/aws" -) - -func init() { - config = aws.Config{} - config.Region = "mock-region" - config.Credentials = StubCredentialsProvider{} -} - -// StubCredentialsProvider provides a stub credential provider that returns -// static credentials that never expire. -type StubCredentialsProvider struct{} - -// Retrieve satisfies the CredentialsProvider interface. Returns stub -// credential value, and never error. -func (StubCredentialsProvider) Retrieve(context.Context) (aws.Credentials, error) { - return aws.Credentials{ - AccessKeyID: "AKID", SecretAccessKey: "SECRET", SessionToken: "SESSION", - Source: "unit test credentials", - }, nil -} - -var config aws.Config - -// Config returns a copy of the mock configuration for unit tests. -func Config() aws.Config { return config.Copy() } - -// RSAPrivateKey is used for testing functionality that requires some -// sort of private key. Taken from crypto/rsa/rsa_test.go -// -// Credit to golang 1.11 -var RSAPrivateKey = &rsa.PrivateKey{ - PublicKey: rsa.PublicKey{ - N: fromBase10("14314132931241006650998084889274020608918049032671858325988396851334124245188214251956198731333464217832226406088020736932173064754214329009979944037640912127943488972644697423190955557435910767690712778463524983667852819010259499695177313115447116110358524558307947613422897787329221478860907963827160223559690523660574329011927531289655711860504630573766609239332569210831325633840174683944553667352219670930408593321661375473885147973879086994006440025257225431977751512374815915392249179976902953721486040787792801849818254465486633791826766873076617116727073077821584676715609985777563958286637185868165868520557"), - E: 3, - }, - D: fromBase10("9542755287494004433998723259516013739278699355114572217325597900889416163458809501304132487555642811888150937392013824621448709836142886006653296025093941418628992648429798282127303704957273845127141852309016655778568546006839666463451542076964744073572349705538631742281931858219480985907271975884773482372966847639853897890615456605598071088189838676728836833012254065983259638538107719766738032720239892094196108713378822882383694456030043492571063441943847195939549773271694647657549658603365629458610273821292232646334717612674519997533901052790334279661754176490593041941863932308687197618671528035670452762731"), - Primes: []*big.Int{ - fromBase10("130903255182996722426771613606077755295583329135067340152947172868415809027537376306193179624298874215608270802054347609836776473930072411958753044562214537013874103802006369634761074377213995983876788718033850153719421695468704276694983032644416930879093914927146648402139231293035971427838068945045019075433"), - fromBase10("109348945610485453577574767652527472924289229538286649661240938988020367005475727988253438647560958573506159449538793540472829815903949343191091817779240101054552748665267574271163617694640513549693841337820602726596756351006149518830932261246698766355347898158548465400674856021497190430791824869615170301029"), - }, -} - -// Taken from crypto/rsa/rsa_test.go -// -// Credit to golang 1.11 -func fromBase10(base10 string) *big.Int { - i, ok := new(big.Int).SetString(base10, 10) - if !ok { - panic("bad number: " + base10) - } - return i -} diff --git a/aws/signer/internal/v4/cache.go b/aws/signer/internal/v4/cache.go deleted file mode 100644 index cbf22f1d..00000000 --- a/aws/signer/internal/v4/cache.go +++ /dev/null @@ -1,115 +0,0 @@ -package v4 - -import ( - "strings" - "sync" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" -) - -func lookupKey(service, region string) string { - var s strings.Builder - s.Grow(len(region) + len(service) + 3) - s.WriteString(region) - s.WriteRune('/') - s.WriteString(service) - return s.String() -} - -type derivedKey struct { - AccessKey string - Date time.Time - Credential []byte -} - -type derivedKeyCache struct { - values map[string]derivedKey - mutex sync.RWMutex -} - -func newDerivedKeyCache() derivedKeyCache { - return derivedKeyCache{ - values: make(map[string]derivedKey), - } -} - -func (s *derivedKeyCache) Get(credentials aws.Credentials, service, region string, signingTime SigningTime) []byte { - key := lookupKey(service, region) - s.mutex.RLock() - if cred, ok := s.get(key, credentials, signingTime.Time); ok { - s.mutex.RUnlock() - return cred - } - s.mutex.RUnlock() - - s.mutex.Lock() - if cred, ok := s.get(key, credentials, signingTime.Time); ok { - s.mutex.Unlock() - return cred - } - cred := deriveKey(credentials.SecretAccessKey, service, region, signingTime) - entry := derivedKey{ - AccessKey: credentials.AccessKeyID, - Date: signingTime.Time, - Credential: cred, - } - s.values[key] = entry - s.mutex.Unlock() - - return cred -} - -func (s *derivedKeyCache) get(key string, credentials aws.Credentials, signingTime time.Time) ([]byte, bool) { - cacheEntry, ok := s.retrieveFromCache(key) - if ok && cacheEntry.AccessKey == credentials.AccessKeyID && isSameDay(signingTime, cacheEntry.Date) { - return cacheEntry.Credential, true - } - return nil, false -} - -func (s *derivedKeyCache) retrieveFromCache(key string) (derivedKey, bool) { - if v, ok := s.values[key]; ok { - return v, true - } - return derivedKey{}, false -} - -// SigningKeyDeriver derives a signing key from a set of credentials -type SigningKeyDeriver struct { - cache derivedKeyCache -} - -// NewSigningKeyDeriver returns a new SigningKeyDeriver -func NewSigningKeyDeriver() *SigningKeyDeriver { - return &SigningKeyDeriver{ - cache: newDerivedKeyCache(), - } -} - -// DeriveKey returns a derived signing key from the given credentials to be used with SigV4 signing. -func (k *SigningKeyDeriver) DeriveKey(credential aws.Credentials, service, region string, signingTime SigningTime) []byte { - return k.cache.Get(credential, service, region, signingTime) -} - -func deriveKey(secret, service, region string, t SigningTime) []byte { - hmacDate := HMACSHA256([]byte("AWS4"+secret), []byte(t.ShortTimeFormat())) - hmacRegion := HMACSHA256(hmacDate, []byte(region)) - hmacService := HMACSHA256(hmacRegion, []byte(service)) - return HMACSHA256(hmacService, []byte("aws4_request")) -} - -func isSameDay(x, y time.Time) bool { - xYear, xMonth, xDay := x.Date() - yYear, yMonth, yDay := y.Date() - - if xYear != yYear { - return false - } - - if xMonth != yMonth { - return false - } - - return xDay == yDay -} diff --git a/aws/signer/internal/v4/const.go b/aws/signer/internal/v4/const.go deleted file mode 100644 index a23cb003..00000000 --- a/aws/signer/internal/v4/const.go +++ /dev/null @@ -1,40 +0,0 @@ -package v4 - -// Signature Version 4 (SigV4) Constants -const ( - // EmptyStringSHA256 is the hex encoded sha256 value of an empty string - EmptyStringSHA256 = `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855` - - // UnsignedPayload indicates that the request payload body is unsigned - UnsignedPayload = "UNSIGNED-PAYLOAD" - - // AmzAlgorithmKey indicates the signing algorithm - AmzAlgorithmKey = "X-Amz-Algorithm" - - // AmzSecurityTokenKey indicates the security token to be used with temporary credentials - AmzSecurityTokenKey = "X-Amz-Security-Token" - - // AmzDateKey is the UTC timestamp for the request in the format YYYYMMDD'T'HHMMSS'Z' - AmzDateKey = "X-Amz-Date" - - // AmzCredentialKey is the access key ID and credential scope - AmzCredentialKey = "X-Amz-Credential" - - // AmzSignedHeadersKey is the set of headers signed for the request - AmzSignedHeadersKey = "X-Amz-SignedHeaders" - - // AmzSignatureKey is the query parameter to store the SigV4 signature - AmzSignatureKey = "X-Amz-Signature" - - // TimeFormat is the time format to be used in the X-Amz-Date header or query parameter - TimeFormat = "20060102T150405Z" - - // ShortTimeFormat is the shorten time format used in the credential scope - ShortTimeFormat = "20060102" - - // ContentSHAKey is the SHA256 of request body - ContentSHAKey = "X-Amz-Content-Sha256" - - // StreamingEventsPayload indicates that the request payload body is a signed event stream. - StreamingEventsPayload = "STREAMING-AWS4-HMAC-SHA256-EVENTS" -) diff --git a/aws/signer/internal/v4/header_rules.go b/aws/signer/internal/v4/header_rules.go deleted file mode 100644 index ea08c4e1..00000000 --- a/aws/signer/internal/v4/header_rules.go +++ /dev/null @@ -1,92 +0,0 @@ -package v4 - -import ( - "strings" -) - -// Rules houses a set of Rule needed for validation of a -// string value -type Rules []Rule - -// Rule interface allows for more flexible rules and just simply -// checks whether or not a value adheres to that Rule -type Rule interface { - IsValid(value string) bool -} - -// IsValid will iterate through all rules and see if any rules -// apply to the value and supports nested rules -func (r Rules) IsValid(value string) bool { - for _, rule := range r { - if rule.IsValid(value) { - return true - } - } - return false -} - -// MapRule generic Rule for maps -type MapRule map[string]struct{} - -// IsValid for the map Rule satisfies whether it exists in the map -func (m MapRule) IsValid(value string) bool { - for key := range m { - if strings.EqualFold(key, value) { - return true - } - } - return false -} - -// AllowList is a generic Rule for include listing -type AllowList struct { - Rule -} - -// IsValid for AllowList checks if the value is within the AllowList -func (w AllowList) IsValid(value string) bool { - return w.Rule.IsValid(value) -} - -// ExcludeList is a generic Rule for exclude listing -type ExcludeList struct { - Rule -} - -// IsValid for AllowList checks if the value is within the AllowList -func (b ExcludeList) IsValid(value string) bool { - return !b.Rule.IsValid(value) -} - -// Patterns is a list of strings to match against -type Patterns []string - -// IsValid for Patterns checks each pattern and returns if a match has -// been found -func (p Patterns) IsValid(value string) bool { - for _, pattern := range p { - if hasPrefixFold(value, pattern) { - return true - } - } - return false -} - -// InclusiveRules rules allow for rules to depend on one another -type InclusiveRules []Rule - -// IsValid will return true if all rules are true -func (r InclusiveRules) IsValid(value string) bool { - for _, rule := range r { - if !rule.IsValid(value) { - return false - } - } - return true -} - -// hasPrefixFold tests whether the string s begins with prefix, interpreted as UTF-8 strings, -// under Unicode case-folding. -func hasPrefixFold(s, prefix string) bool { - return len(s) >= len(prefix) && strings.EqualFold(s[0:len(prefix)], prefix) -} diff --git a/aws/signer/internal/v4/headers.go b/aws/signer/internal/v4/headers.go deleted file mode 100644 index cbd22107..00000000 --- a/aws/signer/internal/v4/headers.go +++ /dev/null @@ -1,32 +0,0 @@ -package v4 - -// IgnoredHeaders is a list of headers that are ignored during signing -var IgnoredHeaders = Rules{ - ExcludeList{ - MapRule{ - "Authorization": struct{}{}, - "User-Agent": struct{}{}, - "X-Amzn-Trace-Id": struct{}{}, - "Expect": struct{}{}, - "Transfer-Encoding": struct{}{}, - }, - }, -} - -// RequiredSignedHeaders are request headers that must be part of SignedHeaders -// whenever they are present on the request. -var RequiredSignedHeaders = Rules{ - AllowList{ - MapRule{ - "Host": struct{}{}, - }, - }, - Patterns{"X-Amz-"}, -} - -// AllowedQueryHoisting is a allowed list for Build query headers. The boolean value -// represents whether or not it is a pattern. -var AllowedQueryHoisting = InclusiveRules{ - ExcludeList{RequiredSignedHeaders}, - Patterns{"X-Amz-"}, -} diff --git a/aws/signer/internal/v4/headers_test.go b/aws/signer/internal/v4/headers_test.go deleted file mode 100644 index 4484c294..00000000 --- a/aws/signer/internal/v4/headers_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package v4 - -import "testing" - -func TestAllowedQueryHoisting(t *testing.T) { - cases := map[string]struct { - Header string - ExpectHoist bool - }{ - "object-lock": { - Header: "X-Amz-Object-Lock-Mode", - ExpectHoist: false, - }, - "s3 metadata": { - Header: "X-Amz-Meta-SomeName", - ExpectHoist: false, - }, - "another header": { - Header: "X-Amz-SomeOtherHeader", - ExpectHoist: false, - }, - "lowercase amz header": { - Header: "x-amz-someotherheader", - ExpectHoist: false, - }, - "mixed case amz header": { - Header: "x-AmZ-someotherheader", - ExpectHoist: false, - }, - "non-amz content header": { - Header: "Content-Type", - ExpectHoist: false, - }, - "non X-AMZ header": { - Header: "X-SomeOtherHeader", - ExpectHoist: false, - }, - } - - for name, c := range cases { - t.Run(name, func(t *testing.T) { - if e, a := c.ExpectHoist, AllowedQueryHoisting.IsValid(c.Header); e != a { - t.Errorf("expect hoist %v, was %v", e, a) - } - }) - } -} - -func TestRequiredSignedHeaders(t *testing.T) { - cases := map[string]struct { - Header string - ExpectRequired bool - }{ - "known content header": { - Header: "Content-Type", - ExpectRequired: false, - }, - "known content header lowercase": { - Header: "content-type", - ExpectRequired: false, - }, - "known conditional header": { - Header: "If-Match", - ExpectRequired: false, - }, - "range header": { - Header: "Range", - ExpectRequired: false, - }, - "content md5 header": { - Header: "Content-Md5", - ExpectRequired: false, - }, - "arbitrary amz header": { - Header: "X-Amz-SomeOtherHeader", - ExpectRequired: true, - }, - "arbitrary amz header lowercase": { - Header: "x-amz-someotherheader", - ExpectRequired: true, - }, - "object-lock amz header": { - Header: "X-Amz-Object-Lock-Mode", - ExpectRequired: true, - }, - "metadata amz header": { - Header: "X-Amz-Meta-SomeName", - ExpectRequired: true, - }, - "non-amz custom header": { - Header: "X-SomeOtherHeader", - ExpectRequired: false, - }, - } - - for name, c := range cases { - t.Run(name, func(t *testing.T) { - if e, a := c.ExpectRequired, RequiredSignedHeaders.IsValid(c.Header); e != a { - t.Errorf("expect required %v, was %v", e, a) - } - }) - } -} - -func TestIgnoredHeaders(t *testing.T) { - cases := map[string]struct { - Header string - ExpectIgnored bool - }{ - "expect": { - Header: "Expect", - ExpectIgnored: true, - }, - "user-agent": { - Header: "User-Agent", - ExpectIgnored: true, - }, - "transfer-encoding": { - Header: "Transfer-Encoding", - ExpectIgnored: true, - }, - "authorization": { - Header: "Authorization", - ExpectIgnored: true, - }, - "authorization lowercase": { - Header: "authorization", - ExpectIgnored: true, - }, - "trace id lowercase": { - Header: "x-amzn-trace-id", - ExpectIgnored: true, - }, - "X-AMZ header": { - Header: "X-Amz-Content-Sha256", - ExpectIgnored: false, - }, - } - - for name, c := range cases { - t.Run(name, func(t *testing.T) { - if e, a := c.ExpectIgnored, IgnoredHeaders.IsValid(c.Header); e == a { - t.Errorf("expect ignored %v, was %v", e, a) - } - }) - } -} diff --git a/aws/signer/internal/v4/hmac.go b/aws/signer/internal/v4/hmac.go deleted file mode 100644 index e7fa7a1b..00000000 --- a/aws/signer/internal/v4/hmac.go +++ /dev/null @@ -1,13 +0,0 @@ -package v4 - -import ( - "crypto/hmac" - "crypto/sha256" -) - -// HMACSHA256 computes a HMAC-SHA256 of data given the provided key. -func HMACSHA256(key []byte, data []byte) []byte { - hash := hmac.New(sha256.New, key) - hash.Write(data) - return hash.Sum(nil) -} diff --git a/aws/signer/internal/v4/host.go b/aws/signer/internal/v4/host.go deleted file mode 100644 index 0c5a3e87..00000000 --- a/aws/signer/internal/v4/host.go +++ /dev/null @@ -1,75 +0,0 @@ -package v4 - -import ( - "net/http" - "strings" -) - -// SanitizeHostForHeader removes default port from host and updates request.Host -func SanitizeHostForHeader(r *http.Request) { - host := getHost(r) - port := portOnly(host) - if port != "" && isDefaultPort(r.URL.Scheme, port) { - r.Host = stripPort(host) - } -} - -// Returns host from request -func getHost(r *http.Request) string { - if r.Host != "" { - return r.Host - } - - return r.URL.Host -} - -// Hostname returns u.Host, without any port number. -// -// If Host is an IPv6 literal with a port number, Hostname returns the -// IPv6 literal without the square brackets. IPv6 literals may include -// a zone identifier. -// -// Copied from the Go 1.8 standard library (net/url) -func stripPort(hostport string) string { - before, _, ok := strings.Cut(hostport, ":") - if !ok { - return hostport - } - if before, _, ok := strings.Cut(hostport, "]"); ok { - return strings.TrimPrefix(before, "[") - } - return before -} - -// Port returns the port part of u.Host, without the leading colon. -// If u.Host doesn't contain a port, Port returns an empty string. -// -// Copied from the Go 1.8 standard library (net/url) -func portOnly(hostport string) string { - _, after, ok := strings.Cut(hostport, ":") - if !ok { - return "" - } - if _, after, ok := strings.Cut(hostport, "]:"); ok { - return after - } - if strings.Contains(hostport, "]") { - return "" - } - return after -} - -// Returns true if the specified URI is using the standard port -// (i.e. port 80 for HTTP URIs or 443 for HTTPS URIs) -func isDefaultPort(scheme, port string) bool { - if port == "" { - return true - } - - lowerCaseScheme := strings.ToLower(scheme) - if (lowerCaseScheme == "http" && port == "80") || (lowerCaseScheme == "https" && port == "443") { - return true - } - - return false -} diff --git a/aws/signer/internal/v4/scope.go b/aws/signer/internal/v4/scope.go deleted file mode 100644 index fc788790..00000000 --- a/aws/signer/internal/v4/scope.go +++ /dev/null @@ -1,13 +0,0 @@ -package v4 - -import "strings" - -// BuildCredentialScope builds the Signature Version 4 (SigV4) signing scope -func BuildCredentialScope(signingTime SigningTime, region, service string) string { - return strings.Join([]string{ - signingTime.ShortTimeFormat(), - region, - service, - "aws4_request", - }, "/") -} diff --git a/aws/signer/internal/v4/time.go b/aws/signer/internal/v4/time.go deleted file mode 100644 index 1de06a76..00000000 --- a/aws/signer/internal/v4/time.go +++ /dev/null @@ -1,36 +0,0 @@ -package v4 - -import "time" - -// SigningTime provides a wrapper around a time.Time which provides cached values for SigV4 signing. -type SigningTime struct { - time.Time - timeFormat string - shortTimeFormat string -} - -// NewSigningTime creates a new SigningTime given a time.Time -func NewSigningTime(t time.Time) SigningTime { - return SigningTime{ - Time: t, - } -} - -// TimeFormat provides a time formatted in the X-Amz-Date format. -func (m *SigningTime) TimeFormat() string { - return m.format(&m.timeFormat, TimeFormat) -} - -// ShortTimeFormat provides a time formatted of 20060102. -func (m *SigningTime) ShortTimeFormat() string { - return m.format(&m.shortTimeFormat, ShortTimeFormat) -} - -func (m *SigningTime) format(target *string, format string) string { - if len(*target) > 0 { - return *target - } - v := m.Time.Format(format) - *target = v - return v -} diff --git a/aws/signer/internal/v4/util.go b/aws/signer/internal/v4/util.go deleted file mode 100644 index d025dbaa..00000000 --- a/aws/signer/internal/v4/util.go +++ /dev/null @@ -1,80 +0,0 @@ -package v4 - -import ( - "net/url" - "strings" -) - -const doubleSpace = " " - -// StripExcessSpaces will rewrite the passed in slice's string values to not -// contain multiple side-by-side spaces. -func StripExcessSpaces(str string) string { - var j, k, l, m, spaces int - // Trim trailing spaces - for j = len(str) - 1; j >= 0 && str[j] == ' '; j-- { - } - - // Trim leading spaces - for k = 0; k < j && str[k] == ' '; k++ { - } - str = str[k : j+1] - - // Strip multiple spaces. - j = strings.Index(str, doubleSpace) - if j < 0 { - return str - } - - buf := []byte(str) - for k, m, l = j, j, len(buf); k < l; k++ { - if buf[k] == ' ' { - if spaces == 0 { - // First space. - buf[m] = buf[k] - m++ - } - spaces++ - } else { - // End of multiple spaces. - spaces = 0 - buf[m] = buf[k] - m++ - } - } - - return string(buf[:m]) -} - -// GetURIPath returns the escaped URI component from the provided URL. -func GetURIPath(u *url.URL) string { - var uriPath string - - if len(u.Opaque) > 0 { - const schemeSep, pathSep, queryStart = "//", "/", "?" - - opaque := u.Opaque - // Cut off the query string if present. - if idx := strings.Index(opaque, queryStart); idx >= 0 { - opaque = opaque[:idx] - } - - // Cutout the scheme separator if present. - if strings.Index(opaque, schemeSep) == 0 { - opaque = opaque[len(schemeSep):] - } - - // capture URI path starting with first path separator. - if idx := strings.Index(opaque, pathSep); idx >= 0 { - uriPath = opaque[idx:] - } - } else { - uriPath = u.EscapedPath() - } - - if len(uriPath) == 0 { - uriPath = "/" - } - - return uriPath -} diff --git a/aws/signer/internal/v4/util_test.go b/aws/signer/internal/v4/util_test.go deleted file mode 100644 index 277f87b6..00000000 --- a/aws/signer/internal/v4/util_test.go +++ /dev/null @@ -1,158 +0,0 @@ -package v4 - -import ( - "net/http" - "net/url" - "testing" -) - -func lazyURLParse(v string) func() (*url.URL, error) { - return func() (*url.URL, error) { - return url.Parse(v) - } -} - -func TestGetURIPath(t *testing.T) { - cases := map[string]struct { - getURL func() (*url.URL, error) - expect string - }{ - // Cases - "with scheme": { - getURL: lazyURLParse("https://localhost:9000"), - expect: "/", - }, - "no port, with scheme": { - getURL: lazyURLParse("https://localhost"), - expect: "/", - }, - "without scheme": { - getURL: lazyURLParse("localhost:9000"), - expect: "/", - }, - "without scheme, with path": { - getURL: lazyURLParse("localhost:9000/abc123"), - expect: "/abc123", - }, - "without scheme, with separator": { - getURL: lazyURLParse("//localhost:9000"), - expect: "/", - }, - "no port, without scheme, with separator": { - getURL: lazyURLParse("//localhost"), - expect: "/", - }, - "without scheme, with separator, with path": { - getURL: lazyURLParse("//localhost:9000/abc123"), - expect: "/abc123", - }, - "no port, without scheme, with separator, with path": { - getURL: lazyURLParse("//localhost/abc123"), - expect: "/abc123", - }, - "opaque with query string": { - getURL: lazyURLParse("localhost:9000/abc123?efg=456"), - expect: "/abc123", - }, - "failing test": { - getURL: func() (*url.URL, error) { - endpoint := "https://service.region.amazonaws.com" - req, _ := http.NewRequest("POST", endpoint, nil) - u := req.URL - - u.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()" - - query := u.Query() - query.Set("some-query-key", "value") - u.RawQuery = query.Encode() - - return u, nil - }, - expect: "/bucket/key-._~,!@#$%^&*()", - }, - } - - for name, c := range cases { - t.Run(name, func(t *testing.T) { - u, err := c.getURL() - if err != nil { - t.Fatalf("failed to get URL, %v", err) - } - - actual := GetURIPath(u) - if e, a := c.expect, actual; e != a { - t.Errorf("expect %v path, got %v", e, a) - } - }) - } -} - -func TestStripExcessHeaders(t *testing.T) { - vals := []string{ - "", - "123", - "1 2 3", - "1 2 3 ", - " 1 2 3", - "1 2 3", - "1 23", - "1 2 3", - "1 2 ", - " 1 2 ", - "12 3", - "12 3 1", - "12 3 1", - "12 3 1abc123", - } - - expected := []string{ - "", - "123", - "1 2 3", - "1 2 3", - "1 2 3", - "1 2 3", - "1 23", - "1 2 3", - "1 2", - "1 2", - "12 3", - "12 3 1", - "12 3 1", - "12 3 1abc123", - } - - for i := range vals { - r := StripExcessSpaces(vals[i]) - if e, a := expected[i], r; e != a { - t.Errorf("%d, expect %v, got %v", i, e, a) - } - } -} - -var stripExcessSpaceCases = []string{ - `AWS4-HMAC-SHA256 Credential=AKIDFAKEIDFAKEID/20160628/us-west-2/s3/aws4_request, SignedHeaders=host;x-amz-date, Signature=1234567890abcdef1234567890abcdef1234567890abcdef`, - `123 321 123 321`, - ` 123 321 123 321 `, - ` 123 321 123 321 `, - "123", - "1 2 3", - " 1 2 3", - "1 2 3", - "1 23", - "1 2 3", - "1 2 ", - " 1 2 ", - "12 3", - "12 3 1", - "12 3 1", - "12 3 1abc123", -} - -func BenchmarkStripExcessSpaces(b *testing.B) { - for i := 0; i < b.N; i++ { - for _, v := range stripExcessSpaceCases { - StripExcessSpaces(v) - } - } -} diff --git a/aws/signer/v4/functional_test.go b/aws/signer/v4/functional_test.go deleted file mode 100644 index a7d4f738..00000000 --- a/aws/signer/v4/functional_test.go +++ /dev/null @@ -1,139 +0,0 @@ -package v4_test - -import ( - "context" - "fmt" - "net/http" - "testing" - "time" - - v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" - "github.com/versity/versitygw/aws/internal/awstesting/unit" - v4Internal "github.com/versity/versitygw/aws/signer/internal/v4" -) - -var standaloneSignCases = []struct { - OrigURI string - OrigQuery string - Region, Service, SubDomain string - ExpSig string - EscapedURI string -}{ - { - OrigURI: `/logs-*/_search`, - OrigQuery: `pretty=true`, - Region: "us-west-2", Service: "es", SubDomain: "hostname-clusterkey", - EscapedURI: `/logs-%2A/_search`, - ExpSig: `AWS4-HMAC-SHA256 Credential=AKID/19700101/us-west-2/es/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=79d0760751907af16f64a537c1242416dacf51204a7dd5284492d15577973b91`, - }, -} - -func TestStandaloneSign_CustomURIEscape(t *testing.T) { - var expectSig = `AWS4-HMAC-SHA256 Credential=AKID/19700101/us-east-1/es/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=6601e883cc6d23871fd6c2a394c5677ea2b8c82b04a6446786d64cd74f520967` - - creds, err := unit.Config().Credentials.Retrieve(context.Background()) - if err != nil { - t.Fatalf("expect no error, got %v", err) - } - signer := v4.NewSigner(func(signer *v4.SignerOptions) { - signer.DisableURIPathEscaping = true - }) - - host := "https://subdomain.us-east-1.es.amazonaws.com" - req, err := http.NewRequest("GET", host, nil) - if err != nil { - t.Fatalf("expect no error, got %v", err) - } - - req.URL.Path = `/log-*/_search` - req.URL.Opaque = "//subdomain.us-east-1.es.amazonaws.com/log-%2A/_search" - - err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, "es", "us-east-1", time.Unix(0, 0)) - if err != nil { - t.Fatalf("expect no error, got %v", err) - } - - actual := req.Header.Get("Authorization") - if e, a := expectSig, actual; e != a { - t.Errorf("expect %v, got %v", e, a) - } -} - -func TestStandaloneSign(t *testing.T) { - creds, err := unit.Config().Credentials.Retrieve(context.Background()) - if err != nil { - t.Fatalf("expect no error, got %v", err) - } - signer := v4.NewSigner() - - for _, c := range standaloneSignCases { - host := fmt.Sprintf("https://%s.%s.%s.amazonaws.com", - c.SubDomain, c.Region, c.Service) - - req, err := http.NewRequest("GET", host, nil) - if err != nil { - t.Errorf("expected no error, but received %v", err) - } - - // URL.EscapedPath() will be used by the signer to get the - // escaped form of the request's URI path. - req.URL.Path = c.OrigURI - req.URL.RawQuery = c.OrigQuery - - err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, c.Service, c.Region, time.Unix(0, 0)) - if err != nil { - t.Errorf("expected no error, but received %v", err) - } - - actual := req.Header.Get("Authorization") - if e, a := c.ExpSig, actual; e != a { - t.Errorf("expected %v, but received %v", e, a) - } - if e, a := c.OrigURI, req.URL.Path; e != a { - t.Errorf("expected %v, but received %v", e, a) - } - if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a { - t.Errorf("expected %v, but received %v", e, a) - } - } -} - -func TestStandaloneSign_RawPath(t *testing.T) { - creds, err := unit.Config().Credentials.Retrieve(context.Background()) - if err != nil { - t.Fatalf("expect no error, got %v", err) - } - signer := v4.NewSigner() - - for _, c := range standaloneSignCases { - host := fmt.Sprintf("https://%s.%s.%s.amazonaws.com", - c.SubDomain, c.Region, c.Service) - - req, err := http.NewRequest("GET", host, nil) - if err != nil { - t.Errorf("expected no error, but received %v", err) - } - - // URL.EscapedPath() will be used by the signer to get the - // escaped form of the request's URI path. - req.URL.Path = c.OrigURI - req.URL.RawPath = c.EscapedURI - req.URL.RawQuery = c.OrigQuery - - err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, c.Service, c.Region, time.Unix(0, 0)) - if err != nil { - t.Errorf("expected no error, but received %v", err) - } - - actual := req.Header.Get("Authorization") - if e, a := c.ExpSig, actual; e != a { - t.Errorf("expected %v, but received %v", e, a) - } - if e, a := c.OrigURI, req.URL.Path; e != a { - t.Errorf("expected %v, but received %v", e, a) - } - if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a { - t.Errorf("expected %v, but received %v", e, a) - } - } -} diff --git a/aws/signer/v4/header_rules.go b/aws/signer/v4/header_rules.go deleted file mode 100644 index dec685b5..00000000 --- a/aws/signer/v4/header_rules.go +++ /dev/null @@ -1,14 +0,0 @@ -package v4 - -import v4Internal "github.com/versity/versitygw/aws/signer/internal/v4" - -// IsRequiredSignedHeader reports whether a header must be signed when it is -// present on an incoming request. -func IsRequiredSignedHeader(header string) bool { - return v4Internal.RequiredSignedHeaders.IsValid(header) -} - -// IsIgnoredHeader reports whether a header is normally excluded from signing. -func IsIgnoredHeader(header string) bool { - return !v4Internal.IgnoredHeaders.IsValid(header) -} diff --git a/aws/signer/v4/v4.go b/aws/signer/v4/v4.go deleted file mode 100644 index 03beebae..00000000 --- a/aws/signer/v4/v4.go +++ /dev/null @@ -1,588 +0,0 @@ -// Package v4 implements signing for AWS V4 signer -// -// Provides request signing for request that need to be signed with -// AWS V4 Signatures. -// -// # Standalone Signer -// -// Generally using the signer outside of the SDK should not require any additional -// -// The signer does this by taking advantage of the URL.EscapedPath method. If your request URI requires -// -// additional escaping you many need to use the URL.Opaque to define what the raw URI should be sent -// to the service as. -// -// The signer will first check the URL.Opaque field, and use its value if set. -// The signer does require the URL.Opaque field to be set in the form of: -// -// "///" -// -// // e.g. -// "//example.com/some/path" -// -// The leading "//" and hostname are required or the URL.Opaque escaping will -// not work correctly. -// -// If URL.Opaque is not set the signer will fallback to the URL.EscapedPath() -// method and using the returned value. -// -// AWS v4 signature validation requires that the canonical string's URI path -// element must be the URI escaped form of the HTTP request's path. -// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html -// -// The Go HTTP client will perform escaping automatically on the request. Some -// of these escaping may cause signature validation errors because the HTTP -// request differs from the URI path or query that the signature was generated. -// https://golang.org/pkg/net/url/#URL.EscapedPath -// -// Because of this, it is recommended that when using the signer outside of the -// SDK that explicitly escaping the request prior to being signed is preferable, -// and will help prevent signature validation errors. This can be done by setting -// the URL.Opaque or URL.RawPath. The SDK will use URL.Opaque first and then -// call URL.EscapedPath() if Opaque is not set. -// -// Test `TestStandaloneSign` provides a complete example of using the signer -// outside of the SDK and pre-escaping the URI path. -package v4 - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "hash" - "net/http" - "net/textproto" - "net/url" - "slices" - "sort" - "strconv" - "strings" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/smithy-go/encoding/httpbinding" - "github.com/aws/smithy-go/logging" - v4Internal "github.com/versity/versitygw/aws/signer/internal/v4" -) - -const ( - signingAlgorithm = "AWS4-HMAC-SHA256" - authorizationHeader = "Authorization" - - // Version of signing v4 - Version = "SigV4" -) - -// HTTPSigner is an interface to a SigV4 signer that can sign HTTP requests -type HTTPSigner interface { - SignHTTP(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*SignerOptions)) error -} - -type keyDerivator interface { - DeriveKey(credential aws.Credentials, service, region string, signingTime v4Internal.SigningTime) []byte -} - -type SignMetadata struct { - StringToSign string - CanonicalString string -} - -// SignerOptions is the SigV4 Signer options. -type SignerOptions struct { - // Disables the Signer's moving HTTP header key/value pairs from the HTTP - // request header to the request's query string. This is most commonly used - // with pre-signed requests preventing headers from being added to the - // request's query string. - DisableHeaderHoisting bool - - // Disables the automatic escaping of the URI path of the request for the - // siganture's canonical string's path. For services that do not need additional - // escaping then use this to disable the signer escaping the path. - // - // S3 is an example of a service that does not need additional escaping. - // - // http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html - DisableURIPathEscaping bool - - // The logger to send log messages to. - Logger logging.Logger - - // Enable logging of signed requests. - // This will enable logging of the canonical request, the string to sign, and for presigning the subsequent - // presigned URL. - LogSigning bool - - // Disables setting the session token on the request as part of signing - // through X-Amz-Security-Token. This is needed for variations of v4 that - // present the token elsewhere. - DisableSessionToken bool -} - -// Signer applies AWS v4 signing to given request. Use this to sign requests -// that need to be signed with AWS V4 Signatures. -type Signer struct { - options SignerOptions - keyDerivator keyDerivator -} - -// NewSigner returns a new SigV4 Signer -func NewSigner(optFns ...func(signer *SignerOptions)) *Signer { - options := SignerOptions{} - - for _, fn := range optFns { - fn(&options) - } - - return &Signer{options: options, keyDerivator: v4Internal.NewSigningKeyDeriver()} -} - -type httpSigner struct { - Request *http.Request - ServiceName string - Region string - Time v4Internal.SigningTime - Credentials aws.Credentials - KeyDerivator keyDerivator - IsPreSign bool - SignedHdrs []string - - PayloadHash string - - DisableHeaderHoisting bool - DisableURIPathEscaping bool - DisableSessionToken bool -} - -func (s *httpSigner) Build() (signedRequest, error) { - req := s.Request - - query := req.URL.Query() - headers := req.Header - - s.setRequiredSigningFields(headers, query) - - // Sort Each Query Key's Values - for key := range query { - sort.Strings(query[key]) - } - - v4Internal.SanitizeHostForHeader(req) - - credentialScope := s.buildCredentialScope() - credentialStr := s.Credentials.AccessKeyID + "/" + credentialScope - if s.IsPreSign { - query.Set(v4Internal.AmzCredentialKey, credentialStr) - } - - unsignedHeaders := headers - if s.IsPreSign && !s.DisableHeaderHoisting { - var urlValues url.Values - urlValues, unsignedHeaders = buildQuery(v4Internal.AllowedQueryHoisting, headers) - for k := range urlValues { - query[k] = urlValues[k] - } - } - - host := req.URL.Host - if len(req.Host) > 0 { - host = req.Host - } - - signedHeaders, signedHeadersStr, canonicalHeaderStr := s.buildCanonicalHeaders(host, v4Internal.IgnoredHeaders, unsignedHeaders, s.Request.ContentLength) - - if s.IsPreSign { - query.Set(v4Internal.AmzSignedHeadersKey, signedHeadersStr) - } - - var rawQuery strings.Builder - rawQuery.WriteString(strings.Replace(query.Encode(), "+", "%20", -1)) - - canonicalURI := v4Internal.GetURIPath(req.URL) - if !s.DisableURIPathEscaping { - canonicalURI = httpbinding.EscapePath(canonicalURI, false) - } - - canonicalString := s.buildCanonicalString( - req.Method, - canonicalURI, - rawQuery.String(), - signedHeadersStr, - canonicalHeaderStr, - ) - - strToSign := s.buildStringToSign(credentialScope, canonicalString) - signingSignature, err := s.buildSignature(strToSign) - if err != nil { - return signedRequest{}, err - } - - if s.IsPreSign { - rawQuery.WriteString("&X-Amz-Signature=") - rawQuery.WriteString(signingSignature) - } else { - headers[authorizationHeader] = append(headers[authorizationHeader][:0], buildAuthorizationHeader(credentialStr, signedHeadersStr, signingSignature)) - } - - req.URL.RawQuery = rawQuery.String() - - return signedRequest{ - Request: req, - SignedHeaders: signedHeaders, - CanonicalString: canonicalString, - StringToSign: strToSign, - PreSigned: s.IsPreSign, - }, nil -} - -func buildAuthorizationHeader(credentialStr, signedHeadersStr, signingSignature string) string { - const credential = "Credential=" - const signedHeaders = "SignedHeaders=" - const signature = "Signature=" - const commaSpace = ", " - - var parts strings.Builder - parts.Grow(len(signingAlgorithm) + 1 + - len(credential) + len(credentialStr) + 2 + - len(signedHeaders) + len(signedHeadersStr) + 2 + - len(signature) + len(signingSignature), - ) - parts.WriteString(signingAlgorithm) - parts.WriteRune(' ') - parts.WriteString(credential) - parts.WriteString(credentialStr) - parts.WriteString(commaSpace) - parts.WriteString(signedHeaders) - parts.WriteString(signedHeadersStr) - parts.WriteString(commaSpace) - parts.WriteString(signature) - parts.WriteString(signingSignature) - return parts.String() -} - -// SignHTTP signs AWS v4 requests with the provided payload hash, service name, region the -// request is made to, and time the request is signed at. The signTime allows -// you to specify that a request is signed for the future, and cannot be -// used until then. -// -// The payloadHash is the hex encoded SHA-256 hash of the request payload, and -// must be provided. Even if the request has no payload (aka body). If the -// request has no payload you should use the hex encoded SHA-256 of an empty -// string as the payloadHash value. -// -// "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" -// -// Some services such as Amazon S3 accept alternative values for the payload -// hash, such as "UNSIGNED-PAYLOAD" for requests where the body will not be -// included in the request signature. -// -// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html -// -// Sign differs from Presign in that it will sign the request using HTTP -// header values. This type of signing is intended for http.Request values that -// will not be shared, or are shared in a way the header values on the request -// will not be lost. -// -// The passed in request will be modified in place. -func (s Signer) SignHTTP(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, signedHdrs []string, optFns ...func(options *SignerOptions)) (*SignMetadata, error) { - options := s.options - - for _, fn := range optFns { - fn(&options) - } - - signer := &httpSigner{ - Request: r, - PayloadHash: payloadHash, - ServiceName: service, - Region: region, - Credentials: credentials, - Time: v4Internal.NewSigningTime(signingTime.UTC()), - DisableHeaderHoisting: options.DisableHeaderHoisting, - DisableURIPathEscaping: options.DisableURIPathEscaping, - DisableSessionToken: options.DisableSessionToken, - KeyDerivator: s.keyDerivator, - SignedHdrs: signedHdrs, - } - - signedRequest, err := signer.Build() - if err != nil { - return nil, err - } - - logSigningInfo(ctx, options, &signedRequest, false) - - return &SignMetadata{ - StringToSign: signedRequest.StringToSign, - CanonicalString: signedRequest.CanonicalString, - }, nil -} - -// PresignHTTP signs AWS v4 requests with the payload hash, service name, region -// the request is made to, and time the request is signed at. The signTime -// allows you to specify that a request is signed for the future, and cannot -// be used until then. -// -// Returns the signed URL and the map of HTTP headers that were included in the -// signature or an error if signing the request failed. For presigned requests -// these headers and their values must be included on the HTTP request when it -// is made. This is helpful to know what header values need to be shared with -// the party the presigned request will be distributed to. -// -// The payloadHash is the hex encoded SHA-256 hash of the request payload, and -// must be provided. Even if the request has no payload (aka body). If the -// request has no payload you should use the hex encoded SHA-256 of an empty -// string as the payloadHash value. -// -// "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" -// -// Some services such as Amazon S3 accept alternative values for the payload -// hash, such as "UNSIGNED-PAYLOAD" for requests where the body will not be -// included in the request signature. -// -// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html -// -// PresignHTTP differs from SignHTTP in that it will sign the request using -// query string instead of header values. This allows you to share the -// Presigned Request's URL with third parties, or distribute it throughout your -// system with minimal dependencies. -// -// PresignHTTP will not set the expires time of the presigned request -// automatically. To specify the expire duration for a request add the -// "X-Amz-Expires" query parameter on the request with the value as the -// duration in seconds the presigned URL should be considered valid for. This -// parameter is not used by all AWS services, and is most notable used by -// Amazon S3 APIs. -// -// expires := 20 * time.Minute -// query := req.URL.Query() -// query.Set("X-Amz-Expires", strconv.FormatInt(int64(expires/time.Second), 10)) -// req.URL.RawQuery = query.Encode() -// -// This method does not modify the provided request. -func (s *Signer) PresignHTTP( - ctx context.Context, credentials aws.Credentials, r *http.Request, - payloadHash string, service string, region string, signingTime time.Time, - signedHdrs []string, - optFns ...func(*SignerOptions), -) (string, http.Header, *SignMetadata, error) { - options := s.options - - for _, fn := range optFns { - fn(&options) - } - - signer := &httpSigner{ - Request: r.Clone(r.Context()), - PayloadHash: payloadHash, - ServiceName: service, - Region: region, - Credentials: credentials, - Time: v4Internal.NewSigningTime(signingTime.UTC()), - IsPreSign: true, - DisableHeaderHoisting: options.DisableHeaderHoisting, - DisableURIPathEscaping: options.DisableURIPathEscaping, - DisableSessionToken: options.DisableSessionToken, - KeyDerivator: s.keyDerivator, - SignedHdrs: signedHdrs, - } - - signedRequest, err := signer.Build() - if err != nil { - return "", nil, nil, err - } - - logSigningInfo(ctx, options, &signedRequest, true) - - signedHeaders := make(http.Header) - - // For the signed headers we canonicalize the header keys in the returned map. - // This avoids situations where can standard library double headers like host header. For example the standard - // library will set the Host header, even if it is present in lower-case form. - for k, v := range signedRequest.SignedHeaders { - key := textproto.CanonicalMIMEHeaderKey(k) - signedHeaders[key] = append(signedHeaders[key], v...) - } - - return signedRequest.Request.URL.String(), signedHeaders, &SignMetadata{ - StringToSign: signedRequest.StringToSign, - CanonicalString: signedRequest.CanonicalString, - }, nil -} - -func (s *httpSigner) buildCredentialScope() string { - return v4Internal.BuildCredentialScope(s.Time, s.Region, s.ServiceName) -} - -func buildQuery(r v4Internal.Rule, header http.Header) (url.Values, http.Header) { - query := url.Values{} - unsignedHeaders := http.Header{} - for k, h := range header { - if r.IsValid(k) { - query[k] = h - } else { - unsignedHeaders[k] = h - } - } - - return query, unsignedHeaders -} - -func (s *httpSigner) buildCanonicalHeaders(host string, rule v4Internal.Rule, header http.Header, length int64) (signed http.Header, signedHeaders, canonicalHeadersStr string) { - signed = make(http.Header) - - var headers []string - const hostHeader = "host" - headers = append(headers, hostHeader) - signed[hostHeader] = append(signed[hostHeader], host) - - const contentLengthHeader = "content-length" - if slices.Contains(s.SignedHdrs, contentLengthHeader) { - headers = append(headers, contentLengthHeader) - signed[contentLengthHeader] = append(signed[contentLengthHeader], strconv.FormatInt(length, 10)) - } - - for k, v := range header { - if !s.shouldSignHeader(k, rule) { - continue // ignored header - } - if strings.EqualFold(k, contentLengthHeader) { - // prevent signing already handled content-length header. - continue - } - - lowerCaseKey := strings.ToLower(k) - if _, ok := signed[lowerCaseKey]; ok { - // include additional values - signed[lowerCaseKey] = append(signed[lowerCaseKey], v...) - continue - } - - headers = append(headers, lowerCaseKey) - signed[lowerCaseKey] = v - } - sort.Strings(headers) - - signedHeaders = strings.Join(headers, ";") - - var canonicalHeaders strings.Builder - n := len(headers) - const colon = ':' - for i := range n { - if headers[i] == hostHeader { - canonicalHeaders.WriteString(hostHeader) - canonicalHeaders.WriteRune(colon) - canonicalHeaders.WriteString(v4Internal.StripExcessSpaces(host)) - } else { - canonicalHeaders.WriteString(headers[i]) - canonicalHeaders.WriteRune(colon) - // Trim out leading, trailing, and dedup inner spaces from signed header values. - values := signed[headers[i]] - for j, v := range values { - cleanedValue := strings.TrimSpace(v4Internal.StripExcessSpaces(v)) - canonicalHeaders.WriteString(cleanedValue) - if j < len(values)-1 { - canonicalHeaders.WriteRune(',') - } - } - } - canonicalHeaders.WriteRune('\n') - } - canonicalHeadersStr = canonicalHeaders.String() - - return signed, signedHeaders, canonicalHeadersStr -} - -func (s *httpSigner) shouldSignHeader(header string, rule v4Internal.Rule) bool { - if strings.EqualFold(header, authorizationHeader) { - return false - } - if s.SignedHdrs != nil { - return slices.ContainsFunc(s.SignedHdrs, func(signedHeader string) bool { - return strings.EqualFold(signedHeader, header) - }) - } - return rule.IsValid(header) -} - -func (s *httpSigner) buildCanonicalString(method, uri, query, signedHeaders, canonicalHeaders string) string { - return strings.Join([]string{ - method, - uri, - query, - canonicalHeaders, - signedHeaders, - s.PayloadHash, - }, "\n") -} - -func (s *httpSigner) buildStringToSign(credentialScope, canonicalRequestString string) string { - return strings.Join([]string{ - signingAlgorithm, - s.Time.TimeFormat(), - credentialScope, - hex.EncodeToString(makeHash(sha256.New(), []byte(canonicalRequestString))), - }, "\n") -} - -func makeHash(hash hash.Hash, b []byte) []byte { - hash.Reset() - hash.Write(b) - return hash.Sum(nil) -} - -func (s *httpSigner) buildSignature(strToSign string) (string, error) { - key := s.KeyDerivator.DeriveKey(s.Credentials, s.ServiceName, s.Region, s.Time) - return hex.EncodeToString(v4Internal.HMACSHA256(key, []byte(strToSign))), nil -} - -func (s *httpSigner) setRequiredSigningFields(headers http.Header, query url.Values) { - amzDate := s.Time.TimeFormat() - - if s.IsPreSign { - query.Set(v4Internal.AmzAlgorithmKey, signingAlgorithm) - sessionToken := s.Credentials.SessionToken - if !s.DisableSessionToken && len(sessionToken) > 0 { - query.Set("X-Amz-Security-Token", sessionToken) - } - - query.Set(v4Internal.AmzDateKey, amzDate) - return - } - - headers[v4Internal.AmzDateKey] = append(headers[v4Internal.AmzDateKey][:0], amzDate) - - if !s.DisableSessionToken && len(s.Credentials.SessionToken) > 0 { - headers[v4Internal.AmzSecurityTokenKey] = append(headers[v4Internal.AmzSecurityTokenKey][:0], s.Credentials.SessionToken) - } -} - -func logSigningInfo(ctx context.Context, options SignerOptions, request *signedRequest, isPresign bool) { - if !options.LogSigning { - return - } - signedURLMsg := "" - if isPresign { - signedURLMsg = fmt.Sprintf(logSignedURLMsg, request.Request.URL.String()) - } - logger := logging.WithContext(ctx, options.Logger) - logger.Logf(logging.Debug, logSignInfoMsg, request.CanonicalString, request.StringToSign, signedURLMsg) -} - -type signedRequest struct { - Request *http.Request - SignedHeaders http.Header - CanonicalString string - StringToSign string - PreSigned bool -} - -const logSignInfoMsg = `Request Signature: ----[ CANONICAL STRING ]----------------------------- -%s ----[ STRING TO SIGN ]-------------------------------- -%s%s ------------------------------------------------------` -const logSignedURLMsg = ` ----[ SIGNED URL ]------------------------------------ -%s` diff --git a/aws/signer/v4/v4_test.go b/aws/signer/v4/v4_test.go deleted file mode 100644 index 1fe0bb20..00000000 --- a/aws/signer/v4/v4_test.go +++ /dev/null @@ -1,380 +0,0 @@ -package v4 - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - "net/http" - "net/url" - "strings" - "testing" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/google/go-cmp/cmp" - v4Internal "github.com/versity/versitygw/aws/signer/internal/v4" -) - -var testCredentials = aws.Credentials{AccessKeyID: "AKID", SecretAccessKey: "SECRET", SessionToken: "SESSION"} - -func buildRequest(serviceName, region, body string) (*http.Request, string) { - reader := strings.NewReader(body) - return buildRequestWithBodyReader(serviceName, region, reader) -} - -func buildRequestWithBodyReader(serviceName, region string, body io.Reader) (*http.Request, string) { - var bodyLen int - - type lenner interface { - Len() int - } - if lr, ok := body.(lenner); ok { - bodyLen = lr.Len() - } - - endpoint := "https://" + serviceName + "." + region + ".amazonaws.com" - req, _ := http.NewRequest("POST", endpoint, body) - req.URL.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()" - req.Header.Set("X-Amz-Target", "prefix.Operation") - req.Header.Set("Content-Type", "application/x-amz-json-1.0") - - if bodyLen > 0 { - req.ContentLength = int64(bodyLen) - } - - req.Header.Set("X-Amz-Meta-Other-Header", "some-value=!@#$%^&* (+)") - req.Header.Add("X-Amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)") - req.Header.Add("X-amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)") - - h := sha256.New() - _, _ = io.Copy(h, body) - payloadHash := hex.EncodeToString(h.Sum(nil)) - - return req, payloadHash -} - -func TestPresignRequest(t *testing.T) { - req, body := buildRequest("dynamodb", "us-east-1", "{}") - - query := req.URL.Query() - query.Set("X-Amz-Expires", "300") - req.URL.RawQuery = query.Encode() - - signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"} - signer := NewSigner() - signed, headers, _, err := signer.PresignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - expectedDate := "19700101T000000Z" - expectedHeaders := "content-length;content-type;host;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore;x-amz-target" - expectedSig := "266528f4c66b4b20807f199141c606c7aa81dd793592b4c6f8dc301c05691e54" - expectedCred := "AKID/19700101/us-east-1/dynamodb/aws4_request" - - q, err := url.ParseQuery(signed[strings.Index(signed, "?"):]) - if err != nil { - t.Errorf("expect no error, got %v", err) - } - - if e, a := expectedSig, q.Get("X-Amz-Signature"); e != a { - t.Errorf("expect %v, got %v", e, a) - } - if e, a := expectedCred, q.Get("X-Amz-Credential"); e != a { - t.Errorf("expect %v, got %v", e, a) - } - if e, a := expectedHeaders, q.Get("X-Amz-SignedHeaders"); e != a { - t.Errorf("expect %v, got %v", e, a) - } - if e, a := expectedDate, q.Get("X-Amz-Date"); e != a { - t.Errorf("expect %v, got %v", e, a) - } - if a := q.Get("X-Amz-Meta-Other-Header"); len(a) != 0 { - t.Errorf("expect %v to be empty", a) - } - if a := q.Get("X-Amz-Target"); len(a) != 0 { - t.Errorf("expect X-Amz-Target to be empty, got %v", a) - } - - for h := range strings.SplitSeq(expectedHeaders, ";") { - v := headers.Get(h) - if len(v) == 0 { - t.Errorf("expect %v, to be present in header map", h) - } - } -} - -func TestPresignBodyWithArrayRequest(t *testing.T) { - req, body := buildRequest("dynamodb", "us-east-1", "{}") - req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a" - - query := req.URL.Query() - query.Set("X-Amz-Expires", "300") - req.URL.RawQuery = query.Encode() - - signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"} - signer := NewSigner() - signed, headers, _, err := signer.PresignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs) - if err != nil { - t.Fatalf("expect no error, got %v", err) - } - - q, err := url.ParseQuery(signed[strings.Index(signed, "?"):]) - if err != nil { - t.Errorf("expect no error, got %v", err) - } - - expectedDate := "19700101T000000Z" - expectedHeaders := "content-length;content-type;host;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore;x-amz-target" - expectedSig := "f8a1f60771366686c04045b64ae1381d302c83d67d84a02567926000e3e653c4" - expectedCred := "AKID/19700101/us-east-1/dynamodb/aws4_request" - - if e, a := expectedSig, q.Get("X-Amz-Signature"); e != a { - t.Errorf("expect %v, got %v", e, a) - } - if e, a := expectedCred, q.Get("X-Amz-Credential"); e != a { - t.Errorf("expect %v, got %v", e, a) - } - if e, a := expectedHeaders, q.Get("X-Amz-SignedHeaders"); e != a { - t.Errorf("expect %v, got %v", e, a) - } - if e, a := expectedDate, q.Get("X-Amz-Date"); e != a { - t.Errorf("expect %v, got %v", e, a) - } - if a := q.Get("X-Amz-Meta-Other-Header"); len(a) != 0 { - t.Errorf("expect %v to be empty, was not", a) - } - if a := q.Get("X-Amz-Target"); len(a) != 0 { - t.Errorf("expect X-Amz-Target to be empty, got %v", a) - } - - for h := range strings.SplitSeq(expectedHeaders, ";") { - v := headers.Get(h) - if len(v) == 0 { - t.Errorf("expect %v, to be present in header map", h) - } - } -} - -func TestSignRequest(t *testing.T) { - req, body := buildRequest("dynamodb", "us-east-1", "{}") - signer := NewSigner() - signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"} - _, err := signer.SignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs) - if err != nil { - t.Fatalf("expect no error, got %v", err) - } - - expectedDate := "19700101T000000Z" - expectedSig := "AWS4-HMAC-SHA256 Credential=AKID/19700101/us-east-1/dynamodb/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore;x-amz-security-token;x-amz-target, Signature=a518299330494908a70222cec6899f6f32f297f8595f6df1776d998936652ad9" - - q := req.Header - if e, a := expectedSig, q.Get("Authorization"); e != a { - t.Errorf("expect %v, got %v", e, a) - } - if e, a := expectedDate, q.Get("X-Amz-Date"); e != a { - t.Errorf("expect %v, got %v", e, a) - } -} - -func TestSignRequestUsesExplicitSignedHeaders(t *testing.T) { - req, payloadHash := buildRequest("dynamodb", "us-east-1", "{}") - reqWithUnsignedHeaders, _ := buildRequest("dynamodb", "us-east-1", "{}") - reqWithUnsignedHeaders.Header.Set("Content-Type", "text/plain") - reqWithUnsignedHeaders.Header.Set("X-Unsigned-Header", "ignored") - signer := NewSigner() - signedHdrs := []string{"host", "x-amz-date"} - - for _, request := range []*http.Request{req, reqWithUnsignedHeaders} { - _, err := signer.SignHTTP(context.Background(), testCredentials, request, payloadHash, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs) - if err != nil { - t.Fatalf("expect no error, got %v", err) - } - } - - authorization := req.Header.Get("Authorization") - if !strings.Contains(authorization, "SignedHeaders=host;x-amz-date,") { - t.Fatalf("expected only explicit signed headers, got %q", authorization) - } - if authorization != reqWithUnsignedHeaders.Header.Get("Authorization") { - t.Fatalf("unsigned headers changed the signature") - } -} - -func TestBuildCanonicalRequest(t *testing.T) { - req, _ := buildRequest("dynamodb", "us-east-1", "{}") - req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a" - - ctx := &httpSigner{ - ServiceName: "dynamodb", - Region: "us-east-1", - Request: req, - Time: v4Internal.NewSigningTime(time.Now()), - KeyDerivator: v4Internal.NewSigningKeyDeriver(), - } - - build, err := ctx.Build() - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - expected := "https://example.org/bucket/key-._~,!@#$%^&*()?Foo=a&Foo=m&Foo=o&Foo=z" - if e, a := expected, build.Request.URL.String(); e != a { - t.Errorf("expect %v, got %v", e, a) - } -} - -func TestSigner_SignHTTP_NoReplaceRequestBody(t *testing.T) { - req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}") - req.Body = io.NopCloser(bytes.NewReader([]byte{})) - - s := NewSigner() - - origBody := req.Body - - _, err := s.SignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{}) - if err != nil { - t.Fatalf("expect no error, got %v", err) - } - - if req.Body != origBody { - t.Errorf("expect request body to not be chagned") - } -} - -func TestRequestHost(t *testing.T) { - req, _ := buildRequest("dynamodb", "us-east-1", "{}") - req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a" - req.Host = "myhost" - - query := req.URL.Query() - query.Set("X-Amz-Expires", "5") - req.URL.RawQuery = query.Encode() - - ctx := &httpSigner{ - ServiceName: "dynamodb", - Region: "us-east-1", - Request: req, - Time: v4Internal.NewSigningTime(time.Now()), - KeyDerivator: v4Internal.NewSigningKeyDeriver(), - } - - build, err := ctx.Build() - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - if !strings.Contains(build.CanonicalString, "host:"+req.Host) { - t.Errorf("canonical host header invalid") - } -} - -func TestSign_buildCanonicalHeadersContentLengthPresent(t *testing.T) { - body := `{"description": "this is a test"}` - req, _ := buildRequest("dynamodb", "us-east-1", body) - req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a" - req.Host = "myhost" - - contentLength := fmt.Sprintf("%d", len([]byte(body))) - req.Header.Add("Content-Length", contentLength) - - query := req.URL.Query() - query.Set("X-Amz-Expires", "5") - req.URL.RawQuery = query.Encode() - - ctx := &httpSigner{ - ServiceName: "dynamodb", - Region: "us-east-1", - Request: req, - Time: v4Internal.NewSigningTime(time.Now()), - KeyDerivator: v4Internal.NewSigningKeyDeriver(), - } - - _, err := ctx.Build() - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - //if !strings.Contains(build.CanonicalString, "content-length:"+contentLength+"\n") { - // t.Errorf("canonical header content-length invalid") - //} -} - -func TestSign_buildCanonicalHeaders(t *testing.T) { - serviceName := "mockAPI" - region := "mock-region" - endpoint := "https://" + serviceName + "." + region + ".amazonaws.com" - - req, err := http.NewRequest("POST", endpoint, nil) - if err != nil { - t.Fatalf("failed to create request, %v", err) - } - - req.Header.Set("FooInnerSpace", " inner space ") - req.Header.Set("FooLeadingSpace", " leading-space") - req.Header.Add("FooMultipleSpace", "no-space") - req.Header.Add("FooMultipleSpace", "\ttab-space") - req.Header.Add("FooMultipleSpace", "trailing-space ") - req.Header.Set("FooNoSpace", "no-space") - req.Header.Set("FooTabSpace", "\ttab-space\t") - req.Header.Set("FooTrailingSpace", "trailing-space ") - req.Header.Set("FooWrappedSpace", " wrapped-space ") - - ctx := &httpSigner{ - ServiceName: serviceName, - Region: region, - Request: req, - Time: v4Internal.NewSigningTime(time.Date(2021, 10, 20, 12, 42, 0, 0, time.UTC)), - KeyDerivator: v4Internal.NewSigningKeyDeriver(), - } - - build, err := ctx.Build() - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - expectCanonicalString := strings.Join([]string{ - `POST`, - `/`, - ``, - `fooinnerspace:inner space`, - `fooleadingspace:leading-space`, - `foomultiplespace:no-space,tab-space,trailing-space`, - `foonospace:no-space`, - `footabspace:tab-space`, - `footrailingspace:trailing-space`, - `foowrappedspace:wrapped-space`, - `host:mockAPI.mock-region.amazonaws.com`, - `x-amz-date:20211020T124200Z`, - ``, - `fooinnerspace;fooleadingspace;foomultiplespace;foonospace;footabspace;footrailingspace;foowrappedspace;host;x-amz-date`, - ``, - }, "\n") - if diff := cmp.Diff(expectCanonicalString, build.CanonicalString); diff != "" { - t.Errorf("expect match, got\n%s", diff) - } -} - -func BenchmarkPresignRequest(b *testing.B) { - signer := NewSigner() - req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}") - - query := req.URL.Query() - query.Set("X-Amz-Expires", "5") - req.URL.RawQuery = query.Encode() - - for i := 0; i < b.N; i++ { - signer.PresignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{}) - } -} - -func BenchmarkSignRequest(b *testing.B) { - signer := NewSigner() - req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}") - for i := 0; i < b.N; i++ { - _, _ = signer.SignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{}) - } -} diff --git a/backend/azure/azure.go b/backend/azure/azure.go index b18708f7..361e6828 100644 --- a/backend/azure/azure.go +++ b/backend/azure/azure.go @@ -1107,22 +1107,7 @@ func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput if err == nil { delResult = append(delResult, types.DeletedObject{Key: obj.Key}) } else { - serr, ok := err.(s3err.S3Error) - if ok { - code := serr.BaseError().Code - message := serr.BaseError().Description - errs = append(errs, types.Error{ - Key: obj.Key, - Code: &code, - Message: &message, - }) - } else { - errs = append(errs, types.Error{ - Key: obj.Key, - Code: backend.GetPtrFromString("InternalError"), - Message: backend.GetPtrFromString(err.Error()), - }) - } + errs = append(errs, s3err.ObjectDeleteError(obj.Key, obj.VersionId, err)) } } diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 6259c74c..77c5cbbc 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -4750,22 +4750,7 @@ func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) delResult = append(delResult, delEntity) } else { - serr, ok := err.(s3err.S3Error) - if ok { - errCode := serr.BaseError().Code - errMessage := serr.BaseError().Code - errs = append(errs, types.Error{ - Key: obj.Key, - Code: &errCode, - Message: &errMessage, - }) - } else { - errs = append(errs, types.Error{ - Key: obj.Key, - Code: backend.GetPtrFromString("InternalError"), - Message: backend.GetPtrFromString(err.Error()), - }) - } + errs = append(errs, s3err.ObjectDeleteError(obj.Key, obj.VersionId, err)) } } @@ -7049,17 +7034,23 @@ func (p *Posix) ListBucketsAndOwners(ctx context.Context) (buckets []s3response. return buckets, nil } +// NormalizeObjectKey resolves object relative to bucket the same way the +// filesystem will (collapsing ".."/"." segments, catching a traversal +// attempt that escapes bucket), but the result names an S3 key, not a host +// path: on Windows filepath.Join/Rel would return it with "\" separators, +// which callers building a policy-resource ARN or match string must never +// see, so it's converted back to "/" before returning. func (p *Posix) NormalizeObjectKey(bucket, object string) string { fullPath := filepath.Join(bucket, object) key, err := filepath.Rel(filepath.Clean(bucket), fullPath) if err != nil { - return fullPath + return filepath.ToSlash(fullPath) } if key == "." { return "" } - return key + return filepath.ToSlash(key) } func (p *Posix) storeChecksums(f *os.File, bucket, object string, chs s3response.Checksum) error { diff --git a/cmd/internal/gwcli/iam.go b/cmd/internal/gwcli/iam.go index 183aa5f0..496dd8bf 100644 --- a/cmd/internal/gwcli/iam.go +++ b/cmd/internal/gwcli/iam.go @@ -114,6 +114,31 @@ func IAMCommand() *cli.Command { Usage: "reject CreateOpenIDConnectProvider requests that omit ThumbprintList instead of auto-fetching it over an outbound TLS connection", EnvVars: []string{"VGW_IAM_DISABLE_OIDC_THUMBPRINT_AUTOFETCH"}, }, + &cli.StringSliceFlag{ + Name: "private-ports", + Usage: "private endpoint listen address: a unix socket path, or :/: when mTLS (--private-cert/--private-cert-key/--private-client-ca) is also configured — refuses to start otherwise (can be specified multiple times)", + EnvVars: []string{"VGW_IAM_PRIVATE_PORTS"}, + }, + &cli.StringFlag{ + Name: "private-cert", + Usage: "TLS server certificate for the private endpoint listener (required for a non-unix-socket --private-ports address)", + EnvVars: []string{"VGW_IAM_PRIVATE_CERT"}, + }, + &cli.StringFlag{ + Name: "private-cert-key", + Usage: "TLS private key for --private-cert", + EnvVars: []string{"VGW_IAM_PRIVATE_CERT_KEY"}, + }, + &cli.StringFlag{ + Name: "private-client-ca", + Usage: "PEM-encoded CA bundle used to verify the S3 gateway's client certificate on the private endpoint listener (required for a non-unix-socket --private-ports address, together with --private-cert/--private-cert-key)", + EnvVars: []string{"VGW_IAM_PRIVATE_CLIENT_CA"}, + }, + &cli.StringFlag{ + Name: "private-socket-perm", + Usage: "octal file-mode permission for a file-backed unix-socket --private-ports address (e.g. '0660'); no effect on TCP or abstract-namespace sockets", + EnvVars: []string{"VGW_IAM_PRIVATE_SOCKET_PERM"}, + }, }, } } diff --git a/cmd/versitygw/iam.go b/cmd/versitygw/iam.go index fbd6fa54..c1bf9783 100644 --- a/cmd/versitygw/iam.go +++ b/cmd/versitygw/iam.go @@ -51,6 +51,11 @@ func runIAM(ctx *cli.Context) error { KeepAlive: keepAlive, HealthPath: healthPath, SocketPerm: socketPerm, + PrivatePorts: ctx.StringSlice("private-ports"), + PrivateCertFile: ctx.String("private-cert"), + PrivateKeyFile: ctx.String("private-cert-key"), + PrivateClientCAFile: ctx.String("private-client-ca"), + PrivateSocketPerm: ctx.String("private-socket-perm"), IAMDir: ctx.String("dir"), VaultEndpointURL: ctx.String("vault-endpoint-url"), VaultNamespace: ctx.String("vault-namespace"), diff --git a/cmd/versitygw/main.go b/cmd/versitygw/main.go index 2a04adcc..970a820b 100644 --- a/cmd/versitygw/main.go +++ b/cmd/versitygw/main.go @@ -27,77 +27,84 @@ import ( "github.com/versity/versitygw/cmd/internal/gwcli" "github.com/versity/versitygw/debuglogger" "github.com/versity/versitygw/embedgw" - "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/internal/netutil" ) var ( - ports []string - admPorts []string - region string - maxConnections, maxRequests int - adminMaxConnections, adminMaxRequests int - corsAllowOrigin string - admCertFile, admKeyFile string - certFile, keyFile string - kafkaURL, kafkaTopic, kafkaKey string - natsURL, natsTopic string - rabbitmqURL, rabbitmqExchange string - rabbitmqRoutingKey string - eventWebhookURL string - eventConfigFilePath string - logWebhookURL, accessLog string - adminLogFile string - healthPath string - virtualDomain string - logLevel string - debug bool - keepAlive bool - pprof string - quiet bool - readonly bool - iamDir string - ldapURL, ldapBindDN, ldapPassword string - ldapQueryBase, ldapObjClasses string - ldapAccessAtr, ldapSecAtr, ldapRoleAtr string - ldapUserIdAtr, ldapGroupIdAtr string - ldapProjectIdAtr string - ldapTLSSkipVerify bool - vaultEndpointURL, vaultNamespace string - vaultSecretStoragePath string - vaultSecretStorageNamespace string - vaultAuthMethod, vaultAuthNamespace string - vaultMountPath string - vaultRootToken, vaultRoleId string - vaultRoleSecret, vaultServerCert string - vaultClientCert, vaultClientCertKey string - s3IamAccess, s3IamSecret string - s3IamRegion, s3IamBucket string - s3IamEndpoint string - s3IamSslNoVerify bool - iamCacheDisable bool - iamCacheTTL int - iamCachePrune int - metricsService string - statsdServers string - dogstatsServers string - ipaHost, ipaVaultName string - ipaUser, ipaPassword string - ipaInsecure bool - iamDebug bool - webuiPorts []string - webuiCertFile, webuiKeyFile string - webuiNoTLS bool - webuiGateways []string - webuiAdminGateways []string - webuiPathPrefix string - webuiS3Prefix string - websitePorts []string - websiteDomain string - websiteCertFile, websiteKeyFile string - websiteNoTLS bool - disableACLs bool - mpMaxParts int - socketPerm string + ports []string + admPorts []string + region string + maxConnections, maxRequests int + adminMaxConnections, adminMaxRequests int + corsAllowOrigin string + admCertFile, admKeyFile string + certFile, keyFile string + kafkaURL, kafkaTopic, kafkaKey string + natsURL, natsTopic string + rabbitmqURL, rabbitmqExchange string + rabbitmqRoutingKey string + eventWebhookURL string + eventConfigFilePath string + logWebhookURL, accessLog string + adminLogFile string + healthPath string + virtualDomain string + logLevel string + debug bool + keepAlive bool + pprof string + quiet bool + readonly bool + iamDir string + ldapURL, ldapBindDN, ldapPassword string + ldapQueryBase, ldapObjClasses string + ldapAccessAtr, ldapSecAtr, ldapRoleAtr string + ldapUserIdAtr, ldapGroupIdAtr string + ldapProjectIdAtr string + ldapTLSSkipVerify bool + vaultEndpointURL, vaultNamespace string + vaultSecretStoragePath string + vaultSecretStorageNamespace string + vaultAuthMethod, vaultAuthNamespace string + vaultMountPath string + vaultRootToken, vaultRoleId string + vaultRoleSecret, vaultServerCert string + vaultClientCert, vaultClientCertKey string + s3IamAccess, s3IamSecret string + s3IamRegion, s3IamBucket string + s3IamEndpoint string + s3IamSslNoVerify bool + iamCacheDisable bool + iamCacheTTL int + iamCachePrune int + metricsService string + statsdServers string + dogstatsServers string + ipaHost, ipaVaultName string + ipaUser, ipaPassword string + ipaInsecure bool + standaloneIAMEndpoint string + standaloneIAMAccess, standaloneIAMSecret string + standaloneClientCert, standaloneClientCertKey string + standaloneServerCA string + standaloneDefaultUserID int + standaloneDefaultGroupID int + standaloneDefaultProjectID int + iamDebug bool + webuiPorts []string + webuiCertFile, webuiKeyFile string + webuiNoTLS bool + webuiGateways []string + webuiAdminGateways []string + webuiPathPrefix string + webuiS3Prefix string + websitePorts []string + websiteDomain string + websiteCertFile, websiteKeyFile string + websiteNoTLS bool + disableACLs bool + mpMaxParts int + socketPerm string ) var ( @@ -163,16 +170,16 @@ documentation can be found in the GitHub wiki.`, // Resolve relative UNIX socket paths to absolute before any backend // (e.g. posix) can change the working directory via os.Chdir. var err error - if ports, err = utils.AbsSocketPaths(ports); err != nil { + if ports, err = netutil.AbsSocketPaths(ports); err != nil { return err } - if admPorts, err = utils.AbsSocketPaths(admPorts); err != nil { + if admPorts, err = netutil.AbsSocketPaths(admPorts); err != nil { return err } - if webuiPorts, err = utils.AbsSocketPaths(webuiPorts); err != nil { + if webuiPorts, err = netutil.AbsSocketPaths(webuiPorts); err != nil { return err } - if websitePorts, err = utils.AbsSocketPaths(websitePorts); err != nil { + if websitePorts, err = netutil.AbsSocketPaths(websitePorts); err != nil { return err } return nil @@ -797,6 +804,60 @@ func initFlags() []cli.Flag { EnvVars: []string{"VGW_IPA_INSECURE"}, Destination: &ipaInsecure, }, + &cli.StringFlag{ + Name: "iam-standalone-endpoint", + Usage: "standalone IAM service private-endpoint address: a unix socket path, or : when mTLS (--iam-standalone-client-cert/-key/--iam-standalone-server-ca) is also configured", + EnvVars: []string{"VGW_IAM_STANDALONE_ENDPOINT"}, + Destination: &standaloneIAMEndpoint, + }, + &cli.StringFlag{ + Name: "iam-standalone-access", + Usage: "access key this gateway signs its own calls to the standalone IAM service with (defaults to --access/root)", + EnvVars: []string{"VGW_IAM_STANDALONE_ACCESS"}, + Destination: &standaloneIAMAccess, + }, + &cli.StringFlag{ + Name: "iam-standalone-secret", + Usage: "secret key this gateway signs its own calls to the standalone IAM service with (defaults to --secret/root)", + EnvVars: []string{"VGW_IAM_STANDALONE_SECRET"}, + Destination: &standaloneIAMSecret, + }, + &cli.StringFlag{ + Name: "iam-standalone-client-cert", + Usage: "client TLS certificate this gateway presents to the standalone IAM service (required for a non-unix-socket --iam-standalone-endpoint)", + EnvVars: []string{"VGW_IAM_STANDALONE_CLIENT_CERT"}, + Destination: &standaloneClientCert, + }, + &cli.StringFlag{ + Name: "iam-standalone-client-cert-key", + Usage: "private key for --iam-standalone-client-cert", + EnvVars: []string{"VGW_IAM_STANDALONE_CLIENT_CERT_KEY"}, + Destination: &standaloneClientCertKey, + }, + &cli.StringFlag{ + Name: "iam-standalone-server-ca", + Usage: "PEM-encoded CA bundle used to verify the standalone IAM service's server certificate (required for a non-unix-socket --iam-standalone-endpoint)", + EnvVars: []string{"VGW_IAM_STANDALONE_SERVER_CA"}, + Destination: &standaloneServerCA, + }, + &cli.IntFlag{ + Name: "iam-standalone-default-uid", + Usage: "POSIX uid assigned to every account resolved through the standalone IAM backend (it has no per-user POSIX identity of its own)", + EnvVars: []string{"VGW_IAM_STANDALONE_DEFAULT_UID"}, + Destination: &standaloneDefaultUserID, + }, + &cli.IntFlag{ + Name: "iam-standalone-default-gid", + Usage: "POSIX gid assigned to every account resolved through the standalone IAM backend", + EnvVars: []string{"VGW_IAM_STANDALONE_DEFAULT_GID"}, + Destination: &standaloneDefaultGroupID, + }, + &cli.IntFlag{ + Name: "iam-standalone-default-project-id", + Usage: "project id assigned to every account resolved through the standalone IAM backend", + EnvVars: []string{"VGW_IAM_STANDALONE_DEFAULT_PROJECT_ID"}, + Destination: &standaloneDefaultProjectID, + }, &cli.IntFlag{ Name: "mp-max-parts", Usage: "maximum number of parts allowed in a multipart upload", @@ -920,6 +981,15 @@ func runGateway(ctx context.Context, be backend.Backend) error { IpaUser: ipaUser, IpaPassword: ipaPassword, IpaInsecure: ipaInsecure, + StandaloneIAMEndpoint: standaloneIAMEndpoint, + StandaloneIAMAccess: standaloneIAMAccess, + StandaloneIAMSecret: standaloneIAMSecret, + StandaloneClientCert: standaloneClientCert, + StandaloneClientCertKey: standaloneClientCertKey, + StandaloneServerCA: standaloneServerCA, + StandaloneDefaultUserID: standaloneDefaultUserID, + StandaloneDefaultGroupID: standaloneDefaultGroupID, + StandaloneDefaultProjectID: standaloneDefaultProjectID, AccessLog: accessLog, LogWebhookURL: logWebhookURL, AdminLogFile: adminLogFile, diff --git a/cmd/versitygw/test.go b/cmd/versitygw/test.go index 075305fe..61190d76 100644 --- a/cmd/versitygw/test.go +++ b/cmd/versitygw/test.go @@ -26,6 +26,7 @@ var ( awsID string awsSecret string endpoint string + iamEndpoint string websiteSchemeTest string websiteDomainTest string websitePortTest string @@ -82,6 +83,12 @@ func initTestFlags() []cli.Flag { Destination: &endpoint, Aliases: []string{"e"}, }, + &cli.StringFlag{ + Name: "iam-endpoint", + Usage: "standalone IAM/STS service endpoint, when it is a separate process from the s3 endpoint (defaults to --endpoint)", + Destination: &iamEndpoint, + Aliases: []string{"ie"}, + }, &cli.BoolFlag{ Name: "host-style", Usage: "Use host-style bucket addressing", @@ -212,6 +219,24 @@ func initTestCommands() []*cli.Command { Usage: "Tests gateway access control with bucket ACLs and Policies", Action: getAction(integration.TestAccessControl), }, + { + Name: "s3-iam", + Usage: "Tests s3 gateway access control backed by the standalone IAM service", + Description: `Runs the access-control tests for an s3 gateway configured with --iam-standalone-endpoint: + IAM user identity policies, their interaction with bucket policies, governance-retention + bypass, and bucket creation. Requires --iam-endpoint pointing at the IAM service's + control-plane API, since the tests create the users and policies they then exercise.`, + Action: getAction(integration.TestS3IAMAccessControl), + }, + { + Name: "s3-iam-session", + Usage: "Tests s3 gateway access control for assumed-role session credentials", + Description: `Runs the role/session access-control tests against an s3 gateway backed by the + standalone IAM service. Every test mints real temporary credentials via + AssumeRoleWithWebIdentity against GitHub Actions' OIDC issuer, so the whole group skips + itself outside a GitHub Actions job holding id-token: write permission.`, + Action: getAction(integration.TestS3IAMSessionAccessControl), + }, { Name: "noacl", Usage: "Tests gateway in ACL-disabled mode", @@ -434,6 +459,7 @@ func getAction(tf testFunc) func(ctx *cli.Context) error { integration.WithSecret(awsSecret), integration.WithRegion(region), integration.WithEndpoint(endpoint), + integration.WithIAMEndpoint(iamEndpoint), integration.WithTLSStatus(tlsStatus), } if testDebug { @@ -484,6 +510,7 @@ func extractIntTests() (commands []*cli.Command) { integration.WithSecret(awsSecret), integration.WithRegion(region), integration.WithEndpoint(endpoint), + integration.WithIAMEndpoint(iamEndpoint), integration.WithTLSStatus(tlsStatus), } if testDebug { diff --git a/cmd/vgwrdma/main.go b/cmd/vgwrdma/main.go index 03c344f0..0596ec9c 100644 --- a/cmd/vgwrdma/main.go +++ b/cmd/vgwrdma/main.go @@ -31,9 +31,9 @@ import ( "github.com/versity/versitygw/cumiddleware" "github.com/versity/versitygw/debuglogger" "github.com/versity/versitygw/embedgw" + "github.com/versity/versitygw/internal/netutil" "github.com/versity/versitygw/rdma" "github.com/versity/versitygw/s3api" - "github.com/versity/versitygw/s3api/utils" ) var ( @@ -184,16 +184,16 @@ documentation can be found in the GitHub wiki.`, // Resolve relative UNIX socket paths to absolute before any backend // (e.g. posix) can change the working directory via os.Chdir. var err error - if ports, err = utils.AbsSocketPaths(ports); err != nil { + if ports, err = netutil.AbsSocketPaths(ports); err != nil { return err } - if admPorts, err = utils.AbsSocketPaths(admPorts); err != nil { + if admPorts, err = netutil.AbsSocketPaths(admPorts); err != nil { return err } - if webuiPorts, err = utils.AbsSocketPaths(webuiPorts); err != nil { + if webuiPorts, err = netutil.AbsSocketPaths(webuiPorts); err != nil { return err } - if websitePorts, err = utils.AbsSocketPaths(websitePorts); err != nil { + if websitePorts, err = netutil.AbsSocketPaths(websitePorts); err != nil { return err } diff --git a/embedgw/embedgw.go b/embedgw/embedgw.go index ba5e66ab..847a6b2a 100644 --- a/embedgw/embedgw.go +++ b/embedgw/embedgw.go @@ -36,6 +36,7 @@ import ( "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/debuglogger" + "github.com/versity/versitygw/internal/netutil" "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api" "github.com/versity/versitygw/s3api/middlewares" @@ -160,15 +161,16 @@ type Config struct { // IAM Backends // - // The gateway supports five external IAM backends. At most one may be + // The gateway supports six external IAM backends. At most one may be // active at a time. When the fields for more than one backend are // populated, the first match in the following priority order wins: // - // 1. IAMDir -- local directory - // 2. LDAPServerURL -- LDAP - // 3. S3IAMEndpoint -- S3-backed - // 4. VaultEndpointURL -- HashiCorp Vault - // 5. IpaHost -- FreeIPA + // 1. StandaloneIAMEndpoint -- standalone IAM service + // 2. IAMDir -- local directory + // 3. LDAPServerURL -- LDAP + // 4. S3IAMEndpoint -- S3-backed + // 5. VaultEndpointURL -- HashiCorp Vault + // 6. IpaHost -- FreeIPA // // Configuring an IAM backend is optional. When none of the trigger fields // above are set, the gateway runs in single-account mode: only the root @@ -280,6 +282,44 @@ type Config struct { // connection. IpaInsecure bool + // Standalone IAM service backend. This is an AWS compatible IAM system + // Activated when StandaloneIAMEndpoint is non-empty. Unlike the other + // backends, this one never holds a plaintext secret for any account but + // its own signing identity and the local root account — every other account's + // secret and inline policy documents stay inside the IAM service process. + // Because of that, user management (CreateUser/UpdateUser/DeleteUser/ListUsers) + // is unavailable through this gateway's own admin API when this + // backend is active; manage users via the standalone IAM service's own + // control-plane API instead. + + // StandaloneIAMEndpoint is either a "host:port" TCP address (mTLS + // required — see StandaloneClientCert/ClientCertKey/ServerCA) or a + // unix socket path for the standalone IAM service's private endpoints. + StandaloneIAMEndpoint string + // StandaloneIAMAccess/StandaloneIAMSecret are this gateway's own + // signing identity for its calls to the private endpoints — not a + // fetched account. Both default to RootUserAccess/RootUserSecret when + // unset. + StandaloneIAMAccess string + StandaloneIAMSecret string + // StandaloneClientCert/ClientCertKey are this gateway's client + // certificate/key for outbound mTLS to the private endpoints. Required + // (together with StandaloneServerCA) unless StandaloneIAMEndpoint is a + // unix socket. + StandaloneClientCert string + StandaloneClientCertKey string + // StandaloneServerCA verifies the standalone IAM service's server + // certificate. + StandaloneServerCA string + // StandaloneDefaultUserID/GroupID/ProjectID are the POSIX uid/gid/ + // project-id assigned to every account resolved through this backend. + // The standalone IAM service's user model has no per-user POSIX + // identity concept, so every standalone-backed account shares these + // one configured values for backend file-ownership purposes. + StandaloneDefaultUserID int + StandaloneDefaultGroupID int + StandaloneDefaultProjectID int + // IAM Cache // // The gateway maintains an in-memory cache of IAM account lookups to @@ -600,7 +640,7 @@ func RunVersityGW(ctx context.Context, be backend.Backend, cfg *Config) error { if cfg.KeyFile == "" { return fmt.Errorf("TLS cert specified without key file") } - cs := utils.NewCertStorage() + cs := netutil.NewCertStorage() if err := cs.SetCertificate(cfg.CertFile, cfg.KeyFile); err != nil { return fmt.Errorf("tls: load certs: %v", err) } @@ -681,6 +721,15 @@ func RunVersityGW(ctx context.Context, be backend.Backend, cfg *Config) error { IpaUser: cfg.IpaUser, IpaPassword: cfg.IpaPassword, IpaInsecure: cfg.IpaInsecure, + StandaloneIAMEndpoint: cfg.StandaloneIAMEndpoint, + StandaloneIAMAccess: cfg.StandaloneIAMAccess, + StandaloneIAMSecret: cfg.StandaloneIAMSecret, + StandaloneClientCert: cfg.StandaloneClientCert, + StandaloneClientCertKey: cfg.StandaloneClientCertKey, + StandaloneServerCA: cfg.StandaloneServerCA, + StandaloneDefaultUserID: cfg.StandaloneDefaultUserID, + StandaloneDefaultGroupID: cfg.StandaloneDefaultGroupID, + StandaloneDefaultProjectID: cfg.StandaloneDefaultProjectID, }) if err != nil { return fmt.Errorf("setup iam: %w", err) @@ -800,7 +849,7 @@ func RunVersityGW(ctx context.Context, be backend.Backend, cfg *Config) error { if cfg.AdminKeyFile == "" { return fmt.Errorf("TLS cert specified without key file") } - cs := utils.NewCertStorage() + cs := netutil.NewCertStorage() if err = cs.SetCertificate(cfg.AdminCertFile, cfg.AdminKeyFile); err != nil { return fmt.Errorf("tls: load certs: %v", err) } @@ -824,7 +873,7 @@ func RunVersityGW(ctx context.Context, be backend.Backend, cfg *Config) error { webTLSKey := "" if len(cfg.WebuiPorts) > 0 { for _, addr := range cfg.WebuiPorts { - if utils.IsUnixSocketPath(addr) { + if netutil.IsUnixSocketPath(addr) { continue } _, webPrt, err := net.SplitHostPort(addr) @@ -855,7 +904,7 @@ func RunVersityGW(ctx context.Context, be backend.Backend, cfg *Config) error { if webTLSKey == "" { return fmt.Errorf("webui TLS cert specified without key file") } - cs := utils.NewCertStorage() + cs := netutil.NewCertStorage() if err := cs.SetCertificate(webTLSCert, webTLSKey); err != nil { return fmt.Errorf("tls: load certs: %v", err) } @@ -923,7 +972,7 @@ func RunVersityGW(ctx context.Context, be backend.Backend, cfg *Config) error { wsTLSKey := "" if len(cfg.WebsitePorts) > 0 { for _, addr := range cfg.WebsitePorts { - if utils.IsUnixSocketPath(addr) { + if netutil.IsUnixSocketPath(addr) { continue } _, wsPrt, err := net.SplitHostPort(addr) @@ -954,7 +1003,7 @@ func RunVersityGW(ctx context.Context, be backend.Backend, cfg *Config) error { if wsTLSKey == "" { return fmt.Errorf("website TLS cert specified without key file") } - cs := utils.NewCertStorage() + cs := netutil.NewCertStorage() if err := cs.SetCertificate(wsTLSCert, wsTLSKey); err != nil { return fmt.Errorf("tls: load certs: %v", err) } @@ -1146,7 +1195,7 @@ func (cfg Config) printBanner() { interfaceMap := make(map[string]bool) for _, portSpec := range cfg.Ports { - if utils.IsUnixSocketPath(portSpec) { + if netutil.IsUnixSocketPath(portSpec) { allPorts = append(allPorts, portSpec) if !interfaceMap[portSpec] { interfaceMap[portSpec] = true @@ -1182,7 +1231,7 @@ func (cfg Config) printBanner() { var allAdmInterfaces []string admInterfaceMap := make(map[string]bool) for _, admPort := range cfg.AdminPorts { - if utils.IsUnixSocketPath(admPort) { + if netutil.IsUnixSocketPath(admPort) { if !admInterfaceMap[admPort] { admInterfaceMap[admPort] = true allAdmInterfaces = append(allAdmInterfaces, admPort) @@ -1215,7 +1264,7 @@ func (cfg Config) printBanner() { var urls []string for _, addrPort := range allInterfaces { - if utils.IsUnixSocketPath(addrPort) { + if netutil.IsUnixSocketPath(addrPort) { urls = append(urls, "unix:"+addrPort) continue } @@ -1233,7 +1282,7 @@ func (cfg Config) printBanner() { var boundHost string if len(cfg.Ports) == 1 { - if utils.IsUnixSocketPath(cfg.Ports[0]) { + if netutil.IsUnixSocketPath(cfg.Ports[0]) { boundHost = fmt.Sprintf("(unix socket: %s)", cfg.Ports[0]) } else { hst, prt, _ := net.SplitHostPort(cfg.Ports[0]) @@ -1267,7 +1316,7 @@ func (cfg Config) printBanner() { if len(allAdmInterfaces) > 0 { lines = append(lines, centerText(""), leftText("Admin service listening on:")) for _, addrPort := range allAdmInterfaces { - if utils.IsUnixSocketPath(addrPort) { + if netutil.IsUnixSocketPath(addrPort) { lines = append(lines, leftText(" unix:"+addrPort)) continue } @@ -1292,7 +1341,7 @@ func (cfg Config) printBanner() { if strings.TrimSpace(webuiAddr) == "" { continue } - if utils.IsUnixSocketPath(webuiAddr) { + if netutil.IsUnixSocketPath(webuiAddr) { if !webInterfaceMap[webuiAddr] { webInterfaceMap[webuiAddr] = true allWebInterfaces = append(allWebInterfaces, webuiAddr) @@ -1321,7 +1370,7 @@ func (cfg Config) printBanner() { if len(allWebInterfaces) > 0 { lines = append(lines, centerText(""), leftText("WebUI listening on:")) for _, addrPort := range allWebInterfaces { - if utils.IsUnixSocketPath(addrPort) { + if netutil.IsUnixSocketPath(addrPort) { lines = append(lines, leftText(" unix:"+addrPort)) continue } @@ -1363,7 +1412,7 @@ func (cfg Config) printBanner() { if strings.TrimSpace(websiteAddr) == "" { continue } - if utils.IsUnixSocketPath(websiteAddr) { + if netutil.IsUnixSocketPath(websiteAddr) { if !websiteInterfaceMap[websiteAddr] { websiteInterfaceMap[websiteAddr] = true allWebsiteInterfaces = append(allWebsiteInterfaces, websiteAddr) @@ -1399,7 +1448,7 @@ func (cfg Config) printBanner() { leftText("Website endpoint listening on:"+domainInfo), ) for _, addrPort := range allWebsiteInterfaces { - if utils.IsUnixSocketPath(addrPort) { + if netutil.IsUnixSocketPath(addrPort) { lines = append(lines, leftText(" unix:"+addrPort)) continue } @@ -1439,11 +1488,11 @@ func leftText(text string) string { // getMatchingIPs returns all IP addresses that the server will listen on // for the given address specification. func getMatchingIPs(spec string) ([]string, error) { - if utils.IsUnixSocketPath(spec) { + if netutil.IsUnixSocketPath(spec) { return []string{spec}, nil } - ips, err := utils.ResolveHostnameIPs(spec) + ips, err := netutil.ResolveHostnameIPs(spec) if err != nil { return nil, fmt.Errorf("resolve hostname: %v", err) } @@ -1497,7 +1546,7 @@ func getAllLocalIPs() ([]string, error) { } func buildServiceURLs(spec string, ssl bool) ([]string, error) { - if utils.IsUnixSocketPath(spec) { + if netutil.IsUnixSocketPath(spec) { return nil, nil } @@ -1628,7 +1677,7 @@ func validatePortConflicts(ports, admPorts, webuiPorts, websitePorts []string) e var allSpecs []portSpec for _, p := range ports { - if utils.IsUnixSocketPath(p) { + if netutil.IsUnixSocketPath(p) { allSpecs = append(allSpecs, portSpec{spec: p, port: p, isUnix: true, portType: "s3"}) continue } @@ -1645,7 +1694,7 @@ func validatePortConflicts(ports, admPorts, webuiPorts, websitePorts []string) e } for _, p := range admPorts { - if utils.IsUnixSocketPath(p) { + if netutil.IsUnixSocketPath(p) { allSpecs = append(allSpecs, portSpec{spec: p, port: p, isUnix: true, portType: "admin"}) continue } @@ -1662,7 +1711,7 @@ func validatePortConflicts(ports, admPorts, webuiPorts, websitePorts []string) e } for _, p := range webuiPorts { - if utils.IsUnixSocketPath(p) { + if netutil.IsUnixSocketPath(p) { allSpecs = append(allSpecs, portSpec{spec: p, port: p, isUnix: true, portType: "webui"}) continue } @@ -1679,7 +1728,7 @@ func validatePortConflicts(ports, admPorts, webuiPorts, websitePorts []string) e } for _, p := range websitePorts { - if utils.IsUnixSocketPath(p) { + if netutil.IsUnixSocketPath(p) { allSpecs = append(allSpecs, portSpec{spec: p, port: p, isUnix: true, portType: "website"}) continue } diff --git a/embedgw/iam.go b/embedgw/iam.go index 5d638171..d8180ade 100644 --- a/embedgw/iam.go +++ b/embedgw/iam.go @@ -26,8 +26,9 @@ import ( "github.com/versity/versitygw/debuglogger" "github.com/versity/versitygw/iamapi" + "github.com/versity/versitygw/iamapi/private" "github.com/versity/versitygw/iamapi/storage" - "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/internal/netutil" ) const iamTitle = "VersityGW IAM API" @@ -79,6 +80,28 @@ type IAMConfig struct { // abstract namespace sockets. SocketPerm string + // PrivatePorts is the list of listening addresses for the standalone + // IAM service's private endpoints (derive-signing-key, evaluate-policy, resolve-identity) + // — see private.PrivateAPI. Each address must be a unix socket, or a TCP + // address with PrivateCertFile/PrivateKeyFile/PrivateClientCAFile all + // set (mTLS with mandatory client-certificate verification); anything + // else fails startup rather than serving these endpoints in the clear. + // Empty disables the private endpoints entirely. + PrivatePorts []string + // PrivateCertFile/PrivateKeyFile are the private listener's own TLS + // server certificate, distinct from CertFile/KeyFile (the public + // control-plane listener's certificate) since the two listeners have + // different security requirements. + PrivateCertFile string + PrivateKeyFile string + // PrivateClientCAFile verifies the S3 gateway's client certificate on + // the private listener. Required, together with PrivateCertFile/ + // PrivateKeyFile, for any non-unix-socket PrivatePorts address. + PrivateClientCAFile string + // PrivateSocketPerm is the octal file-mode string for a file-backed + // unix-socket PrivatePorts address. + PrivateSocketPerm string + // IAMDir enables local file-backed IAM API storage. Set to the directory // path where the IAM API user database is stored. IAMDir string @@ -132,6 +155,56 @@ type IAMConfig struct { DisableOIDCThumbprintAutoFetch bool } +// newPrivateAPI builds the standalone IAM service's private endpoint set +// and the TLS options ServeMultiPort will enforce (mTLS, or nothing at all +// for a unix-socket-only deployment — see netutil.RequireSecureTransport). +func newPrivateAPI(store storage.Storer, cfg *IAMConfig) (*private.PrivateAPI, netutil.TLSOptions, error) { + allSet := cfg.PrivateCertFile != "" && cfg.PrivateKeyFile != "" && cfg.PrivateClientCAFile != "" + noneSet := cfg.PrivateCertFile == "" && cfg.PrivateKeyFile == "" && cfg.PrivateClientCAFile == "" + if !allSet && !noneSet { + return nil, netutil.TLSOptions{}, fmt.Errorf("--private-cert, --private-cert-key, and --private-client-ca must all be set together, or all left empty for a unix-socket-only private listener") + } + + var tlsOpts netutil.TLSOptions + if allSet { + cs := netutil.NewCertStorage() + if err := cs.SetCertificate(cfg.PrivateCertFile, cfg.PrivateKeyFile); err != nil { + return nil, netutil.TLSOptions{}, fmt.Errorf("private listener: load certs: %w", err) + } + pool, err := netutil.LoadCACertPool(cfg.PrivateClientCAFile) + if err != nil { + return nil, netutil.TLSOptions{}, fmt.Errorf("private listener: %w", err) + } + tlsOpts = netutil.TLSOptions{ + GetCertificate: cs.GetCertificate, + ClientCAs: pool, + RequireClientCert: true, + } + } + + var privOpts []private.PrivateAPIOption + if cfg.PrivateSocketPerm != "" { + perm, err := strconv.ParseUint(cfg.PrivateSocketPerm, 8, 32) + if err != nil { + return nil, netutil.TLSOptions{}, fmt.Errorf("invalid PrivateSocketPerm value %q: must be an octal integer (e.g. '0660'): %w", cfg.PrivateSocketPerm, err) + } + privOpts = append(privOpts, private.WithPrivateSocketPerm(os.FileMode(perm))) + } + if cfg.Quiet { + privOpts = append(privOpts, private.WithPrivateQuiet()) + } + + p, err := private.New(store, iamapi.RootCredentials{ + Access: cfg.RootUserAccess, + Secret: cfg.RootUserSecret, + }, privOpts...) + if err != nil { + return nil, netutil.TLSOptions{}, fmt.Errorf("init private IAM API: %w", err) + } + + return p, tlsOpts, nil +} + var iamAPIRunning atomic.Bool // RunIAMAPI starts the VersityGW IAM API with the supplied configuration. It @@ -194,10 +267,6 @@ func RunIAMAPI(ctx context.Context, cfg *IAMConfig) error { opts := []iamapi.Option{ iamapi.WithConcurrencyLimiter(cfg.MaxConnections, cfg.MaxRequests), - iamapi.WithRootUserCreds(iamapi.RootCredentials{ - Access: cfg.RootUserAccess, - Secret: cfg.RootUserSecret, - }), } if cfg.HealthPath != "" { opts = append(opts, iamapi.WithHealth(cfg.HealthPath)) @@ -233,20 +302,38 @@ func RunIAMAPI(ctx context.Context, cfg *IAMConfig) error { opts = append(opts, iamapi.WithTLS(cs)) } - server, err := iamapi.New(store, opts...) + server, err := iamapi.New(store, iamapi.RootCredentials{ + Access: cfg.RootUserAccess, + Secret: cfg.RootUserSecret, + }, opts...) if err != nil { return fmt.Errorf("init IAM API server: %w", err) } + var privateAPI *private.PrivateAPI + var privateTLSOpts netutil.TLSOptions + if len(cfg.PrivatePorts) > 0 { + privateAPI, privateTLSOpts, err = newPrivateAPI(store, cfg) + if err != nil { + return err + } + } + if !cfg.Quiet { cfg.printBanner() } - errCh := make(chan error, 1) + errCh := make(chan error, 2) go func() { errCh <- server.ServeMultiPort(cfg.Ports) }() + if privateAPI != nil { + go func() { + errCh <- privateAPI.ServeMultiPort(cfg.PrivatePorts, privateTLSOpts) + }() + } + var sigHup <-chan struct{} if cfg.SigHup != nil { sigHup = cfg.SigHup @@ -277,6 +364,11 @@ Loop: if err := server.Shutdown(); err != nil { fmt.Fprintf(os.Stderr, "shutdown IAM API server: %v\n", err) } + if privateAPI != nil { + if err := privateAPI.Shutdown(); err != nil { + fmt.Fprintf(os.Stderr, "shutdown private IAM API server: %v\n", err) + } + } return saveErr } @@ -310,6 +402,16 @@ func (cfg IAMConfig) printBanner() { lines = append(lines, leftText(" "+u)) } + if len(cfg.PrivatePorts) > 0 { + privateInterfaces, _ := resolveIAMBannerInterfaces(cfg.PrivatePorts) + if len(privateInterfaces) > 0 { + lines = append(lines, centerText(""), leftText("IAM private service listening on:")) + for _, u := range buildIAMBannerURLs(privateInterfaces, cfg.PrivateCertFile != "" || cfg.PrivateKeyFile != "") { + lines = append(lines, leftText(" "+u)) + } + } + } + fmt.Println("┌" + strings.Repeat("─", columnWidth-2) + "┐") for _, line := range lines { fmt.Printf("│%-*s│\n", columnWidth-2, line) @@ -323,7 +425,7 @@ func resolveIAMBannerInterfaces(ports []string) ([]string, []string) { interfaceMap := make(map[string]bool) for _, portSpec := range ports { - if utils.IsUnixSocketPath(portSpec) { + if netutil.IsUnixSocketPath(portSpec) { allPorts = append(allPorts, portSpec) if !interfaceMap[portSpec] { interfaceMap[portSpec] = true @@ -358,7 +460,7 @@ func resolveIAMBannerInterfaces(ports []string) ([]string, []string) { func formatIAMBannerBoundHost(ports, allPorts []string) string { if len(ports) == 1 { - if utils.IsUnixSocketPath(ports[0]) { + if netutil.IsUnixSocketPath(ports[0]) { return fmt.Sprintf("(unix socket: %s)", ports[0]) } hst, prt, _ := net.SplitHostPort(ports[0]) @@ -379,7 +481,7 @@ func buildIAMBannerURLs(interfaces []string, tls bool) []string { } for _, addrPort := range interfaces { - if utils.IsUnixSocketPath(addrPort) { + if netutil.IsUnixSocketPath(addrPort) { urls = append(urls, "unix:"+addrPort) continue } diff --git a/genmtlscerts.sh b/genmtlscerts.sh new file mode 100755 index 00000000..7d9f9235 --- /dev/null +++ b/genmtlscerts.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# +# Generate the mTLS material the S3 gateway needs to talk to a standalone IAM +# service's private endpoints over TCP: +# +# /ca.pem CA certificate, trusted by both sides +# /iam-server.pem IAM private-listener server certificate +# /iam-server.key +# /gw-client.pem S3 gateway client certificate +# /gw-client.key +# +# Usage: genmtlscerts.sh [server-ip] +# +# The server certificate carries an IP SAN for server-ip (default 127.0.0.1) +# because the gateway dials the private endpoint as "https://:" +# with standard Go certificate verification and no hostname override — an IP +# endpoint therefore needs an IP SAN, not a CN or a DNS SAN, or the handshake +# fails with a name-mismatch error. + +set -Eeuo pipefail + +if [[ $# -lt 1 ]]; then + echo "usage: $0 [server-ip]" >&2 + exit 1 +fi + +CERT_DIR="$1" +SERVER_IP="${2:-127.0.0.1}" + +mkdir -p "$CERT_DIR" + +EXT_FILE="$CERT_DIR/openssl-ext.cnf" +# Written as a file rather than passed via -addext so this works on both +# OpenSSL and the LibreSSL +cat >"$EXT_FILE" </dev/null +openssl req -new -x509 -key "$CERT_DIR/ca.key" -out "$CERT_DIR/ca.pem" -days 1 \ + -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versitygw-test-ca" + +# IAM private-listener server certificate +openssl genpkey -algorithm RSA -out "$CERT_DIR/iam-server.key" -pkeyopt rsa_keygen_bits:2048 2>/dev/null +openssl req -new -key "$CERT_DIR/iam-server.key" -out "$CERT_DIR/iam-server.csr" \ + -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versitygw-iam-private" +openssl x509 -req -in "$CERT_DIR/iam-server.csr" -CA "$CERT_DIR/ca.pem" -CAkey "$CERT_DIR/ca.key" \ + -CAcreateserial -out "$CERT_DIR/iam-server.pem" -days 1 \ + -extfile "$EXT_FILE" -extensions server 2>/dev/null + +# S3 gateway client certificate. The IAM service verifies it against the CA +# but does not authorize on its identity — authorization is the root SigV4 +# credential the gateway signs each private request with. +openssl genpkey -algorithm RSA -out "$CERT_DIR/gw-client.key" -pkeyopt rsa_keygen_bits:2048 2>/dev/null +openssl req -new -key "$CERT_DIR/gw-client.key" -out "$CERT_DIR/gw-client.csr" \ + -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versitygw-s3-gateway" +openssl x509 -req -in "$CERT_DIR/gw-client.csr" -CA "$CERT_DIR/ca.pem" -CAkey "$CERT_DIR/ca.key" \ + -CAcreateserial -out "$CERT_DIR/gw-client.pem" -days 1 \ + -extfile "$EXT_FILE" -extensions client 2>/dev/null + +rm -f "$CERT_DIR"/*.csr "$EXT_FILE" diff --git a/go.mod b/go.mod index d934280b..60c719a5 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,6 @@ require ( github.com/go-ldap/ldap/v3 v3.4.14 github.com/gofiber/fiber/v3 v3.4.0 github.com/golang-jwt/jwt/v5 v5.3.1 - github.com/google/go-cmp v0.7.0 github.com/google/uuid v1.6.0 github.com/hashicorp/vault-client-go v0.4.3 github.com/minio/crc64nvme v1.1.1 diff --git a/go.sum b/go.sum index 37ffe0c4..a8988af0 100644 --- a/go.sum +++ b/go.sum @@ -89,8 +89,6 @@ github.com/gofiber/utils/v2 v2.4.1/go.mod h1:I+RTsgMUdzFuifVc3LOEkfh32wQW9BfRl7l github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= diff --git a/iamapi/authentication_test.go b/iamapi/authentication_test.go index a99b5ed0..3aaf2ff3 100644 --- a/iamapi/authentication_test.go +++ b/iamapi/authentication_test.go @@ -31,7 +31,6 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" awsv4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" "github.com/gofiber/fiber/v3" - vgwv4 "github.com/versity/versitygw/aws/signer/v4" "github.com/versity/versitygw/iamapi/iamerr" "github.com/versity/versitygw/iamapi/internal/iammiddleware" "github.com/versity/versitygw/internal/sigv4auth" @@ -339,12 +338,11 @@ func TestVerifyIAMAuthRejectsQueryWrongCredentialRegion(t *testing.T) { // TestVerifyIAMAuthRejectsExpiredQueryRequest confirms a presigned IAM // request signed too long ago is rejected by the same fixed ±15-minute -// freshness window (ValidateDateAt) header auth uses — confirmed live -// (niksis02 profile): real IAM's query-auth ignores X-Amz-Expires entirely -// (see TestVerifyIAMAuthQueryIgnoresXAmzExpires) and instead rejects a -// stale signing time with SignatureDoesNotMatch: "Signature expired: ... -// is now earlier than ... (... - 15 min.)" — byte-for-byte what this -// codebase's own SignatureDoesNotMatchExpired already produces. +// freshness window (ValidateDateAt) header auth uses: real IAM's +// query-auth ignores X-Amz-Expires entirely and instead rejects a stale +// signing time with SignatureDoesNotMatch: "Signature expired: ... is now +// earlier than ... (... - 15 min.)" — byte-for-byte what this codebase's +// own SignatureDoesNotMatchExpired already produces. func TestVerifyIAMAuthRejectsExpiredQueryRequest(t *testing.T) { app := newIAMAuthTestApp(t) signedTwoHoursAgo := time.Now().UTC().Add(-2 * time.Hour) @@ -373,10 +371,10 @@ func TestVerifyIAMAuthRejectsExpiredQueryRequest(t *testing.T) { } // TestVerifyIAMAuthQueryIgnoresXAmzExpires confirms IAM/STS query-auth -// neither requires nor validates X-Amz-Expires, unlike S3's presigned URLs -// — confirmed live (niksis02 profile) that real IAM's ListUsers accepts a -// presigned request with X-Amz-Expires omitted, non-numeric, negative, or -// far beyond S3's 604800-second maximum, every time. +// neither requires nor validates X-Amz-Expires, unlike S3's presigned URLs: +// real IAM's ListUsers accepts a presigned request with X-Amz-Expires +// omitted, non-numeric, negative, or far beyond S3's 604800-second +// maximum, every time. func TestVerifyIAMAuthQueryIgnoresXAmzExpires(t *testing.T) { for _, expires := range []string{"", "abc", "-5", "9999999"} { t.Run(expires, func(t *testing.T) { @@ -418,15 +416,21 @@ func TestVerifyIAMAuthRejectsSessionTokenHeaderNotSigned(t *testing.T) { hash := sha256.Sum256(body) payloadHash := hex.EncodeToString(hash[:]) - signer := vgwv4.NewSigner() // Sign with only "host" listed — the security-token header is present // on the wire but deliberately excluded from SignedHeaders, simulating // a client (or tampering party) that never binds it to the signature. - if _, err := signer.SignHTTP(context.Background(), - aws.Credentials{AccessKeyID: session.AccessKeyId, SecretAccessKey: session.SecretAccessKey}, - req, payloadHash, "iam", iammiddleware.SigningRegion, time.Now().UTC(), []string{"host"}); err != nil { - t.Fatalf("sign request: %v", err) - } + signingTime := time.Now().UTC() + yyyymmdd := signingTime.Format(sigv4auth.YYYYMMDD) + derivedKey := sigv4auth.DeriveKey(session.SecretAccessKey, yyyymmdd, iammiddleware.SigningRegion, "iam") + in := sigv4auth.SigningInputFromRequest(req) + in.AccessKeyID = session.AccessKeyId + in.CredentialScope = sigv4auth.BuildCredentialScope(yyyymmdd, iammiddleware.SigningRegion, "iam") + in.SignedHdrs = []string{"host"} + in.PayloadHash = payloadHash + in.SigningTime = signingTime + result := sigv4auth.BuildAndSign(derivedKey, in) + req.Header.Set("X-Amz-Date", result.AmzDate) + req.Header.Set("Authorization", result.AuthorizationHeader) resp, err := server.app.Test(req) if err != nil { @@ -672,23 +676,21 @@ func querySignedIAMRequest(t *testing.T, method, target string, body []byte, sec hash := sha256.Sum256(body) payloadHash := hex.EncodeToString(hash[:]) - signer := vgwv4.NewSigner() - signedURL, signedHeaders, _, err := signer.PresignHTTP( - context.Background(), - aws.Credentials{AccessKeyID: testRoot.Access, SecretAccessKey: secret}, - req, - payloadHash, - "iam", - region, - signingTime, - nil, - ) - if err != nil { - t.Fatalf("presign request: %v", err) - } + yyyymmdd := signingTime.Format(sigv4auth.YYYYMMDD) + derivedKey := sigv4auth.DeriveKey(secret, yyyymmdd, region, "iam") + in := sigv4auth.SigningInputFromRequest(req) + in.AccessKeyID = testRoot.Access + in.CredentialScope = sigv4auth.BuildCredentialScope(yyyymmdd, region, "iam") + in.PayloadHash = payloadHash + in.SigningTime = signingTime + in.IsPreSign = true + result := sigv4auth.BuildAndSign(derivedKey, in) - signedReq := httptest.NewRequest(method, signedURL, bytes.NewReader(body)) - for key, values := range signedHeaders { + signedURL := *req.URL + signedURL.RawQuery = result.RawQuery + + signedReq := httptest.NewRequest(method, signedURL.String(), bytes.NewReader(body)) + for key, values := range result.SignedHeaders { for _, value := range values { signedReq.Header.Add(key, value) } diff --git a/iamapi/controller_test.go b/iamapi/controller_test.go index 2f12d765..39bf5156 100644 --- a/iamapi/controller_test.go +++ b/iamapi/controller_test.go @@ -171,10 +171,9 @@ func TestIAMApiControllerUserLifecycle(t *testing.T) { // TestIAMApiControllerGetRootUser confirms GetUser's self-lookup form // (UserName omitted, the only way any real AWS SDK/CLI ever invokes it, // since Query-protocol clients simply don't serialize an absent optional -// field — confirmed live: `aws iam get-user` with no --user-name, as root, -// succeeds and returns the root pseudo-user) and its non-standard explicit- -// empty-string equivalent both resolve to the actual authenticated caller — -// root, here, since doIAMAction always signs as root. +// field) and its non-standard explicit-empty-string equivalent both +// resolve to the actual authenticated caller — root, here, since +// doIAMAction always signs as root. func TestIAMApiControllerGetRootUser(t *testing.T) { server := newIAMControllerTestServer(t) @@ -2025,7 +2024,7 @@ func TestIAMApiControllerOIDCThumbprintAutoFetchDisabled(t *testing.T) { if err != nil { t.Fatalf("storage.New: %v", err) } - server, err := New(store, WithQuiet(), WithRootUserCreds(testRoot), WithOIDCThumbprintAutoFetchDisabled()) + server, err := New(store, testRoot, WithQuiet(), WithOIDCThumbprintAutoFetchDisabled()) if err != nil { t.Fatalf("New: %v", err) } @@ -2061,7 +2060,7 @@ func newIAMControllerTestServer(t *testing.T) *IAMApiServer { if err != nil { t.Fatalf("storage.New: %v", err) } - server, err := New(store, WithQuiet(), WithRootUserCreds(testRoot)) + server, err := New(store, testRoot, WithQuiet()) if err != nil { t.Fatalf("New: %v", err) } @@ -2723,10 +2722,9 @@ func TestIAMApiControllerAssumeRoleWithWebIdentityMultiplePrincipalsInArray(t *t "WebIdentityToken": {token}, }) // Passes trust evaluation and the audience check; fails only at the - // network-dependent signature verification step (see the IDP - // communication error test below for that path exercised - // deterministically) — here it's enough to confirm it gets that far - // rather than being rejected as AccessDenied/InvalidIdentityToken. + // network-dependent signature verification step — here it's enough to + // confirm it gets that far rather than being rejected as + // AccessDenied/InvalidIdentityToken. requireSTSError(t, resp, http.StatusBadRequest, "Sender", "InvalidIdentityToken", "Couldn't retrieve verification key from your identity provider, please reference AssumeRoleWithWebIdentity documentation for requirements") } @@ -2954,8 +2952,7 @@ func TestIAMApiControllerGetCallerIdentityIncorrectServiceScope(t *testing.T) { // URL), signed with the given credentials. When sessionToken is non-empty, // the real v4 signer adds X-Amz-Security-Token to the query string itself // — the same way AWS's own SDKs presign a request for temporary -// credentials (confirmed live against real AWS: such a request, submitted -// as a plain HTTP GET with no Authorization header, succeeds). +// credentials. func querySignedSTSRequest(t *testing.T, access, secret, sessionToken, target string) *http.Request { t.Helper() @@ -2975,10 +2972,8 @@ func querySignedSTSRequest(t *testing.T, access, secret, sessionToken, target st // TestIAMApiControllerGetCallerIdentityQueryAuthWithSessionToken confirms a // temporary (ASIA…) session CAN authenticate via query-string (presigned -// URL) auth when X-Amz-Security-Token matches the session — confirmed live -// against real AWS (a genuine sts.PresignClient-generated presigned -// GetCallerIdentity request, signed with real ASIA… credentials and -// submitted as a plain HTTP GET, returns 200). +// URL) auth when X-Amz-Security-Token matches the session, matching a +// genuine sts.PresignClient-generated presigned GetCallerIdentity request. func TestIAMApiControllerGetCallerIdentityQueryAuthWithSessionToken(t *testing.T) { server := newIAMControllerTestServer(t) diff --git a/iamapi/internal/iammiddleware/auth.go b/iamapi/internal/iammiddleware/auth.go index c79284ee..91834588 100644 --- a/iamapi/internal/iammiddleware/auth.go +++ b/iamapi/internal/iammiddleware/auth.go @@ -14,7 +14,6 @@ package iammiddleware import ( - "context" "errors" "strconv" "time" @@ -43,10 +42,10 @@ const ( // canonical request and left unbound to the signature. // // This only applies to header auth. Query-string (presigned) auth carries -// the token as a query parameter instead, which createPresignedHTTPRequestFromCtx -// already includes in the signed canonical query string regardless of -// SignedHeaders, so requiredSignedHeaders (unconditionally "host") is used -// for both root/permanent and session query-auth requests. +// the token as a query parameter instead, which sigv4auth's presign query +// extraction already includes in the signed canonical query string +// regardless of SignedHeaders, so requiredSignedHeaders (unconditionally +// "host") is used for both root/permanent and session query-auth requests. var ( requiredSignedHeaders = []string{"host"} requiredTempSignedHeaders = []string{"host", sigv4auth.HeaderSecurityToken} @@ -67,18 +66,6 @@ type RootCredentials struct { Secret string } -// IdentityStore resolves an access key id to the session or long-term user -// that owns it, and resolves named resources for policy evaluation. -// storage.Storer satisfies this directly. -type IdentityStore interface { - GetSession(ctx context.Context, accessKeyID string) (*types.Session, error) - GetRole(ctx context.Context, roleName string) (*types.Role, error) - GetUserByAccessKeyID(ctx context.Context, accessKeyID string) (*types.User, error) - GetUser(ctx context.Context, username string) (*types.User, error) - GetOIDCProvider(ctx context.Context, arn string) (*types.OIDCProvider, error) - RecordAccessKeyUsage(ctx context.Context, accessKeyID, service, region string, when time.Time) error -} - // VerifyIAMAuth authenticates a request against service (sigv4auth.ServiceIAM // or sigv4auth.ServiceSTS). // @@ -88,21 +75,21 @@ type IdentityStore interface { // identity (and, for a user/session, its policy documents) is stored via // httpctx.ContextKeyCallerIdentity for the policy middleware and controllers // to read back. Root bypasses the policy middleware entirely -func VerifyIAMAuth(service string, root *RootCredentials, store IdentityStore) fiber.Handler { +func VerifyIAMAuth(service string, root *RootCredentials, store iamutil.IdentityStore) fiber.Handler { return func(ctx fiber.Ctx) error { authData, tdate, queryAuth, err := parseIAMAuth(ctx, service) if err != nil { return err } - // A security token in the query string is only ever legitimate - // alongside a temporary (ASIA…) access key — reject it outright for - // root or any long-term (AKIA…) credential before any signature - // work, the same way for both, rather than letting it fall through - // to a signature-mismatch error once a tampered/unsigned token - // param invalidates the canonical query string. - if queryAuth && !iamutil.IsTempAccessKeyID(authData.Access) && - ctx.Request().URI().QueryArgs().Has(sigv4auth.QuerySecurityToken) { + // A security token paired with root or any long-term (AKIA…) + // credential is rejected inside sigv4auth.ParseQueryAuthorization + // for query auth, and just below for header auth — before any + // signature work either way, rather than letting it fall through to + // a signature-mismatch error once a tampered/unsigned token + // invalidates the canonical request. + if !queryAuth && !sigv4auth.IsTempAccessKeyID(authData.Access) && + ctx.Get(sigv4auth.HeaderSecurityToken) != "" { return iamerr.GetAPIError(iamerr.ErrInvalidClientTokenID) } @@ -124,24 +111,42 @@ func VerifyIAMAuth(service string, root *RootCredentials, store IdentityStore) f } httpctx.ContextKeyCallerIdentity.Set(ctx, *identity) + // Best-effort update of a permanent access key's GetAccessKeyLastUsed + // metadata, matching real IAM's behavior. A failure is only logged, + // never returned: this is purely informational, and a lost update + // under concurrent use is immaterial. Called synchronously: a Storer + // implementation for which this is network-bound (e.g. Vault) is + // expected to make it non-blocking itself. if identity.User != nil { - recordAccessKeyUsage(ctx.Context(), store, authData.Access, service) + if err := store.RecordAccessKeyUsage(ctx.Context(), authData.Access, service, SigningRegion, time.Now().UTC()); err != nil { + debuglogger.Logf("failed to record access key last-used metadata for %q: %v", authData.Access, err) + } } return nil } } -// recordAccessKeyUsage best-effort-updates a permanent access key's -// GetAccessKeyLastUsed metadata (service, region, and timestamp) after it -// successfully authenticates a request, matching real IAM's behavior. A -// failure is only logged, never returned, since this is purely -// informational metadata and a lost update under concurrent use is -// immaterial. Called synchronously: a Storer implementation for which this -// update is network-bound (e.g. Vault) is expected to make it non-blocking -// itself rather than adding that latency to every authenticated request -func recordAccessKeyUsage(reqCtx context.Context, store IdentityStore, accessKeyID, service string) { - if err := store.RecordAccessKeyUsage(reqCtx, accessKeyID, service, SigningRegion, time.Now().UTC()); err != nil { - debuglogger.Logf("failed to record access key last-used metadata for %q: %v", accessKeyID, err) +// VerifyRootOnlySigV4 authenticates a request as strictly the configured +// root credential — used by the standalone IAM service's private +// endpoints, which only the S3 gateway itself ever calls, signing as its +// own configured IAM-client identity (root, or a dedicated IAM-access +// credential that defaults to root). Unlike VerifyIAMAuth, any +// other access key — valid IAM user, session, or unknown — is rejected +// outright before any signature work: there is no identity to resolve on +// behalf of here, and these two endpoints exist specifically so no identity +// other than the gateway's own ever needs to reach them. +func VerifyRootOnlySigV4(service string, root *RootCredentials) fiber.Handler { + return func(ctx fiber.Ctx) error { + authData, tdate, queryAuth, err := parseIAMAuth(ctx, service) + if err != nil { + return err + } + + if authData.Access != root.Access { + return iamerr.GetAPIError(iamerr.ErrInvalidClientTokenID) + } + + return checkSignature(ctx, authData, root.Secret, tdate, queryAuth, service) } } @@ -154,12 +159,10 @@ func recordAccessKeyUsage(reqCtx context.Context, store IdentityStore, accessKey // // A temporary session can be used via query-string (presigned URL) // authentication — real AWS accepts X-Amz-Security-Token as a query -// parameter for exactly this (confirmed live: a genuine presigned -// sts:GetCallerIdentity request signed with temporary/session credentials, -// carrying X-Amz-Security-Token in the query string, succeeds against real -// AWS). VerifyIAMAuth already rejects a security token paired with any -// non-temporary credential (root included) before this is ever reached. -func resolveIdentity(ctx fiber.Ctx, store IdentityStore, authData sigv4auth.AuthData, queryAuth bool) (*types.Identity, string, error) { +// parameter for exactly this. VerifyIAMAuth already rejects a security +// token paired with any non-temporary credential (root included) before +// this is ever reached. +func resolveIdentity(ctx fiber.Ctx, store iamutil.IdentityStore, authData sigv4auth.AuthData, queryAuth bool) (*types.Identity, string, error) { if store == nil { return nil, "", iamerr.GetAPIError(iamerr.ErrInvalidClientTokenID) } @@ -167,73 +170,30 @@ func resolveIdentity(ctx fiber.Ctx, store IdentityStore, authData sigv4auth.Auth if iamutil.IsTempAccessKeyID(authData.Access) { return resolveSessionIdentity(ctx, store, authData, queryAuth) } - return resolveUserIdentity(ctx, store, authData) -} -func resolveSessionIdentity(ctx fiber.Ctx, store IdentityStore, authData sigv4auth.AuthData, queryAuth bool) (*types.Identity, string, error) { - session, err := store.GetSession(ctx.Context(), authData.Access) + identity, secret, err := iamutil.ResolveUserIdentity(ctx, store, authData.Access) if err != nil { return nil, "", iamerr.GetAPIError(iamerr.ErrInvalidClientTokenID) } + return identity, secret, nil +} +// resolveSessionIdentity extracts the security token from wherever this +// request carries it and delegates to iamutil.ResolveSessionByToken, mapping +// its sentinel errors onto the control plane's single public-facing error — +// which deliberately does not distinguish "no such session" from "wrong +// token" for an unauthenticated caller. +func resolveSessionIdentity(ctx fiber.Ctx, store iamutil.IdentityStore, authData sigv4auth.AuthData, queryAuth bool) (*types.Identity, string, error) { token := ctx.Get(sigv4auth.HeaderSecurityToken) if queryAuth { token = ctx.Query(sigv4auth.QuerySecurityToken) } - if token == "" || !sigv4auth.SecureCompare(token, session.SessionToken) { - return nil, "", iamerr.GetAPIError(iamerr.ErrInvalidClientTokenID) - } - // A signature-valid, unexpired session still authenticates even if its - // role has since been deleted — real STS credentials are self-contained - // and don't re-check role existence on every call. What such a session - // can no longer do is get any IAM action past the policy middleware: - // with Role/IdentityPolicies left unset, EvaluateIdentityPolicies denies - // by default, same effective outcome as an explicit rejection here would - // have had for every pipeline except GetCallerIdentity, which needs - // none of this and must keep working regardless. - // - // The reloaded role must also still be the *same* role the session was - // originally minted against — RoleID and Arn, both captured in the - // session at AssumeRoleWithWebIdentity time, must match the freshly - // loaded role's own values. Without this check, deleting a role and - // recreating one of the same name (necessarily getting a new RoleID) - // would let every pre-existing session for the old role silently - // inherit whatever policies the new role happens to carry. - identity := &types.Identity{ - Session: session, - SessionPolicy: session.Policy, - } - if role, err := store.GetRole(ctx.Context(), session.RoleName); err == nil && - role.RoleID == session.RoleID && role.Arn == session.RoleArn { - identity.Role = role - identity.IdentityPolicies = role.Policies.Inline - } - return identity, session.SecretAccessKey, nil -} - -func resolveUserIdentity(ctx fiber.Ctx, store IdentityStore, authData sigv4auth.AuthData) (*types.Identity, string, error) { - user, err := store.GetUserByAccessKeyID(ctx.Context(), authData.Access) + identity, secret, err := iamutil.ResolveSessionByToken(ctx.Context(), store, authData.Access, token) if err != nil { return nil, "", iamerr.GetAPIError(iamerr.ErrInvalidClientTokenID) } - - var keyEntry *types.AccessKeyEntry - for i := range user.AccessKeys { - if user.AccessKeys[i].AccessKeyId == authData.Access { - keyEntry = &user.AccessKeys[i] - break - } - } - if keyEntry == nil || keyEntry.Status != iamutil.AccessKeyStatusActive { - return nil, "", iamerr.GetAPIError(iamerr.ErrInvalidClientTokenID) - } - - identity := &types.Identity{ - User: user, - IdentityPolicies: user.Policies.Inline, - } - return identity, keyEntry.SecretAccessKey, nil + return identity, secret, nil } func checkSignature(ctx fiber.Ctx, authData sigv4auth.AuthData, secret string, tdate time.Time, queryAuth bool, service string) error { @@ -242,14 +202,16 @@ func checkSignature(ctx fiber.Ctx, authData sigv4auth.AuthData, secret string, t return err } + derivedKey := sigv4auth.DeriveKey(secret, tdate.Format(sigv4auth.YYYYMMDD), authData.Region, service) + payloadHash := sigv4auth.PayloadSHA256Hex(ctx.BodyRaw()) if queryAuth { - _, err = sigv4auth.CheckQuerySignature(ctx, authData, secret, payloadHash, tdate, contentLength, sigv4auth.CheckOptions{ + _, err = sigv4auth.CheckQuerySignature(ctx, authData, derivedKey, payloadHash, tdate, contentLength, sigv4auth.CheckOptions{ Service: service, RequiredSignedHeaders: requiredSignedHeaders, }) } else { - _, err = sigv4auth.CheckSignature(ctx, authData, secret, payloadHash, tdate, contentLength, sigv4auth.CheckOptions{ + _, err = sigv4auth.CheckSignature(ctx, authData, derivedKey, payloadHash, tdate, contentLength, sigv4auth.CheckOptions{ Service: service, RequiredSignedHeaders: requiredHeaderAuthSignedHeaders(authData.Access), }) @@ -311,12 +273,10 @@ func parseIAMHeaderAuth(ctx fiber.Ctx, expectedService string) (sigv4auth.AuthDa } // parseIAMQueryAuth parses SigV4 query-string (presigned URL) authentication -// parameters. Unlike S3 (see s3api/utils/presign-auth-reader.go), IAM/STS -// query-auth does not use X-Amz-Expires at all: confirmed live (niksis02 -// profile) against real IAM's ListUsers — a presigned request with -// X-Amz-Expires omitted, non-numeric ("abc"), negative ("-5"), or far -// beyond the 604800-second S3 maximum ("9999999") is accepted every time, -// while a request merely signed too long ago is rejected with +// parameters. Unlike S3, IAM/STS query-auth does not use X-Amz-Expires at +// all: a presigned request with X-Amz-Expires omitted, non-numeric, +// negative, or far beyond S3's 604800-second maximum is accepted every +// time, while a request merely signed too long ago is rejected with // SignatureDoesNotMatch ("Signature expired: ... is now earlier than ... // (... - 15 min.)") — byte-for-byte the same message this codebase's own // SignatureDoesNotMatchExpired already produces. So X-Amz-Expires is diff --git a/iamapi/internal/iammiddleware/errors.go b/iamapi/internal/iammiddleware/errors.go index fb8df834..41342797 100644 --- a/iamapi/internal/iammiddleware/errors.go +++ b/iamapi/internal/iammiddleware/errors.go @@ -15,6 +15,7 @@ package iammiddleware import ( "errors" + "strings" "github.com/gofiber/fiber/v3" "github.com/versity/versitygw/debuglogger" @@ -33,6 +34,13 @@ func GlobalErrorHandler(ctx fiber.Ctx, er error) error { return ctx.Status(apiErr.StatusCode()).Send(apiErr.XMLBody(requestID)) } + var fiberErr *fiber.Error + if errors.As(er, &fiberErr) && strings.Contains(strings.ToLower(fiberErr.Message), "cannot parse content-length") { + debuglogger.Logf("failed to parse Content-Length") + ctx.Status(fiber.StatusBadRequest) + return nil + } + if httpctx.ContextKeyStack.IsSet(ctx) { debuglogger.Panic(er) } else { diff --git a/iamapi/internal/iammiddleware/policy.go b/iamapi/internal/iammiddleware/policy.go index 855753fb..71510693 100644 --- a/iamapi/internal/iammiddleware/policy.go +++ b/iamapi/internal/iammiddleware/policy.go @@ -15,6 +15,7 @@ package iammiddleware import ( + "maps" "strconv" "time" @@ -51,7 +52,7 @@ const iamActionPrefix = "iam:" // requestConditionContext supplies the request's aws:SourceIp/aws:username/ // aws:PrincipalArn/aws:CurrentTime/aws:EpochTime values for a statement's // Condition block. -func VerifyIAMPolicy(store IdentityStore) fiber.Handler { +func VerifyIAMPolicy(store iamutil.IdentityStore) fiber.Handler { return func(ctx fiber.Ctx) error { identity, _ := httpctx.ContextKeyCallerIdentity.Get(ctx).(types.Identity) if identity.IsRoot { @@ -68,8 +69,8 @@ func VerifyIAMPolicy(store IdentityStore) fiber.Handler { Condition: requestConditionContext(ctx, identity, action, resourceTags), } - if !authorizeRequest(identity, reqCtx) { - return iamerr.AccessDeniedIAMAction(callerArn(identity), fullAction) + if Authorize(identity, reqCtx) != policy.DecisionAllow { + return iamerr.AccessDeniedIAMAction(CallerArn(identity), fullAction) } // A rename/path-move is a two-resource transition: AWS's UpdateUser @@ -79,8 +80,8 @@ func VerifyIAMPolicy(store IdentityStore) fiber.Handler { if target := updateUserTargetResource(ctx, store); target != "" { targetCtx := reqCtx targetCtx.Resource = target - if !authorizeRequest(identity, targetCtx) { - return iamerr.AccessDeniedIAMAction(callerArn(identity), fullAction) + if Authorize(identity, targetCtx) != policy.DecisionAllow { + return iamerr.AccessDeniedIAMAction(CallerArn(identity), fullAction) } } } @@ -89,20 +90,58 @@ func VerifyIAMPolicy(store IdentityStore) fiber.Handler { } } -// authorizeRequest reports whether reqCtx is allowed by identity's own -// inline policies and, for a session with a session policy attached, the -// narrowing session policy as well. -func authorizeRequest(identity types.Identity, reqCtx policy.RequestContext) bool { - if !policy.EvaluateIdentityPolicies(identity.IdentityPolicies, reqCtx) { - return false +// Authorize reports how identity's own inline policies and, for a session +// with a session policy attached, the narrowing session policy as well, +// decide reqCtx. +// +// A session policy can only narrow, never widen, what the role's identity +// policies otherwise allow — matching AWS's permission-boundary semantics +// for AssumeRole session policies — so this is an intersection, not the +// "either source is independently sufficient" combination VerifyAccess uses +// for S3 bucket-policy-vs-identity-policy: an explicit Deny from either +// layer here always wins outright, and the result is DecisionAllow only +// when both layers (or just the identity layer, absent a session policy) +// independently reach DecisionAllow. +func Authorize(identity types.Identity, reqCtx policy.RequestContext) policy.Decision { + d, sd, hasSessionPolicy := AuthorizeSplit(identity, reqCtx) + if d == policy.DecisionDeny { + return policy.DecisionDeny } - if identity.Session != nil && identity.SessionPolicy != "" { - sessionPolicy := []types.PolicyEntry{{PolicyDocument: identity.SessionPolicy}} - if !policy.EvaluateIdentityPolicies(sessionPolicy, reqCtx) { - return false - } + if !hasSessionPolicy { + return d } - return true + if sd == policy.DecisionDeny { + return policy.DecisionDeny + } + if d != policy.DecisionAllow || sd != policy.DecisionAllow { + return policy.DecisionNoMatch + } + return policy.DecisionAllow +} + +// AuthorizeSplit reports the identity-policy and session-policy decisions +// separately, rather than folded together as Authorize does, plus whether a +// session policy applied at all. +// +// The two must stay separable for the S3 data plane, where a *resource* +// policy is also in play. A session policy filters everything, including +// permissions that came from the bucket policy rather than from the role +// with a role carrying no identity policy at all, a bucket policy granting +// s3:GetObject and s3:PutObject to that role, and a session policy allowing only +// s3:GetObject, the Get succeeds and the Put is denied. Collapsing the two into +// one decision here would lose the distinction between "the session policy did +// not permit this" (which must deny even against a bucket-policy Allow) and +// "the role's own policies did not permit this" (which a bucket-policy +// Allow may still grant). +func AuthorizeSplit(identity types.Identity, reqCtx policy.RequestContext) (identityDecision, sessionDecision policy.Decision, hasSessionPolicy bool) { + identityDecision = policy.EvaluateIdentityPolicies(identity.IdentityPolicies, reqCtx) + + if identity.Session == nil || identity.SessionPolicy == "" { + return identityDecision, policy.DecisionNoMatch, false + } + + sessionPolicy := []types.PolicyEntry{{PolicyDocument: identity.SessionPolicy}} + return identityDecision, policy.EvaluateIdentityPolicies(sessionPolicy, reqCtx), true } // resourceForAction resolves the ARN action targets and, when that ARN names @@ -127,7 +166,7 @@ func authorizeRequest(identity types.Identity, reqCtx policy.RequestContext) boo // request still reaches the controller afterward, which reports the // specific NoSuchEntity/MissingValue error if authorization happens to pass // on a wildcard grant, or AccessDenied first if it doesn't. -func resourceForAction(ctx fiber.Ctx, store IdentityStore, action string) (string, []types.Tag) { +func resourceForAction(ctx fiber.Ctx, store iamutil.IdentityStore, action string) (string, []types.Tag) { switch action { case "CreateUser": return newUserResource(ctx), nil @@ -177,7 +216,7 @@ func newUserResource(ctx fiber.Ctx) string { // used elsewhere — none of this group's actions actually accept an omitted // UserName (the controller layer requires it), so this only guards against // a malformed request reaching here. -func existingUserResource(ctx fiber.Ctx, store IdentityStore) (string, []types.Tag) { +func existingUserResource(ctx fiber.Ctx, store iamutil.IdentityStore) (string, []types.Tag) { userName, ok := iamutil.RequestParam(ctx, "UserName") if !ok || userName == "" { return "", nil @@ -195,7 +234,7 @@ func existingUserResource(ctx fiber.Ctx, store IdentityStore) (string, []types.T // own Arn and Tags. A session (assumed role) has no self IAM user to // resolve, so it falls back to ("", nil), the same lookup-failure fallback // used elsewhere. -func getUserResource(ctx fiber.Ctx, store IdentityStore) (string, []types.Tag) { +func getUserResource(ctx fiber.Ctx, store iamutil.IdentityStore) (string, []types.Tag) { userName, ok := iamutil.RequestParam(ctx, "UserName") if !ok || userName == "" { identity, _ := httpctx.ContextKeyCallerIdentity.Get(ctx).(types.Identity) @@ -216,7 +255,7 @@ func getUserResource(ctx fiber.Ctx, store IdentityStore) (string, []types.Tag) { // AccessKeyId being queried, so the resource-level check is against the IAM // user that owns that key, matching real IAM's resource-type classification // for this action. -func accessKeyOwnerResource(ctx fiber.Ctx, store IdentityStore) (string, []types.Tag) { +func accessKeyOwnerResource(ctx fiber.Ctx, store iamutil.IdentityStore) (string, []types.Tag) { accessKeyID, ok := iamutil.RequestParam(ctx, "AccessKeyId") if !ok || accessKeyID == "" { return "", nil @@ -235,7 +274,7 @@ func accessKeyOwnerResource(ctx fiber.Ctx, store IdentityStore) (string, []types // "" when the request doesn't actually relocate the user (neither NewPath // nor NewUserName supplied) or when the source user can't be resolved, the // same fallback used elsewhere when a lookup fails. -func updateUserTargetResource(ctx fiber.Ctx, store IdentityStore) string { +func updateUserTargetResource(ctx fiber.Ctx, store iamutil.IdentityStore) string { newPath, _ := iamutil.RequestParam(ctx, "NewPath") newUserName, _ := iamutil.RequestParam(ctx, "NewUserName") if newPath == "" && newUserName == "" { @@ -272,7 +311,7 @@ func newRoleResource(ctx fiber.Ctx) string { return iamutil.BuildRoleArn(iamutil.DefaultAccountID, path, roleName) } -func existingRoleResource(ctx fiber.Ctx, store IdentityStore) (string, []types.Tag) { +func existingRoleResource(ctx fiber.Ctx, store iamutil.IdentityStore) (string, []types.Tag) { roleName, ok := iamutil.RequestParam(ctx, "RoleName") if !ok || roleName == "" { return "*", nil @@ -333,21 +372,7 @@ func requestConditionContext(ctx fiber.Ctx, identity types.Identity, action stri if ip := ctx.IP(); ip != "" { condCtx["aws:SourceIp"] = []string{ip} } - if arn := callerArn(identity); arn != "" { - condCtx["aws:PrincipalArn"] = []string{arn} - condCtx["aws:PrincipalAccount"] = []string{iamutil.DefaultAccountID} - } - switch { - case identity.User != nil: - condCtx["aws:username"] = []string{identity.User.UserName} - condCtx["aws:userid"] = []string{identity.User.UserID} - addPrincipalTagContext(condCtx, identity.User.Tags) - case identity.Session != nil: - condCtx["aws:userid"] = []string{identity.Session.RoleID + ":" + identity.Session.RoleSessionName} - if identity.Role != nil { - addPrincipalTagContext(condCtx, identity.Role.Tags) - } - } + maps.Copy(condCtx, IdentityConditionContext(identity)) for _, tag := range resourceTags { condCtx["iam:ResourceTag/"+tag.Key] = []string{tag.Value} @@ -362,6 +387,59 @@ func requestConditionContext(ctx fiber.Ctx, identity types.Identity, action stri return condCtx } +// IdentityConditionContext builds the condition keys that describe *who* is +// calling — as opposed to the request-derived keys (time, source IP, +// transport) that its callers add around it. +// +// Splitting these out is what lets the standalone-IAM private +// evaluate-policy endpoint serve the S3 gateway: the gateway knows the +// request but not the identity behind the access key, so it sends only the +// request-derived keys and this side fills in the rest from the identity it +// resolved. The gateway is never trusted to supply these keys itself, even +// though it authenticates as root. +func IdentityConditionContext(identity types.Identity) map[string][]string { + condCtx := map[string][]string{} + if arn := CallerArn(identity); arn != "" { + condCtx["aws:PrincipalArn"] = []string{arn} + condCtx["aws:PrincipalAccount"] = []string{iamutil.DefaultAccountID} + } + switch { + case identity.User != nil: + condCtx["aws:PrincipalType"] = []string{"User"} + condCtx["aws:username"] = []string{identity.User.UserName} + condCtx["aws:userid"] = []string{identity.User.UserID} + addPrincipalTagContext(condCtx, identity.User.Tags) + case identity.Session != nil: + condCtx["aws:PrincipalType"] = []string{"AssumedRole"} + condCtx["aws:userid"] = []string{identity.Session.RoleID + ":" + identity.Session.RoleSessionName} + if identity.Role != nil { + addPrincipalTagContext(condCtx, identity.Role.Tags) + } + } + return condCtx +} + +// IdentityConditionKeyPrefixes lists the condition-key namespaces that +// describe the caller or the resource, and that therefore only the IAM +// service may populate. handleEvaluatePolicy strips every one of them from +// a gateway-supplied context before overlaying its own — an +// override-on-collision merge would leave any key the service happens *not* +// to set (aws:PrincipalTag/x for an untagged role, say) under the +// gateway's control, which is exactly what a StringNotEquals-guarded Allow +// keys off. +var IdentityConditionKeyPrefixes = []string{ + "aws:PrincipalArn", + "aws:PrincipalAccount", + "aws:PrincipalType", + "aws:username", + "aws:userid", + "aws:PrincipalTag/", + "aws:ResourceTag/", + "iam:ResourceTag/", + "aws:RequestTag/", + "aws:TagKeys", +} + // addPrincipalTagContext populates aws:PrincipalTag/ from tags, the // calling principal's own tags. func addPrincipalTagContext(condCtx map[string][]string, tags []types.Tag) { @@ -390,9 +468,9 @@ func addRequestTagContext(condCtx map[string][]string, ctx fiber.Ctx) { condCtx["aws:TagKeys"] = keys } -// callerArn identifies identity the way real IAM error messages do: the +// CallerArn identifies identity the way real IAM error messages do: the // user's own Arn, or the assumed-role session Arn. -func callerArn(identity types.Identity) string { +func CallerArn(identity types.Identity) string { if identity.Session != nil { return iamutil.BuildAssumedRoleArn(iamutil.DefaultAccountID, identity.Session.RoleName, identity.Session.RoleSessionName) } diff --git a/iamapi/internal/iamutil/access_key.go b/iamapi/internal/iamutil/access_key.go index a60320db..9458a837 100644 --- a/iamapi/internal/iamutil/access_key.go +++ b/iamapi/internal/iamutil/access_key.go @@ -18,10 +18,10 @@ import ( "crypto/rand" "encoding/base64" "regexp" - "strings" "github.com/versity/versitygw/debuglogger" "github.com/versity/versitygw/iamapi/iamerr" + "github.com/versity/versitygw/internal/sigv4auth" ) const ( @@ -34,11 +34,7 @@ const ( maxAccessKeyIDLen = 128 secretAccessKeyBytes = 30 - // tempAccessKeyIDPrefix marks temporary credentials minted by - // AssumeRoleWithWebIdentity, matching AWS's ASIA… convention that - // distinguishes them from long-term AKIA… access keys. - tempAccessKeyIDPrefix = "ASIA" - sessionTokenBytes = 128 + sessionTokenBytes = 128 ) var accessKeyIDPattern = regexp.MustCompile(`^[\w]+$`) @@ -69,7 +65,7 @@ func GenerateSecretAccessKey() (string, error) { // access key id in the ASIA… format, for credentials minted by // AssumeRoleWithWebIdentity. func GenerateTempAccessKeyID() (string, error) { - id, err := generateAWSID(tempAccessKeyIDPrefix, accessKeyIDRandomLen) + id, err := generateAWSID(sigv4auth.TempAccessKeyIDPrefix, accessKeyIDRandomLen) if err != nil { debuglogger.Logf("failed to generate temporary IAM access key id: %v", err) return "", err @@ -93,9 +89,10 @@ func GenerateSessionToken() (string, error) { // IsTempAccessKeyID reports whether accessKeyID has the ASIA… prefix used // for temporary credentials minted by AssumeRoleWithWebIdentity, as opposed -// to a long-term AKIA… access key. +// to a long-term AKIA… access key. It delegates to sigv4auth so the S3 +// gateway, which cannot import this package, shares one definition. func IsTempAccessKeyID(accessKeyID string) bool { - return strings.HasPrefix(accessKeyID, tempAccessKeyIDPrefix) + return sigv4auth.IsTempAccessKeyID(accessKeyID) } // ValidateAccessKeyID checks that accessKeyID fits within the allowed length diff --git a/iamapi/internal/iamutil/identity.go b/iamapi/internal/iamutil/identity.go new file mode 100644 index 00000000..b5ac8275 --- /dev/null +++ b/iamapi/internal/iamutil/identity.go @@ -0,0 +1,145 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iamutil + +import ( + "context" + "errors" + "time" + + "github.com/versity/versitygw/iamapi/types" + "github.com/versity/versitygw/internal/sigv4auth" +) + +// ErrIdentityNotFound and ErrInvalidSessionToken are returned by +// ResolveSessionByToken and ResolveUserIdentity, so a caller that needs to +// know *which* failure occurred — the standalone-IAM private endpoints, and +// iammiddleware's own sigv4 pipeline, which turn them into S3's own +// InvalidAccessKeyId and InvalidToken respectively — can distinguish them. +// +// The public IAM control plane deliberately collapses both into a single +// InvalidClientTokenId: an unauthenticated caller must not learn whether an +// access key exists. +var ( + ErrIdentityNotFound = errors.New("identity not found") + ErrInvalidSessionToken = errors.New("invalid session token") +) + +// IdentityStore resolves an access key id to the session or long-term user +// that owns it, and resolves named resources for policy evaluation. +// storage.Storer satisfies this directly. +type IdentityStore interface { + GetSession(ctx context.Context, accessKeyID string) (*types.Session, error) + GetRole(ctx context.Context, roleName string) (*types.Role, error) + GetUserByAccessKeyID(ctx context.Context, accessKeyID string) (*types.User, error) + GetUser(ctx context.Context, username string) (*types.User, error) + GetOIDCProvider(ctx context.Context, arn string) (*types.OIDCProvider, error) + RecordAccessKeyUsage(ctx context.Context, accessKeyID, service, region string, when time.Time) error +} + +// ResolveSessionByToken resolves a temporary (ASIA…) access key to the +// session that owns it, requiring token to match the session's stored +// SessionToken. The empty-token rejection and the constant-time comparison +// both live here rather than in any caller: this is the only function that +// may turn a session access key id into a secret, so no caller can be +// written that skips them. +// +// It does not itself verify a SigV4 signature — request-pipeline callers do +// that next, so a stolen or guessed access key id plus token is never +// sufficient on its own. +func ResolveSessionByToken(ctx context.Context, store IdentityStore, accessKeyID, token string) (*types.Identity, string, error) { + // No token means there is nothing to resolve the access key against, so + // the key is reported as simply not existing rather than as a bad token + // — matching real S3, which answers InvalidAccessKeyId for a temporary + // access key presented with no X-Amz-Security-Token, and InvalidToken + // only once a token is actually present and wrong. + if token == "" { + return nil, "", ErrIdentityNotFound + } + + session, err := store.GetSession(ctx, accessKeyID) + if err != nil { + return nil, "", ErrIdentityNotFound + } + + if !sigv4auth.SecureCompare(token, session.SessionToken) { + return nil, "", ErrInvalidSessionToken + } + + // A signature-valid, unexpired session still authenticates even if its + // role has since been deleted — real STS credentials are self-contained + // and don't re-check role existence on every call. What such a session + // can no longer do is get any IAM action past the policy middleware: + // with Role/IdentityPolicies left unset, EvaluateIdentityPolicies denies + // by default, same effective outcome as an explicit rejection here would + // have had for every pipeline except GetCallerIdentity, which needs + // none of this and must keep working regardless. + // + // The reloaded role must also still be the *same* role the session was + // originally minted against — RoleID and Arn, both captured in the + // session at AssumeRoleWithWebIdentity time, must match the freshly + // loaded role's own values. Without this check, deleting a role and + // recreating one of the same name (necessarily getting a new RoleID) + // would let every pre-existing session for the old role silently + // inherit whatever policies the new role happens to carry. + identity := &types.Identity{ + Session: session, + SessionPolicy: session.Policy, + } + if role, err := store.GetRole(ctx, session.RoleName); err == nil && + role.RoleID == session.RoleID && role.Arn == session.RoleArn { + identity.Role = role + identity.IdentityPolicies = role.Policies.Inline + } + return identity, session.SecretAccessKey, nil +} + +// ResolveUserIdentity resolves accessKeyID to its long-term (AKIA…) IAM user +// and secret, reporting the ErrIdentityNotFound/ErrInvalidSessionToken +// sentinels rather than an opaque API error — callers on the public control +// plane that want the opaque error do that translation themselves. +// +// Temporary (ASIA…) session access keys are rejected here: resolving one +// safely requires validating its security token, which this function has no +// parameter for. A caller that can supply a token uses ResolveSessionByToken +// instead. Silently resolving a session's secret from its access key id +// alone, with no token check at all, would let anyone who merely knows the +// id impersonate the session. +func ResolveUserIdentity(ctx context.Context, store IdentityStore, accessKeyID string) (*types.Identity, string, error) { + if sigv4auth.IsTempAccessKeyID(accessKeyID) { + return nil, "", ErrInvalidSessionToken + } + + user, err := store.GetUserByAccessKeyID(ctx, accessKeyID) + if err != nil { + return nil, "", ErrIdentityNotFound + } + + var keyEntry *types.AccessKeyEntry + for i := range user.AccessKeys { + if user.AccessKeys[i].AccessKeyId == accessKeyID { + keyEntry = &user.AccessKeys[i] + break + } + } + if keyEntry == nil || keyEntry.Status != AccessKeyStatusActive { + return nil, "", ErrIdentityNotFound + } + + identity := &types.Identity{ + User: user, + IdentityPolicies: user.Policies.Inline, + } + return identity, keyEntry.SecretAccessKey, nil +} diff --git a/iamapi/internal/iamutil/webidentity.go b/iamapi/internal/iamutil/webidentity.go index 9be96f66..61782ac1 100644 --- a/iamapi/internal/iamutil/webidentity.go +++ b/iamapi/internal/iamutil/webidentity.go @@ -382,11 +382,11 @@ func VerifyWebIdentityExpiration(claims jwt.MapClaims, now time.Time) error { // web identity token claims beyond exp (already checked separately by // VerifyWebIdentityExpiration): iat and sub must both be present, and nbf // (if present) must not be in the future beyond webIdentityExpLeeway of -// clock skew. Confirmed against real AWS (niksis02 profile): a token with -// exp but no iat, or with iat but no sub, is rejected with -// InvalidIdentityToken "Missing a required claim: ." — without -// this check, such a token would otherwise obtain credentials whenever the -// role's trust policy doesn't itself require sub via Condition. +// clock skew. A token with exp but no iat, or with iat but no sub, is +// rejected with InvalidIdentityToken "Missing a required claim: +// ." — without this check, such a token would otherwise obtain +// credentials whenever the role's trust policy doesn't itself require sub +// via Condition. func VerifyWebIdentityRequiredClaims(claims jwt.MapClaims, now time.Time) error { if _, ok := claims["iat"].(float64); !ok { debuglogger.Logf("web identity token has no iat claim") @@ -580,8 +580,8 @@ type jwksCacheEntry struct { keys *jwkSet expiresAt time.Time // lastForcedRefresh is when an unknown-kid lookup last bypassed - // expiresAt to force a fetch for this issuer, gating - // jwksMinForcedRefreshInterval (see forceRefreshJWKSCache). + // expiresAt to force a fetch for this issuer, gated by + // jwksMinForcedRefreshInterval. lastForcedRefresh time.Time } diff --git a/iamapi/policy/identity.go b/iamapi/policy/identity.go index 4ee29dab..a4696fae 100644 --- a/iamapi/policy/identity.go +++ b/iamapi/policy/identity.go @@ -19,6 +19,7 @@ import ( "github.com/versity/versitygw/debuglogger" "github.com/versity/versitygw/iamapi/types" + "github.com/versity/versitygw/internal/condition" ) // MaxSessionPolicyBytes is the maximum length, in bytes, of the optional @@ -45,31 +46,53 @@ type RequestContext struct { Condition map[string][]string } -// EvaluateIdentityPolicies reports whether reqCtx is allowed by documents -// (each a user's or role's inline policy entry), using IAM's evaluation -// semantics: a statement must cover the action, the resource, and (if -// present) its Condition block to be considered at all; an explicit Deny -// statement that does so makes the whole evaluation deny regardless of any -// Allow found elsewhere (in the same or another document), and absent an -// explicit deny, at least one covering Allow statement is required — so an -// identity with no matching statement at all is denied by default. +// Decision is the tri-state result of evaluating a set of identity policy +// documents. A caller combining this with another policy source (e.g. an S3 +// bucket policy) needs this distinction, not a plain bool, to implement +// AWS's real cross-policy precedence: an explicit Deny from either source +// wins outright over an Allow from the other, but a NoMatch from one source +// leaves the other free to grant access on its own. +type Decision int + +const ( + // DecisionNoMatch means no statement in any document matched reqCtx at + // all — neither an Allow nor a Deny. + DecisionNoMatch Decision = iota + // DecisionAllow means at least one statement matched with Effect Allow, + // and no statement matched with Effect Deny. + DecisionAllow + // DecisionDeny means a statement matched with Effect Deny, or the + // evaluation failed closed (unparseable/invalid document, or a + // Condition that couldn't be evaluated). + DecisionDeny +) + +// EvaluateIdentityPolicies reports how documents (each a user's or role's +// inline policy entry) decide reqCtx, using IAM's evaluation semantics: a +// statement must cover the action, the resource, and (if present) its +// Condition block to be considered at all; a matching explicit Deny +// statement makes the whole evaluation DecisionDeny regardless of any Allow +// found elsewhere (in the same or another document); absent an explicit +// deny, at least one covering Allow statement is required for DecisionAllow +// — an identity with no matching statement at all gets DecisionNoMatch, not +// DecisionAllow. // -// A document that fails to parse, or a statement whose Condition block can't -// be evaluated (see evaluateCondition's ok return), denies the whole -// evaluation rather than being skipped: PutUserPolicy/PutRolePolicy already -// reject any policy document that wouldn't parse or whose Condition uses an +// A document that fails to parse, or a statement whose Condition block +// can't be evaluated, returns DecisionDeny +// rather than being skipped: PutUserPolicy/PutRolePolicy already reject any +// policy document that wouldn't parse or whose Condition uses an // unrecognized operator, so this only matters for documents written before // that validation existed - and for exactly that legacy-data case, we can't // rule out a hidden Deny inside the part we can't evaluate, so the safe // outcome is to deny rather than silently proceed as if it wasn't there. -func EvaluateIdentityPolicies(documents []types.PolicyEntry, reqCtx RequestContext) bool { +func EvaluateIdentityPolicies(documents []types.PolicyEntry, reqCtx RequestContext) Decision { allowed := false for _, entry := range documents { var doc Document if err := json.Unmarshal([]byte(entry.PolicyDocument), &doc); err != nil { debuglogger.Logf("identity policy document failed to parse: %v", err) - return false + return DecisionDeny } // PutUserPolicy/PutRolePolicy already reject a document that // wouldn't pass Validate (e.g. both Action and NotAction on one @@ -81,7 +104,7 @@ func EvaluateIdentityPolicies(documents []types.PolicyEntry, reqCtx RequestConte // not just at ingress. if err := doc.Validate(); err != nil { debuglogger.Logf("identity policy document failed validation: %v", err) - return false + return DecisionDeny } for _, stmt := range doc.Statement { @@ -94,10 +117,10 @@ func EvaluateIdentityPolicies(documents []types.PolicyEntry, reqCtx RequestConte if !statementCoversResource(stmt, reqCtx.Resource, reqCtx.Condition, doc.Version) { continue } - matched, ok := evaluateCondition(stmt.Condition, reqCtx.Condition, doc.Version) + matched, ok := condition.Evaluate(stmt.Condition, reqCtx.Condition, doc.Version) if !ok { debuglogger.Logf("identity policy evaluation: statement condition could not be evaluated, denying") - return false + return DecisionDeny } if !matched { continue @@ -105,13 +128,16 @@ func EvaluateIdentityPolicies(documents []types.PolicyEntry, reqCtx RequestConte if stmt.Effect == "Deny" { debuglogger.Logf("identity policy evaluation: action %q on resource %q explicitly denied", reqCtx.Action, reqCtx.Resource) - return false + return DecisionDeny } allowed = true } } - return allowed + if allowed { + return DecisionAllow + } + return DecisionNoMatch } // statementCoversResource reports whether stmt's Resource/NotResource @@ -140,9 +166,9 @@ func matchAnyResource(patterns []string, resource string, ctxVars map[string][]s for _, p := range patterns { pattern := p if version == Version2012 { - pattern = substitutePolicyVariables(p, ctxVars) + pattern = condition.SubstitutePolicyVariables(p, ctxVars) } - if globMatch(pattern, resource) { + if condition.GlobMatch(pattern, resource) { return true } } diff --git a/iamapi/policy/identity_test.go b/iamapi/policy/identity_test.go index c6b1ff58..0580d722 100644 --- a/iamapi/policy/identity_test.go +++ b/iamapi/policy/identity_test.go @@ -33,31 +33,31 @@ func TestEvaluateIdentityPolicies(t *testing.T) { name string documents []types.PolicyEntry reqCtx RequestContext - want bool + want Decision }{ { name: "no documents denies by default", documents: nil, reqCtx: RequestContext{Action: "iam:CreateUser", Resource: "*"}, - want: false, + want: DecisionNoMatch, }, { name: "no matching statement denies by default", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:GetUser","Resource":"*"}]}`), reqCtx: RequestContext{Action: "iam:CreateUser", Resource: "*"}, - want: false, + want: DecisionNoMatch, }, { name: "matching allow statement allows", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:CreateUser","Resource":"*"}]}`), reqCtx: RequestContext{Action: "iam:CreateUser", Resource: "*"}, - want: true, + want: DecisionAllow, }, { name: "wildcard action allows", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:*","Resource":"*"}]}`), reqCtx: RequestContext{Action: "iam:CreateUser", Resource: "*"}, - want: true, + want: DecisionAllow, }, { name: "explicit deny overrides an allow in another document", @@ -66,19 +66,19 @@ func TestEvaluateIdentityPolicies(t *testing.T) { `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"iam:CreateUser","Resource":"*"}]}`, ), reqCtx: RequestContext{Action: "iam:CreateUser", Resource: "*"}, - want: false, + want: DecisionDeny, }, { name: "explicit deny overrides an allow in the same document", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:*","Resource":"*"},{"Effect":"Deny","Action":"iam:CreateUser","Resource":"*"}]}`), reqCtx: RequestContext{Action: "iam:CreateUser", Resource: "*"}, - want: false, + want: DecisionDeny, }, { name: "action match is case-insensitive", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"IAM:CREATEUSER","Resource":"*"}]}`), reqCtx: RequestContext{Action: "iam:CreateUser", Resource: "*"}, - want: true, + want: DecisionAllow, }, { // A malformed document might have contained a Deny we can no @@ -87,13 +87,13 @@ func TestEvaluateIdentityPolicies(t *testing.T) { name: "malformed document denies the whole evaluation, even with a valid Allow elsewhere", documents: policyEntries(`not json`, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:CreateUser","Resource":"*"}]}`), reqCtx: RequestContext{Action: "iam:CreateUser", Resource: "*"}, - want: false, + want: DecisionDeny, }, { name: "malformed document denies the whole evaluation regardless of document order", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:CreateUser","Resource":"*"}]}`, `not json`), reqCtx: RequestContext{Action: "iam:CreateUser", Resource: "*"}, - want: false, + want: DecisionDeny, }, { // A Deny guarded by a Condition operator this package doesn't @@ -106,7 +106,7 @@ func TestEvaluateIdentityPolicies(t *testing.T) { `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:CreateUser","Resource":"*"},{"Effect":"Deny","Action":"iam:CreateUser","Resource":"*","Condition":{"FooBarOperator":{"aws:username":"alice"}}}]}`, ), reqCtx: RequestContext{Action: "iam:CreateUser", Resource: "*", Condition: map[string][]string{"aws:username": {"alice"}}}, - want: false, + want: DecisionDeny, }, { // Fail-closed on a condition-evaluation error isn't scoped to @@ -115,7 +115,7 @@ func TestEvaluateIdentityPolicies(t *testing.T) { name: "unrecognized operator on an Allow-only statement still denies (fail-closed is not Deny-specific)", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:CreateUser","Resource":"*","Condition":{"FooBarOperator":{"aws:username":"alice"}}}]}`), reqCtx: RequestContext{Action: "iam:CreateUser", Resource: "*", Condition: map[string][]string{"aws:username": {"alice"}}}, - want: false, + want: DecisionDeny, }, { // A document containing any statement Validate() would @@ -128,73 +128,73 @@ func TestEvaluateIdentityPolicies(t *testing.T) { name: "unrecognized operator in an unrelated statement invalidates the whole document", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:CreateUser","Resource":"*"},{"Effect":"Deny","Action":"iam:DeleteUser","Resource":"*","Condition":{"FooBarOperator":{"aws:username":"alice"}}}]}`), reqCtx: RequestContext{Action: "iam:CreateUser", Resource: "*"}, - want: false, + want: DecisionDeny, }, { name: "Null operator end-to-end: denies presence of aws:username", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:CreateUser","Resource":"*"},{"Effect":"Deny","Action":"iam:CreateUser","Resource":"*","Condition":{"Null":{"aws:username":"false"}}}]}`), reqCtx: RequestContext{Action: "iam:CreateUser", Resource: "*", Condition: map[string][]string{"aws:username": {"alice"}}}, - want: false, + want: DecisionDeny, }, { name: "Null operator end-to-end: allows when aws:username is absent (session, not user)", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:CreateUser","Resource":"*"},{"Effect":"Deny","Action":"iam:CreateUser","Resource":"*","Condition":{"Null":{"aws:username":"false"}}}]}`), reqCtx: RequestContext{Action: "iam:CreateUser", Resource: "*", Condition: map[string][]string{"aws:userid": {"role-id:session"}}}, - want: true, + want: DecisionAllow, }, { name: "NotAction denies coverage for the excluded action", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","NotAction":"iam:CreateUser","Resource":"*"}]}`), reqCtx: RequestContext{Action: "iam:CreateUser", Resource: "*"}, - want: false, + want: DecisionNoMatch, }, { name: "NotAction allows actions outside the exclusion", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","NotAction":"iam:CreateUser","Resource":"*"}]}`), reqCtx: RequestContext{Action: "iam:DeleteUser", Resource: "*"}, - want: true, + want: DecisionAllow, }, { name: "resource-scoped allow matches the named resource", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:GetRole","Resource":"arn:aws:iam::000000000000:role/role-a"}]}`), reqCtx: RequestContext{Action: "iam:GetRole", Resource: "arn:aws:iam::000000000000:role/role-a"}, - want: true, + want: DecisionAllow, }, { name: "resource-scoped allow does not cover a different resource", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:GetRole","Resource":"arn:aws:iam::000000000000:role/role-a"}]}`), reqCtx: RequestContext{Action: "iam:GetRole", Resource: "arn:aws:iam::000000000000:role/role-b"}, - want: false, + want: DecisionNoMatch, }, { name: "resource match is case-sensitive", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:GetRole","Resource":"arn:aws:iam::000000000000:role/Role-A"}]}`), reqCtx: RequestContext{Action: "iam:GetRole", Resource: "arn:aws:iam::000000000000:role/role-a"}, - want: false, + want: DecisionNoMatch, }, { name: "resource-scoped deny only affects the named resource", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:GetRole","Resource":"*"},{"Effect":"Deny","Action":"iam:GetRole","Resource":"arn:aws:iam::000000000000:role/role-a"}]}`), reqCtx: RequestContext{Action: "iam:GetRole", Resource: "arn:aws:iam::000000000000:role/role-b"}, - want: true, + want: DecisionAllow, }, { name: "resource-scoped deny denies the named resource", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:GetRole","Resource":"*"},{"Effect":"Deny","Action":"iam:GetRole","Resource":"arn:aws:iam::000000000000:role/role-a"}]}`), reqCtx: RequestContext{Action: "iam:GetRole", Resource: "arn:aws:iam::000000000000:role/role-a"}, - want: false, + want: DecisionDeny, }, { name: "NotResource excludes the named resource", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:GetRole","NotResource":"arn:aws:iam::000000000000:role/role-a"}]}`), reqCtx: RequestContext{Action: "iam:GetRole", Resource: "arn:aws:iam::000000000000:role/role-a"}, - want: false, + want: DecisionNoMatch, }, { name: "NotResource allows resources outside the exclusion", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:GetRole","NotResource":"arn:aws:iam::000000000000:role/role-a"}]}`), reqCtx: RequestContext{Action: "iam:GetRole", Resource: "arn:aws:iam::000000000000:role/role-b"}, - want: true, + want: DecisionAllow, }, { // ${aws:username} in Resource must resolve to the requesting @@ -203,19 +203,19 @@ func TestEvaluateIdentityPolicies(t *testing.T) { name: "policy variable in Resource matches the caller's own resource", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:GetUser","Resource":"arn:aws:iam::000000000000:user/${aws:username}"}]}`), reqCtx: RequestContext{Action: "iam:GetUser", Resource: "arn:aws:iam::000000000000:user/alice", Condition: map[string][]string{"aws:username": {"alice"}}}, - want: true, + want: DecisionAllow, }, { name: "policy variable in Resource does not match a different principal's resource", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:GetUser","Resource":"arn:aws:iam::000000000000:user/${aws:username}"}]}`), reqCtx: RequestContext{Action: "iam:GetUser", Resource: "arn:aws:iam::000000000000:user/bob", Condition: map[string][]string{"aws:username": {"alice"}}}, - want: false, + want: DecisionNoMatch, }, { name: "unresolvable policy variable in Resource is left literal and so does not match a real ARN", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:GetUser","Resource":"arn:aws:iam::000000000000:user/${aws:username}"}]}`), reqCtx: RequestContext{Action: "iam:GetUser", Resource: "arn:aws:iam::000000000000:user/alice"}, - want: false, + want: DecisionNoMatch, }, { // AWS requires Version 2012-10-17 to use policy variables at @@ -224,37 +224,37 @@ func TestEvaluateIdentityPolicies(t *testing.T) { name: "policy variable in Resource is not substituted under version 2008-10-17", documents: policyEntries(`{"Version":"2008-10-17","Statement":[{"Effect":"Allow","Action":"iam:GetUser","Resource":"arn:aws:iam::000000000000:user/${aws:username}"}]}`), reqCtx: RequestContext{Action: "iam:GetUser", Resource: "arn:aws:iam::000000000000:user/alice", Condition: map[string][]string{"aws:username": {"alice"}}}, - want: false, + want: DecisionNoMatch, }, { name: "policy variable in Resource is not substituted with no Version at all", documents: policyEntries(`{"Statement":[{"Effect":"Allow","Action":"iam:GetUser","Resource":"arn:aws:iam::000000000000:user/${aws:username}"}]}`), reqCtx: RequestContext{Action: "iam:GetUser", Resource: "arn:aws:iam::000000000000:user/alice", Condition: map[string][]string{"aws:username": {"alice"}}}, - want: false, + want: DecisionNoMatch, }, { name: "condition must match", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:GetRole","Resource":"*","Condition":{"StringEquals":{"aws:username":"alice"}}}]}`), reqCtx: RequestContext{Action: "iam:GetRole", Resource: "*", Condition: map[string][]string{"aws:username": {"alice"}}}, - want: true, + want: DecisionAllow, }, { name: "condition mismatch denies by default", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:GetRole","Resource":"*","Condition":{"StringEquals":{"aws:username":"alice"}}}]}`), reqCtx: RequestContext{Action: "iam:GetRole", Resource: "*", Condition: map[string][]string{"aws:username": {"bob"}}}, - want: false, + want: DecisionNoMatch, }, { name: "deny condition must also match to take effect", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:GetRole","Resource":"*"},{"Effect":"Deny","Action":"iam:GetRole","Resource":"*","Condition":{"IpAddress":{"aws:SourceIp":"10.0.0.0/8"}}}]}`), reqCtx: RequestContext{Action: "iam:GetRole", Resource: "*", Condition: map[string][]string{"aws:SourceIp": {"203.0.113.5"}}}, - want: true, + want: DecisionAllow, }, { name: "deny condition matching denies", documents: policyEntries(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:GetRole","Resource":"*"},{"Effect":"Deny","Action":"iam:GetRole","Resource":"*","Condition":{"IpAddress":{"aws:SourceIp":"10.0.0.0/8"}}}]}`), reqCtx: RequestContext{Action: "iam:GetRole", Resource: "*", Condition: map[string][]string{"aws:SourceIp": {"10.1.2.3"}}}, - want: false, + want: DecisionDeny, }, } diff --git a/iamapi/policy/trust.go b/iamapi/policy/trust.go index 9c09a737..7ce1832b 100644 --- a/iamapi/policy/trust.go +++ b/iamapi/policy/trust.go @@ -20,12 +20,13 @@ import ( "strings" "github.com/versity/versitygw/iamapi/iamerr" + "github.com/versity/versitygw/internal/condition" ) // trustPrincipalKeys are the only keys IAM accepts inside a trust policy // statement's Principal object. CanonicalUser is deliberately not accepted -// here (see errTrustInvalidPrincipalKey) since it identifies an S3 canonical -// user id which is the legacy s3 user identifier and is not planned to support +// since it identifies an S3 canonical user id, the legacy S3 user +// identifier, which is not planned to be supported here. var trustPrincipalKeys = map[string]bool{ "AWS": true, "Service": true, @@ -175,7 +176,7 @@ func (d Document) ValidateTrust() error { // valid Effect, a required Principal (never NotPrincipal), an Action or // NotAction with only "sts:"-prefixed values, no Resource/NotResource, and - // if present - a Condition block whose operators are all recognized (see -// conditionShapeValid, shared with the identity-policy side; condition +// condition.ShapeValid, shared with the identity-policy side; condition // *keys* and operand *values* are deliberately not validated here, matching // AWS behavior). func (s Statement) ValidateTrust() error { @@ -197,7 +198,7 @@ func (s Statement) ValidateTrust() error { return err } - if !conditionShapeValid(s.Condition) { + if !condition.ShapeValid(s.Condition) { return errTrustSyntax } @@ -324,9 +325,9 @@ func validateSharedProviderTenancy(s Statement, federated []string) error { } // oidcProviderURLFromFederatedArn extracts the provider Url from a Federated -// principal ARN shaped like "arn:aws:iam:::oidc-provider/" -// (see iamutil.BuildOIDCProviderArn), reporting ok=false for any value not -// shaped like an OIDC provider ARN at all — a bare federation identifier +// principal ARN shaped like "arn:aws:iam:::oidc-provider/", +// reporting ok=false for any value not shaped like an OIDC provider ARN at +// all — a bare federation identifier // (e.g. "cognito-identity.amazonaws.com") or a malformed value, both handled // elsewhere (this is deliberately a lightweight shape check, not full ARN // validation: an actually-malformed ARN is caught later, when the runtime @@ -356,23 +357,23 @@ func oidcProviderURLFromFederatedArn(value string) (string, bool) { // and StringEqualsIgnoreCase don't treat '*'/'?' as wildcards at all, so // only the plain "empty or exactly '*'" check applies to them. A block that // fails to parse reports false, same as an absent one — -// conditionShapeValid/evaluateCondition are responsible for rejecting or +// condition.ShapeValid/condition.Evaluate are responsible for rejecting or // fail-closing a block this can't understand; this check only ever adds a // stricter write-time requirement on top of that. func conditionScopesClaim(raw json.RawMessage, key string) bool { if len(raw) == 0 { return false } - var block map[string]map[string]ConditionValues - if err := json.Unmarshal(raw, &block); err != nil { + block, err := condition.Parse(raw) + if err != nil { return false } for operator, kvs := range block { - op, ok := parseOperatorName(operator) + op, ok := condition.ParseOperatorName(operator) if !ok { continue } - switch op.base { + switch op.Base { case "StringEquals", "StringLike", "StringEqualsIgnoreCase": default: continue @@ -385,7 +386,7 @@ func conditionScopesClaim(raw json.RawMessage, key string) bool { if v == "" || v == "*" { continue } - if op.base == "StringLike" && !hasNonWildcardCharacter(v) { + if op.Base == "StringLike" && !hasNonWildcardCharacter(v) { continue } return true diff --git a/iamapi/policy/validate.go b/iamapi/policy/validate.go index 3987897b..d4a8c74f 100644 --- a/iamapi/policy/validate.go +++ b/iamapi/policy/validate.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/versity/versitygw/iamapi/iamerr" + "github.com/versity/versitygw/internal/condition" ) // MaxDocumentLength is IAM's parameter-level maximum length for a @@ -122,8 +123,8 @@ func (d Document) Validate() error { // no Principal/NotPrincipal, an Action or NotAction (not both) with // vendor-prefixed values, a Resource or NotResource (not both) with // ARN-shaped values, and - if present - a Condition block whose operators -// are all recognized (see conditionShapeValid; condition *keys* and operand -// *values* are deliberately not validated here, matching AWS behavior). +// are all recognized (condition *keys* and operand *values* are +// deliberately not validated here, matching AWS behavior). func (s Statement) Validate() error { switch s.Effect { case "Allow", "Deny": @@ -135,7 +136,7 @@ func (s Statement) Validate() error { return errPrincipalNotAllowed } - if !conditionShapeValid(s.Condition) { + if !condition.ShapeValid(s.Condition) { return errSyntax } diff --git a/iamapi/policy/webidentity.go b/iamapi/policy/webidentity.go index 954acb93..23afd157 100644 --- a/iamapi/policy/webidentity.go +++ b/iamapi/policy/webidentity.go @@ -20,6 +20,7 @@ import ( "time" "github.com/versity/versitygw/debuglogger" + "github.com/versity/versitygw/internal/condition" ) // AssumeRoleWithWebIdentityAction is the sts action name role trust @@ -182,7 +183,7 @@ func EvaluateWebIdentityTrust(document string, lookup ProviderLookup, wctx WebId } anyIssuerMatch = true - matched, condOk := evaluateCondition(stmt.Condition, ctxVars, doc.Version) + matched, condOk := condition.Evaluate(stmt.Condition, ctxVars, doc.Version) if !condOk { debuglogger.Logf("web identity trust evaluation: statement condition could not be evaluated, denying") denied = true @@ -260,7 +261,7 @@ func matchAny(patterns []string, action string) bool { // IAM-style glob ('*' any run of characters, '?' any single character) — // e.g. "sts:*" or "sts:AssumeRole*" both match "sts:AssumeRoleWithWebIdentity". func matchActionPattern(pattern, action string) bool { - return globMatch(toLowerASCII(pattern), toLowerASCII(action)) + return condition.GlobMatch(toLowerASCII(pattern), toLowerASCII(action)) } func toLowerASCII(s string) string { @@ -272,32 +273,3 @@ func toLowerASCII(s string) string { } return string(b) } - -// globMatch implements the small wildcard grammar IAM Action/Resource -// patterns use: '*' matches any run of characters (including none), '?' -// matches exactly one character, everything else matches literally. -func globMatch(pattern, s string) bool { - var pi, si, star, match int - star = -1 - for si < len(s) { - switch { - case pi < len(pattern) && (pattern[pi] == '?' || pattern[pi] == s[si]): - pi++ - si++ - case pi < len(pattern) && pattern[pi] == '*': - star = pi - match = si - pi++ - case star != -1: - pi = star + 1 - match++ - si = match - default: - return false - } - } - for pi < len(pattern) && pattern[pi] == '*' { - pi++ - } - return pi == len(pattern) -} diff --git a/iamapi/policy/webidentity_test.go b/iamapi/policy/webidentity_test.go index 46aa2290..f2c96493 100644 --- a/iamapi/policy/webidentity_test.go +++ b/iamapi/policy/webidentity_test.go @@ -161,7 +161,7 @@ func TestEvaluateWebIdentityTrust(t *testing.T) { // RequestContext.Condition on the identity-policy side - this // is the most realistic place to exercise the multivalue // aggregation semantics documented on aggregate() in - // condition.go. "banned" is present among the claim's values, + // internal/condition. "banned" is present among the claim's values, // so unqualified StringNotEquals (pre-existing, unchanged // semantics: fails to match if any actual value matches) fails // to match, and the Allow's condition doesn't hold. diff --git a/iamapi/private/errors.go b/iamapi/private/errors.go new file mode 100644 index 00000000..1cceb2c8 --- /dev/null +++ b/iamapi/private/errors.go @@ -0,0 +1,119 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package private + +import ( + "errors" + "net/http" + + "github.com/gofiber/fiber/v3" + "github.com/versity/versitygw/debuglogger" + "github.com/versity/versitygw/iamapi/internal/iamutil" +) + +// Error codes carried in the JSON error body's "code" field. The S3 +// gateway maps them to distinct S3 errors — CodeNoSuchIdentity to +// InvalidAccessKeyId, CodeInvalidToken to InvalidToken — so an end user +// gets an accurate diagnosis instead of one catch-all. Without them every +// 403 looks identical on the wire, and a gateway whose own IAM-client +// credential was rotated would tell the *user* their access key doesn't +// exist. +const ( + CodeNoSuchIdentity = "NoSuchIdentity" + CodeInvalidToken = "InvalidToken" + CodeBadRequest = "BadRequest" +) + +// privateAPIError is a minimal local error for failures (like a malformed +// request body) that don't map to any of iamerr's AWS-IAM-specific error +// codes — this protocol is plain JSON, not the rest of iamapi's +// AWS-Query/XML wire format, so there's no need to force every error +// through iamerr.APIError's XML-rendering machinery. +type privateAPIError struct { + status int + code string + message string +} + +func (e *privateAPIError) Error() string { return e.message } +func (e *privateAPIError) StatusCode() int { return e.status } +func (e *privateAPIError) Code() string { return e.code } + +var ( + errMalformedRequestBody = &privateAPIError{ + status: http.StatusBadRequest, + code: CodeBadRequest, + message: "malformed request body", + } + errNoSuchIdentity = &privateAPIError{ + status: http.StatusForbidden, + code: CodeNoSuchIdentity, + message: "no identity for the given access key id", + } + errInvalidSessionToken = &privateAPIError{ + status: http.StatusForbidden, + code: CodeInvalidToken, + message: "the given session token is missing, invalid, or does not belong to the given access key id", + } +) + +// mapResolveError translates iamutil's identity-resolution sentinels into +// the wire errors this protocol reports. Anything unrecognized falls through +// unchanged and renders as a 500, which is the correct signal: it is a fault +// in the IAM service, not a problem with the caller's identity. +func mapResolveError(err error) error { + switch { + case errors.Is(err, iamutil.ErrIdentityNotFound): + return errNoSuchIdentity + case errors.Is(err, iamutil.ErrInvalidSessionToken): + return errInvalidSessionToken + default: + return err + } +} + +// coder is implemented by errors carrying a stable machine-readable code +// for the "code" field of the JSON error body. +type coder interface { + Code() string +} + +// statusCoder is satisfied by both iamerr.APIError (used by +// iammiddleware.VerifyRootOnlySigV4) and privateAPIError, so errorHandler +// can extract the right HTTP status from either without depending on +// iamerr's XML-specific interface methods. +type statusCoder interface { + StatusCode() int +} + +// errorHandler renders any error as a small JSON body with the matching +// HTTP status (defaulting to 500 for an error with no known status) and, +// where the error carries one, a machine-readable code the S3 gateway +// dispatches on. +func (p *PrivateAPI) errorHandler(ctx fiber.Ctx, err error) error { + status := http.StatusInternalServerError + if sc, ok := err.(statusCoder); ok { + status = sc.StatusCode() + } else { + debuglogger.InternalError(err) + } + + body := map[string]string{"error": err.Error()} + if c, ok := err.(coder); ok { + body["code"] = c.Code() + } + + ctx.Status(status) + return ctx.JSON(body) +} diff --git a/iamapi/private/handlers.go b/iamapi/private/handlers.go new file mode 100644 index 00000000..adf3ecdc --- /dev/null +++ b/iamapi/private/handlers.go @@ -0,0 +1,174 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package private + +import ( + "encoding/json" + "maps" + "strings" + + "github.com/gofiber/fiber/v3" + "github.com/versity/versitygw/iamapi/internal/iammiddleware" + "github.com/versity/versitygw/iamapi/policy" + "github.com/versity/versitygw/iamapi/types" + "github.com/versity/versitygw/internal/sigv4auth" +) + +func (p *PrivateAPI) handleDeriveSigningKey(ctx fiber.Ctx) error { + var req DeriveSigningKeyRequest + if err := json.Unmarshal(ctx.Body(), &req); err != nil { + return errMalformedRequestBody + } + + _, secret, err := resolvePrivateIdentity(ctx.Context(), p.store, req.AccessKeyID, req.SessionToken) + if err != nil { + return mapResolveError(err) + } + + derivedKey := sigv4auth.DeriveKey(secret, req.Date, req.Region, req.Service) + + return ctx.JSON(DeriveSigningKeyResponse{DerivedKey: derivedKey}) +} + +// handleResolveIdentity answers "does this access key exist, and what +// principal is it" for a batch of access key ids, returning no credential +// material at all — see ResolveIdentityResponse for why that is what makes +// answering for a session, with no session token, safe. +func (p *PrivateAPI) handleResolveIdentity(ctx fiber.Ctx) error { + var req ResolveIdentityRequest + if err := json.Unmarshal(ctx.Body(), &req); err != nil { + return errMalformedRequestBody + } + + resolved := resolveIdentityMetadata(ctx.Context(), p.store, req.AccessKeyIDs) + + identities := make([]ResolvedIdentity, len(resolved)) + for i, r := range resolved { + if !r.Found { + continue + } + identities[i] = ResolvedIdentity{ + Found: true, + Kind: identityKindWireValue(r.Kind), + PrincipalArn: r.PrincipalArn, + } + } + + return ctx.JSON(ResolveIdentityResponse{Identities: identities}) +} + +// identityKindWireValue converts identityKind to its wire representation. +func identityKindWireValue(k identityKind) string { + if k == identityKindSession { + return KindSession + } + return KindUser +} + +func (p *PrivateAPI) handleEvaluatePolicy(ctx fiber.Ctx) error { + var req EvaluatePolicyRequest + if err := json.Unmarshal(ctx.Body(), &req); err != nil { + return errMalformedRequestBody + } + + identity, _, err := resolvePrivateIdentity(ctx.Context(), p.store, req.AccessKeyID, req.SessionToken) + if err != nil { + return mapResolveError(err) + } + + condition := conditionContextFor(*identity, req.Condition) + + decisions := make([][]string, len(req.Resources)) + sessionDecisions := make([][]string, len(req.Resources)) + hasSessionPolicy := false + + for i, resource := range req.Resources { + perAction := make([]string, len(req.Actions)) + perActionSession := make([]string, len(req.Actions)) + for j, action := range req.Actions { + identityDecision, sessionDecision, hasSession := iammiddleware.AuthorizeSplit(*identity, policy.RequestContext{ + Action: action, + Resource: resource, + Condition: condition, + }) + perAction[j] = decisionWireValue(identityDecision) + perActionSession[j] = decisionWireValue(sessionDecision) + hasSessionPolicy = hasSession + } + decisions[i] = perAction + sessionDecisions[i] = perActionSession + } + + resp := EvaluatePolicyResponse{ + Decisions: decisions, + PrincipalArn: iammiddleware.CallerArn(*identity), + } + if hasSessionPolicy { + resp.HasSessionPolicy = true + resp.SessionDecisions = sessionDecisions + } + + return ctx.JSON(resp) +} + +// conditionContextFor combines the request-derived condition keys the S3 +// gateway observed (source IP, time, transport) with the identity-derived +// keys only this service can know (aws:PrincipalArn, aws:username, …). +// +// Every key in an identity or resource namespace is dropped from the +// gateway's contribution first, then this side's own values are laid over +// the remainder. Filtering rather than merging matters: an +// override-on-collision merge would leave any key this service happens +// *not* to set — aws:PrincipalTag/x for an untagged role, say — under the +// gateway's control, which is precisely what a StringNotEquals-guarded +// Allow keys off. The gateway authenticates as root, so this is defense in +// depth rather than a trust boundary, but the layering costs nothing. +func conditionContextFor(identity types.Identity, requestKeys map[string][]string) map[string][]string { + condition := make(map[string][]string, len(requestKeys)) + for k, v := range requestKeys { + if isIdentityConditionKey(k) { + continue + } + condition[k] = v + } + maps.Copy(condition, iammiddleware.IdentityConditionContext(identity)) + return condition +} + +// isIdentityConditionKey reports whether key names the caller or the +// resource, and so may only be set by this service. Matching is +// case-insensitive because policy condition-key lookup is +// (iamapi/policy.lookupContextValues) — a caller must not be able to smuggle +// "AWS:PrincipalArn" past a case-sensitive filter. +func isIdentityConditionKey(key string) bool { + for _, prefix := range iammiddleware.IdentityConditionKeyPrefixes { + if strings.EqualFold(key, prefix) || + (strings.HasSuffix(prefix, "/") && len(key) > len(prefix) && strings.EqualFold(key[:len(prefix)], prefix)) { + return true + } + } + return false +} + +// decisionWireValue converts policy.Decision to its wire representation. +func decisionWireValue(d policy.Decision) string { + switch d { + case policy.DecisionAllow: + return DecisionAllow + case policy.DecisionDeny: + return DecisionDeny + default: + return DecisionNoMatch + } +} diff --git a/iamapi/private/identity.go b/iamapi/private/identity.go new file mode 100644 index 00000000..6da4825f --- /dev/null +++ b/iamapi/private/identity.go @@ -0,0 +1,103 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package private + +import ( + "context" + + "github.com/versity/versitygw/iamapi/internal/iammiddleware" + "github.com/versity/versitygw/iamapi/internal/iamutil" + "github.com/versity/versitygw/iamapi/types" + "github.com/versity/versitygw/internal/sigv4auth" +) + +// resolvePrivateIdentity resolves accessKeyID — long-term (AKIA…) or +// temporary (ASIA…) — to its identity and secret, for the S3 gateway to +// authenticate and authorize one of its own data-plane callers. +// +// sessionToken is required for, and only meaningful to, a temporary access +// key; it is what makes resolving a session here safe (see +// iamutil.ResolveSessionByToken). Errors are the iamutil.ErrIdentityNotFound +// / iamutil.ErrInvalidSessionToken sentinels, so the caller can report which +// failure occurred. +func resolvePrivateIdentity(ctx context.Context, store iamutil.IdentityStore, accessKeyID, sessionToken string) (*types.Identity, string, error) { + if sigv4auth.IsTempAccessKeyID(accessKeyID) { + return iamutil.ResolveSessionByToken(ctx, store, accessKeyID, sessionToken) + } + if sessionToken != "" { + // A token alongside a permanent credential is always a caller + // error, and accepting it silently would mask a misrouted request. + return nil, "", iamutil.ErrInvalidSessionToken + } + return iamutil.ResolveUserIdentity(ctx, store, accessKeyID) +} + +// identityKind labels what sort of principal an access key belongs to, for +// callers that need to tell an ephemeral session apart from a long-term user +// without holding a session token. +type identityKind string + +const ( + identityKindUser identityKind = "user" + identityKindSession identityKind = "session" +) + +// resolveIdentityMetadata answers "does this access key exist, and what +// principal is it" for each of accessKeyIDs, returning nothing that could +// authenticate anyone — no secret, no derived key, no policy. That is what +// makes it safe to resolve a temporary (ASIA…) key here with no session +// token: knowing a session exists grants nothing, whereas knowing its secret +// grants everything. +// +// It backs the S3 gateway's IAMService.GetUserAccount, whose only real +// consumer is auth.CheckIfAccountsExist — validating the principals named in +// a bucket policy or ACL, one batch per PutBucketPolicy/PutBucketAcl. +// Results are positional: one entry per input, with Found false for keys +// that don't resolve, rather than an error for the whole batch. +func resolveIdentityMetadata(ctx context.Context, store iamutil.IdentityStore, accessKeyIDs []string) []identityMetadata { + out := make([]identityMetadata, len(accessKeyIDs)) + for i, accessKeyID := range accessKeyIDs { + if sigv4auth.IsTempAccessKeyID(accessKeyID) { + session, err := store.GetSession(ctx, accessKeyID) + if err != nil { + continue + } + out[i] = identityMetadata{ + Found: true, + Kind: identityKindSession, + PrincipalArn: iamutil.BuildAssumedRoleArn(iamutil.DefaultAccountID, session.RoleName, session.RoleSessionName), + } + continue + } + + identity, _, err := iamutil.ResolveUserIdentity(ctx, store, accessKeyID) + if err != nil { + continue + } + out[i] = identityMetadata{ + Found: true, + Kind: identityKindUser, + PrincipalArn: iammiddleware.CallerArn(*identity), + } + } + return out +} + +// identityMetadata is one resolveIdentityMetadata result. The zero value +// means "no such access key". +type identityMetadata struct { + Found bool + Kind identityKind + PrincipalArn string +} diff --git a/iamapi/private/listener.go b/iamapi/private/listener.go new file mode 100644 index 00000000..5e56d559 --- /dev/null +++ b/iamapi/private/listener.go @@ -0,0 +1,68 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package private + +import ( + "fmt" + "net" + "time" + + "github.com/gofiber/fiber/v3" + "github.com/versity/versitygw/internal/netutil" +) + +const shutDownDuration = time.Second * 10 + +// ServeMultiPort binds and serves the private endpoints on every address in +// addrs. Each address is checked with netutil.RequireSecureTransport before +// binding anything — mTLS (server cert + mandatory client-cert +// verification) or a unix socket, nothing else — so a misconfiguration +// fails startup instead of silently serving these endpoints in the clear. +// tlsOpts is only applied to non-unix-socket addresses, or to a unix +// socket address if a server certificate is configured for it too. +func (p *PrivateAPI) ServeMultiPort(addrs []string, tlsOpts netutil.TLSOptions) error { + if len(addrs) == 0 { + return fmt.Errorf("no private listener addresses specified") + } + + hasMTLS := tlsOpts.GetCertificate != nil && tlsOpts.ClientCAs != nil && tlsOpts.RequireClientCert + for _, addr := range addrs { + if err := netutil.RequireSecureTransport(addr, hasMTLS); err != nil { + return err + } + } + + var listeners []net.Listener + for _, addr := range addrs { + var ln net.Listener + var err error + if netutil.IsUnixSocketPath(addr) && tlsOpts.GetCertificate == nil { + ln, err = netutil.NewMultiAddrListener(fiber.NetworkTCP, addr, netutil.ListenerOptions{SocketPerm: p.socketPerm}) + } else { + ln, err = netutil.NewMultiAddrTLSListenerWithOptions(fiber.NetworkTCP, addr, tlsOpts, netutil.ListenerOptions{SocketPerm: p.socketPerm}) + } + if err != nil { + return fmt.Errorf("failed to bind private iam listener %s: %w", addr, err) + } + listeners = append(listeners, ln) + } + + finalListener := netutil.NewMultiListener(listeners...) + return p.app.Listener(finalListener, fiber.ListenConfig{DisableStartupMessage: true}) +} + +// Shutdown gracefully stops the private endpoint listeners. +func (p *PrivateAPI) Shutdown() error { + return p.app.ShutdownWithTimeout(shutDownDuration) +} diff --git a/iamapi/private/private_test.go b/iamapi/private/private_test.go new file mode 100644 index 00000000..bf424251 --- /dev/null +++ b/iamapi/private/private_test.go @@ -0,0 +1,694 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package private + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/versity/versitygw/iamapi/internal/iammiddleware" + "github.com/versity/versitygw/iamapi/internal/iamutil" + "github.com/versity/versitygw/iamapi/storage" + "github.com/versity/versitygw/iamapi/types" + "github.com/versity/versitygw/internal/sigv4auth" +) + +var testRoot = iammiddleware.RootCredentials{Access: "AKIDTESTROOT", Secret: "TESTROOTSECRET"} + +// newTestServer builds a fresh file-backed store rooted at t.TempDir() and a +// PrivateAPI on top of it — no public control-plane IAMApiServer involved, +// since this package's handlers only ever need a populated storage.Storer. +func newTestServer(t *testing.T) (*PrivateAPI, storage.Storer) { + t.Helper() + + store, err := storage.New(storage.Config{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("storage.New: %v", err) + } + + p, err := New(store, testRoot) + if err != nil { + t.Fatalf("New: %v", err) + } + return p, store +} + +// createTestUser creates a user with the given name, access key, and +// (optional) inline policy directly against store — bypassing the public +// control-plane API entirely, since it isn't under test here. Arn is set +// explicitly (iamutil.BuildUserArn, matching what the control-plane +// controller computes before calling storage.CreateUser — storage.CreateUser +// itself never populates it) so tests can assert on a realistic principal +// ARN in an evaluate-policy response. +func createTestUser(t *testing.T, store storage.Storer, userName, accessKeyID, secret, policyDocument string) { + t.Helper() + ctx := context.Background() + + if _, err := store.CreateUser(ctx, types.User{ + UserName: userName, + Path: "/", + Arn: iamutil.BuildUserArn(iamutil.DefaultAccountID, "/", userName), + CreateDate: time.Now().UTC(), + }); err != nil { + t.Fatalf("CreateUser: %v", err) + } + + if _, err := store.CreateAccessKey(ctx, storage.CreateAccessKeyInput{ + UserName: userName, + AccessKeyID: accessKeyID, + SecretAccessKey: secret, + Status: "Active", + CreateDate: time.Now().UTC(), + }); err != nil { + t.Fatalf("CreateAccessKey: %v", err) + } + + if policyDocument != "" { + if err := store.PutUserPolicy(ctx, storage.PutUserPolicyInput{ + UserName: userName, + PolicyName: "P", + PolicyDocument: policyDocument, + }); err != nil { + t.Fatalf("PutUserPolicy: %v", err) + } + } +} + +// signPrivateRequest signs req as access/secret for the private endpoints' +// SigV4 protocol (service "iam", iammiddleware.SigningRegion), mutating its +// Authorization/X-Amz-Date headers in place. +func signPrivateRequest(t *testing.T, req *http.Request, access, secret string, payloadHash string) { + t.Helper() + + signingTime := time.Now().UTC() + yyyymmdd := signingTime.Format(sigv4auth.YYYYMMDD) + derivedKey := sigv4auth.DeriveKey(secret, yyyymmdd, iammiddleware.SigningRegion, privateService) + in := sigv4auth.SigningInputFromRequest(req) + in.AccessKeyID = access + in.CredentialScope = sigv4auth.BuildCredentialScope(yyyymmdd, iammiddleware.SigningRegion, privateService) + in.PayloadHash = payloadHash + in.SigningTime = signingTime + result := sigv4auth.BuildAndSign(derivedKey, in) + req.Header.Set("X-Amz-Date", result.AmzDate) + req.Header.Set("Authorization", result.AuthorizationHeader) +} + +func doPrivateRequest(t *testing.T, p *PrivateAPI, method, target, access, secret string, body []byte) *http.Response { + t.Helper() + + req := httptest.NewRequest(method, target, bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.ContentLength = int64(len(body)) + + hash := sigv4auth.PayloadSHA256Hex(body) + signPrivateRequest(t, req, access, secret, hash) + + resp, err := p.app.Test(req) + if err != nil { + t.Fatalf("app.Test: %v", err) + } + return resp +} + +func readBody(t *testing.T, resp *http.Response) string { + t.Helper() + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + return string(body) +} + +func TestPrivateAPIDeriveSigningKey(t *testing.T) { + p, store := newTestServer(t) + createTestUser(t, store, "alice", "AKIAALICE", "alicesecret", "") + + yyyymmdd := time.Now().UTC().Format(sigv4auth.YYYYMMDD) + body, _ := json.Marshal(DeriveSigningKeyRequest{ + AccessKeyID: "AKIAALICE", + Date: yyyymmdd, + Region: "us-east-1", + Service: "s3", + }) + + resp := doPrivateRequest(t, p, http.MethodPost, DerivePath, testRoot.Access, testRoot.Secret, body) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, body=%s", resp.StatusCode, readBody(t, resp)) + } + + var got DeriveSigningKeyResponse + if err := json.Unmarshal([]byte(readBody(t, resp)), &got); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + + want := sigv4auth.DeriveKey("alicesecret", yyyymmdd, "us-east-1", "s3") + if !bytes.Equal(got.DerivedKey, want) { + t.Errorf("DerivedKey = %x, want %x", got.DerivedKey, want) + } +} + +func TestPrivateAPIDeriveSigningKeyRejectsUnknownAccessKey(t *testing.T) { + p, _ := newTestServer(t) + + body, _ := json.Marshal(DeriveSigningKeyRequest{ + AccessKeyID: "AKIADOESNOTEXIST", + Date: time.Now().UTC().Format(sigv4auth.YYYYMMDD), + Region: "us-east-1", + Service: "s3", + }) + + resp := doPrivateRequest(t, p, http.MethodPost, DerivePath, testRoot.Access, testRoot.Secret, body) + if resp.StatusCode != http.StatusForbidden { + t.Errorf("status = %d, want %d; body=%s", resp.StatusCode, http.StatusForbidden, readBody(t, resp)) + } +} + +// TestPrivateAPIDeriveSigningKeySessionToken covers every way a temporary +// (ASIA…) access key can be presented to derive-signing-key. The security of +// the whole session path rests on exactly one thing — that a signing key is +// handed out only for a session token matching the one stored — so each +// wrong-token shape is pinned, along with the error code that tells the S3 +// gateway to report InvalidToken rather than InvalidAccessKeyId. +func TestPrivateAPIDeriveSigningKeySessionToken(t *testing.T) { + p, store := newTestServer(t) + role := createTestRole(t, store, "testrole", "") + session := createTestSessionForRole(t, store, role, "ASIASOMESESSIONKEY", "sessionsecret", "correct-session-token", "") + + tests := []struct { + name string + token string + wantStatus int + wantCode string + }{ + {name: "no token at all", token: "", wantStatus: http.StatusForbidden, wantCode: CodeNoSuchIdentity}, + {name: "wrong token", token: "wrong-session-token", wantStatus: http.StatusForbidden, wantCode: CodeInvalidToken}, + {name: "correct token", token: session.SessionToken, wantStatus: http.StatusOK}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + body, _ := json.Marshal(DeriveSigningKeyRequest{ + AccessKeyID: session.AccessKeyId, + SessionToken: tt.token, + Date: time.Now().UTC().Format(sigv4auth.YYYYMMDD), + Region: "us-east-1", + Service: "s3", + }) + + resp := doPrivateRequest(t, p, http.MethodPost, DerivePath, testRoot.Access, testRoot.Secret, body) + raw := readBody(t, resp) + if resp.StatusCode != tt.wantStatus { + t.Fatalf("status = %d, want %d; body=%s", resp.StatusCode, tt.wantStatus, raw) + } + if tt.wantCode != "" { + var errBody struct{ Code string } + if err := json.Unmarshal([]byte(raw), &errBody); err != nil { + t.Fatalf("unmarshal error body %s: %v", raw, err) + } + if errBody.Code != tt.wantCode { + t.Fatalf("error code = %q, want %q", errBody.Code, tt.wantCode) + } + return + } + + var out DeriveSigningKeyResponse + if err := json.Unmarshal([]byte(raw), &out); err != nil { + t.Fatalf("unmarshal %s: %v", raw, err) + } + want := sigv4auth.DeriveKey("sessionsecret", time.Now().UTC().Format(sigv4auth.YYYYMMDD), "us-east-1", "s3") + if string(out.DerivedKey) != string(want) { + t.Errorf("derived key = %x, want %x", out.DerivedKey, want) + } + }) + } +} + +// TestPrivateAPIDeriveSigningKeyRejectsTokenWithPermanentKey confirms a +// session token offered alongside a long-term (AKIA…) key is rejected rather +// than ignored — accepting it silently would mask a misrouted request. +func TestPrivateAPIDeriveSigningKeyRejectsTokenWithPermanentKey(t *testing.T) { + p, store := newTestServer(t) + createTestUser(t, store, "alice", "AKIAALICE", "alicesecret", "") + + body, _ := json.Marshal(DeriveSigningKeyRequest{ + AccessKeyID: "AKIAALICE", + SessionToken: "some-session-token", + Date: time.Now().UTC().Format(sigv4auth.YYYYMMDD), + Region: "us-east-1", + Service: "s3", + }) + + resp := doPrivateRequest(t, p, http.MethodPost, DerivePath, testRoot.Access, testRoot.Secret, body) + if resp.StatusCode != http.StatusForbidden { + t.Errorf("status = %d, want %d; body=%s", resp.StatusCode, http.StatusForbidden, readBody(t, resp)) + } +} + +// TestPrivateAPIEvaluatePolicySessionPolicy confirms the role's own policies +// and the session policy are reported *separately*, not folded together. +// +// The S3 gateway needs them apart because a bucket policy is also in play +// there: a session policy filters permissions that came from the bucket +// policy too, while the role's own decision does not. See +// iammiddleware.AuthorizeSplit. +func TestPrivateAPIEvaluatePolicySessionPolicy(t *testing.T) { + rolePolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject"],"Resource":"*"}]}` + + tests := []struct { + name string + sessionPolicy string + action string + want string + wantSession string + wantHasSessionPo bool + }{ + { + name: "no session policy: role decision stands alone", + action: "s3:GetObject", + want: DecisionAllow, + }, + { + name: "session policy narrows to a subset", + sessionPolicy: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}`, + action: "s3:PutObject", + want: DecisionAllow, + wantSession: DecisionNoMatch, + wantHasSessionPo: true, + }, + { + name: "session policy cannot widen beyond the role", + sessionPolicy: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}`, + action: "s3:DeleteObject", + want: DecisionNoMatch, + wantSession: DecisionAllow, + wantHasSessionPo: true, + }, + { + name: "session policy explicit deny against the role's allow", + sessionPolicy: `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"s3:GetObject","Resource":"*"}]}`, + action: "s3:GetObject", + want: DecisionAllow, + wantSession: DecisionDeny, + wantHasSessionPo: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p, store := newTestServer(t) + role := createTestRole(t, store, "testrole", rolePolicy) + session := createTestSessionForRole(t, store, role, "ASIASESSION", "sessionsecret", "tok", tt.sessionPolicy) + + body, _ := json.Marshal(EvaluatePolicyRequest{ + AccessKeyID: session.AccessKeyId, + SessionToken: session.SessionToken, + Actions: []string{tt.action}, + Resources: []string{"*"}, + }) + + resp := doPrivateRequest(t, p, http.MethodPost, EvaluatePath, testRoot.Access, testRoot.Secret, body) + raw := readBody(t, resp) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", resp.StatusCode, raw) + } + + var out EvaluatePolicyResponse + if err := json.Unmarshal([]byte(raw), &out); err != nil { + t.Fatalf("unmarshal %s: %v", raw, err) + } + if len(out.Decisions) != 1 || len(out.Decisions[0]) != 1 || out.Decisions[0][0] != tt.want { + t.Errorf("Decisions = %v, want [[%v]]", out.Decisions, tt.want) + } + if out.HasSessionPolicy != tt.wantHasSessionPo { + t.Errorf("HasSessionPolicy = %v, want %v", out.HasSessionPolicy, tt.wantHasSessionPo) + } + if tt.wantHasSessionPo { + if len(out.SessionDecisions) != 1 || len(out.SessionDecisions[0]) != 1 || out.SessionDecisions[0][0] != tt.wantSession { + t.Errorf("SessionDecisions = %v, want [[%v]]", out.SessionDecisions, tt.wantSession) + } + } else if len(out.SessionDecisions) != 0 { + t.Errorf("SessionDecisions = %v, want none when no session policy applies", out.SessionDecisions) + } + wantArn := iamutil.BuildAssumedRoleArn(iamutil.DefaultAccountID, role.RoleName, session.RoleSessionName) + if out.PrincipalArn != wantArn { + t.Errorf("PrincipalArn = %q, want %q", out.PrincipalArn, wantArn) + } + }) + } +} + +// TestPrivateAPIEvaluatePolicyStripsCallerSuppliedIdentityKeys confirms the +// gateway cannot influence an identity-namespace condition key by sending +// one itself. Stripping rather than overriding matters for keys the service +// does not set at all: aws:PrincipalTag/team below has no value for an +// untagged user, and a policy that Allows on its *absence* must not be +// satisfiable by a value the caller supplied. +func TestPrivateAPIEvaluatePolicyStripsCallerSuppliedIdentityKeys(t *testing.T) { + p, store := newTestServer(t) + createTestUser(t, store, "alice", "AKIAALICE", "alicesecret", + `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*",`+ + `"Condition":{"StringEquals":{"aws:PrincipalTag/team":"admins"}}}]}`) + + body, _ := json.Marshal(EvaluatePolicyRequest{ + AccessKeyID: "AKIAALICE", + Actions: []string{"s3:GetObject"}, + Resources: []string{"*"}, + Condition: map[string][]string{ + "aws:PrincipalTag/team": {"admins"}, + // Case-varied spellings must be stripped too: policy key lookup + // is case-insensitive, so a case-sensitive filter would be no + // filter at all. + "AWS:PrincipalArn": {"arn:aws:iam::000000000000:user/somebodyelse"}, + }, + }) + + resp := doPrivateRequest(t, p, http.MethodPost, EvaluatePath, testRoot.Access, testRoot.Secret, body) + raw := readBody(t, resp) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", resp.StatusCode, raw) + } + + var out EvaluatePolicyResponse + if err := json.Unmarshal([]byte(raw), &out); err != nil { + t.Fatalf("unmarshal %s: %v", raw, err) + } + if len(out.Decisions) != 1 || len(out.Decisions[0]) != 1 || out.Decisions[0][0] != DecisionNoMatch { + t.Errorf("Decisions = %v, want [[%v]]: a caller-supplied aws:PrincipalTag must not satisfy the condition", out.Decisions, DecisionNoMatch) + } +} + +// TestPrivateAPIEvaluatePolicyUsesRequestConditionKeys is the counterpart to +// the test above: the request-derived keys the gateway *is* the authority +// for must reach the policy evaluator intact. +func TestPrivateAPIEvaluatePolicyUsesRequestConditionKeys(t *testing.T) { + p, store := newTestServer(t) + createTestUser(t, store, "alice", "AKIAALICE", "alicesecret", + `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*",`+ + `"Condition":{"IpAddress":{"aws:SourceIp":"10.1.2.0/24"}}}]}`) + + tests := []struct { + name string + sourceIP string + want string + }{ + {name: "matching source ip", sourceIP: "10.1.2.3", want: DecisionAllow}, + {name: "non-matching source ip", sourceIP: "10.9.9.9", want: DecisionNoMatch}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + body, _ := json.Marshal(EvaluatePolicyRequest{ + AccessKeyID: "AKIAALICE", + Actions: []string{"s3:GetObject"}, + Resources: []string{"*"}, + Condition: map[string][]string{"aws:SourceIp": {tt.sourceIP}}, + }) + + resp := doPrivateRequest(t, p, http.MethodPost, EvaluatePath, testRoot.Access, testRoot.Secret, body) + raw := readBody(t, resp) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", resp.StatusCode, raw) + } + + var out EvaluatePolicyResponse + if err := json.Unmarshal([]byte(raw), &out); err != nil { + t.Fatalf("unmarshal %s: %v", raw, err) + } + if len(out.Decisions) != 1 || len(out.Decisions[0]) != 1 || out.Decisions[0][0] != tt.want { + t.Errorf("Decisions = %v, want [%v]", out.Decisions, tt.want) + } + }) + } +} + +// TestPrivateAPIResolveIdentity covers the metadata-only endpoint: it must +// answer positionally for a whole batch, resolve a session with no token +// (the disclosure is harmless, since nothing it returns authenticates +// anyone), and label session versus user so the gateway can refuse to +// persist a reference to an ephemeral principal. +func TestPrivateAPIResolveIdentity(t *testing.T) { + p, store := newTestServer(t) + createTestUser(t, store, "alice", "AKIAALICE", "alicesecret", "") + role := createTestRole(t, store, "testrole", "") + session := createTestSessionForRole(t, store, role, "ASIASESSION", "sessionsecret", "tok", "") + + body, _ := json.Marshal(ResolveIdentityRequest{ + AccessKeyIDs: []string{"AKIAALICE", "AKIADOESNOTEXIST", session.AccessKeyId}, + }) + + resp := doPrivateRequest(t, p, http.MethodPost, ResolveIdentityPath, testRoot.Access, testRoot.Secret, body) + raw := readBody(t, resp) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", resp.StatusCode, raw) + } + + var out ResolveIdentityResponse + if err := json.Unmarshal([]byte(raw), &out); err != nil { + t.Fatalf("unmarshal %s: %v", raw, err) + } + + want := []ResolvedIdentity{ + {Found: true, Kind: KindUser, PrincipalArn: iamutil.BuildUserArn(iamutil.DefaultAccountID, "/", "alice")}, + {}, + {Found: true, Kind: KindSession, PrincipalArn: iamutil.BuildAssumedRoleArn(iamutil.DefaultAccountID, role.RoleName, session.RoleSessionName)}, + } + if len(out.Identities) != len(want) { + t.Fatalf("Identities = %+v, want %d entries", out.Identities, len(want)) + } + for i := range want { + if out.Identities[i] != want[i] { + t.Errorf("Identities[%d] = %+v, want %+v", i, out.Identities[i], want[i]) + } + } +} + +func TestPrivateAPIEvaluatePolicy(t *testing.T) { + p, store := newTestServer(t) + createTestUser(t, store, "alice", "AKIAALICE", "alicesecret", + `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"},{"Effect":"Deny","Action":"s3:DeleteObject","Resource":"*"}]}`) + wantArn := iamutil.BuildUserArn(iamutil.DefaultAccountID, "/", "alice") + + tests := []struct { + name string + action string + want string + }{ + {name: "allowed action", action: "s3:GetObject", want: DecisionAllow}, + {name: "action not granted", action: "s3:PutObject", want: DecisionNoMatch}, + {name: "explicitly denied action", action: "s3:DeleteObject", want: DecisionDeny}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + body, _ := json.Marshal(EvaluatePolicyRequest{ + AccessKeyID: "AKIAALICE", + Actions: []string{tt.action}, + Resources: []string{"*"}, + }) + + resp := doPrivateRequest(t, p, http.MethodPost, EvaluatePath, testRoot.Access, testRoot.Secret, body) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, body=%s", resp.StatusCode, readBody(t, resp)) + } + + var got EvaluatePolicyResponse + if err := json.Unmarshal([]byte(readBody(t, resp)), &got); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + if len(got.Decisions) != 1 || len(got.Decisions[0]) != 1 || got.Decisions[0][0] != tt.want { + t.Errorf("Decisions = %v, want [[%v]]", got.Decisions, tt.want) + } + if got.PrincipalArn != wantArn { + t.Errorf("PrincipalArn = %q, want %q", got.PrincipalArn, wantArn) + } + }) + } +} + +// TestPrivateAPIEvaluatePolicyBatchesMultipleActions confirms multiple +// actions supplied in one EvaluatePolicyRequest are each evaluated +// independently against the same resource, in a single request, with +// Decisions returned in the same order as Actions. +func TestPrivateAPIEvaluatePolicyBatchesMultipleActions(t *testing.T) { + p, store := newTestServer(t) + createTestUser(t, store, "alice", "AKIAALICE", "alicesecret", + `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"},{"Effect":"Deny","Action":"s3:DeleteObject","Resource":"*"}]}`) + + body, _ := json.Marshal(EvaluatePolicyRequest{ + AccessKeyID: "AKIAALICE", + Actions: []string{"s3:GetObject", "s3:PutObject", "s3:DeleteObject"}, + Resources: []string{"*"}, + }) + + resp := doPrivateRequest(t, p, http.MethodPost, EvaluatePath, testRoot.Access, testRoot.Secret, body) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, body=%s", resp.StatusCode, readBody(t, resp)) + } + + var got EvaluatePolicyResponse + if err := json.Unmarshal([]byte(readBody(t, resp)), &got); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + want := []string{DecisionAllow, DecisionNoMatch, DecisionDeny} + if len(got.Decisions) != 1 || len(got.Decisions[0]) != len(want) { + t.Fatalf("Decisions = %v, want [%v]", got.Decisions, want) + } + for i := range want { + if got.Decisions[0][i] != want[i] { + t.Errorf("Decisions[0][%d] = %v, want %v", i, got.Decisions[0][i], want[i]) + } + } +} + +// TestPrivateAPIEvaluatePolicyBatchesMultipleResources confirms several +// resources are each evaluated against every action in one request — what +// keeps a 1000-key DeleteObjects a single round trip. +func TestPrivateAPIEvaluatePolicyBatchesMultipleResources(t *testing.T) { + p, store := newTestServer(t) + createTestUser(t, store, "alice", "AKIAALICE", "alicesecret", + `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:DeleteObject","Resource":"arn:aws:s3:::b/allowed/*"}]}`) + + body, _ := json.Marshal(EvaluatePolicyRequest{ + AccessKeyID: "AKIAALICE", + Actions: []string{"s3:DeleteObject"}, + Resources: []string{"arn:aws:s3:::b/allowed/one", "arn:aws:s3:::b/denied/two", "arn:aws:s3:::b/allowed/three"}, + }) + + resp := doPrivateRequest(t, p, http.MethodPost, EvaluatePath, testRoot.Access, testRoot.Secret, body) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, body=%s", resp.StatusCode, readBody(t, resp)) + } + + var got EvaluatePolicyResponse + if err := json.Unmarshal([]byte(readBody(t, resp)), &got); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + want := []string{DecisionAllow, DecisionNoMatch, DecisionAllow} + if len(got.Decisions) != len(want) { + t.Fatalf("Decisions = %v, want %d resource entries", got.Decisions, len(want)) + } + for i := range want { + if len(got.Decisions[i]) != 1 || got.Decisions[i][0] != want[i] { + t.Errorf("Decisions[%d] = %v, want [%v]", i, got.Decisions[i], want[i]) + } + } +} + +// TestPrivateAPIRejectsNonRootCredential confirms a validly-signed request +// from a real (non-root) IAM user's own credentials is rejected outright — +// only the S3 gateway's own root-equivalent identity may ever call these +// endpoints. +func TestPrivateAPIRejectsNonRootCredential(t *testing.T) { + p, store := newTestServer(t) + createTestUser(t, store, "alice", "AKIAALICE", "alicesecret", "") + + body, _ := json.Marshal(DeriveSigningKeyRequest{ + AccessKeyID: "AKIAALICE", + Date: time.Now().UTC().Format(sigv4auth.YYYYMMDD), + Region: "us-east-1", + Service: "s3", + }) + + // Signed with alice's own, otherwise-valid credentials — not root. + resp := doPrivateRequest(t, p, http.MethodPost, DerivePath, "AKIAALICE", "alicesecret", body) + if resp.StatusCode != http.StatusForbidden { + t.Errorf("status = %d, want %d; body=%s", resp.StatusCode, http.StatusForbidden, readBody(t, resp)) + } +} + +func TestPrivateAPIRejectsMalformedBody(t *testing.T) { + p, _ := newTestServer(t) + + resp := doPrivateRequest(t, p, http.MethodPost, DerivePath, testRoot.Access, testRoot.Secret, []byte("not json")) + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("status = %d, want %d; body=%s", resp.StatusCode, http.StatusBadRequest, readBody(t, resp)) + } +} + +func TestPrivateAPIRejectsUnsignedRequest(t *testing.T) { + p, _ := newTestServer(t) + + body, _ := json.Marshal(DeriveSigningKeyRequest{AccessKeyID: "AKIAX", Date: "20260101", Region: "us-east-1", Service: "s3"}) + req := httptest.NewRequest(http.MethodPost, DerivePath, bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + + resp, err := p.app.Test(req) + if err != nil { + t.Fatalf("app.Test: %v", err) + } + if resp.StatusCode == http.StatusOK { + t.Errorf("expected an unsigned request to be rejected, got 200") + } +} + +// createTestRole creates a role with an optional inline permission policy +// directly against store, the same way createTestUser bypasses the +// control-plane API. Arn and RoleID are set explicitly because +// storage.CreateRole doesn't populate them, and iamutil.ResolveSessionByToken +// re-checks both against the session before attaching the role's policies. +func createTestRole(t *testing.T, store storage.Storer, roleName, policyDocument string) *types.Role { + t.Helper() + ctx := context.Background() + + role, err := store.CreateRole(ctx, types.Role{ + RoleName: roleName, + Path: "/", + RoleID: "AROA" + roleName, + Arn: iamutil.BuildRoleArn(iamutil.DefaultAccountID, "/", roleName), + CreateDate: time.Now().UTC(), + }) + if err != nil { + t.Fatalf("CreateRole: %v", err) + } + + if policyDocument != "" { + if err := store.PutRolePolicy(ctx, storage.PutRolePolicyInput{ + RoleName: roleName, + PolicyName: "P", + PolicyDocument: policyDocument, + }); err != nil { + t.Fatalf("PutRolePolicy: %v", err) + } + } + return role +} + +// createTestSessionForRole creates a session against role as +// AssumeRoleWithWebIdentity would, with an optional inline session policy. +// RoleID and RoleArn are copied from role so the session survives +// iamutil.ResolveSessionByToken's same-role re-check. +func createTestSessionForRole(t *testing.T, store storage.Storer, role *types.Role, accessKeyID, secret, token, sessionPolicy string) *types.Session { + t.Helper() + + session, err := store.CreateSession(context.Background(), types.Session{ + AccessKeyId: accessKeyID, + SecretAccessKey: secret, + SessionToken: token, + RoleArn: role.Arn, + RoleName: role.RoleName, + RoleID: role.RoleID, + RoleSessionName: "testsession", + CreateDate: time.Now().UTC(), + Expiration: time.Now().UTC().Add(time.Hour), + Policy: sessionPolicy, + }) + if err != nil { + t.Fatalf("CreateSession: %v", err) + } + return session +} diff --git a/iamapi/private/server.go b/iamapi/private/server.go new file mode 100644 index 00000000..6b465007 --- /dev/null +++ b/iamapi/private/server.go @@ -0,0 +1,124 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package private implements the standalone IAM service's private endpoint +// set: derive a SigV4 signing key, evaluate IAM identity policies, and +// resolve an access key id to the principal that owns it. All three exist +// purely so the S3 gateway can authenticate and authorize its own callers +// without ever holding a plaintext secret or a policy document itself. +// This is a separate fiber app from the public control-plane +// iamapi.IAMApiServer, meant to be served on its own listener(s), never the +// public one. +// +// The endpoints authenticate strictly as the configured root credential +// — only the S3 gateway itself, signing as its own IAM-client identity, ever +// legitimately calls them. Transport security is enforced by ServeMultiPort, +// not by request handling +package private + +import ( + "fmt" + "os" + + "github.com/gofiber/fiber/v3" + "github.com/gofiber/fiber/v3/middleware/logger" + "github.com/versity/versitygw/iamapi/internal/iammiddleware" + "github.com/versity/versitygw/iamapi/storage" + "github.com/versity/versitygw/internal/sigv4auth" +) + +const ( + // These are exported so auth.IAMServiceStandalone (the S3-side client) + // shares one source of truth for the routes rather than duplicating the + // literal path strings. + DerivePath = "/private/derive-signing-key" + EvaluatePath = "/private/evaluate-policy" + ResolveIdentityPath = "/private/resolve-identity" + + // privateService is the SigV4 credential-scope service name the S3 + // gateway signs its own requests to these endpoints with. It's an + // internal detail — these routes aren't part of any AWS-compatible + // API — reusing "iam" is simplest and avoids inventing a new constant + // consumers on both sides would have to agree on. + privateService = sigv4auth.ServiceIAM +) + +// PrivateAPI is the standalone IAM service's private endpoint set +type PrivateAPI struct { + app *fiber.App + store storage.Storer + socketPerm os.FileMode + quiet bool +} + +type PrivateAPIOption func(*PrivateAPI) + +// WithPrivateSocketPerm sets the file-mode permission applied to any +// file-backed unix-socket listener address (no effect on TCP addresses or +// Linux abstract-namespace sockets). +func WithPrivateSocketPerm(perm os.FileMode) PrivateAPIOption { + return func(p *PrivateAPI) { p.socketPerm = perm } +} + +// WithPrivateQuiet suppresses per-request summary logging, mirroring +// iamapi.WithQuiet for the public API. Callers should gate both on the same +// flag so the two log streams turn on and off together. +func WithPrivateQuiet() PrivateAPIOption { + return func(p *PrivateAPI) { p.quiet = true } +} + +// New constructs the private endpoint set. root is the identity these +// endpoints authenticate every request against. +func New(store storage.Storer, root iammiddleware.RootCredentials, opts ...PrivateAPIOption) (*PrivateAPI, error) { + if store == nil { + return nil, fmt.Errorf("iamapi/private: storer is required") + } + + p := &PrivateAPI{store: store} + for _, opt := range opts { + opt(p) + } + + app := fiber.New(fiber.Config{ + AppName: "versitygw-iam-private", + ServerHeader: "VERSITYGW", + ErrorHandler: p.errorHandler, + }) + p.app = app + + if !p.quiet { + app.Use("*", logger.New(logger.Config{ + Format: "${time} | vgw-iam-private | ${status} | ${latency} | ${ip} | ${method} | ${path} | ${error} | ${queryParams}\n", + })) + } + + rootAuth := iammiddleware.VerifyRootOnlySigV4(privateService, &root) + app.Post(DerivePath, chainHandlers(rootAuth, p.handleDeriveSigningKey)) + app.Post(EvaluatePath, chainHandlers(rootAuth, p.handleEvaluatePolicy)) + app.Post(ResolveIdentityPath, chainHandlers(rootAuth, p.handleResolveIdentity)) + + return p, nil +} + +// chainHandlers composes handlers into one, calling each in turn and +// stopping at the first error. +func chainHandlers(handlers ...fiber.Handler) fiber.Handler { + return func(ctx fiber.Ctx) error { + for _, h := range handlers { + if err := h(ctx); err != nil { + return err + } + } + return nil + } +} diff --git a/iamapi/private/types.go b/iamapi/private/types.go new file mode 100644 index 00000000..2b438ec5 --- /dev/null +++ b/iamapi/private/types.go @@ -0,0 +1,114 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package private + +// DeriveSigningKeyRequest is the derive-signing-key request body. Date, +// Region, and Service are the request's credential-scope components +// (yyyymmdd/region/service), matching sigv4auth.DeriveKey's parameters. +// +// SessionToken is required when AccessKeyID is a temporary (ASIA…) key and +// must be absent otherwise. It is what makes resolving a session here safe: +// without it, anyone who learned a session's access key id could ask for +// that session's signing key. +type DeriveSigningKeyRequest struct { + AccessKeyID string `json:"accessKeyId"` + SessionToken string `json:"sessionToken,omitempty"` + Date string `json:"date"` + Region string `json:"region"` + Service string `json:"service"` +} + +// DeriveSigningKeyResponse carries the derived signing key (kSigning) — +// never the underlying secret. +type DeriveSigningKeyResponse struct { + DerivedKey []byte `json:"derivedKey"` +} + +// EvaluatePolicyRequest is the evaluate-policy request body +type EvaluatePolicyRequest struct { + AccessKeyID string `json:"accessKeyId"` + SessionToken string `json:"sessionToken,omitempty"` + Actions []string `json:"actions"` + Resources []string `json:"resources"` + Condition map[string][]string `json:"condition,omitempty"` +} + +// ResolveIdentityRequest asks whether each access key id exists and what +// principal it names. It carries no session token because the response +// carries no credential material +type ResolveIdentityRequest struct { + AccessKeyIDs []string `json:"accessKeyIds"` +} + +// ResolveIdentityResponse answers ResolveIdentityRequest positionally: one +// entry per requested access key id, in the same order, with Found false +// for one that doesn't resolve. It deliberately carries no secret, no +// derived key and no policy — only that makes it safe to answer for a +// temporary (ASIA…) key with no session token, since knowing a session +// exists grants nothing. +type ResolveIdentityResponse struct { + Identities []ResolvedIdentity `json:"identities"` +} + +// ResolvedIdentity is one ResolveIdentityResponse entry. Kind is +// KindUser or KindSession. +type ResolvedIdentity struct { + Found bool `json:"found"` + Kind string `json:"kind,omitempty"` + PrincipalArn string `json:"principalArn,omitempty"` +} + +// Kind values for ResolvedIdentity.Kind — strings on the wire for the same +// reason the Decision values below are: self-documenting, and immune to +// iota drift between the independently-built gateway and IAM service. +const ( + KindUser = "user" + KindSession = "session" +) + +// Decision values for each entry of EvaluatePolicyResponse.Decisions. +// Deliberately strings, not policy.Decision's int: self-documenting on the +// wire, and immune to iota drift between the S3 gateway and standalone IAM +// service — two independently-built processes speaking this protocol. +const ( + DecisionAllow = "allow" + DecisionDeny = "deny" + DecisionNoMatch = "no_match" +) + +// EvaluatePolicyResponse carries the full tri-state result for the whole +// requested matrix — not just whether each cell is allowed — so the S3 +// gateway can distinguish an explicit Deny (which must override an +// otherwise-allowing bucket policy) from a plain NoMatch (which doesn't), +// and can build an AWS-shaped denial message. +// +// Decisions[i][j] is the decision for the request's Resources[i] and +// Actions[j], in the order both were sent. PrincipalArn is the resolved +// identity's own ARN, best-effort: "" when it can't be resolved (e.g. a +// session whose role no longer exists), in which case the caller falls back +// to the access key. It is shared by the whole batch — one request always +// evaluates against a single identity. +// SessionDecisions is the same matrix evaluated against the caller's +// session policy alone, and HasSessionPolicy says whether one applied — the +// caller must ignore SessionDecisions when it is false. They are reported +// separately from Decisions rather than folded into them because a session +// policy filters *everything*, including permissions the S3 gateway's own +// bucket policy grants, which this service knows nothing about. See +// iammiddleware.AuthorizeSplit. +type EvaluatePolicyResponse struct { + Decisions [][]string `json:"decisions"` + SessionDecisions [][]string `json:"sessionDecisions,omitempty"` + HasSessionPolicy bool `json:"hasSessionPolicy,omitempty"` + PrincipalArn string `json:"principalArn,omitempty"` +} diff --git a/iamapi/server.go b/iamapi/server.go index 43d09a2c..a2dabad0 100644 --- a/iamapi/server.go +++ b/iamapi/server.go @@ -63,13 +63,14 @@ type IAMApiServer struct { oidcThumbprintAutoFetchDisabled bool } -func New(store storage.Storer, opts ...Option) (*IAMApiServer, error) { +func New(store storage.Storer, root RootCredentials, opts ...Option) (*IAMApiServer, error) { if store == nil { return nil, fmt.Errorf("iamapi: storer is required") } server := &IAMApiServer{ - store: store, + store: store, + rootCreds: &root, Router: &IAMApiRouter{ store: store, }, @@ -162,12 +163,6 @@ func WithOnListen(fn func()) Option { return func(s *IAMApiServer) { s.onListen = fn } } -func WithRootUserCreds(root RootCredentials) Option { - return func(s *IAMApiServer) { - s.rootCreds = &root - } -} - // WithOIDCThumbprintAutoFetchDisabled disables CreateOpenIDConnectProvider's // TLS auto-fetch fallback for when ThumbprintList is omitted. When set, an // omitted ThumbprintList is rejected with a MissingValue error instead of diff --git a/iamapi/storage/vault.go b/iamapi/storage/vault.go index db356bbf..e5d10da1 100644 --- a/iamapi/storage/vault.go +++ b/iamapi/storage/vault.go @@ -40,8 +40,7 @@ const vaultRequestTimeout = 10 * time.Second // withRoleCAS/withOIDCProviderCAS run when a version-checked (CAS) write // loses a race against a concurrent writer updating the same entity — // mirroring the 3-attempt collision-retry loops already used elsewhere in -// this package for ID generation (see controller.go's CreateUser/CreateRole/ -// CreateAccessKey). +// this package for ID generation. const maxCASRetries = 3 // errConcurrentModification is withUserCAS/withRoleCAS/withOIDCProviderCAS's @@ -359,7 +358,7 @@ func (s *VaultStore) GetUser(_ context.Context, username string) (*types.User, e // readUserVersion resolves username the same way GetUser does, additionally // returning the KV version the record was read at, so a mutation can write // back with a matching CAS value instead of racing on a blind -// delete-then-recreate (see replaceUser). +// delete-then-recreate. func (s *VaultStore) readUserVersion(username string) (*types.User, int32, error) { key := caseFoldKey(username) path := s.usersPath() + "/" + key @@ -752,13 +751,12 @@ const recordAccessKeyUsageTimeout = 5 * time.Second // RecordAccessKeyUsage updates accessKeyID's GetAccessKeyLastUsed metadata // in its own background goroutine, detached from ctx, and always returns -// nil immediately: this runs on the hot path of every authenticated request -// (see iammiddleware.recordAccessKeyUsage), and a Vault round trip — plus, -// on a CAS conflict, withUserCAS's retry loop — is too expensive to add -// synchronously to every one of them. A failure (including one that -// exhausts those retries) is only logged, never surfaced: this is purely -// informational metadata, and a lost update under concurrent use is -// immaterial. +// nil immediately: this runs on the hot path of every authenticated +// request, and a Vault round trip — plus, on a CAS conflict, withUserCAS's +// retry loop — is too expensive to add synchronously to every one of them. +// A failure (including one that exhausts those retries) is only logged, +// never surfaced: this is purely informational metadata, and a lost update +// under concurrent use is immaterial. func (s *VaultStore) RecordAccessKeyUsage(_ context.Context, accessKeyID, service, region string, when time.Time) error { go func() { ctx, cancel := context.WithTimeout(context.Background(), recordAccessKeyUsageTimeout) @@ -1043,7 +1041,7 @@ func (s *VaultStore) GetRole(_ context.Context, roleName string) (*types.Role, e // readRoleVersion is GetRole's counterpart to readUserVersion: it // additionally returns the KV version the record was read at, so a // mutation can write back with a matching CAS value instead of racing on a -// blind delete-then-recreate (see replaceRole). +// blind delete-then-recreate. func (s *VaultStore) readRoleVersion(roleName string) (*types.Role, int32, error) { key := caseFoldKey(roleName) path := s.rolesPath() + "/" + key @@ -1858,12 +1856,12 @@ func (s *VaultStore) GetSession(_ context.Context, accessKeyID string) (*types.S if err != nil { if vault.IsErrorStatus(err, http.StatusNotFound) { // Either this access key never existed, or Vault's own - // delete_version_after TTL (see setSessionTTL) already - // soft-deleted the version — confirmed live: Vault answers a - // read for a soft-deleted-but-not-yet-destroyed version with - // 404, not 200-with-null-data. Either way, best-effort purge - // the lingering metadata record now, since Vault doesn't - // appear to reclaim it on its own once merely soft-deleted. + // delete_version_after TTL already soft-deleted the version — + // Vault answers a read for a soft-deleted-but-not-yet-destroyed + // version with 404, not 200-with-null-data. Either way, + // best-effort purge the lingering metadata record now, since + // Vault doesn't appear to reclaim it on its own once merely + // soft-deleted. s.purgeSession(accessKeyID) return nil, ErrSessionNotFound } diff --git a/iamapi/policy/condition.go b/internal/condition/condition.go similarity index 61% rename from iamapi/policy/condition.go rename to internal/condition/condition.go index 64fa9a86..3b7d0f9a 100644 --- a/iamapi/policy/condition.go +++ b/internal/condition/condition.go @@ -12,11 +12,20 @@ // specific language governing permissions and limitations // under the License. -package policy +// Package condition implements AWS IAM's policy Condition grammar and +// evaluation semantics: the operator registry (StringEquals, IpAddress, +// DateGreaterThan, ...), the ForAllValues/ForAnyValue/IfExists modifiers, +// and ${...} policy-variable substitution. It has no knowledge of any +// particular policy type (identity, trust, or resource-based) — callers +// supply a statement's raw Condition block, a request's context-key values, +// and the enclosing document's Version, and get back whether the condition +// holds. This lets both iamapi/policy (IAM identity/trust policies) and +// auth (S3 bucket policies) share one implementation and one AWS-verified +// behavior, rather than maintaining two. +package condition import ( "bytes" - "encoding/base64" "encoding/json" "fmt" "net" @@ -28,14 +37,14 @@ import ( "github.com/versity/versitygw/debuglogger" ) -// ConditionValues decodes the value(s) of a single Condition operator/key -// pair. Unlike Action/Resource's string-only StringOrSlice, a Condition -// value may also be a bare JSON number or boolean rather than -// being re-serialized, so e.g. "5.50" round-trips as "5.50", not "5.5". A -// JSON null value or a non-scalar (object/array) element is rejected. -type ConditionValues []string +// Values decodes the value(s) of a single Condition operator/key pair. +// Unlike Action/Resource's string-only representation, a Condition value may +// also be a bare JSON number or boolean rather than a string, so e.g. "5.50" +// round-trips as "5.50", not "5.5". A JSON null value or a non-scalar +// (object/array) element is rejected. +type Values []string -func (c *ConditionValues) UnmarshalJSON(data []byte) error { +func (c *Values) UnmarshalJSON(data []byte) error { trimmed := bytes.TrimSpace(data) if len(trimmed) > 0 && trimmed[0] == '[' { var raws []json.RawMessage @@ -58,7 +67,7 @@ func (c *ConditionValues) UnmarshalJSON(data []byte) error { if !ok { return fmt.Errorf("policy: invalid condition value %s", trimmed) } - *c = ConditionValues{s} + *c = Values{s} return nil } @@ -90,14 +99,18 @@ func decodeConditionScalar(raw json.RawMessage) (string, bool) { return num.String(), true } -// conditionQualifier is IAM's multivalued-context-key set operator, given as -// a "ForAllValues:"/"ForAnyValue:" prefix on a condition operator name. -type conditionQualifier int +// Block is a statement's Condition object, decoded to operator name -> key +// -> value(s). +type Block map[string]map[string]Values + +// Qualifier is IAM's multivalued-context-key set operator, given as a +// "ForAllValues:"/"ForAnyValue:" prefix on a condition operator name. +type Qualifier int const ( - qualifierNone conditionQualifier = iota - qualifierForAllValues - qualifierForAnyValue + QualifierNone Qualifier = iota + QualifierForAllValues + QualifierForAnyValue ) // conditionComparator is a single (policy value, request value) match test @@ -145,10 +158,17 @@ var conditionRegistry = map[string]conditionOperatorDef{ "Bool": {compare: boolMatch}, - "BinaryEquals": {compare: binaryMatch}, + // BinaryEquals is a plain string comparison, not a base64-decode-then- + // compare: AWS's own IAM condition-operator reference documents the + // request context value as itself the base64 text (the same string + // that appears in the policy on a match), never the decoded raw bytes + // - live-verified via iam:SimulateCustomPolicy, which also rejects a + // non-base64 binary-typed context value outright. Do not "fix" this to + // decode either side. + "BinaryEquals": {compare: stringExact}, // ArnEquals and ArnLike behave identically in real AWS (both wildcard - // -aware), and are matched here with the same whole-string globMatch + // -aware), and are matched here with the same whole-string GlobMatch // already used for Action/Resource - do not "fix" ArnEquals to a strict // == later, that would diverge from AWS behavior. "ArnEquals": {compare: stringLike}, @@ -162,7 +182,7 @@ var conditionRegistry = map[string]conditionOperatorDef{ func stringExact(expected, actual string) bool { return expected == actual } func stringFold(expected, actual string) bool { return strings.EqualFold(expected, actual) } -func stringLike(expected, actual string) bool { return globMatch(expected, actual) } +func stringLike(expected, actual string) bool { return GlobMatch(expected, actual) } // numericCompare builds a comparator from a (actual, expected float64) -> // bool test, matching AWS's direction convention (the request's value is @@ -210,10 +230,26 @@ func boolMatch(expected, actual string) bool { return eerr == nil && aerr == nil && e == a } -func binaryMatch(expected, actual string) bool { - e, eerr := base64.StdEncoding.DecodeString(expected) - a, aerr := base64.StdEncoding.DecodeString(actual) - return eerr == nil && aerr == nil && bytes.Equal(e, a) +// normalizeIPOrCIDR appends a full-length prefix ("/32" or "/128") to s when +// it names a bare address rather than a CIDR range, so a single address and +// its equivalent /32 or /128 range are always handled the same way. +func normalizeIPOrCIDR(s string) string { + if strings.Contains(s, "/") { + return s + } + if ip := net.ParseIP(s); ip != nil && ip.To4() != nil { + return s + "/32" + } + return s + "/128" +} + +// ParseIPOrCIDR reports whether s is a valid IP address or CIDR range, for +// write-time validation of an IP-semantic condition key's value (e.g. AWS +// rejects PutBucketPolicy for a non-IP aws:SourceIp value with "Invalid IP +// address in Conditions", independent of which operator wraps it). +func ParseIPOrCIDR(s string) bool { + _, _, err := net.ParseCIDR(normalizeIPOrCIDR(s)) + return err == nil } // ipMatch reports whether actual (an address) falls within cidr (a CIDR @@ -221,15 +257,7 @@ func binaryMatch(expected, actual string) bool { // IpAddress/NotIpAddress condition operators. An unparseable operand on // either side never matches (fails closed) rather than erroring. func ipMatch(cidr, actual string) bool { - c := cidr - if !strings.Contains(c, "/") { - if ip := net.ParseIP(c); ip != nil && ip.To4() != nil { - c += "/32" - } else { - c += "/128" - } - } - _, network, err := net.ParseCIDR(c) + _, network, err := net.ParseCIDR(normalizeIPOrCIDR(cidr)) if err != nil { return false } @@ -237,63 +265,74 @@ func ipMatch(cidr, actual string) bool { return ip != nil && network.Contains(ip) } -// parsedOperator is a condition operator name decomposed into its set +// ParsedOperator is a condition operator name decomposed into its set // qualifier, base operator, and IfExists flag. -type parsedOperator struct { - qualifier conditionQualifier - base string - ifExists bool +type ParsedOperator struct { + Qualifier Qualifier + Base string + IfExists bool } -// parseOperatorName decomposes name (e.g. "ForAllValues:StringNotEqualsIfExists") -// into a parsedOperator, reporting ok=false if the base operator (after +// ParseOperatorName decomposes name (e.g. "ForAllValues:StringNotEqualsIfExists") +// into a ParsedOperator, reporting ok=false if the base operator (after // stripping a recognized qualifier prefix and IfExists suffix) isn't one // conditionRegistry recognizes, or is "Null" (Null has no IfExists variant - // "NullIfExists" is rejected here since after suffix-stripping "Null" isn't // itself in conditionRegistry). A bare "Null", optionally qualifier-prefixed, is accepted -func parseOperatorName(name string) (parsedOperator, bool) { +func ParseOperatorName(name string) (ParsedOperator, bool) { op := name - qualifier := qualifierNone + qualifier := QualifierNone switch { case strings.HasPrefix(op, "ForAllValues:"): - qualifier = qualifierForAllValues + qualifier = QualifierForAllValues op = strings.TrimPrefix(op, "ForAllValues:") case strings.HasPrefix(op, "ForAnyValue:"): - qualifier = qualifierForAnyValue + qualifier = QualifierForAnyValue op = strings.TrimPrefix(op, "ForAnyValue:") } if op == "Null" { - return parsedOperator{qualifier: qualifier, base: "Null"}, true + return ParsedOperator{Qualifier: qualifier, Base: "Null"}, true } base := strings.TrimSuffix(op, "IfExists") ifExists := base != op if _, ok := conditionRegistry[base]; !ok { - return parsedOperator{}, false + return ParsedOperator{}, false } - return parsedOperator{qualifier: qualifier, base: base, ifExists: ifExists}, true + return ParsedOperator{Qualifier: qualifier, Base: base, IfExists: ifExists}, true } -// conditionShapeValid checks raw (a statement's Condition block) against -// IAM's condition grammar for write-time validation: an object of operator -// -> (key -> value), where every operator name is recognized by -// parseOperatorName. An absent, null, or empty Condition is valid (matches -// evaluateCondition's "always matches" contract). -func conditionShapeValid(raw json.RawMessage) bool { +// Parse decodes raw (a statement's Condition block) into a Block, validating +// only its JSON shape and that every operator name is one ParseOperatorName +// recognizes - not condition key names, which are meaningful only to a +// specific policy type (IAM identity policies accept arbitrary custom/tag +// keys; S3 bucket policies validate against AWS's fixed key catalogue) and +// so are the caller's responsibility. An absent, null, or empty raw decodes +// to a nil Block with no error, matching Evaluate's "always matches" +// contract for a statement with no Condition at all. +func Parse(raw json.RawMessage) (Block, error) { if len(raw) == 0 || string(bytes.TrimSpace(raw)) == "null" { - return true + return nil, nil } - var block map[string]map[string]ConditionValues + var block Block if err := json.Unmarshal(raw, &block); err != nil { - return false + return nil, err } for operator := range block { - if _, ok := parseOperatorName(operator); !ok { - return false + if _, ok := ParseOperatorName(operator); !ok { + return nil, fmt.Errorf("policy: unrecognized condition operator %q", operator) } } - return true + return block, nil +} + +// ShapeValid reports whether raw (a statement's Condition block) satisfies +// Parse without error - write-time validation of the condition grammar +// alone (operator names), with no opinion on condition keys. +func ShapeValid(raw json.RawMessage) bool { + _, err := Parse(raw) + return err == nil } // conditionVariableOperators is the subset of conditionRegistry that AWS @@ -316,48 +355,39 @@ var conditionVariableOperators = map[string]bool{ "ArnNotLike": true, } -// evaluateCondition evaluates a policy statement's Condition block against -// ctxVars - a ":" keyed context for trust-policy -// evaluation, or an "aws:" keyed context for identity-policy -// evaluation. An absent or empty Condition always matches. version is the -// enclosing document's Version element: a ${...} policy variable in a -// Condition value is only ever substituted when version is exactly -// Version2012 AND the operator is one of conditionVariableOperators - -// AWS requires the 2012-10-17 policy version to use variables at all, and -// never expands them for Numeric/Date/Bool/Binary/IP/Null operators even -// then. A variable that doesn't qualify is left as literal text, the -// same fallback used for an absent/multivalued context key - so it simply -// won't match a real condition value, rather than silently expanding into -// something AWS itself wouldn't. +// Evaluate evaluates a policy statement's Condition block against ctxVars - +// context-key values keyed however the caller's policy type documents them +// (e.g. "aws:" for IAM identity/S3 bucket policies, +// ":" for trust-policy evaluation). An absent or empty +// Condition always matches. version is the enclosing document's Version +// element: a ${...} policy variable in a Condition value is only ever +// substituted when version is exactly "2012-10-17" AND the operator is one +// of conditionVariableOperators - AWS requires the 2012-10-17 policy version +// to use variables at all, and never expands them for +// Numeric/Date/Bool/Binary/IP/Null operators even then. A variable that +// doesn't qualify is left as literal text, the same fallback used for an +// absent/multivalued context key - so it simply won't match a real +// condition value, rather than silently expanding into something AWS itself +// wouldn't. // // matched reports whether the condition holds; ok reports whether it could -// be evaluated at all. ok is false only for a Condition block whose JSON -// shape or operator name conditionShapeValid would already reject - i.e. -// only for a document stored before that write-time validation existed, or -// containing a future operator this package doesn't yet recognize. Callers -// MUST treat ok=false as "cannot rule out a hidden Deny" and deny the whole -// evaluation, never as a non-match - see EvaluateIdentityPolicies and -// EvaluateWebIdentityTrust. -func evaluateCondition(raw json.RawMessage, ctxVars map[string][]string, version string) (matched bool, ok bool) { - if len(raw) == 0 || string(bytes.TrimSpace(raw)) == "null" { - return true, true - } - - var block map[string]map[string]ConditionValues - if err := json.Unmarshal(raw, &block); err != nil { +// be evaluated at all. ok is false only for a Condition block Parse would +// already reject - i.e. only for a document stored before write-time +// validation existed, or containing a future operator this package doesn't +// yet recognize. Callers MUST treat ok=false as "cannot rule out a hidden +// Deny" and deny the whole evaluation, never as a non-match. +func Evaluate(raw json.RawMessage, ctxVars map[string][]string, version string) (matched bool, ok bool) { + block, err := Parse(raw) + if err != nil { debuglogger.Logf("policy condition block failed to parse: %v", err) return false, false } for operator, kvs := range block { - op, recognized := parseOperatorName(operator) - if !recognized { - debuglogger.Logf("policy condition: unrecognized operator %q", operator) - return false, false - } + op, _ := ParseOperatorName(operator) // Parse already validated every operator name for key, expected := range kvs { actual, present := lookupContextValues(ctxVars, key) - if version == Version2012 && conditionVariableOperators[op.base] { + if version == version2012 && conditionVariableOperators[op.Base] { expected = substituteConditionValues(expected, ctxVars) } if !evaluateConditionKey(op, expected, actual, present) { @@ -368,6 +398,12 @@ func evaluateCondition(raw json.RawMessage, ctxVars map[string][]string, version return true, true } +// version2012 is AWS's "2012-10-17" policy-document version string, the +// only one that enables ${...} policy-variable substitution. Duplicated +// here (rather than imported) since this package has no dependency on any +// specific policy type's Version constants. +const version2012 = "2012-10-17" + // lookupContextValues retrieves ctxVars[key], matching key // case-insensitively: AWS documents condition (and policy-variable) key // *names* as case-insensitive - "aws:SourceIp" and "AWS:SOURCEIP" name the @@ -390,14 +426,14 @@ func lookupContextValues(ctxVars map[string][]string, key string) ([]string, boo // placeholder, e.g. "${aws:username}". var policyVariablePattern = regexp.MustCompile(`\$\{([A-Za-z0-9_:.\-]+)\}`) -// substitutePolicyVariables replaces every ${key} placeholder in s with the +// SubstitutePolicyVariables replaces every ${key} placeholder in s with the // single value ctxVars holds for key, looked up the same case-insensitive // way as a Condition key. AWS only allows a single-valued context key to be // used as a policy variable; a placeholder naming an absent or multivalued // key is left as literal text, same as any other substring - so it simply // won't match a real resource ARN or condition value, rather than being // silently dropped and turning a Deny that relies on it into a no-op. -func substitutePolicyVariables(s string, ctxVars map[string][]string) string { +func SubstitutePolicyVariables(s string, ctxVars map[string][]string) string { if !strings.Contains(s, "${") { return s } @@ -411,14 +447,14 @@ func substitutePolicyVariables(s string, ctxVars map[string][]string) string { }) } -// substituteConditionValues applies substitutePolicyVariables to every +// substituteConditionValues applies SubstitutePolicyVariables to every // element of values, so e.g. a Condition of // {"StringEquals":{"iam:ResourceTag/owner":"${aws:username}"}} compares // against the requester's own username rather than the literal text. -func substituteConditionValues(values ConditionValues, ctxVars map[string][]string) ConditionValues { - out := make(ConditionValues, len(values)) +func substituteConditionValues(values Values, ctxVars map[string][]string) Values { + out := make(Values, len(values)) for i, v := range values { - out[i] = substitutePolicyVariables(v, ctxVars) + out[i] = SubstitutePolicyVariables(v, ctxVars) } return out } @@ -426,25 +462,25 @@ func substituteConditionValues(values ConditionValues, ctxVars map[string][]stri // evaluateConditionKey evaluates one operator/key pair of an already // -parsed Condition block against actual (ctxVars[key]) and present // (whether key was in ctxVars at all). -func evaluateConditionKey(op parsedOperator, expected ConditionValues, actual []string, present bool) bool { - if op.base == "Null" { +func evaluateConditionKey(op ParsedOperator, expected Values, actual []string, present bool) bool { + if op.Base == "Null" { return evaluateNull(expected, present) } - entry := conditionRegistry[op.base] // guaranteed present - parseOperatorName already validated op.base + entry := conditionRegistry[op.Base] // guaranteed present - ParseOperatorName already validated op.Base - if op.qualifier == qualifierForAllValues && !present { + if op.Qualifier == QualifierForAllValues && !present { return true } if entry.negate { if !present { return true } - return aggregate(op.qualifier, true, expected, actual, entry.compare) + return aggregate(op.Qualifier, true, expected, actual, entry.compare) } if !present { - return op.ifExists + return op.IfExists } - return aggregate(op.qualifier, false, expected, actual, entry.compare) + return aggregate(op.Qualifier, false, expected, actual, entry.compare) } // evaluateNull implements the Null condition operator: true if expected @@ -452,7 +488,7 @@ func evaluateConditionKey(op parsedOperator, expected ConditionValues, actual [] // must be absent ("true") and it is, or must be present ("false") and it // is. A value that's neither "true" nor "false" never satisfies the // condition (fails closed) -func evaluateNull(expected ConditionValues, present bool) bool { +func evaluateNull(expected Values, present bool) bool { for _, e := range expected { switch { case strings.EqualFold(e, "true"): @@ -472,7 +508,7 @@ func evaluateNull(expected ConditionValues, present bool) bool { // under qualifier's multivalued-context-key semantics. negate selects the // Not-operator family, sharing the same per-pair comparator as its positive // counterpart (see conditionRegistry). -func aggregate(qualifier conditionQualifier, negate bool, expected ConditionValues, actual []string, cmp conditionComparator) bool { +func aggregate(qualifier Qualifier, negate bool, expected Values, actual []string, cmp conditionComparator) bool { matchesAny := func(a string) bool { for _, e := range expected { if cmp(e, a) { @@ -482,7 +518,7 @@ func aggregate(qualifier conditionQualifier, negate bool, expected ConditionValu return false } - useForAll := qualifier == qualifierForAllValues || (qualifier == qualifierNone && negate) + useForAll := qualifier == QualifierForAllValues || (qualifier == QualifierNone && negate) if useForAll { for _, a := range actual { if ok := matchesAny(a); ok == negate { @@ -498,3 +534,32 @@ func aggregate(qualifier conditionQualifier, negate bool, expected ConditionValu } return false // vacuously false over an empty/absent actual } + +// GlobMatch implements the small wildcard grammar IAM Action/Resource/Arn +// patterns use: '*' matches any run of characters (including none), '?' +// matches exactly one character, everything else matches literally. +func GlobMatch(pattern, s string) bool { + var pi, si, star, match int + star = -1 + for si < len(s) { + switch { + case pi < len(pattern) && (pattern[pi] == '?' || pattern[pi] == s[si]): + pi++ + si++ + case pi < len(pattern) && pattern[pi] == '*': + star = pi + match = si + pi++ + case star != -1: + pi = star + 1 + match++ + si = match + default: + return false + } + } + for pi < len(pattern) && pattern[pi] == '*' { + pi++ + } + return pi == len(pattern) +} diff --git a/iamapi/policy/condition_test.go b/internal/condition/condition_test.go similarity index 90% rename from iamapi/policy/condition_test.go rename to internal/condition/condition_test.go index 121d4839..72c161c3 100644 --- a/iamapi/policy/condition_test.go +++ b/internal/condition/condition_test.go @@ -12,25 +12,31 @@ // specific language governing permissions and limitations // under the License. -package policy +package condition import ( "reflect" "testing" ) +// testVersion2012 is the "2012-10-17" policy-document version string, +// duplicated here rather than exported from the package (it's meaningful +// only to a caller's own Version type, e.g. iamapi/policy.Version2012 or +// auth.PolicyVersion2012). +const testVersion2012 = "2012-10-17" + // evalCondTest is the shared table shape for every TestEvaluateCondition* -// function below. wantErr means "evaluateCondition's ok return should be -// false" (the block's shape or an operator name couldn't be recognized) - -// distinct from want=false, which means the condition was evaluated fine -// but didn't match. +// function below. wantErr means "Evaluate's ok return should be false" (the +// block's shape or an operator name couldn't be recognized) - distinct from +// want=false, which means the condition was evaluated fine but didn't +// match. type evalCondTest struct { name string raw string ctxVars map[string][]string // version is the enclosing document's Version element: a Condition // value's ${...} policy variable is only ever substituted - // when this is exactly Version2012. Left "" (no Version) for every + // when this is exactly testVersion2012. Left "" (no Version) for every // existing case except the ones specifically testing substitution. version string want bool @@ -41,13 +47,13 @@ func runEvalCondTests(t *testing.T, tests []evalCondTest) { t.Helper() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - matched, ok := evaluateCondition([]byte(tt.raw), tt.ctxVars, tt.version) + matched, ok := Evaluate([]byte(tt.raw), tt.ctxVars, tt.version) wantOk := !tt.wantErr if ok != wantOk { - t.Fatalf("evaluateCondition() ok = %v, want %v", ok, wantOk) + t.Fatalf("Evaluate() ok = %v, want %v", ok, wantOk) } if ok && matched != tt.want { - t.Errorf("evaluateCondition() matched = %v, want %v", matched, tt.want) + t.Errorf("Evaluate() matched = %v, want %v", matched, tt.want) } }) } @@ -224,14 +230,14 @@ func TestEvaluateCondition(t *testing.T) { name: "policy variable in condition value is substituted under version 2012-10-17", raw: `{"StringEquals":{"iam:ResourceTag/owner":"${aws:username}"}}`, ctxVars: map[string][]string{"aws:username": {"alice"}, "iam:ResourceTag/owner": {"alice"}}, - version: Version2012, + version: testVersion2012, want: true, }, { name: "policy variable naming an absent key is left literal and so fails to match", raw: `{"StringEquals":{"iam:ResourceTag/owner":"${aws:nonexistent}"}}`, ctxVars: map[string][]string{"iam:ResourceTag/owner": {"alice"}}, - version: Version2012, + version: testVersion2012, want: false, }, { @@ -251,7 +257,7 @@ func TestEvaluateCondition(t *testing.T) { name: "policy variable is not substituted inside NumericEquals even under version 2012-10-17", raw: `{"NumericEquals":{"aws:EpochTime":"${aws:EpochTime}"}}`, ctxVars: map[string][]string{"aws:EpochTime": {"1700000000"}}, - version: Version2012, + version: testVersion2012, want: false, }, }) @@ -467,9 +473,9 @@ func TestEvaluateConditionBinary(t *testing.T) { want: false, }, { - name: "BinaryEquals invalid base64 fails closed, not an error", + name: "BinaryEquals decoded raw bytes do not match the base64 policy value", raw: `{"BinaryEquals":{"example.com:token":"aGVsbG8="}}`, - ctxVars: map[string][]string{"example.com:token": {"not-valid-base64!!"}}, + ctxVars: map[string][]string{"example.com:token": {"hello"}}, want: false, }, }) @@ -656,20 +662,20 @@ func TestEvaluateConditionQualifiers(t *testing.T) { }) } -func TestConditionValuesUnmarshalJSON(t *testing.T) { +func TestValuesUnmarshalJSON(t *testing.T) { tests := []struct { name string json string - want ConditionValues + want Values wantErr bool }{ - {"string", `"alice"`, ConditionValues{"alice"}, false}, - {"integer number, unquoted", `5`, ConditionValues{"5"}, false}, - {"decimal number preserves literal text", `5.50`, ConditionValues{"5.50"}, false}, - {"bool true", `true`, ConditionValues{"true"}, false}, - {"bool false", `false`, ConditionValues{"false"}, false}, - {"array of strings", `["a","b"]`, ConditionValues{"a", "b"}, false}, - {"array mixing string/number/bool", `["a",5,true]`, ConditionValues{"a", "5", "true"}, false}, + {"string", `"alice"`, Values{"alice"}, false}, + {"integer number, unquoted", `5`, Values{"5"}, false}, + {"decimal number preserves literal text", `5.50`, Values{"5.50"}, false}, + {"bool true", `true`, Values{"true"}, false}, + {"bool false", `false`, Values{"false"}, false}, + {"array of strings", `["a","b"]`, Values{"a", "b"}, false}, + {"array mixing string/number/bool", `["a",5,true]`, Values{"a", "5", "true"}, false}, {"null is rejected", `null`, nil, true}, {"null array element is rejected", `["a",null]`, nil, true}, {"nested array element is rejected", `[["a"]]`, nil, true}, @@ -678,7 +684,7 @@ func TestConditionValuesUnmarshalJSON(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var got ConditionValues + var got Values err := got.UnmarshalJSON([]byte(tt.json)) if tt.wantErr { if err == nil { @@ -703,7 +709,7 @@ func TestParseOperatorName(t *testing.T) { wantOk bool wantBase string wantIfExists bool - wantQualif conditionQualifier + wantQualif Qualifier }{ {name: "StringEquals", op: "StringEquals", wantOk: true, wantBase: "StringEquals"}, {name: "StringEqualsIfExists", op: "StringEqualsIfExists", wantOk: true, wantBase: "StringEquals", wantIfExists: true}, @@ -715,9 +721,9 @@ func TestParseOperatorName(t *testing.T) { {name: "ArnLike", op: "ArnLike", wantOk: true, wantBase: "ArnLike"}, {name: "IpAddress", op: "IpAddress", wantOk: true, wantBase: "IpAddress"}, {name: "Null", op: "Null", wantOk: true, wantBase: "Null"}, - {name: "ForAllValues:StringEquals", op: "ForAllValues:StringEquals", wantOk: true, wantBase: "StringEquals", wantQualif: qualifierForAllValues}, - {name: "ForAnyValue:StringNotEqualsIfExists", op: "ForAnyValue:StringNotEqualsIfExists", wantOk: true, wantBase: "StringNotEquals", wantIfExists: true, wantQualif: qualifierForAnyValue}, - {name: "ForAllValues:Null accepted, qualifier is a no-op", op: "ForAllValues:Null", wantOk: true, wantBase: "Null", wantQualif: qualifierForAllValues}, + {name: "ForAllValues:StringEquals", op: "ForAllValues:StringEquals", wantOk: true, wantBase: "StringEquals", wantQualif: QualifierForAllValues}, + {name: "ForAnyValue:StringNotEqualsIfExists", op: "ForAnyValue:StringNotEqualsIfExists", wantOk: true, wantBase: "StringNotEquals", wantIfExists: true, wantQualif: QualifierForAnyValue}, + {name: "ForAllValues:Null accepted, qualifier is a no-op", op: "ForAllValues:Null", wantOk: true, wantBase: "Null", wantQualif: QualifierForAllValues}, {name: "NullIfExists rejected", op: "NullIfExists", wantOk: false}, {name: "unrecognized base", op: "FooBarOperator", wantOk: false}, {name: "unrecognized qualifier prefix left as part of the name", op: "ForSomeValues:StringEquals", wantOk: false}, @@ -726,15 +732,15 @@ func TestParseOperatorName(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, ok := parseOperatorName(tt.op) + got, ok := ParseOperatorName(tt.op) if ok != tt.wantOk { - t.Fatalf("parseOperatorName(%q) ok = %v, want %v", tt.op, ok, tt.wantOk) + t.Fatalf("ParseOperatorName(%q) ok = %v, want %v", tt.op, ok, tt.wantOk) } if !ok { return } - if got.base != tt.wantBase || got.ifExists != tt.wantIfExists || got.qualifier != tt.wantQualif { - t.Fatalf("parseOperatorName(%q) = %+v, want {base:%q ifExists:%v qualifier:%v}", tt.op, got, tt.wantBase, tt.wantIfExists, tt.wantQualif) + if got.Base != tt.wantBase || got.IfExists != tt.wantIfExists || got.Qualifier != tt.wantQualif { + t.Fatalf("ParseOperatorName(%q) = %+v, want {base:%q ifExists:%v qualifier:%v}", tt.op, got, tt.wantBase, tt.wantIfExists, tt.wantQualif) } }) } @@ -754,8 +760,29 @@ func TestGlobMatch(t *testing.T) { {pattern: "exact", s: "exacts", want: false}, } for _, tt := range tests { - if got := globMatch(tt.pattern, tt.s); got != tt.want { - t.Errorf("globMatch(%q, %q) = %v, want %v", tt.pattern, tt.s, got, tt.want) + if got := GlobMatch(tt.pattern, tt.s); got != tt.want { + t.Errorf("GlobMatch(%q, %q) = %v, want %v", tt.pattern, tt.s, got, tt.want) } } } + +func TestParseIPOrCIDR(t *testing.T) { + tests := []struct { + name string + s string + want bool + }{ + {"bare IPv4 valid", "203.0.113.5", true}, + {"CIDR valid", "10.0.0.0/8", true}, + {"bare IPv6 valid", "::1", true}, + {"garbage", "not-an-ip", false}, + {"empty", "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ParseIPOrCIDR(tt.s); got != tt.want { + t.Errorf("ParseIPOrCIDR(%q) = %v, want %v", tt.s, got, tt.want) + } + }) + } +} diff --git a/internal/httpctx/context_keys.go b/internal/httpctx/context_keys.go index 5d9fa59e..615eb15e 100644 --- a/internal/httpctx/context_keys.go +++ b/internal/httpctx/context_keys.go @@ -14,7 +14,9 @@ package httpctx -import "github.com/gofiber/fiber/v3" +import ( + "github.com/gofiber/fiber/v3" +) // ContextKey names a request-local value stored in fiber.Ctx locals. type ContextKey string @@ -38,6 +40,7 @@ const ( ContextKeyHostID ContextKey = "host-id" ContextKeyWebsiteConfig ContextKey = "website-config" ContextKeyCallerIdentity ContextKey = "iam-caller-identity" + ContextKeyOriginalURIPath ContextKey = "original-uri-path" ) func (ck ContextKey) Set(ctx fiber.Ctx, val any) { diff --git a/internal/netutil/clientcert.go b/internal/netutil/clientcert.go new file mode 100644 index 00000000..3017c0e1 --- /dev/null +++ b/internal/netutil/clientcert.go @@ -0,0 +1,49 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package netutil + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "os" +) + +// LoadClientCert loads a client certificate/key pair for presenting on an +// outbound mTLS connection +func LoadClientCert(certFile, keyFile string) (tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return tls.Certificate{}, fmt.Errorf("load client certificate: %w", err) + } + return cert, nil +} + +// LoadCACertPool loads a PEM-encoded CA bundle for verifying a peer's +// certificate on an outbound connection (the server's cert, from a +// client's perspective) or, on an inbound mTLS listener, a connecting +// client's certificate. +func LoadCACertPool(caFile string) (*x509.CertPool, error) { + pemBytes, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("read CA certificate %q: %w", caFile, err) + } + + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pemBytes) { + return nil, fmt.Errorf("no valid certificates found in %q", caFile) + } + + return pool, nil +} diff --git a/internal/netutil/multi_listener.go b/internal/netutil/multi_listener.go index 7affd504..5a3ec444 100644 --- a/internal/netutil/multi_listener.go +++ b/internal/netutil/multi_listener.go @@ -16,6 +16,7 @@ package netutil import ( "crypto/tls" + "crypto/x509" "errors" "fmt" "net" @@ -270,10 +271,38 @@ func NewMultiAddrListener(network, address string, opts ListenerOptions) (net.Li return NewMultiListener(listeners...), nil } +// TLSOptions configures the server-side tls.Config for +// NewMultiAddrTLSListenerWithOptions. A non-nil ClientCAs enables mTLS: +// inbound connections must present a certificate verified against that +// pool +type TLSOptions struct { + GetCertificate func(*tls.ClientHelloInfo) (*tls.Certificate, error) + ClientCAs *x509.CertPool + RequireClientCert bool +} + +// NewMultiAddrTLSListener creates TLS listeners for all IP addresses that the +// hostname in the address resolves to. Similar to NewMultiAddrListener but with TLS. func NewMultiAddrTLSListener(network, address string, getCertificateFunc func(*tls.ClientHelloInfo) (*tls.Certificate, error), opts ListenerOptions) (net.Listener, error) { + return NewMultiAddrTLSListenerWithOptions(network, address, TLSOptions{GetCertificate: getCertificateFunc}, opts) +} + +// NewMultiAddrTLSListenerWithOptions is NewMultiAddrTLSListener with control +// over client-certificate verification (mTLS), for listeners — such as the +// standalone IAM service's private endpoints — that must authenticate the +// connecting client, not just the server. +func NewMultiAddrTLSListenerWithOptions(network, address string, tlsOpts TLSOptions, opts ListenerOptions) (net.Listener, error) { config := &tls.Config{ MinVersion: tls.VersionTLS12, - GetCertificate: getCertificateFunc, + GetCertificate: tlsOpts.GetCertificate, + } + if tlsOpts.ClientCAs != nil { + config.ClientCAs = tlsOpts.ClientCAs + if tlsOpts.RequireClientCert { + config.ClientAuth = tls.RequireAndVerifyClientCert + } else { + config.ClientAuth = tls.VerifyClientCertIfGiven + } } if IsUnixSocketPath(address) { @@ -314,3 +343,18 @@ func NewMultiAddrTLSListener(network, address string, getCertificateFunc func(*t return NewMultiListener(listeners...), nil } + +// RequireSecureTransport enforces mTLS or unix socket a unix socket +// is always acceptable (the filesystem is the trust boundary), +// but a TCP address is only acceptable when mTLS (a server cert plus +// mandatory client-certificate verification) is actually configured for +// it +func RequireSecureTransport(address string, hasMTLS bool) error { + if IsUnixSocketPath(address) { + return nil + } + if !hasMTLS { + return fmt.Errorf("private listener %q requires either a unix socket path or mTLS (server cert + client CA); refusing to serve on plain TCP", address) + } + return nil +} diff --git a/s3api/utils/multi_listener_test.go b/internal/netutil/multi_listener_full_test.go similarity index 97% rename from s3api/utils/multi_listener_test.go rename to internal/netutil/multi_listener_full_test.go index 6e7b000b..dae68766 100644 --- a/s3api/utils/multi_listener_test.go +++ b/internal/netutil/multi_listener_full_test.go @@ -12,7 +12,7 @@ // specific language governing permissions and limitations // under the License. -package utils +package netutil import ( "crypto/tls" @@ -349,7 +349,7 @@ func TestNewMultiAddrListener(t *testing.T) { func TestNewMultiAddrTLSListener(t *testing.T) { // Create a simple test certificate getCertFunc := func(*tls.ClientHelloInfo) (*tls.Certificate, error) { - cert, err := tls.X509KeyPair([]byte(testCert), []byte(testKey)) + cert, err := tls.X509KeyPair([]byte(multiListenerTestCert), []byte(multiListenerTestKey)) return &cert, err } @@ -388,7 +388,7 @@ func TestNewMultiAddrTLSListener(t *testing.T) { } // Test certificate and key for TLS tests -const testCert = `-----BEGIN CERTIFICATE----- +const multiListenerTestCert = `-----BEGIN CERTIFICATE----- MIIBhTCCASugAwIBAgIQIRi6zePL6mKjOipn+dNuaTAKBggqhkjOPQQDAjASMRAw DgYDVQQKEwdBY21lIENvMB4XDTE3MTAyMDE5NDMwNloXDTE4MTAyMDE5NDMwNlow EjEQMA4GA1UEChMHQWNtZSBDbzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABD0d @@ -400,7 +400,7 @@ Wf86aX6PepsntZv2GYlA5UpabfT2EZICICpJ5h/iI+i341gBmLiAFQOyTDT+/wQc 6MF9+Yw1Yy0t -----END CERTIFICATE-----` -const testKey = `-----BEGIN EC PRIVATE KEY----- +const multiListenerTestKey = `-----BEGIN EC PRIVATE KEY----- MHcCAQEEIIrYSSNQFaA2Hwf1duRSxKtLYX5CB04fSeQ6tF1aY/PuoAoGCCqGSM49 AwEHoUQDQgAEPR3tU2Fta9ktY+6P9G0cWO+0kETA6SFs38GecTyudlHz6xvCdz8q EKTcWGekdmdDPsHloRNtsiCa697B2O9IFA== diff --git a/internal/netutil/multi_listener_test.go b/internal/netutil/multi_listener_test.go new file mode 100644 index 00000000..cc9b2952 --- /dev/null +++ b/internal/netutil/multi_listener_test.go @@ -0,0 +1,200 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package netutil + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "net" + "testing" + "time" +) + +func TestRequireSecureTransport(t *testing.T) { + tests := []struct { + name string + address string + hasMTLS bool + wantErr bool + }{ + {name: "unix socket without mTLS is fine", address: "/tmp/private.sock", hasMTLS: false, wantErr: false}, + {name: "unix socket with mTLS is fine", address: "/tmp/private.sock", hasMTLS: true, wantErr: false}, + {name: "abstract socket without mTLS is fine", address: "@private", hasMTLS: false, wantErr: false}, + {name: "TCP without mTLS is rejected", address: "127.0.0.1:9443", hasMTLS: false, wantErr: true}, + {name: "TCP with mTLS is fine", address: "127.0.0.1:9443", hasMTLS: true, wantErr: false}, + {name: "bare port without mTLS is rejected", address: ":9443", hasMTLS: false, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := RequireSecureTransport(tt.address, tt.hasMTLS) + if (err != nil) != tt.wantErr { + t.Errorf("RequireSecureTransport(%q, %v) error = %v, wantErr %v", tt.address, tt.hasMTLS, err, tt.wantErr) + } + }) + } +} + +// TestMTLSListenerRejectsClientWithoutCert confirms a listener built with +// TLSOptions{ClientCAs, RequireClientCert: true} — the shape the private +// IAM endpoints use — refuses a TLS handshake from a client presenting no +// certificate, and accepts one presenting a cert signed by the configured +// CA. This is the core security boundary the whole standalone-IAM-service +// design leans on ("otherwise this endpoint should not serve anything"), +// so it's worth verifying the handshake itself, not just the config shape. +func TestMTLSListenerRejectsClientWithoutCert(t *testing.T) { + ca := generateTestCA(t) + serverCert := issueTestCert(t, ca, "server") + clientCert := issueTestCert(t, ca, "client") + + caPool := x509.NewCertPool() + caPool.AddCert(ca.cert) + + addr := "127.0.0.1:0" + ln, err := net.Listen("tcp", addr) + if err != nil { + t.Fatalf("listen: %v", err) + } + // Pinned to TLS 1.2: TLS 1.3 clients can report Dial as successful before + // observing the server's post-handshake rejection alert for a missing + // client cert, making that half of this test flaky. TLS 1.2 client-cert + // verification is synchronous within the initial handshake flight. + tlsLn := tls.NewListener(ln, &tls.Config{ + MinVersion: tls.VersionTLS12, + MaxVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{serverCert}, + ClientCAs: caPool, + ClientAuth: tls.RequireAndVerifyClientCert, + }) + defer tlsLn.Close() + + serverErrCh := make(chan error, 1) + go func() { + conn, err := tlsLn.Accept() + if err != nil { + serverErrCh <- err + return + } + defer conn.Close() + serverErrCh <- conn.(*tls.Conn).Handshake() + }() + + t.Run("no client cert is rejected", func(t *testing.T) { + conn, err := tls.Dial("tcp", tlsLn.Addr().String(), &tls.Config{ + RootCAs: caPool, + MaxVersion: tls.VersionTLS12, + ServerName: "server", + }) + if err == nil { + conn.Close() + t.Fatal("expected handshake to fail without a client certificate") + } + <-serverErrCh + }) + + go func() { + conn, err := tlsLn.Accept() + if err != nil { + serverErrCh <- err + return + } + defer conn.Close() + serverErrCh <- conn.(*tls.Conn).Handshake() + }() + + t.Run("valid client cert is accepted", func(t *testing.T) { + conn, err := tls.Dial("tcp", tlsLn.Addr().String(), &tls.Config{ + RootCAs: caPool, + MaxVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{clientCert}, + ServerName: "server", + }) + if err != nil { + t.Fatalf("expected handshake to succeed with a valid client certificate: %v", err) + } + conn.Close() + if err := <-serverErrCh; err != nil { + t.Fatalf("server-side handshake failed: %v", err) + } + }) +} + +type testCA struct { + cert *x509.Certificate + key *ecdsa.PrivateKey +} + +func generateTestCA(t *testing.T) testCA { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate CA key: %v", err) + } + + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + } + + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatalf("create CA cert: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parse CA cert: %v", err) + } + + return testCA{cert: cert, key: key} +} + +func issueTestCert(t *testing.T, ca testCA, cn string) tls.Certificate { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate %s key: %v", cn, err) + } + + template := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: cn}, + DNSNames: []string{cn}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + } + + der, err := x509.CreateCertificate(rand.Reader, template, ca.cert, &key.PublicKey, ca.key) + if err != nil { + t.Fatalf("create %s cert: %v", cn, err) + } + + return tls.Certificate{ + Certificate: [][]byte{der}, + PrivateKey: key, + } +} diff --git a/internal/sigv4auth/auth.go b/internal/sigv4auth/auth.go index 92dd4120..e57a16d9 100644 --- a/internal/sigv4auth/auth.go +++ b/internal/sigv4auth/auth.go @@ -35,8 +35,19 @@ const ( // HeaderSecurityToken is the header a temporary credential's // SessionToken is presented in, matching AWS's X-Amz-Security-Token. HeaderSecurityToken = "X-Amz-Security-Token" + + // TempAccessKeyIDPrefix marks temporary credentials minted by + // AssumeRoleWithWebIdentity, matching AWS's ASIA… convention that + // distinguishes them from long-term AKIA… access keys. + TempAccessKeyIDPrefix = "ASIA" ) +// IsTempAccessKeyID reports whether accessKeyID is a temporary (session) +// credential rather than a long-term AKIA… access key. +func IsTempAccessKeyID(accessKeyID string) bool { + return strings.HasPrefix(accessKeyID, TempAccessKeyIDPrefix) +} + type ParseErrorKind string const ( diff --git a/internal/sigv4auth/canonical.go b/internal/sigv4auth/canonical.go new file mode 100644 index 00000000..835704d8 --- /dev/null +++ b/internal/sigv4auth/canonical.go @@ -0,0 +1,376 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sigv4auth + +import ( + "crypto/sha256" + "encoding/hex" + "net/http" + "net/url" + "slices" + "sort" + "strconv" + "strings" + "time" +) + +const ( + amzAlgorithmKey = "X-Amz-Algorithm" + amzDateKey = "X-Amz-Date" + amzCredentialKey = "X-Amz-Credential" + amzSignedHeadersKey = "X-Amz-SignedHeaders" + + authorizationHeader = "Authorization" + + hostHeader = "host" + contentLengthHeader = "content-length" +) + +// BuildCredentialScope builds the "yyyymmdd/region/service/aws4_request" +// scope string shared by the credential, the string-to-sign, and (as the +// derivation input) DeriveKey. +func BuildCredentialScope(yyyymmdd, region, service string) string { + return strings.Join([]string{yyyymmdd, region, service, Terminal}, "/") +} + +// SigningInput is everything BuildAndSign needs to reproduce a request's +// SigV4 canonical request, string-to-sign, and signature. +type SigningInput struct { + Method string + Host string + URIPath string // raw/escaped request path; "" is treated as "/" + Query url.Values + Header http.Header + ContentLength int64 + AccessKeyID string + CredentialScope string // BuildCredentialScope(yyyymmdd, region, service) + SignedHdrs []string + PayloadHash string + SigningTime time.Time + DisableURIPathEscaping bool + IsPreSign bool +} + +// SignResult carries everything BuildAndSign computed: the canonical +// request and string-to-sign (for a caller to compare against a +// client-presented signature), the signature itself, and — since +// BuildAndSign never mutates its input — every value that needs to land +// back on the wire for an outbound request to actually be signed. +// AmzDate is the formatted X-Amz-Date value used in canonicalization; +// for header auth (!IsPreSign) a caller signing a real outbound request +// must set it as a header itself (req.Header.Set("X-Amz-Date", ...)) — +// unlike a verification caller, which only ever compares Signature and +// never sends the request anywhere. RawQuery is the sorted, re-encoded +// query string in both modes when SigningInput.IsPreSign, it additionally +// carries the appended "&X-Amz-Signature=..." that makes it the final +// presigned query string — for presign, X-Amz-Date already went into +// RawQuery, so AmzDate itself needs no separate application. +// AuthorizationHeader (header-auth's full Authorization value) is only +// populated when !IsPreSign. +type SignResult struct { + SignedHeaders http.Header + CanonicalString string + StringToSign string + Signature string + AmzDate string + RawQuery string + AuthorizationHeader string +} + +// BuildAndSign rebuilds the canonical request for in the same way a SigV4 +// client would have, and signs it with derivedKey — the kSigning value +// DeriveKey computes from a secret, or the equivalent value a standalone +// IAM service returns without ever revealing that secret. It never mutates +// in.Query or in.Header: verifying a request's signature must not corrupt +// the real inbound query/headers a caller still needs afterward, and +// signing a real outbound request works just as well by having the caller +// apply SignResult's output themselves. The caller compares Signature +// against the one presented on the original request; a match proves the +// request was signed by whoever holds the secret DeriveKey (or the remote +// IAM service) derived derivedKey from. +func BuildAndSign(derivedKey []byte, in SigningInput) *SignResult { + query := cloneQuery(in.Query) + headers := cloneHeader(in.Header) + + amzDate := in.SigningTime.Format(ISO8601Format) + setRequiredSigningFields(headers, query, in.IsPreSign, amzDate) + + for key := range query { + sort.Strings(query[key]) + } + + credentialStr := in.AccessKeyID + "/" + in.CredentialScope + if in.IsPreSign { + query.Set(amzCredentialKey, credentialStr) + } + + unsignedHeaders := headers + if in.IsPreSign { + var hoisted url.Values + hoisted, unsignedHeaders = hoistHeadersToQuery(headers) + for k := range hoisted { + query[k] = hoisted[k] + } + } + + signedHeaders, signedHeadersStr, canonicalHeaderStr := buildCanonicalHeaders(in.Host, in.SignedHdrs, unsignedHeaders, in.ContentLength) + + if in.IsPreSign { + query.Set(amzSignedHeadersKey, signedHeadersStr) + } + + var rawQuery strings.Builder + rawQuery.WriteString(strings.ReplaceAll(query.Encode(), "+", "%20")) + + canonicalURI := in.URIPath + if canonicalURI == "" { + canonicalURI = "/" + } + if !in.DisableURIPathEscaping { + canonicalURI = escapePath(canonicalURI, false) + } + + canonicalString := buildCanonicalString(in.Method, canonicalURI, rawQuery.String(), signedHeadersStr, canonicalHeaderStr, in.PayloadHash) + strToSign := buildStringToSign(in.SigningTime, in.CredentialScope, canonicalString) + signature := hex.EncodeToString(hmacSHA256(derivedKey, []byte(strToSign))) + + result := &SignResult{ + SignedHeaders: signedHeaders, + CanonicalString: canonicalString, + StringToSign: strToSign, + Signature: signature, + AmzDate: amzDate, + RawQuery: rawQuery.String(), + } + + if in.IsPreSign { + result.RawQuery += "&X-Amz-Signature=" + signature + } else { + result.AuthorizationHeader = buildAuthorizationHeader(credentialStr, signedHeadersStr, signature) + } + + return result +} + +func cloneQuery(q url.Values) url.Values { + clone := make(url.Values, len(q)) + for k, v := range q { + clone[k] = append([]string(nil), v...) + } + return clone +} + +func cloneHeader(h http.Header) http.Header { + clone := make(http.Header, len(h)) + for k, v := range h { + clone[k] = append([]string(nil), v...) + } + return clone +} + +func setRequiredSigningFields(headers http.Header, query url.Values, isPreSign bool, amzDate string) { + if isPreSign { + query.Set(amzAlgorithmKey, AlgorithmHMACSHA256) + query.Set(amzDateKey, amzDate) + return + } + headers[amzDateKey] = append(headers[amzDateKey][:0], amzDate) +} + +// hoistHeadersToQuery splits header into the subset eligible for +// query-string hoisting on a presigned request (allowedQueryHoisting) and +// the remainder, which stays as headers. +func hoistHeadersToQuery(header http.Header) (url.Values, http.Header) { + query := url.Values{} + unsignedHeaders := http.Header{} + for k, h := range header { + if allowedQueryHoisting.IsValid(k) { + query[k] = h + } else { + unsignedHeaders[k] = h + } + } + return query, unsignedHeaders +} + +func buildCanonicalHeaders(host string, signedHdrs []string, header http.Header, contentLength int64) (signed http.Header, signedHeadersStr, canonicalHeadersStr string) { + signed = make(http.Header) + + var headerNames []string + headerNames = append(headerNames, hostHeader) + signed[hostHeader] = append(signed[hostHeader], host) + + if slices.Contains(signedHdrs, contentLengthHeader) { + headerNames = append(headerNames, contentLengthHeader) + signed[contentLengthHeader] = append(signed[contentLengthHeader], strconv.FormatInt(contentLength, 10)) + } + + for k, v := range header { + if !shouldSignHeader(k, signedHdrs) { + continue + } + if strings.EqualFold(k, contentLengthHeader) { + // prevent signing the already-handled content-length header. + continue + } + + lowerCaseKey := strings.ToLower(k) + if _, ok := signed[lowerCaseKey]; ok { + signed[lowerCaseKey] = append(signed[lowerCaseKey], v...) + continue + } + + headerNames = append(headerNames, lowerCaseKey) + signed[lowerCaseKey] = v + } + sort.Strings(headerNames) + + signedHeadersStr = strings.Join(headerNames, ";") + + var canonicalHeaders strings.Builder + for _, name := range headerNames { + if name == hostHeader { + canonicalHeaders.WriteString(hostHeader) + canonicalHeaders.WriteByte(':') + canonicalHeaders.WriteString(stripExcessSpaces(host)) + } else { + canonicalHeaders.WriteString(name) + canonicalHeaders.WriteByte(':') + values := signed[name] + for j, v := range values { + canonicalHeaders.WriteString(strings.TrimSpace(stripExcessSpaces(v))) + if j < len(values)-1 { + canonicalHeaders.WriteByte(',') + } + } + } + canonicalHeaders.WriteByte('\n') + } + + return signed, signedHeadersStr, canonicalHeaders.String() +} + +// shouldSignHeader reports whether header must be included in the +// canonical headers: never Authorization itself, otherwise exactly the +// headers named in signedHdrs (the client's own SignedHeaders list) when +// non-nil, else falling back to ignoredHeaders' default policy. +func shouldSignHeader(header string, signedHdrs []string) bool { + if strings.EqualFold(header, authorizationHeader) { + return false + } + if signedHdrs != nil { + return slices.ContainsFunc(signedHdrs, func(signedHeader string) bool { + return strings.EqualFold(signedHeader, header) + }) + } + return ignoredHeaders.IsValid(header) +} + +func buildCanonicalString(method, uri, query, signedHeaders, canonicalHeaders, payloadHash string) string { + return strings.Join([]string{ + method, + uri, + query, + canonicalHeaders, + signedHeaders, + payloadHash, + }, "\n") +} + +func buildStringToSign(t time.Time, credentialScope, canonicalRequestString string) string { + hash := sha256.Sum256([]byte(canonicalRequestString)) + return strings.Join([]string{ + AlgorithmHMACSHA256, + t.Format(ISO8601Format), + credentialScope, + hex.EncodeToString(hash[:]), + }, "\n") +} + +func buildAuthorizationHeader(credentialStr, signedHeadersStr, signature string) string { + return AlgorithmHMACSHA256 + " Credential=" + credentialStr + + ", SignedHeaders=" + signedHeadersStr + ", Signature=" + signature +} + +const doubleSpace = " " + +// stripExcessSpaces rewrites str to collapse any run of interior spaces to +// a single space, after trimming leading/trailing spaces. +func stripExcessSpaces(str string) string { + var j, k, l, m, spaces int + for j = len(str) - 1; j >= 0 && str[j] == ' '; j-- { + } + for k = 0; k < j && str[k] == ' '; k++ { + } + str = str[k : j+1] + + j = strings.Index(str, doubleSpace) + if j < 0 { + return str + } + + buf := []byte(str) + for k, m, l = j, j, len(buf); k < l; k++ { + if buf[k] == ' ' { + if spaces == 0 { + buf[m] = buf[k] + m++ + } + spaces++ + } else { + spaces = 0 + buf[m] = buf[k] + m++ + } + } + + return string(buf[:m]) +} + +// escapePath URI-encodes path per SigV4's canonical-URI rules: every byte +// except unreserved characters (A-Za-z0-9-._~) is percent-encoded, and '/' +// is additionally preserved unless encodeSep is set (used for the path +// itself, never encodeSep; AWS also reuses this style of encoding for query +// keys/values, always with encodeSep). +func escapePath(path string, encodeSep bool) string { + var buf strings.Builder + buf.Grow(len(path)) + for i := 0; i < len(path); i++ { + c := path[i] + if isUnreservedByte(c) || (c == '/' && !encodeSep) { + buf.WriteByte(c) + continue + } + buf.WriteByte('%') + buf.WriteByte(upperHex[c>>4]) + buf.WriteByte(upperHex[c&0x0f]) + } + return buf.String() +} + +const upperHex = "0123456789ABCDEF" + +func isUnreservedByte(c byte) bool { + switch { + case 'A' <= c && c <= 'Z': + return true + case 'a' <= c && c <= 'z': + return true + case '0' <= c && c <= '9': + return true + case c == '-' || c == '_' || c == '.' || c == '~': + return true + } + return false +} diff --git a/internal/sigv4auth/canonical_test.go b/internal/sigv4auth/canonical_test.go new file mode 100644 index 00000000..d306a239 --- /dev/null +++ b/internal/sigv4auth/canonical_test.go @@ -0,0 +1,151 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sigv4auth + +import ( + "net/http" + "strings" + "testing" + "time" +) + +// TestBuildCanonicalHeaders ports the canonical-header edge cases (leading/ +// trailing/interior space stripping, multi-value merging, sort order) +func TestBuildCanonicalHeaders(t *testing.T) { + req, err := http.NewRequest("POST", "https://mockAPI.mock-region.amazonaws.com", nil) + if err != nil { + t.Fatalf("failed to create request, %v", err) + } + + req.Header.Set("FooInnerSpace", " inner space ") + req.Header.Set("FooLeadingSpace", " leading-space") + req.Header.Add("FooMultipleSpace", "no-space") + req.Header.Add("FooMultipleSpace", "\ttab-space") + req.Header.Add("FooMultipleSpace", "trailing-space ") + req.Header.Set("FooNoSpace", "no-space") + req.Header.Set("FooTabSpace", "\ttab-space\t") + req.Header.Set("FooTrailingSpace", "trailing-space ") + req.Header.Set("FooWrappedSpace", " wrapped-space ") + + signingTime := time.Date(2021, 10, 20, 12, 42, 0, 0, time.UTC) + yyyymmdd := signingTime.Format(YYYYMMDD) + in := SigningInputFromRequest(req) + in.AccessKeyID = "AKID" + in.CredentialScope = BuildCredentialScope(yyyymmdd, "mock-region", "mockAPI") + in.SigningTime = signingTime + result := BuildAndSign([]byte("dummy-derived-key"), in) + + expectCanonicalString := strings.Join([]string{ + `POST`, + `/`, + ``, + `fooinnerspace:inner space`, + `fooleadingspace:leading-space`, + `foomultiplespace:no-space,tab-space,trailing-space`, + `foonospace:no-space`, + `footabspace:tab-space`, + `footrailingspace:trailing-space`, + `foowrappedspace:wrapped-space`, + `host:mockAPI.mock-region.amazonaws.com`, + `x-amz-date:20211020T124200Z`, + ``, + `fooinnerspace;fooleadingspace;foomultiplespace;foonospace;footabspace;footrailingspace;foowrappedspace;host;x-amz-date`, + ``, + }, "\n") + + if result.CanonicalString != expectCanonicalString { + t.Errorf("canonical string mismatch:\ngot:\n%s\nwant:\n%s", result.CanonicalString, expectCanonicalString) + } +} + +func TestBuildAndSignOpaqueURLAndQuerySorting(t *testing.T) { + req, err := http.NewRequest("POST", "https://dynamodb.us-east-1.amazonaws.com", nil) + if err != nil { + t.Fatalf("failed to create request, %v", err) + } + req.URL.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()" + req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a" + + in := SigningInputFromRequest(req) + if want := "/bucket/key-._~,!@#$%^&*()"; in.URIPath != want { + t.Errorf("URIPath = %q, want %q (the pre-escaped Opaque path used verbatim, not re-derived from URL.Path)", in.URIPath, want) + } + + signingTime := time.Unix(0, 0) + yyyymmdd := signingTime.Format(YYYYMMDD) + in.AccessKeyID = "AKID" + in.CredentialScope = BuildCredentialScope(yyyymmdd, "us-east-1", "dynamodb") + in.SigningTime = signingTime + result := BuildAndSign([]byte("dummy-derived-key"), in) + + expected := "Foo=a&Foo=m&Foo=o&Foo=z" + if result.RawQuery != expected { + t.Errorf("RawQuery = %q, want %q", result.RawQuery, expected) + } +} + +// TestSanitizeHostForHeader confirms a request's explicit Host takes +// precedence over URL.Host and is reflected verbatim in the canonical +// "host" header +func TestSanitizeHostForHeader(t *testing.T) { + req, err := http.NewRequest("POST", "https://dynamodb.us-east-1.amazonaws.com", nil) + if err != nil { + t.Fatalf("failed to create request, %v", err) + } + req.Host = "myhost" + + in := SigningInputFromRequest(req) + signingTime := time.Now() + yyyymmdd := signingTime.Format(YYYYMMDD) + in.AccessKeyID = "AKID" + in.CredentialScope = BuildCredentialScope(yyyymmdd, "us-east-1", "dynamodb") + in.SigningTime = signingTime + result := BuildAndSign([]byte("dummy-derived-key"), in) + + if !strings.Contains(result.CanonicalString, "host:"+req.Host) { + t.Errorf("canonical host header invalid:\n%s", result.CanonicalString) + } +} + +// TestBuildAndSignExplicitSignedHeadersIgnoresUnsignedHeaders confirms that, +// with an explicit SignedHdrs list, extra headers present on the request +// but absent from that list never affect the resulting signature. +func TestBuildAndSignExplicitSignedHeadersIgnoresUnsignedHeaders(t *testing.T) { + build := func(extraHeaders bool) string { + req, err := http.NewRequest("POST", "https://dynamodb.us-east-1.amazonaws.com", nil) + if err != nil { + t.Fatalf("failed to create request, %v", err) + } + if extraHeaders { + req.Header.Set("Content-Type", "text/plain") + req.Header.Set("X-Unsigned-Header", "ignored") + } + + signingTime := time.Unix(0, 0) + yyyymmdd := signingTime.Format(YYYYMMDD) + derivedKey := DeriveKey("SECRET", yyyymmdd, "us-east-1", "dynamodb") + + in := SigningInputFromRequest(req) + in.AccessKeyID = "AKID" + in.CredentialScope = BuildCredentialScope(yyyymmdd, "us-east-1", "dynamodb") + in.SignedHdrs = []string{"host", "x-amz-date"} + in.SigningTime = signingTime + result := BuildAndSign(derivedKey, in) + return result.Signature + } + + if got, want := build(false), build(true); got != want { + t.Errorf("unsigned headers changed the signature: %q != %q", got, want) + } +} diff --git a/internal/sigv4auth/ctx.go b/internal/sigv4auth/ctx.go new file mode 100644 index 00000000..347d6b79 --- /dev/null +++ b/internal/sigv4auth/ctx.go @@ -0,0 +1,144 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sigv4auth + +import ( + "net/http" + "net/url" + "strings" + + "github.com/gofiber/fiber/v3" + "github.com/versity/versitygw/debuglogger" + "github.com/versity/versitygw/internal/httpctx" +) + +// signingInputFromCtx builds a SigningInput straight from ctx's underlying +// fasthttp request. isPreSign selects between header-auth's and presigned (query) auth's +// slightly different query/header handling; see queryFromCtx/ +// presignQueryFromCtx and headersFromCtx. +func signingInputFromCtx(ctx fiber.Ctx, signedHdrs []string, contentLength int64, requiredSignedHdrs []string, isPreSign bool) (SigningInput, error) { + if err := validateRequiredSignedHeaders(signedHdrs, requiredSignedHdrs); err != nil { + return SigningInput{}, err + } + + headers, err := headersFromCtx(ctx, signedHdrs, requiredSignedHdrs, isPreSign) + if err != nil { + return SigningInput{}, err + } + + query := queryFromCtx(ctx) + if isPreSign { + query = presignQueryFromCtx(ctx) + } + + return SigningInput{ + Method: methodFromCtx(ctx), + Host: hostFromCtx(ctx), + URIPath: uriPathFromCtx(ctx), + Query: query, + Header: headers, + ContentLength: contentLength, + IsPreSign: isPreSign, + }, nil +} + +// methodFromCtx returns ctx's HTTP method from the underlying fasthttp +// request header directly +func methodFromCtx(ctx fiber.Ctx) string { + return string(ctx.Request().Header.Method()) +} + +// headersFromCtx collects ctx's request headers eligible for signing: every +// header naming itself in signedHdrs, plus any header ignoredHeaders always +// signs regardless. Reads straight off the underlying fasthttp request +// (Header.All(), preserving every duplicate key exactly as sent). +func headersFromCtx(ctx fiber.Ctx, signedHdrs, requiredSignedHdrs []string, isPreSign bool) (http.Header, error) { + headers := http.Header{} + headersNotSigned := []string{} + for key, value := range ctx.Request().Header.All() { + keyStr := string(key) + if includeHeader(keyStr, signedHdrs) || IsIgnoredHeader(keyStr) { + headers.Add(keyStr, string(value)) + continue + } + if isRequiredSignedHeader(keyStr, requiredSignedHdrs) { + headersNotSigned = append(headersNotSigned, strings.ToLower(keyStr)) + } + } + + if len(headersNotSigned) != 0 { + debuglogger.Logf("headers present in request but not included in SignedHeaders: %q", strings.Join(headersNotSigned, ", ")) + return nil, &HeadersNotSignedError{Headers: headersNotSigned} + } + + if !isPreSign { + for _, header := range signedHdrs { + if headers.Get(header) == "" { + headers.Set(header, "") + } + } + } + + return headers, nil +} + +// queryFromCtx returns ctx's full, unfiltered query string as url.Values — +// the header-auth path, where any existing query parameters (e.g. +// ?partNumber=2) are simply part of the canonical request, untouched. +func queryFromCtx(ctx fiber.Ctx) url.Values { + query := url.Values{} + for key, value := range ctx.Request().URI().QueryArgs().All() { + query.Add(string(key), string(value)) + } + return query +} + +// presignQueryFromCtx returns ctx's query string as url.Values with the +// generated SigV4 auth parameters excluded (generatedQueryAuthParams) — +// the presign path, which must recompute and re-add its own +// X-Amz-Credential/X-Amz-SignedHeaders/X-Amz-Signature rather than sign the +// client-presented ones. +func presignQueryFromCtx(ctx fiber.Ctx) url.Values { + query := url.Values{} + for key, value := range ctx.Request().URI().QueryArgs().All() { + keyStr := string(key) + if _, ok := generatedQueryAuthParams[keyStr]; ok { + continue + } + query.Add(keyStr, string(value)) + } + return query +} + +// uriPathFromCtx returns ctx's raw request path exactly as received on the +// wire (fasthttp's PathOriginal — unnormalized, unescaped-or-not exactly as +// sent, no dot-segment collapsing), "/" if empty. HostStyleParser rewrites +// PathOriginal itself to move a virtual-hosted-style request's bucket from +// the Host header into the path for routing, so the true original is read +// back from where it stashed it rather than from PathOriginal directly. +func uriPathFromCtx(ctx fiber.Ctx) string { + path := string(ctx.Request().URI().PathOriginal()) + if httpctx.ContextKeyOriginalURIPath.IsSet(ctx) { + path, _ = httpctx.ContextKeyOriginalURIPath.Get(ctx).(string) + } + if path == "" { + return "/" + } + return path +} + +// hostFromCtx returns ctx's Host header verbatim. +func hostFromCtx(ctx fiber.Ctx) string { + return string(ctx.Request().Header.Host()) +} diff --git a/internal/sigv4auth/derive.go b/internal/sigv4auth/derive.go new file mode 100644 index 00000000..1ae8105a --- /dev/null +++ b/internal/sigv4auth/derive.go @@ -0,0 +1,45 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sigv4auth + +import ( + "crypto/hmac" + "crypto/sha256" +) + +// DeriveKey computes the SigV4 signing key (kSigning) for a secret access +// key and a request's credential scope: +// +// kDate = HMAC-SHA256("AWS4"+secret, yyyymmdd) +// kRegion = HMAC-SHA256(kDate, region) +// kService = HMAC-SHA256(kRegion, service) +// kSigning = HMAC-SHA256(kService, "aws4_request") +// +// This is the one artifact that's safe to hand across a process boundary: a +// standalone IAM service can compute and return it without ever exposing +// the secret itself. Every SigV4 consumer in this codebase (header auth, +// presigned/query auth, POST-policy, chunked upload) is built on top of this +// single implementation rather than each deriving its own key. +func DeriveKey(secret, yyyymmdd, region, service string) []byte { + kDate := hmacSHA256([]byte("AWS4"+secret), []byte(yyyymmdd)) + kRegion := hmacSHA256(kDate, []byte(region)) + kService := hmacSHA256(kRegion, []byte(service)) + return hmacSHA256(kService, []byte(Terminal)) +} + +func hmacSHA256(key, data []byte) []byte { + h := hmac.New(sha256.New, key) + h.Write(data) + return h.Sum(nil) +} diff --git a/internal/sigv4auth/derive_test.go b/internal/sigv4auth/derive_test.go new file mode 100644 index 00000000..8b3f5afa --- /dev/null +++ b/internal/sigv4auth/derive_test.go @@ -0,0 +1,34 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sigv4auth + +import ( + "encoding/hex" + "testing" +) + +func TestDeriveKey(t *testing.T) { + const wantHex = "2c94c0cf5378ada6887f09bb697df8fc0affdb34ba1cdd5bda32b664bd55b73c" + + got := DeriveKey("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "20150830", "us-east-1", "iam") + + want, err := hex.DecodeString(wantHex) + if err != nil { + t.Fatalf("decode want hex: %v", err) + } + + if hex.EncodeToString(got) != hex.EncodeToString(want) { + t.Errorf("DeriveKey() = %x, want %x", got, want) + } +} diff --git a/internal/sigv4auth/header_rules.go b/internal/sigv4auth/header_rules.go new file mode 100644 index 00000000..51801b18 --- /dev/null +++ b/internal/sigv4auth/header_rules.go @@ -0,0 +1,129 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sigv4auth + +import "strings" + +// rule reports whether a header name adheres to some signing policy — which +// headers are excluded from signing, which must be signed when present, and +// which are eligible for query-string hoisting on a presigned request. +type rule interface { + IsValid(value string) bool +} + +// rules is a set of rule; IsValid reports whether any rule in the set +// matches (nested/composable rules). +type rules []rule + +func (r rules) IsValid(value string) bool { + for _, rl := range r { + if rl.IsValid(value) { + return true + } + } + return false +} + +// mapRule is a case-insensitive set-membership rule. +type mapRule map[string]struct{} + +func (m mapRule) IsValid(value string) bool { + for key := range m { + if strings.EqualFold(key, value) { + return true + } + } + return false +} + +// allowList and excludeList wrap another rule, only for readability at the +// table-definition call site — allowList is a no-op wrapper, excludeList +// inverts. +type allowList struct{ rule } + +func (w allowList) IsValid(value string) bool { return w.rule.IsValid(value) } + +type excludeList struct{ rule } + +func (b excludeList) IsValid(value string) bool { return !b.rule.IsValid(value) } + +// patterns matches by case-insensitive prefix. +type patterns []string + +func (p patterns) IsValid(value string) bool { + for _, pattern := range p { + if hasPrefixFold(value, pattern) { + return true + } + } + return false +} + +// inclusiveRules requires every rule in the set to match. +type inclusiveRules []rule + +func (r inclusiveRules) IsValid(value string) bool { + for _, rl := range r { + if !rl.IsValid(value) { + return false + } + } + return true +} + +func hasPrefixFold(s, prefix string) bool { + return len(s) >= len(prefix) && strings.EqualFold(s[0:len(prefix)], prefix) +} + +// ignoredHeaders is excluded from signing regardless of SignedHeaders. +var ignoredHeaders = rules{ + excludeList{ + mapRule{ + "Authorization": struct{}{}, + "User-Agent": struct{}{}, + "X-Amzn-Trace-Id": struct{}{}, + "Expect": struct{}{}, + "Transfer-Encoding": struct{}{}, + }, + }, +} + +// requiredSignedHeadersRule is the header-auth SignedHeaders policy: which +// request headers, if present, must appear in SignedHeaders. +var requiredSignedHeadersRule = rules{ + allowList{ + mapRule{ + "Host": struct{}{}, + }, + }, + patterns{"X-Amz-"}, +} + +// allowedQueryHoisting selects which unsigned headers a presigned request +// may hoist into the query string. +var allowedQueryHoisting = inclusiveRules{ + excludeList{requiredSignedHeadersRule}, + patterns{"X-Amz-"}, +} + +// IsIgnoredHeader reports whether a header is normally excluded from signing. +func IsIgnoredHeader(header string) bool { + return !ignoredHeaders.IsValid(header) +} + +// IsRequiredSignedHeader reports whether a header must be signed when it is +// present on an incoming request. +func IsRequiredSignedHeader(header string) bool { + return requiredSignedHeadersRule.IsValid(header) +} diff --git a/internal/sigv4auth/query.go b/internal/sigv4auth/query.go index 6ad04c6c..176f9932 100644 --- a/internal/sigv4auth/query.go +++ b/internal/sigv4auth/query.go @@ -14,19 +14,12 @@ package sigv4auth import ( - "errors" "fmt" - "net/http" - "net/url" - "os" "strconv" "strings" "time" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/smithy-go/logging" "github.com/gofiber/fiber/v3" - "github.com/versity/versitygw/aws/signer/v4" "github.com/versity/versitygw/debuglogger" ) @@ -127,6 +120,15 @@ func ParseQueryAuthorization(ctx fiber.Ctx, opts QueryAuthOptions) (AuthData, Qu return a, details, err } + // A security token in the query string is only ever legitimate alongside + // a temporary (ASIA…) access key. Reject it outright for root or any + // long-term (AKIA…) credential before any signature work, rather than + // letting it fall through to a signature mismatch once the tampered or + // unsigned token parameter invalidates the canonical query string. + if ctx.Request().URI().QueryArgs().Has(QuerySecurityToken) && !IsTempAccessKeyID(creds.Access) { + return a, details, &QueryError{Kind: ErrQuerySecurityToken} + } + if opts.Region != "" && creds.Region != opts.Region { return a, details, &QueryError{ Kind: ErrQueryIncorrectRegion, @@ -255,62 +257,59 @@ func missingQueryParameterError(parameter string) *QueryError { return &QueryError{Kind: ErrQueryMissingRequiredParams, Value: parameter} } -// CheckQuerySignature rebuilds a SigV4 query-auth request and compares the -// generated query signature to the signature presented by the client. -func CheckQuerySignature(ctx fiber.Ctx, auth AuthData, secret, payloadHash string, tdate time.Time, contentLen int64, opts CheckOptions) (*CheckResult, error) { +// CheckQuerySignature rebuilds a SigV4 query-auth request — reading +// everything it needs straight from ctx, with no intermediate +// *http.Request — and compares the generated query signature to the +// signature presented by the client. derivedKey is the request's kSigning +// value — either computed locally via DeriveKey from a known secret, or +// obtained from a standalone IAM service that never reveals the secret +// itself. +func CheckQuerySignature(ctx fiber.Ctx, auth AuthData, derivedKey []byte, payloadHash string, tdate time.Time, contentLen int64, opts CheckOptions) (*CheckResult, error) { service := opts.Service if service == "" { service = auth.Service } signedHdrs := strings.Split(auth.SignedHeaders, ";") - req, err := createPresignedHTTPRequestFromCtx(ctx, signedHdrs, contentLen, opts.RequiredSignedHeaders) + in, err := signingInputFromCtx(ctx, signedHdrs, contentLen, opts.RequiredSignedHeaders, true) if err != nil { return nil, err } + in.AccessKeyID = auth.Access + in.CredentialScope = BuildCredentialScope(tdate.Format(YYYYMMDD), auth.Region, service) + in.SignedHdrs = signedHdrs + in.PayloadHash = payloadHash + in.SigningTime = tdate + in.DisableURIPathEscaping = opts.DisableURIPathEscaping - signer := v4.NewSigner() - uri, _, signMeta, err := signer.PresignHTTP(ctx.RequestCtx(), - aws.Credentials{ - AccessKeyID: auth.Access, - SecretAccessKey: secret, - }, - req, payloadHash, service, auth.Region, tdate, signedHdrs, - func(options *v4.SignerOptions) { - options.DisableURIPathEscaping = opts.DisableURIPathEscaping - // See the identical comment in verify.go's CheckSignature: this - // logger dumps a complete, replayable signed URL (including - // X-Amz-Signature and any session token) unredacted, so it may - // only run at LevelUnsafe. - if debuglogger.IsUnsafeEnabled() { - options.LogSigning = true - options.Logger = logging.NewStandardLogger(os.Stderr) - } - }) - if err != nil { - return nil, fmt.Errorf("presign generated http request: %w", err) + result := BuildAndSign(derivedKey, in) + + // See the identical comment in verify.go's CheckSignature: this dumps a + // complete, replayable signed query string unredacted, so it may only + // run at LevelUnsafe. + if debuglogger.IsUnsafeEnabled() { + debuglogger.Logf("Request Signature:\n"+ + "---[ CANONICAL STRING ]-----------------------------\n%s\n"+ + "---[ STRING TO SIGN ]--------------------------------\n%s\n"+ + "---[ SIGNED QUERY ]-----------------------------------\n%s\n"+ + "-----------------------------------------------------", + result.CanonicalString, result.StringToSign, result.RawQuery) } - urlParts, err := url.Parse(uri) - if err != nil { - return nil, fmt.Errorf("parse presigned url: %w", err) - } - - signature := urlParts.Query().Get(QuerySignature) - if !SecureCompare(signature, auth.Signature) { + if !SecureCompare(result.Signature, auth.Signature) { return nil, &SignatureMismatchError{ AccessKeyID: auth.Access, - StringToSign: signMeta.StringToSign, + StringToSign: result.StringToSign, SignatureProvided: auth.Signature, - StringToSignBytes: HexBytes(signMeta.StringToSign), - CanonicalRequest: signMeta.CanonicalString, - CanonicalRequestBytes: HexBytes(signMeta.CanonicalString), + StringToSignBytes: HexBytes(result.StringToSign), + CanonicalRequest: result.CanonicalString, + CanonicalRequestBytes: HexBytes(result.CanonicalString), } } return &CheckResult{ - CanonicalString: signMeta.CanonicalString, - StringToSign: signMeta.StringToSign, + CanonicalString: result.CanonicalString, + StringToSign: result.StringToSign, }, nil } @@ -322,52 +321,6 @@ var generatedQueryAuthParams = map[string]struct{}{ QuerySignature: {}, } -func createPresignedHTTPRequestFromCtx(ctx fiber.Ctx, signedHdrs []string, contentLength int64, requiredSignedHdrs []string) (*http.Request, error) { - req := ctx.Request() - if err := validateRequiredSignedHeaders(signedHdrs, requiredSignedHdrs); err != nil { - return nil, err - } - - uri, _, _ := strings.Cut(ctx.OriginalURL(), "?") - query := strings.Builder{} - - for key, value := range ctx.Request().URI().QueryArgs().All() { - keyStr := string(key) - if _, ok := generatedQueryAuthParams[keyStr]; ok { - continue - } - - if query.Len() > 0 { - query.WriteByte('&') - } - query.WriteString(url.QueryEscape(keyStr)) - query.WriteByte('=') - query.WriteString(url.QueryEscape(string(value))) - } - - if query.Len() > 0 { - uri += "?" + query.String() - } - - httpReq, err := http.NewRequest(string(req.Header.Method()), uri, nil) - if err != nil { - return nil, errors.New("error in creating an http request") - } - if err := addRequestHeadersFromCtx(ctx, httpReq, signedHdrs, requiredSignedHdrs); err != nil { - return nil, err - } - - if !includeHeader("Content-Length", signedHdrs) { - httpReq.ContentLength = 0 - } else { - httpReq.ContentLength = contentLength - } - - httpReq.Host = string(req.Header.Host()) - - return httpReq, nil -} - // IsQueryAuth determines if a request uses SigV4 query-string auth. func IsQueryAuth(ctx fiber.Ctx) bool { algo := ctx.Query(QueryAlgorithm) diff --git a/internal/sigv4auth/request.go b/internal/sigv4auth/request.go new file mode 100644 index 00000000..ec93fff9 --- /dev/null +++ b/internal/sigv4auth/request.go @@ -0,0 +1,112 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sigv4auth + +import ( + "net/http" + "net/url" + "strings" +) + +// SigningInputFromRequest extracts a SigningInput's request-shaped fields +// (Method, Host, URIPath, Query, Header, ContentLength) from a real +// *http.Request being signed for an outbound call +func SigningInputFromRequest(req *http.Request) SigningInput { + return SigningInput{ + Method: req.Method, + Host: sanitizedHost(req), + URIPath: getURIPath(req.URL), + Query: req.URL.Query(), + Header: req.Header, + ContentLength: req.ContentLength, + } +} + +// sanitizedHost resolves req's effective Host header value (req.Host takes +// precedence over req.URL.Host) and strips a default port (80 for http, 443 +// for https) so the canonical "host" header matches what a well-behaved +// SigV4 client signs. +func sanitizedHost(req *http.Request) string { + host := req.URL.Host + if len(req.Host) > 0 { + host = req.Host + } + port := portOnly(host) + if port != "" && isDefaultPort(req.URL.Scheme, port) { + return stripPort(host) + } + return host +} + +func stripPort(hostport string) string { + before, _, ok := strings.Cut(hostport, ":") + if !ok { + return hostport + } + if before, _, ok := strings.Cut(hostport, "]"); ok { + return strings.TrimPrefix(before, "[") + } + return before +} + +func portOnly(hostport string) string { + _, after, ok := strings.Cut(hostport, ":") + if !ok { + return "" + } + if _, after, ok := strings.Cut(hostport, "]:"); ok { + return after + } + if strings.Contains(hostport, "]") { + return "" + } + return after +} + +func isDefaultPort(scheme, port string) bool { + if port == "" { + return true + } + lowerCaseScheme := strings.ToLower(scheme) + return (lowerCaseScheme == "http" && port == "80") || (lowerCaseScheme == "https" && port == "443") +} + +// getURIPath returns the escaped URI path component of u, preferring +// u.Opaque (set when the caller pre-escaped the path) over u.EscapedPath(). +func getURIPath(u *url.URL) string { + var uriPath string + + if len(u.Opaque) > 0 { + const schemeSep, pathSep, queryStart = "//", "/", "?" + + opaque := u.Opaque + if idx := strings.Index(opaque, queryStart); idx >= 0 { + opaque = opaque[:idx] + } + if strings.Index(opaque, schemeSep) == 0 { + opaque = opaque[len(schemeSep):] + } + if idx := strings.Index(opaque, pathSep); idx >= 0 { + uriPath = opaque[idx:] + } + } else { + uriPath = u.EscapedPath() + } + + if len(uriPath) == 0 { + uriPath = "/" + } + + return uriPath +} diff --git a/internal/sigv4auth/verify.go b/internal/sigv4auth/verify.go index ca02567d..499511c0 100644 --- a/internal/sigv4auth/verify.go +++ b/internal/sigv4auth/verify.go @@ -14,18 +14,12 @@ package sigv4auth import ( - "errors" "fmt" - "net/http" - "os" "slices" "strings" "time" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/smithy-go/logging" "github.com/gofiber/fiber/v3" - "github.com/versity/versitygw/aws/signer/v4" "github.com/versity/versitygw/debuglogger" ) @@ -64,127 +58,59 @@ func (e *SignatureMismatchError) Error() string { } // CheckSignature rebuilds the canonical request with the supplied service, -// region, payload hash, signing time, and signed headers, then compares the +// region, payload hash, signing time, and signed headers. Then compares the // generated signature to the signature presented by the client. -func CheckSignature(ctx fiber.Ctx, auth AuthData, secret, payloadHash string, tdate time.Time, contentLen int64, opts CheckOptions) (*CheckResult, error) { +// derivedKey is the request's kSigning value — either computed +// locally via DeriveKey from a known secret, or obtained from a standalone +// IAM service that never reveals the secret itself. +func CheckSignature(ctx fiber.Ctx, auth AuthData, derivedKey []byte, payloadHash string, tdate time.Time, contentLen int64, opts CheckOptions) (*CheckResult, error) { service := opts.Service if service == "" { service = auth.Service } signedHdrs := strings.Split(auth.SignedHeaders, ";") - req, err := createHTTPRequestFromCtx(ctx, signedHdrs, contentLen, opts.RequiredSignedHeaders) + in, err := signingInputFromCtx(ctx, signedHdrs, contentLen, opts.RequiredSignedHeaders, false) if err != nil { return nil, err } + in.AccessKeyID = auth.Access + in.CredentialScope = BuildCredentialScope(tdate.Format(YYYYMMDD), auth.Region, service) + in.SignedHdrs = signedHdrs + in.PayloadHash = payloadHash + in.SigningTime = tdate + in.DisableURIPathEscaping = opts.DisableURIPathEscaping - signer := v4.NewSigner() + result := BuildAndSign(derivedKey, in) - signMeta, err := signer.SignHTTP(req.Context(), - aws.Credentials{ - AccessKeyID: auth.Access, - SecretAccessKey: secret, - }, - req, payloadHash, service, auth.Region, tdate, signedHdrs, - func(options *v4.SignerOptions) { - options.DisableURIPathEscaping = opts.DisableURIPathEscaping - // The signer's diagnostic logger prints the canonical request, - // string-to-sign, and (for presigned requests) the complete - // signed URL verbatim, bypassing the redaction layer entirely. - // That's replayable signature/session-token material, so only - // enable it at LevelUnsafe, never at plain debug. - if debuglogger.IsUnsafeEnabled() { - options.LogSigning = true - options.Logger = logging.NewStandardLogger(os.Stderr) - } - }) - if err != nil { - return nil, fmt.Errorf("sign generated http request: %w", err) + // This prints the canonical request and string-to-sign verbatim, + // bypassing the redaction layer entirely — replayable signature + // material, so only ever log it at LevelUnsafe, never at plain debug. + if debuglogger.IsUnsafeEnabled() { + debuglogger.Logf("Request Signature:\n"+ + "---[ CANONICAL STRING ]-----------------------------\n%s\n"+ + "---[ STRING TO SIGN ]--------------------------------\n%s\n"+ + "-----------------------------------------------------", + result.CanonicalString, result.StringToSign) } - genAuth, err := ParseAuthorization(req.Header.Get("Authorization"), service) - if err != nil { - return nil, err - } - - if !SecureCompare(auth.Signature, genAuth.Signature) { + if !SecureCompare(auth.Signature, result.Signature) { return nil, &SignatureMismatchError{ AccessKeyID: auth.Access, - StringToSign: signMeta.StringToSign, + StringToSign: result.StringToSign, SignatureProvided: auth.Signature, - StringToSignBytes: HexBytes(signMeta.StringToSign), - CanonicalRequest: signMeta.CanonicalString, - CanonicalRequestBytes: HexBytes(signMeta.CanonicalString), + StringToSignBytes: HexBytes(result.StringToSign), + CanonicalRequest: result.CanonicalString, + CanonicalRequestBytes: HexBytes(result.CanonicalString), } } return &CheckResult{ - CanonicalString: signMeta.CanonicalString, - StringToSign: signMeta.StringToSign, + CanonicalString: result.CanonicalString, + StringToSign: result.StringToSign, }, nil } -func CreateHTTPRequestFromCtx(ctx fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) { - return createHTTPRequestFromCtx(ctx, signedHdrs, contentLength, nil) -} - -func createHTTPRequestFromCtx(ctx fiber.Ctx, signedHdrs []string, contentLength int64, requiredSignedHdrs []string) (*http.Request, error) { - req := ctx.Request() - if err := validateRequiredSignedHeaders(signedHdrs, requiredSignedHdrs); err != nil { - return nil, err - } - - httpReq, err := http.NewRequest(string(req.Header.Method()), ctx.OriginalURL(), nil) - if err != nil { - return nil, errors.New("error in creating an http request") - } - - if err := addRequestHeadersFromCtx(ctx, httpReq, signedHdrs, requiredSignedHdrs); err != nil { - return nil, err - } - - for _, header := range signedHdrs { - if httpReq.Header.Get(header) == "" { - httpReq.Header.Set(header, "") - } - } - - if !includeHeader("Content-Length", signedHdrs) { - httpReq.ContentLength = 0 - } else { - httpReq.ContentLength = contentLength - } - - httpReq.Host = string(req.Header.Host()) - - return httpReq, nil -} - -func AddRequestHeadersFromCtx(ctx fiber.Ctx, httpReq *http.Request, signedHdrs []string) error { - return addRequestHeadersFromCtx(ctx, httpReq, signedHdrs, nil) -} - -func addRequestHeadersFromCtx(ctx fiber.Ctx, httpReq *http.Request, signedHdrs, requiredSignedHdrs []string) error { - headersNotSigned := []string{} - for key, value := range ctx.Request().Header.All() { - keyStr := string(key) - if includeHeader(keyStr, signedHdrs) || v4.IsIgnoredHeader(keyStr) { - httpReq.Header.Add(keyStr, string(value)) - continue - } - if isRequiredSignedHeader(keyStr, requiredSignedHdrs) { - headersNotSigned = append(headersNotSigned, strings.ToLower(keyStr)) - } - } - - if len(headersNotSigned) != 0 { - debuglogger.Logf("headers present in request but not included in SignedHeaders: %q", strings.Join(headersNotSigned, ", ")) - return &HeadersNotSignedError{Headers: headersNotSigned} - } - - return nil -} - func validateRequiredSignedHeaders(signedHdrs, requiredSignedHdrs []string) error { if requiredSignedHdrs == nil { return nil @@ -205,7 +131,7 @@ func validateRequiredSignedHeaders(signedHdrs, requiredSignedHdrs []string) erro func isRequiredSignedHeader(header string, requiredSignedHdrs []string) bool { if requiredSignedHdrs == nil { - return v4.IsRequiredSignedHeader(header) + return IsRequiredSignedHeader(header) } return includeHeader(header, requiredSignedHdrs) diff --git a/runoidctests.sh b/runoidctests.sh new file mode 100755 index 00000000..d24ce1bd --- /dev/null +++ b/runoidctests.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# +# Run the test groups that need a real, signed OIDC ID token. +# +# AssumeRoleWithWebIdentity verifies a token's signature against its issuer's +# live JWKS, so these tests need a genuine identity provider rather than a +# fake token. GitHub Actions' own OIDC issuer is the one publicly reachable +# IdP available from inside CI, and only a job holding `id-token: write` can +# mint a token from it — which is why this script lives behind +# .github/workflows/functional-iam-oidc.yml rather than the general +# functional suite. Outside such a job the tests skip themselves, so running +# this locally is harmless but proves little. +# +# It brings up two processes: a standalone IAM service holding every user, +# role, policy and secret, and an s3 gateway that reaches its private +# endpoints over mTLS for signing keys and policy decisions. + +set -Eeuo pipefail + +readonly IAM_PORT=7078 +readonly IAM_PRIVATE_PORT=7079 +readonly GW_PORT=7077 + +readonly IAM_DIR=/tmp/iam-oidc +readonly GW_DIR=/tmp/s3iam-oidc-gw +readonly CERT_DIR=/tmp/s3iam-oidc-certs + +IAM_PID="" +GW_PID="" + +cleanup() { + local status=$? + trap - EXIT + for pid in "$GW_PID" "$IAM_PID"; do + if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + fi + if [[ -n "$pid" ]]; then + wait "$pid" 2>/dev/null || true + fi + done + exit "$status" +} + +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +wait_for_server() { + local name="$1" + local url="$2" + local pid="$3" + + for _ in {1..50}; do + if curl --fail --silent --max-time 1 "$url" >/dev/null 2>&1; then + return 0 + fi + if ! kill -0 "$pid" 2>/dev/null; then + echo "$name stopped before becoming ready" >&2 + wait "$pid" 2>/dev/null || true + return 1 + fi + sleep 0.2 + done + + echo "timed out waiting for $name at $url" >&2 + return 1 +} + +rm -rf "$IAM_DIR" "$GW_DIR" "$CERT_DIR" +mkdir -p "$IAM_DIR" "$GW_DIR" + +# The gateway verifies the IAM service's certificate normally, with no +# hostname override, so the server certificate needs an IP SAN matching the +# address --iam-standalone-endpoint names. +./genmtlscerts.sh "$CERT_DIR" 127.0.0.1 + +echo "Starting the standalone IAM service" +./versitygw --health /healthz -p ":$IAM_PORT" -a user -s pass iam \ + --dir "$IAM_DIR" \ + --private-ports "127.0.0.1:$IAM_PRIVATE_PORT" \ + --private-cert "$CERT_DIR/iam-server.pem" \ + --private-cert-key "$CERT_DIR/iam-server.key" \ + --private-client-ca "$CERT_DIR/ca.pem" & +IAM_PID=$! +wait_for_server "IAM API server" "http://127.0.0.1:$IAM_PORT/healthz" "$IAM_PID" + +echo "Starting the s3 gateway backed by it" +./versitygw --health /healthz -p ":$GW_PORT" -a user -s pass \ + --iam-standalone-endpoint "127.0.0.1:$IAM_PRIVATE_PORT" \ + --iam-standalone-client-cert "$CERT_DIR/gw-client.pem" \ + --iam-standalone-client-cert-key "$CERT_DIR/gw-client.key" \ + --iam-standalone-server-ca "$CERT_DIR/ca.pem" \ + posix "$GW_DIR" & +GW_PID=$! +wait_for_server "s3 gateway" "http://127.0.0.1:$GW_PORT/healthz" "$GW_PID" + +echo "Running the live GitHub OIDC web-identity test" +./versitygw test -a user -s pass -e "http://127.0.0.1:$IAM_PORT" \ + IAMAssumeRoleWithWebIdentity_github_oidc_live + +echo "Running the s3 assumed-role session access control tests" +./versitygw test -a user -s pass \ + -e "http://127.0.0.1:$GW_PORT" \ + --iam-endpoint "http://127.0.0.1:$IAM_PORT" \ + s3-iam-session diff --git a/runtests.sh b/runtests.sh index 7fa11c79..2a1fbee2 100755 --- a/runtests.sh +++ b/runtests.sh @@ -229,6 +229,60 @@ fi # kill off server kill $GW_NO_ACL_PID +ECHO "Running the s3 + standalone IAM access control tests" +# This stage is the only one that runs two versitygw processes at once: a +# standalone IAM service holding every user, policy and secret, and an s3 +# gateway that reaches it over mTLS for signing keys and policy decisions. +# ports: 7080 IAM control plane, 7081 IAM private endpoint, 7082 s3 gateway +rm -rf /tmp/s3iam /tmp/s3iamgw /tmp/s3iamcerts /tmp/s3iam.covdata /tmp/s3iamgw.covdata +mkdir -p /tmp/s3iam /tmp/s3iamgw /tmp/s3iam.covdata /tmp/s3iamgw.covdata + +# The gateway verifies the IAM service's certificate normally, with no +# hostname override, so the server certificate needs an IP SAN matching the +# --iam-standalone-endpoint host. +./genmtlscerts.sh /tmp/s3iamcerts 127.0.0.1 + +GOCOVERDIR=/tmp/s3iam.covdata ./versitygw --health /healthz -p :7080 -a user -s pass iam \ + --dir /tmp/s3iam \ + --private-ports 127.0.0.1:7081 \ + --private-cert /tmp/s3iamcerts/iam-server.pem \ + --private-cert-key /tmp/s3iamcerts/iam-server.key \ + --private-client-ca /tmp/s3iamcerts/ca.pem & +IAM_PID=$! + +sleep 2 + +if ! kill -0 $IAM_PID; then + echo "standalone IAM service no longer running" + exit 1 +fi + +GOCOVERDIR=/tmp/s3iamgw.covdata ./versitygw -p :7082 -a user -s pass \ + --iam-standalone-endpoint 127.0.0.1:7081 \ + --iam-standalone-client-cert /tmp/s3iamcerts/gw-client.pem \ + --iam-standalone-client-cert-key /tmp/s3iamcerts/gw-client.key \ + --iam-standalone-server-ca /tmp/s3iamcerts/ca.pem \ + posix $SIDECAR_FLAG /tmp/s3iamgw & +GW_S3IAM_PID=$! + +sleep 2 + +if ! kill -0 $GW_S3IAM_PID; then + echo "s3 gateway backed by standalone IAM no longer running" + kill $IAM_PID + exit 1 +fi + +if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7082 --iam-endpoint http://127.0.0.1:7080 s3-iam; then + echo "s3 + standalone IAM access control tests failed" + kill $GW_S3IAM_PID + kill $IAM_PID + exit 1 +fi + +kill $GW_S3IAM_PID +kill $IAM_PID + exit 0 # ----------------------------------------------------------------------------- @@ -256,6 +310,8 @@ exit 0 # /tmp/versioning.covdata # /tmp/versioning.https.covdata # /tmp/noacl.covdata +# /tmp/s3iam.covdata (standalone IAM service) +# /tmp/s3iamgw.covdata (s3 gateway backed by it) # # This gives you coverage metrics isolated per test suite / server mode. # @@ -265,7 +321,7 @@ exit 0 # If you want a unified report combining all environments: # # go tool covdata merge \ -# -i=/tmp/covdata,/tmp/https.covdata,/tmp/versioning.covdata,/tmp/versioning.https.covdata,/tmp/noacl.covdata \ +# -i=/tmp/covdata,/tmp/https.covdata,/tmp/versioning.covdata,/tmp/versioning.https.covdata,/tmp/noacl.covdata,/tmp/s3iam.covdata,/tmp/s3iamgw.covdata \ # -o /tmp/allcovdata # # go tool covdata percent -i=/tmp/allcovdata diff --git a/s3api/admin-server.go b/s3api/admin-server.go index c9f40048..d06b98e5 100644 --- a/s3api/admin-server.go +++ b/s3api/admin-server.go @@ -25,9 +25,9 @@ import ( "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/debuglogger" + "github.com/versity/versitygw/internal/netutil" "github.com/versity/versitygw/s3api/controllers" "github.com/versity/versitygw/s3api/middlewares" - "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3log" ) @@ -35,7 +35,7 @@ type S3AdminServer struct { app *fiber.App backend backend.Backend router *S3AdminRouter - CertStorage *utils.CertStorage + CertStorage *netutil.CertStorage quiet bool debug bool corsAllowOrigin string @@ -100,7 +100,7 @@ func NewAdminServer(be backend.Backend, root middlewares.RootUserConfig, region type AdminOpt func(s *S3AdminServer) -func WithAdminSrvTLS(cs *utils.CertStorage) AdminOpt { +func WithAdminSrvTLS(cs *netutil.CertStorage) AdminOpt { return func(s *S3AdminServer) { s.CertStorage = cs } } @@ -152,9 +152,9 @@ func (sa *S3AdminServer) ServeMultiPort(ports []string) error { var err error if sa.CertStorage != nil { - ln, err = utils.NewMultiAddrTLSListener(fiber.NetworkTCP, portSpec, sa.CertStorage.GetCertificate, utils.ListenerOptions{SocketPerm: sa.socketPerm}) + ln, err = netutil.NewMultiAddrTLSListener(fiber.NetworkTCP, portSpec, sa.CertStorage.GetCertificate, netutil.ListenerOptions{SocketPerm: sa.socketPerm}) } else { - ln, err = utils.NewMultiAddrListener(fiber.NetworkTCP, portSpec, utils.ListenerOptions{SocketPerm: sa.socketPerm}) + ln, err = netutil.NewMultiAddrListener(fiber.NetworkTCP, portSpec, netutil.ListenerOptions{SocketPerm: sa.socketPerm}) } if err != nil { @@ -169,7 +169,7 @@ func (sa *S3AdminServer) ServeMultiPort(ports []string) error { } // Combine all listeners - finalListener := utils.NewMultiListener(listeners...) + finalListener := netutil.NewMultiListener(listeners...) return sa.app.Listener(finalListener, fiber.ListenConfig{ DisableStartupMessage: true, diff --git a/s3api/controllers/admin.go b/s3api/controllers/admin.go index 089a4719..f627497e 100644 --- a/s3api/controllers/admin.go +++ b/s3api/controllers/admin.go @@ -135,7 +135,7 @@ func (c AdminController) ChangeBucketOwner(ctx fiber.Ctx) (*Response, error) { owner := ctx.Query("owner") bucket := ctx.Query("bucket") - accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam) + accs, err := c.iam.ResolveAccounts([]string{owner}) if err != nil { return &Response{ MetaOpts: &MetaOptions{}, diff --git a/s3api/controllers/admin_test.go b/s3api/controllers/admin_test.go index 4fdfbb04..10c6cfa4 100644 --- a/s3api/controllers/admin_test.go +++ b/s3api/controllers/admin_test.go @@ -18,6 +18,7 @@ import ( "context" "encoding/xml" "errors" + "fmt" "net/http" "testing" @@ -487,8 +488,15 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { iam := &IAMServiceMock{ - GetUserAccountFunc: func(access string) (auth.Account, error) { - return auth.Account{}, tt.input.extraMockErr + ResolveAccountsFunc: func(accessKeyIDs []string) ([]string, error) { + switch tt.input.extraMockErr { + case nil: + return []string{}, nil + case auth.ErrNoSuchUser: + return accessKeyIDs, nil + default: + return nil, fmt.Errorf("check user account: %w", tt.input.extraMockErr) + } }, } be := &BackendMock{ @@ -674,6 +682,9 @@ func TestAdminController_CreateBucket(t *testing.T) { GetUserAccountFunc: func(access string) (auth.Account, error) { return auth.Account{}, tt.input.extraMockErr }, + ResolveAccountsFunc: func(accessKeyIDs []string) ([]string, error) { + return []string{}, nil + }, } be := &BackendMock{ CreateBucketFunc: func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput, defaultACL []byte) error { diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 03dda5b6..11be7824 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -21,6 +21,7 @@ import ( "sort" "strings" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gofiber/fiber/v3" "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" @@ -76,6 +77,34 @@ func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs } } +// verifyAccess wraps auth.VerifyAccess, always injecting the controller's +// own configured IAM backend, readonly mode, and disableACL setting into opts +func (c S3ApiController) verifyAccess(ctx fiber.Ctx, opts auth.AccessOptions) error { + opts.Iam = c.iam + opts.Readonly = c.readonly + opts.DisableACL = c.disableACL + return auth.VerifyAccess(ctx, c.be, opts) +} + +// verifyObjectsAccess wraps auth.VerifyObjectsAccess, for the one request +// shape (DeleteObjects) that names several objects at once. The returned +// slice has one entry per object (nil where it may proceed); the error +// return is a whole-request failure, not about any one object. +func (c S3ApiController) verifyObjectsAccess(ctx fiber.Ctx, opts auth.AccessOptions, objects []types.ObjectIdentifier, bypass auth.BypassMode) ([]error, error) { + opts.Iam = c.iam + opts.Readonly = c.readonly + opts.DisableACL = c.disableACL + return auth.VerifyObjectsAccess(ctx, c.be, opts, objects, bypass) +} + +// verifyObjectCopyAccess wraps auth.VerifyObjectCopyAccess +func (c S3ApiController) verifyObjectCopyAccess(ctx fiber.Ctx, copySource string, opts auth.AccessOptions) error { + opts.Iam = c.iam + opts.Readonly = c.readonly + opts.DisableACL = c.disableACL + return auth.VerifyObjectCopyAccess(ctx, c.be, copySource, opts) +} + func (c S3ApiController) getAclHeaderValue(ctx fiber.Ctx, key string, defaultValues ...string) string { if c.disableACL { return "" diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index fb403825..46e01d9b 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -76,6 +76,8 @@ type testInput struct { beErr error extraMockErr error extraMockResp any + readonly bool + disableACL bool } type testOutput struct { diff --git a/s3api/controllers/bucket-delete.go b/s3api/controllers/bucket-delete.go index b847e919..c110f243 100644 --- a/s3api/controllers/bucket-delete.go +++ b/s3api/controllers/bucket-delete.go @@ -29,9 +29,8 @@ func (c S3ApiController) DeleteBucketTagging(ctx fiber.Ctx) (*Response, error) { parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -39,7 +38,6 @@ func (c S3ApiController) DeleteBucketTagging(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.PutBucketTaggingAction}, IsPublicRequest: IsBucketPublic, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -64,16 +62,14 @@ func (c S3ApiController) DeleteBucketOwnershipControls(ctx fiber.Ctx) (*Response isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, Acc: acct, Bucket: bucket, Actions: []auth.Action{auth.PutBucketOwnershipControlsAction}, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -98,16 +94,14 @@ func (c S3ApiController) DeleteBucketPolicy(ctx fiber.Ctx) (*Response, error) { isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, Acc: acct, Bucket: bucket, Actions: []auth.Action{auth.DeleteBucketPolicyAction}, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -133,9 +127,8 @@ func (c S3ApiController) DeleteBucketCors(ctx fiber.Ctx) (*Response, error) { parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -143,7 +136,6 @@ func (c S3ApiController) DeleteBucketCors(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.PutBucketCorsAction}, IsPublicRequest: IsBucketPublic, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -169,9 +161,8 @@ func (c S3ApiController) DeleteBucketWebsite(ctx fiber.Ctx) (*Response, error) { parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -179,7 +170,6 @@ func (c S3ApiController) DeleteBucketWebsite(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.DeleteBucketWebsiteAction}, IsPublicRequest: IsBucketPublic, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -205,9 +195,8 @@ func (c S3ApiController) DeleteBucket(ctx fiber.Ctx) (*Response, error) { parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -215,7 +204,6 @@ func (c S3ApiController) DeleteBucket(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.DeleteBucketAction}, IsPublicRequest: IsBucketPublic, - DisableACL: c.disableACL, }) if err != nil { return &Response{ diff --git a/s3api/controllers/bucket-get.go b/s3api/controllers/bucket-get.go index 14962e8c..9e287fd7 100644 --- a/s3api/controllers/bucket-get.go +++ b/s3api/controllers/bucket-get.go @@ -32,8 +32,7 @@ func (c S3ApiController) GetBucketTagging(ctx fiber.Ctx) (*Response, error) { isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -41,7 +40,6 @@ func (c S3ApiController) GetBucketTagging(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.GetBucketTaggingAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -85,8 +83,7 @@ func (c S3ApiController) GetBucketOwnershipControls(ctx fiber.Ctx) (*Response, e isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -94,7 +91,6 @@ func (c S3ApiController) GetBucketOwnershipControls(ctx fiber.Ctx) (*Response, e Bucket: bucket, Actions: []auth.Action{auth.GetBucketOwnershipControlsAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -126,8 +122,7 @@ func (c S3ApiController) GetBucketVersioning(ctx fiber.Ctx) (*Response, error) { isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -135,7 +130,6 @@ func (c S3ApiController) GetBucketVersioning(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.GetBucketVersioningAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -169,8 +163,7 @@ func (c S3ApiController) GetBucketCors(ctx fiber.Ctx) (*Response, error) { isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -178,7 +171,6 @@ func (c S3ApiController) GetBucketCors(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.GetBucketCorsAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -213,8 +205,7 @@ func (c S3ApiController) GetBucketWebsite(ctx fiber.Ctx) (*Response, error) { isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -222,7 +213,6 @@ func (c S3ApiController) GetBucketWebsite(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.GetBucketWebsiteAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -257,8 +247,7 @@ func (c S3ApiController) GetBucketPolicy(ctx fiber.Ctx) (*Response, error) { isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -266,7 +255,6 @@ func (c S3ApiController) GetBucketPolicy(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.GetBucketPolicyAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -292,8 +280,7 @@ func (c S3ApiController) GetBucketPolicyStatus(ctx fiber.Ctx) (*Response, error) isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -301,7 +288,6 @@ func (c S3ApiController) GetBucketPolicyStatus(ctx fiber.Ctx) (*Response, error) Bucket: bucket, Actions: []auth.Action{auth.GetBucketPolicyStatusAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -354,8 +340,7 @@ func (c S3ApiController) ListObjectVersions(ctx fiber.Ctx) (*Response, error) { isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -363,7 +348,6 @@ func (c S3ApiController) ListObjectVersions(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.ListBucketVersionsAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -408,8 +392,7 @@ func (c S3ApiController) GetObjectLockConfiguration(ctx fiber.Ctx) (*Response, e isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -417,7 +400,6 @@ func (c S3ApiController) GetObjectLockConfiguration(ctx fiber.Ctx) (*Response, e Bucket: bucket, Actions: []auth.Action{auth.GetBucketObjectLockConfigurationAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -454,8 +436,7 @@ func (c S3ApiController) GetBucketAcl(ctx fiber.Ctx) (*Response, error) { isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionReadAcp, IsRoot: isRoot, @@ -463,7 +444,6 @@ func (c S3ApiController) GetBucketAcl(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.GetBucketAclAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -506,8 +486,7 @@ func (c S3ApiController) ListMultipartUploads(ctx fiber.Ctx) (*Response, error) isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -515,7 +494,6 @@ func (c S3ApiController) ListMultipartUploads(ctx fiber.Ctx) (*Response, error) Bucket: bucket, Actions: []auth.Action{auth.ListBucketMultipartUploadsAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -568,8 +546,7 @@ func (c S3ApiController) ListObjectsV2(ctx fiber.Ctx) (*Response, error) { region = defaultRegion } - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -577,7 +554,6 @@ func (c S3ApiController) ListObjectsV2(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.ListBucketAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -641,8 +617,7 @@ func (c S3ApiController) ListObjects(ctx fiber.Ctx) (*Response, error) { region = defaultRegion } - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -650,7 +625,6 @@ func (c S3ApiController) ListObjects(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.ListBucketAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -704,8 +678,7 @@ func (c S3ApiController) GetBucketLocation(ctx fiber.Ctx) (*Response, error) { isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -713,7 +686,6 @@ func (c S3ApiController) GetBucketLocation(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.GetBucketLocationAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ diff --git a/s3api/controllers/bucket-head.go b/s3api/controllers/bucket-head.go index f54214db..67c74c2f 100644 --- a/s3api/controllers/bucket-head.go +++ b/s3api/controllers/bucket-head.go @@ -32,9 +32,8 @@ func (c S3ApiController) HeadBucket(ctx fiber.Ctx) (*Response, error) { parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -42,7 +41,6 @@ func (c S3ApiController) HeadBucket(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.ListBucketAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ diff --git a/s3api/controllers/bucket-post.go b/s3api/controllers/bucket-post.go index 40dfec08..564fbf81 100644 --- a/s3api/controllers/bucket-post.go +++ b/s3api/controllers/bucket-post.go @@ -34,34 +34,19 @@ import ( func (c S3ApiController) DeleteObjects(ctx fiber.Ctx) (*Response, error) { bucket := ctx.Params("bucket") - bypass := strings.EqualFold(ctx.Get("X-Amz-Bypass-Governance-Retention"), "true") + bypass := auth.BypassModeForRequest(strings.EqualFold(ctx.Get("X-Amz-Bypass-Governance-Retention"), "true")) acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Actions: []auth.Action{auth.DeleteObjectAction}, - IsPublicRequest: IsBucketPublic, - DisableACL: c.disableACL, - }) - if err != nil { - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, err - } - + // The body has to be parsed before authorization, not after: real AWS + // authorizes s3:DeleteObject against each object's own ARN, so the keys + // are part of what is being authorized. The parsed objects go straight + // to VerifyObjectsAccess, which checks policy and object locks for all + // of them in one pass. var dObj s3response.DeleteObjects - err = xml.Unmarshal(ctx.BodyRaw(), &dObj) + err := xml.Unmarshal(ctx.BodyRaw(), &dObj) if err != nil { debuglogger.Logf("error unmarshalling delete objects: %v", err) return &Response{ @@ -71,7 +56,23 @@ func (c S3ApiController) DeleteObjects(ctx fiber.Ctx) (*Response, error) { }, s3err.GetAPIError(s3err.ErrInvalidRequest) } - err = auth.CheckObjectAccess(ctx.RequestCtx(), bucket, acct.Access, dObj.Objects, bypass, IsBucketPublic, c.be, false) + // checkErrs holds one entry per requested object — nil where it may + // proceed to the backend, an AWS-shaped denial otherwise. DeleteObjects + // supports partial success, so a denial on one object (policy or object + // lock) must not fail any other: only the objects that clear this check + // are sent to the backend, and the rest are reported as per-object + // errors directly from checkErrs. err here is a whole-request failure + // (readonly mode, or an error resolving policy/lock state), not about + // any one object. + checkErrs, err := c.verifyObjectsAccess(ctx, + auth.AccessOptions{ + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + IsPublicRequest: IsBucketPublic, + }, dObj.Objects, bypass) if err != nil { return &Response{ MetaOpts: &MetaOptions{ @@ -80,15 +81,26 @@ func (c S3ApiController) DeleteObjects(ctx fiber.Ctx) (*Response, error) { }, err } - res, err := c.be.DeleteObjects(ctx.RequestCtx(), - &s3.DeleteObjectsInput{ - Bucket: &bucket, - Delete: &types.Delete{ - Objects: dObj.Objects, - }, - }) + toDelete := make([]types.ObjectIdentifier, 0, len(dObj.Objects)) + for i, obj := range dObj.Objects { + if checkErrs[i] == nil { + toDelete = append(toDelete, obj) + } + } + + var backendResult s3response.DeleteResult + if len(toDelete) > 0 { + backendResult, err = c.be.DeleteObjects(ctx.RequestCtx(), + &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{ + Objects: toDelete, + }, + }) + } + return &Response{ - Data: res, + Data: utils.MergeDeleteObjectsResult(dObj.Objects, checkErrs, backendResult), MetaOpts: &MetaOptions{ ObjectCount: int64(len(dObj.Objects)), BucketOwner: parsedAcl.Owner, @@ -115,9 +127,8 @@ func (c S3ApiController) POSTObject(ctx fiber.Ctx) (*Response, error) { key := parsed.Fields["key"] - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -125,7 +136,6 @@ func (c S3ApiController) POSTObject(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.PutObjectAction}, IsPublicRequest: IsBucketPublic, - DisableACL: c.disableACL, }) if err != nil { return &Response{ diff --git a/s3api/controllers/bucket-post_test.go b/s3api/controllers/bucket-post_test.go index 4d4aa968..20a50dad 100644 --- a/s3api/controllers/bucket-post_test.go +++ b/s3api/controllers/bucket-post_test.go @@ -46,19 +46,36 @@ func TestS3ApiController_DeleteObjects(t *testing.T) { validRes := s3response.DeleteResult{ Deleted: []types.DeletedObject{ - {Key: utils.GetStringPtr("key")}, + {Key: utils.GetStringPtr("obj")}, }, } + partialSuccessBody, err := xml.Marshal(s3response.DeleteObjects{ + Objects: []types.ObjectIdentifier{ + {Key: utils.GetStringPtr("locked")}, + {Key: utils.GetStringPtr("ok")}, + }, + }) + assert.NoError(t, err) + + lockConfig, err := json.Marshal(auth.BucketLockConfig{Enabled: true}) + assert.NoError(t, err) + + legalHoldOn, legalHoldOff := true, false + lockedObjectCode := "AccessDenied" + lockedObjectMessage := "Access Denied because object protected by object lock." + tests := []struct { - name string - input testInput - output testOutput + name string + input testInput + output testOutput + configureMock func(be *BackendMock) }{ { name: "verify access fails", input: testInput{ locals: accessDeniedLocals, + body: validBody, }, output: testOutput{ response: &Response{ @@ -140,6 +157,56 @@ func TestS3ApiController_DeleteObjects(t *testing.T) { }, }, }, + { + name: "partial success: one object locked, one succeeds", + input: testInput{ + locals: defaultLocals, + body: partialSuccessBody, + }, + output: testOutput{ + response: &Response{ + Data: s3response.DeleteResult{ + Deleted: []types.DeletedObject{ + {Key: utils.GetStringPtr("ok")}, + }, + Error: []types.Error{ + {Key: utils.GetStringPtr("locked"), Code: &lockedObjectCode, Message: &lockedObjectMessage}, + }, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + EventName: s3event.EventObjectRemovedDeleteObjects, + ObjectCount: 2, + }, + }, + }, + configureMock: func(be *BackendMock) { + be.GetObjectLockConfigurationFunc = func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return lockConfig, nil + } + be.GetBucketVersioningFunc = func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) { + return s3response.GetBucketVersioningOutput{}, nil + } + be.GetObjectRetentionFunc = func(contextMoqParam context.Context, bucket, object, versionId string) ([]byte, error) { + return []byte("{}"), nil + } + be.GetObjectLegalHoldFunc = func(contextMoqParam context.Context, bucket, object, versionId string) (*bool, error) { + if object == "locked" { + return &legalHoldOn, nil + } + return &legalHoldOff, nil + } + be.DeleteObjectsFunc = func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) { + // Only the object that cleared the lock check should + // ever reach the backend. + assert.Len(t, deleteObjectsInput.Delete.Objects, 1) + assert.Equal(t, "ok", *deleteObjectsInput.Delete.Objects[0].Key) + return s3response.DeleteResult{ + Deleted: []types.DeletedObject{{Key: utils.GetStringPtr("ok")}}, + }, nil + } + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -154,6 +221,9 @@ func TestS3ApiController_DeleteObjects(t *testing.T) { return nil, tt.input.extraMockErr }, } + if tt.configureMock != nil { + tt.configureMock(be) + } ctrl := S3ApiController{ be: be, diff --git a/s3api/controllers/bucket-put.go b/s3api/controllers/bucket-put.go index f3edbefa..2b507abd 100644 --- a/s3api/controllers/bucket-put.go +++ b/s3api/controllers/bucket-put.go @@ -37,8 +37,7 @@ func (c S3ApiController) PutBucketTagging(ctx fiber.Ctx) (*Response, error) { isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -46,7 +45,6 @@ func (c S3ApiController) PutBucketTagging(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.PutBucketTaggingAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -80,15 +78,13 @@ func (c S3ApiController) PutBucketOwnershipControls(ctx fiber.Ctx) (*Response, e acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - if err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + if err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, Acc: acct, Bucket: bucket, Actions: []auth.Action{auth.PutBucketOwnershipControlsAction}, - DisableACL: c.disableACL, }); err != nil { return &Response{ MetaOpts: &MetaOptions{ @@ -140,8 +136,7 @@ func (c S3ApiController) PutBucketVersioning(ctx fiber.Ctx) (*Response, error) { isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -149,7 +144,6 @@ func (c S3ApiController) PutBucketVersioning(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.PutBucketVersioningAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -195,8 +189,7 @@ func (c S3ApiController) PutObjectLockConfiguration(ctx fiber.Ctx) (*Response, e isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) - if err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + if err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -204,7 +197,6 @@ func (c S3ApiController) PutObjectLockConfiguration(ctx fiber.Ctx) (*Response, e Bucket: bucket, Actions: []auth.Action{auth.PutBucketObjectLockConfigurationAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }); err != nil { return &Response{ MetaOpts: &MetaOptions{ @@ -237,8 +229,7 @@ func (c S3ApiController) PutBucketCors(ctx fiber.Ctx) (*Response, error) { isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -246,7 +237,6 @@ func (c S3ApiController) PutBucketCors(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.PutBucketCorsAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -294,8 +284,7 @@ func (c S3ApiController) PutBucketWebsite(ctx fiber.Ctx) (*Response, error) { isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -303,7 +292,6 @@ func (c S3ApiController) PutBucketWebsite(ctx fiber.Ctx) (*Response, error) { Bucket: bucket, Actions: []auth.Action{auth.PutBucketWebsiteAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -357,15 +345,13 @@ func (c S3ApiController) PutBucketPolicy(ctx fiber.Ctx) (*Response, error) { acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, Acc: acct, Bucket: bucket, Actions: []auth.Action{auth.PutBucketPolicyAction}, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -409,16 +395,14 @@ func (c S3ApiController) PutBucketAcl(ctx fiber.Ctx) (*Response, error) { grants := grantFullControl + grantRead + grantReadACP + grantWrite + grantWriteACP var input *auth.PutBucketAclInput - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionWriteAcp, IsRoot: isRoot, Acc: acct, Bucket: bucket, Actions: []auth.Action{auth.PutBucketAclAction}, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -585,11 +569,12 @@ func (c S3ApiController) CreateBucket(ctx fiber.Ctx) (*Response, error) { utils.ContextKeyBucketOwner.Set(ctx, creator) } bucketOwner := utils.ContextKeyBucketOwner.Get(ctx).(auth.Account) + isRoot, _ := utils.ContextKeyIsRoot.Get(ctx).(bool) - if creator.Role != auth.RoleAdmin && creator.Role != auth.RoleUserPlus { + if err := auth.VerifyCreateBucketAccess(ctx, c.iam, isRoot, creator, bucket); err != nil { return &Response{ MetaOpts: &MetaOptions{}, - }, s3err.GetAPIError(s3err.ErrAccessDenied) + }, err } // validate the bucket name diff --git a/s3api/controllers/bucket-put_test.go b/s3api/controllers/bucket-put_test.go index c699eb8a..b4045881 100644 --- a/s3api/controllers/bucket-put_test.go +++ b/s3api/controllers/bucket-put_test.go @@ -716,6 +716,10 @@ func TestS3ApiController_CreateBucket(t *testing.T) { Access: "user", Role: auth.RoleUser, } + userPlusAcc := auth.Account{ + Access: "userplus", + Role: auth.RoleUserPlus, + } invLocConstBody, err := xml.Marshal(s3response.CreateBucketConfiguration{ LocationConstraint: utils.GetStringPtr("us-west-1"), @@ -732,6 +736,7 @@ func TestS3ApiController_CreateBucket(t *testing.T) { input: testInput{ locals: map[utils.ContextKey]any{ utils.ContextKeyAccount: userAcc, + utils.ContextKeyIsRoot: false, }, }, output: testOutput{ @@ -916,6 +921,47 @@ func TestS3ApiController_CreateBucket(t *testing.T) { }, }, }, + { + name: "userplus role can create bucket", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyAccount: userPlusAcc, + }, + bucket: "my-bucket", + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: userPlusAcc.Access, + }, + Headers: map[string]*string{ + "Location": utils.GetStringPtr("/my-bucket"), + "x-amz-bucket-arn": utils.GetStringPtr("arn:aws:s3:::my-bucket"), + }, + }, + }, + }, + { + name: "root bypasses role check", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyAccount: userAcc, + utils.ContextKeyIsRoot: true, + }, + bucket: "my-bucket", + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: userAcc.Access, + }, + Headers: map[string]*string{ + "Location": utils.GetStringPtr("/my-bucket"), + "x-amz-bucket-arn": utils.GetStringPtr("arn:aws:s3:::my-bucket"), + }, + }, + }, + }, } for _, tt := range tests { diff --git a/s3api/controllers/iam_moq_test.go b/s3api/controllers/iam_moq_test.go index b131ec2e..c2150dfe 100644 --- a/s3api/controllers/iam_moq_test.go +++ b/s3api/controllers/iam_moq_test.go @@ -30,6 +30,9 @@ var _ auth.IAMService = &IAMServiceMock{} // ListUserAccountsFunc: func() ([]auth.Account, error) { // panic("mock out the ListUserAccounts method") // }, +// ResolveAccountsFunc: func(accessKeyIDs []string) ([]string, error) { +// panic("mock out the ResolveAccounts method") +// }, // ShutdownFunc: func() error { // panic("mock out the Shutdown method") // }, @@ -55,6 +58,9 @@ type IAMServiceMock struct { // ListUserAccountsFunc mocks the ListUserAccounts method. ListUserAccountsFunc func() ([]auth.Account, error) + // ResolveAccountsFunc mocks the ResolveAccounts method. + ResolveAccountsFunc func(accessKeyIDs []string) ([]string, error) + // ShutdownFunc mocks the Shutdown method. ShutdownFunc func() error @@ -81,6 +87,11 @@ type IAMServiceMock struct { // ListUserAccounts holds details about calls to the ListUserAccounts method. ListUserAccounts []struct { } + // ResolveAccounts holds details about calls to the ResolveAccounts method. + ResolveAccounts []struct { + // AccessKeyIDs is the accessKeyIDs argument value. + AccessKeyIDs []string + } // Shutdown holds details about calls to the Shutdown method. Shutdown []struct { } @@ -96,6 +107,7 @@ type IAMServiceMock struct { lockDeleteUserAccount sync.RWMutex lockGetUserAccount sync.RWMutex lockListUserAccounts sync.RWMutex + lockResolveAccounts sync.RWMutex lockShutdown sync.RWMutex lockUpdateUserAccount sync.RWMutex } @@ -223,6 +235,38 @@ func (mock *IAMServiceMock) ListUserAccountsCalls() []struct { return calls } +// ResolveAccounts calls ResolveAccountsFunc. +func (mock *IAMServiceMock) ResolveAccounts(accessKeyIDs []string) ([]string, error) { + if mock.ResolveAccountsFunc == nil { + panic("IAMServiceMock.ResolveAccountsFunc: method is nil but IAMService.ResolveAccounts was just called") + } + callInfo := struct { + AccessKeyIDs []string + }{ + AccessKeyIDs: accessKeyIDs, + } + mock.lockResolveAccounts.Lock() + mock.calls.ResolveAccounts = append(mock.calls.ResolveAccounts, callInfo) + mock.lockResolveAccounts.Unlock() + return mock.ResolveAccountsFunc(accessKeyIDs) +} + +// ResolveAccountsCalls gets all the calls that were made to ResolveAccounts. +// Check the length with: +// +// len(mockedIAMService.ResolveAccountsCalls()) +func (mock *IAMServiceMock) ResolveAccountsCalls() []struct { + AccessKeyIDs []string +} { + var calls []struct { + AccessKeyIDs []string + } + mock.lockResolveAccounts.RLock() + calls = mock.calls.ResolveAccounts + mock.lockResolveAccounts.RUnlock() + return calls +} + // Shutdown calls ShutdownFunc. func (mock *IAMServiceMock) Shutdown() error { if mock.ShutdownFunc == nil { diff --git a/s3api/controllers/object-delete.go b/s3api/controllers/object-delete.go index 111b3957..58bf0f54 100644 --- a/s3api/controllers/object-delete.go +++ b/s3api/controllers/object-delete.go @@ -41,9 +41,8 @@ func (c S3ApiController) DeleteObjectTagging(ctx fiber.Ctx) (*Response, error) { action = auth.DeleteObjectVersionTaggingAction } - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -52,7 +51,6 @@ func (c S3ApiController) DeleteObjectTagging(ctx fiber.Ctx) (*Response, error) { Object: key, Actions: []auth.Action{action}, IsPublicRequest: isBucketPublic, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -85,9 +83,8 @@ func (c S3ApiController) AbortMultipartUpload(ctx fiber.Ctx) (*Response, error) isBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -96,7 +93,6 @@ func (c S3ApiController) AbortMultipartUpload(ctx fiber.Ctx) (*Response, error) Object: key, Actions: []auth.Action{auth.AbortMultipartUploadAction}, IsPublicRequest: isBucketPublic, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -125,7 +121,7 @@ func (c S3ApiController) DeleteObject(ctx fiber.Ctx) (*Response, error) { bucket := ctx.Params("bucket") key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) versionId := ctx.Query("versionId") - bypass := strings.EqualFold(ctx.Get("X-Amz-Bypass-Governance-Retention"), "true") + bypass := auth.BypassModeForRequest(strings.EqualFold(ctx.Get("X-Amz-Bypass-Governance-Retention"), "true")) ifMatch := utils.GetStringPtr(strings.Trim(ctx.Get("If-Match"), `"`)) ifMatchLastModTime := utils.ParsePreconditionDateHeader(ctx.Get("X-Amz-If-Match-Last-Modified-Time")) ifMatchSize := utils.ParseIfMatchSize(ctx) @@ -140,9 +136,8 @@ func (c S3ApiController) DeleteObject(ctx fiber.Ctx) (*Response, error) { action = auth.DeleteObjectVersionAction } - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -151,7 +146,6 @@ func (c S3ApiController) DeleteObject(ctx fiber.Ctx) (*Response, error) { Object: key, Actions: []auth.Action{action}, IsPublicRequest: isBucketPublic, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -162,9 +156,9 @@ func (c S3ApiController) DeleteObject(ctx fiber.Ctx) (*Response, error) { } err = auth.CheckObjectAccess( - ctx.RequestCtx(), + ctx, bucket, - acct.Access, + acct, []types.ObjectIdentifier{ { Key: &key, @@ -174,6 +168,7 @@ func (c S3ApiController) DeleteObject(ctx fiber.Ctx) (*Response, error) { bypass, isBucketPublic, c.be, + c.iam, false, ) if err != nil { diff --git a/s3api/controllers/object-get.go b/s3api/controllers/object-get.go index 51f29bd8..c109d499 100644 --- a/s3api/controllers/object-get.go +++ b/s3api/controllers/object-get.go @@ -45,8 +45,7 @@ func (c S3ApiController) GetObjectTagging(ctx fiber.Ctx) (*Response, error) { action = auth.GetObjectVersionTaggingAction } - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -55,7 +54,6 @@ func (c S3ApiController) GetObjectTagging(ctx fiber.Ctx) (*Response, error) { Object: key, Actions: []auth.Action{action}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -103,8 +101,7 @@ func (c S3ApiController) GetObjectRetention(ctx fiber.Ctx) (*Response, error) { parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -113,7 +110,6 @@ func (c S3ApiController) GetObjectRetention(ctx fiber.Ctx) (*Response, error) { Object: key, Actions: []auth.Action{auth.GetObjectRetentionAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -151,8 +147,7 @@ func (c S3ApiController) GetObjectLegalHold(ctx fiber.Ctx) (*Response, error) { parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -161,7 +156,6 @@ func (c S3ApiController) GetObjectLegalHold(ctx fiber.Ctx) (*Response, error) { Object: key, Actions: []auth.Action{auth.GetObjectLegalHoldAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -189,8 +183,7 @@ func (c S3ApiController) GetObjectAcl(ctx fiber.Ctx) (*Response, error) { parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionReadAcp, IsRoot: isRoot, @@ -199,7 +192,6 @@ func (c S3ApiController) GetObjectAcl(ctx fiber.Ctx) (*Response, error) { Object: key, Actions: []auth.Action{auth.GetObjectAclAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -232,8 +224,7 @@ func (c S3ApiController) ListParts(ctx fiber.Ctx) (*Response, error) { parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -242,7 +233,6 @@ func (c S3ApiController) ListParts(ctx fiber.Ctx) (*Response, error) { Object: key, Actions: []auth.Action{auth.ListMultipartUploadPartsAction}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -304,8 +294,7 @@ func (c S3ApiController) GetObjectAttributes(ctx fiber.Ctx) (*Response, error) { action = auth.GetObjectVersionAttributesAction } - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -314,7 +303,6 @@ func (c S3ApiController) GetObjectAttributes(ctx fiber.Ctx) (*Response, error) { Object: key, Actions: []auth.Action{action}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -429,8 +417,7 @@ func (c S3ApiController) GetObject(ctx fiber.Ctx) (*Response, error) { action = auth.GetObjectVersionAction } - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -439,7 +426,6 @@ func (c S3ApiController) GetObject(ctx fiber.Ctx) (*Response, error) { Object: key, Actions: []auth.Action{action}, IsPublicRequest: isPublicBucketRequest, - DisableACL: c.disableACL, }) if err != nil { return &Response{ diff --git a/s3api/controllers/object-head.go b/s3api/controllers/object-head.go index 5974e638..8776ae64 100644 --- a/s3api/controllers/object-head.go +++ b/s3api/controllers/object-head.go @@ -76,9 +76,8 @@ func (c S3ApiController) HeadObject(ctx fiber.Ctx) (*Response, error) { action = auth.GetObjectVersionAction } - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -87,7 +86,6 @@ func (c S3ApiController) HeadObject(ctx fiber.Ctx) (*Response, error) { Object: key, Actions: []auth.Action{action}, IsPublicRequest: isPublicBucket, - DisableACL: c.disableACL, }) if err != nil { return &Response{ diff --git a/s3api/controllers/object-post.go b/s3api/controllers/object-post.go index fb2052db..bc785972 100644 --- a/s3api/controllers/object-post.go +++ b/s3api/controllers/object-post.go @@ -39,9 +39,8 @@ func (c S3ApiController) RestoreObject(ctx fiber.Ctx) (*Response, error) { isBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -50,7 +49,6 @@ func (c S3ApiController) RestoreObject(ctx fiber.Ctx) (*Response, error) { Object: key, Actions: []auth.Action{auth.RestoreObjectAction}, IsPublicRequest: isBucketPublic, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -91,9 +89,8 @@ func (c S3ApiController) SelectObjectContent(ctx fiber.Ctx) (*Response, error) { isBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionRead, IsRoot: isRoot, @@ -102,7 +99,6 @@ func (c S3ApiController) SelectObjectContent(ctx fiber.Ctx) (*Response, error) { Object: key, Actions: []auth.Action{auth.GetObjectAction}, IsPublicRequest: isBucketPublic, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -175,9 +171,8 @@ func (c S3ApiController) CreateMultipartUpload(ctx fiber.Ctx) (*Response, error) actions = append(actions, auth.PutObjectRetentionAction) } - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -185,7 +180,6 @@ func (c S3ApiController) CreateMultipartUpload(ctx fiber.Ctx) (*Response, error) Bucket: bucket, Object: key, Actions: actions, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -278,9 +272,8 @@ func (c S3ApiController) CompleteMultipartUpload(ctx fiber.Ctx) (*Response, erro isBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -289,7 +282,6 @@ func (c S3ApiController) CompleteMultipartUpload(ctx fiber.Ctx) (*Response, erro Object: key, Actions: []auth.Action{auth.PutObjectAction}, IsPublicRequest: isBucketPublic, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -363,7 +355,7 @@ func (c S3ApiController) CompleteMultipartUpload(ctx fiber.Ctx) (*Response, erro ifMatch, ifNoneMatch := utils.ParsePreconditionMatchHeaders(ctx) - err = auth.CheckObjectAccess(ctx.RequestCtx(), bucket, acct.Access, []types.ObjectIdentifier{{Key: &key}}, true, isBucketPublic, c.be, true) + err = auth.CheckObjectAccess(ctx, bucket, acct, []types.ObjectIdentifier{{Key: &key}}, auth.BypassOverwrite, isBucketPublic, c.be, c.iam, true) if err != nil { return &Response{ MetaOpts: &MetaOptions{ diff --git a/s3api/controllers/object-put.go b/s3api/controllers/object-put.go index 12331cd3..f88260ce 100644 --- a/s3api/controllers/object-put.go +++ b/s3api/controllers/object-put.go @@ -47,8 +47,7 @@ func (c S3ApiController) PutObjectTagging(ctx fiber.Ctx) (*Response, error) { action = auth.PutObjectVersionTaggingAction } - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -57,7 +56,6 @@ func (c S3ApiController) PutObjectTagging(ctx fiber.Ctx) (*Response, error) { Object: key, Actions: []auth.Action{action}, IsPublicRequest: IsBucketPublic, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -98,8 +96,7 @@ func (c S3ApiController) PutObjectRetention(ctx fiber.Ctx) (*Response, error) { IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -108,7 +105,6 @@ func (c S3ApiController) PutObjectRetention(ctx fiber.Ctx) (*Response, error) { Object: key, Actions: []auth.Action{auth.PutObjectRetentionAction}, IsPublicRequest: IsBucketPublic, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -129,7 +125,7 @@ func (c S3ApiController) PutObjectRetention(ctx fiber.Ctx) (*Response, error) { } // check if the operation is allowed - err = auth.IsObjectLockRetentionPutAllowed(ctx.RequestCtx(), c.be, bucket, key, versionId, acct.Access, retention, bypass) + err = auth.IsObjectLockRetentionPutAllowed(ctx, c.be, c.iam, bucket, key, versionId, acct, retention, bypass) if err != nil { return &Response{ MetaOpts: &MetaOptions{ @@ -165,8 +161,7 @@ func (c S3ApiController) PutObjectLegalHold(ctx fiber.Ctx) (*Response, error) { IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, auth.AccessOptions{ - Readonly: c.readonly, + err := c.verifyAccess(ctx, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -175,7 +170,6 @@ func (c S3ApiController) PutObjectLegalHold(ctx fiber.Ctx) (*Response, error) { Object: key, Actions: []auth.Action{auth.PutObjectLegalHoldAction}, IsPublicRequest: IsBucketPublic, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -234,9 +228,8 @@ func (c S3ApiController) UploadPart(ctx fiber.Ctx) (*Response, error) { contentLengthStr = decodedLength } - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -245,7 +238,6 @@ func (c S3ApiController) UploadPart(ctx fiber.Ctx) (*Response, error) { Object: key, Actions: []auth.Action{auth.PutObjectAction}, IsPublicRequest: IsBucketPublic, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -361,7 +353,7 @@ func (c S3ApiController) UploadPartCopy(ctx fiber.Ctx) (*Response, error) { }, err } - err = auth.VerifyObjectCopyAccess(ctx.RequestCtx(), c.be, copySource, + err = c.verifyObjectCopyAccess(ctx, copySource, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionWrite, @@ -371,7 +363,6 @@ func (c S3ApiController) UploadPartCopy(ctx fiber.Ctx) (*Response, error) { Object: key, Actions: []auth.Action{auth.PutObjectAction}, IsPublicRequest: IsBucketPublic, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -444,9 +435,8 @@ func (c S3ApiController) PutObjectAcl(ctx fiber.Ctx) (*Response, error) { isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -525,7 +515,7 @@ func (c S3ApiController) CopyObject(ctx fiber.Ctx) (*Response, error) { actions = append(actions, auth.PutObjectRetentionAction) } - err = auth.VerifyObjectCopyAccess(ctx.RequestCtx(), c.be, copySource, + err = c.verifyObjectCopyAccess(ctx, copySource, auth.AccessOptions{ Acl: parsedAcl, AclPermission: auth.PermissionWrite, @@ -609,7 +599,7 @@ func (c S3ApiController) CopyObject(ctx fiber.Ctx) (*Response, error) { preconditionHdrs := utils.ParsePreconditionHeaders(ctx, utils.WithCopySource()) - err = auth.CheckObjectAccess(ctx.RequestCtx(), bucket, acct.Access, []types.ObjectIdentifier{{Key: &key}}, true, false, c.be, true) + err = auth.CheckObjectAccess(ctx, bucket, acct, []types.ObjectIdentifier{{Key: &key}}, auth.BypassOverwrite, false, c.be, c.iam, true) if err != nil { return &Response{ MetaOpts: &MetaOptions{ @@ -710,9 +700,8 @@ func (c S3ApiController) PutObject(ctx fiber.Ctx) (*Response, error) { actions = append(actions, auth.PutObjectRetentionAction) } - err := auth.VerifyAccess(ctx.RequestCtx(), c.be, + err := c.verifyAccess(ctx, auth.AccessOptions{ - Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionWrite, IsRoot: isRoot, @@ -721,7 +710,6 @@ func (c S3ApiController) PutObject(ctx fiber.Ctx) (*Response, error) { Object: key, Actions: actions, IsPublicRequest: IsBucketPublic, - DisableACL: c.disableACL, }) if err != nil { return &Response{ @@ -750,7 +738,7 @@ func (c S3ApiController) PutObject(ctx fiber.Ctx) (*Response, error) { }, err } - err = auth.CheckObjectAccess(ctx.RequestCtx(), bucket, acct.Access, []types.ObjectIdentifier{{Key: &key}}, true, IsBucketPublic, c.be, true) + err = auth.CheckObjectAccess(ctx, bucket, acct, []types.ObjectIdentifier{{Key: &key}}, auth.BypassOverwrite, IsBucketPublic, c.be, c.iam, true) if err != nil { return &Response{ MetaOpts: &MetaOptions{ diff --git a/s3api/controllers/object-put_test.go b/s3api/controllers/object-put_test.go index deb3641f..4aa7347a 100644 --- a/s3api/controllers/object-put_test.go +++ b/s3api/controllers/object-put_test.go @@ -603,6 +603,27 @@ func TestS3ApiController_UploadPartCopy(t *testing.T) { err: s3err.GetAPIError(s3err.ErrAccessDenied), }, }, + { + name: "readonly mode blocks upload part copy", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "X-Amz-Copy-Source": "bucket/key", + }, + queries: map[string]string{ + "partNumber": "2", + }, + readonly: true, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, { name: "invalid copy source", input: testInput{ @@ -729,7 +750,9 @@ func TestS3ApiController_UploadPartCopy(t *testing.T) { } ctrl := S3ApiController{ - be: be, + be: be, + readonly: tt.input.readonly, + disableACL: tt.input.disableACL, } testController( @@ -805,7 +828,7 @@ func TestS3ApiController_PutObjectAcl(t *testing.T) { return tt.input.beErr }, GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { - return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + return nil, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy) }, } @@ -849,6 +872,58 @@ func TestS3ApiController_CopyObject(t *testing.T) { err: s3err.GetAPIError(s3err.ErrAccessDenied), }, }, + { + name: "readonly mode blocks copy object", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "X-Amz-Copy-Source": "bucket/object", + }, + readonly: true, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "disableACL blocks a non-owner's grantee-based access", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyIsRoot: false, + utils.ContextKeyParsedAcl: auth.ACL{ + Owner: "root", + Grantees: []auth.Grantee{ + { + Access: "user", + Permission: auth.PermissionWrite, + Type: types.TypeCanonicalUser, + }, + }, + }, + utils.ContextKeyAccount: auth.Account{ + Access: "user", + Role: auth.RoleUser, + }, + }, + headers: map[string]string{ + "X-Amz-Copy-Source": "bucket/object", + }, + disableACL: true, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, { name: "invalid copy source", input: testInput{ @@ -1059,7 +1134,7 @@ func TestS3ApiController_CopyObject(t *testing.T) { return tt.input.beRes.(s3response.CopyObjectOutput), tt.input.beErr }, GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { - return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + return nil, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy) }, GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) { return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented) @@ -1070,7 +1145,9 @@ func TestS3ApiController_CopyObject(t *testing.T) { } ctrl := S3ApiController{ - be: be, + be: be, + readonly: tt.input.readonly, + disableACL: tt.input.disableACL, } testController( diff --git a/s3api/middlewares/authentication.go b/s3api/middlewares/authentication.go index 2f1cea90..7d8ab356 100644 --- a/s3api/middlewares/authentication.go +++ b/s3api/middlewares/authentication.go @@ -17,12 +17,14 @@ package middlewares import ( "crypto/sha256" "encoding/hex" + "errors" "io" "strconv" "time" "github.com/gofiber/fiber/v3" "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/internal/sigv4auth" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" ) @@ -39,7 +41,7 @@ type RootUserConfig struct { } func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string, streamBody, requireContentSha256, allowDefaultRegion bool) fiber.Handler { - acct := accounts{root: root, iam: iam} + rootAccount := auth.Account{Access: root.Access, Secret: root.Secret, Role: auth.RoleAdmin} return func(ctx fiber.Ctx) error { // The bucket is public, no need to check this signature @@ -89,10 +91,15 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string, utils.ContextKeyIsRoot.Set(ctx, authData.Access == root.Access) - account, err := acct.getAccount(authData.Access) + sessionToken := ctx.Get(sigv4auth.HeaderSecurityToken) + + derivedKey, account, err := auth.ResolveDerivedKey(iam, rootAccount, authData.Access, sessionToken, authData.Date, authData.Region, sigv4auth.ServiceS3) if err == auth.ErrNoSuchUser { return s3err.GetInvalidAccessKeyIdErr(authData.Access) } + if errors.Is(err, auth.ErrInvalidSessionToken) { + return s3err.GetAPIError(s3err.ErrInvalidToken) + } if err != nil { return err } @@ -126,7 +133,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string, return s3err.GetAPIError(s3err.ErrInvalidSHA256PayloadUsage) } - canonicalString, err := utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength) + canonicalString, err := utils.CheckValidSignature(ctx, authData, derivedKey, hashPayload, tdate, contentLength) if err != nil { return err } @@ -153,7 +160,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string, if utils.IsStreamingPayload(hashPayload) { wrapBodyReader(ctx, func(r io.Reader) io.Reader { var cr io.Reader - cr, err = utils.NewChunkReader(ctx, r, authData, canonicalString, account.Secret, tdate) + cr, err = utils.NewChunkReader(ctx, r, authData, canonicalString, derivedKey, tdate) return cr }) if err != nil { @@ -190,20 +197,3 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string, return nil } } - -type accounts struct { - root RootUserConfig - iam auth.IAMService -} - -func (a accounts) getAccount(access string) (auth.Account, error) { - if access == a.root.Access { - return auth.Account{ - Access: a.root.Access, - Secret: a.root.Secret, - Role: auth.RoleAdmin, - }, nil - } - - return a.iam.GetUserAccount(access) -} diff --git a/s3api/middlewares/host-style-parser.go b/s3api/middlewares/host-style-parser.go index e0d5afc2..80966cc7 100644 --- a/s3api/middlewares/host-style-parser.go +++ b/s3api/middlewares/host-style-parser.go @@ -19,6 +19,7 @@ import ( "strings" "github.com/gofiber/fiber/v3" + "github.com/versity/versitygw/internal/httpctx" ) // HostStyleParser is a middleware which parses the bucket name @@ -31,6 +32,11 @@ func HostStyleParser(virtualDomain string) fiber.Handler { if !found || bucket == "" { return ctx.Next() } + // SigV4 verification signs the request's original, on-the-wire path, + // not the bucket-prefixed one used for routing here — save it + // before ctx.Path() below overwrites fasthttp's URI.PathOriginal too. + httpctx.ContextKeyOriginalURIPath.Set(ctx, string(ctx.Request().URI().PathOriginal())) + path := ctx.Path() if path == "/" { // omit the trailing / for bucket operations diff --git a/s3api/middlewares/object-post-auth.go b/s3api/middlewares/object-post-auth.go index 6e55f3a9..34494b00 100644 --- a/s3api/middlewares/object-post-auth.go +++ b/s3api/middlewares/object-post-auth.go @@ -16,6 +16,7 @@ package middlewares import ( "bytes" + "errors" "mime" "strconv" "time" @@ -23,16 +24,18 @@ import ( "github.com/gofiber/fiber/v3" "github.com/versity/versitygw/auth" "github.com/versity/versitygw/debuglogger" + "github.com/versity/versitygw/internal/sigv4auth" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" ) const ( - formFieldPolicy = "policy" - formFieldAlgorithm = "x-amz-algorithm" - formFieldCredential = "x-amz-credential" - formFieldDate = "x-amz-date" - formFieldSignature = "x-amz-signature" + formFieldPolicy = "policy" + formFieldAlgorithm = "x-amz-algorithm" + formFieldCredential = "x-amz-credential" + formFieldDate = "x-amz-date" + formFieldSignature = "x-amz-signature" + formFieldSecurityToken = "x-amz-security-token" aws4HMACSHA256 = "AWS4-HMAC-SHA256" hourSeconds = 60 * 60 @@ -47,7 +50,7 @@ type PostObjectResult struct { } func AuthorizePostObject(root RootUserConfig, iam auth.IAMService, region string) fiber.Handler { - acct := accounts{root: root, iam: iam} + rootAccount := auth.Account{Access: root.Access, Secret: root.Secret, Role: auth.RoleAdmin} return func(ctx fiber.Ctx) error { contentLengthStr := ctx.Get("Content-Length") @@ -164,11 +167,15 @@ func AuthorizePostObject(root RootUserConfig, iam auth.IAMService, region string return s3err.PostAuth.IncorrectRegion(credentialStr, region, creds.Region) } - account, err := acct.getAccount(creds.Access) + derivedKey, account, err := auth.ResolveDerivedKey(iam, rootAccount, creds.Access, fields[formFieldSecurityToken], creds.Date, creds.Region, sigv4auth.ServiceS3) if err == auth.ErrNoSuchUser { debuglogger.Logf("POST object access key not found: %s", creds.Access) return s3err.GetInvalidAccessKeyIdErr(creds.Access) } + if errors.Is(err, auth.ErrInvalidSessionToken) { + debuglogger.Logf("invalid POST object security token for access key %s", creds.Access) + return s3err.GetAPIError(s3err.ErrInvalidToken) + } if err != nil { debuglogger.Logf("failed to resolve POST object account %q: %v", creds.Access, err) return err @@ -177,7 +184,7 @@ func AuthorizePostObject(root RootUserConfig, iam auth.IAMService, region string utils.ContextKeyAccount.Set(ctx, account) utils.ContextKeyIsRoot.Set(ctx, account.Access == root.Access) - expectedSig, err := utils.SignPostPolicy(policyB64, creds.Date, region, account.Secret) + expectedSig, err := utils.SignPostPolicy(policyB64, derivedKey) if err != nil { return err } diff --git a/s3api/middlewares/object-post-auth_test.go b/s3api/middlewares/object-post-auth_test.go index db278747..6e9e7877 100644 --- a/s3api/middlewares/object-post-auth_test.go +++ b/s3api/middlewares/object-post-auth_test.go @@ -28,6 +28,7 @@ import ( "github.com/gofiber/fiber/v3" "github.com/stretchr/testify/assert" + "github.com/versity/versitygw/internal/sigv4auth" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" ) @@ -186,7 +187,8 @@ func TestAuthorizePostObject_SignedRequest(t *testing.T) { map[string]string{"bucket": "mybucket"}, []any{"starts-with", "$key", "uploads/"}, }) - sig, err := utils.SignPostPolicy(policyB64, dateShort, region, secretKey) + derivedKey := sigv4auth.DeriveKey(secretKey, dateShort, region, sigv4auth.ServiceS3) + sig, err := utils.SignPostPolicy(policyB64, derivedKey) assert.NoError(t, err) var gotAuthenticated bool diff --git a/s3api/middlewares/presign-auth.go b/s3api/middlewares/presign-auth.go index ceae9bd3..29a416fa 100644 --- a/s3api/middlewares/presign-auth.go +++ b/s3api/middlewares/presign-auth.go @@ -15,17 +15,19 @@ package middlewares import ( + "errors" "io" "strconv" "github.com/gofiber/fiber/v3" "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/internal/sigv4auth" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" ) func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region string, streamBody bool) fiber.Handler { - acct := accounts{root: root, iam: iam} + rootAccount := auth.Account{Access: root.Access, Secret: root.Secret, Role: auth.RoleAdmin} return func(ctx fiber.Ctx) error { // The bucket is public, no need to check this signature @@ -40,11 +42,6 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region return s3err.GetAPIError(s3err.ErrUnsupportedAuthorizationMechanism) } - if ctx.Request().URI().QueryArgs().Has("X-Amz-Security-Token") { - // OIDC Authorization with X-Amz-Security-Token is not supported - return s3err.QueryAuthErrors.SecurityTokenNotSupported() - } - // Set in the context the "authenticated" key, in case the authentication succeeds, // otherwise the middleware will return the caucht error utils.ContextKeyAuthenticated.Set(ctx, true) @@ -56,10 +53,15 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region utils.ContextKeyIsRoot.Set(ctx, authData.Access == root.Access) - account, err := acct.getAccount(authData.Access) + sessionToken := ctx.Query(sigv4auth.QuerySecurityToken) + + derivedKey, account, err := auth.ResolveDerivedKey(iam, rootAccount, authData.Access, sessionToken, authData.Date[:8], authData.Region, sigv4auth.ServiceS3) if err == auth.ErrNoSuchUser { return s3err.GetInvalidAccessKeyIdErr(authData.Access) } + if errors.Is(err, auth.ErrInvalidSessionToken) { + return s3err.GetAPIError(s3err.ErrInvalidToken) + } if err != nil { return err } @@ -75,7 +77,7 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region } } - err = utils.CheckPresignedSignature(ctx, authData, account.Secret) + err = utils.CheckPresignedSignature(ctx, authData, derivedKey) if err != nil { return err } diff --git a/s3api/middlewares/public-bucket.go b/s3api/middlewares/public-bucket.go index 28be6bed..ac39bc37 100644 --- a/s3api/middlewares/public-bucket.go +++ b/s3api/middlewares/public-bucket.go @@ -57,7 +57,7 @@ func AuthorizePublicBucketAccess(be backend.Backend, s3action string, policyPerm } bucket, object := parsePath(ctx.Path()) - err := auth.VerifyPublicAccess(ctx.RequestCtx(), be, policyPermission, permission, bucket, object) + err := auth.VerifyPublicAccess(ctx, be, policyPermission, permission, bucket, object) if err != nil { if s3action == metrics.ActionHeadBucket { // add the bucket region header for HeadBucket diff --git a/s3api/server.go b/s3api/server.go index c0d542ee..e8563f3b 100644 --- a/s3api/server.go +++ b/s3api/server.go @@ -29,6 +29,7 @@ import ( "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/debuglogger" + "github.com/versity/versitygw/internal/netutil" "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/controllers" "github.com/versity/versitygw/s3api/middlewares" @@ -48,7 +49,7 @@ type S3ApiServer struct { Router *S3ApiRouter app *fiber.App backend backend.Backend - CertStorage *utils.CertStorage + CertStorage *netutil.CertStorage quiet bool keepAlive bool health string @@ -237,7 +238,7 @@ func validateMiddlewareMount(mount middlewareMount) error { type Option func(*S3ApiServer) // WithTLS sets TLS Credentials -func WithTLS(cs *utils.CertStorage) Option { +func WithTLS(cs *netutil.CertStorage) Option { return func(s *S3ApiServer) { s.CertStorage = cs } } @@ -365,9 +366,9 @@ func (sa *S3ApiServer) ServeMultiPort(ports []string) error { var err error if sa.CertStorage != nil { - ln, err = utils.NewMultiAddrTLSListener(fiber.NetworkTCP, portSpec, sa.CertStorage.GetCertificate, utils.ListenerOptions{SocketPerm: sa.socketPerm}) + ln, err = netutil.NewMultiAddrTLSListener(fiber.NetworkTCP, portSpec, sa.CertStorage.GetCertificate, netutil.ListenerOptions{SocketPerm: sa.socketPerm}) } else { - ln, err = utils.NewMultiAddrListener(fiber.NetworkTCP, portSpec, utils.ListenerOptions{SocketPerm: sa.socketPerm}) + ln, err = netutil.NewMultiAddrListener(fiber.NetworkTCP, portSpec, netutil.ListenerOptions{SocketPerm: sa.socketPerm}) } if err != nil { return fmt.Errorf("failed to bind s3 listener %s: %w", portSpec, err) @@ -381,7 +382,7 @@ func (sa *S3ApiServer) ServeMultiPort(ports []string) error { } // Combine all listeners - finalListener := utils.NewMultiListener(listeners...) + finalListener := netutil.NewMultiListener(listeners...) if sa.onListen != nil { fn := sa.onListen diff --git a/s3api/server_test.go b/s3api/server_test.go index c76999f0..0a845bb6 100644 --- a/s3api/server_test.go +++ b/s3api/server_test.go @@ -25,8 +25,8 @@ import ( "github.com/gofiber/fiber/v3" "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/internal/netutil" "github.com/versity/versitygw/s3api/middlewares" - "github.com/versity/versitygw/s3api/utils" ) func newTestS3ApiServer(opts ...Option) (*S3ApiServer, error) { @@ -69,7 +69,7 @@ func TestS3ApiServer_Serve(t *testing.T) { app: fiber.New(), backend: backend.BackendUnsupported{}, Router: &S3ApiRouter{}, - CertStorage: &utils.CertStorage{}, + CertStorage: &netutil.CertStorage{}, }, port: "localhost:notaport", }, diff --git a/s3api/utils/auth-reader.go b/s3api/utils/auth-reader.go index 12884d2b..8161b5e0 100644 --- a/s3api/utils/auth-reader.go +++ b/s3api/utils/auth-reader.go @@ -39,11 +39,13 @@ const ( service = sigv4auth.ServiceS3 ) -// CheckValidSignature validates the ctx v4 auth signature -func CheckValidSignature(ctx fiber.Ctx, auth AuthData, secret, checksum string, tdate time.Time, contentLen int64) (string, error) { - result, err := sigv4auth.CheckSignature(ctx, auth, secret, checksum, tdate, contentLen, sigv4auth.CheckOptions{ - Service: service, - DisableURIPathEscaping: true, +// CheckValidSignature validates the ctx v4 auth signature against +// derivedKey — the request's kSigning value, either derived locally from a +// known secret or obtained from a standalone IAM service that never reveals +// the secret itself. +func CheckValidSignature(ctx fiber.Ctx, auth AuthData, derivedKey []byte, checksum string, tdate time.Time, contentLen int64) (string, error) { + result, err := sigv4auth.CheckSignature(ctx, auth, derivedKey, checksum, tdate, contentLen, sigv4auth.CheckOptions{ + Service: service, }) if err != nil { return "", mapSigV4Error(err) @@ -86,24 +88,11 @@ func ParseCredentials(input string, errHandler CredsError) (*CredentialsScope, e return creds, nil } -func SignPostPolicy(base64Policy, yyyymmdd, region, secretKey string) (string, error) { - signingKey := deriveSigningKey(secretKey, yyyymmdd, region) - sig := hmacSHA256(signingKey, []byte(base64Policy)) - return hex.EncodeToString(sig), nil -} - -func deriveSigningKey(secretKey, yyyymmdd, region string) []byte { - kDate := hmacSHA256([]byte("AWS4"+secretKey), []byte(yyyymmdd)) - kRegion := hmacSHA256(kDate, []byte(region)) - kService := hmacSHA256(kRegion, []byte(service)) - kSigning := hmacSHA256(kService, []byte("aws4_request")) - return kSigning -} - -func hmacSHA256(key, data []byte) []byte { - h := hmac.New(sha256.New, key) - h.Write(data) - return h.Sum(nil) +// SignPostPolicy signs a POST-policy document with derivedKey +func SignPostPolicy(base64Policy string, derivedKey []byte) (string, error) { + h := hmac.New(sha256.New, derivedKey) + h.Write([]byte(base64Policy)) + return hex.EncodeToString(h.Sum(nil)), nil } func mapSigV4Error(err error) error { diff --git a/s3api/utils/auth_test.go b/s3api/utils/auth_test.go index 7997b257..af1f97a2 100644 --- a/s3api/utils/auth_test.go +++ b/s3api/utils/auth_test.go @@ -16,14 +16,14 @@ package utils import ( "net" + "strings" "testing" "time" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/gofiber/fiber/v3" "github.com/valyala/fasthttp" "github.com/valyala/fasthttp/fasthttputil" - v4 "github.com/versity/versitygw/aws/signer/v4" + "github.com/versity/versitygw/internal/sigv4auth" ) func TestAuthParse(t *testing.T) { @@ -93,36 +93,19 @@ func Test_Client_UserAgent(t *testing.T) { } app.Get("/", func(c fiber.Ctx) error { - req, err := createHttpRequestFromCtx(c, signedHdrs, int64(c.Request().Header.ContentLength())) - if err != nil { - t.Fatal(err) + auth := sigv4auth.AuthData{ + Access: access, + Region: region, + Service: service, + SignedHeaders: strings.Join(signedHdrs, ";"), + Signature: expectedSig, } + derivedKey := sigv4auth.DeriveKey(secret, dateStr[:8], region, service) + opts := sigv4auth.CheckOptions{DisableURIPathEscaping: true} + contentLen := int64(c.Request().Header.ContentLength()) - req.Host = host - req.Header.Set("X-Amz-Content-Sha256", zeroLenSig) - - signer := v4.NewSigner() - - _, signErr := signer.SignHTTP(req.Context(), - aws.Credentials{ - AccessKeyID: access, - SecretAccessKey: secret, - }, - req, zeroLenSig, service, region, tdate, signedHdrs, - func(options *v4.SignerOptions) { - options.DisableURIPathEscaping = true - }) - if signErr != nil { - t.Fatalf("sign generated http request: %v", err) - } - - genAuth, err := ParseAuthorization(req.Header.Get("Authorization")) - if err != nil { - return err - } - - if genAuth.Signature != expectedSig { - t.Errorf("SIG: %v\nexpected: %v\n", genAuth.Signature, expectedSig) + if _, err := sigv4auth.CheckSignature(c, auth, derivedKey, zeroLenSig, tdate, contentLen, opts); err != nil { + t.Errorf("CheckSignature: %v", err) } return c.Send(c.Request().Header.UserAgent()) @@ -145,9 +128,19 @@ func Test_Client_UserAgent(t *testing.T) { defer fasthttp.ReleaseRequest(req) defer fasthttp.ReleaseResponse(resp) - req.SetRequestURI("http://example.com") + // Host/User-Agent/X-Amz-Content-Sha256/X-Amz-Date are sent as real + // headers, reproducing the captured request verbatim, so CheckSignature + // extracts them straight off the live fiber.Ctx like it does for any + // real request. + req.SetRequestURI("http://" + host + "/") req.Header.SetUserAgent(agent) + req.Header.Set("X-Amz-Content-Sha256", zeroLenSig) + req.Header.Set("X-Amz-Date", dateStr) if err := client.Do(req, resp); err != nil { t.Fatal(err) } + + if got := string(resp.Body()); got != agent { + t.Errorf("user-agent got %q, expected %q", got, agent) + } } diff --git a/s3api/utils/chunk-reader.go b/s3api/utils/chunk-reader.go index 62f5d6e6..f966202b 100644 --- a/s3api/utils/chunk-reader.go +++ b/s3api/utils/chunk-reader.go @@ -192,7 +192,7 @@ func ParseDecodedContentLength(ctx fiber.Ctx) (int64, error) { return decContLength, nil } -func NewChunkReader(ctx fiber.Ctx, r io.Reader, authdata AuthData, canonicalString, secret string, date time.Time) (io.Reader, error) { +func NewChunkReader(ctx fiber.Ctx, r io.Reader, authdata AuthData, canonicalString string, derivedKey []byte, date time.Time) (io.Reader, error) { cLength, err := ParseDecodedContentLength(ctx) if err != nil { return nil, err @@ -214,9 +214,9 @@ func NewChunkReader(ctx fiber.Ctx, r io.Reader, authdata AuthData, canonicalStri case payloadTypeStreamingUnsignedTrailer: return NewUnsignedChunkReader(r, checksumType, cLength) case payloadTypeStreamingSignedTrailer: - return NewSignedChunkReader(r, authdata, canonicalString, secret, date, checksumType, true, cLength) + return NewSignedChunkReader(r, authdata, canonicalString, derivedKey, date, checksumType, true, cLength) case payloadTypeStreamingSigned: - return NewSignedChunkReader(r, authdata, canonicalString, secret, date, "", false, cLength) + return NewSignedChunkReader(r, authdata, canonicalString, derivedKey, date, "", false, cLength) // return not supported for: // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER diff --git a/s3api/utils/multi_listener.go b/s3api/utils/multi_listener.go deleted file mode 100644 index d03f9402..00000000 --- a/s3api/utils/multi_listener.go +++ /dev/null @@ -1,399 +0,0 @@ -// Copyright 2026 Versity Software -// This file is licensed under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package utils - -import ( - "crypto/tls" - "errors" - "fmt" - "net" - "os" - "path/filepath" - "strings" - "sync" -) - -// MultiListener implements net.Listener and accepts connections from multiple -// underlying listeners. This is useful for listening on multiple IP addresses -// that a hostname resolves to (e.g., both IPv4 and IPv6 for "localhost"). -type MultiListener struct { - listeners []net.Listener - acceptCh chan acceptResult - closeCh chan struct{} - closeOnce sync.Once - wg sync.WaitGroup -} - -type acceptResult struct { - conn net.Conn - err error -} - -// NewMultiListener creates a new MultiListener that accepts connections from -// all provided listeners. -func NewMultiListener(listeners ...net.Listener) *MultiListener { - if len(listeners) == 0 { - return nil - } - - ml := &MultiListener{ - listeners: listeners, - acceptCh: make(chan acceptResult, 2*len(listeners)), - closeCh: make(chan struct{}), - } - - // Start accepting from each listener in its own goroutine - for _, ln := range listeners { - ml.wg.Add(1) - go ml.acceptLoop(ln) - } - - return ml -} - -// acceptLoop continuously accepts connections from a single listener -// and forwards them to the accept channel -func (ml *MultiListener) acceptLoop(ln net.Listener) { - defer ml.wg.Done() - - for { - conn, err := ln.Accept() - - select { - case <-ml.closeCh: - // MultiListener is closing - if conn != nil { - conn.Close() - } - return - case ml.acceptCh <- acceptResult{conn: conn, err: err}: - // Connection or error sent successfully - if err != nil { - return - } - } - } -} - -// Accept waits for and returns the next connection from any of the listeners -func (ml *MultiListener) Accept() (net.Conn, error) { - select { - case <-ml.closeCh: - return nil, errors.New("listener closed") - case result, ok := <-ml.acceptCh: - if !ok { - // Channel closed - return nil, errors.New("listener closed") - } - return result.conn, result.err - } -} - -// Close closes all underlying listeners -func (ml *MultiListener) Close() error { - var errs []error - - ml.closeOnce.Do(func() { - close(ml.closeCh) - - // Close all listeners - for _, ln := range ml.listeners { - if err := ln.Close(); err != nil { - errs = append(errs, err) - } - } - - // Wait for all accept loops to finish - ml.wg.Wait() - - // Drain any remaining accepts - close(ml.acceptCh) - for range ml.acceptCh { - } - }) - - if len(errs) > 0 { - return fmt.Errorf("errors closing listeners: %v", errs) - } - return nil -} - -// Addr returns the address of the first listener -func (ml *MultiListener) Addr() net.Addr { - if len(ml.listeners) > 0 { - return ml.listeners[0].Addr() - } - return nil -} - -// IsUnixSocketPath reports whether addr should be treated as a UNIX domain -// socket path rather than a TCP/IP address. It does so by attempting to parse -// addr as a host:port spec using net.SplitHostPort; anything that cannot be -// parsed that way (e.g. "/path/to/socket", "./rel/socket", "@abstract") is -// considered a socket path. -func IsUnixSocketPath(addr string) bool { - _, _, err := net.SplitHostPort(addr) - return err != nil -} - -// AbsSocketPaths converts any relative UNIX socket paths in addrs to absolute -// paths using the current working directory. Non-socket addresses (TCP/IP) and -// abstract sockets ("@name") are returned unchanged. This should be called -// early in program startup — before any backend that calls os.Chdir — so that -// relative paths are resolved against the shell's working directory. -func AbsSocketPaths(addrs []string) ([]string, error) { - result := make([]string, len(addrs)) - for i, addr := range addrs { - if strings.HasPrefix(addr, "./") { - abs, err := filepath.Abs(addr) - if err != nil { - return nil, fmt.Errorf("failed to resolve socket path %q: %w", addr, err) - } - result[i] = abs - } else { - result[i] = addr - } - } - return result, nil -} - -// isAbstractSocket reports whether addr is a Linux abstract namespace socket. -// Abstract sockets start with "@"; Go's net package maps this to a leading -// null byte (\0) in the sockaddr, so no socket file is created on disk. -func isAbstractSocket(addr string) bool { - return strings.HasPrefix(addr, "@") -} - -// removeStaleSocket removes a leftover UNIX socket file at path so the -// address can be reused. It returns an error if the path exists but is not -// a socket, protecting regular files and directories from accidental deletion. -func removeStaleSocket(path string) error { - fi, err := os.Stat(path) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return fmt.Errorf("failed to stat socket path %q: %w", path, err) - } - if fi.Mode()&os.ModeSocket == 0 { - return fmt.Errorf("path %q already exists and is not a socket (mode %s)", path, fi.Mode()) - } - return os.Remove(path) -} - -// ResolveHostnameIPs resolves a hostname to all its IP addresses (IPv4 and IPv6). -// If the input is already an IP address or empty, it returns it as-is. -// This is useful for determining all addresses a server will listen on. -func ResolveHostnameIPs(address string) ([]string, error) { - if IsUnixSocketPath(address) { - return []string{address}, nil - } - - host, _, err := net.SplitHostPort(address) - if err != nil { - return nil, fmt.Errorf("invalid address %q: %w", address, err) - } - - // Handle empty host (e.g., ":8080" means all interfaces) - if host == "" { - return []string{""}, nil - } - - // If already an IP address, return as is - if net.ParseIP(host) != nil { - return []string{host}, nil - } - - // Resolve hostname to all IP addresses - ips, err := net.LookupIP(host) - if err != nil { - return nil, fmt.Errorf("failed to resolve hostname %q: %w", host, err) - } - - if len(ips) == 0 { - return nil, fmt.Errorf("no addresses found for hostname %q", host) - } - - // Convert IPs to strings - result := make([]string, 0, len(ips)) - for _, ip := range ips { - result = append(result, ip.String()) - } - - return result, nil -} - -// resolveHostnameAddrs resolves a hostname to all its IP addresses (IPv4 and IPv6) -// and returns them as a list of addresses with the port attached. -func resolveHostnameAddrs(address string) ([]string, error) { - if IsUnixSocketPath(address) { - return []string{address}, nil - } - - host, port, err := net.SplitHostPort(address) - if err != nil { - return nil, fmt.Errorf("invalid address %q: %w", address, err) - } - - // If host is empty or already an IP address, return as is - if host == "" || net.ParseIP(host) != nil { - return []string{address}, nil - } - - // Resolve hostname to all IP addresses - ips, err := net.LookupIP(host) - if err != nil { - return nil, fmt.Errorf("failed to resolve hostname %q: %w", host, err) - } - - if len(ips) == 0 { - return nil, fmt.Errorf("no addresses found for hostname %q", host) - } - - // Build list of addresses with port - addrs := make([]string, 0, len(ips)) - for _, ip := range ips { - addr := net.JoinHostPort(ip.String(), port) - addrs = append(addrs, addr) - } - - return addrs, nil -} - -// ListenerOptions configures optional behaviour for NewMultiAddrListener and -// NewMultiAddrTLSListener. -type ListenerOptions struct { - // SocketPerm, when non-zero, sets the file-mode permissions on file-backed - // UNIX sockets after binding. It is ignored for TCP/IP addresses and - // abstract namespace sockets. - SocketPerm os.FileMode -} - -// NewMultiAddrListener creates listeners for all IP addresses that the hostname -// in the address resolves to. If the address is already an IP, it creates a -// single listener. Returns a MultiListener if multiple addresses are resolved, -// or a single listener if only one address is found. -// -// UNIX domain socket forms are also supported: -// - "/path/to/socket" or "./rel/socket" — file-backed socket; any stale -// socket file is removed before binding. -// - "@name" — Linux abstract namespace socket; no file is created or removed. -// -// opts.SocketPerm, when non-zero, sets the file-mode permissions on file-backed -// sockets after binding. It is ignored for TCP/IP addresses and abstract sockets. -func NewMultiAddrListener(network, address string, opts ListenerOptions) (net.Listener, error) { - if IsUnixSocketPath(address) { - // For file-backed sockets, remove any stale socket file so re-binding works cleanly. - // Abstract sockets (@name) have no filesystem entry; skip removal for them. - if !isAbstractSocket(address) { - if err := removeStaleSocket(address); err != nil { - return nil, err - } - } - ln, err := net.Listen("unix", address) - if err != nil { - return nil, fmt.Errorf("failed to bind unix socket listener %s: %w", address, err) - } - if opts.SocketPerm != 0 && !isAbstractSocket(address) { - if err := os.Chmod(address, opts.SocketPerm); err != nil { - ln.Close() - return nil, fmt.Errorf("failed to set permissions on socket %s: %w", address, err) - } - } - return NewMultiListener(ln), nil - } - - addrs, err := resolveHostnameAddrs(address) - if err != nil { - return nil, err - } - - // Create listeners for all resolved addresses - listeners := make([]net.Listener, 0, len(addrs)) - - for _, addr := range addrs { - ln, err := net.Listen(network, addr) - if err != nil { - // Close any listeners we've already created - for _, l := range listeners { - l.Close() - } - return nil, fmt.Errorf("failed to bind listener %s: %w", addr, err) - } - listeners = append(listeners, ln) - } - - // Return MultiListener for multiple addresses - return NewMultiListener(listeners...), nil -} - -// NewMultiAddrTLSListener creates TLS listeners for all IP addresses that the -// hostname in the address resolves to. Similar to NewMultiAddrListener but with TLS. -// -// UNIX domain socket forms are also supported: -// - "/path/to/socket" or "./rel/socket" — file-backed socket; any stale -// socket file is removed before binding. -// - "@name" — Linux abstract namespace socket; no file is created or removed. -// -// opts.SocketPerm, when non-zero, sets the file-mode permissions on file-backed -// sockets after binding. It is ignored for TCP/IP addresses and abstract sockets. -func NewMultiAddrTLSListener(network, address string, getCertificateFunc func(*tls.ClientHelloInfo) (*tls.Certificate, error), opts ListenerOptions) (net.Listener, error) { - config := &tls.Config{ - MinVersion: tls.VersionTLS12, - GetCertificate: getCertificateFunc, - } - - if IsUnixSocketPath(address) { - if !isAbstractSocket(address) { - if err := removeStaleSocket(address); err != nil { - return nil, err - } - } - ln, err := net.Listen("unix", address) - if err != nil { - return nil, fmt.Errorf("failed to bind unix TLS socket listener %s: %w", address, err) - } - if opts.SocketPerm != 0 && !isAbstractSocket(address) { - if err := os.Chmod(address, opts.SocketPerm); err != nil { - ln.Close() - return nil, fmt.Errorf("failed to set permissions on socket %s: %w", address, err) - } - } - return NewMultiListener(tls.NewListener(ln, config)), nil - } - - addrs, err := resolveHostnameAddrs(address) - if err != nil { - return nil, err - } - - // Create TLS listeners for all resolved addresses - listeners := make([]net.Listener, 0, len(addrs)) - - for _, addr := range addrs { - ln, err := net.Listen(network, addr) - if err != nil { - // Close any listeners we've already created - for _, l := range listeners { - l.Close() - } - return nil, fmt.Errorf("failed to bind TLS listener %s: %w", addr, err) - } - listeners = append(listeners, tls.NewListener(ln, config)) - } - - // Return MultiListener for multiple addresses - return NewMultiListener(listeners...), nil -} diff --git a/s3api/utils/presign-auth-reader.go b/s3api/utils/presign-auth-reader.go index c22463f3..d376524b 100644 --- a/s3api/utils/presign-auth-reader.go +++ b/s3api/utils/presign-auth-reader.go @@ -28,8 +28,9 @@ const ( unsignedPayload string = "UNSIGNED-PAYLOAD" ) -// CheckPresignedSignature validates presigned request signature -func CheckPresignedSignature(ctx fiber.Ctx, auth AuthData, secret string) error { +// CheckPresignedSignature validates a presigned request's signature against +// derivedKey +func CheckPresignedSignature(ctx fiber.Ctx, auth AuthData, derivedKey []byte) error { var contentLength int64 var err error contentLengthStr := ctx.Get("Content-Length") @@ -42,9 +43,8 @@ func CheckPresignedSignature(ctx fiber.Ctx, auth AuthData, secret string) error date, _ := time.Parse(iso8601Format, auth.Date) - _, err = sigv4auth.CheckQuerySignature(ctx, auth, secret, unsignedPayload, date, contentLength, sigv4auth.CheckOptions{ - Service: service, - DisableURIPathEscaping: true, + _, err = sigv4auth.CheckQuerySignature(ctx, auth, derivedKey, unsignedPayload, date, contentLength, sigv4auth.CheckOptions{ + Service: service, }) if err != nil { return mapSigV4Error(err) @@ -133,7 +133,7 @@ func mapQueryAuthError(err error) error { queryErr.ServerTime.Format(time.RFC3339), ) case sigv4auth.ErrQuerySecurityToken: - return s3err.QueryAuthErrors.SecurityTokenNotSupported() + return s3err.GetAPIError(s3err.ErrInvalidToken) } } diff --git a/s3api/utils/signed-chunk-reader.go b/s3api/utils/signed-chunk-reader.go index d9e591f0..26e63097 100644 --- a/s3api/utils/signed-chunk-reader.go +++ b/s3api/utils/signed-chunk-reader.go @@ -40,7 +40,6 @@ import ( const ( zeroLenSig = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - awsV4 = "AWS4" awsS3Service = "s3" awsV4Request = "aws4_request" trailerSignatureHeader = "x-amz-trailer-signature:" @@ -84,11 +83,13 @@ type ChunkReader struct { // NewChunkReader reads from request body io.Reader and parses out the // chunk metadata in stream. The headers are validated for proper signatures. // Reading from the chunk reader will read only the object data stream -// without the chunk headers/trailers. -func NewSignedChunkReader(r io.Reader, authdata AuthData, canonicalString, secret string, date time.Time, chType checksumType, requireTrailer bool, cLength int64) (io.Reader, error) { +// without the chunk headers/trailers. derivedKey is the same SigV4 kSigning +// value the seed request's Authorization header was already checked +// against, reused here rather than re-derived or re-fetched. +func NewSignedChunkReader(r io.Reader, authdata AuthData, canonicalString string, derivedKey []byte, date time.Time, chType checksumType, requireTrailer bool, cLength int64) (io.Reader, error) { chRdr := &ChunkReader{ r: r, - signingKey: getSigningKey(secret, authdata.Region, date), + signingKey: derivedKey, // the authdata.Signature is validated in the auth-reader, // so we can use that here without any other checks prevSig: authdata.Signature, @@ -353,18 +354,6 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) { return n, nil } -// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html -// Task 3: Calculate Signature -// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro -func getSigningKey(secret, region string, date time.Time) []byte { - dateKey := hmac256([]byte(awsV4+secret), []byte(date.Format(yyyymmdd))) - dateRegionKey := hmac256(dateKey, []byte(region)) - dateRegionServiceKey := hmac256(dateRegionKey, []byte(awsS3Service)) - signingKey := hmac256(dateRegionServiceKey, []byte(awsV4Request)) - debuglogger.Infof("signing key: %s", hex.EncodeToString(signingKey)) - return signingKey -} - func hmac256(key []byte, data []byte) []byte { hash := hmac.New(sha256.New, key) hash.Write(data) diff --git a/s3api/utils/signed_headers_test.go b/s3api/utils/signed_headers_test.go index 12a7db98..01938d5f 100644 --- a/s3api/utils/signed_headers_test.go +++ b/s3api/utils/signed_headers_test.go @@ -14,23 +14,24 @@ package utils import ( - "context" "net/http" "net/url" "testing" "time" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/gofiber/fiber/v3" "github.com/stretchr/testify/require" "github.com/valyala/fasthttp" - v4 "github.com/versity/versitygw/aws/signer/v4" + "github.com/versity/versitygw/internal/sigv4auth" "github.com/versity/versitygw/s3err" ) const signedHeadersTestRegion = "us-east-1" -var signedHeadersTestCreds = aws.Credentials{ +var signedHeadersTestCreds = struct { + AccessKeyID string + SecretAccessKey string +}{ AccessKeyID: "AKID", SecretAccessKey: "SECRET", } @@ -43,7 +44,7 @@ func TestCheckPresignedSignatureRejectsUnsignedAmzHeader(t *testing.T) { authData, err := ParsePresignedURIParts(ctx, signedHeadersTestRegion) require.NoError(t, err) - err = CheckPresignedSignature(ctx, authData, signedHeadersTestCreds.SecretAccessKey) + err = CheckPresignedSignature(ctx, authData, derivedKeyFor(authData)) requireHeadersNotSigned(t, err, "x-amz-copy-source") } @@ -56,7 +57,7 @@ func TestCheckPresignedSignatureAllowsSignedAmzHeader(t *testing.T) { authData, err := ParsePresignedURIParts(ctx, signedHeadersTestRegion) require.NoError(t, err) - err = CheckPresignedSignature(ctx, authData, signedHeadersTestCreds.SecretAccessKey) + err = CheckPresignedSignature(ctx, authData, derivedKeyFor(authData)) require.NoError(t, err) } @@ -69,7 +70,7 @@ func TestCheckPresignedSignatureAllowsUnsignedNonAmzHeader(t *testing.T) { authData, err := ParsePresignedURIParts(ctx, signedHeadersTestRegion) require.NoError(t, err) - err = CheckPresignedSignature(ctx, authData, signedHeadersTestCreds.SecretAccessKey) + err = CheckPresignedSignature(ctx, authData, derivedKeyFor(authData)) require.NoError(t, err) } @@ -78,7 +79,7 @@ func TestCheckValidSignatureRejectsUnsignedAmzHeader(t *testing.T) { "X-Amz-Tagging": []string{"a=b"}, }) - _, err := CheckValidSignature(ctx, authData, signedHeadersTestCreds.SecretAccessKey, unsignedPayload, signingTime, 0) + _, err := CheckValidSignature(ctx, authData, derivedKeyFor(authData), unsignedPayload, signingTime, 0) requireHeadersNotSigned(t, err, "x-amz-tagging") } @@ -87,7 +88,7 @@ func TestCheckValidSignatureAllowsSignedAmzHeader(t *testing.T) { "X-Amz-Tagging": []string{"a=b"}, }, nil) - _, err := CheckValidSignature(ctx, authData, signedHeadersTestCreds.SecretAccessKey, unsignedPayload, signingTime, 0) + _, err := CheckValidSignature(ctx, authData, derivedKeyFor(authData), unsignedPayload, signingTime, 0) require.NoError(t, err) } @@ -97,7 +98,7 @@ func TestCheckValidSignatureAllowsUnsignedNonAmzHeader(t *testing.T) { "X-Custom-Header": []string{"value"}, }) - _, err := CheckValidSignature(ctx, authData, signedHeadersTestCreds.SecretAccessKey, unsignedPayload, signingTime, 0) + _, err := CheckValidSignature(ctx, authData, derivedKeyFor(authData), unsignedPayload, signingTime, 0) require.NoError(t, err) } @@ -109,7 +110,7 @@ func TestCheckPresignedSignatureRejectsUnsignedAmzHeaderPattern(t *testing.T) { authData, err := ParsePresignedURIParts(ctx, signedHeadersTestRegion) require.NoError(t, err) - err = CheckPresignedSignature(ctx, authData, signedHeadersTestCreds.SecretAccessKey) + err = CheckPresignedSignature(ctx, authData, derivedKeyFor(authData)) requireHeadersNotSigned(t, err, "x-amz-some-other-header") } @@ -118,10 +119,14 @@ func TestCheckValidSignatureRejectsUnsignedAmzHeaderPattern(t *testing.T) { "X-Amz-Some-Other-Header": []string{"value"}, }) - _, err := CheckValidSignature(ctx, authData, signedHeadersTestCreds.SecretAccessKey, unsignedPayload, signingTime, 0) + _, err := CheckValidSignature(ctx, authData, derivedKeyFor(authData), unsignedPayload, signingTime, 0) requireHeadersNotSigned(t, err, "x-amz-some-other-header") } +func derivedKeyFor(authData AuthData) []byte { + return sigv4auth.DeriveKey(signedHeadersTestCreds.SecretAccessKey, authData.Date[:8], signedHeadersTestRegion, service) +} + func buildPresignedURL(t *testing.T, headers http.Header) string { t.Helper() @@ -132,23 +137,22 @@ func buildPresignedURL(t *testing.T, headers http.Header) string { req.Header = make(http.Header) } - signer := v4.NewSigner() - signedURL, _, _, err := signer.PresignHTTP( - context.Background(), - signedHeadersTestCreds, - req, - unsignedPayload, - service, - signedHeadersTestRegion, - time.Now().UTC(), - nil, - func(options *v4.SignerOptions) { - options.DisableURIPathEscaping = true - }, - ) - require.NoError(t, err) + signingTime := time.Now().UTC() + yyyymmdd := signingTime.Format(sigv4auth.YYYYMMDD) + derivedKey := sigv4auth.DeriveKey(signedHeadersTestCreds.SecretAccessKey, yyyymmdd, signedHeadersTestRegion, service) - return signedURL + in := sigv4auth.SigningInputFromRequest(req) + in.AccessKeyID = signedHeadersTestCreds.AccessKeyID + in.CredentialScope = sigv4auth.BuildCredentialScope(yyyymmdd, signedHeadersTestRegion, service) + in.PayloadHash = unsignedPayload + in.SigningTime = signingTime + in.DisableURIPathEscaping = true + in.IsPreSign = true + result := sigv4auth.BuildAndSign(derivedKey, in) + + signedURL := *req.URL + signedURL.RawQuery = result.RawQuery + return signedURL.String() } func signedHeaderAuthCtx(t *testing.T, signedHeaders, extraHeaders http.Header) (fiber.Ctx, AuthData, time.Time) { @@ -162,21 +166,18 @@ func signedHeaderAuthCtx(t *testing.T, signedHeaders, extraHeaders http.Header) req.Header = make(http.Header) } - signer := v4.NewSigner() - _, err = signer.SignHTTP( - context.Background(), - signedHeadersTestCreds, - req, - unsignedPayload, - service, - signedHeadersTestRegion, - signingTime, - nil, - func(options *v4.SignerOptions) { - options.DisableURIPathEscaping = true - }, - ) - require.NoError(t, err) + yyyymmdd := signingTime.Format(sigv4auth.YYYYMMDD) + derivedKey := sigv4auth.DeriveKey(signedHeadersTestCreds.SecretAccessKey, yyyymmdd, signedHeadersTestRegion, service) + + in := sigv4auth.SigningInputFromRequest(req) + in.AccessKeyID = signedHeadersTestCreds.AccessKeyID + in.CredentialScope = sigv4auth.BuildCredentialScope(yyyymmdd, signedHeadersTestRegion, service) + in.PayloadHash = unsignedPayload + in.SigningTime = signingTime + in.DisableURIPathEscaping = true + result := sigv4auth.BuildAndSign(derivedKey, in) + req.Header.Set("X-Amz-Date", result.AmzDate) + req.Header.Set("Authorization", result.AuthorizationHeader) headers := req.Header.Clone() for key, values := range extraHeaders { @@ -225,3 +226,46 @@ func requireHeadersNotSigned(t *testing.T, err error, expected string) { require.Equal(t, "AccessDenied", serr.Code) require.Equal(t, expected, serr.HeadersNotSigned) } + +// TestCheckValidSignatureRejectsUnsignedSecurityToken pins the entire +// binding argument for a session credential presented via header auth. +// +// The gateway adds no RequiredSignedHeaders entry for +// X-Amz-Security-Token, and deliberately so: passing a non-nil list +// *replaces* sigv4auth's default rule (host plus every X-Amz-* header) +// rather than adding to it, which would weaken the binding for every other +// X-Amz-* header. What keeps the token bound to the signature is that +// default rule alone — so if anyone ever hands CheckValidSignature an +// explicit list, this test is what catches it. +func TestCheckValidSignatureRejectsUnsignedSecurityToken(t *testing.T) { + ctx, authData, signingTime := signedHeaderAuthCtx(t, nil, http.Header{ + "X-Amz-Security-Token": []string{"a-session-token"}, + }) + + _, err := CheckValidSignature(ctx, authData, derivedKeyFor(authData), unsignedPayload, signingTime, 0) + requireHeadersNotSigned(t, err, "x-amz-security-token") +} + +// TestCheckValidSignatureAllowsSignedSecurityToken is the positive half: +// a token that *was* part of the signed request passes, so a legitimate +// session credential is not rejected by the rule above. +func TestCheckValidSignatureAllowsSignedSecurityToken(t *testing.T) { + ctx, authData, signingTime := signedHeaderAuthCtx(t, http.Header{ + "X-Amz-Security-Token": []string{"a-session-token"}, + }, nil) + + _, err := CheckValidSignature(ctx, authData, derivedKeyFor(authData), unsignedPayload, signingTime, 0) + require.NoError(t, err) +} + +// TestCheckValidSignatureRejectsSwappedSecurityToken confirms the token +// cannot be swapped for another session's after signing: it is part of the +// canonical request, so altering it invalidates the signature. +func TestCheckValidSignatureRejectsSwappedSecurityToken(t *testing.T) { + signed := http.Header{"X-Amz-Security-Token": []string{"the-real-session-token"}} + ctx, authData, signingTime := signedHeaderAuthCtx(t, signed, nil) + ctx.Request().Header.Set("X-Amz-Security-Token", "somebody-elses-session-token") + + _, err := CheckValidSignature(ctx, authData, derivedKeyFor(authData), unsignedPayload, signingTime, 0) + require.Error(t, err, "swapping the security token after signing must invalidate the signature") +} diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index d1271954..f34edcf4 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -18,14 +18,11 @@ import ( "crypto/tls" "encoding/base64" "encoding/xml" - "errors" "fmt" "io" "net" - "net/http" "net/url" "regexp" - "slices" "strconv" "strings" "sync/atomic" @@ -34,7 +31,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gofiber/fiber/v3" "github.com/valyala/fasthttp" - signerV4 "github.com/versity/versitygw/aws/signer/v4" + "github.com/versity/versitygw/backend" "github.com/versity/versitygw/debuglogger" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3response" @@ -135,41 +132,6 @@ func ExtractMetadataFromFields(fields map[string]string) (map[string]string, err return metadata, nil } -func createHttpRequestFromCtx(ctx fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) { - req := ctx.Request() - - uri := ctx.OriginalURL() - - httpReq, err := http.NewRequest(string(req.Header.Method()), uri, nil) - if err != nil { - return nil, errors.New("error in creating an http request") - } - - if err := addRequestHeadersFromCtx(ctx, httpReq, signedHdrs); err != nil { - return nil, err - } - - // make sure all headers in the signed headers are present - for _, header := range signedHdrs { - if httpReq.Header.Get(header) == "" { - httpReq.Header.Set(header, "") - } - } - - // Check if Content-Length in signed headers - // If content length is non 0, then the header will be included - if !includeHeader("Content-Length", signedHdrs) { - httpReq.ContentLength = 0 - } else { - httpReq.ContentLength = contentLength - } - - // Set the Host header - httpReq.Host = string(req.Header.Host()) - - return httpReq, nil -} - func SetMetaHeaders(ctx fiber.Ctx, meta map[string]string) { ctx.Response().Header.DisableNormalizing() for key, val := range meta { @@ -296,34 +258,6 @@ func IsValidBucketName(bucket string) bool { return true } -func includeHeader(hdr string, signedHdrs []string) bool { - return slices.ContainsFunc(signedHdrs, func(shdr string) bool { - return strings.EqualFold(hdr, shdr) - }) -} - -func addRequestHeadersFromCtx(ctx fiber.Ctx, httpReq *http.Request, signedHdrs []string) error { - headersNotSigned := []string{} - for key, value := range ctx.Request().Header.All() { - keyStr := string(key) - if includeHeader(keyStr, signedHdrs) || signerV4.IsIgnoredHeader(keyStr) { - httpReq.Header.Add(keyStr, string(value)) - continue - } - if signerV4.IsRequiredSignedHeader(keyStr) { - lowerKey := strings.ToLower(keyStr) - headersNotSigned = append(headersNotSigned, lowerKey) - } - } - - if len(headersNotSigned) != 0 { - debuglogger.Logf("headers present in request but not included in SignedHeaders: %q", strings.Join(headersNotSigned, ", ")) - return s3err.GetHeadersNotSignedErr(headersNotSigned) - } - - return nil -} - // expiration time window // https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html#RESTAuthenticationTimeStamp const timeExpirationSec = 15 * 60 // seconds @@ -1059,29 +993,6 @@ func GenerateObjectLocation(ctx fiber.Ctx, virtualDomain, bucket, object string) ) } -type CertStorage struct { - cert atomic.Pointer[tls.Certificate] -} - -func NewCertStorage() *CertStorage { - return &CertStorage{} -} - -func (cs *CertStorage) GetCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate, error) { - return cs.cert.Load(), nil -} - -func (cs *CertStorage) SetCertificate(certFile string, keyFile string) error { - cert, err := tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - return fmt.Errorf("unable to set certificate: %w", err) - } - - cs.cert.Store(&cert) - - return nil -} - func NewTLSListener(network string, address string, getCertificateFunc func(*tls.ClientHelloInfo) (*tls.Certificate, error)) (net.Listener, error) { config := &tls.Config{ MinVersion: tls.VersionTLS12, @@ -1095,6 +1006,55 @@ func NewTLSListener(network string, address string, getCertificateFunc func(*tls return tls.NewListener(ln, config), nil } +// MergeDeleteObjectsResult builds the final DeleteObjects response, +// preserving the order objects were requested in across both the Deleted +// and Error lists. objects is the full request; checkErrs is +// VerifyObjectsAccess's per-object result for it (nil entries were sent to +// the backend); backendResult is the backend's response for just those. +// +// The backend's own Deleted/Error order is not assumed to match the order +// its objects were sent in, so objects are matched back to their backend +// result by identity (key + version) rather than by position, with a FIFO +// queue per identity to keep duplicate keys in the same request each paired +// with their own result. versionID is "" for a keyed (unversioned) delete, +// matching how both the request and every backend's response represent "no +// version specified" — as a nil pointer. +func MergeDeleteObjectsResult(objects []types.ObjectIdentifier, checkErrs []error, backendResult s3response.DeleteResult) s3response.DeleteResult { + type objectKey struct{ key, versionID string } + + deletedByKey := make(map[objectKey][]types.DeletedObject, len(backendResult.Deleted)) + for _, d := range backendResult.Deleted { + k := objectKey{backend.GetStringFromPtr(d.Key), backend.GetStringFromPtr(d.VersionId)} + deletedByKey[k] = append(deletedByKey[k], d) + } + errorByKey := make(map[objectKey][]types.Error, len(backendResult.Error)) + for _, e := range backendResult.Error { + k := objectKey{backend.GetStringFromPtr(e.Key), backend.GetStringFromPtr(e.VersionId)} + errorByKey[k] = append(errorByKey[k], e) + } + + var result s3response.DeleteResult + for i, obj := range objects { + if checkErrs[i] != nil { + result.Error = append(result.Error, s3err.ObjectDeleteError(obj.Key, obj.VersionId, checkErrs[i])) + continue + } + + k := objectKey{backend.GetStringFromPtr(obj.Key), backend.GetStringFromPtr(obj.VersionId)} + if queue := deletedByKey[k]; len(queue) > 0 { + result.Deleted = append(result.Deleted, queue[0]) + deletedByKey[k] = queue[1:] + continue + } + if queue := errorByKey[k]; len(queue) > 0 { + result.Error = append(result.Error, queue[0]) + errorByKey[k] = queue[1:] + } + } + + return result +} + func DetectResourceType(ctx fiber.Ctx) s3err.ResourceType { path := ctx.Path() if path == "" || path == "/" { diff --git a/s3api/utils/utils_test.go b/s3api/utils/utils_test.go index d496997a..10390de7 100644 --- a/s3api/utils/utils_test.go +++ b/s3api/utils/utils_test.go @@ -20,7 +20,6 @@ import ( "encoding/xml" "errors" "math/rand" - "net/http" "net/url" "reflect" "strings" @@ -28,7 +27,6 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/gofiber/fiber/v3" "github.com/stretchr/testify/assert" "github.com/valyala/fasthttp" "github.com/versity/versitygw/backend" @@ -36,67 +34,6 @@ import ( "github.com/versity/versitygw/s3response" ) -func TestCreateHttpRequestFromCtx(t *testing.T) { - type args struct { - ctx fiber.Ctx - } - - app := fiber.New() - - // Expected output, Case 1 - ctx := app.AcquireCtx(&fasthttp.RequestCtx{}) - req := ctx.Request() - request, _ := http.NewRequest(string(req.Header.Method()), req.URI().String(), bytes.NewReader(req.Body())) - - // Case 2 - ctx2 := app.AcquireCtx(&fasthttp.RequestCtx{}) - req2 := ctx2.Request() - req2.Header.Add("X-Amz-Mfa", "Some valid Mfa") - - request2, _ := http.NewRequest(string(req2.Header.Method()), req2.URI().String(), bytes.NewReader(req2.Body())) - request2.Header.Add("X-Amz-Mfa", "Some valid Mfa") - - tests := []struct { - name string - args args - want *http.Request - wantErr bool - hdrs []string - }{ - { - name: "Success-response", - args: args{ - ctx: ctx, - }, - want: request, - wantErr: false, - hdrs: []string{}, - }, - { - name: "Success-response-With-Headers", - args: args{ - ctx: ctx2, - }, - want: request2, - wantErr: false, - hdrs: []string{"X-Amz-Mfa"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := createHttpRequestFromCtx(tt.args.ctx, tt.hdrs, 0) - if (err != nil) != tt.wantErr { - t.Errorf("CreateHttpRequestFromCtx() error = %v, wantErr %v", err, tt.wantErr) - return - } - - if !reflect.DeepEqual(got.Header, tt.want.Header) { - t.Errorf("CreateHttpRequestFromCtx() got = %v, want %v", got, tt.want) - } - }) - } -} - // a helper method to construct a raw http request with the given http request headers // to further parse with fasthttp.Request.Read and return fasthttp.RequestHeader func createHeadersFromRawRequest(t *testing.T, hdrs [][2]string) *fasthttp.RequestHeader { @@ -229,42 +166,6 @@ func TestGetUserMetaData(t *testing.T) { } } -func Test_includeHeader(t *testing.T) { - type args struct { - hdr string - signedHdrs []string - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "include-header-falsy-case", - args: args{ - hdr: "Content-Type", - signedHdrs: []string{"X-Amz-Acl", "Content-Encoding"}, - }, - want: false, - }, - { - name: "include-header-falsy-case", - args: args{ - hdr: "Content-Type", - signedHdrs: []string{"X-Amz-Acl", "Content-Type"}, - }, - want: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := includeHeader(tt.args.hdr, tt.args.signedHdrs); got != tt.want { - t.Errorf("includeHeader() = %v, want %v", got, tt.want) - } - }) - } -} - func TestIsValidBucketName(t *testing.T) { type args struct { bucket string diff --git a/s3err/presigned-urls.go b/s3err/presigned-urls.go index 20bcfa3d..2204e7a6 100644 --- a/s3err/presigned-urls.go +++ b/s3err/presigned-urls.go @@ -83,8 +83,4 @@ func (queryAuthErrors) OnlyHMACSupported() S3Error { return authQueryParamError("X-Amz-Algorithm only supports \"AWS4-HMAC-SHA256\"") } -func (queryAuthErrors) SecurityTokenNotSupported() S3Error { - return authQueryParamError("Authorization with X-Amz-Security-Token is not supported") -} - var QueryAuthErrors queryAuthErrors diff --git a/s3err/s3err.go b/s3err/s3err.go index 1b1348ff..5fe14131 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -126,6 +126,7 @@ const ( ErrMissingContentLength ErrContentLengthMismatch ErrInvalidAccessKeyID + ErrInvalidToken ErrRequestNotReadyYet ErrMissingDateHeader ErrGetUploadsWithKey @@ -397,6 +398,11 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "The AWS Access Key Id you provided does not exist in our records.", HTTPStatusCode: http.StatusForbidden, }, + ErrInvalidToken: { + Code: "InvalidToken", + Description: "The provided token is malformed or otherwise invalid.", + HTTPStatusCode: http.StatusBadRequest, + }, ErrRequestNotReadyYet: { Code: "AccessDenied", Description: "Request is not valid yet.", @@ -1002,6 +1008,28 @@ func GetWebsiteRoutingRulesLimitedErr(rules int) APIError { } } +func GetExplicitDenyAccessErr(principal, action, resourceArn, source string) APIError { + return APIError{ + Code: "AccessDenied", + Description: fmt.Sprintf( + "User: %s is not authorized to perform: %s on resource: %q with an explicit deny in %s", + principal, action, resourceArn, source, + ), + HTTPStatusCode: http.StatusForbidden, + } +} + +func GetImplicitDenyAccessErr(principal, action, resourceArn string) APIError { + return APIError{ + Code: "AccessDenied", + Description: fmt.Sprintf( + "User: %s is not authorized to perform: %s on resource: %q because no identity-based policy allows the %s action", + principal, action, resourceArn, action, + ), + HTTPStatusCode: http.StatusForbidden, + } +} + type ResourceType string const ( @@ -1011,3 +1039,23 @@ const ( ResourceTypeBucketPolicy ResourceType = "BUCKETPOLICY" ResourceTypeUpload ResourceType = "UPLOAD" ) + +func ObjectDeleteError(key, versionId *string, err error) types.Error { + if serr, ok := err.(S3Error); ok { + base := serr.BaseError() + return types.Error{ + Key: key, + VersionId: versionId, + Code: &base.Code, + Message: &base.Description, + } + } + message := err.Error() + code := "InternalError" + return types.Error{ + Key: key, + VersionId: versionId, + Code: &code, + Message: &message, + } +} diff --git a/tests/integration/Access_Control.go b/tests/integration/Access_Control.go index ad7d44df..ec4474a3 100644 --- a/tests/integration/Access_Control.go +++ b/tests/integration/Access_Control.go @@ -17,8 +17,11 @@ package integration import ( "bytes" "context" + "encoding/base64" + "encoding/json" "fmt" "io" + "strings" "time" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -287,7 +290,9 @@ func AccessControl_multi_statement_policy(s *S3Conf) error { Bucket: &bucket, }) cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + if err := checkApiErr(err, s3err.GetExplicitDenyAccessErr( + testuser.access, "s3:DeleteBucket", fmt.Sprintf("arn:aws:s3:::%s", bucket), "a resource-based policy", + )); err != nil { return err } @@ -1110,3 +1115,565 @@ func AccessControl_CopyObject_with_retention_policy(s *S3Conf) error { return cleanupLockedObjects(s3client, bucket, []objToDelete{{key: dstObj}}) }, withLock()) } + +// AccessControl_bucket_policy_condition_ip_allow covers a bucket-policy +// Allow statement scoped by an IpAddress Condition matching the caller's +// real source IP: 0.0.0.0/0 matches any IPv4 address, so this exercises +// the Condition machinery without depending on the test runner's actual +// address. +func AccessControl_bucket_policy_condition_ip_allow(s *S3Conf) error { + testName := "AccessControl_bucket_policy_condition_ip_allow" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser := getUser("user") + if err := createUsers(s, []user{testuser}); err != nil { + return err + } + + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: "Allow", + Principal: testuser.access, + Action: "s3:PutObject", + Resource: fmt.Sprintf("arn:aws:s3:::%s/*", bucket), + Condition: json.RawMessage(`{"IpAddress":{"aws:SourceIp":"0.0.0.0/0"}}`), + }); err != nil { + return err + } + + userClient := s.getUserClient(testuser) + _, err := putObjects(userClient, []string{"my-obj"}, bucket) + return err + }) +} + +// AccessControl_bucket_policy_condition_ip_deny_no_match covers the same +// shape as AccessControl_bucket_policy_condition_ip_allow with a CIDR +// (TEST-NET-3, RFC 5737) that can never match a real caller, so the Allow +// statement never applies and the request falls through to an implicit +// deny. +func AccessControl_bucket_policy_condition_ip_deny_no_match(s *S3Conf) error { + testName := "AccessControl_bucket_policy_condition_ip_deny_no_match" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser := getUser("user") + if err := createUsers(s, []user{testuser}); err != nil { + return err + } + + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: "Allow", + Principal: testuser.access, + Action: "s3:PutObject", + Resource: fmt.Sprintf("arn:aws:s3:::%s/*", bucket), + Condition: json.RawMessage(`{"IpAddress":{"aws:SourceIp":"203.0.113.0/24"}}`), + }); err != nil { + return err + } + + userClient := s.getUserClient(testuser) + _, err := putObjects(userClient, []string{"my-obj"}, bucket) + return checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)) + }) +} + +// AccessControl_bucket_policy_condition_explicit_deny_overrides_allow +// covers a Deny statement scoped by a matching IpAddress Condition +// overriding a broader, unconditional Allow — the same explicit-deny-wins +// precedence bucket policies already have for unconditional statements, +// now confirmed to hold once one side's match depends on Condition +// evaluation too. +func AccessControl_bucket_policy_condition_explicit_deny_overrides_allow(s *S3Conf) error { + testName := "AccessControl_bucket_policy_condition_explicit_deny_overrides_allow" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser := getUser("user") + if err := createUsers(s, []user{testuser}); err != nil { + return err + } + + if err := putBucketPolicyDoc(s, bucket, + bucketStatement{ + Effect: "Allow", + Principal: testuser.access, + Action: "s3:PutObject", + Resource: fmt.Sprintf("arn:aws:s3:::%s/*", bucket), + }, + bucketStatement{ + Effect: "Deny", + Principal: testuser.access, + Action: "s3:PutObject", + Resource: fmt.Sprintf("arn:aws:s3:::%s/*", bucket), + Condition: json.RawMessage(`{"IpAddress":{"aws:SourceIp":"0.0.0.0/0"}}`), + }, + ); err != nil { + return err + } + + userClient := s.getUserClient(testuser) + _, err := putObjects(userClient, []string{"my-obj"}, bucket) + return checkApiErr(err, s3err.GetExplicitDenyAccessErr(testuser.access, "s3:PutObject", + fmt.Sprintf("arn:aws:s3:::%s/my-obj", bucket), "a resource-based policy")) + }) +} + +// AccessControl_bucket_policy_condition_s3_prefix covers the s3:prefix +// condition key, populated from a ListObjectsV2 request's own Prefix +// parameter: an Allow scoped to a specific prefix grants a request naming +// that exact prefix and denies one that doesn't. +func AccessControl_bucket_policy_condition_s3_prefix(s *S3Conf) error { + testName := "AccessControl_bucket_policy_condition_s3_prefix" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser := getUser("user") + if err := createUsers(s, []user{testuser}); err != nil { + return err + } + + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: "Allow", + Principal: testuser.access, + Action: "s3:ListBucket", + Resource: fmt.Sprintf("arn:aws:s3:::%s", bucket), + Condition: json.RawMessage(`{"StringEquals":{"s3:prefix":"photos/"}}`), + }); err != nil { + return err + } + + userClient := s.getUserClient(testuser) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := userClient.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucket, + Prefix: getPtr("photos/"), + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = userClient.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucket, + Prefix: getPtr("videos/"), + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)) + }) +} + +// The tests below cover every Condition operator family bucket policies +// support with at least one Allow and one Deny case each, each scoped to a +// real condition-context key the S3 gateway actually populates from the +// request, so the whole round trip - PutBucketPolicy, the live request, +// and the resulting Allow/Deny - is exercised end to end, not just the +// shared evaluator in isolation (already covered exhaustively by +// internal/condition's own unit tests). +// +// ArnEquals/ArnLike/ArnNotEquals/ArnNotLike are deliberately not covered +// here: they'd need aws:PrincipalArn, which bucket-policy Condition doesn't +// populate today (only identity-policy Condition does - see +// project_s3_bucket_policy_condition memory for why). The operator logic +// itself is still covered by internal/condition's unit tests +// (TestEvaluateConditionArn); what's untested is only the wiring, because +// there's nothing to wire yet. + +// AccessControl_bucket_policy_condition_string_operators covers the full +// String family (Equals/NotEquals/Like/NotLike, both plain and IgnoreCase) +// against s3:prefix, populated from a ListObjectsV2 request's own Prefix +// parameter. +func AccessControl_bucket_policy_condition_string_operators(s *S3Conf) error { + testName := "AccessControl_bucket_policy_condition_string_operators" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser := getUser("user") + if err := createUsers(s, []user{testuser}); err != nil { + return err + } + userClient := s.getUserClient(testuser) + + for _, tc := range []struct { + name string + condition string + prefix string + wantAllow bool + }{ + {"StringEquals matches", `{"StringEquals":{"s3:prefix":"photos/"}}`, "photos/", true}, + {"StringEquals mismatches", `{"StringEquals":{"s3:prefix":"photos/"}}`, "videos/", false}, + {"StringNotEquals passes on a different value", `{"StringNotEquals":{"s3:prefix":"photos/"}}`, "videos/", true}, + {"StringNotEquals fails on the same value", `{"StringNotEquals":{"s3:prefix":"photos/"}}`, "photos/", false}, + {"StringLike wildcard matches", `{"StringLike":{"s3:prefix":"photos/*"}}`, "photos/vacation", true}, + {"StringLike wildcard mismatches", `{"StringLike":{"s3:prefix":"photos/*"}}`, "videos/vacation", false}, + {"StringNotLike passes when the pattern doesn't match", `{"StringNotLike":{"s3:prefix":"photos/*"}}`, "videos/vacation", true}, + {"StringNotLike fails when the pattern matches", `{"StringNotLike":{"s3:prefix":"photos/*"}}`, "photos/vacation", false}, + {"StringEqualsIgnoreCase matches regardless of case", `{"StringEqualsIgnoreCase":{"s3:prefix":"Photos/"}}`, "photos/", true}, + {"StringNotEqualsIgnoreCase fails when equal regardless of case", `{"StringNotEqualsIgnoreCase":{"s3:prefix":"Photos/"}}`, "photos/", false}, + } { + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: "Allow", + Principal: testuser.access, + Action: "s3:ListBucket", + Resource: fmt.Sprintf("arn:aws:s3:::%s", bucket), + Condition: json.RawMessage(tc.condition), + }); err != nil { + return fmt.Errorf("%s: %w", tc.name, err) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := userClient.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucket, + Prefix: getPtr(tc.prefix), + }) + cancel() + + if tc.wantAllow { + if err != nil { + return fmt.Errorf("%s: expected success, got %w", tc.name, err) + } + continue + } + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return fmt.Errorf("%s: %w", tc.name, err) + } + } + return nil + }) +} + +// AccessControl_bucket_policy_condition_numeric_operators covers the full +// Numeric family against s3:max-keys, populated from a ListObjectsV2 +// request's own MaxKeys parameter, binding to the request's actual +// MaxKeys value, not some default. +func AccessControl_bucket_policy_condition_numeric_operators(s *S3Conf) error { + testName := "AccessControl_bucket_policy_condition_numeric_operators" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser := getUser("user") + if err := createUsers(s, []user{testuser}); err != nil { + return err + } + userClient := s.getUserClient(testuser) + + for _, tc := range []struct { + name string + condition string + maxKeys int32 + wantAllow bool + }{ + {"NumericEquals matches", `{"NumericEquals":{"s3:max-keys":"5"}}`, 5, true}, + {"NumericEquals mismatches", `{"NumericEquals":{"s3:max-keys":"5"}}`, 6, false}, + {"NumericNotEquals passes on a different value", `{"NumericNotEquals":{"s3:max-keys":"5"}}`, 6, true}, + {"NumericNotEquals fails on the same value", `{"NumericNotEquals":{"s3:max-keys":"5"}}`, 5, false}, + {"NumericLessThan matches", `{"NumericLessThan":{"s3:max-keys":"10"}}`, 5, true}, + {"NumericLessThan boundary does not match", `{"NumericLessThan":{"s3:max-keys":"10"}}`, 10, false}, + {"NumericLessThanEquals boundary matches", `{"NumericLessThanEquals":{"s3:max-keys":"10"}}`, 10, true}, + {"NumericGreaterThan matches", `{"NumericGreaterThan":{"s3:max-keys":"5"}}`, 10, true}, + {"NumericGreaterThan boundary does not match", `{"NumericGreaterThan":{"s3:max-keys":"5"}}`, 5, false}, + {"NumericGreaterThanEquals boundary matches", `{"NumericGreaterThanEquals":{"s3:max-keys":"5"}}`, 5, true}, + } { + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: "Allow", + Principal: testuser.access, + Action: "s3:ListBucket", + Resource: fmt.Sprintf("arn:aws:s3:::%s", bucket), + Condition: json.RawMessage(tc.condition), + }); err != nil { + return fmt.Errorf("%s: %w", tc.name, err) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := userClient.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucket, + MaxKeys: getPtr(tc.maxKeys), + }) + cancel() + + if tc.wantAllow { + if err != nil { + return fmt.Errorf("%s: expected success, got %w", tc.name, err) + } + continue + } + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return fmt.Errorf("%s: %w", tc.name, err) + } + } + return nil + }) +} + +// AccessControl_bucket_policy_condition_date_operators covers the +// Less/Greater halves of the Date family against aws:CurrentTime, using +// dates 48 hours in the past/future so the outcome is never flaky +// regardless of test-runner clock skew or how long the request takes. +// DateEquals/DateNotEquals aren't covered here - matching an exact instant +// against a live "now" is inherently flaky at this layer - but are +// exercised at internal/condition's unit-test layer +// (TestEvaluateConditionDate), which is what actually implements the +// comparison; only the request-to-context wiring is new here. +func AccessControl_bucket_policy_condition_date_operators(s *S3Conf) error { + testName := "AccessControl_bucket_policy_condition_date_operators" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser := getUser("user") + if err := createUsers(s, []user{testuser}); err != nil { + return err + } + userClient := s.getUserClient(testuser) + + past := time.Now().Add(-48 * time.Hour).UTC().Format(time.RFC3339) + future := time.Now().Add(48 * time.Hour).UTC().Format(time.RFC3339) + + for _, tc := range []struct { + name string + condition string + wantAllow bool + }{ + {"DateLessThan a future date matches", fmt.Sprintf(`{"DateLessThan":{"aws:CurrentTime":%q}}`, future), true}, + {"DateLessThan a past date does not match", fmt.Sprintf(`{"DateLessThan":{"aws:CurrentTime":%q}}`, past), false}, + {"DateLessThanEquals a future date matches", fmt.Sprintf(`{"DateLessThanEquals":{"aws:CurrentTime":%q}}`, future), true}, + {"DateGreaterThan a past date matches", fmt.Sprintf(`{"DateGreaterThan":{"aws:CurrentTime":%q}}`, past), true}, + {"DateGreaterThan a future date does not match", fmt.Sprintf(`{"DateGreaterThan":{"aws:CurrentTime":%q}}`, future), false}, + {"DateGreaterThanEquals a past date matches", fmt.Sprintf(`{"DateGreaterThanEquals":{"aws:CurrentTime":%q}}`, past), true}, + } { + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: "Allow", + Principal: testuser.access, + Action: "s3:ListBucket", + Resource: fmt.Sprintf("arn:aws:s3:::%s", bucket), + Condition: json.RawMessage(tc.condition), + }); err != nil { + return fmt.Errorf("%s: %w", tc.name, err) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := userClient.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) + cancel() + + if tc.wantAllow { + if err != nil { + return fmt.Errorf("%s: expected success, got %w", tc.name, err) + } + continue + } + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return fmt.Errorf("%s: %w", tc.name, err) + } + } + return nil + }) +} + +// AccessControl_bucket_policy_condition_bool_operator covers Bool against +// aws:SecureTransport, comparing it to whether s's own endpoint is actually +// using TLS - so this passes the same way against either an HTTP or HTTPS +// test target. +func AccessControl_bucket_policy_condition_bool_operator(s *S3Conf) error { + testName := "AccessControl_bucket_policy_condition_bool_operator" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser := getUser("user") + if err := createUsers(s, []user{testuser}); err != nil { + return err + } + userClient := s.getUserClient(testuser) + + secure := strings.HasPrefix(s.endpoint, "https://") + + for _, tc := range []struct { + name string + condition string + wantAllow bool + }{ + {"Bool matches the request's actual transport", fmt.Sprintf(`{"Bool":{"aws:SecureTransport":"%t"}}`, secure), true}, + {"Bool mismatches the request's actual transport", fmt.Sprintf(`{"Bool":{"aws:SecureTransport":"%t"}}`, !secure), false}, + } { + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: "Allow", + Principal: testuser.access, + Action: "s3:ListBucket", + Resource: fmt.Sprintf("arn:aws:s3:::%s", bucket), + Condition: json.RawMessage(tc.condition), + }); err != nil { + return fmt.Errorf("%s: %w", tc.name, err) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := userClient.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) + cancel() + + if tc.wantAllow { + if err != nil { + return fmt.Errorf("%s: expected success, got %w", tc.name, err) + } + continue + } + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return fmt.Errorf("%s: %w", tc.name, err) + } + } + return nil + }) +} + +// AccessControl_bucket_policy_condition_binary_operator covers BinaryEquals +// against s3:prefix: AWS's own condition-operator reference documents the +// match as a literal string comparison between the policy's base64 text and +// the request context value +func AccessControl_bucket_policy_condition_binary_operator(s *S3Conf) error { + testName := "AccessControl_bucket_policy_condition_binary_operator" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser := getUser("user") + if err := createUsers(s, []user{testuser}); err != nil { + return err + } + userClient := s.getUserClient(testuser) + + encoded := base64.StdEncoding.EncodeToString([]byte("photos/")) + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: "Allow", + Principal: testuser.access, + Action: "s3:ListBucket", + Resource: fmt.Sprintf("arn:aws:s3:::%s", bucket), + Condition: json.RawMessage(fmt.Sprintf(`{"BinaryEquals":{"s3:prefix":%q}}`, encoded)), + }); err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := userClient.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucket, + Prefix: getPtr(encoded), + }) + cancel() + if err != nil { + return fmt.Errorf("matching prefix: expected success, got %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = userClient.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucket, + Prefix: getPtr("photos/"), + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)) + }) +} + +// AccessControl_bucket_policy_condition_null_operator covers Null against +// s3:prefix's presence/absence: Null:"true" requires the key be absent (no +// Prefix parameter on the request at all), Null:"false" requires it be +// present. +func AccessControl_bucket_policy_condition_null_operator(s *S3Conf) error { + testName := "AccessControl_bucket_policy_condition_null_operator" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser := getUser("user") + if err := createUsers(s, []user{testuser}); err != nil { + return err + } + userClient := s.getUserClient(testuser) + + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: "Allow", + Principal: testuser.access, + Action: "s3:ListBucket", + Resource: fmt.Sprintf("arn:aws:s3:::%s", bucket), + Condition: json.RawMessage(`{"Null":{"s3:prefix":"true"}}`), + }); err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := userClient.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) + cancel() + if err != nil { + return fmt.Errorf("null:true, no prefix param: expected success, got %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = userClient.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucket, + Prefix: getPtr("photos/"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return fmt.Errorf("null:true, prefix param present: %w", err) + } + + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: "Allow", + Principal: testuser.access, + Action: "s3:ListBucket", + Resource: fmt.Sprintf("arn:aws:s3:::%s", bucket), + Condition: json.RawMessage(`{"Null":{"s3:prefix":"false"}}`), + }); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = userClient.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucket, + Prefix: getPtr("photos/"), + }) + cancel() + if err != nil { + return fmt.Errorf("null:false, prefix param present: expected success, got %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = userClient.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return fmt.Errorf("null:false, no prefix param: %w", err) + } + return nil + }) +} + +// AccessControl_bucket_policy_condition_not_ip_address_allow and +// ..._deny cover NotIpAddress, the negated counterpart of the IpAddress +// coverage above (AccessControl_bucket_policy_condition_ip_allow/ +// ..._ip_deny_no_match): it grants access when the caller's address falls +// OUTSIDE the given range. +func AccessControl_bucket_policy_condition_not_ip_address_allow(s *S3Conf) error { + testName := "AccessControl_bucket_policy_condition_not_ip_address_allow" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser := getUser("user") + if err := createUsers(s, []user{testuser}); err != nil { + return err + } + + // TEST-NET-3 (RFC 5737) can never match a real caller, so + // NotIpAddress against it is always true. + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: "Allow", + Principal: testuser.access, + Action: "s3:PutObject", + Resource: fmt.Sprintf("arn:aws:s3:::%s/*", bucket), + Condition: json.RawMessage(`{"NotIpAddress":{"aws:SourceIp":"203.0.113.0/24"}}`), + }); err != nil { + return err + } + + userClient := s.getUserClient(testuser) + _, err := putObjects(userClient, []string{"my-obj"}, bucket) + return err + }) +} + +func AccessControl_bucket_policy_condition_not_ip_address_deny(s *S3Conf) error { + testName := "AccessControl_bucket_policy_condition_not_ip_address_deny" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser := getUser("user") + if err := createUsers(s, []user{testuser}); err != nil { + return err + } + + // 0.0.0.0/0 matches any IPv4 address, so NotIpAddress against it is + // always false. + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: "Allow", + Principal: testuser.access, + Action: "s3:PutObject", + Resource: fmt.Sprintf("arn:aws:s3:::%s/*", bucket), + Condition: json.RawMessage(`{"NotIpAddress":{"aws:SourceIp":"0.0.0.0/0"}}`), + }); err != nil { + return err + } + + userClient := s.getUserClient(testuser) + _, err := putObjects(userClient, []string{"my-obj"}, bucket) + return checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)) + }) +} diff --git a/tests/integration/DeleteObjects.go b/tests/integration/DeleteObjects.go index db22d08e..71ad96aa 100644 --- a/tests/integration/DeleteObjects.go +++ b/tests/integration/DeleteObjects.go @@ -18,8 +18,10 @@ import ( "context" "fmt" + "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/versity/versitygw/s3err" ) func DeleteObjects_empty_input(s *S3Conf) error { @@ -156,3 +158,194 @@ func DeleteObjects_success(s *S3Conf) error { return nil }) } + +// DeleteObjects_iam_mixed_denials_and_success covers a single batch mixing +// every DeleteObjects outcome at once: a key the identity policy denies, a +// governance-locked key with no bypass, and keys the caller may freely +// delete. All three outcomes land in one response — no top-level error — +// with Deleted and Errors each preserving the order the keys were +// requested in. +func DeleteObjects_iam_mixed_denials_and_success(s *S3Conf) error { + testName := "DeleteObjects_iam_mixed_denials_and_success" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + for _, key := range []string{"allowed/one", "allowed/two", "denied/one"} { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: &key}) + cancel() + if err != nil { + return err + } + } + if err := putGovernanceLockedObject(s, bucket, "locked/one"); err != nil { + return err + } + + user, cleanup, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: actS3DeleteObject, + Resource: []string{objectArn(bucket, "allowed/*"), objectArn(bucket, "locked/*")}, + }), + }) + if err != nil { + return err + } + defer cleanup() + + delObjects := []types.ObjectIdentifier{ + {Key: getPtr("allowed/one")}, + {Key: getPtr("denied/one")}, + {Key: getPtr("locked/one")}, + {Key: getPtr("allowed/two")}, + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := user.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{Objects: delObjects}, + }) + cancel() + if err != nil { + return fmt.Errorf("expected DeleteObjects to succeed with per-object denials, not fail outright: %w", err) + } + + if err := checkDeletedKeysInOrder(out.Deleted, []string{"allowed/one", "allowed/two"}); err != nil { + return err + } + return checkDeleteObjectsErrsInOrder(out.Errors, []struct { + key string + err s3err.S3Error + }{ + {"denied/one", wantImplicitDeny(user.arn, actS3DeleteObject, objectArn(bucket, "denied/one"))}, + {"locked/one", s3err.GetAPIError(s3err.ErrObjectLocked)}, + }) + }, withLock()) +} + +// DeleteObjects_iam_all_access_denied covers a batch where the identity +// policy grants nothing at all: the call itself still succeeds — no +// top-level error — with every object reported denied in Errors, in +// request order, and nothing in Deleted. +func DeleteObjects_iam_all_access_denied(s *S3Conf) error { + testName := "DeleteObjects_iam_all_access_denied" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + for _, key := range []string{"one", "two", "three"} { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: &key}) + cancel() + if err != nil { + return err + } + } + + user, cleanup, err := newS3IAMUser(root, s, nil) + if err != nil { + return err + } + defer cleanup() + + delObjects := []types.ObjectIdentifier{ + {Key: getPtr("one")}, {Key: getPtr("two")}, {Key: getPtr("three")}, + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := user.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{Objects: delObjects}, + }) + cancel() + if err != nil { + return fmt.Errorf("expected DeleteObjects to succeed with per-object denials, not fail outright: %w", err) + } + + if len(out.Deleted) != 0 { + return fmt.Errorf("expected nothing deleted, got %+v", out.Deleted) + } + return checkDeleteObjectsErrsInOrder(out.Errors, []struct { + key string + err s3err.S3Error + }{ + {"one", wantImplicitDeny(user.arn, actS3DeleteObject, objectArn(bucket, "one"))}, + {"two", wantImplicitDeny(user.arn, actS3DeleteObject, objectArn(bucket, "two"))}, + {"three", wantImplicitDeny(user.arn, actS3DeleteObject, objectArn(bucket, "three"))}, + }) + }) +} + +// DeleteObjects_iam_all_locked covers a batch where every object is +// governance-locked and none is deleted: the call still succeeds — no +// top-level error — with every object reported denied in Errors, in +// request order, and nothing in Deleted. Omitting the bypass header +// entirely reports the generic object-lock message; sending the header +// without s3:BypassGovernanceRetention reports the specific AccessDenied +// naming that action instead — the same distinction the single-object +// DELETE path makes, now confirmed for the batch path too. +func DeleteObjects_iam_all_locked(s *S3Conf) error { + testName := "DeleteObjects_iam_all_locked" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + for _, key := range []string{"locked/one", "locked/two"} { + if err := putGovernanceLockedObject(s, bucket, key); err != nil { + return err + } + } + + user, cleanup, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: actS3DeleteObject, Resource: objectsArn(bucket), + }), + }) + if err != nil { + return err + } + defer cleanup() + + delObjects := []types.ObjectIdentifier{{Key: getPtr("locked/one")}, {Key: getPtr("locked/two")}} + + // No bypass header at all: the generic object-lock message. + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := user.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{Objects: delObjects}, + }) + cancel() + if err != nil { + return fmt.Errorf("expected DeleteObjects to succeed with per-object denials, not fail outright: %w", err) + } + if len(out.Deleted) != 0 { + return fmt.Errorf("expected nothing deleted, got %+v", out.Deleted) + } + if err := checkDeleteObjectsErrsInOrder(out.Errors, []struct { + key string + err s3err.S3Error + }{ + {"locked/one", s3err.GetAPIError(s3err.ErrObjectLocked)}, + {"locked/two", s3err.GetAPIError(s3err.ErrObjectLocked)}, + }); err != nil { + return fmt.Errorf("without bypass header: %w", err) + } + + // Bypass header sent, but the identity policy doesn't grant + // s3:BypassGovernanceRetention: a specific AccessDenied naming that + // action, not the generic object-lock message. + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err = user.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{Objects: delObjects}, + BypassGovernanceRetention: getPtr(true), + }) + cancel() + if err != nil { + return fmt.Errorf("expected DeleteObjects to succeed with per-object denials, not fail outright: %w", err) + } + if len(out.Deleted) != 0 { + return fmt.Errorf("expected nothing deleted, got %+v", out.Deleted) + } + if err := checkDeleteObjectsErrsInOrder(out.Errors, []struct { + key string + err s3err.S3Error + }{ + {"locked/one", wantImplicitDeny(user.arn, actS3BypassGovernance, objectArn(bucket, "locked/one"))}, + {"locked/two", wantImplicitDeny(user.arn, actS3BypassGovernance, objectArn(bucket, "locked/two"))}, + }); err != nil { + return fmt.Errorf("with bypass header, no permission: %w", err) + } + return nil + }, withLock()) +} diff --git a/tests/integration/GetObjectRetention.go b/tests/integration/GetObjectRetention.go index d0ff789b..b454492e 100644 --- a/tests/integration/GetObjectRetention.go +++ b/tests/integration/GetObjectRetention.go @@ -113,7 +113,7 @@ func GetObjectRetention_success(s *S3Conf) error { return err } - date := time.Now().Add(time.Hour * 3) + date := time.Now().Add(complianceTestRetention) retention := types.ObjectLockRetention{ Mode: types.ObjectLockRetentionModeCompliance, RetainUntilDate: &date, diff --git a/tests/integration/PutBucketPolicy.go b/tests/integration/PutBucketPolicy.go index 7dd7e6cc..a9d4ecaa 100644 --- a/tests/integration/PutBucketPolicy.go +++ b/tests/integration/PutBucketPolicy.go @@ -523,7 +523,9 @@ func PutBucketPolicy_explicit_deny(s *S3Conf) error { Key: getPtr("someprefix/hello"), }) cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + if err := checkApiErr(err, s3err.GetExplicitDenyAccessErr( + testuser2.access, "s3:PutObject", fmt.Sprintf("%v/someprefix/hello", resource), "a resource-based policy", + )); err != nil { return err } @@ -717,3 +719,141 @@ func PutBucketPolicy_status(s *S3Conf) error { return nil }) } + +func PutBucketPolicy_condition_invalid_operator(s *S3Conf) error { + testName := "PutBucketPolicy_condition_invalid_operator" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + for _, tc := range []struct { + condition string + // operatorName is the exact, as-written operator name AWS's + // "Invalid Condition type : " message echoes back. + operatorName string + }{ + // completely unrecognized operator + {`{"NotARealOperator":{"aws:SourceIp":"1.2.3.4/32"}}`, "NotARealOperator"}, + // operator names are case-sensitive - lowercase is unrecognized + {`{"stringequals":{"aws:UserAgent":"foo"}}`, "stringequals"}, + // IfExists suffix on an operator that doesn't take one + {`{"NullIfExists":{"aws:UserAgent":"true"}}`, "NullIfExists"}, + // unrecognized ForAllValues/ForAnyValue qualifier prefix + {`{"ForSomeValues:StringEquals":{"aws:UserAgent":"foo"}}`, "ForSomeValues:StringEquals"}, + } { + doc := fmt.Sprintf(`{"Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject", + "Resource":"arn:aws:s3:::%s/*","Condition":%s}]}`, bucket, tc.condition) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ + Bucket: &bucket, + Policy: &doc, + }) + cancel() + + if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("Invalid Condition type : %s", tc.operatorName))); err != nil { + return err + } + } + return nil + }) +} + +func PutBucketPolicy_condition_invalid_key(s *S3Conf) error { + testName := "PutBucketPolicy_condition_invalid_key" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + for _, condition := range []string{ + // unrecognized key under a String operator + `{"StringEquals":{"s3:FakeKeyDoesNotExist":"foo"}}`, + // unrecognized key under a Numeric operator + `{"NumericEquals":{"s3:NotARealKey":"5"}}`, + // unrecognized key under IpAddress + `{"IpAddress":{"aws:NotARealIpKey":"10.0.0.0/8"}}`, + // a plausible-looking but non-existent aws: global key + `{"StringEquals":{"aws:NotARealGlobalKey":"foo"}}`, + } { + doc := fmt.Sprintf(`{"Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject", + "Resource":"arn:aws:s3:::%s/*","Condition":%s}]}`, bucket, condition) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ + Bucket: &bucket, + Policy: &doc, + }) + cancel() + + if err := checkApiErr(err, getMalformedPolicyError("Policy has an invalid condition key")); err != nil { + return err + } + } + return nil + }) +} + +func PutBucketPolicy_condition_action_mismatch(s *S3Conf) error { + testName := "PutBucketPolicy_condition_action_mismatch" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + for _, tc := range []struct { + action string + condition string + }{ + // s3:prefix only applies to s3:ListBucket/s3:ListBucketVersions, + // not s3:GetObject. + {`"s3:GetObject"`, `{"StringEquals":{"s3:prefix":"foo"}}`}, + // ... and notably not s3:ListBucketMultipartUploads either, + // despite also being a List-shaped action. + {`"s3:ListBucketMultipartUploads"`, `{"StringEquals":{"s3:prefix":"foo"}}`}, + // s3:x-amz-acl only applies to s3:PutObject/PutBucketAcl/PutObjectAcl. + {`"s3:GetObject"`, `{"StringEquals":{"s3:x-amz-acl":"public-read"}}`}, + // s3:VersionId only applies to the *Version* action family. + {`"s3:GetObject"`, `{"StringEquals":{"s3:VersionId":"abc123"}}`}, + // an explicit multi-action list requires every action to + // support the key, even though s3:PutObject alone would. + {`["s3:GetObject","s3:PutObject"]`, `{"StringEquals":{"s3:x-amz-acl":"public-read"}}`}, + } { + doc := fmt.Sprintf(`{"Statement":[{"Effect":"Allow","Principal":"*","Action":%s, + "Resource":"arn:aws:s3:::%s/*","Condition":%s}]}`, tc.action, bucket, tc.condition) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ + Bucket: &bucket, + Policy: &doc, + }) + cancel() + + if err := checkApiErr(err, getMalformedPolicyError("Conditions do not apply to combination of actions and resources in statement")); err != nil { + return err + } + } + return nil + }) +} + +func PutBucketPolicy_condition_invalid_ip(s *S3Conf) error { + testName := "PutBucketPolicy_condition_invalid_ip" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + for _, condition := range []string{ + // not an IP address at all + `{"IpAddress":{"aws:SourceIp":"not-an-ip"}}`, + // malformed CIDR notation (octet out of range, bad prefix length) + `{"IpAddress":{"aws:SourceIp":"300.1.1.1/40"}}`, + // the same bad value under NotIpAddress + `{"NotIpAddress":{"aws:SourceIp":"not-an-ip"}}`, + // the IP-format check fires regardless of which operator wraps + // the key - even Null, which has nothing to do with IP syntax. + `{"Null":{"aws:SourceIp":"true"}}`, + } { + doc := fmt.Sprintf(`{"Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject", + "Resource":"arn:aws:s3:::%s/*","Condition":%s}]}`, bucket, condition) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ + Bucket: &bucket, + Policy: &doc, + }) + cancel() + + if err := checkApiErr(err, getMalformedPolicyError("Invalid IP address in Conditions")); err != nil { + return err + } + } + return nil + }) +} diff --git a/tests/integration/PutObject.go b/tests/integration/PutObject.go index 835232b2..d2622d3b 100644 --- a/tests/integration/PutObject.go +++ b/tests/integration/PutObject.go @@ -263,7 +263,7 @@ func PutObject_with_object_lock(s *S3Conf) error { testName := "PutObject_with_object_lock" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" - retainUntilDate := time.Now().AddDate(1, 0, 0) + retainUntilDate := time.Now().Add(complianceTestRetention) _, err := putObjectWithData(10, &s3.PutObjectInput{ Bucket: &bucket, diff --git a/tests/integration/PutObjectRetention.go b/tests/integration/PutObjectRetention.go index e384ca1f..758f6576 100644 --- a/tests/integration/PutObjectRetention.go +++ b/tests/integration/PutObjectRetention.go @@ -140,13 +140,13 @@ func PutObjectRetention_invalid_mode(s *S3Conf) error { func PutObjectRetention_overwrite_compliance_mode(s *S3Conf) error { testName := "PutObjectRetention_overwrite_compliance_mode" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - date := time.Now().Add(time.Hour * 3) obj := "my-obj" _, err := putObjects(s3client, []string{obj}, bucket) if err != nil { return err } + date := time.Now().Add(complianceTestRetention) ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ Bucket: &bucket, @@ -161,13 +161,18 @@ func PutObjectRetention_overwrite_compliance_mode(s *S3Conf) error { return err } + // A fresh date, not the one above: COMPLIANCE denies a mode switch + // unconditionally regardless of what date is requested, so this only + // needs to still be in the future by request time, not tied to what + // was stored a round trip ago. + attempted := time.Now().Add(complianceTestRetention) ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ Bucket: &bucket, Key: &obj, Retention: &types.ObjectLockRetention{ Mode: types.ObjectLockRetentionModeGovernance, - RetainUntilDate: &date, + RetainUntilDate: &attempted, }, }) cancel() @@ -182,13 +187,13 @@ func PutObjectRetention_overwrite_compliance_mode(s *S3Conf) error { func PutObjectRetention_overwrite_compliance_with_compliance(s *S3Conf) error { testName := "PutObjectRetention_overwrite_compliance_with_compliance" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - date := time.Now().Add(time.Hour * 200) obj := "my-obj" _, err := putObjects(s3client, []string{obj}, bucket) if err != nil { return err } + date := time.Now().Add(complianceTestRetention) ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ Bucket: &bucket, @@ -203,7 +208,10 @@ func PutObjectRetention_overwrite_compliance_with_compliance(s *S3Conf) error { return err } - newDate := date.AddDate(2, 0, 0) + // Extending stays within complianceTestRetention's budget so the + // object can still be waited out: a COMPLIANCE retention pushed years + // into the future could never be cleaned up. + newDate := date.Add(complianceTestRetention) ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ @@ -247,7 +255,7 @@ func PutObjectRetention_overwrite_governance_with_governance(s *S3Conf) error { return err } - newDate := date.AddDate(2, 0, 0) + newDate := date.Add(time.Hour) ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ @@ -312,13 +320,13 @@ func PutObjectRetention_overwrite_governance_without_bypass_specified(s *S3Conf) func PutObjectRetention_overwrite_governance_with_permission(s *S3Conf) error { testName := "PutObjectRetention_overwrite_governance_with_permission" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - date := time.Now().Add(time.Hour * 3) obj := "my-obj" _, err := putObjects(s3client, []string{obj}, bucket) if err != nil { return err } + date := time.Now().Add(complianceTestRetention) ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ Bucket: &bucket, @@ -346,13 +354,14 @@ func PutObjectRetention_overwrite_governance_with_permission(s *S3Conf) error { return err } + complianceDate := time.Now().Add(complianceTestRetention) ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ Bucket: &bucket, Key: &obj, Retention: &types.ObjectLockRetention{ Mode: types.ObjectLockRetentionModeCompliance, - RetainUntilDate: &date, + RetainUntilDate: &complianceDate, }, BypassGovernanceRetention: &bypass, }) @@ -365,10 +374,249 @@ func PutObjectRetention_overwrite_governance_with_permission(s *S3Conf) error { }, withLock()) } +func PutObjectRetention_shorten_governance_without_bypass(s *S3Conf) error { + testName := "PutObjectRetention_shorten_governance_without_bypass" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + date := time.Now().Add(time.Hour) + obj := "my-obj" + _, err := putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: &date, + }, + }) + cancel() + if err != nil { + return err + } + + shorter := time.Now().Add(complianceTestRetention / 2) + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: &shorter, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return err + } + + return cleanupLockedObjects(s3client, bucket, []objToDelete{{key: obj}}) + }, withLock()) +} + +func PutObjectRetention_shorten_governance_with_bypass(s *S3Conf) error { + testName := "PutObjectRetention_shorten_governance_with_bypass" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + date := time.Now().Add(time.Hour) + obj := "my-obj" + _, err := putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: &date, + }, + }) + cancel() + if err != nil { + return err + } + + policy := genPolicyDoc("Allow", fmt.Sprintf(`"%v"`, s.awsID), `["s3:BypassGovernanceRetention"]`, fmt.Sprintf(`"arn:aws:s3:::%v/*"`, bucket)) + bypass := true + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ + Bucket: &bucket, + Policy: &policy, + }) + cancel() + if err != nil { + return err + } + + shorter := time.Now().Add(complianceTestRetention / 2) + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: &shorter, + }, + BypassGovernanceRetention: &bypass, + }) + cancel() + if err != nil { + return err + } + + return cleanupLockedObjects(s3client, bucket, []objToDelete{{key: obj}}) + }, withLock()) +} + +func PutObjectRetention_shorten_compliance_denied(s *S3Conf) error { + testName := "PutObjectRetention_shorten_compliance_denied" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + date := time.Now().Add(complianceTestRetention) + obj := "my-obj" + _, err := putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + RetainUntilDate: &date, + }, + }) + cancel() + if err != nil { + return err + } + + policy := genPolicyDoc("Allow", fmt.Sprintf(`"%v"`, s.awsID), `["s3:BypassGovernanceRetention"]`, fmt.Sprintf(`"arn:aws:s3:::%v/*"`, bucket)) + bypass := true + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ + Bucket: &bucket, + Policy: &policy, + }) + cancel() + if err != nil { + return err + } + + // Recomputed fresh right before each request below, rather than once + // up front: the server rejects a RetainUntilDate that has already + // passed (InvalidArgument) before it ever gets to the object-lock + // comparison this test is exercising (ErrObjectLocked) + newShorterDate := func() time.Time { + return time.Now().Add(time.Until(date) / 2) + } + + // Neither asking to bypass nor holding the permission helps. + shorter := newShorterDate() + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + RetainUntilDate: &shorter, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return err + } + + shorter = newShorterDate() + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + RetainUntilDate: &shorter, + }, + BypassGovernanceRetention: &bypass, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return err + } + + return cleanupLockedObjects(s3client, bucket, []objToDelete{{key: obj, isCompliance: true}}) + }, withLock()) +} + +func PutObjectRetention_rewrite_same_date(s *S3Conf) error { + testName := "PutObjectRetention_rewrite_same_date" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + govObj, compObj := "my-obj-governance", "my-obj-compliance" + _, err := putObjects(s3client, []string{govObj, compObj}, bucket) + if err != nil { + return err + } + + for _, obj := range []struct { + key string + mode types.ObjectLockRetentionMode + }{ + {key: govObj, mode: types.ObjectLockRetentionModeGovernance}, + {key: compObj, mode: types.ObjectLockRetentionModeCompliance}, + } { + date := time.Now().Add(complianceTestRetention) + if obj.mode == types.ObjectLockRetentionModeGovernance { + date = time.Now().Add(time.Hour) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj.key, + Retention: &types.ObjectLockRetention{ + Mode: obj.mode, + RetainUntilDate: &date, + }, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj.key, + Retention: &types.ObjectLockRetention{ + Mode: obj.mode, + RetainUntilDate: &date, + }, + }) + cancel() + if err != nil { + return fmt.Errorf("%v: rewriting the identical date must be allowed: %w", obj.mode, err) + } + } + + return cleanupLockedObjects(s3client, bucket, []objToDelete{ + {key: govObj}, + {key: compObj, isCompliance: true}, + }) + }, withLock()) +} + func PutObjectRetention_success(s *S3Conf) error { testName := "PutObjectRetention_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - date := time.Now().Add(time.Hour * 3) key := "my-obj" _, err := putObjects(s3client, []string{key}, bucket) @@ -376,6 +624,7 @@ func PutObjectRetention_success(s *S3Conf) error { return err } + date := time.Now().Add(complianceTestRetention) ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ Bucket: &bucket, diff --git a/tests/integration/WORM_protection.go b/tests/integration/WORM_protection.go index 1a095fc0..72b12fd0 100644 --- a/tests/integration/WORM_protection.go +++ b/tests/integration/WORM_protection.go @@ -210,6 +210,58 @@ func WORMProtection_bucket_object_lock_governance_bypass_delete_multiple(s *S3Co }, withLock()) } +func WORMProtection_delete_objects_locked_object_partial_success(s *S3Conf) error { + testName := "WORMProtection_delete_objects_locked_object_partial_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + locked, unlocked := "locked-obj", "unlocked-obj" + if _, err := putObjects(s3client, []string{locked, unlocked}, bucket); err != nil { + return err + } + + date := time.Now().Add(time.Hour) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &locked, + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: &date, + }, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{ + Objects: []types.ObjectIdentifier{ + {Key: &locked}, + {Key: &unlocked}, + }, + }, + }) + cancel() + if err != nil { + return fmt.Errorf("expected DeleteObjects to succeed with a per-object denial, not fail outright: %w", err) + } + + if len(out.Errors) != 1 { + return fmt.Errorf("expected exactly 1 per-object error, got %+v", out.Errors) + } + if err := checkDeleteObjectsErr(out.Errors[0], locked, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return err + } + if err := checkDeletedKeysInOrder(out.Deleted, []string{unlocked}); err != nil { + return err + } + + return cleanupLockedObjects(s3client, bucket, []objToDelete{{key: locked, isCompliance: false}}) + }, withLock()) +} + func WORMProtection_object_lock_retention_compliance_locked(s *S3Conf) error { testName := "WORMProtection_object_lock_retention_compliance_locked" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -220,7 +272,7 @@ func WORMProtection_object_lock_retention_compliance_locked(s *S3Conf) error { return err } - date := time.Now().Add(time.Hour * 3) + date := time.Now().Add(2 * complianceTestRetention) ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ Bucket: &bucket, diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 28ed43d7..b4b1f84e 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -44,7 +44,7 @@ func TestAuthentication(ts *TestState) { } func TestPresignedAuthentication(ts *TestState) { - ts.Run(PresignedAuth_security_token_not_supported) + ts.Run(PresignedAuth_security_token_with_permanent_credentials) ts.Run(PresignedAuth_unsupported_algorithm) ts.Run(PresignedAuth_ECDSA_not_supported) ts.Run(PresignedAuth_missing_signature_query_param) @@ -626,6 +626,10 @@ func TestPutBucketPolicy(ts *TestState) { ts.Run(PutBucketPolicy_version) ts.Run(PutBucketPolicy_success) ts.Run(PutBucketPolicy_status) + ts.Run(PutBucketPolicy_condition_invalid_operator) + ts.Run(PutBucketPolicy_condition_invalid_key) + ts.Run(PutBucketPolicy_condition_action_mismatch) + ts.Run(PutBucketPolicy_condition_invalid_ip) } func TestGetBucketPolicy(ts *TestState) { @@ -767,6 +771,10 @@ func TestPutObjectRetention(ts *TestState) { ts.Run(PutObjectRetention_overwrite_governance_with_governance) ts.Run(PutObjectRetention_overwrite_governance_without_bypass_specified) ts.Run(PutObjectRetention_overwrite_governance_with_permission) + ts.Run(PutObjectRetention_shorten_governance_without_bypass) + ts.Run(PutObjectRetention_shorten_governance_with_bypass) + ts.Run(PutObjectRetention_shorten_compliance_denied) + ts.Run(PutObjectRetention_rewrite_same_date) ts.Run(PutObjectRetention_success) } @@ -854,6 +862,7 @@ func TestWORMProtection(ts *TestState) { ts.Run(WORMProtection_bucket_object_lock_configuration_governance_mode) ts.Run(WORMProtection_bucket_object_lock_governance_bypass_delete) ts.Run(WORMProtection_bucket_object_lock_governance_bypass_delete_multiple) + ts.Run(WORMProtection_delete_objects_locked_object_partial_success) ts.Run(WORMProtection_object_lock_retention_compliance_locked) ts.Run(WORMProtection_object_lock_retention_governance_locked) ts.Run(WORMProtection_object_lock_retention_governance_bypass_overwrite_put) @@ -1553,6 +1562,67 @@ func TestIAMAccessControl(ts *TestState) { ts.Run(IAMAccessControl_CrossIdentity_AssumeRoleWithWebIdentityHasNoCallerIdentityCheck) } +func TestS3IAMAccessControl(ts *TestState) { + ts.Run(S3IAMAccessControl_no_policy_denies) + ts.Run(S3IAMAccessControl_root_bypasses_policies) + ts.Run(S3IAMAccessControl_identity_policy_allows_without_bucket_policy) + ts.Run(S3IAMAccessControl_identity_policy_action_wildcards) + ts.Run(S3IAMAccessControl_identity_policy_resource_scoping) + ts.Run(S3IAMAccessControl_identity_policy_bucket_vs_object_arn) + ts.Run(S3IAMAccessControl_identity_policy_not_action_and_not_resource) + ts.Run(S3IAMAccessControl_identity_policy_explicit_deny_wins) + ts.Run(S3IAMAccessControl_multiple_inline_policies_combine) + ts.Run(S3IAMAccessControl_bucket_policy_allows_without_identity_policy) + ts.Run(S3IAMAccessControl_bucket_policy_explicit_deny) + ts.Run(S3IAMAccessControl_policy_combinations) + ts.Run(S3IAMAccessControl_copy_object_requires_both_sides) + ts.Run(S3IAMAccessControl_create_bucket) + ts.Run(S3IAMAccessControl_governance_bypass_sources) + ts.Run(S3IAMAccessControl_governance_without_bypass_header) + ts.Run(S3IAMAccessControl_compliance_mode_not_bypassable) + ts.Run(S3IAMAccessControl_delete_objects_authorizes_each_key) + ts.Run(S3IAMAccessControl_delete_objects_version_needs_separate_permission) + ts.Run(S3IAMAccessControl_governance_bypass_delete_objects) + ts.Run(DeleteObjects_iam_mixed_denials_and_success) + ts.Run(DeleteObjects_iam_all_access_denied) + ts.Run(DeleteObjects_iam_all_locked) + ts.Run(S3IAMAccessControl_retention_extension_needs_no_bypass) + ts.Run(S3IAMAccessControl_governance_bypass_put_object_retention) + ts.Run(S3IAMAccessControl_retention_shortening_needs_bypass) + ts.Run(S3IAMAccessControl_condition_source_ip) + ts.Run(S3IAMAccessControl_condition_negated_operator_needs_context) + ts.Run(S3IAMAccessControl_condition_request_keys) + ts.Run(S3IAMAccessControl_condition_identity_keys) + ts.Run(S3IAMAccessControl_condition_principal_tag) + ts.Run(S3IAMAccessControl_condition_on_deny_statement) + ts.Run(S3IAMAccessControl_condition_multiple_keys_anded) + ts.Run(S3IAMAccessControl_inactive_and_deleted_credentials) + ts.Run(S3IAMAccessControl_bucket_policy_unknown_principal_rejected) +} + +func TestS3IAMSessionAccessControl(ts *TestState) { + ts.Run(S3IAMSession_role_policy_allows) + ts.Run(S3IAMSession_role_without_policy_denied) + ts.Run(S3IAMSession_role_policy_explicit_deny_wins) + ts.Run(S3IAMSession_role_policy_resource_scoped) + ts.Run(S3IAMSession_session_policy_narrows_role) + ts.Run(S3IAMSession_session_policy_cannot_widen_role) + ts.Run(S3IAMSession_session_policy_explicit_deny_overrides_role) + ts.Run(S3IAMSession_role_policy_deny_overrides_session_allow) + ts.Run(S3IAMSession_session_policy_without_role_policy_denied) + ts.Run(S3IAMSession_bucket_policy_allows_without_role_policy) + ts.Run(S3IAMSession_session_policy_filters_bucket_policy_grant) + ts.Run(S3IAMSession_bucket_policy_deny_overrides_role_allow) + ts.Run(S3IAMSession_missing_and_wrong_security_token) + ts.Run(S3IAMSession_presigned_url_with_session_credentials) + ts.Run(S3IAMSession_deleted_role_denies) + ts.Run(S3IAMSession_create_bucket_via_role_policy) + ts.Run(S3IAMSession_governance_bypass_via_role_policy) + ts.Run(S3IAMSession_delete_objects_authorizes_each_key) + ts.Run(S3IAMSession_condition_identity_keys) + ts.Run(S3IAMSession_get_caller_identity_matches_s3_principal) +} + func TestIAM(ts *TestState) { TestIAMAuth(ts) TestIAMQueryAuth(ts) @@ -1615,6 +1685,18 @@ func TestAccessControl(ts *TestState) { if !ts.conf.azureTests { ts.Run(AccessControl_policy_normalizes_object_key_for_get_put_delete) } + ts.Run(AccessControl_bucket_policy_condition_ip_allow) + ts.Run(AccessControl_bucket_policy_condition_ip_deny_no_match) + ts.Run(AccessControl_bucket_policy_condition_not_ip_address_allow) + ts.Run(AccessControl_bucket_policy_condition_not_ip_address_deny) + ts.Run(AccessControl_bucket_policy_condition_explicit_deny_overrides_allow) + ts.Run(AccessControl_bucket_policy_condition_s3_prefix) + ts.Run(AccessControl_bucket_policy_condition_string_operators) + ts.Run(AccessControl_bucket_policy_condition_numeric_operators) + ts.Run(AccessControl_bucket_policy_condition_date_operators) + ts.Run(AccessControl_bucket_policy_condition_bool_operator) + ts.Run(AccessControl_bucket_policy_condition_binary_operator) + ts.Run(AccessControl_bucket_policy_condition_null_operator) } func TestPublicBuckets(ts *TestState) { @@ -1880,6 +1962,58 @@ type IntTests map[string]IntTest func GetIntTests() IntTests { return IntTests{ + "S3IAMAccessControl_retention_shortening_needs_bypass": S3IAMAccessControl_retention_shortening_needs_bypass, + "S3IAMSession_get_caller_identity_matches_s3_principal": S3IAMSession_get_caller_identity_matches_s3_principal, + "S3IAMSession_condition_identity_keys": S3IAMSession_condition_identity_keys, + "S3IAMSession_delete_objects_authorizes_each_key": S3IAMSession_delete_objects_authorizes_each_key, + "S3IAMSession_governance_bypass_via_role_policy": S3IAMSession_governance_bypass_via_role_policy, + "S3IAMSession_create_bucket_via_role_policy": S3IAMSession_create_bucket_via_role_policy, + "S3IAMSession_deleted_role_denies": S3IAMSession_deleted_role_denies, + "S3IAMSession_presigned_url_with_session_credentials": S3IAMSession_presigned_url_with_session_credentials, + "S3IAMSession_missing_and_wrong_security_token": S3IAMSession_missing_and_wrong_security_token, + "S3IAMSession_bucket_policy_deny_overrides_role_allow": S3IAMSession_bucket_policy_deny_overrides_role_allow, + "S3IAMSession_session_policy_filters_bucket_policy_grant": S3IAMSession_session_policy_filters_bucket_policy_grant, + "S3IAMSession_bucket_policy_allows_without_role_policy": S3IAMSession_bucket_policy_allows_without_role_policy, + "S3IAMSession_session_policy_without_role_policy_denied": S3IAMSession_session_policy_without_role_policy_denied, + "S3IAMSession_role_policy_deny_overrides_session_allow": S3IAMSession_role_policy_deny_overrides_session_allow, + "S3IAMSession_session_policy_explicit_deny_overrides_role": S3IAMSession_session_policy_explicit_deny_overrides_role, + "S3IAMSession_session_policy_cannot_widen_role": S3IAMSession_session_policy_cannot_widen_role, + "S3IAMSession_session_policy_narrows_role": S3IAMSession_session_policy_narrows_role, + "S3IAMSession_role_policy_resource_scoped": S3IAMSession_role_policy_resource_scoped, + "S3IAMSession_role_policy_explicit_deny_wins": S3IAMSession_role_policy_explicit_deny_wins, + "S3IAMSession_role_without_policy_denied": S3IAMSession_role_without_policy_denied, + "S3IAMSession_role_policy_allows": S3IAMSession_role_policy_allows, + "S3IAMAccessControl_retention_extension_needs_no_bypass": S3IAMAccessControl_retention_extension_needs_no_bypass, + "S3IAMAccessControl_delete_objects_authorizes_each_key": S3IAMAccessControl_delete_objects_authorizes_each_key, + "S3IAMAccessControl_delete_objects_version_needs_separate_permission": S3IAMAccessControl_delete_objects_version_needs_separate_permission, + "S3IAMAccessControl_no_policy_denies": S3IAMAccessControl_no_policy_denies, + "S3IAMAccessControl_root_bypasses_policies": S3IAMAccessControl_root_bypasses_policies, + "S3IAMAccessControl_identity_policy_allows_without_bucket_policy": S3IAMAccessControl_identity_policy_allows_without_bucket_policy, + "S3IAMAccessControl_identity_policy_action_wildcards": S3IAMAccessControl_identity_policy_action_wildcards, + "S3IAMAccessControl_identity_policy_resource_scoping": S3IAMAccessControl_identity_policy_resource_scoping, + "S3IAMAccessControl_identity_policy_bucket_vs_object_arn": S3IAMAccessControl_identity_policy_bucket_vs_object_arn, + "S3IAMAccessControl_identity_policy_not_action_and_not_resource": S3IAMAccessControl_identity_policy_not_action_and_not_resource, + "S3IAMAccessControl_identity_policy_explicit_deny_wins": S3IAMAccessControl_identity_policy_explicit_deny_wins, + "S3IAMAccessControl_multiple_inline_policies_combine": S3IAMAccessControl_multiple_inline_policies_combine, + "S3IAMAccessControl_bucket_policy_allows_without_identity_policy": S3IAMAccessControl_bucket_policy_allows_without_identity_policy, + "S3IAMAccessControl_bucket_policy_explicit_deny": S3IAMAccessControl_bucket_policy_explicit_deny, + "S3IAMAccessControl_policy_combinations": S3IAMAccessControl_policy_combinations, + "S3IAMAccessControl_copy_object_requires_both_sides": S3IAMAccessControl_copy_object_requires_both_sides, + "S3IAMAccessControl_create_bucket": S3IAMAccessControl_create_bucket, + "S3IAMAccessControl_governance_bypass_sources": S3IAMAccessControl_governance_bypass_sources, + "S3IAMAccessControl_governance_without_bypass_header": S3IAMAccessControl_governance_without_bypass_header, + "S3IAMAccessControl_compliance_mode_not_bypassable": S3IAMAccessControl_compliance_mode_not_bypassable, + "S3IAMAccessControl_governance_bypass_delete_objects": S3IAMAccessControl_governance_bypass_delete_objects, + "S3IAMAccessControl_governance_bypass_put_object_retention": S3IAMAccessControl_governance_bypass_put_object_retention, + "S3IAMAccessControl_condition_source_ip": S3IAMAccessControl_condition_source_ip, + "S3IAMAccessControl_condition_negated_operator_needs_context": S3IAMAccessControl_condition_negated_operator_needs_context, + "S3IAMAccessControl_condition_request_keys": S3IAMAccessControl_condition_request_keys, + "S3IAMAccessControl_condition_identity_keys": S3IAMAccessControl_condition_identity_keys, + "S3IAMAccessControl_condition_principal_tag": S3IAMAccessControl_condition_principal_tag, + "S3IAMAccessControl_condition_on_deny_statement": S3IAMAccessControl_condition_on_deny_statement, + "S3IAMAccessControl_condition_multiple_keys_anded": S3IAMAccessControl_condition_multiple_keys_anded, + "S3IAMAccessControl_inactive_and_deleted_credentials": S3IAMAccessControl_inactive_and_deleted_credentials, + "S3IAMAccessControl_bucket_policy_unknown_principal_rejected": S3IAMAccessControl_bucket_policy_unknown_principal_rejected, "Authentication_invalid_auth_header": Authentication_invalid_auth_header, "Authentication_unsupported_signature_version": Authentication_unsupported_signature_version, "Authentication_missing_components": Authentication_missing_components, @@ -2272,7 +2406,7 @@ func GetIntTests() IntTests { "IAMAccessControl_RoleTrustDenialIndependentOfPermissionPolicy": IAMAccessControl_RoleTrustDenialIndependentOfPermissionPolicy, "IAMAccessControl_CrossIdentity_UnrelatedRoleCannotBeAssumedViaWrongIssuer": IAMAccessControl_CrossIdentity_UnrelatedRoleCannotBeAssumedViaWrongIssuer, "IAMAccessControl_CrossIdentity_AssumeRoleWithWebIdentityHasNoCallerIdentityCheck": IAMAccessControl_CrossIdentity_AssumeRoleWithWebIdentityHasNoCallerIdentityCheck, - "PresignedAuth_security_token_not_supported": PresignedAuth_security_token_not_supported, + "PresignedAuth_security_token_with_permanent_credentials": PresignedAuth_security_token_with_permanent_credentials, "PresignedAuth_unsupported_algorithm": PresignedAuth_unsupported_algorithm, "PresignedAuth_ECDSA_not_supported": PresignedAuth_ECDSA_not_supported, "PresignedAuth_missing_signature_query_param": PresignedAuth_missing_signature_query_param, @@ -2502,6 +2636,9 @@ func GetIntTests() IntTests { "DeleteObjects_empty_input": DeleteObjects_empty_input, "DeleteObjects_non_existing_objects": DeleteObjects_non_existing_objects, "DeleteObjects_success": DeleteObjects_success, + "DeleteObjects_iam_mixed_denials_and_success": DeleteObjects_iam_mixed_denials_and_success, + "DeleteObjects_iam_all_access_denied": DeleteObjects_iam_all_access_denied, + "DeleteObjects_iam_all_locked": DeleteObjects_iam_all_locked, "CopyObject_non_existing_dst_bucket": CopyObject_non_existing_dst_bucket, "CopyObject_not_owned_source_bucket": CopyObject_not_owned_source_bucket, "CopyObject_copy_to_itself": CopyObject_copy_to_itself, @@ -2708,6 +2845,10 @@ func GetIntTests() IntTests { "PutBucketPolicy_version": PutBucketPolicy_version, "PutBucketPolicy_success": PutBucketPolicy_success, "PutBucketPolicy_status": PutBucketPolicy_status, + "PutBucketPolicy_condition_invalid_operator": PutBucketPolicy_condition_invalid_operator, + "PutBucketPolicy_condition_invalid_key": PutBucketPolicy_condition_invalid_key, + "PutBucketPolicy_condition_action_mismatch": PutBucketPolicy_condition_action_mismatch, + "PutBucketPolicy_condition_invalid_ip": PutBucketPolicy_condition_invalid_ip, "GetBucketPolicy_non_existing_bucket": GetBucketPolicy_non_existing_bucket, "GetBucketPolicy_not_set": GetBucketPolicy_not_set, "GetBucketPolicy_success": GetBucketPolicy_success, @@ -2802,6 +2943,10 @@ func GetIntTests() IntTests { "PutObjectRetention_overwrite_governance_with_governance": PutObjectRetention_overwrite_governance_with_governance, "PutObjectRetention_overwrite_governance_without_bypass_specified": PutObjectRetention_overwrite_governance_without_bypass_specified, "PutObjectRetention_overwrite_governance_with_permission": PutObjectRetention_overwrite_governance_with_permission, + "PutObjectRetention_shorten_governance_without_bypass": PutObjectRetention_shorten_governance_without_bypass, + "PutObjectRetention_shorten_governance_with_bypass": PutObjectRetention_shorten_governance_with_bypass, + "PutObjectRetention_shorten_compliance_denied": PutObjectRetention_shorten_compliance_denied, + "PutObjectRetention_rewrite_same_date": PutObjectRetention_rewrite_same_date, "PutObjectRetention_success": PutObjectRetention_success, "GetObjectRetention_non_existing_bucket": GetObjectRetention_non_existing_bucket, "GetObjectRetention_non_existing_object": GetObjectRetention_non_existing_object, @@ -2861,6 +3006,7 @@ func GetIntTests() IntTests { "WORMProtection_bucket_object_lock_configuration_governance_mode": WORMProtection_bucket_object_lock_configuration_governance_mode, "WORMProtection_bucket_object_lock_governance_bypass_delete": WORMProtection_bucket_object_lock_governance_bypass_delete, "WORMProtection_bucket_object_lock_governance_bypass_delete_multiple": WORMProtection_bucket_object_lock_governance_bypass_delete_multiple, + "WORMProtection_delete_objects_locked_object_partial_success": WORMProtection_delete_objects_locked_object_partial_success, "WORMProtection_object_lock_retention_compliance_locked": WORMProtection_object_lock_retention_compliance_locked, "WORMProtection_object_lock_retention_governance_locked": WORMProtection_object_lock_retention_governance_locked, "WORMProtection_object_lock_retention_governance_bypass_overwrite_put": WORMProtection_object_lock_retention_governance_bypass_overwrite_put, @@ -2910,6 +3056,18 @@ func GetIntTests() IntTests { "AccessControl_CopyObject_with_legal_hold_policy": AccessControl_CopyObject_with_legal_hold_policy, "AccessControl_CopyObject_with_retention_policy": AccessControl_CopyObject_with_retention_policy, "AccessControl_policy_normalizes_object_key_for_get_put_delete": AccessControl_policy_normalizes_object_key_for_get_put_delete, + "AccessControl_bucket_policy_condition_ip_allow": AccessControl_bucket_policy_condition_ip_allow, + "AccessControl_bucket_policy_condition_ip_deny_no_match": AccessControl_bucket_policy_condition_ip_deny_no_match, + "AccessControl_bucket_policy_condition_not_ip_address_allow": AccessControl_bucket_policy_condition_not_ip_address_allow, + "AccessControl_bucket_policy_condition_not_ip_address_deny": AccessControl_bucket_policy_condition_not_ip_address_deny, + "AccessControl_bucket_policy_condition_explicit_deny_overrides_allow": AccessControl_bucket_policy_condition_explicit_deny_overrides_allow, + "AccessControl_bucket_policy_condition_s3_prefix": AccessControl_bucket_policy_condition_s3_prefix, + "AccessControl_bucket_policy_condition_string_operators": AccessControl_bucket_policy_condition_string_operators, + "AccessControl_bucket_policy_condition_numeric_operators": AccessControl_bucket_policy_condition_numeric_operators, + "AccessControl_bucket_policy_condition_date_operators": AccessControl_bucket_policy_condition_date_operators, + "AccessControl_bucket_policy_condition_bool_operator": AccessControl_bucket_policy_condition_bool_operator, + "AccessControl_bucket_policy_condition_binary_operator": AccessControl_bucket_policy_condition_binary_operator, + "AccessControl_bucket_policy_condition_null_operator": AccessControl_bucket_policy_condition_null_operator, "PublicBucket_default_private_bucket": PublicBucket_default_private_bucket, "PublicBucket_public_bucket_policy": PublicBucket_public_bucket_policy, "PublicBucket_public_object_policy": PublicBucket_public_object_policy, diff --git a/tests/integration/iam_assume_role_with_web_identity_github_oidc.go b/tests/integration/iam_assume_role_with_web_identity_github_oidc.go index 61a5600e..e218702c 100644 --- a/tests/integration/iam_assume_role_with_web_identity_github_oidc.go +++ b/tests/integration/iam_assume_role_with_web_identity_github_oidc.go @@ -17,6 +17,7 @@ package integration import ( "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -27,6 +28,7 @@ import ( "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/aws/smithy-go" ) const ( @@ -142,14 +144,29 @@ func IAMAssumeRoleWithWebIdentity_github_oidc_live(s *S3Conf) error { // trust policy would grant every workflow run in the repo, on any branch, // the same trust, which is far too broad outside this throwaway context. func createGitHubOIDCTrust(client *iam.Client, repo string) (roleName, roleArn string, cleanup func(), err error) { + // The provider is keyed by URL alone — a second CreateOpenIDConnectProvider + // for the same githubOIDCIssuerURL fails with EntityAlreadyExists, same as + // real AWS. Some tests mint more than one session (and so call this more + // than once) within a single run, so a provider left by an earlier call + // that hasn't been cleaned up yet is expected, not a leak: reuse it rather + // than failing, and only this call's cleanup deletes it if this call is + // the one that actually created it. + ownsProvider := true out, err := createOIDCProvider(client, &iam.CreateOpenIDConnectProviderInput{ Url: aws.String(githubOIDCIssuerURL), ClientIDList: []string{githubOIDCTestAudience}, }) + var providerArn string if err != nil { - return "", "", nil, fmt.Errorf("create GitHub OIDC provider: %w", err) + var ae smithy.APIError + if !errors.As(err, &ae) || ae.ErrorCode() != "EntityAlreadyExists" { + return "", "", nil, fmt.Errorf("create GitHub OIDC provider: %w", err) + } + ownsProvider = false + providerArn = oidcProviderArn(githubOIDCIssuerURL) + } else { + providerArn = aws.ToString(out.OpenIDConnectProviderArn) } - providerArn := aws.ToString(out.OpenIDConnectProviderArn) host := trimProviderScheme(githubOIDCIssuerURL) roleName = "github-oidc-" + genRandString(12) @@ -157,14 +174,18 @@ func createGitHubOIDCTrust(client *iam.Client, repo string) (roleName, roleArn s `"Condition":{"StringEquals":{"%s:aud":%q},"StringLike":{"%s:sub":%q}}}]}`, providerArn, host, githubOIDCTestAudience, host, "repo:"+repo+":*") if _, err := createIAMRole(client, &iam.CreateRoleInput{RoleName: &roleName, AssumeRolePolicyDocument: &trust}); err != nil { - deleteOIDCProvider(client, providerArn) + if ownsProvider { + deleteOIDCProvider(client, providerArn) + } return "", "", nil, fmt.Errorf("create GitHub OIDC trust role: %w", err) } roleArn = "arn:aws:iam::000000000000:role/" + roleName cleanup = func() { deleteIAMRole(client, roleName) - deleteOIDCProvider(client, providerArn) + if ownsProvider { + deleteOIDCProvider(client, providerArn) + } } return roleName, roleArn, cleanup, nil } @@ -233,7 +254,7 @@ func fetchGitHubIDToken(requestURL, requestToken, audience string) (string, erro func getCallerIdentityWithSessionCreds(cfg S3Conf, access, secret, token string) (*sts.GetCallerIdentityOutput, error) { cfg.awsID = access cfg.awsSecret = secret - stsCfg := cfg.Config() + stsCfg := cfg.iamConfig() stsCfg.Credentials = credentials.NewStaticCredentialsProvider(access, secret, token) ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) diff --git a/tests/integration/iam_query_auth.go b/tests/integration/iam_query_auth.go index 0ab03991..c87a44ba 100644 --- a/tests/integration/iam_query_auth.go +++ b/tests/integration/iam_query_auth.go @@ -16,15 +16,12 @@ package integration import ( "bytes" - "context" "crypto/sha256" "encoding/hex" "fmt" "net/http" "strings" - "github.com/aws/aws-sdk-go-v2/aws" - vgwv4 "github.com/versity/versitygw/aws/signer/v4" "github.com/versity/versitygw/iamapi/iamerr" "github.com/versity/versitygw/internal/sigv4auth" ) @@ -273,30 +270,38 @@ func createIAMQuerySignedRequest(endpoint string, cfg *authConfig, access, secre payloadHash = hex.EncodeToString(hash[:]) } - signer := vgwv4.NewSigner() - signedURL, signedHeaders, _, err := signer.PresignHTTP( - context.Background(), - aws.Credentials{AccessKeyID: access, SecretAccessKey: secret}, - req, - payloadHash, - cfg.service, - region, - cfg.date, - nil, - ) - if err != nil { - return nil, fmt.Errorf("sign IAM query auth request: %w", err) - } + yyyymmdd := cfg.date.Format(sigv4auth.YYYYMMDD) + derivedKey := sigv4auth.DeriveKey(secret, yyyymmdd, region, cfg.service) + in := sigv4auth.SigningInputFromRequest(req) + in.AccessKeyID = access + in.CredentialScope = sigv4auth.BuildCredentialScope(yyyymmdd, region, cfg.service) + in.PayloadHash = payloadHash + in.SigningTime = cfg.date + in.IsPreSign = true + result := sigv4auth.BuildAndSign(derivedKey, in) - signedReq, err := http.NewRequest(cfg.method, signedURL, bytes.NewReader(cfg.body)) + signedURL := *req.URL + signedURL.RawQuery = result.RawQuery + + signedReq, err := http.NewRequest(cfg.method, signedURL.String(), bytes.NewReader(cfg.body)) if err != nil { return nil, fmt.Errorf("create signed IAM query auth request: %w", err) } for key, value := range cfg.headers { signedReq.Header.Set(key, value) } - for key, values := range signedHeaders { - signedReq.Header[key] = append([]string(nil), values...) + for key, values := range result.SignedHeaders { + if key == "host" { + // signedReq already carries the correct Host implicitly via its + // URL; result.SignedHeaders holds it under the raw lowercase + // canonical-header key "host" rather than Go's canonicalized + // "Host", so writing it into signedReq.Header would add a + // second, non-excluded Host header on the wire. + continue + } + for _, value := range values { + signedReq.Header.Add(key, value) + } } return signedReq, nil diff --git a/tests/integration/presigned_urls.go b/tests/integration/presigned_urls.go index 9a3dacc0..03791918 100644 --- a/tests/integration/presigned_urls.go +++ b/tests/integration/presigned_urls.go @@ -28,8 +28,8 @@ import ( "github.com/versity/versitygw/s3err" ) -func PresignedAuth_security_token_not_supported(s *S3Conf) error { - testName := "PresignedAuth_security_token_not_supported" +func PresignedAuth_security_token_with_permanent_credentials(s *S3Conf) error { + testName := "PresignedAuth_security_token_with_permanent_credentials" return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &bucket}) @@ -50,7 +50,7 @@ func PresignedAuth_security_token_not_supported(s *S3Conf) error { return err } - return checkHTTPResponseApiErr(resp, s3err.QueryAuthErrors.SecurityTokenNotSupported()) + return checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrInvalidToken)) }) } diff --git a/tests/integration/s3_iam_access_control.go b/tests/integration/s3_iam_access_control.go new file mode 100644 index 00000000..6a725038 --- /dev/null +++ b/tests/integration/s3_iam_access_control.go @@ -0,0 +1,1805 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package integration + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/iam" + iamtypes "github.com/aws/aws-sdk-go-v2/service/iam/types" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/versity/versitygw/s3err" +) + +// S3IAMAccessControl_no_policy_denies verifies a caller with no identity +// policy and no bucket policy is denied by default — there is no implicit +// grant anywhere for an ordinary IAM user. +func S3IAMAccessControl_no_policy_denies(s *S3Conf) error { + testName := "S3IAMAccessControl_no_policy_denies" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + user, cleanup, err := newS3IAMUser(root, s, nil) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + return checkApiErr(err, wantImplicitDeny(user.arn, actS3GetObject, objectArn(bucket, "obj"))) + }) +} + +// S3IAMAccessControl_root_bypasses_policies verifies the gateway's root +// credential is authorized regardless of any policy, including a bucket +// policy that explicitly denies everyone. +func S3IAMAccessControl_root_bypasses_policies(s *S3Conf) error { + testName := "S3IAMAccessControl_root_bypasses_policies" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: "Deny", Principal: "*", Action: "s3:*", Resource: []string{bucketArn(bucket), objectsArn(bucket)}, + }); err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + return err + }) +} + +// S3IAMAccessControl_identity_policy_allows_without_bucket_policy is the +// core same-account behavior: an identity-policy Allow grants the request on +// its own, with no bucket policy and no ACL grant involved at all. +func S3IAMAccessControl_identity_policy_allows_without_bucket_policy(s *S3Conf) error { + testName := "S3IAMAccessControl_identity_policy_allows_without_bucket_policy" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + user, cleanup, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{Effect: "Allow", Action: actS3PutObject, Resource: objectsArn(bucket)}), + }) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return fmt.Errorf("expected PutObject to be allowed: %w", err) + } + + // The same policy grants nothing beyond the action it names. + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + return checkApiErr(err, wantImplicitDeny(user.arn, actS3GetObject, objectArn(bucket, "obj"))) + }) +} + +// S3IAMAccessControl_identity_policy_action_wildcards verifies "s3:*" and +// prefix wildcards ("s3:Get*") match the way an exact action name does. +func S3IAMAccessControl_identity_policy_action_wildcards(s *S3Conf) error { + testName := "S3IAMAccessControl_identity_policy_action_wildcards" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + + cases := []struct { + name string + action any + wantGetOK bool + wantPutOK bool + }{ + {name: "full wildcard", action: "s3:*", wantGetOK: true, wantPutOK: true}, + {name: "prefix wildcard", action: "s3:Get*", wantGetOK: true, wantPutOK: false}, + {name: "bare wildcard", action: "*", wantGetOK: true, wantPutOK: true}, + } + for _, tc := range cases { + if err := func() error { + user, cleanup, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: tc.action, + Resource: []string{bucketArn(bucket), objectsArn(bucket)}, + }), + }) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if tc.wantGetOK { + if err != nil { + return fmt.Errorf("expected GetObject to be allowed: %w", err) + } + } else if err := checkApiErr(err, wantImplicitDeny(user.arn, actS3GetObject, objectArn(bucket, "obj"))); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj2")}) + cancel() + if tc.wantPutOK { + if err != nil { + return fmt.Errorf("expected PutObject to be allowed: %w", err) + } + return nil + } + return checkApiErr(err, wantImplicitDeny(user.arn, actS3PutObject, objectArn(bucket, "obj2"))) + }(); err != nil { + return fmt.Errorf("%s: %w", tc.name, err) + } + } + return nil + }) +} + +// S3IAMAccessControl_identity_policy_resource_scoping verifies a Resource +// pattern scopes a grant to matching keys only. +func S3IAMAccessControl_identity_policy_resource_scoping(s *S3Conf) error { + testName := "S3IAMAccessControl_identity_policy_resource_scoping" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + for _, key := range []string{"allowed/obj", "denied/obj"} { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr(key)}) + cancel() + if err != nil { + return err + } + } + + user, cleanup, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: actS3GetObject, + Resource: objectArn(bucket, "allowed/*"), + }), + }) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("allowed/obj")}) + cancel() + if err != nil { + return fmt.Errorf("expected GetObject on the matching key to be allowed: %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("denied/obj")}) + cancel() + return checkApiErr(err, wantImplicitDeny(user.arn, actS3GetObject, objectArn(bucket, "denied/obj"))) + }) +} + +// S3IAMAccessControl_identity_policy_bucket_vs_object_arn verifies a +// bucket-level action evaluates against the bucket ARN, so an object-ARN +// grant ("bucket/*") does not cover it and vice versa. +func S3IAMAccessControl_identity_policy_bucket_vs_object_arn(s *S3Conf) error { + testName := "S3IAMAccessControl_identity_policy_bucket_vs_object_arn" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + objectOnly, cleanupObj, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{Effect: "Allow", Action: actS3ListBucket, Resource: objectsArn(bucket)}), + }) + if err != nil { + return err + } + defer cleanupObj() + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = objectOnly.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) + cancel() + if err := checkApiErr(err, wantImplicitDeny(objectOnly.arn, actS3ListBucket, bucketArn(bucket))); err != nil { + return fmt.Errorf("an object-ARN grant must not cover a bucket-level action: %w", err) + } + + bucketScoped, cleanupBucket, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{Effect: "Allow", Action: actS3ListBucket, Resource: bucketArn(bucket)}), + }) + if err != nil { + return err + } + defer cleanupBucket() + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = bucketScoped.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) + cancel() + if err != nil { + return fmt.Errorf("expected ListObjects to be allowed by a bucket-ARN grant: %w", err) + } + return nil + }) +} + +// S3IAMAccessControl_identity_policy_not_action_and_not_resource verifies +// NotAction and NotResource grant everything *except* what they name. +func S3IAMAccessControl_identity_policy_not_action_and_not_resource(s *S3Conf) error { + testName := "S3IAMAccessControl_identity_policy_not_action_and_not_resource" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + + notAction, cleanupAction, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", NotAction: actS3GetObject, + Resource: []string{bucketArn(bucket), objectsArn(bucket)}, + }), + }) + if err != nil { + return err + } + defer cleanupAction() + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = notAction.client.PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("other")}) + cancel() + if err != nil { + return fmt.Errorf("NotAction must grant an action it does not name: %w", err) + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = notAction.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err := checkApiErr(err, wantImplicitDeny(notAction.arn, actS3GetObject, objectArn(bucket, "obj"))); err != nil { + return fmt.Errorf("NotAction must not grant the action it names: %w", err) + } + + notResource, cleanupResource, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: actS3GetObject, + NotResource: objectArn(bucket, "obj"), + }), + }) + if err != nil { + return err + } + defer cleanupResource() + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = notResource.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("other")}) + cancel() + if err != nil { + return fmt.Errorf("NotResource must grant a resource it does not name: %w", err) + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = notResource.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + return checkApiErr(err, wantImplicitDeny(notResource.arn, actS3GetObject, objectArn(bucket, "obj"))) + }) +} + +// S3IAMAccessControl_identity_policy_explicit_deny_wins verifies an explicit +// Deny beats a matching Allow regardless of statement order or of whether +// the two live in the same inline policy document. +func S3IAMAccessControl_identity_policy_explicit_deny_wins(s *S3Conf) error { + testName := "S3IAMAccessControl_identity_policy_explicit_deny_wins" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + + allow := accessStatement{Effect: "Allow", Action: actS3GetObject, Resource: objectsArn(bucket)} + deny := accessStatement{Effect: "Deny", Action: actS3GetObject, Resource: objectsArn(bucket)} + + cases := []struct { + name string + policies map[string]string + }{ + {"deny after allow, same document", map[string]string{"p": policyDoc(allow, deny)}}, + {"deny before allow, same document", map[string]string{"p": policyDoc(deny, allow)}}, + {"allow and deny in separate documents", map[string]string{"a": policyDoc(allow), "d": policyDoc(deny)}}, + } + for _, tc := range cases { + if err := func() error { + user, cleanup, err := newS3IAMUser(root, s, tc.policies) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + return checkApiErr(err, wantExplicitIdentityDeny(user.arn, actS3GetObject, objectArn(bucket, "obj"))) + }(); err != nil { + return fmt.Errorf("%s: %w", tc.name, err) + } + } + return nil + }) +} + +// S3IAMAccessControl_multiple_inline_policies_combine verifies separate +// inline policy documents are unioned, so an action allowed by either one is +// allowed overall. +func S3IAMAccessControl_multiple_inline_policies_combine(s *S3Conf) error { + testName := "S3IAMAccessControl_multiple_inline_policies_combine" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + user, cleanup, err := newS3IAMUser(root, s, map[string]string{ + "reader": policyDoc(accessStatement{Effect: "Allow", Action: actS3GetObject, Resource: objectsArn(bucket)}), + "writer": policyDoc(accessStatement{Effect: "Allow", Action: actS3PutObject, Resource: objectsArn(bucket)}), + }) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return fmt.Errorf("expected PutObject to be allowed by the second document: %w", err) + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return fmt.Errorf("expected GetObject to be allowed by the first document: %w", err) + } + return nil + }) +} + +// S3IAMAccessControl_bucket_policy_allows_without_identity_policy verifies +// the resource side is independently sufficient too: a bucket policy naming +// the user's access key grants the request with no identity policy at all. +func S3IAMAccessControl_bucket_policy_allows_without_identity_policy(s *S3Conf) error { + testName := "S3IAMAccessControl_bucket_policy_allows_without_identity_policy" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + user, cleanup, err := newS3IAMUser(root, s, nil) + if err != nil { + return err + } + defer cleanup() + + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: "Allow", Principal: user.conf.awsID, Action: actS3PutObject, Resource: objectsArn(bucket), + }); err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return fmt.Errorf("expected PutObject to be allowed by the bucket policy: %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + return checkApiErr(err, wantImplicitDeny(user.arn, actS3GetObject, objectArn(bucket, "obj"))) + }) +} + +// S3IAMAccessControl_bucket_policy_explicit_deny verifies a bucket-policy +// Deny denies on its own, and reports the resource-based-policy message. +func S3IAMAccessControl_bucket_policy_explicit_deny(s *S3Conf) error { + testName := "S3IAMAccessControl_bucket_policy_explicit_deny" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + user, cleanup, err := newS3IAMUser(root, s, nil) + if err != nil { + return err + } + defer cleanup() + + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: "Deny", Principal: user.conf.awsID, Action: actS3GetObject, Resource: objectsArn(bucket), + }); err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + // The resource-based denial names the access key, not the ARN: + // bucket-policy principals are access-key-based for every backend, + // so the gateway has no ARN in hand at that point. + return checkApiErr(err, wantExplicitResourceDeny(user.conf.awsID, actS3GetObject, objectArn(bucket, "obj"))) + }) +} + +// S3IAMAccessControl_policy_combinations walks the full precedence matrix +// between the identity policy and the bucket policy for one action, checking +// the exact outcome and message for each of the nine combinations that +// matter. +func S3IAMAccessControl_policy_combinations(s *S3Conf) error { + testName := "S3IAMAccessControl_policy_combinations" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + + const ( + silent = "silent" + allow = "allow" + deny = "deny" + ) + cases := []struct { + identity string + resource string + // wantErr builds the expected error, or is nil when the request + // must succeed. + wantErr func(user *s3IAMPrincipal) s3err.S3Error + }{ + {identity: silent, resource: silent, wantErr: func(u *s3IAMPrincipal) s3err.S3Error { + return wantImplicitDeny(u.arn, actS3GetObject, objectArn(bucket, "obj")) + }}, + {identity: allow, resource: silent}, + {identity: silent, resource: allow}, + {identity: allow, resource: allow}, + {identity: deny, resource: silent, wantErr: func(u *s3IAMPrincipal) s3err.S3Error { + return wantExplicitIdentityDeny(u.arn, actS3GetObject, objectArn(bucket, "obj")) + }}, + {identity: deny, resource: allow, wantErr: func(u *s3IAMPrincipal) s3err.S3Error { + return wantExplicitIdentityDeny(u.arn, actS3GetObject, objectArn(bucket, "obj")) + }}, + {identity: silent, resource: deny, wantErr: func(u *s3IAMPrincipal) s3err.S3Error { + return wantExplicitResourceDeny(u.conf.awsID, actS3GetObject, objectArn(bucket, "obj")) + }}, + {identity: allow, resource: deny, wantErr: func(u *s3IAMPrincipal) s3err.S3Error { + return wantExplicitResourceDeny(u.conf.awsID, actS3GetObject, objectArn(bucket, "obj")) + }}, + // A Deny on both sides is reported as the resource-based one: + // VerifyAccess evaluates the bucket policy first and returns + // immediately, which also saves an IAM round trip. + {identity: deny, resource: deny, wantErr: func(u *s3IAMPrincipal) s3err.S3Error { + return wantExplicitResourceDeny(u.conf.awsID, actS3GetObject, objectArn(bucket, "obj")) + }}, + } + + for _, tc := range cases { + if err := func() error { + policies := map[string]string{} + if tc.identity != silent { + effect := "Allow" + if tc.identity == deny { + effect = "Deny" + } + policies["p"] = policyDoc(accessStatement{ + Effect: effect, Action: actS3GetObject, Resource: objectsArn(bucket), + }) + } + + user, cleanup, err := newS3IAMUser(root, s, policies) + if err != nil { + return err + } + defer cleanup() + + if tc.resource == silent { + if err := deleteBucketPolicyIfAny(s, bucket); err != nil { + return err + } + } else { + effect := "Allow" + if tc.resource == deny { + effect = "Deny" + } + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: effect, Principal: user.conf.awsID, Action: actS3GetObject, Resource: objectsArn(bucket), + }); err != nil { + return err + } + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if tc.wantErr == nil { + if err != nil { + return fmt.Errorf("expected the request to be allowed: %w", err) + } + return nil + } + return checkApiErr(err, tc.wantErr(user)) + }(); err != nil { + return fmt.Errorf("identity=%s resource=%s: %w", tc.identity, tc.resource, err) + } + } + return nil + }) +} + +// S3IAMAccessControl_copy_object_requires_both_sides verifies a CopyObject +// is authorized against both its source (GetObject) and its destination +// (PutObject), so a policy granting only one of the two is not enough. +func S3IAMAccessControl_copy_object_requires_both_sides(s *S3Conf) error { + testName := "S3IAMAccessControl_copy_object_requires_both_sides" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("src")}) + cancel() + if err != nil { + return err + } + + cases := []struct { + name string + action any + wantAction string + wantArn string + }{ + {name: "destination only", action: actS3PutObject, wantAction: actS3GetObject, wantArn: objectArn(bucket, "src")}, + {name: "source only", action: actS3GetObject, wantAction: actS3PutObject, wantArn: objectArn(bucket, "dst")}, + } + for _, tc := range cases { + if err := func() error { + user, cleanup, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{Effect: "Allow", Action: tc.action, Resource: objectsArn(bucket)}), + }) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: aws.String("dst"), + CopySource: aws.String(bucket + "/src"), + }) + cancel() + return checkApiErr(err, wantImplicitDeny(user.arn, tc.wantAction, tc.wantArn)) + }(); err != nil { + return fmt.Errorf("%s: %w", tc.name, err) + } + } + + // Granting both sides completes the copy. + user, cleanup, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: []string{actS3GetObject, actS3PutObject}, Resource: objectsArn(bucket), + }), + }) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + defer cancel() + if _, err := user.client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: aws.String("dst"), + CopySource: aws.String(bucket + "/src"), + }); err != nil { + return fmt.Errorf("expected CopyObject to be allowed once both sides are granted: %w", err) + } + return nil + }) +} + +// S3IAMAccessControl_create_bucket verifies s3:CreateBucket is gated by the +// identity policy alone — the bucket doesn't exist yet, so there is no +// bucket policy or ACL to consult — and that the grant is resource-scoped to +// the bucket name. +func S3IAMAccessControl_create_bucket(s *S3Conf) error { + testName := "S3IAMAccessControl_create_bucket" + return actionHandlerNoSetup(s, testName, func(_ *s3.Client, _ string) error { + root := s.GetIAMClient() + allowedName, otherName := getBucketName(), getBucketName() + + cases := []struct { + name string + policy func() string + bucket string + wantErr func(user *s3IAMPrincipal, bucket string) s3err.S3Error + }{ + { + name: "no policy denies", + policy: func() string { return "" }, + bucket: otherName, + wantErr: func(u *s3IAMPrincipal, b string) s3err.S3Error { + return wantImplicitDeny(u.arn, actS3CreateBucket, bucketArn(b)) + }, + }, + { + name: "explicit deny", + policy: func() string { + return policyDoc(accessStatement{Effect: "Deny", Action: actS3CreateBucket, Resource: "*"}) + }, + bucket: otherName, + wantErr: func(u *s3IAMPrincipal, b string) s3err.S3Error { + return wantExplicitIdentityDeny(u.arn, actS3CreateBucket, bucketArn(b)) + }, + }, + { + name: "scoped grant allows the named bucket", + policy: func() string { + return policyDoc(accessStatement{Effect: "Allow", Action: actS3CreateBucket, Resource: bucketArn(allowedName)}) + }, + bucket: allowedName, + }, + { + name: "scoped grant denies another bucket", + policy: func() string { + return policyDoc(accessStatement{Effect: "Allow", Action: actS3CreateBucket, Resource: bucketArn(allowedName)}) + }, + bucket: otherName, + wantErr: func(u *s3IAMPrincipal, b string) s3err.S3Error { + return wantImplicitDeny(u.arn, actS3CreateBucket, bucketArn(b)) + }, + }, + { + name: "wildcard grant allows any bucket", + policy: func() string { + return policyDoc(accessStatement{Effect: "Allow", Action: actS3CreateBucket, Resource: "arn:aws:s3:::*"}) + }, + bucket: otherName, + }, + } + + for _, tc := range cases { + if err := func() error { + policies := map[string]string{} + if doc := tc.policy(); doc != "" { + policies["p"] = doc + } + user, cleanup, err := newS3IAMUser(root, s, policies) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: &tc.bucket}) + cancel() + + if tc.wantErr == nil { + if err != nil { + return fmt.Errorf("expected CreateBucket to be allowed: %w", err) + } + return teardown(s, tc.bucket) + } + return checkApiErr(err, tc.wantErr(user, tc.bucket)) + }(); err != nil { + return fmt.Errorf("%s: %w", tc.name, err) + } + } + return nil + }) +} + +// S3IAMAccessControl_governance_bypass_sources verifies s3:BypassGovernance +// Retention follows the same precedence as any other action: an Allow from +// either the identity policy or the bucket policy is enough on its own, and +// an explicit Deny from either wins over the other's Allow. +func S3IAMAccessControl_governance_bypass_sources(s *S3Conf) error { + testName := "S3IAMAccessControl_governance_bypass_sources" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + const ( + silent = "silent" + allow = "allow" + deny = "deny" + ) + cases := []struct { + identity string + resource string + wantDenied bool + }{ + {identity: allow, resource: silent}, + {identity: silent, resource: allow}, + {identity: allow, resource: allow}, + {identity: silent, resource: silent, wantDenied: true}, + {identity: deny, resource: allow, wantDenied: true}, + {identity: allow, resource: deny, wantDenied: true}, + } + + for i, tc := range cases { + if err := func() error { + key := fmt.Sprintf("locked-%d", i) + if err := putGovernanceLockedObject(s, bucket, key); err != nil { + return err + } + + // Deleting the object always needs s3:DeleteObject as well; + // only the bypass permission is what varies per case. + statements := []accessStatement{ + {Effect: "Allow", Action: actS3DeleteObject, Resource: objectsArn(bucket)}, + } + if tc.identity != silent { + effect := "Allow" + if tc.identity == deny { + effect = "Deny" + } + statements = append(statements, accessStatement{ + Effect: effect, Action: actS3BypassGovernance, Resource: objectsArn(bucket), + }) + } + + user, cleanup, err := newS3IAMUser(root, s, map[string]string{"p": policyDoc(statements...)}) + if err != nil { + return err + } + defer cleanup() + + if tc.resource == silent { + if err := deleteBucketPolicyIfAny(s, bucket); err != nil { + return err + } + } else { + effect := "Allow" + if tc.resource == deny { + effect = "Deny" + } + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: effect, Principal: user.conf.awsID, + Action: actS3BypassGovernance, Resource: objectsArn(bucket), + }); err != nil { + return err + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &key, + BypassGovernanceRetention: aws.Bool(true), + }) + cancel() + + if !tc.wantDenied { + if err != nil { + return fmt.Errorf("expected the governance-bypassing delete to be allowed: %w", err) + } + return nil + } + if err == nil { + return fmt.Errorf("expected the governance-bypassing delete to be denied") + } + // Whichever way bypass was denied, the error names the + // bypass action specifically — not the generic + // "object protected by object lock" message, which the + // gateway reserves for a request with no bypass header. + if err := checkSdkApiErr(err, "AccessDenied"); err != nil { + return err + } + if !strings.Contains(err.Error(), actS3BypassGovernance) { + return fmt.Errorf("expected the denial to name %s, got: %v", actS3BypassGovernance, err) + } + return nil + }(); err != nil { + return fmt.Errorf("identity=%s resource=%s: %w", tc.identity, tc.resource, err) + } + } + + // Release the keys still under retention: the cases that expected a + // denial left theirs locked, and teardown cannot remove those. + var locked []objToDelete + for i, tc := range cases { + if tc.wantDenied { + locked = append(locked, objToDelete{key: fmt.Sprintf("locked-%d", i)}) + } + } + return cleanupLockedObjects(s.GetClient(), bucket, locked) + }, withLock()) +} + +// S3IAMAccessControl_governance_without_bypass_header verifies the bypass +// permission is irrelevant when the request doesn't ask to bypass: the +// object stays protected, and the error is the generic object-lock one. +func S3IAMAccessControl_governance_without_bypass_header(s *S3Conf) error { + testName := "S3IAMAccessControl_governance_without_bypass_header" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + const key = "locked" + if err := putGovernanceLockedObject(s, bucket, key); err != nil { + return err + } + + user, cleanup, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: []string{actS3DeleteObject, actS3BypassGovernance}, + Resource: objectsArn(bucket), + }), + }) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &bucket, Key: aws.String(key)}) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return err + } + + return cleanupLockedObjects(s.GetClient(), bucket, []objToDelete{{key: key}}) + }, withLock()) +} + +// S3IAMAccessControl_compliance_mode_not_bypassable verifies COMPLIANCE +// retention is absolute: no identity or bucket policy can grant a bypass of +// it, unlike GOVERNANCE. +func S3IAMAccessControl_compliance_mode_not_bypassable(s *S3Conf) error { + testName := "S3IAMAccessControl_compliance_mode_not_bypassable" + return s3IAMComplianceActionHandler(s, testName, func(root *iam.Client, bucket string) error { + const key = "compliance-locked" + retainUntil := time.Now().UTC().Add(time.Hour) + if _, err := putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: aws.String(key), + ObjectLockMode: types.ObjectLockModeCompliance, + ObjectLockRetainUntilDate: &retainUntil, + }, s.GetClient()); err != nil { + return err + } + + user, cleanup, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: []string{actS3DeleteObject, actS3BypassGovernance}, + Resource: objectsArn(bucket), + }), + }) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: aws.String(key), + BypassGovernanceRetention: aws.Bool(true), + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)) + }) +} + +// S3IAMAccessControl_delete_objects_authorizes_each_key verifies the batch +// DeleteObjects path authorizes s3:DeleteObject against each object's own +// ARN, the way real AWS does — a policy naming only "bucket/*" is +// sufficient — and that it supports partial success: verified live against +// real AWS (niksis02, account 792168558830), a key outside the granted +// prefix denies only that key, reported in the response's Errors list, while +// every other key in the same batch is still deleted and reported in +// Deleted. Both lists preserve the order the keys were requested in. +func S3IAMAccessControl_delete_objects_authorizes_each_key(s *S3Conf) error { + testName := "S3IAMAccessControl_delete_objects_authorizes_each_key" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + for _, key := range []string{"allowed/one", "allowed/two", "denied/three"} { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr(key)}) + cancel() + if err != nil { + return err + } + } + + user, cleanup, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: actS3DeleteObject, + Resource: objectArn(bucket, "allowed/*"), + }), + }) + if err != nil { + return err + } + defer cleanup() + + // A key outside the grant, mixed in with two that aren't, denies + // only that key — the request as a whole succeeds. + out, err := deleteObjectsWithBypass(user.client, bucket, "allowed/one", "denied/three", "allowed/two") + if err != nil { + return fmt.Errorf("expected DeleteObjects to succeed with a per-object denial, not fail outright: %w", err) + } + wantErr := wantImplicitDeny(user.arn, actS3DeleteObject, objectArn(bucket, "denied/three")) + if len(out.Errors) != 1 { + return fmt.Errorf("expected exactly 1 per-object error, got %+v", out.Errors) + } + if err := checkDeleteObjectsErr(out.Errors[0], "denied/three", wantErr); err != nil { + return err + } + wantDeleted := []string{"allowed/one", "allowed/two"} + if err := checkDeletedKeysInOrder(out.Deleted, wantDeleted); err != nil { + return err + } + + // Every key inside the grant succeeds, with no bucket-ARN grant + // anywhere, and no per-object errors. + out, err = deleteObjectsWithBypass(user.client, bucket, "allowed/one") + if err != nil { + return fmt.Errorf("expected DeleteObjects to be allowed by an object-ARN-only grant: %w", err) + } + if len(out.Errors) != 0 { + return fmt.Errorf("expected no per-object errors, got %+v", out.Errors) + } + return nil + }) +} + +// S3IAMAccessControl_delete_objects_version_needs_separate_permission +// verifies that naming a VersionId in a DeleteObjects entry is authorized +// against s3:DeleteObjectVersion, a distinct permission from the +// s3:DeleteObject a keyed (unversioned) delete needs — verified live against +// real AWS (niksis02, account 792168558830): a policy granting only +// s3:DeleteObject denies the versioned deletes in a batch while its keyed +// deletes in the same batch still succeed, each independently, matching the +// single-object DELETE path's existing behavior for the same distinction. +// +// The denial happens at authorization, before the backend ever resolves the +// named version, so this doesn't need a real object version (and the +// gateway this test group runs against has no --versioning-dir configured +// to produce one): an arbitrary VersionId is enough to exercise the +// s3:DeleteObjectVersion check and prove the batch still partially +// succeeds. +func S3IAMAccessControl_delete_objects_version_needs_separate_permission(s *S3Conf) error { + testName := "S3IAMAccessControl_delete_objects_version_needs_separate_permission" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + + user, cleanup, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: actS3DeleteObject, + Resource: objectsArn(bucket), + }), + }) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + defer cancel() + out, err := user.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{ + Objects: []types.ObjectIdentifier{ + {Key: aws.String("obj")}, + {Key: aws.String("versioned-obj"), VersionId: aws.String("some-version-id")}, + }, + }, + }) + if err != nil { + return fmt.Errorf("expected DeleteObjects to succeed with a per-object denial, not fail outright: %w", err) + } + + wantErr := wantImplicitDeny(user.arn, actS3DeleteObjectVersion, objectArn(bucket, "versioned-obj")) + if len(out.Errors) != 1 { + return fmt.Errorf("expected exactly 1 per-object error, got %+v", out.Errors) + } + if err := checkDeleteObjectsErr(out.Errors[0], "versioned-obj", wantErr); err != nil { + return err + } + if len(out.Deleted) != 1 || out.Deleted[0].Key == nil || *out.Deleted[0].Key != "obj" { + return fmt.Errorf("expected the keyed delete to succeed, got %+v", out.Deleted) + } + return nil + }) +} + +// S3IAMAccessControl_governance_bypass_delete_objects verifies the batch +// DeleteObjects path enforces the bypass permission per object, the same way +// the single-object delete does. +func S3IAMAccessControl_governance_bypass_delete_objects(s *S3Conf) error { + testName := "S3IAMAccessControl_governance_bypass_delete_objects" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + const key = "locked" + if err := putGovernanceLockedObject(s, bucket, key); err != nil { + return err + } + + withoutBypass, cleanupWithout, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{Effect: "Allow", Action: actS3DeleteObject, Resource: objectsArn(bucket)}), + }) + if err != nil { + return err + } + defer cleanupWithout() + + out, err := deleteObjectsWithBypass(withoutBypass.client, bucket, key) + if err != nil { + return fmt.Errorf("expected DeleteObjects to succeed with a per-object denial, not fail outright: %w", err) + } + if err := checkDeleteObjectsErr(out.Errors[0], key, + wantImplicitDeny(withoutBypass.arn, actS3BypassGovernance, objectArn(bucket, key))); err != nil { + return fmt.Errorf("expected DeleteObjects to be denied without the bypass permission: %w", err) + } + + withBypass, cleanupWith, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: []string{actS3DeleteObject, actS3BypassGovernance}, + Resource: objectsArn(bucket), + }), + }) + if err != nil { + return err + } + defer cleanupWith() + + out, err = deleteObjectsWithBypass(withBypass.client, bucket, key) + if err != nil { + return fmt.Errorf("expected DeleteObjects to be allowed with the bypass permission: %w", err) + } + if len(out.Errors) != 0 { + return fmt.Errorf("expected no per-object errors, got %+v", out.Errors) + } + return nil + }, withLock()) +} + +// S3IAMAccessControl_retention_extension_needs_no_bypass verifies the +// direction of the change is what decides whether a bypass is needed: +// pushing a retention date further out only strengthens the lock, so it +// needs nothing beyond s3:PutObjectRetention — in either mode. Shortening +// is the case that needs a bypass, covered by the test below. +func S3IAMAccessControl_retention_extension_needs_no_bypass(s *S3Conf) error { + testName := "S3IAMAccessControl_retention_extension_needs_no_bypass" + return s3IAMComplianceActionHandler(s, testName, func(root *iam.Client, bucket string) error { + user, cleanup, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: "s3:PutObjectRetention", Resource: objectsArn(bucket), + }), + }) + if err != nil { + return err + } + defer cleanup() + + modes := []types.ObjectLockRetentionMode{ + types.ObjectLockRetentionModeGovernance, + types.ObjectLockRetentionModeCompliance, + } + for _, mode := range modes { + key := "extend-" + strings.ToLower(string(mode)) + retainUntil := time.Now().UTC().Add(time.Minute) + if _, err := putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + ObjectLockMode: types.ObjectLockMode(mode), + ObjectLockRetainUntilDate: &retainUntil, + }, s.GetClient()); err != nil { + return err + } + + extended := retainUntil.Add(time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := user.client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &key, + Retention: &types.ObjectLockRetention{Mode: mode, RetainUntilDate: &extended}, + }) + cancel() + if err != nil { + return fmt.Errorf("%s: expected extending a retention to need no bypass: %w", mode, err) + } + } + return nil + }) +} + +// S3IAMAccessControl_governance_bypass_put_object_retention verifies the +// bypass permission gates weakening a GOVERNANCE retention through +// PutObjectRetention — here by switching its mode to COMPLIANCE, which the +// gateway only permits with the bypass header. +func S3IAMAccessControl_governance_bypass_put_object_retention(s *S3Conf) error { + testName := "S3IAMAccessControl_governance_bypass_put_object_retention" + return s3IAMComplianceActionHandler(s, testName, func(root *iam.Client, bucket string) error { + retainUntil := time.Now().UTC().Add(time.Hour) + toCompliance := func(client *s3.Client, key string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + defer cancel() + _, err := client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: aws.String(key), + BypassGovernanceRetention: aws.Bool(true), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + RetainUntilDate: &retainUntil, + }, + }) + return err + } + + withoutBypass, cleanupWithout, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{Effect: "Allow", Action: "s3:PutObjectRetention", Resource: objectsArn(bucket)}), + }) + if err != nil { + return err + } + defer cleanupWithout() + + if err := putGovernanceLockedObject(s, bucket, "no-bypass"); err != nil { + return err + } + if err := toCompliance(withoutBypass.client, "no-bypass"); err == nil { + return fmt.Errorf("expected the retention mode change to be denied without the bypass permission") + } + + withBypass, cleanupWith, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: []string{"s3:PutObjectRetention", actS3BypassGovernance}, + Resource: objectsArn(bucket), + }), + }) + if err != nil { + return err + } + defer cleanupWith() + + if err := putGovernanceLockedObject(s, bucket, "with-bypass"); err != nil { + return err + } + if err := toCompliance(withBypass.client, "with-bypass"); err != nil { + return fmt.Errorf("expected the retention mode change to be allowed with the bypass permission: %w", err) + } + return nil + }) +} + +// S3IAMAccessControl_retention_shortening_needs_bypass verifies that moving +// a retention date earlier — weakening the lock without changing its mode — +// needs both the bypass header and s3:BypassGovernanceRetention for +// GOVERNANCE, and is refused outright for COMPLIANCE however the caller +// asks. +// +// Extending is the control: it only ever strengthens the lock, so it needs +// neither, in either mode. +func S3IAMAccessControl_retention_shortening_needs_bypass(s *S3Conf) error { + testName := "S3IAMAccessControl_retention_shortening_needs_bypass" + return s3IAMComplianceActionHandler(s, testName, func(root *iam.Client, bucket string) error { + withBypass, cleanupWith, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: []string{"s3:PutObjectRetention", actS3BypassGovernance}, + Resource: objectsArn(bucket), + }), + }) + if err != nil { + return err + } + defer cleanupWith() + + withoutBypass, cleanupWithout, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: "s3:PutObjectRetention", Resource: objectsArn(bucket), + }), + }) + if err != nil { + return err + } + defer cleanupWithout() + + cases := []struct { + name string + mode types.ObjectLockRetentionMode + user *s3IAMPrincipal + shorten bool + sendHeader bool + // wantErr is nil when the change must be allowed. + wantErr func(user *s3IAMPrincipal, key string) s3err.S3Error + }{ + { + name: "governance extended needs nothing", + mode: types.ObjectLockRetentionModeGovernance, user: withoutBypass, + }, + { + name: "compliance extended needs nothing", + mode: types.ObjectLockRetentionModeCompliance, user: withoutBypass, + }, + { + // No header at all: the object is simply reported as locked, + // with no mention of a permission the caller never invoked. + name: "governance shortened without the bypass header", + mode: types.ObjectLockRetentionModeGovernance, user: withBypass, shorten: true, + wantErr: func(*s3IAMPrincipal, string) s3err.S3Error { + return s3err.GetAPIError(s3err.ErrObjectLocked) + }, + }, + { + // Header sent but the permission missing: the denial names + // the permission that was needed. + name: "governance shortened without the bypass permission", + mode: types.ObjectLockRetentionModeGovernance, user: withoutBypass, shorten: true, sendHeader: true, + wantErr: func(u *s3IAMPrincipal, key string) s3err.S3Error { + return wantImplicitDeny(u.arn, actS3BypassGovernance, objectArn(bucket, key)) + }, + }, + { + name: "governance shortened with header and permission", + mode: types.ObjectLockRetentionModeGovernance, user: withBypass, shorten: true, sendHeader: true, + }, + { + // COMPLIANCE is absolute: neither the header nor the + // permission can weaken it. + name: "compliance shortened even with header and permission", + mode: types.ObjectLockRetentionModeCompliance, user: withBypass, shorten: true, sendHeader: true, + wantErr: func(*s3IAMPrincipal, string) s3err.S3Error { + return s3err.GetAPIError(s3err.ErrObjectLocked) + }, + }, + } + + for i, tc := range cases { + if err := func() error { + key := fmt.Sprintf("retained-%d", i) + original := time.Now().UTC().Add(time.Hour) + if _, err := putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + ObjectLockMode: types.ObjectLockMode(tc.mode), + ObjectLockRetainUntilDate: &original, + }, s.GetClient()); err != nil { + return err + } + + want := original.Add(time.Minute) + if tc.shorten { + want = original.Add(-30 * time.Second) + } + + input := &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &key, + Retention: &types.ObjectLockRetention{ + Mode: tc.mode, + RetainUntilDate: &want, + }, + } + if tc.sendHeader { + input.BypassGovernanceRetention = aws.Bool(true) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := tc.user.client.PutObjectRetention(ctx, input) + cancel() + + if tc.wantErr == nil { + if err != nil { + return fmt.Errorf("expected the retention change to be allowed: %w", err) + } + return nil + } + return checkApiErr(err, tc.wantErr(tc.user, key)) + }(); err != nil { + return fmt.Errorf("%s: %w", tc.name, err) + } + } + return nil + }) +} + +// S3IAMAccessControl_condition_source_ip verifies aws:SourceIp is populated +// from the real request, both as a grant that matches and as one that +// doesn't. +func S3IAMAccessControl_condition_source_ip(s *S3Conf) error { + testName := "S3IAMAccessControl_condition_source_ip" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + callerIP, err := callerSourceIP(s) + if err != nil { + return err + } + + return runS3ConditionCases(root, s, bucket, "obj", []s3ConditionCase{ + { + name: "matching source ip", + condition: cond("IpAddress", "aws:SourceIp", callerIP+"/32"), + wantAllowed: true, + }, + { + name: "non-matching source ip", + condition: cond("IpAddress", "aws:SourceIp", "203.0.113.0/24"), + }, + { + name: "negated operator with a matching key", + condition: cond("NotIpAddress", "aws:SourceIp", "203.0.113.0/24"), + wantAllowed: true, + }, + }) + }) +} + +// S3IAMAccessControl_condition_negated_operator_needs_context is a +// regression test for a fail-open bug: the gateway used to send no condition +// context at all for S3 requests, and iamapi/policy treats a negated +// operator over an absent key as vacuously true — so a Deny guarded by +// NotIpAddress silently never fired, and an Allow guarded by one fired for +// everybody. With the context populated, a NotIpAddress Deny naming the +// caller's own address must actually deny. +func S3IAMAccessControl_condition_negated_operator_needs_context(s *S3Conf) error { + testName := "S3IAMAccessControl_condition_negated_operator_needs_context" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + callerIP, err := callerSourceIP(s) + if err != nil { + return err + } + + user, cleanup, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc( + accessStatement{Effect: "Allow", Action: actS3GetObject, Resource: objectsArn(bucket)}, + accessStatement{ + Effect: "Deny", Action: actS3GetObject, Resource: objectsArn(bucket), + Condition: cond("NotIpAddress", "aws:SourceIp", "203.0.113.0/24"), + }, + ), + }) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err := checkApiErr(err, wantExplicitIdentityDeny(user.arn, actS3GetObject, objectArn(bucket, "obj"))); err != nil { + return fmt.Errorf("a NotIpAddress Deny must fire when the caller's address (%s) is outside the named range: %w", callerIP, err) + } + return nil + }) +} + +// S3IAMAccessControl_condition_request_keys covers the remaining condition +// keys the gateway derives from the request itself. +func S3IAMAccessControl_condition_request_keys(s *S3Conf) error { + testName := "S3IAMAccessControl_condition_request_keys" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + + // The integration harness always drives the gateway over plain + // HTTP or TLS, never both in one run, so aws:SecureTransport is + // asserted against whichever this run actually uses rather than + // hardcoded. + secure := strings.HasPrefix(s.endpoint, "https") + + return runS3ConditionCases(root, s, bucket, "obj", []s3ConditionCase{ + { + name: "secure transport matches the endpoint scheme", + condition: cond("Bool", "aws:SecureTransport", fmt.Sprintf("%t", secure)), + wantAllowed: true, + }, + { + name: "secure transport mismatch", + condition: cond("Bool", "aws:SecureTransport", fmt.Sprintf("%t", !secure)), + }, + { + name: "current time inside a broad window", + condition: cond("DateLessThan", "aws:CurrentTime", "2999-01-01T00:00:00Z"), + wantAllowed: true, + }, + { + name: "current time outside the window", + condition: cond("DateLessThan", "aws:CurrentTime", "2000-01-01T00:00:00Z"), + }, + { + name: "epoch time inside a broad window", + condition: cond("NumericGreaterThan", "aws:EpochTime", "1000000000"), + wantAllowed: true, + }, + { + name: "user agent is present", + condition: cond("Null", "aws:UserAgent", "false"), + wantAllowed: true, + }, + }) + }) +} + +// S3IAMAccessControl_condition_identity_keys verifies the identity-derived +// condition keys — which the IAM service fills in, since the gateway never +// learns who an access key belongs to — reach policy evaluation. +func S3IAMAccessControl_condition_identity_keys(s *S3Conf) error { + testName := "S3IAMAccessControl_condition_identity_keys" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + + user, cleanup, err := newS3IAMUser(root, s, nil) + if err != nil { + return err + } + defer cleanup() + + cases := []struct { + name string + condition func() []byte + wantAllowed bool + }{ + { + name: "principal arn matches", + condition: func() []byte { return cond("StringEquals", "aws:PrincipalArn", user.arn) }, + wantAllowed: true, + }, + { + name: "principal arn mismatch", + condition: func() []byte { + return cond("StringEquals", "aws:PrincipalArn", "arn:aws:iam::000000000000:user/somebodyelse") + }, + }, + { + name: "username matches", + condition: func() []byte { return cond("StringEquals", "aws:username", user.name) }, + wantAllowed: true, + }, + { + name: "principal type is User", + condition: func() []byte { return cond("StringEquals", "aws:PrincipalType", "User") }, + wantAllowed: true, + }, + { + name: "principal account matches", + condition: func() []byte { return cond("StringEquals", "aws:PrincipalAccount", testAccountID) }, + wantAllowed: true, + }, + } + + for _, tc := range cases { + if err := func() error { + if err := putS3IAMUserPolicy(root, user, "p", policyDoc(accessStatement{ + Effect: "Allow", Action: actS3GetObject, Resource: objectsArn(bucket), + Condition: tc.condition(), + })); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if tc.wantAllowed { + if err != nil { + return fmt.Errorf("expected the request to be allowed: %w", err) + } + return nil + } + return checkApiErr(err, wantImplicitDeny(user.arn, actS3GetObject, objectArn(bucket, "obj"))) + }(); err != nil { + return fmt.Errorf("%s: %w", tc.name, err) + } + } + return nil + }) +} + +// S3IAMAccessControl_condition_principal_tag verifies aws:PrincipalTag/ +// is populated from the calling user's own IAM tags, and that a tag the user +// doesn't carry is treated as absent rather than as an empty match. +func S3IAMAccessControl_condition_principal_tag(s *S3Conf) error { + testName := "S3IAMAccessControl_condition_principal_tag" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + + userName := newIAMUserName() + createOut, err := createIAMUser(root, &iam.CreateUserInput{ + UserName: aws.String(userName), + Tags: []iamtypes.Tag{{Key: aws.String("team"), Value: aws.String("storage")}}, + }) + if err != nil { + return err + } + defer deleteS3IAMUser(root, userName) + + keyOut, err := createIAMAccessKey(root, &iam.CreateAccessKeyInput{UserName: aws.String(userName)}) + if err != nil { + return err + } + conf := *s + conf.awsID = aws.ToString(keyOut.AccessKey.AccessKeyId) + conf.awsSecret = aws.ToString(keyOut.AccessKey.SecretAccessKey) + user := &s3IAMPrincipal{name: userName, arn: aws.ToString(createOut.User.Arn), conf: conf, client: conf.GetClient()} + + cases := []struct { + name string + condition []byte + wantAllowed bool + }{ + {name: "matching tag value", condition: cond("StringEquals", "aws:PrincipalTag/team", "storage"), wantAllowed: true}, + {name: "wrong tag value", condition: cond("StringEquals", "aws:PrincipalTag/team", "networking")}, + {name: "tag the user does not carry", condition: cond("StringEquals", "aws:PrincipalTag/other", "anything")}, + {name: "absent tag reported by Null", condition: cond("Null", "aws:PrincipalTag/other", "true"), wantAllowed: true}, + } + for _, tc := range cases { + if err := func() error { + if err := putS3IAMUserPolicy(root, user, "p", policyDoc(accessStatement{ + Effect: "Allow", Action: actS3GetObject, Resource: objectsArn(bucket), + Condition: tc.condition, + })); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if tc.wantAllowed { + if err != nil { + return fmt.Errorf("expected the request to be allowed: %w", err) + } + return nil + } + return checkApiErr(err, wantImplicitDeny(user.arn, actS3GetObject, objectArn(bucket, "obj"))) + }(); err != nil { + return fmt.Errorf("%s: %w", tc.name, err) + } + } + return nil + }) +} + +// S3IAMAccessControl_condition_on_deny_statement verifies a Condition +// attached to a Deny narrows that Deny — when the condition doesn't hold, +// the statement contributes nothing and an unconditional Allow still stands. +func S3IAMAccessControl_condition_on_deny_statement(s *S3Conf) error { + testName := "S3IAMAccessControl_condition_on_deny_statement" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + callerIP, err := callerSourceIP(s) + if err != nil { + return err + } + + user, cleanup, err := newS3IAMUser(root, s, nil) + if err != nil { + return err + } + defer cleanup() + + allow := accessStatement{Effect: "Allow", Action: actS3GetObject, Resource: objectsArn(bucket)} + + // Deny conditioned on an address the caller does not have: it must + // not fire, leaving the Allow in force. + if err := putS3IAMUserPolicy(root, user, "p", policyDoc(allow, accessStatement{ + Effect: "Deny", Action: actS3GetObject, Resource: objectsArn(bucket), + Condition: cond("IpAddress", "aws:SourceIp", "203.0.113.0/24"), + })); err != nil { + return err + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return fmt.Errorf("a Deny whose condition does not hold must not block an unconditional Allow: %w", err) + } + + // Deny conditioned on the caller's real address: it must fire and + // override the Allow. + if err := putS3IAMUserPolicy(root, user, "p", policyDoc(allow, accessStatement{ + Effect: "Deny", Action: actS3GetObject, Resource: objectsArn(bucket), + Condition: cond("IpAddress", "aws:SourceIp", callerIP+"/32"), + })); err != nil { + return err + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + return checkApiErr(err, wantExplicitIdentityDeny(user.arn, actS3GetObject, objectArn(bucket, "obj"))) + }) +} + +// S3IAMAccessControl_condition_multiple_keys_anded verifies multiple keys +// within one Condition block must all hold, while multiple values for one +// key are ORed. +func S3IAMAccessControl_condition_multiple_keys_anded(s *S3Conf) error { + testName := "S3IAMAccessControl_condition_multiple_keys_anded" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + callerIP, err := callerSourceIP(s) + if err != nil { + return err + } + + user, cleanup, err := newS3IAMUser(root, s, nil) + if err != nil { + return err + } + defer cleanup() + + cases := []struct { + name string + condition []byte + wantAllowed bool + }{ + { + name: "both keys hold", + condition: condAll(map[string]map[string]any{ + "IpAddress": {"aws:SourceIp": callerIP + "/32"}, + "StringEquals": {"aws:username": user.name}, + }), + wantAllowed: true, + }, + { + name: "one key fails", + condition: condAll(map[string]map[string]any{ + "IpAddress": {"aws:SourceIp": callerIP + "/32"}, + "StringEquals": {"aws:username": "somebodyelse"}, + }), + }, + { + name: "one of several values for a key matches", + condition: cond("StringEquals", "aws:username", []string{"somebodyelse", user.name}), + wantAllowed: true, + }, + } + for _, tc := range cases { + if err := func() error { + if err := putS3IAMUserPolicy(root, user, "p", policyDoc(accessStatement{ + Effect: "Allow", Action: actS3GetObject, Resource: objectsArn(bucket), + Condition: tc.condition, + })); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if tc.wantAllowed { + if err != nil { + return fmt.Errorf("expected the request to be allowed: %w", err) + } + return nil + } + return checkApiErr(err, wantImplicitDeny(user.arn, actS3GetObject, objectArn(bucket, "obj"))) + }(); err != nil { + return fmt.Errorf("%s: %w", tc.name, err) + } + } + return nil + }) +} + +// S3IAMAccessControl_inactive_and_deleted_credentials verifies the gateway +// stops accepting an access key as soon as the IAM service stops vouching +// for it — whether it was deactivated, deleted, or its user was removed. +func S3IAMAccessControl_inactive_and_deleted_credentials(s *S3Conf) error { + testName := "S3IAMAccessControl_inactive_and_deleted_credentials" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + grantAll := map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: "s3:*", + Resource: []string{bucketArn(bucket), objectsArn(bucket)}, + }), + } + + cases := []struct { + name string + disable func(user *s3IAMPrincipal) error + }{ + { + name: "deactivated access key", + disable: func(user *s3IAMPrincipal) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + defer cancel() + _, err := root.UpdateAccessKey(ctx, &iam.UpdateAccessKeyInput{ + UserName: aws.String(user.name), + AccessKeyId: aws.String(user.conf.awsID), + Status: iamtypes.StatusTypeInactive, + }) + return err + }, + }, + { + name: "deleted access key", + disable: func(user *s3IAMPrincipal) error { + return deleteIAMAccessKey(root, user.name, user.conf.awsID) + }, + }, + { + name: "deleted user", + disable: func(user *s3IAMPrincipal) error { + return deleteS3IAMUser(root, user.name) + }, + }, + } + + for _, tc := range cases { + if err := func() error { + user, cleanup, err := newS3IAMUser(root, s, grantAll) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) + cancel() + if err != nil { + return fmt.Errorf("expected the credential to work before being disabled: %w", err) + } + + if err := tc.disable(user); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) + cancel() + return checkApiErr(err, s3err.GetInvalidAccessKeyIdErr(user.conf.awsID)) + }(); err != nil { + return fmt.Errorf("%s: %w", tc.name, err) + } + } + return nil + }) +} + +// S3IAMAccessControl_bucket_policy_unknown_principal_rejected verifies +// PutBucketPolicy validates its principals against the IAM service, so a +// policy naming somebody who doesn't exist is rejected instead of being +// stored as a statement that can never match. +func S3IAMAccessControl_bucket_policy_unknown_principal_rejected(s *S3Conf) error { + testName := "S3IAMAccessControl_bucket_policy_unknown_principal_rejected" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: "Allow", Principal: "AKIADOESNOTEXIST", Action: actS3GetObject, Resource: objectsArn(bucket), + }) + return checkApiErr(err, s3err.APIError{ + Code: "MalformedPolicy", + Description: "Invalid principal in policy", + HTTPStatusCode: 400, + }) + }) +} diff --git a/tests/integration/s3_iam_session_access_control.go b/tests/integration/s3_iam_session_access_control.go new file mode 100644 index 00000000..2e670229 --- /dev/null +++ b/tests/integration/s3_iam_session_access_control.go @@ -0,0 +1,786 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package integration + +import ( + "context" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/versity/versitygw/s3err" +) + +// S3IAMSession_role_policy_allows verifies a session inherits the assumed +// role's inline policies, and that they are sufficient on their own with no +// bucket policy in play. +func S3IAMSession_role_policy_allows(s *S3Conf) error { + testName := "S3IAMSession_role_policy_allows" + return s3IAMSessionActionHandler(s, testName, func(root *iam.Client, bucket string) error { + session, cleanup, err := newGitHubSession(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: "s3:*", + Resource: []string{bucketArn(bucket), objectsArn(bucket)}, + }), + }, "") + if err != nil { + return err + } + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return fmt.Errorf("expected PutObject to be allowed by the role policy: %w", err) + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return fmt.Errorf("expected GetObject to be allowed by the role policy: %w", err) + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) + cancel() + if err != nil { + return fmt.Errorf("expected ListObjects to be allowed by the role policy: %w", err) + } + return nil + }) +} + +// S3IAMSession_role_without_policy_denied verifies a session with no role +// policy and no bucket policy is denied, and that the denial names the +// assumed-role session ARN rather than the temporary access key. +func S3IAMSession_role_without_policy_denied(s *S3Conf) error { + testName := "S3IAMSession_role_without_policy_denied" + return s3IAMSessionActionHandler(s, testName, func(root *iam.Client, bucket string) error { + session, cleanup, err := newGitHubSession(root, s, nil, "") + if err != nil { + return err + } + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + return checkApiErr(err, wantImplicitDeny(session.arn, actS3GetObject, objectArn(bucket, "obj"))) + }) +} + +// S3IAMSession_role_policy_explicit_deny_wins verifies an explicit Deny in +// the role's own policy overrides its Allow, exactly as for a long-term +// user. +func S3IAMSession_role_policy_explicit_deny_wins(s *S3Conf) error { + testName := "S3IAMSession_role_policy_explicit_deny_wins" + return s3IAMSessionActionHandler(s, testName, func(root *iam.Client, bucket string) error { + session, cleanup, err := newGitHubSession(root, s, map[string]string{ + "p": policyDoc( + accessStatement{Effect: "Allow", Action: "s3:*", Resource: objectsArn(bucket)}, + accessStatement{Effect: "Deny", Action: actS3GetObject, Resource: objectsArn(bucket)}, + ), + }, "") + if err != nil { + return err + } + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return fmt.Errorf("expected PutObject to still be allowed: %w", err) + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + return checkApiErr(err, wantExplicitIdentityDeny(session.arn, actS3GetObject, objectArn(bucket, "obj"))) + }) +} + +// S3IAMSession_role_policy_resource_scoped verifies a role policy's Resource +// pattern scopes what the session may touch. +func S3IAMSession_role_policy_resource_scoped(s *S3Conf) error { + testName := "S3IAMSession_role_policy_resource_scoped" + return s3IAMSessionActionHandler(s, testName, func(root *iam.Client, bucket string) error { + session, cleanup, err := newGitHubSession(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: "s3:*", Resource: objectArn(bucket, "allowed/*"), + }), + }, "") + if err != nil { + return err + } + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("allowed/obj")}) + cancel() + if err != nil { + return fmt.Errorf("expected the in-scope key to be allowed: %w", err) + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("denied/obj")}) + cancel() + return checkApiErr(err, wantImplicitDeny(session.arn, actS3PutObject, objectArn(bucket, "denied/obj"))) + }) +} + +// S3IAMSession_session_policy_narrows_role verifies a session policy +// restricts what the role would otherwise permit — the primary reason to +// pass one. +func S3IAMSession_session_policy_narrows_role(s *S3Conf) error { + testName := "S3IAMSession_session_policy_narrows_role" + return s3IAMSessionActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + + session, cleanup, err := newGitHubSession(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: "s3:*", + Resource: []string{bucketArn(bucket), objectsArn(bucket)}, + }), + }, policyDoc(accessStatement{ + Effect: "Allow", Action: actS3GetObject, Resource: objectsArn(bucket), + })) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return fmt.Errorf("expected GetObject to be allowed by both layers: %w", err) + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("other")}) + cancel() + return checkApiErr(err, wantImplicitDeny(session.arn, actS3PutObject, objectArn(bucket, "other"))) + }) +} + +// S3IAMSession_session_policy_cannot_widen_role verifies a session policy +// can only ever subtract: granting more than the role has does not add +// anything. +func S3IAMSession_session_policy_cannot_widen_role(s *S3Conf) error { + testName := "S3IAMSession_session_policy_cannot_widen_role" + return s3IAMSessionActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + + session, cleanup, err := newGitHubSession(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: actS3GetObject, Resource: objectsArn(bucket), + }), + }, policyDoc(accessStatement{ + Effect: "Allow", Action: "s3:*", Resource: "*", + })) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return fmt.Errorf("expected GetObject to be allowed by both layers: %w", err) + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("other")}) + cancel() + return checkApiErr(err, wantImplicitDeny(session.arn, actS3PutObject, objectArn(bucket, "other"))) + }) +} + +// S3IAMSession_session_policy_explicit_deny_overrides_role verifies an +// explicit Deny in the session policy beats the role's Allow. +func S3IAMSession_session_policy_explicit_deny_overrides_role(s *S3Conf) error { + testName := "S3IAMSession_session_policy_explicit_deny_overrides_role" + return s3IAMSessionActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + + session, cleanup, err := newGitHubSession(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: "s3:*", + Resource: []string{bucketArn(bucket), objectsArn(bucket)}, + }), + }, policyDoc( + accessStatement{Effect: "Allow", Action: "s3:*", Resource: "*"}, + accessStatement{Effect: "Deny", Action: actS3GetObject, Resource: objectsArn(bucket)}, + )) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + return checkApiErr(err, wantExplicitIdentityDeny(session.arn, actS3GetObject, objectArn(bucket, "obj"))) + }) +} + +// S3IAMSession_role_policy_deny_overrides_session_allow verifies the reverse +// direction: an explicit Deny in the role's policy is not escapable by a +// permissive session policy. +func S3IAMSession_role_policy_deny_overrides_session_allow(s *S3Conf) error { + testName := "S3IAMSession_role_policy_deny_overrides_session_allow" + return s3IAMSessionActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + + session, cleanup, err := newGitHubSession(root, s, map[string]string{ + "p": policyDoc( + accessStatement{Effect: "Allow", Action: "s3:*", Resource: objectsArn(bucket)}, + accessStatement{Effect: "Deny", Action: actS3GetObject, Resource: objectsArn(bucket)}, + ), + }, policyDoc(accessStatement{Effect: "Allow", Action: "s3:*", Resource: "*"})) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + return checkApiErr(err, wantExplicitIdentityDeny(session.arn, actS3GetObject, objectArn(bucket, "obj"))) + }) +} + +// S3IAMSession_session_policy_without_role_policy_denied verifies a session +// policy alone grants nothing: with the role carrying no policy and no +// bucket policy in play, there is nothing for it to narrow. +func S3IAMSession_session_policy_without_role_policy_denied(s *S3Conf) error { + testName := "S3IAMSession_session_policy_without_role_policy_denied" + return s3IAMSessionActionHandler(s, testName, func(root *iam.Client, bucket string) error { + session, cleanup, err := newGitHubSession(root, s, nil, + policyDoc(accessStatement{Effect: "Allow", Action: "s3:*", Resource: "*"})) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + return checkApiErr(err, wantImplicitDeny(session.arn, actS3GetObject, objectArn(bucket, "obj"))) + }) +} + +// S3IAMSession_bucket_policy_allows_without_role_policy verifies the bucket +// policy is independently sufficient for a session too, exactly as it is for +// a long-term user. +// +// The bucket policy names "*" rather than the session: this gateway matches +// bucket-policy principals against the caller's access key, and a session's +// key is ephemeral, so auth.CheckIfAccountsExist rejects one as a principal +// outright rather than let a policy come to reference a principal that stops +// existing. See bucketStatement. +func S3IAMSession_bucket_policy_allows_without_role_policy(s *S3Conf) error { + testName := "S3IAMSession_bucket_policy_allows_without_role_policy" + return s3IAMSessionActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: "Allow", Principal: "*", Action: actS3GetObject, Resource: objectsArn(bucket), + }); err != nil { + return err + } + + session, cleanup, err := newGitHubSession(root, s, nil, "") + if err != nil { + return err + } + defer cleanup() + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return fmt.Errorf("expected GetObject to be allowed by the bucket policy: %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("other")}) + cancel() + return checkApiErr(err, wantImplicitDeny(session.arn, actS3PutObject, objectArn(bucket, "other"))) + }) +} + +// S3IAMSession_session_policy_filters_bucket_policy_grant is the property +// that distinguishes a session policy from an ordinary identity policy: it +// filters *everything* the session can do, including permissions that came +// from the bucket policy rather than from the role. +// +// Verified against real AWS with a role carrying no identity policy at all, +// a bucket policy granting it both s3:GetObject and s3:PutObject, and a +// session policy allowing only s3:GetObject — the Get succeeds and the Put +// is denied. +func S3IAMSession_session_policy_filters_bucket_policy_grant(s *S3Conf) error { + testName := "S3IAMSession_session_policy_filters_bucket_policy_grant" + return s3IAMSessionActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: "Allow", Principal: "*", + Action: []string{actS3GetObject, actS3PutObject}, Resource: objectsArn(bucket), + }); err != nil { + return err + } + + session, cleanup, err := newGitHubSession(root, s, nil, + policyDoc(accessStatement{Effect: "Allow", Action: actS3GetObject, Resource: objectsArn(bucket)})) + if err != nil { + return err + } + defer cleanup() + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return fmt.Errorf("expected GetObject to be allowed by the bucket policy within the session policy: %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("other")}) + cancel() + return checkApiErr(err, wantImplicitDeny(session.arn, actS3PutObject, objectArn(bucket, "other"))) + }) +} + +// S3IAMSession_bucket_policy_deny_overrides_role_allow verifies a +// bucket-policy Deny beats the role's Allow for a session, and reports the +// resource-based-policy message. +func S3IAMSession_bucket_policy_deny_overrides_role_allow(s *S3Conf) error { + testName := "S3IAMSession_bucket_policy_deny_overrides_role_allow" + return s3IAMSessionActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + if err := putBucketPolicyDoc(s, bucket, bucketStatement{ + Effect: "Deny", Principal: "*", Action: actS3GetObject, Resource: objectsArn(bucket), + }); err != nil { + return err + } + + session, cleanup, err := newGitHubSession(root, s, map[string]string{ + "p": policyDoc(accessStatement{Effect: "Allow", Action: "s3:*", Resource: objectsArn(bucket)}), + }, "") + if err != nil { + return err + } + defer cleanup() + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + // A resource-based denial names the raw access key: bucket-policy + // principals are access-key-based for every backend, so no ARN is in + // hand at that point. + return checkApiErr(err, wantExplicitResourceDeny(session.conf.awsID, actS3GetObject, objectArn(bucket, "obj"))) + }) +} + +// S3IAMSession_missing_and_wrong_security_token verifies the two ways a +// session credential can be presented wrongly, each with the error real S3 +// returns for it. +func S3IAMSession_missing_and_wrong_security_token(s *S3Conf) error { + testName := "S3IAMSession_missing_and_wrong_security_token" + return s3IAMSessionActionHandler(s, testName, func(root *iam.Client, bucket string) error { + session, cleanup, err := newGitHubSession(root, s, map[string]string{ + "p": policyDoc(accessStatement{Effect: "Allow", Action: "s3:*", Resource: objectsArn(bucket)}), + }, "") + if err != nil { + return err + } + defer cleanup() + + // No token at all: with nothing to resolve the temporary access key + // against, it simply does not name any identity. + noToken := s3ClientWithSessionCreds(s, session.conf.awsID, session.conf.awsSecret, "") + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = noToken.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err := checkApiErr(err, s3err.GetInvalidAccessKeyIdErr(session.conf.awsID)); err != nil { + return fmt.Errorf("missing security token: %w", err) + } + + // A token that doesn't match the session it names. + wrongToken := s3ClientWithSessionCreds(s, session.conf.awsID, session.conf.awsSecret, "not-the-real-session-token") + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = wrongToken.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidToken)); err != nil { + return fmt.Errorf("wrong security token: %w", err) + } + return nil + }) +} + +// S3IAMSession_presigned_url_with_session_credentials verifies a presigned +// URL signed with temporary credentials works: the security token rides in +// the query string, where it is part of the signed canonical request. +func S3IAMSession_presigned_url_with_session_credentials(s *S3Conf) error { + testName := "S3IAMSession_presigned_url_with_session_credentials" + return s3IAMSessionActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + + session, cleanup, err := newGitHubSession(root, s, map[string]string{ + "p": policyDoc(accessStatement{Effect: "Allow", Action: actS3GetObject, Resource: objectsArn(bucket)}), + }, "") + if err != nil { + return err + } + defer cleanup() + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + presigned, err := s3.NewPresignClient(session.client).PresignGetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, Key: aws.String("obj"), + }) + cancel() + if err != nil { + return fmt.Errorf("presign: %w", err) + } + if !strings.Contains(presigned.URL, "X-Amz-Security-Token") { + return fmt.Errorf("expected the presigned URL to carry X-Amz-Security-Token") + } + + resp, err := s.httpClient.Get(presigned.URL) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return fmt.Errorf("expected the presigned request to succeed, got status %d", resp.StatusCode) + } + return nil + }) +} + +// S3IAMSession_deleted_role_denies verifies a session outlives its role's +// deletion as a credential — it still authenticates — but loses every +// permission the role gave it. +func S3IAMSession_deleted_role_denies(s *S3Conf) error { + testName := "S3IAMSession_deleted_role_denies" + return s3IAMSessionActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + + session, cleanup, err := newGitHubSession(root, s, map[string]string{ + "p": policyDoc(accessStatement{Effect: "Allow", Action: "s3:*", Resource: objectsArn(bucket)}), + }, "") + if err != nil { + return err + } + defer cleanup() + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return fmt.Errorf("expected GetObject to be allowed before the role is deleted: %w", err) + } + + if err := deleteIAMRoleAndPolicies(root, session.name); err != nil { + return fmt.Errorf("delete role: %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + return checkApiErr(err, wantImplicitDeny(session.arn, actS3GetObject, objectArn(bucket, "obj"))) + }) +} + +// S3IAMSession_create_bucket_via_role_policy verifies s3:CreateBucket is +// grantable to a session by its role policy, and denied without it. +func S3IAMSession_create_bucket_via_role_policy(s *S3Conf) error { + testName := "S3IAMSession_create_bucket_via_role_policy" + // The skip is checked before actionHandlerNoSetup rather than inside it, + // so a skipped run doesn't also report itself as a pass. + if _, ok := gitHubOIDCToken(); !ok { + skipF("%v: %v", testName, gitHubOIDCSkipReason) + return nil + } + + return actionHandlerNoSetup(s, testName, func(_ *s3.Client, _ string) error { + root := s.GetIAMClient() + allowed, denied := getBucketName(), getBucketName() + + session, cleanup, err := newGitHubSession(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: actS3CreateBucket, Resource: bucketArn(allowed), + }), + }, "") + if err != nil { + return err + } + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: &allowed}) + cancel() + if err != nil { + return fmt.Errorf("expected CreateBucket to be allowed for the granted name: %w", err) + } + defer teardown(s, allowed) + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: &denied}) + cancel() + return checkApiErr(err, wantImplicitDeny(session.arn, actS3CreateBucket, bucketArn(denied))) + }) +} + +// S3IAMSession_governance_bypass_via_role_policy verifies a session can be +// granted s3:BypassGovernanceRetention through its role, and that a session +// policy withholding it takes it away again. +func S3IAMSession_governance_bypass_via_role_policy(s *S3Conf) error { + testName := "S3IAMSession_governance_bypass_via_role_policy" + return s3IAMSessionActionHandler(s, testName, func(root *iam.Client, bucket string) error { + grantAll := map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: []string{actS3DeleteObject, actS3BypassGovernance}, + Resource: objectsArn(bucket), + }), + } + + // Role grants the bypass, session policy withholds it: denied. + withheld, cleanupWithheld, err := newGitHubSession(root, s, grantAll, + policyDoc(accessStatement{Effect: "Allow", Action: actS3DeleteObject, Resource: objectsArn(bucket)})) + if err != nil { + return err + } + defer cleanupWithheld() + + if err := putGovernanceLockedObject(s, bucket, "locked-withheld"); err != nil { + return err + } + if err := deleteObjectBypassingGovernance(withheld.client, bucket, "locked-withheld"); err == nil { + return fmt.Errorf("expected the delete to be denied when the session policy withholds the bypass permission") + } + + // Role grants it and no session policy narrows it: allowed. + granted, cleanupGranted, err := newGitHubSession(root, s, grantAll, "") + if err != nil { + return err + } + defer cleanupGranted() + + if err := putGovernanceLockedObject(s, bucket, "locked-granted"); err != nil { + return err + } + if err := deleteObjectBypassingGovernance(granted.client, bucket, "locked-granted"); err != nil { + return fmt.Errorf("expected the delete to be allowed by the role's bypass grant: %w", err) + } + return nil + }, withLock()) +} + +// S3IAMSession_delete_objects_authorizes_each_key verifies the per-key +// authorization of a batch delete applies to a session's role policy too. +func S3IAMSession_delete_objects_authorizes_each_key(s *S3Conf) error { + testName := "S3IAMSession_delete_objects_authorizes_each_key" + return s3IAMSessionActionHandler(s, testName, func(root *iam.Client, bucket string) error { + for _, key := range []string{"allowed/one", "denied/two"} { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr(key)}) + cancel() + if err != nil { + return err + } + } + + session, cleanup, err := newGitHubSession(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: actS3DeleteObject, Resource: objectArn(bucket, "allowed/*"), + }), + }, "") + if err != nil { + return err + } + defer cleanup() + + out, err := deleteObjectsWithBypass(session.client, bucket, "allowed/one", "denied/two") + if err != nil { + return fmt.Errorf("expected DeleteObjects to succeed with a per-object denial, not fail outright: %w", err) + } + if len(out.Errors) != 1 { + return fmt.Errorf("expected exactly 1 per-object error, got %+v", out.Errors) + } + if err := checkDeleteObjectsErr(out.Errors[0], "denied/two", wantImplicitDeny(session.arn, actS3DeleteObject, objectArn(bucket, "denied/two"))); err != nil { + return err + } + + if _, err := deleteObjectsWithBypass(session.client, bucket, "allowed/one"); err != nil { + return fmt.Errorf("expected the in-scope key to be deletable: %w", err) + } + return nil + }) +} + +// S3IAMSession_condition_identity_keys verifies the identity-derived +// condition keys describe the *session*, not the underlying role: aws:userid +// carries the role id and session name, and aws:PrincipalArn the +// assumed-role ARN. +func S3IAMSession_condition_identity_keys(s *S3Conf) error { + testName := "S3IAMSession_condition_identity_keys" + return s3IAMSessionActionHandler(s, testName, func(root *iam.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if err != nil { + return err + } + + cases := []struct { + name string + condition func(session *s3IAMPrincipal) []byte + wantAllowed bool + }{ + { + name: "principal arn matches the assumed-role session", + condition: func(p *s3IAMPrincipal) []byte { return cond("StringEquals", "aws:PrincipalArn", p.arn) }, + wantAllowed: true, + }, + { + name: "principal type is AssumedRole", + condition: func(p *s3IAMPrincipal) []byte { return cond("StringEquals", "aws:PrincipalType", "AssumedRole") }, + wantAllowed: true, + }, + { + name: "userid ends with the session name", + condition: func(p *s3IAMPrincipal) []byte { return cond("StringLike", "aws:userid", "*:"+sessionNameFor(p)) }, + wantAllowed: true, + }, + { + name: "principal arn mismatch", + condition: func(p *s3IAMPrincipal) []byte { + return cond("StringEquals", "aws:PrincipalArn", "arn:aws:sts::000000000000:assumed-role/other/other") + }, + }, + { + name: "aws:username is absent for a session", + condition: func(p *s3IAMPrincipal) []byte { return cond("Null", "aws:username", "false") }, + }, + } + + for _, tc := range cases { + if err := func() error { + session, cleanup, err := newGitHubSession(root, s, nil, "") + if err != nil { + return err + } + defer cleanup() + + if _, err := putIAMRolePolicy(root, &iam.PutRolePolicyInput{ + RoleName: aws.String(session.name), + PolicyName: aws.String("p"), + PolicyDocument: aws.String(policyDoc(accessStatement{ + Effect: "Allow", Action: actS3GetObject, Resource: objectsArn(bucket), + Condition: tc.condition(session), + })), + }); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + if tc.wantAllowed { + if err != nil { + return fmt.Errorf("expected the request to be allowed: %w", err) + } + return nil + } + return checkApiErr(err, wantImplicitDeny(session.arn, actS3GetObject, objectArn(bucket, "obj"))) + }(); err != nil { + return fmt.Errorf("%s: %w", tc.name, err) + } + } + return nil + }) +} + +// S3IAMSession_get_caller_identity_matches_s3_principal verifies STS and the +// S3 data plane agree on who the session is: the ARN GetCallerIdentity +// reports is the one an S3 denial names. +func S3IAMSession_get_caller_identity_matches_s3_principal(s *S3Conf) error { + testName := "S3IAMSession_get_caller_identity_matches_s3_principal" + return s3IAMSessionActionHandler(s, testName, func(root *iam.Client, bucket string) error { + session, cleanup, err := newGitHubSession(root, s, nil, "") + if err != nil { + return err + } + defer cleanup() + + callerOut, err := getCallerIdentityWithSessionCreds(*s, session.conf.awsID, session.conf.awsSecret, session.sessionToken) + if err != nil { + return fmt.Errorf("GetCallerIdentity: %w", err) + } + if aws.ToString(callerOut.Arn) != session.arn { + return fmt.Errorf("GetCallerIdentity reported Arn %q, want %q", aws.ToString(callerOut.Arn), session.arn) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = session.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr("obj")}) + cancel() + return checkApiErr(err, wantImplicitDeny(session.arn, actS3GetObject, objectArn(bucket, "obj"))) + }) +} diff --git a/tests/integration/s3_iam_utils.go b/tests/integration/s3_iam_utils.go new file mode 100644 index 00000000..c5d21b10 --- /dev/null +++ b/tests/integration/s3_iam_utils.go @@ -0,0 +1,550 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package integration + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "sync" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/versity/versitygw/s3err" +) + +const ( + actS3GetObject = "s3:GetObject" + actS3PutObject = "s3:PutObject" + actS3DeleteObject = "s3:DeleteObject" + actS3DeleteObjectVersion = "s3:DeleteObjectVersion" + actS3ListBucket = "s3:ListBucket" + actS3CreateBucket = "s3:CreateBucket" + actS3BypassGovernance = "s3:BypassGovernanceRetention" +) + +// s3IAMPrincipal is an identity that can make S3 requests: an IAM user with +// a long-term access key, or an assumed-role session with temporary +// credentials. Tests assert against arn when checking a denial message, +// since the gateway names the principal by ARN once a PolicyEvaluator +// resolves it. +type s3IAMPrincipal struct { + // name is the IAM user name or, for a session, the role name. + name string + arn string + // conf is a copy of the suite's S3Conf carrying this principal's + // credentials, so tests can build additional clients (presign, STS) + // beyond the plain s3 one. + conf S3Conf + client *s3.Client + // sessionToken is set only for an assumed-role session, for the tests + // that need to build a differently-credentialed client from the same + // session (a presigned URL, an STS call, a deliberately wrong token). + sessionToken string +} + +// s3IAMActionHandler is actionHandler for the S3+IAM groups: it runs handler +// with a root-owned bucket and the root IAM client the fixtures below need, +// then tears the bucket down. Root creates every bucket and object a test +// operates on, so that what the test measures is the principal's +// authorization, never its ability to set the scene. +func s3IAMActionHandler(s *S3Conf, testName string, handler func(root *iam.Client, bucket string) error, opts ...setupOpt) error { + return actionHandler(s, testName, func(_ *s3.Client, bucket string) error { + return handler(s.GetIAMClient(), bucket) + }, opts...) +} + +// s3IAMComplianceActionHandler is s3IAMActionHandler for the tests that put +// an object under COMPLIANCE retention. Such an object cannot be deleted +// before its retention expires — by anyone, with any permission, by design — +// so its bucket cannot be torn down either. +// +// Rather than fail teardown, the bucket is left behind, and its name gets a +// random suffix so that a leftover from an earlier run against the same data +// directory can't collide with this one. The shared getBucketName counter +// restarts with each test process, so without the suffix a second local run +// would fail every one of these tests with BucketAlreadyOwnedByYou. +func s3IAMComplianceActionHandler(s *S3Conf, testName string, handler func(root *iam.Client, bucket string) error) error { + runF(testName) + + // Lower-cased because genRandString's charset includes capitals, which + // bucket names do not allow. + bucket := getBucketName() + "-" + strings.ToLower(genRandString(8)) + if err := setup(s, bucket, withLock()); err != nil { + failF("%v: failed to create a bucket: %v", testName, err) + return fmt.Errorf("%v: failed to create a bucket: %w", testName, err) + } + + if err := handler(s.GetIAMClient(), bucket); err != nil { + failF("%v: %v", testName, err) + return fmt.Errorf("%v: %w", testName, err) + } + + passF(testName) + return nil +} + +// newS3IAMUser creates an IAM user with the given inline policies +// (policyName -> document, may be nil) and one long-term access key, and +// returns a principal whose S3 client is authenticated as that user, plus a +// cleanup func removing the key, the policies, and the user. +func newS3IAMUser(root *iam.Client, s *S3Conf, policies map[string]string) (*s3IAMPrincipal, func(), error) { + userName := newIAMUserName() + + createOut, err := createIAMUser(root, &iam.CreateUserInput{UserName: aws.String(userName)}) + if err != nil { + return nil, nil, fmt.Errorf("create user: %w", err) + } + + cleanup := func() { deleteS3IAMUser(root, userName) } + + for name, doc := range policies { + if _, err := putIAMUserPolicy(root, &iam.PutUserPolicyInput{ + UserName: aws.String(userName), PolicyName: aws.String(name), PolicyDocument: aws.String(doc), + }); err != nil { + cleanup() + return nil, nil, fmt.Errorf("attach policy %q: %w", name, err) + } + } + + keyOut, err := createIAMAccessKey(root, &iam.CreateAccessKeyInput{UserName: aws.String(userName)}) + if err != nil { + cleanup() + return nil, nil, fmt.Errorf("create access key: %w", err) + } + + conf := *s + conf.awsID = aws.ToString(keyOut.AccessKey.AccessKeyId) + conf.awsSecret = aws.ToString(keyOut.AccessKey.SecretAccessKey) + + return &s3IAMPrincipal{ + name: userName, + arn: aws.ToString(createOut.User.Arn), + conf: conf, + client: conf.GetClient(), + }, cleanup, nil +} + +// putS3IAMUserPolicy attaches (or replaces) one inline policy on an existing +// principal, for tests that vary a policy in place across sub-cases rather +// than recreating the whole user each time. +func putS3IAMUserPolicy(root *iam.Client, principal *s3IAMPrincipal, policyName, document string) error { + _, err := putIAMUserPolicy(root, &iam.PutUserPolicyInput{ + UserName: aws.String(principal.name), + PolicyName: aws.String(policyName), + PolicyDocument: aws.String(document), + }) + return err +} + +// deleteS3IAMUser removes every dependency DeleteUser would otherwise reject +// — inline policies and access keys — before deleting the user. The existing +// deleteIAMUserAndPolicies/deleteIAMUserAndAccessKeys helpers each cover +// only one of the two, and these fixtures always create both. +func deleteS3IAMUser(root *iam.Client, userName string) error { + polOut, err := listIAMUserPolicies(root, &iam.ListUserPoliciesInput{UserName: aws.String(userName)}) + if err != nil { + return err + } + for _, name := range polOut.PolicyNames { + if err := deleteIAMUserPolicy(root, userName, name); err != nil { + return err + } + } + + keyOut, err := listIAMAccessKeys(root, &iam.ListAccessKeysInput{UserName: aws.String(userName)}) + if err != nil { + return err + } + for _, key := range keyOut.AccessKeyMetadata { + if err := deleteIAMAccessKey(root, userName, aws.ToString(key.AccessKeyId)); err != nil { + return err + } + } + + return deleteIAMUser(root, userName) +} + +// putBucketPolicyDoc installs a bucket policy as root. Statements are built +// with bucketStatement so a test's intent stays readable and a typo becomes +// a compile error rather than a silently-malformed document. +func putBucketPolicyDoc(s *S3Conf, bucket string, statements ...bucketStatement) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + defer cancel() + + doc := bucketPolicyDoc(statements...) + _, err := s.GetClient().PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ + Bucket: &bucket, + Policy: &doc, + }) + return err +} + +// bucketStatement is one S3 bucket-policy statement, built as a typed value +// rather than a formatted JSON string so a test typo is a compile error. +// It mirrors accessStatement (iam_access_control.go) for identity policies; +// the difference is Principal, which bucket policies require and identity +// policies forbid. +// +// Principal is matched against the caller's raw access key by this gateway +// (auth.Principals.Contains) — deliberately not against an ARN, for +// compatibility with the non-IAM backends that have no ARNs at all. A +// long-term user is therefore named by its AKIA… access key. An assumed-role +// session cannot be named at all: its ASIA… key is ephemeral, so +// auth.IAMService.ResolveAccounts rejects it outright rather than let a bucket +// policy come to reference a principal that stops existing. Session tests +// use "*" for that reason. +type bucketStatement struct { + Sid string `json:"Sid,omitempty"` + Effect string `json:"Effect"` + Principal any `json:"Principal"` + Action any `json:"Action"` + Resource any `json:"Resource"` + Condition json.RawMessage `json:"Condition,omitempty"` +} + +// bucketPolicyDoc marshals statements into a complete bucket-policy +// document. Marshaling a fixed struct of strings cannot fail in practice; a +// panic here means a test itself is malformed. +func bucketPolicyDoc(statements ...bucketStatement) string { + doc := struct { + Version string `json:"Version"` + Statement []bucketStatement `json:"Statement"` + }{"2012-10-17", statements} + b, err := json.Marshal(doc) + if err != nil { + panic(fmt.Sprintf("s3_iam_utils: bucketPolicyDoc: %v", err)) + } + return string(b) +} + +// bucketArn and objectArn build the resource ARNs an S3 policy statement +// names, matching how the gateway builds the resource it evaluates against. +func bucketArn(bucket string) string { return "arn:aws:s3:::" + bucket } +func objectArn(bucket, key string) string { return "arn:aws:s3:::" + bucket + "/" + key } +func objectsArn(bucket string) string { return "arn:aws:s3:::" + bucket + "/*" } + +// wantExplicitIdentityDeny, wantExplicitResourceDeny and wantImplicitDeny +// name the three denial shapes VerifyAccess produces. All three share Code +// AccessDenied and HTTP 403 and differ only in message text, which is +// exactly why these tests assert on the full message: a test checking only +// the code could not tell an identity-policy deny from a bucket-policy one, +// and the precedence between them is the whole point of this group. +func wantExplicitIdentityDeny(principal, action, resourceArn string) s3err.S3Error { + return s3err.GetExplicitDenyAccessErr(principal, action, resourceArn, "an identity-based policy") +} + +func wantExplicitResourceDeny(principal, action, resourceArn string) s3err.S3Error { + return s3err.GetExplicitDenyAccessErr(principal, action, resourceArn, "a resource-based policy") +} + +func wantImplicitDeny(principal, action, resourceArn string) s3err.S3Error { + return s3err.GetImplicitDenyAccessErr(principal, action, resourceArn) +} + +// s3ClientWithSessionCreds builds an *s3.Client authenticated with a full +// access/secret/session-token triple, for the assumed-role session tests. +func s3ClientWithSessionCreds(s *S3Conf, access, secret, token string) *s3.Client { + conf := *s + conf.awsID = access + conf.awsSecret = secret + + cfg := conf.Config() + cfg.Credentials = credentials.NewStaticCredentialsProvider(access, secret, token) + return s3.NewFromConfig(cfg, func(o *s3.Options) { + if s.hostStyle { + o.BaseEndpoint = &s.endpoint + o.UsePathStyle = false + } + }) +} + +// s3ConditionCase is one row of a table-driven condition test: the Condition +// block to attach to an otherwise-unconditional GetObject Allow, and whether +// it should grant. +type s3ConditionCase struct { + name string + condition []byte + wantAllowed bool +} + +// runS3ConditionCases attaches each case's condition to a fresh user's +// GetObject Allow and checks whether the resulting request is authorized. +// A failing condition voids the statement entirely, leaving nothing to +// grant — hence the implicit-deny expectation rather than an explicit one. +func runS3ConditionCases(root *iam.Client, s *S3Conf, bucket, key string, cases []s3ConditionCase) error { + user, cleanup, err := newS3IAMUser(root, s, nil) + if err != nil { + return err + } + defer cleanup() + + for _, tc := range cases { + if err := func() error { + if err := putS3IAMUserPolicy(root, user, "p", policyDoc(accessStatement{ + Effect: "Allow", Action: actS3GetObject, Resource: objectsArn(bucket), + Condition: tc.condition, + })); err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = user.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: getPtr(key)}) + cancel() + if tc.wantAllowed { + if err != nil { + return fmt.Errorf("expected the request to be allowed: %w", err) + } + return nil + } + return checkApiErr(err, wantImplicitDeny(user.arn, actS3GetObject, objectArn(bucket, key))) + }(); err != nil { + return fmt.Errorf("%s: %w", tc.name, err) + } + } + return nil +} + +func deleteObjectsWithBypass(client *s3.Client, bucket string, keys ...string) (*s3.DeleteObjectsOutput, error) { + objects := make([]types.ObjectIdentifier, len(keys)) + for i, key := range keys { + objects[i] = types.ObjectIdentifier{Key: aws.String(key)} + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + defer cancel() + return client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{Objects: objects}, + BypassGovernanceRetention: aws.Bool(true), + }) +} + +// checkDeleteObjectsErr checks one DeleteObjects response entry against the +// key and denial it's expected to carry. +func checkDeleteObjectsErr(got types.Error, wantKey string, wantErr s3err.S3Error) error { + if got.Key == nil || *got.Key != wantKey { + return fmt.Errorf("expected the per-object error to be for key %q, got %+v", wantKey, got) + } + base := wantErr.BaseError() + if got.Code == nil || *got.Code != base.Code { + return fmt.Errorf("expected error code %q for key %q, got %+v", base.Code, wantKey, got) + } + if got.Message == nil || *got.Message != base.Description { + return fmt.Errorf("expected error message %q for key %q, got %+v", base.Description, wantKey, got) + } + return nil +} + +// checkDeletedKeysInOrder checks that a DeleteObjects response's Deleted +// list names exactly wantKeys, in that order — DeleteObjects preserves the +// order objects were requested in across both the Deleted and Error lists. +func checkDeletedKeysInOrder(got []types.DeletedObject, wantKeys []string) error { + if len(got) != len(wantKeys) { + return fmt.Errorf("expected %d deleted objects %v, got %+v", len(wantKeys), wantKeys, got) + } + for i, want := range wantKeys { + if got[i].Key == nil || *got[i].Key != want { + return fmt.Errorf("expected deleted object %d to be %q, got %+v", i, want, got) + } + } + return nil +} + +// putGovernanceLockedObject writes an object under GOVERNANCE retention, as +// root, for the bypass-permission tests to then try to delete. +func putGovernanceLockedObject(s *S3Conf, bucket, key string) error { + retainUntil := time.Now().UTC().Add(time.Hour) + _, err := putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + ObjectLockMode: types.ObjectLockModeGovernance, + ObjectLockRetainUntilDate: &retainUntil, + }, s.GetClient()) + return err +} + +// deleteBucketPolicyIfAny clears the bucket policy for a sub-case that needs +// the resource side silent, tolerating there being none to delete — the +// table-driven tests reuse one bucket across cases rather than paying for a +// fresh bucket per row. +func deleteBucketPolicyIfAny(s *S3Conf, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + defer cancel() + + _, err := s.GetClient().DeleteBucketPolicy(ctx, &s3.DeleteBucketPolicyInput{Bucket: &bucket}) + if err != nil && checkSdkApiErr(err, "NoSuchBucketPolicy") == nil { + return nil + } + return err +} + +// gitHubOIDCSkipReason explains, in the skip message, why a run outside the +// OIDC workflow can't exercise any of this. +const gitHubOIDCSkipReason = "ACTIONS_ID_TOKEN_REQUEST_URL/ACTIONS_ID_TOKEN_REQUEST_TOKEN not set " + + "(expected outside a GitHub Actions job with id-token: write permission)" + +var ( + gitHubOIDCTokenOnce sync.Once + gitHubOIDCTokenVal string + gitHubOIDCTokenOK bool +) + +// gitHubOIDCToken fetches one real ID token for the whole group and reuses +// it. Every test needs a token, and they all want the same audience and the +// same repo subject, so fetching one per test would only add round trips to +// GitHub's runtime endpoint for no additional coverage. +func gitHubOIDCToken() (string, bool) { + gitHubOIDCTokenOnce.Do(func() { + reqURL := os.Getenv("ACTIONS_ID_TOKEN_REQUEST_URL") + reqToken := os.Getenv("ACTIONS_ID_TOKEN_REQUEST_TOKEN") + if reqURL == "" || reqToken == "" { + return + } + token, err := fetchGitHubIDToken(reqURL, reqToken, githubOIDCTestAudience) + if err != nil { + // The error is deliberately not propagated as a token: a fetch + // failure inside the workflow shows up as every test failing to + // assume a role, with the reason on the first one. + return + } + gitHubOIDCTokenVal, gitHubOIDCTokenOK = token, true + }) + return gitHubOIDCTokenVal, gitHubOIDCTokenOK +} + +// s3IAMSessionActionHandler is s3IAMActionHandler that first skips the test +// when no GitHub OIDC token can be minted — which is every environment but +// the one workflow holding id-token: write permission. +func s3IAMSessionActionHandler(s *S3Conf, testName string, handler func(root *iam.Client, bucket string) error, opts ...setupOpt) error { + if _, ok := gitHubOIDCToken(); !ok { + skipF("%v: %v", testName, gitHubOIDCSkipReason) + return nil + } + return s3IAMActionHandler(s, testName, handler, opts...) +} + +// newGitHubSession registers a throwaway OIDC provider for GitHub Actions' +// issuer and a role trusting it, attaches rolePolicies as the role's inline +// permission policies, then assumes it with a real ID token and (when +// sessionPolicy is non-empty) an inline session policy. +// +// The returned principal's name is the role name, so a test can put another +// role policy on it or delete the role mid-test; arn is the assumed-role +// session ARN, which is what a denial message names. +func newGitHubSession(root *iam.Client, s *S3Conf, rolePolicies map[string]string, sessionPolicy string) (*s3IAMPrincipal, func(), error) { + token, ok := gitHubOIDCToken() + if !ok { + return nil, nil, fmt.Errorf("no GitHub OIDC token available") + } + repo := os.Getenv("GITHUB_REPOSITORY") + if repo == "" { + return nil, nil, fmt.Errorf("GITHUB_REPOSITORY is not set, but the OIDC token request variables are - unexpected environment") + } + + roleName, _, cleanup, err := createGitHubOIDCTrust(root, repo) + if err != nil { + return nil, nil, err + } + + for name, doc := range rolePolicies { + if _, err := putIAMRolePolicy(root, &iam.PutRolePolicyInput{ + RoleName: aws.String(roleName), PolicyName: aws.String(name), PolicyDocument: aws.String(doc), + }); err != nil { + cleanup() + return nil, nil, fmt.Errorf("attach role policy %q: %w", name, err) + } + } + + sessionName := "s3-sess-" + genRandString(8) + out, err := assumeRoleWithWebIdentitySessionPolicy(s, roleArnFor(roleName), sessionName, token, sessionPolicy) + if err != nil { + cleanup() + return nil, nil, fmt.Errorf("AssumeRoleWithWebIdentity: %w", err) + } + + access := aws.ToString(out.Credentials.AccessKeyId) + secret := aws.ToString(out.Credentials.SecretAccessKey) + sessionToken := aws.ToString(out.Credentials.SessionToken) + + conf := *s + conf.awsID = access + conf.awsSecret = secret + + principal := &s3IAMPrincipal{ + name: roleName, + arn: aws.ToString(out.AssumedRoleUser.Arn), + conf: conf, + client: s3ClientWithSessionCreds(s, access, secret, sessionToken), + sessionToken: sessionToken, + } + // The role may already have been deleted by the test itself + // (S3IAMSession_deleted_role_denies); cleanup tolerates that. + return principal, cleanup, nil +} + +// assumeRoleWithWebIdentitySessionPolicy is assumeRoleWithWebIdentity with +// the optional inline session-policy parameter, which no other test in this +// package needs. +func assumeRoleWithWebIdentitySessionPolicy(s *S3Conf, roleArn, sessionName, token, sessionPolicy string) (*sts.AssumeRoleWithWebIdentityOutput, error) { + input := &sts.AssumeRoleWithWebIdentityInput{ + RoleArn: aws.String(roleArn), + RoleSessionName: aws.String(sessionName), + WebIdentityToken: aws.String(token), + } + if sessionPolicy != "" { + input.Policy = aws.String(sessionPolicy) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + defer cancel() + return s.GetSTSClient().AssumeRoleWithWebIdentity(ctx, input) +} + +// roleArnFor builds the ARN of a role in this gateway's single fixed +// account. +func roleArnFor(roleName string) string { + return "arn:aws:iam::" + testAccountID + ":role/" + roleName +} + +// sessionNameFor recovers the session name from an assumed-role ARN, whose +// last path element it is. +func sessionNameFor(p *s3IAMPrincipal) string { + idx := strings.LastIndex(p.arn, "/") + if idx < 0 { + return "" + } + return p.arn[idx+1:] +} + +// deleteObjectBypassingGovernance deletes one object with the +// bypass-governance-retention header set. +func deleteObjectBypassingGovernance(client *s3.Client, bucket, key string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + defer cancel() + _, err := client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &key, + BypassGovernanceRetention: aws.Bool(true), + }) + return err +} diff --git a/tests/integration/s3conf.go b/tests/integration/s3conf.go index d47acb4d..41c5ef5b 100644 --- a/tests/integration/s3conf.go +++ b/tests/integration/s3conf.go @@ -38,6 +38,7 @@ type S3Conf struct { awsSecret string awsRegion string endpoint string + iamEndpoint string websiteScheme string websiteDomain string websitePort string @@ -94,6 +95,13 @@ func WithRegion(r string) Option { func WithEndpoint(e string) Option { return func(s *S3Conf) { s.endpoint = e } } + +// WithIAMEndpoint points the IAM/STS clients at a standalone IAM service +// separate from the S3 endpoint, for the test groups that drive both +// processes at once +func WithIAMEndpoint(e string) Option { + return func(s *S3Conf) { s.iamEndpoint = e } +} func WithWebsiteScheme(scheme string) Option { return func(s *S3Conf) { s.websiteScheme = scheme } } @@ -156,12 +164,22 @@ func (c *S3Conf) GetClient() *s3.Client { } func (c *S3Conf) GetIAMClient() *iam.Client { - return iam.NewFromConfig(c.Config()) + return iam.NewFromConfig(c.iamConfig()) } // GetSTSClient returns an SDK client for STS actions func (c *S3Conf) GetSTSClient() *sts.Client { - return sts.NewFromConfig(c.Config()) + return sts.NewFromConfig(c.iamConfig()) +} + +// iamConfig is Config with the base endpoint pointed at the IAM service +// when one was configured separately from the S3 endpoint. +func (c *S3Conf) iamConfig() aws.Config { + cfg := c.Config() + if c.iamEndpoint != "" { + cfg.BaseEndpoint = &c.iamEndpoint + } + return cfg } func (c *S3Conf) GetPresignClient() *s3.PresignClient { diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 48e5f63d..93f7d2cf 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -2120,7 +2120,7 @@ func checkWORMProtection(client *s3.Client, bucket, object string) error { } ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + out, err := client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ Bucket: &bucket, Delete: &types.Delete{ Objects: []types.ObjectIdentifier{ @@ -2131,7 +2131,13 @@ func checkWORMProtection(client *s3.Client, bucket, object string) error { }, }) cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + if err != nil { + return fmt.Errorf("expected DeleteObjects to succeed with a per-object denial, not fail outright: %w", err) + } + if len(out.Errors) != 1 { + return fmt.Errorf("expected exactly 1 per-object error, got %+v", out.Errors) + } + if err := checkDeleteObjectsErr(out.Errors[0], object, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { return err } @@ -2841,6 +2847,18 @@ const ( maxDelObjWorkers int64 = 20 // Maximum number of concurrent delete workers maxRetryAttempts int = 3 // Maximum retries for object deletion lockWaitTime time.Duration = time.Second * 3 // Wait time for lock expiration before retrying delete + + // complianceTestRetention is how far out a test should set a COMPLIANCE + // retention it means to clean up afterwards. A COMPLIANCE retention can + // never be shortened or removed — by anyone, with any permission, which + // is the whole point of the mode — so the only way to release the object + // is to outlast it, which cleanupLockedObjects does. Long enough for the + // test body to run against a genuinely locked object, short enough to + // wait out. + complianceTestRetention time.Duration = time.Second * 10 + // maxComplianceCleanupWait bounds that wait, so a test that locks an + // object for hours fails quickly and clearly instead of hanging. + maxComplianceCleanupWait time.Duration = time.Second * 30 ) // cleanupLockedObjects removes objects from a bucket that may be protected by @@ -2889,21 +2907,28 @@ func cleanupLockedObjects(client *s3.Client, bucket string, objs []objToDelete) } } - // Apply temporary retention policy to allow deletion - // RetainUntilDate is set a few seconds in the future to handle network delays - retDate := time.Now().Add(lockWaitTime) - mode := types.ObjectLockRetentionModeGovernance + // A COMPLIANCE retention can only be waited out — it cannot be + // shortened or removed by anyone. Tests that mean to clean up + // therefore lock for complianceTestRetention, and this sleeps + // until that has passed. if obj.isCompliance { - mode = types.ObjectLockRetentionModeCompliance + return waitOutComplianceRetention(client, bucket, obj) } + // A GOVERNANCE retention can be weakened with the bypass + // permission and header, so shorten it to a few seconds out + // rather than waiting for the original date. The margin absorbs + // network delay. + retDate := time.Now().Add(lockWaitTime) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ - Bucket: &bucket, - Key: &obj.key, - VersionId: getPtr(obj.versionId), + Bucket: &bucket, + Key: &obj.key, + VersionId: getPtr(obj.versionId), + BypassGovernanceRetention: getBoolPtr(true), Retention: &types.ObjectLockRetention{ - Mode: mode, + Mode: types.ObjectLockRetentionModeGovernance, RetainUntilDate: &retDate, }, }) @@ -2930,6 +2955,71 @@ func cleanupLockedObjects(client *s3.Client, bucket string, objs []objToDelete) return eg.Wait() } +// waitOutComplianceRetention blocks until obj's retention has passed, the +// only way to release a COMPLIANCE-locked object. A retention further out +// than maxComplianceCleanupWait is reported as an error rather than waited +// on: such an object cannot be cleaned up within a test run at all, and the +// test that locked it should either use complianceTestRetention or skip +// teardown. +func waitOutComplianceRetention(client *s3.Client, bucket string, obj objToDelete) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ + Bucket: &bucket, + Key: &obj.key, + VersionId: getPtr(obj.versionId), + }) + cancel() + + // Already deleted: nothing to wait for. + if err != nil && checkSdkApiErr(err, "NoSuchKey") == nil { + return nil + } + + // No retention of its own means the object is protected only by the + // bucket's default retention. Nothing forbids giving it a short one of + // its own — an object-level retention supersedes the bucket default, and + // there is no existing retention here to weaken — so that is how such an + // object gets released. + noRetention := err != nil && checkSdkApiErr(err, "NoSuchObjectLockConfiguration") == nil + if err != nil && !noRetention { + return err + } + if !noRetention && (out.Retention == nil || out.Retention.RetainUntilDate == nil) { + noRetention = true + } + if noRetention { + retDate := time.Now().Add(lockWaitTime) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj.key, + VersionId: getPtr(obj.versionId), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + RetainUntilDate: &retDate, + }, + }) + cancel() + if err != nil && checkSdkApiErr(err, "NoSuchKey") != nil { + return err + } + time.Sleep(lockWaitTime) + return nil + } + + // The extra second absorbs clock skew between this process and the + // gateway, which compares the retention against its own clock. + wait := time.Until(*out.Retention.RetainUntilDate) + time.Second + if wait > maxComplianceCleanupWait { + return fmt.Errorf("object %q is under COMPLIANCE retention until %v, too far out to wait for: use complianceTestRetention, or skip teardown", + obj.key, out.Retention.RetainUntilDate) + } + if wait > 0 { + time.Sleep(wait) + } + return nil +} + type objectLockMode string const ( @@ -3735,3 +3825,21 @@ func hexBytes(s string) string { } return strings.Join(parts, " ") } + +// checkDeleteObjectsErrsInOrder checks that got names exactly the (key, +// error) pairs in want, in that order — DeleteObjects preserves the order +// objects were requested in across both the Deleted and Error lists. +func checkDeleteObjectsErrsInOrder(got []types.Error, want []struct { + key string + err s3err.S3Error +}) error { + if len(got) != len(want) { + return fmt.Errorf("expected %d per-object errors, got %d: %+v", len(want), len(got), got) + } + for i, w := range want { + if err := checkDeleteObjectsErr(got[i], w.key, w.err); err != nil { + return fmt.Errorf("error %d: %w", i, err) + } + } + return nil +} diff --git a/tests/integration/versioning.go b/tests/integration/versioning.go index 2662a45a..6fe70238 100644 --- a/tests/integration/versioning.go +++ b/tests/integration/versioning.go @@ -2629,7 +2629,7 @@ func Versioning_WORM_obj_version_locked_with_compliance_retention(s *S3Conf) err } version := objVersions[0] - rDate := time.Now().Add(time.Hour * 48) + rDate := time.Now().Add(2 * complianceTestRetention) ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ Bucket: &bucket, @@ -2820,7 +2820,7 @@ func Versioning_WORM_delete_marker_locked_object_compliance_retention(s *S3Conf) Key: &obj, Retention: &types.ObjectLockRetention{ Mode: types.ObjectLockRetentionModeCompliance, - RetainUntilDate: getPtr(time.Now().AddDate(1, 0, 0)), + RetainUntilDate: getPtr(time.Now().Add(complianceTestRetention)), }, }) cancel() diff --git a/website/handler.go b/website/handler.go index c84d7623..b4965d3b 100644 --- a/website/handler.go +++ b/website/handler.go @@ -331,7 +331,7 @@ func resolveIndexKey(key string, config *s3response.WebsiteConfiguration) string } func (c *websiteController) getObject(ctx fiber.Ctx, bucket, key string) websiteResult { - if err := auth.VerifyPublicAccess(ctx.RequestCtx(), c.be, auth.GetObjectAction, auth.PermissionRead, bucket, key); err != nil { + if err := auth.VerifyPublicAccess(ctx, c.be, auth.GetObjectAction, auth.PermissionRead, bucket, key); err != nil { return websiteResult{ Key: key, StatusCode: statusCodeFromError(err), @@ -364,7 +364,7 @@ func (c *websiteController) getObject(ctx fiber.Ctx, bucket, key string) website } func (c *websiteController) headObject(ctx fiber.Ctx, bucket, key string) websiteResult { - if err := auth.VerifyPublicAccess(ctx.RequestCtx(), c.be, auth.GetObjectAction, auth.PermissionRead, bucket, key); err != nil { + if err := auth.VerifyPublicAccess(ctx, c.be, auth.GetObjectAction, auth.PermissionRead, bucket, key); err != nil { return websiteResult{ Key: key, StatusCode: statusCodeFromError(err), diff --git a/website/server.go b/website/server.go index a82a31cb..fb8b5a3b 100644 --- a/website/server.go +++ b/website/server.go @@ -24,14 +24,14 @@ import ( "github.com/gofiber/fiber/v3/middleware/recover" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/debuglogger" + "github.com/versity/versitygw/internal/netutil" "github.com/versity/versitygw/s3api/middlewares" - "github.com/versity/versitygw/s3api/utils" ) // Server is the static website hosting endpoint. type Server struct { app *fiber.App - CertStorage *utils.CertStorage + CertStorage *netutil.CertStorage domain string quiet bool socketPerm os.FileMode @@ -46,7 +46,7 @@ func WithQuiet() Option { } // WithTLS sets TLS credentials. -func WithTLS(cs *utils.CertStorage) Option { +func WithTLS(cs *netutil.CertStorage) Option { return func(s *Server) { s.CertStorage = cs } } @@ -116,9 +116,9 @@ func (s *Server) ServeMultiPort(ports []string) error { var err error if s.CertStorage != nil { - ln, err = utils.NewMultiAddrTLSListener(fiber.NetworkTCP, addrSpec, s.CertStorage.GetCertificate, utils.ListenerOptions{SocketPerm: s.socketPerm}) + ln, err = netutil.NewMultiAddrTLSListener(fiber.NetworkTCP, addrSpec, s.CertStorage.GetCertificate, netutil.ListenerOptions{SocketPerm: s.socketPerm}) } else { - ln, err = utils.NewMultiAddrListener(fiber.NetworkTCP, addrSpec, utils.ListenerOptions{SocketPerm: s.socketPerm}) + ln, err = netutil.NewMultiAddrListener(fiber.NetworkTCP, addrSpec, netutil.ListenerOptions{SocketPerm: s.socketPerm}) } if err != nil { @@ -132,7 +132,7 @@ func (s *Server) ServeMultiPort(ports []string) error { return fmt.Errorf("failed to create any website listeners") } - finalListener := utils.NewMultiListener(listeners...) + finalListener := netutil.NewMultiListener(listeners...) return s.app.Listener(finalListener, fiber.ListenConfig{ DisableStartupMessage: true, diff --git a/webui/webserver.go b/webui/webserver.go index 92970e68..37bfd5db 100644 --- a/webui/webserver.go +++ b/webui/webserver.go @@ -26,7 +26,7 @@ import ( "github.com/gofiber/fiber/v3/middleware/logger" "github.com/gofiber/fiber/v3/middleware/recover" "github.com/gofiber/fiber/v3/middleware/static" - "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/internal/netutil" ) // ServerConfig holds the server configuration @@ -40,7 +40,7 @@ type ServerConfig struct { // Server is the main GUI server type Server struct { app *fiber.App - CertStorage *utils.CertStorage + CertStorage *netutil.CertStorage config *ServerConfig pathPrefix string quiet bool @@ -56,7 +56,7 @@ func WithQuiet() Option { } // WithTLS sets TLS Credentials -func WithTLS(cs *utils.CertStorage) Option { +func WithTLS(cs *netutil.CertStorage) Option { return func(s *Server) { s.CertStorage = cs } } @@ -187,9 +187,9 @@ func (s *Server) ServeMultiPort(ports []string) error { var err error if s.CertStorage != nil { - ln, err = utils.NewMultiAddrTLSListener(fiber.NetworkTCP, addrSpec, s.CertStorage.GetCertificate, utils.ListenerOptions{SocketPerm: s.socketPerm}) + ln, err = netutil.NewMultiAddrTLSListener(fiber.NetworkTCP, addrSpec, s.CertStorage.GetCertificate, netutil.ListenerOptions{SocketPerm: s.socketPerm}) } else { - ln, err = utils.NewMultiAddrListener(fiber.NetworkTCP, addrSpec, utils.ListenerOptions{SocketPerm: s.socketPerm}) + ln, err = netutil.NewMultiAddrListener(fiber.NetworkTCP, addrSpec, netutil.ListenerOptions{SocketPerm: s.socketPerm}) } if err != nil { @@ -204,7 +204,7 @@ func (s *Server) ServeMultiPort(ports []string) error { } // Combine all listeners - finalListener := utils.NewMultiListener(listeners...) + finalListener := netutil.NewMultiListener(listeners...) return s.app.Listener(finalListener, fiber.ListenConfig{ DisableStartupMessage: true,