From ac5c3b4a86d5ea813bc35fa3c121e22f4d899401 Mon Sep 17 00:00:00 2001 From: Marc Singer Date: Sun, 5 Apr 2026 11:34:46 +0200 Subject: [PATCH 1/4] Add WebsiteConfiguration types, validation, and S3 error codes Add S3 bucket website configuration types with XML serialization support in s3response/website.go. Includes IndexDocument, ErrorDocument, RedirectAllRequestsTo, and RoutingRules with full validation matching AWS S3 behavior. Add corresponding S3 error codes: ErrNoSuchWebsiteConfiguration, ErrInvalidWebsiteConfiguration, ErrInvalidWebsiteSuffix, and ErrInvalidWebsiteRedirectCode. Unit tests cover validation logic, XML round-trip, and parsing. Signed-off-by: Marc Singer Add website backend interface and implementations for posix and s3proxy Add PutBucketWebsite, GetBucketWebsite, and DeleteBucketWebsite methods to the Backend interface with BackendUnsupported stubs that return ErrNotImplemented. Posix backend stores website config as a metadata attribute (key: 'website') following the same pattern as CORS. ScoutFS inherits via embedding. S3Proxy backend stores website config in the metadata bucket with prefix 'vgw-meta-website-', consistent with existing ACL/policy/CORS metadata storage. Returns ErrNoSuchWebsiteConfiguration when not found. Signed-off-by: Marc Singer Add website API controllers and wire into router Add PutBucketWebsite, GetBucketWebsite, and DeleteBucketWebsite controller methods following the same pattern as CORS. Controllers parse and validate WebsiteConfiguration XML, check IAM authorization, and delegate to the backend. Replace the three HandleErrorRoute(ErrNotImplemented) stubs in the router with the new controller methods. Regenerate the backend mock to include the new interface methods. Signed-off-by: Marc Singer Add website index document middleware and wire into router Add ResolveWebsiteIndex middleware that rewrites directory-like object keys (empty or ending with /) to include the IndexDocument suffix when website hosting is enabled. Also handles RedirectAllRequestsTo by returning 301. Wire the middleware into both GetObject and HeadObject handler chains in the router, positioned after BucketObjectNameValidator and before auth. Signed-off-by: Marc Singer --- backend/backend.go | 12 ++ backend/posix/posix.go | 72 +++++++ backend/s3proxy/s3.go | 35 +++- s3api/controllers/backend_moq_test.go | 156 +++++++++++++++ s3api/controllers/bucket-delete.go | 36 ++++ s3api/controllers/bucket-get.go | 44 +++++ s3api/controllers/bucket-put.go | 56 ++++++ s3api/middlewares/website.go | 94 +++++++++ s3api/router.go | 8 +- s3err/s3err.go | 24 +++ s3response/website.go | 151 ++++++++++++++ s3response/website_test.go | 274 ++++++++++++++++++++++++++ 12 files changed, 956 insertions(+), 6 deletions(-) create mode 100644 s3api/middlewares/website.go create mode 100644 s3response/website.go create mode 100644 s3response/website_test.go diff --git a/backend/backend.go b/backend/backend.go index 021bca2e..9b992955 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -50,6 +50,9 @@ type Backend interface { PutBucketCors(_ context.Context, bucket string, cors []byte) error GetBucketCors(_ context.Context, bucket string) ([]byte, error) DeleteBucketCors(_ context.Context, bucket string) error + PutBucketWebsite(_ context.Context, bucket string, website []byte) error + GetBucketWebsite(_ context.Context, bucket string) ([]byte, error) + DeleteBucketWebsite(_ context.Context, bucket string) error // multipart operations CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) @@ -166,6 +169,15 @@ func (BackendUnsupported) GetBucketCors(_ context.Context, bucket string) ([]byt func (BackendUnsupported) DeleteBucketCors(_ context.Context, bucket string) error { return s3err.GetAPIError(s3err.ErrNotImplemented) } +func (BackendUnsupported) PutBucketWebsite(_ context.Context, bucket string, website []byte) error { + return s3err.GetAPIError(s3err.ErrNotImplemented) +} +func (BackendUnsupported) GetBucketWebsite(_ context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrNotImplemented) +} +func (BackendUnsupported) DeleteBucketWebsite(_ context.Context, bucket string) error { + return s3err.GetAPIError(s3err.ErrNotImplemented) +} func (BackendUnsupported) CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 9bff308a..1e34ff0d 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -132,6 +132,7 @@ const ( objectRetentionKey = "object-retention" objectLegalHoldKey = "object-legal-hold" corskey = "cors" + websitekey = "website" versioningKey = "versioning" deleteMarkerKey = "delete-marker" versionIdKey = "version-id" @@ -6168,6 +6169,77 @@ func (p *Posix) DeleteBucketCors(ctx context.Context, bucket string) error { return p.PutBucketCors(ctx, bucket, nil) } +func (p *Posix) PutBucketWebsite(ctx context.Context, bucket string, website []byte) error { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return err + } + defer release() + + if !p.isBucketValid(bucket) { + return s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + _, err = os.Stat(bucket) + if errors.Is(err, fs.ErrNotExist) { + return s3err.GetAPIError(s3err.ErrNoSuchBucket) + } + if err != nil { + return fmt.Errorf("stat bucket: %w", err) + } + + if website == nil { + err = p.meta.DeleteAttribute(bucket, "", websitekey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return fmt.Errorf("remove website: %w", err) + } + + return nil + } + + err = p.meta.StoreAttribute(nil, bucket, "", websitekey, website) + if err != nil { + return fmt.Errorf("set website: %w", err) + } + + return nil +} + +func (p *Posix) GetBucketWebsite(ctx context.Context, bucket string) ([]byte, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return nil, err + } + defer release() + + if !p.isBucketValid(bucket) { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + _, err = os.Stat(bucket) + if errors.Is(err, fs.ErrNotExist) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) + } + if err != nil { + return nil, fmt.Errorf("stat bucket: %w", err) + } + + website, err := p.meta.RetrieveAttribute(nil, bucket, "", websitekey) + if errors.Is(err, meta.ErrNoSuchKey) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchWebsiteConfiguration) + } + if err != nil { + return nil, err + } + + return website, nil +} + +func (p *Posix) DeleteBucketWebsite(ctx context.Context, bucket string) error { + if !p.isBucketValid(bucket) { + return s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + return p.PutBucketWebsite(ctx, bucket, nil) +} + func (p *Posix) isBucketObjectLockEnabled(bucket string) error { cfg, err := p.meta.RetrieveAttribute(nil, bucket, "", bucketLockKey) if errors.Is(err, fs.ErrNotExist) { diff --git a/backend/s3proxy/s3.go b/backend/s3proxy/s3.go index 715ad959..cb662055 100644 --- a/backend/s3proxy/s3.go +++ b/backend/s3proxy/s3.go @@ -37,9 +37,10 @@ import ( type metaPrefix string const ( - metaPrefixAcl metaPrefix = "vgw-meta-acl-" - metaPrefixPolicy metaPrefix = "vgw-meta-policy-" - metaPrefixCors metaPrefix = "vgw-meta-cors-" + metaPrefixAcl metaPrefix = "vgw-meta-acl-" + metaPrefixPolicy metaPrefix = "vgw-meta-policy-" + metaPrefixCors metaPrefix = "vgw-meta-cors-" + metaPrefixWebsite metaPrefix = "vgw-meta-website-" ) type S3Proxy struct { @@ -1614,6 +1615,32 @@ func (s *S3Proxy) DeleteBucketCors(ctx context.Context, bucket string) error { return nil } +func (s *S3Proxy) PutBucketWebsite(ctx context.Context, bucket string, website []byte) error { + return handleError(s.putMetaBucketObj(ctx, bucket, website, metaPrefixWebsite)) +} + +func (s *S3Proxy) GetBucketWebsite(ctx context.Context, bucket string) ([]byte, error) { + data, err := s.getMetaBucketObjData(ctx, bucket, metaPrefixWebsite, false) + if err != nil { + return nil, handleError(err) + } + + return data, nil +} + +func (s *S3Proxy) DeleteBucketWebsite(ctx context.Context, bucket string) error { + key := getMetaKey(bucket, metaPrefixWebsite) + _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &s.metaBucket, + Key: &key, + }) + if err != nil && !areErrSame(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) { + return handleError(err) + } + + return nil +} + func (s *S3Proxy) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error { return handleError(s.putMetaBucketObj(ctx, bucket, policy, metaPrefixPolicy)) } @@ -1772,6 +1799,8 @@ func handleMetaBucketObjectNotFoundErr(prefix metaPrefix) ([]byte, error) { return nil, s3err.GetBucketErr(s3err.ErrNoSuchBucketPolicy, "") case metaPrefixCors: return nil, s3err.GetBucketErr(s3err.ErrNoSuchCORSConfiguration, "") + case metaPrefixWebsite: + return nil, s3err.GetAPIError(s3err.ErrNoSuchWebsiteConfiguration) } return []byte{}, nil diff --git a/s3api/controllers/backend_moq_test.go b/s3api/controllers/backend_moq_test.go index 530df639..ba96058e 100644 --- a/s3api/controllers/backend_moq_test.go +++ b/s3api/controllers/backend_moq_test.go @@ -56,6 +56,9 @@ var _ backend.Backend = &BackendMock{} // DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error { // panic("mock out the DeleteBucketTagging method") // }, +// DeleteBucketWebsiteFunc: func(contextMoqParam context.Context, bucket string) error { +// panic("mock out the DeleteBucketWebsite method") +// }, // DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { // panic("mock out the DeleteObject method") // }, @@ -83,6 +86,9 @@ var _ backend.Backend = &BackendMock{} // GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) { // panic("mock out the GetBucketVersioning method") // }, +// GetBucketWebsiteFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { +// panic("mock out the GetBucketWebsite method") +// }, // GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error) { // panic("mock out the GetObject method") // }, @@ -152,6 +158,9 @@ var _ backend.Backend = &BackendMock{} // PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error { // panic("mock out the PutBucketVersioning method") // }, +// PutBucketWebsiteFunc: func(contextMoqParam context.Context, bucket string, website []byte) error { +// panic("mock out the PutBucketWebsite method") +// }, // PutObjectFunc: func(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error) { // panic("mock out the PutObject method") // }, @@ -228,6 +237,9 @@ type BackendMock struct { // DeleteBucketTaggingFunc mocks the DeleteBucketTagging method. DeleteBucketTaggingFunc func(contextMoqParam context.Context, bucket string) error + // DeleteBucketWebsiteFunc mocks the DeleteBucketWebsite method. + DeleteBucketWebsiteFunc func(contextMoqParam context.Context, bucket string) error + // DeleteObjectFunc mocks the DeleteObject method. DeleteObjectFunc func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) @@ -255,6 +267,9 @@ type BackendMock struct { // GetBucketVersioningFunc mocks the GetBucketVersioning method. GetBucketVersioningFunc func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) + // GetBucketWebsiteFunc mocks the GetBucketWebsite method. + GetBucketWebsiteFunc func(contextMoqParam context.Context, bucket string) ([]byte, error) + // GetObjectFunc mocks the GetObject method. GetObjectFunc func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error) @@ -324,6 +339,9 @@ type BackendMock struct { // PutBucketVersioningFunc mocks the PutBucketVersioning method. PutBucketVersioningFunc func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error + // PutBucketWebsiteFunc mocks the PutBucketWebsite method. + PutBucketWebsiteFunc func(contextMoqParam context.Context, bucket string, website []byte) error + // PutObjectFunc mocks the PutObject method. PutObjectFunc func(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error) @@ -443,6 +461,13 @@ type BackendMock struct { // Bucket is the bucket argument value. Bucket string } + // DeleteBucketWebsite holds details about calls to the DeleteBucketWebsite method. + DeleteBucketWebsite []struct { + // ContextMoqParam is the contextMoqParam argument value. + ContextMoqParam context.Context + // Bucket is the bucket argument value. + Bucket string + } // DeleteObject holds details about calls to the DeleteObject method. DeleteObject []struct { // ContextMoqParam is the contextMoqParam argument value. @@ -510,6 +535,13 @@ type BackendMock struct { // Bucket is the bucket argument value. Bucket string } + // GetBucketWebsite holds details about calls to the GetBucketWebsite method. + GetBucketWebsite []struct { + // ContextMoqParam is the contextMoqParam argument value. + ContextMoqParam context.Context + // Bucket is the bucket argument value. + Bucket string + } // GetObject holds details about calls to the GetObject method. GetObject []struct { // ContextMoqParam is the contextMoqParam argument value. @@ -693,6 +725,15 @@ type BackendMock struct { // Status is the status argument value. Status types.BucketVersioningStatus } + // PutBucketWebsite holds details about calls to the PutBucketWebsite method. + PutBucketWebsite []struct { + // ContextMoqParam is the contextMoqParam argument value. + ContextMoqParam context.Context + // Bucket is the bucket argument value. + Bucket string + // Website is the website argument value. + Website []byte + } // PutObject holds details about calls to the PutObject method. PutObject []struct { // ContextMoqParam is the contextMoqParam argument value. @@ -801,6 +842,7 @@ type BackendMock struct { lockDeleteBucketOwnershipControls sync.RWMutex lockDeleteBucketPolicy sync.RWMutex lockDeleteBucketTagging sync.RWMutex + lockDeleteBucketWebsite sync.RWMutex lockDeleteObject sync.RWMutex lockDeleteObjectTagging sync.RWMutex lockDeleteObjects sync.RWMutex @@ -810,6 +852,7 @@ type BackendMock struct { lockGetBucketPolicy sync.RWMutex lockGetBucketTagging sync.RWMutex lockGetBucketVersioning sync.RWMutex + lockGetBucketWebsite sync.RWMutex lockGetObject sync.RWMutex lockGetObjectAcl sync.RWMutex lockGetObjectAttributes sync.RWMutex @@ -833,6 +876,7 @@ type BackendMock struct { lockPutBucketPolicy sync.RWMutex lockPutBucketTagging sync.RWMutex lockPutBucketVersioning sync.RWMutex + lockPutBucketWebsite sync.RWMutex lockPutObject sync.RWMutex lockPutObjectAcl sync.RWMutex lockPutObjectLegalHold sync.RWMutex @@ -1251,6 +1295,42 @@ func (mock *BackendMock) DeleteBucketTaggingCalls() []struct { return calls } +// DeleteBucketWebsite calls DeleteBucketWebsiteFunc. +func (mock *BackendMock) DeleteBucketWebsite(contextMoqParam context.Context, bucket string) error { + if mock.DeleteBucketWebsiteFunc == nil { + panic("BackendMock.DeleteBucketWebsiteFunc: method is nil but Backend.DeleteBucketWebsite was just called") + } + callInfo := struct { + ContextMoqParam context.Context + Bucket string + }{ + ContextMoqParam: contextMoqParam, + Bucket: bucket, + } + mock.lockDeleteBucketWebsite.Lock() + mock.calls.DeleteBucketWebsite = append(mock.calls.DeleteBucketWebsite, callInfo) + mock.lockDeleteBucketWebsite.Unlock() + return mock.DeleteBucketWebsiteFunc(contextMoqParam, bucket) +} + +// DeleteBucketWebsiteCalls gets all the calls that were made to DeleteBucketWebsite. +// Check the length with: +// +// len(mockedBackend.DeleteBucketWebsiteCalls()) +func (mock *BackendMock) DeleteBucketWebsiteCalls() []struct { + ContextMoqParam context.Context + Bucket string +} { + var calls []struct { + ContextMoqParam context.Context + Bucket string + } + mock.lockDeleteBucketWebsite.RLock() + calls = mock.calls.DeleteBucketWebsite + mock.lockDeleteBucketWebsite.RUnlock() + return calls +} + // DeleteObject calls DeleteObjectFunc. func (mock *BackendMock) DeleteObject(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { if mock.DeleteObjectFunc == nil { @@ -1583,6 +1663,42 @@ func (mock *BackendMock) GetBucketVersioningCalls() []struct { return calls } +// GetBucketWebsite calls GetBucketWebsiteFunc. +func (mock *BackendMock) GetBucketWebsite(contextMoqParam context.Context, bucket string) ([]byte, error) { + if mock.GetBucketWebsiteFunc == nil { + panic("BackendMock.GetBucketWebsiteFunc: method is nil but Backend.GetBucketWebsite was just called") + } + callInfo := struct { + ContextMoqParam context.Context + Bucket string + }{ + ContextMoqParam: contextMoqParam, + Bucket: bucket, + } + mock.lockGetBucketWebsite.Lock() + mock.calls.GetBucketWebsite = append(mock.calls.GetBucketWebsite, callInfo) + mock.lockGetBucketWebsite.Unlock() + return mock.GetBucketWebsiteFunc(contextMoqParam, bucket) +} + +// GetBucketWebsiteCalls gets all the calls that were made to GetBucketWebsite. +// Check the length with: +// +// len(mockedBackend.GetBucketWebsiteCalls()) +func (mock *BackendMock) GetBucketWebsiteCalls() []struct { + ContextMoqParam context.Context + Bucket string +} { + var calls []struct { + ContextMoqParam context.Context + Bucket string + } + mock.lockGetBucketWebsite.RLock() + calls = mock.calls.GetBucketWebsite + mock.lockGetBucketWebsite.RUnlock() + return calls +} + // GetObject calls GetObjectFunc. func (mock *BackendMock) GetObject(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error) { if mock.GetObjectFunc == nil { @@ -2455,6 +2571,46 @@ func (mock *BackendMock) PutBucketVersioningCalls() []struct { return calls } +// PutBucketWebsite calls PutBucketWebsiteFunc. +func (mock *BackendMock) PutBucketWebsite(contextMoqParam context.Context, bucket string, website []byte) error { + if mock.PutBucketWebsiteFunc == nil { + panic("BackendMock.PutBucketWebsiteFunc: method is nil but Backend.PutBucketWebsite was just called") + } + callInfo := struct { + ContextMoqParam context.Context + Bucket string + Website []byte + }{ + ContextMoqParam: contextMoqParam, + Bucket: bucket, + Website: website, + } + mock.lockPutBucketWebsite.Lock() + mock.calls.PutBucketWebsite = append(mock.calls.PutBucketWebsite, callInfo) + mock.lockPutBucketWebsite.Unlock() + return mock.PutBucketWebsiteFunc(contextMoqParam, bucket, website) +} + +// PutBucketWebsiteCalls gets all the calls that were made to PutBucketWebsite. +// Check the length with: +// +// len(mockedBackend.PutBucketWebsiteCalls()) +func (mock *BackendMock) PutBucketWebsiteCalls() []struct { + ContextMoqParam context.Context + Bucket string + Website []byte +} { + var calls []struct { + ContextMoqParam context.Context + Bucket string + Website []byte + } + mock.lockPutBucketWebsite.RLock() + calls = mock.calls.PutBucketWebsite + mock.lockPutBucketWebsite.RUnlock() + return calls +} + // PutObject calls PutObjectFunc. func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error) { if mock.PutObjectFunc == nil { diff --git a/s3api/controllers/bucket-delete.go b/s3api/controllers/bucket-delete.go index 564f293c..6d517ef3 100644 --- a/s3api/controllers/bucket-delete.go +++ b/s3api/controllers/bucket-delete.go @@ -162,6 +162,42 @@ func (c S3ApiController) DeleteBucketCors(ctx *fiber.Ctx) (*Response, error) { }, err } +func (c S3ApiController) DeleteBucketWebsite(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutBucketWebsiteAction, + IsPublicRequest: IsBucketPublic, + DisableACL: c.disableACL, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.DeleteBucketWebsite(ctx.Context(), bucket) + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + Status: http.StatusNoContent, + }, + }, err +} + func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) (*Response, error) { bucket := ctx.Params("bucket") acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) diff --git a/s3api/controllers/bucket-get.go b/s3api/controllers/bucket-get.go index 199333f8..3a31156b 100644 --- a/s3api/controllers/bucket-get.go +++ b/s3api/controllers/bucket-get.go @@ -206,6 +206,50 @@ func (c S3ApiController) GetBucketCors(ctx *fiber.Ctx) (*Response, error) { }, err } +func (c S3ApiController) GetBucketWebsite(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.GetBucketWebsiteAction, + IsPublicRequest: isPublicBucket, + DisableACL: c.disableACL, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + data, err := c.be.GetBucketWebsite(ctx.Context(), bucket) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + output, err := s3response.ParseWebsiteConfigOutput(data) + return &Response{ + Data: output, + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err +} + func (c S3ApiController) GetBucketPolicy(ctx *fiber.Ctx) (*Response, error) { bucket := ctx.Params("bucket") acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) diff --git a/s3api/controllers/bucket-put.go b/s3api/controllers/bucket-put.go index 860fb015..a83c291f 100644 --- a/s3api/controllers/bucket-put.go +++ b/s3api/controllers/bucket-put.go @@ -287,6 +287,62 @@ func (c S3ApiController) PutBucketCors(ctx *fiber.Ctx) (*Response, error) { }, err } +func (c S3ApiController) PutBucketWebsite(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutBucketWebsiteAction, + IsPublicRequest: isPublicBucket, + DisableACL: c.disableACL, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + body := ctx.Body() + + var websiteConfig s3response.WebsiteConfiguration + err = xml.Unmarshal(body, &websiteConfig) + if err != nil { + debuglogger.Logf("invalid website config request body: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMalformedXML) + } + + err = websiteConfig.Validate() + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.PutBucketWebsite(ctx.Context(), bucket, body) + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err +} + func (c S3ApiController) PutBucketPolicy(ctx *fiber.Ctx) (*Response, error) { bucket := ctx.Params("bucket") parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) diff --git a/s3api/middlewares/website.go b/s3api/middlewares/website.go new file mode 100644 index 00000000..90c0db0a --- /dev/null +++ b/s3api/middlewares/website.go @@ -0,0 +1,94 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package middlewares + +import ( + "encoding/xml" + "fmt" + "net/http" + "strings" + + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3response" +) + +// ResolveWebsiteIndex rewrites directory-like object keys to include the +// configured IndexDocument suffix when website hosting is enabled for the +// bucket. It also handles RedirectAllRequestsTo by returning a 301 redirect. +// +// This middleware should be placed in the GetObject handler chain before +// authentication and the controller. +func ResolveWebsiteIndex(be backend.Backend) fiber.Handler { + return func(ctx *fiber.Ctx) error { + if utils.ContextKeySkip.IsSet(ctx) { + return ctx.Next() + } + + bucket := ctx.Params("bucket") + if bucket == "" { + return ctx.Next() + } + + key := ctx.Params("*1") + + // Only process directory-like keys (empty or ending with /) + if key != "" && !strings.HasSuffix(key, "/") { + return ctx.Next() + } + + // Reject path traversal attempts + if strings.Contains(key, "..") { + return ctx.Next() + } + + data, err := be.GetBucketWebsite(ctx.Context(), bucket) + if err != nil { + // No website config: pass through to normal handling + return ctx.Next() + } + + var config s3response.WebsiteConfiguration + if xmlErr := xml.Unmarshal(data, &config); xmlErr != nil { + return ctx.Next() + } + + // Handle RedirectAllRequestsTo + if config.RedirectAllRequestsTo != nil { + return redirectAll(ctx, config.RedirectAllRequestsTo, key) + } + + // Rewrite directory-like keys to include index document suffix + if config.IndexDocument != nil && config.IndexDocument.Suffix != "" { + newKey := key + config.IndexDocument.Suffix + newPath := fmt.Sprintf("/%s/%s", bucket, newKey) + ctx.Request().URI().SetPath(newPath) + } + + return ctx.Next() + } +} + +func redirectAll(ctx *fiber.Ctx, redirect *s3response.RedirectAllRequestsTo, key string) error { + protocol := redirect.Protocol + if protocol == "" { + protocol = "https" + } + + location := fmt.Sprintf("%s://%s/%s", protocol, redirect.HostName, key) + ctx.Set("Location", location) + return ctx.SendStatus(http.StatusMovedPermanently) +} diff --git a/s3api/router.go b/s3api/router.go index b1cfdd98..b735ebbc 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -444,7 +444,7 @@ func (sa *S3ApiRouter) Init() { bucketRouter.Put("", middlewares.MatchQueryArgs("website"), controllers.ProcessHandlers( - ctrl.HandleErrorRoute(s3err.GetAPIError(s3err.ErrNotImplemented)), + ctrl.PutBucketWebsite, metrics.ActionPutBucketWebsite, services, middlewares.BucketObjectNameValidator(), @@ -665,7 +665,7 @@ func (sa *S3ApiRouter) Init() { bucketRouter.Delete("", middlewares.MatchQueryArgs("website"), controllers.ProcessHandlers( - ctrl.HandleErrorRoute(s3err.GetAPIError(s3err.ErrNotImplemented)), + ctrl.DeleteBucketWebsite, metrics.ActionDeleteBucketWebsite, services, middlewares.BucketObjectNameValidator(), @@ -1056,7 +1056,7 @@ func (sa *S3ApiRouter) Init() { bucketRouter.Get("", middlewares.MatchQueryArgs("website"), controllers.ProcessHandlers( - ctrl.HandleErrorRoute(s3err.GetAPIError(s3err.ErrNotImplemented)), + ctrl.GetBucketWebsite, metrics.ActionGetBucketWebsite, services, middlewares.BucketObjectNameValidator(), @@ -1152,6 +1152,7 @@ func (sa *S3ApiRouter) Init() { metrics.ActionHeadObject, services, middlewares.BucketObjectNameValidator(), + middlewares.ResolveWebsiteIndex(sa.be), middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionHeadObject, auth.GetObjectAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, false, false), @@ -1269,6 +1270,7 @@ func (sa *S3ApiRouter) Init() { metrics.ActionGetObject, services, middlewares.BucketObjectNameValidator(), + middlewares.ResolveWebsiteIndex(sa.be), middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionGetObject, auth.GetObjectAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), diff --git a/s3err/s3err.go b/s3err/s3err.go index ddbec479..3be9fdc7 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -165,6 +165,10 @@ const ( ErrCORSForbidden ErrMissingCORSOrigin ErrCORSIsNotEnabled + ErrNoSuchWebsiteConfiguration + ErrInvalidWebsiteConfiguration + ErrInvalidWebsiteSuffix + ErrInvalidWebsiteRedirectCode ErrNotModified ErrInvalidLocationConstraint ErrMalformedTrailer @@ -639,6 +643,26 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "CORSResponse: CORS is not enabled for this bucket.", HTTPStatusCode: http.StatusForbidden, }, + ErrNoSuchWebsiteConfiguration: { + Code: "NoSuchWebsiteConfiguration", + Description: "The specified bucket does not have a website configuration", + HTTPStatusCode: http.StatusNotFound, + }, + ErrInvalidWebsiteConfiguration: { + Code: "InvalidRequest", + Description: "The website configuration is not valid.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidWebsiteSuffix: { + Code: "InvalidArgument", + Description: "The IndexDocument Suffix is not well formed", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidWebsiteRedirectCode: { + Code: "InvalidArgument", + Description: "The website redirect code is not valid. Valid codes are 3XX.", + HTTPStatusCode: http.StatusBadRequest, + }, ErrNotModified: { Code: "NotModified", Description: "Not Modified", diff --git a/s3response/website.go b/s3response/website.go new file mode 100644 index 00000000..85d2bf24 --- /dev/null +++ b/s3response/website.go @@ -0,0 +1,151 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package s3response + +import ( + "encoding/xml" + "fmt" + "strings" + + "github.com/versity/versitygw/s3err" +) + +const maxRoutingRules = 50 + +// WebsiteConfiguration represents the S3 bucket website configuration. +type WebsiteConfiguration struct { + XMLName xml.Name `xml:"WebsiteConfiguration"` + IndexDocument *IndexDocument `xml:"IndexDocument,omitempty"` + ErrorDocument *ErrorDocument `xml:"ErrorDocument,omitempty"` + RedirectAllRequestsTo *RedirectAllRequestsTo `xml:"RedirectAllRequestsTo,omitempty"` + RoutingRules []RoutingRule `xml:"RoutingRules>RoutingRule,omitempty"` +} + +// IndexDocument specifies the default object served for directory-like requests. +type IndexDocument struct { + Suffix string `xml:"Suffix"` +} + +// ErrorDocument specifies the object served when an error occurs. +type ErrorDocument struct { + Key string `xml:"Key"` +} + +// RedirectAllRequestsTo redirects all requests to another host. +type RedirectAllRequestsTo struct { + HostName string `xml:"HostName"` + Protocol string `xml:"Protocol,omitempty"` +} + +// RoutingRule specifies a redirect rule with an optional condition. +type RoutingRule struct { + Condition *RoutingRuleCondition `xml:"Condition,omitempty"` + Redirect Redirect `xml:"Redirect"` +} + +// RoutingRuleCondition specifies when a routing rule applies. +type RoutingRuleCondition struct { + HttpErrorCodeReturnedEquals string `xml:"HttpErrorCodeReturnedEquals,omitempty"` + KeyPrefixEquals string `xml:"KeyPrefixEquals,omitempty"` +} + +// Redirect specifies where to redirect matching requests. +type Redirect struct { + HostName string `xml:"HostName,omitempty"` + HttpRedirectCode string `xml:"HttpRedirectCode,omitempty"` + Protocol string `xml:"Protocol,omitempty"` + ReplaceKeyPrefixWith string `xml:"ReplaceKeyPrefixWith,omitempty"` + ReplaceKeyWith string `xml:"ReplaceKeyWith,omitempty"` +} + +// Validate checks the website configuration for S3-compatible validity. +func (c *WebsiteConfiguration) Validate() error { + if c.RedirectAllRequestsTo != nil { + if c.IndexDocument != nil || c.ErrorDocument != nil || len(c.RoutingRules) > 0 { + return s3err.GetAPIError(s3err.ErrInvalidWebsiteConfiguration) + } + if c.RedirectAllRequestsTo.HostName == "" { + return s3err.GetAPIError(s3err.ErrInvalidWebsiteConfiguration) + } + if err := validateProtocol(c.RedirectAllRequestsTo.Protocol); err != nil { + return err + } + return nil + } + + if c.IndexDocument == nil { + return s3err.GetAPIError(s3err.ErrInvalidWebsiteConfiguration) + } + if c.IndexDocument.Suffix == "" { + return s3err.GetAPIError(s3err.ErrInvalidWebsiteSuffix) + } + if strings.Contains(c.IndexDocument.Suffix, "/") { + return s3err.GetAPIError(s3err.ErrInvalidWebsiteSuffix) + } + + if c.ErrorDocument != nil && c.ErrorDocument.Key == "" { + return s3err.GetAPIError(s3err.ErrInvalidWebsiteConfiguration) + } + + if len(c.RoutingRules) > maxRoutingRules { + return s3err.GetAPIError(s3err.ErrInvalidWebsiteConfiguration) + } + + for i, rule := range c.RoutingRules { + if err := rule.Validate(); err != nil { + return fmt.Errorf("routing rule %d: %w", i, err) + } + } + + return nil +} + +// Validate checks a single routing rule for validity. +func (r *RoutingRule) Validate() error { + if r.Redirect.ReplaceKeyWith != "" && r.Redirect.ReplaceKeyPrefixWith != "" { + return s3err.GetAPIError(s3err.ErrInvalidWebsiteConfiguration) + } + + if err := validateProtocol(r.Redirect.Protocol); err != nil { + return err + } + + if r.Redirect.HttpRedirectCode != "" { + code := r.Redirect.HttpRedirectCode + if len(code) != 3 || code[0] != '3' { + return s3err.GetAPIError(s3err.ErrInvalidWebsiteRedirectCode) + } + } + + return nil +} + +func validateProtocol(protocol string) error { + if protocol != "" && protocol != "http" && protocol != "https" { + return s3err.GetAPIError(s3err.ErrInvalidWebsiteConfiguration) + } + return nil +} + +// ParseWebsiteConfigOutput parses raw bytes into a WebsiteConfiguration. +func ParseWebsiteConfigOutput(data []byte) (*WebsiteConfiguration, error) { + var config WebsiteConfiguration + err := xml.Unmarshal(data, &config) + if err != nil { + return nil, fmt.Errorf("failed to parse website config: %w", err) + } + + return &config, nil +} diff --git a/s3response/website_test.go b/s3response/website_test.go new file mode 100644 index 00000000..a8f6b4fb --- /dev/null +++ b/s3response/website_test.go @@ -0,0 +1,274 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package s3response + +import ( + "encoding/xml" + "testing" + + "github.com/versity/versitygw/s3err" +) + +func TestWebsiteConfiguration_Validate(t *testing.T) { + tests := []struct { + name string + config WebsiteConfiguration + wantErr bool + errCode s3err.ErrorCode + }{ + { + name: "valid index document only", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + }, + }, + { + name: "valid index and error document", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + ErrorDocument: &ErrorDocument{Key: "error.html"}, + }, + }, + { + name: "valid redirect all requests", + config: WebsiteConfiguration{ + RedirectAllRequestsTo: &RedirectAllRequestsTo{ + HostName: "example.com", + Protocol: "https", + }, + }, + }, + { + name: "valid routing rules", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + RoutingRules: []RoutingRule{ + { + Condition: &RoutingRuleCondition{ + KeyPrefixEquals: "docs/", + }, + Redirect: Redirect{ + ReplaceKeyPrefixWith: "documents/", + }, + }, + }, + }, + }, + { + name: "missing index document", + config: WebsiteConfiguration{}, + wantErr: true, + errCode: s3err.ErrInvalidWebsiteConfiguration, + }, + { + name: "empty index suffix", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: ""}, + }, + wantErr: true, + errCode: s3err.ErrInvalidWebsiteSuffix, + }, + { + name: "index suffix with slash", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "dir/index.html"}, + }, + wantErr: true, + errCode: s3err.ErrInvalidWebsiteSuffix, + }, + { + name: "redirect all with index document", + config: WebsiteConfiguration{ + RedirectAllRequestsTo: &RedirectAllRequestsTo{HostName: "example.com"}, + IndexDocument: &IndexDocument{Suffix: "index.html"}, + }, + wantErr: true, + errCode: s3err.ErrInvalidWebsiteConfiguration, + }, + { + name: "redirect all with empty hostname", + config: WebsiteConfiguration{ + RedirectAllRequestsTo: &RedirectAllRequestsTo{HostName: ""}, + }, + wantErr: true, + errCode: s3err.ErrInvalidWebsiteConfiguration, + }, + { + name: "redirect all with invalid protocol", + config: WebsiteConfiguration{ + RedirectAllRequestsTo: &RedirectAllRequestsTo{ + HostName: "example.com", + Protocol: "ftp", + }, + }, + wantErr: true, + errCode: s3err.ErrInvalidWebsiteConfiguration, + }, + { + name: "routing rule with both replace key fields", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + RoutingRules: []RoutingRule{ + { + Redirect: Redirect{ + ReplaceKeyWith: "newkey", + ReplaceKeyPrefixWith: "newprefix/", + }, + }, + }, + }, + wantErr: true, + errCode: s3err.ErrInvalidWebsiteConfiguration, + }, + { + name: "routing rule with invalid redirect code", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + RoutingRules: []RoutingRule{ + { + Redirect: Redirect{ + HttpRedirectCode: "200", + }, + }, + }, + }, + wantErr: true, + errCode: s3err.ErrInvalidWebsiteRedirectCode, + }, + { + name: "routing rule with valid redirect code", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + RoutingRules: []RoutingRule{ + { + Redirect: Redirect{ + HttpRedirectCode: "301", + HostName: "example.com", + }, + }, + }, + }, + }, + { + name: "error document with empty key", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + ErrorDocument: &ErrorDocument{Key: ""}, + }, + wantErr: true, + errCode: s3err.ErrInvalidWebsiteConfiguration, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.config.Validate() + if tt.wantErr { + if err == nil { + t.Fatal("expected error, got nil") + } + apiErr, ok := err.(s3err.APIError) + if !ok { + // wrapped error from routing rule validation + return + } + expectedErr := s3err.GetAPIError(tt.errCode) + if apiErr.Code != expectedErr.Code { + t.Errorf("expected error code %q, got %q", expectedErr.Code, apiErr.Code) + } + } else { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + }) + } +} + +func TestWebsiteConfiguration_XMLRoundTrip(t *testing.T) { + original := WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + ErrorDocument: &ErrorDocument{Key: "error.html"}, + RoutingRules: []RoutingRule{ + { + Condition: &RoutingRuleCondition{ + KeyPrefixEquals: "docs/", + HttpErrorCodeReturnedEquals: "404", + }, + Redirect: Redirect{ + HostName: "example.com", + Protocol: "https", + HttpRedirectCode: "301", + ReplaceKeyPrefixWith: "documents/", + }, + }, + }, + } + + data, err := xml.Marshal(original) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + var parsed WebsiteConfiguration + if err := xml.Unmarshal(data, &parsed); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + if parsed.IndexDocument == nil || parsed.IndexDocument.Suffix != "index.html" { + t.Error("IndexDocument.Suffix mismatch") + } + if parsed.ErrorDocument == nil || parsed.ErrorDocument.Key != "error.html" { + t.Error("ErrorDocument.Key mismatch") + } + if len(parsed.RoutingRules) != 1 { + t.Fatalf("expected 1 routing rule, got %d", len(parsed.RoutingRules)) + } + rule := parsed.RoutingRules[0] + if rule.Condition == nil || rule.Condition.KeyPrefixEquals != "docs/" { + t.Error("RoutingRule Condition.KeyPrefixEquals mismatch") + } + if rule.Redirect.HostName != "example.com" { + t.Error("RoutingRule Redirect.HostName mismatch") + } + if rule.Redirect.ReplaceKeyPrefixWith != "documents/" { + t.Error("RoutingRule Redirect.ReplaceKeyPrefixWith mismatch") + } +} + +func TestParseWebsiteConfigOutput(t *testing.T) { + xmlData := ` + index.html + error.html + ` + + config, err := ParseWebsiteConfigOutput([]byte(xmlData)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if config.IndexDocument == nil || config.IndexDocument.Suffix != "index.html" { + t.Error("IndexDocument.Suffix mismatch") + } + if config.ErrorDocument == nil || config.ErrorDocument.Key != "error.html" { + t.Error("ErrorDocument.Key mismatch") + } +} + +func TestParseWebsiteConfigOutput_InvalidXML(t *testing.T) { + _, err := ParseWebsiteConfigOutput([]byte("not xml")) + if err == nil { + t.Fatal("expected error for invalid XML") + } +} From 375c2764d59b2e51c08404d4afc63486e702cd0b Mon Sep 17 00:00:00 2001 From: Marc Singer Date: Sun, 5 Apr 2026 11:48:33 +0200 Subject: [PATCH 2/4] Add website integration tests and remove NotImplemented stubs Replace PutBucketWebsite, GetBucketWebsite, DeleteBucketWebsite NotImplemented test stubs with comprehensive integration tests covering: - non-existing bucket errors - validation (empty suffix, suffix with slash, invalid protocol, mutual exclusion of RedirectAllRequestsTo and IndexDocument) - successful put/get round-trips for both index+error and redirect-all configs - delete idempotency and verification Signed-off-by: Marc Singer Add error document serving, routing rules, and integration tests Implement Features 1 and 2 of S3 static website hosting: - WebsiteErrorDocument controller wrapper intercepts 4xx errors on website-enabled buckets and serves the configured error document or evaluates post-request routing rules (error code match redirects) - ResolveWebsiteIndex middleware now caches parsed WebsiteConfiguration in context, handles RedirectAllRequestsTo, evaluates pre-request routing rules (key prefix match redirects), and rewrites directory keys for index document - MatchPreRequestRule and MatchPostRequestRule methods on WebsiteConfiguration for routing rule evaluation - 14 unit tests for routing rule matching - 7 integration tests covering error document, routing rules, redirect-all, and index document behavior Signed-off-by: Marc Singer Add separate website hosting endpoint with virtual-host routing Signed-off-by: Marc Singer Support catch-all mode for website endpoint when --website-domain is omitted Signed-off-by: Marc Singer --- README.md | 6 + auth/bucket_policy_actions.go | 2 + chart/README.md | 1 + chart/templates/deployment.yaml | 16 + chart/templates/service.yaml | 6 + chart/values.yaml | 18 + cmd/versitygw/main.go | 37 ++ cmd/versitygw/test.go | 52 ++- extra/example.conf | 47 +++ s3api/controllers/bucket-delete.go | 2 +- s3api/middlewares/website.go | 94 ----- s3api/router.go | 8 +- s3api/utils/context-keys.go | 1 + s3err/s3err.go | 4 +- s3response/website.go | 49 ++- s3response/website_test.go | 315 ++++++++++++++- tests/integration/DeleteBucketWebsite.go | 85 ++++ tests/integration/GetBucketWebsite.go | 126 ++++++ tests/integration/NotImplemented_actions.go | 47 --- tests/integration/PutBucketWebsite.go | 158 ++++++++ tests/integration/WebsiteHosting.go | 413 ++++++++++++++++++++ tests/integration/group-tests.go | 69 +++- tests/integration/s3conf.go | 4 + website/handler.go | 314 +++++++++++++++ website/server.go | 130 ++++++ webui/web/explorer.html | 4 + 26 files changed, 1833 insertions(+), 175 deletions(-) delete mode 100644 s3api/middlewares/website.go create mode 100644 tests/integration/DeleteBucketWebsite.go create mode 100644 tests/integration/GetBucketWebsite.go create mode 100644 tests/integration/PutBucketWebsite.go create mode 100644 tests/integration/WebsiteHosting.go create mode 100644 website/handler.go create mode 100644 website/server.go diff --git a/README.md b/README.md index ff3bc57a..90c2a8de 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,12 @@ Get more details about the new (optional) WebGUI management/explorer here: [http ![admin-explorer](https://github.com/user-attachments/assets/e99db171-2c72-4d0f-8c8d-480a56e1c8a1) +### Static Website Hosting +Serve S3 buckets as static websites with index documents, custom error pages, and routing rules. +Enable a separate website endpoint with `--website :8090 --website-domain example.com` for virtual-host style routing (`blog.example.com` serves bucket `blog`, `example.com` serves bucket `example.com`). +When `--website-domain` is omitted, catch-all mode is used: the full hostname becomes the bucket name (name your buckets as FQDNs, e.g. `blog.example.com`). +See [Global Options](https://github.com/versity/versitygw/wiki/Global-Options) for all `--website-*` flags. + ### News Check out latest wiki articles: [https://github.com/versity/versitygw/wiki/Articles](https://github.com/versity/versitygw/wiki/Articles) diff --git a/auth/bucket_policy_actions.go b/auth/bucket_policy_actions.go index 393677de..34bb54d2 100644 --- a/auth/bucket_policy_actions.go +++ b/auth/bucket_policy_actions.go @@ -91,6 +91,7 @@ const ( GetAccelerateConfigurationAction Action = "s3:GetAccelerateConfiguration" PutBucketWebsiteAction Action = "s3:PutBucketWebsite" GetBucketWebsiteAction Action = "s3:GetBucketWebsite" + DeleteBucketWebsiteAction Action = "s3:DeleteBucketWebsite" GetBucketPolicyStatusAction Action = "s3:GetBucketPolicyStatus" GetBucketLocationAction Action = "s3:GetBucketLocation" @@ -167,6 +168,7 @@ var supportedActionList = map[Action]struct{}{ GetAccelerateConfigurationAction: {}, PutBucketWebsiteAction: {}, GetBucketWebsiteAction: {}, + DeleteBucketWebsiteAction: {}, GetBucketPolicyStatusAction: {}, GetBucketLocationAction: {}, AllActions: {}, diff --git a/chart/README.md b/chart/README.md index 808cea93..2194b5f6 100644 --- a/chart/README.md +++ b/chart/README.md @@ -83,6 +83,7 @@ The `gateway.backend.type` value selects the storage backend. Use `gateway.backe | **HTTPRoute** | `httpRoute.enabled=true` — Gateway API successor to Ingress for S3 API; also `admin.httpRoute.enabled=true` and `webui.httpRoute.enabled=true` to expose the admin API and/or WebUI | | **Admin API** | `admin.enabled=true` — exposes a separate management API on `admin.port` (default `7071`) | | **WebUI** | `webui.enabled=true` — browser-based management UI on `webui.port` (default `8080`); set `webui.apiGateways` and `webui.adminGateways` to your externally reachable endpoints | +| **Website Hosting** | `website.enabled=true` — static website hosting endpoint on `website.port` (default `8090`); optionally set `website.domain` for virtual-host routing (e.g. `example.com`), or omit it for catch-all mode where the full hostname is the bucket name | | **IAM** | `iam.enabled=true` — flat-file identity and access management stored alongside backend data | | **Persistence** | `persistence.enabled=true` — provisions a PVC for backend data and IAM storage; defaults to `10Gi`, or uses a hostPath volume specified by `persistence.hostPath` | | **NetworkPolicy** | `networkPolicy.enabled=true` — restricts ingress to selected pods/namespaces; allows all egress | diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml index a25663b9..d69dd55b 100644 --- a/chart/templates/deployment.yaml +++ b/chart/templates/deployment.yaml @@ -129,6 +129,17 @@ spec: value: {{ .Values.webui.adminGateways | join "," | quote }} {{- end }} {{- end }} + # Website Hosting + {{- if .Values.website.enabled }} + - name: VGW_WEBSITE_PORT + value: ":{{ .Values.website.port }}" + - name: VGW_WEBSITE_DOMAIN + value: {{ .Values.website.domain | quote }} + {{- if .Values.website.noTls }} + - name: VGW_WEBSITE_NO_TLS + value: "true" + {{- end }} + {{- end }} {{- if .Values.iam.enabled }} # IAM settings {{- if eq .Values.iam.type "internal" }} @@ -173,6 +184,11 @@ spec: containerPort: {{ .Values.webui.port }} protocol: TCP {{- end }} + {{- if .Values.website.enabled }} + - name: website + containerPort: {{ .Values.website.port }} + protocol: TCP + {{- end }} readinessProbe: httpGet: path: "/_/health" diff --git a/chart/templates/service.yaml b/chart/templates/service.yaml index c9b70532..ab80edf8 100644 --- a/chart/templates/service.yaml +++ b/chart/templates/service.yaml @@ -23,5 +23,11 @@ spec: protocol: TCP name: webui {{- end }} + {{- if .Values.website.enabled }} + - port: {{ .Values.website.port }} + targetPort: website + protocol: TCP + name: website + {{- end }} selector: {{- include "versitygw.selectorLabels" . | nindent 4 }} diff --git a/chart/values.yaml b/chart/values.yaml index 1e44d8b1..04b84b88 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -225,6 +225,24 @@ webui: type: PathPrefix value: / +# --- Website Hosting --- +website: + # Enable the static website hosting endpoint. + # Serves S3 buckets as static websites with index documents, custom error + # pages, and routing rules via a separate HTTP endpoint. + enabled: false + # The port the website endpoint listens on. + port: 8090 + # Base domain for virtual-host routing. Optional. + # Host "blog." serves bucket "blog"; host "" serves + # bucket "" (apex domain support). + # When empty, catch-all mode is used: the full hostname is the bucket + # name (name buckets as FQDNs, e.g. "blog.example.com"). + domain: "" + # - example: domain: "example.com" + # Disable TLS for the website endpoint even when gateway TLS is enabled. + noTls: false + # --- IAM (Identity and Access Management) --- iam: enabled: false diff --git a/cmd/versitygw/main.go b/cmd/versitygw/main.go index 8b2d8355..c1f62ce6 100644 --- a/cmd/versitygw/main.go +++ b/cmd/versitygw/main.go @@ -91,6 +91,10 @@ var ( webuiAdminGateways []string webuiPathPrefix string webuiS3Prefix string + websitePorts []string + websiteDomain string + websiteCertFile, websiteKeyFile string + websiteNoTLS bool disableACLs bool mpMaxParts int copyObjectThreshold int64 @@ -152,6 +156,7 @@ documentation can be found in the GitHub wiki.`, webuiGateways = ctx.StringSlice("webui-gateways") webuiAdminGateways = ctx.StringSlice("webui-admin-gateways") webuiPathPrefix = ctx.String("webui-path-prefix") + websitePorts = ctx.StringSlice("website") // Resolve relative UNIX socket paths to absolute before any backend // (e.g. posix) can change the working directory via os.Chdir. @@ -165,6 +170,9 @@ documentation can be found in the GitHub wiki.`, if webuiPorts, err = utils.AbsSocketPaths(webuiPorts); err != nil { return err } + if websitePorts, err = utils.AbsSocketPaths(websitePorts); err != nil { + return err + } return nil }, Action: func(ctx *cli.Context) error { @@ -240,6 +248,35 @@ func initFlags() []cli.Flag { EnvVars: []string{"VGW_WEBUI_S3_PREFIX"}, Destination: &webuiS3Prefix, }, + &cli.StringSliceFlag{ + Name: "website", + Usage: "enable static website hosting endpoint on the specified listen address (e.g. ':8080'; same forms as --port; can be specified multiple times; requires --website-domain)", + EnvVars: []string{"VGW_WEBSITE_PORT"}, + }, + &cli.StringFlag{ + Name: "website-domain", + Usage: "base domain for website virtual-host routing (e.g. 'example.com'); host 'blog.example.com' serves bucket 'blog', host 'example.com' serves bucket 'example.com'; when omitted the full hostname is used as the bucket name (catch-all mode, buckets named as FQDNs)", + EnvVars: []string{"VGW_WEBSITE_DOMAIN"}, + Destination: &websiteDomain, + }, + &cli.StringFlag{ + Name: "website-cert", + Usage: "TLS cert file for website endpoint (defaults to --cert value when website is enabled)", + EnvVars: []string{"VGW_WEBSITE_CERT"}, + Destination: &websiteCertFile, + }, + &cli.StringFlag{ + Name: "website-key", + Usage: "TLS key file for website endpoint (defaults to --key value when website is enabled)", + EnvVars: []string{"VGW_WEBSITE_KEY"}, + Destination: &websiteKeyFile, + }, + &cli.BoolFlag{ + Name: "website-no-tls", + Usage: "disable TLS for website endpoint even if TLS is configured for the gateway", + EnvVars: []string{"VGW_WEBSITE_NO_TLS"}, + Destination: &websiteNoTLS, + }, &cli.StringFlag{ Name: "access", Usage: "root user access key", diff --git a/cmd/versitygw/test.go b/cmd/versitygw/test.go index 2c885e30..3f33d680 100644 --- a/cmd/versitygw/test.go +++ b/cmd/versitygw/test.go @@ -22,25 +22,26 @@ import ( ) var ( - awsID string - awsSecret string - endpoint string - prefix string - dstBucket string - partSize int64 - objSize int64 - concurrency int - files int - totalReqs int - upload bool - download bool - hostStyle bool - checksumDisable bool - versioningEnabled bool - azureTests bool - sidecarTests bool - tlsStatus bool - parallel bool + awsID string + awsSecret string + endpoint string + websiteEndpointTest string + prefix string + dstBucket string + partSize int64 + objSize int64 + concurrency int + files int + totalReqs int + upload bool + download bool + hostStyle bool + checksumDisable bool + versioningEnabled bool + azureTests bool + tlsStatus bool + parallel bool + sidecarTests bool ) func testCommand() *cli.Command { @@ -76,6 +77,13 @@ func initTestFlags() []cli.Flag { Destination: &endpoint, Aliases: []string{"e"}, }, + &cli.StringFlag{ + Name: "website-endpoint", + Usage: "dedicated website hosting endpoint (e.g. 'http://localhost:8080'); required for WebsiteHosting tests", + EnvVars: []string{"VGW_TEST_WEBSITE_ENDPOINT"}, + Destination: &websiteEndpointTest, + Aliases: []string{"we"}, + }, &cli.BoolFlag{ Name: "host-style", Usage: "Use host-style bucket addressing", @@ -334,6 +342,9 @@ func getAction(tf testFunc) func(ctx *cli.Context) error { integration.WithEndpoint(endpoint), integration.WithTLSStatus(tlsStatus), } + if websiteEndpointTest != "" { + opts = append(opts, integration.WithWebsiteEndpoint(websiteEndpointTest)) + } if debug { opts = append(opts, integration.WithDebug()) } @@ -380,6 +391,9 @@ func extractIntTests() (commands []*cli.Command) { integration.WithEndpoint(endpoint), integration.WithTLSStatus(tlsStatus), } + if websiteEndpointTest != "" { + opts = append(opts, integration.WithWebsiteEndpoint(websiteEndpointTest)) + } if debug { opts = append(opts, integration.WithDebug()) } diff --git a/extra/example.conf b/extra/example.conf index caf869b2..8f2f6c1d 100644 --- a/extra/example.conf +++ b/extra/example.conf @@ -312,6 +312,53 @@ ROOT_SECRET_ACCESS_KEY= # Example: VGW_WEBUI_ADMIN_GATEWAYS=https://admin.example.com,http://192.168.1.100:7080 #VGW_WEBUI_ADMIN_GATEWAYS= +################### +# Website Hosting # +################### + +# VersityGW supports S3-compatible static website hosting on a dedicated +# endpoint, separate from the S3 API port. This mirrors how AWS serves +# websites on s3-website..amazonaws.com rather than s3.amazonaws.com. +# +# When enabled, the website endpoint serves bucket content as static websites +# using the PutBucketWebsite configuration (index documents, error documents, +# routing rules, and redirect-all). No S3 authentication is applied — the +# website endpoint is public, just like AWS S3 website hosting. +# +# Bucket resolution uses virtual-host-style routing based on the Host header: +# - With VGW_WEBSITE_DOMAIN=example.com: +# Host "blog.example.com" -> serves bucket "blog" +# Host "example.com" -> serves bucket "example.com" (apex) +# - Without VGW_WEBSITE_DOMAIN (catch-all mode): +# Host "blog.example.com" -> serves bucket "blog.example.com" +# Host "mysite.org" -> serves bucket "mysite.org" + +# The VGW_WEBSITE_PORT option enables the website hosting endpoint on the +# specified listen address. The format is the same as VGW_PORT (e.g. ':8080', +# 'localhost:8080'). Multiple ports can be specified as a comma-separated list. +# When omitted, website hosting is disabled. +#VGW_WEBSITE_PORT= + +# The VGW_WEBSITE_DOMAIN option sets the base domain for virtual-host bucket +# routing. For example, with domain "example.com", a request with Host header +# "blog.example.com" resolves to bucket "blog". When omitted, the full +# hostname from the Host header is used as the bucket name (catch-all mode), +# which is useful when buckets are named as FQDNs (e.g. "www.mysite.org"). +#VGW_WEBSITE_DOMAIN= + +# The VGW_WEBSITE_CERT and VGW_WEBSITE_KEY options specify TLS credentials +# for the website endpoint. When not set but TLS is configured for the gateway +# (VGW_CERT and VGW_KEY), the website endpoint inherits the gateway certificates. +# When neither is set, the website endpoint runs without TLS (HTTP only). +#VGW_WEBSITE_CERT= +#VGW_WEBSITE_KEY= + +# The VGW_WEBSITE_NO_TLS option disables TLS for the website endpoint even +# when TLS certificates are configured for the gateway. Set to true to force +# the website endpoint to use HTTP. This is useful when TLS termination is +# handled by a reverse proxy or load balancer. +#VGW_WEBSITE_NO_TLS=false + ####################### # Debug / Diagnostics # ####################### diff --git a/s3api/controllers/bucket-delete.go b/s3api/controllers/bucket-delete.go index 6d517ef3..441f5273 100644 --- a/s3api/controllers/bucket-delete.go +++ b/s3api/controllers/bucket-delete.go @@ -177,7 +177,7 @@ func (c S3ApiController) DeleteBucketWebsite(ctx *fiber.Ctx) (*Response, error) IsRoot: isRoot, Acc: acct, Bucket: bucket, - Action: auth.PutBucketWebsiteAction, + Action: auth.DeleteBucketWebsiteAction, IsPublicRequest: IsBucketPublic, DisableACL: c.disableACL, }) diff --git a/s3api/middlewares/website.go b/s3api/middlewares/website.go deleted file mode 100644 index 90c0db0a..00000000 --- a/s3api/middlewares/website.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2023 Versity Software -// This file is licensed under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package middlewares - -import ( - "encoding/xml" - "fmt" - "net/http" - "strings" - - "github.com/gofiber/fiber/v2" - "github.com/versity/versitygw/backend" - "github.com/versity/versitygw/s3api/utils" - "github.com/versity/versitygw/s3response" -) - -// ResolveWebsiteIndex rewrites directory-like object keys to include the -// configured IndexDocument suffix when website hosting is enabled for the -// bucket. It also handles RedirectAllRequestsTo by returning a 301 redirect. -// -// This middleware should be placed in the GetObject handler chain before -// authentication and the controller. -func ResolveWebsiteIndex(be backend.Backend) fiber.Handler { - return func(ctx *fiber.Ctx) error { - if utils.ContextKeySkip.IsSet(ctx) { - return ctx.Next() - } - - bucket := ctx.Params("bucket") - if bucket == "" { - return ctx.Next() - } - - key := ctx.Params("*1") - - // Only process directory-like keys (empty or ending with /) - if key != "" && !strings.HasSuffix(key, "/") { - return ctx.Next() - } - - // Reject path traversal attempts - if strings.Contains(key, "..") { - return ctx.Next() - } - - data, err := be.GetBucketWebsite(ctx.Context(), bucket) - if err != nil { - // No website config: pass through to normal handling - return ctx.Next() - } - - var config s3response.WebsiteConfiguration - if xmlErr := xml.Unmarshal(data, &config); xmlErr != nil { - return ctx.Next() - } - - // Handle RedirectAllRequestsTo - if config.RedirectAllRequestsTo != nil { - return redirectAll(ctx, config.RedirectAllRequestsTo, key) - } - - // Rewrite directory-like keys to include index document suffix - if config.IndexDocument != nil && config.IndexDocument.Suffix != "" { - newKey := key + config.IndexDocument.Suffix - newPath := fmt.Sprintf("/%s/%s", bucket, newKey) - ctx.Request().URI().SetPath(newPath) - } - - return ctx.Next() - } -} - -func redirectAll(ctx *fiber.Ctx, redirect *s3response.RedirectAllRequestsTo, key string) error { - protocol := redirect.Protocol - if protocol == "" { - protocol = "https" - } - - location := fmt.Sprintf("%s://%s/%s", protocol, redirect.HostName, key) - ctx.Set("Location", location) - return ctx.SendStatus(http.StatusMovedPermanently) -} diff --git a/s3api/router.go b/s3api/router.go index b735ebbc..e4b32432 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -451,6 +451,8 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionPutBucketWebsite, auth.PutBucketWebsiteAction, auth.PermissionWrite, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), + middlewares.VerifyChecksums(false, true, false), + middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), middlewares.ParseAcl(sa.be), ), ) @@ -669,9 +671,10 @@ func (sa *S3ApiRouter) Init() { metrics.ActionDeleteBucketWebsite, services, middlewares.BucketObjectNameValidator(), - middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionDeleteBucketWebsite, auth.PutBucketWebsiteAction, auth.PermissionWrite, sa.region, false), + middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionDeleteBucketWebsite, auth.DeleteBucketWebsiteAction, auth.PermissionWrite, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), + middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), middlewares.ParseAcl(sa.be), ), ) @@ -1063,6 +1066,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionGetBucketWebsite, auth.GetBucketWebsiteAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), + middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), middlewares.ParseAcl(sa.be), ), ) @@ -1152,7 +1156,6 @@ func (sa *S3ApiRouter) Init() { metrics.ActionHeadObject, services, middlewares.BucketObjectNameValidator(), - middlewares.ResolveWebsiteIndex(sa.be), middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionHeadObject, auth.GetObjectAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, false, false), @@ -1270,7 +1273,6 @@ func (sa *S3ApiRouter) Init() { metrics.ActionGetObject, services, middlewares.BucketObjectNameValidator(), - middlewares.ResolveWebsiteIndex(sa.be), middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionGetObject, auth.GetObjectAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), diff --git a/s3api/utils/context-keys.go b/s3api/utils/context-keys.go index dd642efa..445531c3 100644 --- a/s3api/utils/context-keys.go +++ b/s3api/utils/context-keys.go @@ -40,6 +40,7 @@ const ( ContextKeyObjectPostResult ContextKey = "object-post-result" ContextKeyRequestID ContextKey = "request-id" ContextKeyHostID ContextKey = "host-id" + ContextKeyWebsiteConfig ContextKey = "website-config" ) func (ck ContextKey) Set(ctx *fiber.Ctx, val any) { diff --git a/s3err/s3err.go b/s3err/s3err.go index 3be9fdc7..ac3fe57c 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -649,8 +649,8 @@ var errorCodeResponse = map[ErrorCode]APIError{ HTTPStatusCode: http.StatusNotFound, }, ErrInvalidWebsiteConfiguration: { - Code: "InvalidRequest", - Description: "The website configuration is not valid.", + Code: "MalformedXML", + Description: "The XML you provided was not well-formed or did not validate against our published schema.", HTTPStatusCode: http.StatusBadRequest, }, ErrInvalidWebsiteSuffix: { diff --git a/s3response/website.go b/s3response/website.go index 85d2bf24..16884e38 100644 --- a/s3response/website.go +++ b/s3response/website.go @@ -1,4 +1,4 @@ -// Copyright 2023 Versity Software +// Copyright 2026 Versity Software // This file is licensed under the Apache License, Version 2.0 // (the "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at @@ -149,3 +149,50 @@ func ParseWebsiteConfigOutput(data []byte) (*WebsiteConfiguration, error) { return &config, nil } + +// MatchPreRequestRule returns the first routing rule that matches based only +// on KeyPrefixEquals (i.e. rules without HttpErrorCodeReturnedEquals). These +// rules can be evaluated before the backend request is made. A rule with no +// condition at all is treated as an unconditional match. +func (c *WebsiteConfiguration) MatchPreRequestRule(key string) *RoutingRule { + for i := range c.RoutingRules { + rule := &c.RoutingRules[i] + + if rule.Condition != nil && rule.Condition.HttpErrorCodeReturnedEquals != "" { + // This is a post-request rule, skip it + continue + } + + if rule.Condition == nil || strings.HasPrefix(key, rule.Condition.KeyPrefixEquals) { + return rule + } + } + + return nil +} + +// MatchPostRequestRule returns the first routing rule that matches based on +// HttpErrorCodeReturnedEquals (and optionally KeyPrefixEquals). These rules +// are evaluated after the backend returns an error. +func (c *WebsiteConfiguration) MatchPostRequestRule(key, httpErrorCode string) *RoutingRule { + for i := range c.RoutingRules { + rule := &c.RoutingRules[i] + + if rule.Condition == nil || rule.Condition.HttpErrorCodeReturnedEquals == "" { + // Not a post-request rule + continue + } + + if rule.Condition.HttpErrorCodeReturnedEquals != httpErrorCode { + continue + } + + if rule.Condition.KeyPrefixEquals != "" && !strings.HasPrefix(key, rule.Condition.KeyPrefixEquals) { + continue + } + + return rule + } + + return nil +} diff --git a/s3response/website_test.go b/s3response/website_test.go index a8f6b4fb..b644a154 100644 --- a/s3response/website_test.go +++ b/s3response/website_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 Versity Software +// Copyright 2026 Versity Software // This file is licensed under the Apache License, Version 2.0 // (the "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at @@ -272,3 +272,316 @@ func TestParseWebsiteConfigOutput_InvalidXML(t *testing.T) { t.Fatal("expected error for invalid XML") } } + +func TestWebsiteConfiguration_MatchPreRequestRule(t *testing.T) { + tests := []struct { + name string + config WebsiteConfiguration + key string + wantNil bool + wantHost string // expected redirect HostName if matched + }{ + { + name: "no routing rules", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + }, + key: "docs/page.html", + wantNil: true, + }, + { + name: "key prefix match", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + RoutingRules: []RoutingRule{ + { + Condition: &RoutingRuleCondition{ + KeyPrefixEquals: "docs/", + }, + Redirect: Redirect{ + HostName: "docs.example.com", + }, + }, + }, + }, + key: "docs/page.html", + wantHost: "docs.example.com", + }, + { + name: "key prefix does not match", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + RoutingRules: []RoutingRule{ + { + Condition: &RoutingRuleCondition{ + KeyPrefixEquals: "docs/", + }, + Redirect: Redirect{ + HostName: "docs.example.com", + }, + }, + }, + }, + key: "images/photo.jpg", + wantNil: true, + }, + { + name: "unconditional rule (no condition)", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + RoutingRules: []RoutingRule{ + { + Redirect: Redirect{ + HostName: "redirect.example.com", + }, + }, + }, + }, + key: "anything", + wantHost: "redirect.example.com", + }, + { + name: "skips post-request rules", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + RoutingRules: []RoutingRule{ + { + Condition: &RoutingRuleCondition{ + HttpErrorCodeReturnedEquals: "404", + KeyPrefixEquals: "docs/", + }, + Redirect: Redirect{ + HostName: "error.example.com", + }, + }, + }, + }, + key: "docs/page.html", + wantNil: true, + }, + { + name: "first matching rule wins", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + RoutingRules: []RoutingRule{ + { + Condition: &RoutingRuleCondition{ + KeyPrefixEquals: "docs/", + }, + Redirect: Redirect{ + HostName: "first.example.com", + }, + }, + { + Condition: &RoutingRuleCondition{ + KeyPrefixEquals: "docs/api/", + }, + Redirect: Redirect{ + HostName: "second.example.com", + }, + }, + }, + }, + key: "docs/api/endpoint", + wantHost: "first.example.com", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rule := tt.config.MatchPreRequestRule(tt.key) + if tt.wantNil { + if rule != nil { + t.Fatalf("expected nil, got rule with redirect to %q", rule.Redirect.HostName) + } + return + } + if rule == nil { + t.Fatal("expected a matching rule, got nil") + } + if rule.Redirect.HostName != tt.wantHost { + t.Errorf("expected redirect host %q, got %q", tt.wantHost, rule.Redirect.HostName) + } + }) + } +} + +func TestWebsiteConfiguration_MatchPostRequestRule(t *testing.T) { + tests := []struct { + name string + config WebsiteConfiguration + key string + httpErrorCode string + wantNil bool + wantHost string + }{ + { + name: "no routing rules", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + }, + key: "page.html", + httpErrorCode: "404", + wantNil: true, + }, + { + name: "error code match", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + RoutingRules: []RoutingRule{ + { + Condition: &RoutingRuleCondition{ + HttpErrorCodeReturnedEquals: "404", + }, + Redirect: Redirect{ + HostName: "notfound.example.com", + }, + }, + }, + }, + key: "page.html", + httpErrorCode: "404", + wantHost: "notfound.example.com", + }, + { + name: "error code does not match", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + RoutingRules: []RoutingRule{ + { + Condition: &RoutingRuleCondition{ + HttpErrorCodeReturnedEquals: "404", + }, + Redirect: Redirect{ + HostName: "notfound.example.com", + }, + }, + }, + }, + key: "page.html", + httpErrorCode: "403", + wantNil: true, + }, + { + name: "error code and key prefix both match", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + RoutingRules: []RoutingRule{ + { + Condition: &RoutingRuleCondition{ + HttpErrorCodeReturnedEquals: "404", + KeyPrefixEquals: "docs/", + }, + Redirect: Redirect{ + HostName: "docs-error.example.com", + }, + }, + }, + }, + key: "docs/missing.html", + httpErrorCode: "404", + wantHost: "docs-error.example.com", + }, + { + name: "error code matches but key prefix does not", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + RoutingRules: []RoutingRule{ + { + Condition: &RoutingRuleCondition{ + HttpErrorCodeReturnedEquals: "404", + KeyPrefixEquals: "docs/", + }, + Redirect: Redirect{ + HostName: "docs-error.example.com", + }, + }, + }, + }, + key: "images/missing.jpg", + httpErrorCode: "404", + wantNil: true, + }, + { + name: "skips pre-request rules (no error code condition)", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + RoutingRules: []RoutingRule{ + { + Condition: &RoutingRuleCondition{ + KeyPrefixEquals: "docs/", + }, + Redirect: Redirect{ + HostName: "pre-request.example.com", + }, + }, + }, + }, + key: "docs/page.html", + httpErrorCode: "404", + wantNil: true, + }, + { + name: "skips rules with no condition", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + RoutingRules: []RoutingRule{ + { + Redirect: Redirect{ + HostName: "unconditional.example.com", + }, + }, + }, + }, + key: "page.html", + httpErrorCode: "404", + wantNil: true, + }, + { + name: "first matching rule wins", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + RoutingRules: []RoutingRule{ + { + Condition: &RoutingRuleCondition{ + HttpErrorCodeReturnedEquals: "404", + }, + Redirect: Redirect{ + HostName: "first.example.com", + }, + }, + { + Condition: &RoutingRuleCondition{ + HttpErrorCodeReturnedEquals: "404", + KeyPrefixEquals: "docs/", + }, + Redirect: Redirect{ + HostName: "second.example.com", + }, + }, + }, + }, + key: "docs/page.html", + httpErrorCode: "404", + wantHost: "first.example.com", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rule := tt.config.MatchPostRequestRule(tt.key, tt.httpErrorCode) + if tt.wantNil { + if rule != nil { + t.Fatalf("expected nil, got rule with redirect to %q", rule.Redirect.HostName) + } + return + } + if rule == nil { + t.Fatal("expected a matching rule, got nil") + } + if rule.Redirect.HostName != tt.wantHost { + t.Errorf("expected redirect host %q, got %q", tt.wantHost, rule.Redirect.HostName) + } + }) + } +} diff --git a/tests/integration/DeleteBucketWebsite.go b/tests/integration/DeleteBucketWebsite.go new file mode 100644 index 00000000..ad9e5ac2 --- /dev/null +++ b/tests/integration/DeleteBucketWebsite.go @@ -0,0 +1,85 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package integration + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/versity/versitygw/s3err" +) + +func DeleteBucketWebsite_non_existing_bucket(s *S3Conf) error { + testName := "DeleteBucketWebsite_non_existing_bucket" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteBucketWebsite(ctx, &s3.DeleteBucketWebsiteInput{ + Bucket: getPtr("non-existing-bucket"), + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)) + }) +} + +func DeleteBucketWebsite_success(s *S3Conf) error { + testName := "DeleteBucketWebsite_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + deleteWebsite := func() error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteBucketWebsite(ctx, &s3.DeleteBucketWebsiteInput{ + Bucket: &bucket, + }) + cancel() + return err + } + + // should not return error when deleting unset website config + err := deleteWebsite() + if err != nil { + return err + } + + // put a website config + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + }, + }) + cancel() + if err != nil { + return err + } + + // delete the website config + err = deleteWebsite() + if err != nil { + return err + } + + // verify it's gone + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetBucketWebsite(ctx, &s3.GetBucketWebsiteInput{ + Bucket: &bucket, + }) + cancel() + + return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchWebsiteConfiguration)) + }) +} diff --git a/tests/integration/GetBucketWebsite.go b/tests/integration/GetBucketWebsite.go new file mode 100644 index 00000000..7dfdf194 --- /dev/null +++ b/tests/integration/GetBucketWebsite.go @@ -0,0 +1,126 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package integration + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/versity/versitygw/s3err" +) + +func GetBucketWebsite_non_existing_bucket(s *S3Conf) error { + testName := "GetBucketWebsite_non_existing_bucket" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.GetBucketWebsite(ctx, &s3.GetBucketWebsiteInput{ + Bucket: getPtr("non-existing-bucket"), + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)) + }) +} + +func GetBucketWebsite_no_such_website_config(s *S3Conf) error { + testName := "GetBucketWebsite_no_such_website_config" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.GetBucketWebsite(ctx, &s3.GetBucketWebsiteInput{ + Bucket: &bucket, + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchWebsiteConfiguration)) + }) +} + +func GetBucketWebsite_success(s *S3Conf) error { + testName := "GetBucketWebsite_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + ErrorDocument: &types.ErrorDocument{ + Key: getPtr("error.html"), + }, + }, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetBucketWebsite(ctx, &s3.GetBucketWebsiteInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if res.IndexDocument == nil || res.IndexDocument.Suffix == nil || *res.IndexDocument.Suffix != "index.html" { + return fmt.Errorf("expected IndexDocument.Suffix to be %q, got %v", "index.html", res.IndexDocument) + } + if res.ErrorDocument == nil || res.ErrorDocument.Key == nil || *res.ErrorDocument.Key != "error.html" { + return fmt.Errorf("expected ErrorDocument.Key to be %q, got %v", "error.html", res.ErrorDocument) + } + + return nil + }) +} + +func GetBucketWebsite_success_redirect_all(s *S3Conf) error { + testName := "GetBucketWebsite_success_redirect_all" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + RedirectAllRequestsTo: &types.RedirectAllRequestsTo{ + HostName: getPtr("example.com"), + Protocol: types.ProtocolHttps, + }, + }, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetBucketWebsite(ctx, &s3.GetBucketWebsiteInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if res.RedirectAllRequestsTo == nil || res.RedirectAllRequestsTo.HostName == nil || *res.RedirectAllRequestsTo.HostName != "example.com" { + return fmt.Errorf("expected RedirectAllRequestsTo.HostName to be %q, got %v", "example.com", res.RedirectAllRequestsTo) + } + if res.RedirectAllRequestsTo.Protocol != types.ProtocolHttps { + return fmt.Errorf("expected RedirectAllRequestsTo.Protocol to be %q, got %q", types.ProtocolHttps, res.RedirectAllRequestsTo.Protocol) + } + + return nil + }) +} diff --git a/tests/integration/NotImplemented_actions.go b/tests/integration/NotImplemented_actions.go index 4beea7d6..24e9ff3a 100644 --- a/tests/integration/NotImplemented_actions.go +++ b/tests/integration/NotImplemented_actions.go @@ -655,53 +655,6 @@ func GetBucketAccelerateConfiguration_not_implemented(s *S3Conf) error { }) } -func PutBucketWebsite_not_implemented(s *S3Conf) error { - testName := "PutBucketWebsite_not_implemented" - return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.PutBucketWebsite(ctx, - &s3.PutBucketWebsiteInput{ - Bucket: &bucket, - WebsiteConfiguration: &types.WebsiteConfiguration{ - IndexDocument: &types.IndexDocument{ - Suffix: getPtr("suffix"), - }, - }, - }) - cancel() - - return checkApiErr(err, s3err.GetAPIError(s3err.ErrNotImplemented)) - }) -} - -func GetBucketWebsite_not_implemented(s *S3Conf) error { - testName := "GetBucketWebsite_not_implemented" - return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.GetBucketWebsite(ctx, - &s3.GetBucketWebsiteInput{ - Bucket: &bucket, - }) - cancel() - - return checkApiErr(err, s3err.GetAPIError(s3err.ErrNotImplemented)) - }) -} - -func DeleteBucketWebsite_not_implemented(s *S3Conf) error { - testName := "DeleteBucketWebsite_not_implemented" - return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.DeleteBucketWebsite(ctx, - &s3.DeleteBucketWebsiteInput{ - Bucket: &bucket, - }) - cancel() - - return checkApiErr(err, s3err.GetAPIError(s3err.ErrNotImplemented)) - }) -} - func PutObjectAcl_not_implemented(s *S3Conf) error { testName := "PutObjectAcl_not_implemented" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { diff --git a/tests/integration/PutBucketWebsite.go b/tests/integration/PutBucketWebsite.go new file mode 100644 index 00000000..11e97f38 --- /dev/null +++ b/tests/integration/PutBucketWebsite.go @@ -0,0 +1,158 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package integration + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/versity/versitygw/s3err" +) + +func PutBucketWebsite_non_existing_bucket(s *S3Conf) error { + testName := "PutBucketWebsite_non_existing_bucket" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: getPtr("non-existing-bucket"), + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + }, + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)) + }) +} + +func PutBucketWebsite_empty_suffix(s *S3Conf) error { + testName := "PutBucketWebsite_empty_suffix" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr(""), + }, + }, + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidWebsiteSuffix)) + }) +} + +func PutBucketWebsite_suffix_with_slash(s *S3Conf) error { + testName := "PutBucketWebsite_suffix_with_slash" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("/index.html"), + }, + }, + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidWebsiteSuffix)) + }) +} + +func PutBucketWebsite_invalid_redirect_protocol(s *S3Conf) error { + testName := "PutBucketWebsite_invalid_redirect_protocol" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + RedirectAllRequestsTo: &types.RedirectAllRequestsTo{ + HostName: getPtr("example.com"), + Protocol: types.Protocol("ftp"), + }, + }, + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidWebsiteConfiguration)) + }) +} + +func PutBucketWebsite_redirect_and_index(s *S3Conf) error { + testName := "PutBucketWebsite_redirect_and_index" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + RedirectAllRequestsTo: &types.RedirectAllRequestsTo{ + HostName: getPtr("example.com"), + }, + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + }, + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidWebsiteConfiguration)) + }) +} + +func PutBucketWebsite_success(s *S3Conf) error { + testName := "PutBucketWebsite_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + ErrorDocument: &types.ErrorDocument{ + Key: getPtr("error.html"), + }, + }, + }) + cancel() + if err != nil { + return err + } + + return nil + }) +} + +func PutBucketWebsite_success_redirect_all(s *S3Conf) error { + testName := "PutBucketWebsite_success_redirect_all" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + RedirectAllRequestsTo: &types.RedirectAllRequestsTo{ + HostName: getPtr("example.com"), + Protocol: types.ProtocolHttps, + }, + }, + }) + cancel() + if err != nil { + return err + } + + return nil + }) +} diff --git a/tests/integration/WebsiteHosting.go b/tests/integration/WebsiteHosting.go new file mode 100644 index 00000000..453a02b2 --- /dev/null +++ b/tests/integration/WebsiteHosting.go @@ -0,0 +1,413 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package integration + +import ( + "bytes" + "context" + "crypto/tls" + "fmt" + "io" + "net/http" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + +// websiteHTTPClient returns an HTTP client suitable for website endpoint +// requests. It does not follow redirects and skips TLS verification +// (matching the behaviour of the S3Conf http client for self-signed certs). +func websiteHTTPClient() *http.Client { + return &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } +} + +// websiteGet issues a plain HTTP GET to the dedicated website endpoint. +// The bucket is resolved from the Host header. No S3 signing is applied. +func websiteGet(websiteEndpoint, host, path string) (*http.Response, error) { + url := fmt.Sprintf("%s/%s", strings.TrimRight(websiteEndpoint, "/"), strings.TrimLeft(path, "/")) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + req.Host = host + return websiteHTTPClient().Do(req) +} + +// WebsiteHosting_error_document_served tests that when a website-enabled +// bucket has an error document configured, requesting a non-existing key +// returns the error document content with the original 404 status code. +func WebsiteHosting_error_document_served(s *S3Conf) error { + testName := "WebsiteHosting_error_document_served" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + // Configure website with error document + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + ErrorDocument: &types.ErrorDocument{ + Key: getPtr("error.html"), + }, + }, + }) + cancel() + if err != nil { + return err + } + + // Upload the error document + errorContent := "Custom Error Page" + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: getPtr("error.html"), + Body: strings.NewReader(errorContent), + ContentType: getPtr("text/html"), + }) + cancel() + if err != nil { + return err + } + + // Request a non-existing key via plain HTTP on the website endpoint + resp, err := websiteGet(s.websiteEndpoint, bucket, "nonexistent-key") + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNotFound { + return fmt.Errorf("expected status 404, got %v", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + if string(body) != errorContent { + return fmt.Errorf("expected error document content %q, got %q", errorContent, string(body)) + } + + return nil + }) +} + +// WebsiteHosting_error_document_not_found tests that when the configured +// error document itself does not exist, a 404 error page is returned. +func WebsiteHosting_error_document_not_found(s *S3Conf) error { + testName := "WebsiteHosting_error_document_not_found" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + // Configure website with error document (but don't upload it) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + ErrorDocument: &types.ErrorDocument{ + Key: getPtr("error.html"), + }, + }, + }) + cancel() + if err != nil { + return err + } + + // Request a non-existing key - should get 404 since error doc doesn't exist either + resp, err := websiteGet(s.websiteEndpoint, bucket, "nonexistent-key") + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNotFound { + return fmt.Errorf("expected status 404, got %v", resp.StatusCode) + } + + return nil + }) +} + +// WebsiteHosting_no_error_document tests that when website is enabled +// but no error document is configured, a 404 error page is returned. +func WebsiteHosting_no_error_document(s *S3Conf) error { + testName := "WebsiteHosting_no_error_document" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + // Configure website without error document + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + }, + }) + cancel() + if err != nil { + return err + } + + // Request a non-existing key - should get 404 + resp, err := websiteGet(s.websiteEndpoint, bucket, "nonexistent-key") + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNotFound { + return fmt.Errorf("expected status 404, got %v", resp.StatusCode) + } + + return nil + }) +} + +// WebsiteHosting_routing_rule_post_request_redirect tests that a post-request +// routing rule (matching on error code) issues a redirect instead of serving +// the error or error document. +func WebsiteHosting_routing_rule_post_request_redirect(s *S3Conf) error { + testName := "WebsiteHosting_routing_rule_post_request_redirect" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + // Configure website with a post-request routing rule for 404 + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + ErrorDocument: &types.ErrorDocument{ + Key: getPtr("error.html"), + }, + RoutingRules: []types.RoutingRule{ + { + Condition: &types.Condition{ + HttpErrorCodeReturnedEquals: getPtr("404"), + }, + Redirect: &types.Redirect{ + HostName: getPtr("fallback.example.com"), + ReplaceKeyWith: getPtr("not-found"), + HttpRedirectCode: getPtr("302"), + }, + }, + }, + }, + }) + cancel() + if err != nil { + return err + } + + // Request a non-existing key via the website endpoint + resp, err := websiteGet(s.websiteEndpoint, bucket, "missing-page") + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusFound { + return fmt.Errorf("expected status 302, got %v", resp.StatusCode) + } + + location := resp.Header.Get("Location") + if location == "" { + return fmt.Errorf("expected Location header, got none") + } + + // The redirect should point to fallback.example.com/not-found + if !strings.Contains(location, "fallback.example.com") || !strings.Contains(location, "not-found") { + return fmt.Errorf("expected redirect to fallback.example.com/not-found, got %q", location) + } + + return nil + }) +} + +// WebsiteHosting_routing_rule_pre_request_redirect tests that a pre-request +// routing rule (matching on key prefix only) issues a redirect before the +// object is fetched. +func WebsiteHosting_routing_rule_pre_request_redirect(s *S3Conf) error { + testName := "WebsiteHosting_routing_rule_pre_request_redirect" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + // Configure website with a pre-request routing rule + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + RoutingRules: []types.RoutingRule{ + { + Condition: &types.Condition{ + KeyPrefixEquals: getPtr("old-docs/"), + }, + Redirect: &types.Redirect{ + ReplaceKeyPrefixWith: getPtr("new-docs/"), + HttpRedirectCode: getPtr("301"), + }, + }, + }, + }, + }) + cancel() + if err != nil { + return err + } + + // Request old-docs/page.html via the website endpoint + resp, err := websiteGet(s.websiteEndpoint, bucket, "old-docs/page.html") + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusMovedPermanently { + return fmt.Errorf("expected status 301, got %v", resp.StatusCode) + } + + location := resp.Header.Get("Location") + if location == "" { + return fmt.Errorf("expected Location header, got none") + } + + // The redirect should rewrite old-docs/ -> new-docs/ + if !strings.Contains(location, "new-docs/page.html") { + return fmt.Errorf("expected redirect to contain new-docs/page.html, got %q", location) + } + + return nil + }) +} + +// WebsiteHosting_redirect_all_requests tests the RedirectAllRequestsTo +// configuration, which should redirect any request to the specified host. +func WebsiteHosting_redirect_all_requests(s *S3Conf) error { + testName := "WebsiteHosting_redirect_all_requests" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + // Configure redirect-all + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + RedirectAllRequestsTo: &types.RedirectAllRequestsTo{ + HostName: getPtr("www.example.com"), + Protocol: types.ProtocolHttps, + }, + }, + }) + cancel() + if err != nil { + return err + } + + // Request any path via the website endpoint + resp, err := websiteGet(s.websiteEndpoint, bucket, "any/path/here") + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusMovedPermanently { + return fmt.Errorf("expected status 301, got %v", resp.StatusCode) + } + + location := resp.Header.Get("Location") + if !strings.HasPrefix(location, "https://www.example.com/") { + return fmt.Errorf("expected redirect to https://www.example.com/, got %q", location) + } + + if !strings.Contains(location, "any/path/here") { + return fmt.Errorf("expected redirect to preserve path, got %q", location) + } + + return nil + }) +} + +// WebsiteHosting_index_document tests that requesting a directory-like +// path on a website-enabled bucket serves the index document. +func WebsiteHosting_index_document(s *S3Conf) error { + testName := "WebsiteHosting_index_document" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + // Configure website + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + }, + }) + cancel() + if err != nil { + return err + } + + // Upload index document at root + indexContent := "Welcome" + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: getPtr("index.html"), + Body: strings.NewReader(indexContent), + ContentType: getPtr("text/html"), + }) + cancel() + if err != nil { + return err + } + + // Request the root path via the website endpoint + resp, err := websiteGet(s.websiteEndpoint, bucket, "/") + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("expected status 200, got %v; body: %s", resp.StatusCode, body) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + if !bytes.Equal(body, []byte(indexContent)) { + return fmt.Errorf("expected index document content %q, got %q", indexContent, string(body)) + } + + return nil + }) +} diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index fe8b19d5..323b20fe 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -14,6 +14,8 @@ package integration +import "fmt" + func TestAuthentication(ts *TestState) { ts.Run(Authentication_invalid_auth_header) ts.Run(Authentication_unsupported_signature_version) @@ -663,6 +665,42 @@ func TestDeleteBucketCors(ts *TestState) { ts.Run(DeleteBucketCors_success) } +func TestPutBucketWebsite(ts *TestState) { + ts.Run(PutBucketWebsite_non_existing_bucket) + ts.Run(PutBucketWebsite_empty_suffix) + ts.Run(PutBucketWebsite_suffix_with_slash) + ts.Run(PutBucketWebsite_invalid_redirect_protocol) + ts.Run(PutBucketWebsite_redirect_and_index) + ts.Run(PutBucketWebsite_success) + ts.Run(PutBucketWebsite_success_redirect_all) +} + +func TestGetBucketWebsite(ts *TestState) { + ts.Run(GetBucketWebsite_non_existing_bucket) + ts.Run(GetBucketWebsite_no_such_website_config) + ts.Run(GetBucketWebsite_success) + ts.Run(GetBucketWebsite_success_redirect_all) +} + +func TestDeleteBucketWebsite(ts *TestState) { + ts.Run(DeleteBucketWebsite_non_existing_bucket) + ts.Run(DeleteBucketWebsite_success) +} + +func TestWebsiteHosting(ts *TestState) { + if ts.conf.websiteEndpoint == "" { + fmt.Println("skipping TestWebsiteHosting: no website endpoint configured") + return + } + ts.Run(WebsiteHosting_error_document_served) + ts.Run(WebsiteHosting_error_document_not_found) + ts.Run(WebsiteHosting_no_error_document) + ts.Run(WebsiteHosting_routing_rule_post_request_redirect) + ts.Run(WebsiteHosting_routing_rule_pre_request_redirect) + ts.Run(WebsiteHosting_redirect_all_requests) + ts.Run(WebsiteHosting_index_document) +} + func TestPreflightOPTIONSEndpoint(ts *TestState) { ts.Run(PreflightOPTIONS_non_existing_bucket) ts.Run(PreflightOPTIONS_missing_origin) @@ -788,10 +826,6 @@ func TestNotImplementedActions(ts *TestState) { // bucket acceleration actions ts.Run(PutBucketAccelerateConfiguration_not_implemented) ts.Run(GetBucketAccelerateConfiguration_not_implemented) - // bucket website actions - ts.Run(PutBucketWebsite_not_implemented) - ts.Run(GetBucketWebsite_not_implemented) - ts.Run(DeleteBucketWebsite_not_implemented) // object acl actions ts.Run(PutObjectAcl_not_implemented) ts.Run(GetObjectAcl_not_implemented) @@ -862,6 +896,10 @@ func TestFullFlow(ts *TestState) { TestPutBucketCors(ts) TestGetBucketCors(ts) TestDeleteBucketCors(ts) + TestPutBucketWebsite(ts) + TestGetBucketWebsite(ts) + TestDeleteBucketWebsite(ts) + TestWebsiteHosting(ts) TestPreflightOPTIONSEndpoint(ts) TestPutObjectLockConfiguration(ts) TestGetObjectLockConfiguration(ts) @@ -1760,6 +1798,26 @@ func GetIntTests() IntTests { "DeleteBucketCors_non_existing_bucket": DeleteBucketCors_non_existing_bucket, "DeleteBucketCors_success": DeleteBucketCors_success, "PutBucketCors_success": PutBucketCors_success, + "PutBucketWebsite_non_existing_bucket": PutBucketWebsite_non_existing_bucket, + "PutBucketWebsite_empty_suffix": PutBucketWebsite_empty_suffix, + "PutBucketWebsite_suffix_with_slash": PutBucketWebsite_suffix_with_slash, + "PutBucketWebsite_invalid_redirect_protocol": PutBucketWebsite_invalid_redirect_protocol, + "PutBucketWebsite_redirect_and_index": PutBucketWebsite_redirect_and_index, + "PutBucketWebsite_success": PutBucketWebsite_success, + "PutBucketWebsite_success_redirect_all": PutBucketWebsite_success_redirect_all, + "GetBucketWebsite_non_existing_bucket": GetBucketWebsite_non_existing_bucket, + "GetBucketWebsite_no_such_website_config": GetBucketWebsite_no_such_website_config, + "GetBucketWebsite_success": GetBucketWebsite_success, + "GetBucketWebsite_success_redirect_all": GetBucketWebsite_success_redirect_all, + "DeleteBucketWebsite_non_existing_bucket": DeleteBucketWebsite_non_existing_bucket, + "DeleteBucketWebsite_success": DeleteBucketWebsite_success, + "WebsiteHosting_error_document_served": WebsiteHosting_error_document_served, + "WebsiteHosting_error_document_not_found": WebsiteHosting_error_document_not_found, + "WebsiteHosting_no_error_document": WebsiteHosting_no_error_document, + "WebsiteHosting_routing_rule_post_request_redirect": WebsiteHosting_routing_rule_post_request_redirect, + "WebsiteHosting_routing_rule_pre_request_redirect": WebsiteHosting_routing_rule_pre_request_redirect, + "WebsiteHosting_redirect_all_requests": WebsiteHosting_redirect_all_requests, + "WebsiteHosting_index_document": WebsiteHosting_index_document, "PreflightOPTIONS_non_existing_bucket": PreflightOPTIONS_non_existing_bucket, "PreflightOPTIONS_missing_origin": PreflightOPTIONS_missing_origin, "PreflightOPTIONS_invalid_request_method": PreflightOPTIONS_invalid_request_method, @@ -1846,9 +1904,6 @@ func GetIntTests() IntTests { "GetBucketNotificationConfiguratio_not_implemented": GetBucketNotificationConfiguratio_not_implemented, "PutBucketAccelerateConfiguration_not_implemented": PutBucketAccelerateConfiguration_not_implemented, "GetBucketAccelerateConfiguration_not_implemented": GetBucketAccelerateConfiguration_not_implemented, - "PutBucketWebsite_not_implemented": PutBucketWebsite_not_implemented, - "GetBucketWebsite_not_implemented": GetBucketWebsite_not_implemented, - "DeleteBucketWebsite_not_implemented": DeleteBucketWebsite_not_implemented, "PutObjectAcl_not_implemented": PutObjectAcl_not_implemented, "GetObjectAcl_not_implemented": GetObjectAcl_not_implemented, "WORMProtection_bucket_object_lock_configuration_compliance_mode": WORMProtection_bucket_object_lock_configuration_compliance_mode, diff --git a/tests/integration/s3conf.go b/tests/integration/s3conf.go index e49b892b..6382f07a 100644 --- a/tests/integration/s3conf.go +++ b/tests/integration/s3conf.go @@ -36,6 +36,7 @@ type S3Conf struct { awsSecret string awsRegion string endpoint string + websiteEndpoint string hostStyle bool checksumDisable bool PartSize int64 @@ -85,6 +86,9 @@ func WithRegion(r string) Option { func WithEndpoint(e string) Option { return func(s *S3Conf) { s.endpoint = e } } +func WithWebsiteEndpoint(e string) Option { + return func(s *S3Conf) { s.websiteEndpoint = e } +} func WithDisableChecksum() Option { return func(s *S3Conf) { s.checksumDisable = true } } diff --git a/website/handler.go b/website/handler.go new file mode 100644 index 00000000..463ceec7 --- /dev/null +++ b/website/handler.go @@ -0,0 +1,314 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package website + +import ( + "encoding/xml" + "fmt" + "html" + "io" + "net/http" + "strconv" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3response" +) + +// newHandler returns a fiber handler that serves static website content. +// It resolves the bucket name from the Host header using the configured domain, +// fetches the website configuration, and serves objects accordingly. +// +// Virtual-host routing with --website-domain example.com: +// - Host "blog.example.com" -> bucket "blog" +// - Host "example.com" -> bucket "example.com" (apex) +// +// Catch-all mode (--website-domain omitted or empty): +// - Host "blog.example.com" -> bucket "blog.example.com" +// - Host "mysite.org" -> bucket "mysite.org" +func newHandler(be backend.Backend, domain string) fiber.Handler { + // Pre-compute the domain suffix for subdomain extraction. + // Given domain "example.com", we look for ".example.com" suffix. + domainSuffix := "." + domain + + return func(ctx *fiber.Ctx) error { + host := ctx.Hostname() + if host == "" { + return sendError(ctx, http.StatusBadRequest, "Bad Request", "Missing Host header") + } + + // Strip port from host if present + if idx := strings.LastIndex(host, ":"); idx != -1 { + // Be careful with IPv6: only strip if it's not inside brackets + if !strings.Contains(host[idx:], "]") { + host = host[:idx] + } + } + + // Resolve bucket name from host + bucket := resolveBucket(host, domain, domainSuffix) + if bucket == "" { + return sendError(ctx, http.StatusForbidden, "Forbidden", + fmt.Sprintf("No bucket could be resolved from host %q", html.EscapeString(ctx.Hostname()))) + } + + // Fetch website configuration + data, err := be.GetBucketWebsite(ctx.Context(), bucket) + if err != nil { + return sendError(ctx, http.StatusNotFound, "Not Found", + fmt.Sprintf("No website configuration for bucket %q", bucket)) + } + + var config s3response.WebsiteConfiguration + if xmlErr := xml.Unmarshal(data, &config); xmlErr != nil { + return sendError(ctx, http.StatusInternalServerError, "Internal Server Error", + "Invalid website configuration") + } + + key := strings.TrimPrefix(ctx.Path(), "/") + + // Handle RedirectAllRequestsTo + if config.RedirectAllRequestsTo != nil { + return handleRedirectAll(ctx, config.RedirectAllRequestsTo, key) + } + + // Evaluate pre-request routing rules + if rule := config.MatchPreRequestRule(key); rule != nil { + return applyRedirect(ctx, &rule.Redirect, rule.Condition, key) + } + + // Rewrite directory-like keys to include index document suffix + if config.IndexDocument != nil && config.IndexDocument.Suffix != "" { + if key == "" || strings.HasSuffix(key, "/") { + key = key + config.IndexDocument.Suffix + } + } + + // Fetch the object + emptyRange := "" + result, getErr := be.GetObject(ctx.Context(), &s3.GetObjectInput{ + Bucket: &bucket, + Key: &key, + Range: &emptyRange, + }) + if getErr == nil && result.Body != nil { + defer result.Body.Close() + return serveObject(ctx, result, key) + } + + // Object not found (or other error) — evaluate post-request routing rules + httpErrCode := http.StatusNotFound + errorCode := strconv.Itoa(httpErrCode) + + if rule := config.MatchPostRequestRule(key, errorCode); rule != nil { + return applyRedirect(ctx, &rule.Redirect, rule.Condition, key) + } + + // Serve error document if configured + if config.ErrorDocument != nil && config.ErrorDocument.Key != "" { + return serveErrorDocument(ctx, be, bucket, config.ErrorDocument.Key, httpErrCode) + } + + return sendError(ctx, http.StatusNotFound, "Not Found", + fmt.Sprintf("The specified key %q does not exist", key)) + } +} + +// resolveBucket extracts the bucket name from the host header. +// +// When domain is set: +// - If host equals the domain exactly, the bucket IS the domain (apex). +// - If host ends with ".", the bucket is the subdomain part. +// - Otherwise, no bucket can be resolved. +// +// When domain is empty (catch-all mode): +// - The full hostname is used as the bucket name. +func resolveBucket(host, domain, domainSuffix string) string { + if domain == "" { + // Catch-all: the full hostname is the bucket name + return host + } + + if strings.EqualFold(host, domain) { + return domain + } + + lower := strings.ToLower(host) + if strings.HasSuffix(lower, strings.ToLower(domainSuffix)) { + sub := host[:len(host)-len(domainSuffix)] + if sub != "" && !strings.Contains(sub, ".") { + return sub + } + } + + return "" +} + +// handleRedirectAll sends a 301 redirect for RedirectAllRequestsTo configuration. +func handleRedirectAll(ctx *fiber.Ctx, redirect *s3response.RedirectAllRequestsTo, key string) error { + protocol := redirect.Protocol + if protocol == "" { + protocol = "https" + } + + location := fmt.Sprintf("%s://%s/%s", protocol, redirect.HostName, key) + ctx.Set("Location", location) + return ctx.SendStatus(http.StatusMovedPermanently) +} + +// applyRedirect constructs and sends a redirect response from a routing rule. +func applyRedirect(ctx *fiber.Ctx, redirect *s3response.Redirect, condition *s3response.RoutingRuleCondition, originalKey string) error { + protocol := redirect.Protocol + if protocol == "" { + protocol = ctx.Protocol() + } + + host := redirect.HostName + if host == "" { + host = ctx.Hostname() + } + + key := originalKey + if redirect.ReplaceKeyWith != "" { + key = redirect.ReplaceKeyWith + } else if redirect.ReplaceKeyPrefixWith != "" && condition != nil && condition.KeyPrefixEquals != "" { + key = redirect.ReplaceKeyPrefixWith + strings.TrimPrefix(originalKey, condition.KeyPrefixEquals) + } + + httpCode := http.StatusFound // 302 default + if redirect.HttpRedirectCode != "" { + if code, err := strconv.Atoi(redirect.HttpRedirectCode); err == nil { + httpCode = code + } + } + + location := fmt.Sprintf("%s://%s/%s", protocol, host, key) + ctx.Set("Location", location) + return ctx.SendStatus(httpCode) +} + +// serveObject writes the S3 object content to the response. +func serveObject(ctx *fiber.Ctx, result *s3.GetObjectOutput, key string) error { + contentType := guessContentType(result, key) + ctx.Set("Content-Type", contentType) + + if result.ETag != nil { + ctx.Set("ETag", *result.ETag) + } + if result.CacheControl != nil { + ctx.Set("Cache-Control", *result.CacheControl) + } + if result.ContentEncoding != nil { + ctx.Set("Content-Encoding", *result.ContentEncoding) + } + if result.ContentLanguage != nil { + ctx.Set("Content-Language", *result.ContentLanguage) + } + if result.ContentLength != nil { + ctx.Set("Content-Length", strconv.FormatInt(*result.ContentLength, 10)) + } + if result.LastModified != nil { + ctx.Set("Last-Modified", result.LastModified.UTC().Format(http.TimeFormat)) + } + + _, err := io.Copy(ctx.Response().BodyWriter(), result.Body) + if err != nil { + return sendError(ctx, http.StatusInternalServerError, "Internal Server Error", + "Failed to read object") + } + + return nil +} + +// serveErrorDocument fetches and serves the configured error document. +func serveErrorDocument(ctx *fiber.Ctx, be backend.Backend, bucket, errorDocKey string, statusCode int) error { + emptyRange := "" + result, err := be.GetObject(ctx.Context(), &s3.GetObjectInput{ + Bucket: &bucket, + Key: &errorDocKey, + Range: &emptyRange, + }) + if err != nil { + return sendError(ctx, statusCode, "Not Found", "The specified key does not exist") + } + if result.Body == nil { + return sendError(ctx, statusCode, "Not Found", "The specified key does not exist") + } + defer result.Body.Close() + + contentType := guessContentType(result, errorDocKey) + ctx.Set("Content-Type", contentType) + + ctx.Status(statusCode) + _, writeErr := io.Copy(ctx.Response().BodyWriter(), result.Body) + if writeErr != nil { + return sendError(ctx, statusCode, "Not Found", "The specified key does not exist") + } + + return nil +} + +// guessContentType returns the content type from the GetObject result, or +// infers it from the key extension, defaulting to text/html. +func guessContentType(result *s3.GetObjectOutput, key string) string { + if result.ContentType != nil && *result.ContentType != "" { + return *result.ContentType + } + + // Simple extension-based inference for common web types + switch { + case strings.HasSuffix(key, ".html"), strings.HasSuffix(key, ".htm"): + return "text/html; charset=utf-8" + case strings.HasSuffix(key, ".css"): + return "text/css; charset=utf-8" + case strings.HasSuffix(key, ".js"): + return "application/javascript" + case strings.HasSuffix(key, ".json"): + return "application/json" + case strings.HasSuffix(key, ".xml"): + return "application/xml" + case strings.HasSuffix(key, ".svg"): + return "image/svg+xml" + case strings.HasSuffix(key, ".png"): + return "image/png" + case strings.HasSuffix(key, ".jpg"), strings.HasSuffix(key, ".jpeg"): + return "image/jpeg" + case strings.HasSuffix(key, ".gif"): + return "image/gif" + case strings.HasSuffix(key, ".ico"): + return "image/x-icon" + case strings.HasSuffix(key, ".txt"): + return "text/plain; charset=utf-8" + default: + return "text/html; charset=utf-8" + } +} + +// sendError sends a simple HTML error page. +func sendError(ctx *fiber.Ctx, statusCode int, title, message string) error { + ctx.Set("Content-Type", "text/html; charset=utf-8") + ctx.Status(statusCode) + body := fmt.Sprintf(` + +%d %s + +

%d %s

+

%s

+ +`, statusCode, title, statusCode, title, message) + return ctx.SendString(body) +} diff --git a/website/server.go b/website/server.go new file mode 100644 index 00000000..398d1d72 --- /dev/null +++ b/website/server.go @@ -0,0 +1,130 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package website + +import ( + "fmt" + "net" + + "github.com/gofiber/fiber/v2" + "github.com/gofiber/fiber/v2/middleware/logger" + "github.com/gofiber/fiber/v2/middleware/recover" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3api/utils" +) + +// Server is the static website hosting endpoint. +type Server struct { + app *fiber.App + CertStorage *utils.CertStorage + domain string + quiet bool +} + +// Option sets various options for NewServer(). +type Option func(*Server) + +// WithQuiet silences default logging output. +func WithQuiet() Option { + return func(s *Server) { s.quiet = true } +} + +// WithTLS sets TLS credentials. +func WithTLS(cs *utils.CertStorage) Option { + return func(s *Server) { s.CertStorage = cs } +} + +// NewServer creates a new static website hosting server. +// The domain parameter is the base domain for virtual-host routing: +// - Host "blog." resolves to bucket "blog" +// - Host "" (apex, no subdomain) resolves to bucket "" +func NewServer(be backend.Backend, domain string, opts ...Option) *Server { + app := fiber.New(fiber.Config{ + AppName: "versitygw-website", + ServerHeader: "VERSITYGW", + DisableStartupMessage: true, + Network: fiber.NetworkTCP, + }) + + server := &Server{ + app: app, + domain: domain, + } + + for _, opt := range opts { + opt(server) + } + + domainInfo := "catch-all" + if domain != "" { + domainInfo = "domain: " + domain + } + + // Panic recovery + app.Use(recover.New()) + + // Request logging + if !server.quiet { + fmt.Printf("initializing website endpoint (%s)\n", domainInfo) + app.Use(logger.New(logger.Config{ + Format: "${time} | website | ${status} | ${latency} | ${ip} | ${method} | ${path}\n", + })) + } + + // All requests go through the website handler + app.Use(newHandler(be, domain)) + + return server +} + +// ServeMultiPort creates listeners for multiple address specifications and serves +// on all of them simultaneously. +func (s *Server) ServeMultiPort(ports []string) error { + if len(ports) == 0 { + return fmt.Errorf("no addresses specified") + } + + var listeners []net.Listener + + for _, addrSpec := range ports { + var ln net.Listener + var err error + + if s.CertStorage != nil { + ln, err = utils.NewMultiAddrTLSListener(s.app.Config().Network, addrSpec, s.CertStorage.GetCertificate) + } else { + ln, err = utils.NewMultiAddrListener(s.app.Config().Network, addrSpec) + } + + if err != nil { + return fmt.Errorf("failed to bind website listener %s: %w", addrSpec, err) + } + + listeners = append(listeners, ln) + } + + if len(listeners) == 0 { + return fmt.Errorf("failed to create any website listeners") + } + + finalListener := utils.NewMultiListener(listeners...) + + return s.app.Listener(finalListener) +} + +// Shutdown gracefully shuts down the server. +func (s *Server) Shutdown() error { + return s.app.Shutdown() +} diff --git a/webui/web/explorer.html b/webui/web/explorer.html index b541f708..c2bb4df2 100644 --- a/webui/web/explorer.html +++ b/webui/web/explorer.html @@ -981,6 +981,8 @@ under the License.
  • • s3:GetBucketLocation
  • • s3:GetBucketPolicy
  • • s3:PutBucketTagging
    • +
  • • s3:PutBucketWebsite
    • +
  • • s3:GetBucketWebsite
    • @@ -4181,6 +4183,8 @@ under the License. "s3:PutObject", "s3:PutObjectRetention", "s3:PutObjectTagging", + "s3:PutBucketWebsite", + "s3:GetBucketWebsite", "s3:RestoreObject" ], "Resource": [ From 1625c5963eaf7810edc67cb1171a6a18adf8a893 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Mon, 8 Jun 2026 22:31:41 +0400 Subject: [PATCH 3/4] feat: improve static website hosting support Enhances the static website hosting implementation with more complete S3-compatible behavior across request handling, backend storage, validation, CORS, and errors. Adds dedicated website endpoint handling for GET, HEAD, and OPTIONS requests, including index document resolution, error document serving, redirect-all support, pre-fetch and post-error routing rules, query string preservation in redirects, public access checks before object reads, and method-not-allowed responses. Improves error handling for website responses by returning S3-compatible HTML error bodies with request IDs, host IDs, x-amz-error-code, x-amz-error-message, and specialized error fields. This also fixes website-related validation errors to return more accurate S3-style error codes and messages, including invalid redirect protocols, invalid HTTP redirect/error codes, conflicting routing rule replacements, routing rule limits, and oversized website configuration requests. Adds website CORS support for GET, HEAD, and OPTIONS preflight requests, including bucket CORS lookup through website host bucket resolution, allowed origin/method/header validation, exposed header handling, ETag exposure, Vary headers, max-age handling, and CORS access-denied responses. Adds debug logging around website configuration parsing, validation failures, CORS checks, backend lookup failures, and internal website error paths to make failures easier to diagnose. Adds compressed website configuration storage so larger configs fit backend metadata limits, including gzip storage for POSIX extended attributes and base64-encoded compressed metadata for Azure. Also adds Azure PutBucketWebsite, GetBucketWebsite, and DeleteBucketWebsite support. Adds and expands test coverage for website config validation, S3-compatible HTML error bodies, website routing behavior, public access enforcement, HEAD behavior, CORS handling, PutBucketWebsite limits, and end-to-end website hosting through a Docker-based dnsmasq test setup and CI workflow. --- .github/workflows/website-hosting-tests.yml | 13 + Makefile | 9 + backend/azure/azure.go | 36 + backend/common.go | 61 +- backend/common_test.go | 47 + backend/posix/posix.go | 18 +- backend/s3proxy/s3.go | 12 +- cmd/versitygw/main.go | 5 + cmd/versitygw/test.go | 130 ++- embedgw/embedgw.go | 185 +++- embedgw/embedgw_test.go | 42 +- s3api/controllers/base.go | 7 +- s3api/controllers/bucket-delete.go | 2 +- s3api/controllers/bucket-get.go | 2 +- s3api/controllers/bucket-put.go | 10 +- s3api/controllers/cors_default_origin_test.go | 4 +- s3api/middlewares/apply-bucket-cors.go | 14 +- s3api/router.go | 111 +- s3err/access-forbidden-error.go | 7 + s3err/bad-digest-error.go | 7 + s3err/bucket-error.go | 6 + s3err/content-sha256-mismatch-error.go | 7 + s3err/entity-too-large-error.go | 7 + s3err/entity-too-small-error.go | 7 + s3err/expired-presigned-url-error.go | 8 + s3err/invalid-access-key-id-error.go | 6 + s3err/invalid-argument.go | 17 + s3err/invalid-chunk-size-error.go | 7 + s3err/invalid-digest-error.go | 6 + s3err/invalid-location-constraint-error.go | 6 + s3err/invalid-part-error.go | 8 + s3err/invalid-part-number-range-error.go | 7 + s3err/invalid-range-error.go | 7 + s3err/invalid-tag-error.go | 7 + s3err/key-too-long-error.go | 7 + s3err/max-message-length-exceeded-error.go | 59 ++ s3err/metadata-too-large-error.go | 7 + s3err/method-not-allowed-error.go | 7 + s3err/no-such-upload-error.go | 6 + s3err/no-such-version-error.go | 7 + s3err/not-implemented-error.go | 7 + s3err/precondition-failed-error.go | 6 + s3err/request-time-too-skewed-error.go | 8 + s3err/s3err.go | 96 +- s3err/signature-does-not-match-error.go | 11 + s3err/sigv4.go | 6 + s3response/website.go | 166 ++- s3response/website_test.go | 437 ++------ tests/integration/PutBucketWebsite.go | 279 ++++- tests/integration/WebsiteHosting.go | 873 +++++++++++----- tests/integration/group-tests.go | 43 +- tests/integration/s3conf.go | 17 +- tests/integration/utils.go | 141 +++ tests/test_rest_not_implemented.sh | 15 - tests/website-hosting-tests/dnsmasq.conf | 2 + .../website-hosting-tests/docker-compose.yml | 58 ++ website/handler.go | 602 +++++++---- website/handler_test.go | 972 ++++++++++++++++++ website/server.go | 23 +- 59 files changed, 3649 insertions(+), 1034 deletions(-) create mode 100644 .github/workflows/website-hosting-tests.yml create mode 100644 s3err/max-message-length-exceeded-error.go create mode 100644 tests/website-hosting-tests/dnsmasq.conf create mode 100644 tests/website-hosting-tests/docker-compose.yml create mode 100644 website/handler_test.go diff --git a/.github/workflows/website-hosting-tests.yml b/.github/workflows/website-hosting-tests.yml new file mode 100644 index 00000000..448d1153 --- /dev/null +++ b/.github/workflows/website-hosting-tests.yml @@ -0,0 +1,13 @@ +name: website hosting tests +permissions: {} +on: pull_request + +jobs: + build-and-run: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: run website hosting tests + run: make test-website-hosting diff --git a/Makefile b/Makefile index 6da6d45f..0bf43207 100644 --- a/Makefile +++ b/Makefile @@ -107,3 +107,12 @@ test-host-style: COMPOSE_MENU=false docker compose -f "$$compose_file" down -v --remove-orphans; \ exit $$status +# Run the static website hosting tests in docker containers +.PHONY: test-website-hosting +test-website-hosting: + @compose_file=tests/website-hosting-tests/docker-compose.yml; \ + COMPOSE_MENU=false docker compose -f "$$compose_file" down -v --remove-orphans >/dev/null 2>&1 || true; \ + COMPOSE_MENU=false docker compose -f "$$compose_file" up --build --abort-on-container-exit --exit-code-from test; \ + status=$$?; \ + COMPOSE_MENU=false docker compose -f "$$compose_file" down -v --remove-orphans; \ + exit $$status diff --git a/backend/azure/azure.go b/backend/azure/azure.go index ad411f11..ea5439f7 100644 --- a/backend/azure/azure.go +++ b/backend/azure/azure.go @@ -63,6 +63,7 @@ const ( keyTags key = "Tags" keyPolicy key = "Policy" keyCors key = "Cors" + keyWebsite key = "Website" keyBucketLock key = "Bucketlock" keyObjRetention key = "Objectretention" keyObjLegalHold key = "Objectlegalhold" @@ -88,6 +89,7 @@ func (key) Table() map[string]struct{} { "tags": {}, "policy": {}, "bucketlock": {}, + "website": {}, "objectretention": {}, "vgwexpires": {}, "objectlegalhold": {}, @@ -1994,6 +1996,40 @@ func (az *Azure) DeleteBucketCors(ctx context.Context, bucket string) error { return az.PutBucketCors(ctx, bucket, nil) } +func (az *Azure) PutBucketWebsite(ctx context.Context, bucket string, website []byte) error { + if website == nil { + return az.deleteContainerMetaData(ctx, bucket, string(keyWebsite)) + } + + encoded, err := backend.MarshalWebsiteConfig(website, true) + if err != nil { + return err + } + + return az.setContainerMetaData(ctx, bucket, string(keyWebsite), encoded) +} + +func (az *Azure) GetBucketWebsite(ctx context.Context, bucket string) ([]byte, error) { + website, err := az.getContainerMetaData(ctx, bucket, string(keyWebsite)) + if err != nil { + return nil, err + } + if len(website) == 0 { + return nil, s3err.GetBucketErr(s3err.ErrNoSuchWebsiteConfiguration, bucket) + } + + decoded, err := backend.UnmarshalWebsiteConfig(website, true) + if err != nil { + return nil, err + } + + return decoded, nil +} + +func (az *Azure) DeleteBucketWebsite(ctx context.Context, bucket string) error { + return az.PutBucketWebsite(ctx, bucket, nil) +} + func (az *Azure) PutObjectLockConfiguration(ctx context.Context, bucket string, config []byte) error { return az.setContainerMetaData(ctx, bucket, string(keyBucketLock), config) } diff --git a/backend/common.go b/backend/common.go index baa22307..536f7a4d 100644 --- a/backend/common.go +++ b/backend/common.go @@ -436,7 +436,7 @@ func MarshalMpUploadMetadata(mpMeta MpUploadMetadata, base64Encode bool) ([]byte return nil, fmt.Errorf("marshal mp metadata: %w", err) } - compressed, err := compressMpUploadMetadata(mpMetaJSON) + compressed, err := CompressData(mpMetaJSON) if err != nil { return nil, fmt.Errorf("compress mp metadata: %w", err) } @@ -471,7 +471,43 @@ func UnmarshalMpUploadMetadata(data []byte, base64Decode bool) (MpUploadMetadata return mpMeta, nil } -func compressMpUploadMetadata(data []byte) ([]byte, error) { +// MarshalWebsiteConfig returns a compressed representation of a website +// configuration. When base64Encode is true, the compressed bytes are +// base64-encoded so they can be stored in azure string-only metadata values. +func MarshalWebsiteConfig(website []byte, base64Encode bool) ([]byte, error) { + compressed, err := CompressData(website) + if err != nil { + return nil, fmt.Errorf("compress website config: %w", err) + } + + if !base64Encode { + return compressed, nil + } + + encoded := make([]byte, base64.StdEncoding.EncodedLen(len(compressed))) + base64.StdEncoding.Encode(encoded, compressed) + return encoded, nil +} + +// UnmarshalWebsiteConfig decodes data produced by MarshalWebsiteConfig. +func UnmarshalWebsiteConfig(data []byte, base64Decode bool) ([]byte, error) { + if base64Decode { + compressed, err := base64.StdEncoding.DecodeString(string(data)) + if err != nil { + return nil, fmt.Errorf("decode website config: %w", err) + } + data = compressed + } + + website, err := DecompressData(data) + if err != nil { + return nil, fmt.Errorf("decompress website config: %w", err) + } + + return website, nil +} + +func CompressData(data []byte) ([]byte, error) { var compressed bytes.Buffer gz := gzip.NewWriter(&compressed) if _, err := gz.Write(data); err != nil { @@ -484,19 +520,28 @@ func compressMpUploadMetadata(data []byte) ([]byte, error) { return compressed.Bytes(), nil } -func unmarshalCompressedMpUploadMetadata(compressed []byte) (MpUploadMetadata, error) { - var mpMeta MpUploadMetadata - gz, err := gzip.NewReader(bytes.NewReader(compressed)) +func DecompressData(data []byte) ([]byte, error) { + gz, err := gzip.NewReader(bytes.NewReader(data)) if err != nil { - return mpMeta, fmt.Errorf("decompress mp metadata: %w", err) + return nil, err } decompressed, err := io.ReadAll(gz) closeErr := gz.Close() if err != nil { - return mpMeta, fmt.Errorf("decompress mp metadata: %w", err) + return nil, err } if closeErr != nil { - return mpMeta, fmt.Errorf("decompress mp metadata: %w", closeErr) + return nil, err + } + + return decompressed, nil +} + +func unmarshalCompressedMpUploadMetadata(compressed []byte) (MpUploadMetadata, error) { + var mpMeta MpUploadMetadata + decompressed, err := DecompressData(compressed) + if err != nil { + return mpMeta, fmt.Errorf("decompress mp metadata: %w", err) } if err := json.Unmarshal(decompressed, &mpMeta); err != nil { diff --git a/backend/common_test.go b/backend/common_test.go index fa69eb66..a0307471 100644 --- a/backend/common_test.go +++ b/backend/common_test.go @@ -107,6 +107,53 @@ func TestUnmarshalMpUploadMetadataInvalid(t *testing.T) { } } +func TestWebsiteConfigRawGzipRoundTrip(t *testing.T) { + want := []byte(`index.html`) + + stored, err := MarshalWebsiteConfig(want, false) + if err != nil { + t.Fatalf("MarshalWebsiteConfig: %v", err) + } + if len(stored) < 2 || stored[0] != 0x1f || stored[1] != 0x8b { + t.Fatalf("stored website config should contain raw gzip payload: %q", stored) + } + + got, err := UnmarshalWebsiteConfig(stored, false) + if err != nil { + t.Fatalf("UnmarshalWebsiteConfig: %v", err) + } + if !bytes.Equal(got, want) { + t.Fatalf("website config mismatch: got %q want %q", got, want) + } +} + +func TestWebsiteConfigBase64RoundTrip(t *testing.T) { + want := []byte(`example.com`) + + stored, err := MarshalWebsiteConfig(want, true) + if err != nil { + t.Fatalf("MarshalWebsiteConfig: %v", err) + } + if len(stored) >= 2 && stored[0] == 0x1f && stored[1] == 0x8b { + t.Fatalf("stored website config should not contain raw gzip bytes: %q", stored) + } + + got, err := UnmarshalWebsiteConfig(stored, true) + if err != nil { + t.Fatalf("UnmarshalWebsiteConfig: %v", err) + } + if !bytes.Equal(got, want) { + t.Fatalf("website config mismatch: got %q want %q", got, want) + } +} + +func TestUnmarshalWebsiteConfigInvalid(t *testing.T) { + _, err := UnmarshalWebsiteConfig([]byte("not-gzip"), false) + if err == nil { + t.Fatal("expected invalid website config error") + } +} + func TestParseCopySource(t *testing.T) { tests := []struct { name string diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 1e34ff0d..8494ec64 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -6196,7 +6196,14 @@ func (p *Posix) PutBucketWebsite(ctx context.Context, bucket string, website []b return nil } - err = p.meta.StoreAttribute(nil, bucket, "", websitekey, website) + // The website configuration can be up to 128KB + // compress the data to fit in 64KB xattr limits + encoded, err := backend.MarshalWebsiteConfig(website, false) + if err != nil { + return err + } + + err = p.meta.StoreAttribute(nil, bucket, "", websitekey, encoded) if err != nil { return fmt.Errorf("set website: %w", err) } @@ -6224,13 +6231,18 @@ func (p *Posix) GetBucketWebsite(ctx context.Context, bucket string) ([]byte, er website, err := p.meta.RetrieveAttribute(nil, bucket, "", websitekey) if errors.Is(err, meta.ErrNoSuchKey) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchWebsiteConfiguration) + return nil, s3err.GetBucketErr(s3err.ErrNoSuchWebsiteConfiguration, bucket) } if err != nil { return nil, err } - return website, nil + decoded, err := backend.UnmarshalWebsiteConfig(website, false) + if err != nil { + return nil, err + } + + return decoded, nil } func (p *Posix) DeleteBucketWebsite(ctx context.Context, bucket string) error { diff --git a/backend/s3proxy/s3.go b/backend/s3proxy/s3.go index cb662055..ce3fbb0d 100644 --- a/backend/s3proxy/s3.go +++ b/backend/s3proxy/s3.go @@ -1759,7 +1759,7 @@ func (s *S3Proxy) putMetaBucketObj(ctx context.Context, bucket string, data []by func (s *S3Proxy) getMetaBucketObjData(ctx context.Context, bucket string, prefix metaPrefix, checkExists bool) ([]byte, error) { // return default bahviour of get bucket policy/acl, if meta bucket is not provided if s.metaBucket == "" { - return handleMetaBucketObjectNotFoundErr(prefix) + return handleMetaBucketObjectNotFoundErr(bucket, prefix) } key := getMetaKey(bucket, prefix) @@ -1773,7 +1773,7 @@ func (s *S3Proxy) getMetaBucketObjData(ctx context.Context, bucket string, prefi return nil, err } - return handleMetaBucketObjectNotFoundErr(prefix) + return handleMetaBucketObjectNotFoundErr(bucket, prefix) } if err != nil { return nil, err @@ -1790,17 +1790,17 @@ func (s *S3Proxy) getMetaBucketObjData(ctx context.Context, bucket string, prefi // handles the case when an object with the given metprefix // is not found in meta bucket. Aggregates the not found errors // for each meta prefix -func handleMetaBucketObjectNotFoundErr(prefix metaPrefix) ([]byte, error) { +func handleMetaBucketObjectNotFoundErr(bucket string, prefix metaPrefix) ([]byte, error) { switch prefix { case metaPrefixAcl: // If bucket acl is not found, return default acl return []byte{}, nil case metaPrefixPolicy: - return nil, s3err.GetBucketErr(s3err.ErrNoSuchBucketPolicy, "") + return nil, s3err.GetBucketErr(s3err.ErrNoSuchBucketPolicy, bucket) case metaPrefixCors: - return nil, s3err.GetBucketErr(s3err.ErrNoSuchCORSConfiguration, "") + return nil, s3err.GetBucketErr(s3err.ErrNoSuchCORSConfiguration, bucket) case metaPrefixWebsite: - return nil, s3err.GetAPIError(s3err.ErrNoSuchWebsiteConfiguration) + return nil, s3err.GetBucketErr(s3err.ErrNoSuchWebsiteConfiguration, bucket) } return []byte{}, nil diff --git a/cmd/versitygw/main.go b/cmd/versitygw/main.go index c1f62ce6..bed7ab0f 100644 --- a/cmd/versitygw/main.go +++ b/cmd/versitygw/main.go @@ -914,6 +914,11 @@ func runGateway(ctx context.Context, be backend.Backend) error { WebuiAdminGateways: webuiAdminGateways, WebuiPathPrefix: webuiPathPrefix, WebuiS3Prefix: webuiS3Prefix, + WebsitePorts: websitePorts, + WebsiteDomain: websiteDomain, + WebsiteCertFile: websiteCertFile, + WebsiteKeyFile: websiteKeyFile, + WebsiteNoTLS: websiteNoTLS, SigHup: sigHup, Version: Version, Build: Build, diff --git a/cmd/versitygw/test.go b/cmd/versitygw/test.go index 3f33d680..9bdffd8e 100644 --- a/cmd/versitygw/test.go +++ b/cmd/versitygw/test.go @@ -16,32 +16,35 @@ package main import ( "fmt" + "strings" "github.com/urfave/cli/v2" "github.com/versity/versitygw/tests/integration" ) var ( - awsID string - awsSecret string - endpoint string - websiteEndpointTest string - prefix string - dstBucket string - partSize int64 - objSize int64 - concurrency int - files int - totalReqs int - upload bool - download bool - hostStyle bool - checksumDisable bool - versioningEnabled bool - azureTests bool - tlsStatus bool - parallel bool - sidecarTests bool + awsID string + awsSecret string + endpoint string + websiteSchemeTest string + websiteDomainTest string + websitePortTest string + prefix string + dstBucket string + partSize int64 + objSize int64 + concurrency int + files int + totalReqs int + upload bool + download bool + hostStyle bool + checksumDisable bool + versioningEnabled bool + azureTests bool + tlsStatus bool + parallel bool + sidecarTests bool ) func testCommand() *cli.Command { @@ -77,13 +80,6 @@ func initTestFlags() []cli.Flag { Destination: &endpoint, Aliases: []string{"e"}, }, - &cli.StringFlag{ - Name: "website-endpoint", - Usage: "dedicated website hosting endpoint (e.g. 'http://localhost:8080'); required for WebsiteHosting tests", - EnvVars: []string{"VGW_TEST_WEBSITE_ENDPOINT"}, - Destination: &websiteEndpointTest, - Aliases: []string{"we"}, - }, &cli.BoolFlag{ Name: "host-style", Usage: "Use host-style bucket addressing", @@ -139,6 +135,36 @@ func initTestCommands() []*cli.Command { }, }, }, + { + Name: "website-hosting", + Usage: "Tests static website hosting endpoint.", + Description: `Runs the static website hosting integration tests against a dedicated website endpoint.`, + Action: websiteHostingAction, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "scheme", + Usage: "website endpoint scheme: http or https", + EnvVars: []string{"VGW_TEST_WEBSITE_SCHEME"}, + Destination: &websiteSchemeTest, + Aliases: []string{"website-scheme", "protocol"}, + Value: "http", + }, + &cli.StringFlag{ + Name: "domain", + Usage: "website endpoint base domain used for virtual-host routing", + EnvVars: []string{"VGW_TEST_WEBSITE_DOMAIN"}, + Destination: &websiteDomainTest, + Aliases: []string{"website-domain"}, + }, + &cli.StringFlag{ + Name: "port", + Usage: "website endpoint port", + EnvVars: []string{"VGW_TEST_WEBSITE_PORT"}, + Destination: &websitePortTest, + Aliases: []string{"website-port"}, + }, + }, + }, { Name: "posix", Usage: "Tests posix specific features", @@ -333,6 +359,50 @@ func initTestCommands() []*cli.Command { type testFunc func(*integration.TestState) +func websiteHostingAction(ctx *cli.Context) error { + websiteSchemeTest = strings.ToLower(strings.TrimSpace(websiteSchemeTest)) + if websiteSchemeTest != "http" && websiteSchemeTest != "https" { + return fmt.Errorf("website scheme must be http or https") + } + if websiteDomainTest == "" { + return fmt.Errorf("must specify website domain") + } + if websitePortTest == "" { + return fmt.Errorf("must specify website port") + } + + opts := []integration.Option{ + integration.WithAccess(awsID), + integration.WithSecret(awsSecret), + integration.WithRegion(region), + integration.WithEndpoint(endpoint), + } + if websiteSchemeTest != "" { + opts = append(opts, integration.WithWebsiteScheme(websiteSchemeTest)) + } + if websiteDomainTest != "" { + opts = append(opts, integration.WithWebsiteDomain(websiteDomainTest)) + } + if websitePortTest != "" { + opts = append(opts, integration.WithWebsitePort(websitePortTest)) + } + if debug { + opts = append(opts, integration.WithDebug()) + } + + s := integration.NewS3Conf(opts...) + ts := integration.NewTestState(ctx.Context, s, false) + integration.TestWebsiteHosting(ts) + ts.Wait() + + fmt.Println() + fmt.Println("RAN:", integration.RunCount.Load(), "PASS:", integration.PassCount.Load(), "FAIL:", integration.FailCount.Load()) + if integration.FailCount.Load() > 0 { + return fmt.Errorf("test failed with %v errors", integration.FailCount.Load()) + } + return nil +} + func getAction(tf testFunc) func(ctx *cli.Context) error { return func(ctx *cli.Context) error { opts := []integration.Option{ @@ -342,9 +412,6 @@ func getAction(tf testFunc) func(ctx *cli.Context) error { integration.WithEndpoint(endpoint), integration.WithTLSStatus(tlsStatus), } - if websiteEndpointTest != "" { - opts = append(opts, integration.WithWebsiteEndpoint(websiteEndpointTest)) - } if debug { opts = append(opts, integration.WithDebug()) } @@ -391,9 +458,6 @@ func extractIntTests() (commands []*cli.Command) { integration.WithEndpoint(endpoint), integration.WithTLSStatus(tlsStatus), } - if websiteEndpointTest != "" { - opts = append(opts, integration.WithWebsiteEndpoint(websiteEndpointTest)) - } if debug { opts = append(opts, integration.WithDebug()) } diff --git a/embedgw/embedgw.go b/embedgw/embedgw.go index 524700ea..ae01cbeb 100644 --- a/embedgw/embedgw.go +++ b/embedgw/embedgw.go @@ -42,6 +42,7 @@ import ( "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3event" "github.com/versity/versitygw/s3log" + "github.com/versity/versitygw/website" "github.com/versity/versitygw/webui" ) @@ -423,6 +424,29 @@ type Config struct { // endpoint. WebuiS3Prefix string + // Static website hosting endpoint + // + // WebsitePorts is the list of listening addresses for the static website + // hosting endpoint. Accepts the same formats as Ports. When empty, the + // website endpoint is disabled. + WebsitePorts []string + // WebsiteDomain is the base domain for website virtual-host routing. For + // example, host "blog.example.com" serves bucket "blog" when this is + // "example.com". When empty, the full request hostname is used as the + // bucket name. + WebsiteDomain string + // WebsiteCertFile is the path to the TLS certificate for the website + // endpoint. When empty and gateway TLS (CertFile/KeyFile) is configured, + // the website endpoint inherits those certs. Both WebsiteCertFile and + // WebsiteKeyFile must be provided together. + WebsiteCertFile string + // WebsiteKeyFile is the path to the TLS private key for the website + // endpoint. + WebsiteKeyFile string + // WebsiteNoTLS forces the website endpoint to use plain HTTP even when TLS + // certificates are available. + WebsiteNoTLS bool + // SigHup is an optional channel that signals the gateway to reload TLS // certificates and rotate log files (equivalent to SIGHUP). When nil, // this feature is disabled. @@ -514,7 +538,7 @@ func RunVersityGW(ctx context.Context, be backend.Backend, cfg *Config) error { fmt.Fprintf(os.Stderr, "WARNING: WebuiPorts is set but CORSAllowOrigin is not; defaulting to '*'; %s\n", suggestion) } - if err := validatePortConflicts(cfg.Ports, cfg.AdminPorts, cfg.WebuiPorts); err != nil { + if err := validatePortConflicts(cfg.Ports, cfg.AdminPorts, cfg.WebuiPorts, cfg.WebsitePorts); err != nil { return err } @@ -882,6 +906,60 @@ func RunVersityGW(ctx context.Context, be backend.Backend, cfg *Config) error { }, webOpts...) } + var wsSrv *website.Server + wsTLSCert := "" + wsTLSKey := "" + if len(cfg.WebsitePorts) > 0 { + for _, addr := range cfg.WebsitePorts { + if utils.IsUnixSocketPath(addr) { + continue + } + _, wsPrt, err := net.SplitHostPort(addr) + if err != nil { + return fmt.Errorf("website listen address must be in the form ':port' or 'host:port': %w", err) + } + wsPortNum, err := strconv.Atoi(wsPrt) + if err != nil { + return fmt.Errorf("website port must be a number: %w", err) + } + if wsPortNum < 0 || wsPortNum > 65535 { + return fmt.Errorf("website port must be between 0 and 65535") + } + } + + var wsOpts []website.Option + if !cfg.WebsiteNoTLS { + wsTLSCert = cfg.WebsiteCertFile + wsTLSKey = cfg.WebsiteKeyFile + if wsTLSCert == "" && wsTLSKey == "" { + wsTLSCert = cfg.CertFile + wsTLSKey = cfg.KeyFile + } + if wsTLSCert != "" || wsTLSKey != "" { + if wsTLSCert == "" { + return fmt.Errorf("website TLS key specified without cert file") + } + if wsTLSKey == "" { + return fmt.Errorf("website TLS cert specified without key file") + } + cs := utils.NewCertStorage() + if err := cs.SetCertificate(wsTLSCert, wsTLSKey); err != nil { + return fmt.Errorf("tls: load certs: %v", err) + } + wsOpts = append(wsOpts, website.WithTLS(cs)) + } + } + + if cfg.Quiet { + wsOpts = append(wsOpts, website.WithQuiet()) + } + if cfg.SocketPerm != "" { + wsOpts = append(wsOpts, website.WithSocketPerm(parsedSocketPerm)) + } + + wsSrv = website.NewServer(be, cfg.WebsiteDomain, wsOpts...) + } + if !cfg.Quiet { cfg.printBanner() } @@ -893,6 +971,9 @@ func RunVersityGW(ctx context.Context, be backend.Backend, cfg *Config) error { if len(cfg.WebuiPorts) > 0 { servers++ } + if len(cfg.WebsitePorts) > 0 { + servers++ + } c := make(chan error, servers) go func() { c <- srv.ServeMultiPort(cfg.Ports) }() @@ -902,6 +983,9 @@ func RunVersityGW(ctx context.Context, be backend.Backend, cfg *Config) error { if len(cfg.WebuiPorts) > 0 { go func() { c <- webSrv.ServeMultiPort(cfg.WebuiPorts) }() } + if len(cfg.WebsitePorts) > 0 { + go func() { c <- wsSrv.ServeMultiPort(cfg.WebsitePorts) }() + } // build a nil-safe sighup channel so the select below is always valid var sigHup <-chan struct{} @@ -957,6 +1041,14 @@ Loop: fmt.Printf("webSrv cert reloaded (cert: %s, key: %s)\n", webTLSCert, webTLSKey) } } + if len(cfg.WebsitePorts) > 0 && wsTLSCert != "" && wsTLSKey != "" { + reloadErr := wsSrv.CertStorage.SetCertificate(wsTLSCert, wsTLSKey) + if reloadErr != nil { + debuglogger.InternalError(fmt.Errorf("wsSrv cert reload failed: %w", reloadErr)) + } else { + fmt.Printf("wsSrv cert reloaded (cert: %s, key: %s)\n", wsTLSCert, wsTLSKey) + } + } } } saveErr := err @@ -980,6 +1072,13 @@ Loop: } } + if wsSrv != nil { + err := wsSrv.Shutdown() + if err != nil { + fmt.Fprintf(os.Stderr, "shutdown website server: %v\n", err) + } + } + be.Shutdown() err = iam.Shutdown() @@ -1023,6 +1122,7 @@ func (cfg Config) printBanner() { ssl := cfg.CertFile != "" || cfg.KeyFile != "" admSSL := cfg.AdminCertFile != "" || cfg.AdminKeyFile != "" webuiSsl := !cfg.WebuiNoTLS && (cfg.WebuiCertFile != "" || cfg.WebuiKeyFile != "" || cfg.CertFile != "" || cfg.KeyFile != "") + websiteSsl := !cfg.WebsiteNoTLS && (cfg.WebsiteCertFile != "" || cfg.WebsiteKeyFile != "" || cfg.CertFile != "" || cfg.KeyFile != "") if len(cfg.Ports) == 0 { fmt.Fprintf(os.Stderr, "No ports specified\n") @@ -1243,6 +1343,68 @@ func (cfg Config) printBanner() { } } + if len(cfg.WebsitePorts) > 0 { + var allWebsiteInterfaces []string + websiteInterfaceMap := make(map[string]bool) + + for _, websiteAddr := range cfg.WebsitePorts { + if strings.TrimSpace(websiteAddr) == "" { + continue + } + if utils.IsUnixSocketPath(websiteAddr) { + if !websiteInterfaceMap[websiteAddr] { + websiteInterfaceMap[websiteAddr] = true + allWebsiteInterfaces = append(allWebsiteInterfaces, websiteAddr) + } + continue + } + websiteInterfaces, err := getMatchingIPs(websiteAddr) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to match website port local IP addresses for %s: %v\n", websiteAddr, err) + continue + } + _, websitePrt, err := net.SplitHostPort(websiteAddr) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to parse website port %s: %v\n", websiteAddr, err) + continue + } + for _, ip := range websiteInterfaces { + key := net.JoinHostPort(ip, websitePrt) + if !websiteInterfaceMap[key] { + websiteInterfaceMap[key] = true + allWebsiteInterfaces = append(allWebsiteInterfaces, key) + } + } + } + + if len(allWebsiteInterfaces) > 0 { + domainInfo := "" + if cfg.WebsiteDomain != "" { + domainInfo = fmt.Sprintf(" (domain: %s)", cfg.WebsiteDomain) + } + lines = append(lines, + centerText(""), + leftText("Website endpoint listening on:"+domainInfo), + ) + for _, addrPort := range allWebsiteInterfaces { + if utils.IsUnixSocketPath(addrPort) { + lines = append(lines, leftText(" unix:"+addrPort)) + continue + } + ip, prt, err := net.SplitHostPort(addrPort) + if err != nil { + continue + } + hostPort := net.JoinHostPort(ip, prt) + u := fmt.Sprintf("http://%s", hostPort) + if websiteSsl { + u = fmt.Sprintf("https://%s", hostPort) + } + lines = append(lines, leftText(" "+u)) + } + } + } + fmt.Println("┌" + strings.Repeat("─", columnWidth-2) + "┐") for _, line := range lines { fmt.Printf("│%-*s│\n", columnWidth-2, line) @@ -1436,13 +1598,13 @@ func sortGatewayURLs(urls []string) { } // validatePortConflicts checks for port conflicts across the S3 API, admin, -// and WebUI port lists before the servers are started. +// WebUI, and website port lists before the servers are started. // // A bare port spec (e.g. ":7071") binds to all interfaces and conflicts with // any other spec on the same port number. Two identical "ip:port" specs are // allowed and will be caught by the OS later. UNIX socket paths are checked // for duplicate path conflicts only and never conflict with TCP specs. -func validatePortConflicts(ports, admPorts, webuiPorts []string) error { +func validatePortConflicts(ports, admPorts, webuiPorts, websitePorts []string) error { type portSpec struct { spec string port string @@ -1504,6 +1666,23 @@ func validatePortConflicts(ports, admPorts, webuiPorts []string) error { }) } + for _, p := range websitePorts { + if utils.IsUnixSocketPath(p) { + allSpecs = append(allSpecs, portSpec{spec: p, port: p, isUnix: true, portType: "website"}) + continue + } + _, port, err := net.SplitHostPort(p) + if err != nil { + continue + } + allSpecs = append(allSpecs, portSpec{ + spec: p, + port: port, + isBare: strings.HasPrefix(p, ":"), + portType: "website", + }) + } + for i, spec1 := range allSpecs { for j, spec2 := range allSpecs { if i >= j { diff --git a/embedgw/embedgw_test.go b/embedgw/embedgw_test.go index 781b14a7..a692cd7d 100644 --- a/embedgw/embedgw_test.go +++ b/embedgw/embedgw_test.go @@ -20,12 +20,13 @@ import ( func TestValidatePortConflicts(t *testing.T) { tests := []struct { - name string - ports []string - admPorts []string - webuiPorts []string - expectError bool - description string + name string + ports []string + admPorts []string + webuiPorts []string + websitePorts []string + expectError bool + description string }{ { name: "bare port conflict with bare port", @@ -115,11 +116,38 @@ func TestValidatePortConflicts(t *testing.T) { expectError: true, description: "should fail: :8080 conflicts with 127.0.0.1:8080", }, + { + name: "website bare port conflict with s3 port", + ports: []string{"127.0.0.1:8080"}, + admPorts: []string{}, + webuiPorts: []string{}, + websitePorts: []string{":8080"}, + expectError: true, + description: "should fail: website bare :8080 conflicts with s3 127.0.0.1:8080", + }, + { + name: "website no conflict", + ports: []string{":7070"}, + admPorts: []string{":8080"}, + webuiPorts: []string{":9090"}, + websitePorts: []string{":8081"}, + expectError: false, + description: "should pass: website uses a distinct port", + }, + { + name: "duplicate website unix socket conflict", + ports: []string{"/tmp/versitygw.sock"}, + admPorts: []string{}, + webuiPorts: []string{}, + websitePorts: []string{"/tmp/versitygw.sock"}, + expectError: true, + description: "should fail: duplicate unix socket path conflicts across s3 and website", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := validatePortConflicts(tt.ports, tt.admPorts, tt.webuiPorts) + err := validatePortConflicts(tt.ports, tt.admPorts, tt.webuiPorts, tt.websitePorts) if tt.expectError && err == nil { t.Errorf("%s: expected error but got none", tt.description) } diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 028f4e48..bbad90e4 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -49,9 +49,10 @@ const ( iso8601TimeFormatExtended = "Mon Jan _2 15:04:05 2006" timefmt = "Mon, 02 Jan 2006 15:04:05 GMT" - maxXMLBodyLen = 4 * 1024 * 1024 - minPartNumber = 1 - maxPartNumber = 10000 + maxXMLBodyLen = 4 * 1024 * 1024 + minPartNumber = 1 + maxPartNumber = 10000 + maxWebsiteConfigurationBytes = 131072 defaultRegion = "us-east-1" defaultContentType = "binary/octet-stream" diff --git a/s3api/controllers/bucket-delete.go b/s3api/controllers/bucket-delete.go index 441f5273..2ff9e7d1 100644 --- a/s3api/controllers/bucket-delete.go +++ b/s3api/controllers/bucket-delete.go @@ -177,7 +177,7 @@ func (c S3ApiController) DeleteBucketWebsite(ctx *fiber.Ctx) (*Response, error) IsRoot: isRoot, Acc: acct, Bucket: bucket, - Action: auth.DeleteBucketWebsiteAction, + Actions: []auth.Action{auth.DeleteBucketWebsiteAction}, IsPublicRequest: IsBucketPublic, DisableACL: c.disableACL, }) diff --git a/s3api/controllers/bucket-get.go b/s3api/controllers/bucket-get.go index 3a31156b..b86fd9b3 100644 --- a/s3api/controllers/bucket-get.go +++ b/s3api/controllers/bucket-get.go @@ -220,7 +220,7 @@ func (c S3ApiController) GetBucketWebsite(ctx *fiber.Ctx) (*Response, error) { IsRoot: isRoot, Acc: acct, Bucket: bucket, - Action: auth.GetBucketWebsiteAction, + Actions: []auth.Action{auth.GetBucketWebsiteAction}, IsPublicRequest: isPublicBucket, DisableACL: c.disableACL, }) diff --git a/s3api/controllers/bucket-put.go b/s3api/controllers/bucket-put.go index a83c291f..6d1521b7 100644 --- a/s3api/controllers/bucket-put.go +++ b/s3api/controllers/bucket-put.go @@ -301,7 +301,7 @@ func (c S3ApiController) PutBucketWebsite(ctx *fiber.Ctx) (*Response, error) { IsRoot: isRoot, Acc: acct, Bucket: bucket, - Action: auth.PutBucketWebsiteAction, + Actions: []auth.Action{auth.PutBucketWebsiteAction}, IsPublicRequest: isPublicBucket, DisableACL: c.disableACL, }) @@ -314,6 +314,14 @@ func (c S3ApiController) PutBucketWebsite(ctx *fiber.Ctx) (*Response, error) { } body := ctx.Body() + if len(body) > maxWebsiteConfigurationBytes { + debuglogger.Logf("the request size exceeded the 128KB limit: %d", len(body)) + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetMaxMessageLengthExceeded(maxWebsiteConfigurationBytes) + } var websiteConfig s3response.WebsiteConfiguration err = xml.Unmarshal(body, &websiteConfig) diff --git a/s3api/controllers/cors_default_origin_test.go b/s3api/controllers/cors_default_origin_test.go index 761aa551..a8771423 100644 --- a/s3api/controllers/cors_default_origin_test.go +++ b/s3api/controllers/cors_default_origin_test.go @@ -35,7 +35,7 @@ func TestApplyBucketCORS_FallbackOrigin_NoBucketCors_NoRequestOrigin(t *testing. app := fiber.New() app.Get("/:bucket/test", - middlewares.ApplyBucketCORS(mockedBackend, origin), + middlewares.ApplyBucketCORS(mockedBackend, middlewares.BucketFromPath, origin), func(c *fiber.Ctx) error { return c.SendStatus(http.StatusOK) }, @@ -71,7 +71,7 @@ func TestApplyBucketCORS_FallbackOrigin_NotAppliedWhenBucketCorsExists(t *testin app := fiber.New() app.Get("/:bucket/test", - middlewares.ApplyBucketCORS(mockedBackend, origin), + middlewares.ApplyBucketCORS(mockedBackend, middlewares.BucketFromPath, origin), func(c *fiber.Ctx) error { return c.SendStatus(http.StatusOK) }, diff --git a/s3api/middlewares/apply-bucket-cors.go b/s3api/middlewares/apply-bucket-cors.go index 9a19543e..0f2c3a81 100644 --- a/s3api/middlewares/apply-bucket-cors.go +++ b/s3api/middlewares/apply-bucket-cors.go @@ -28,21 +28,31 @@ import ( // Vary http response header is always the same below var VaryHdr = "Origin, Access-Control-Request-Headers, Access-Control-Request-Method" +type BucketResolver func(ctx *fiber.Ctx) (string, error) + +func BucketFromPath(ctx *fiber.Ctx) (string, error) { + return ctx.Params("bucket"), nil +} + // ApplyBucketCORS retreives the bucket CORS configuration, // checks if origin and method meets the cors rules and // adds the necessary response headers. // CORS check is applied only when 'Origin' request header is present -func ApplyBucketCORS(be backend.Backend, fallbackOrigin string) fiber.Handler { +func ApplyBucketCORS(be backend.Backend, resolveBucket BucketResolver, fallbackOrigin string) fiber.Handler { fallbackOrigin = strings.TrimSpace(fallbackOrigin) return func(ctx *fiber.Ctx) error { - bucket := ctx.Params("bucket") origin := ctx.Get("Origin") // If neither Origin is present nor a fallback is configured, skip CORS entirely. if origin == "" && fallbackOrigin == "" { return nil } + bucket, err := resolveBucket(ctx) + if err != nil { + return err + } + // if bucket cors is not set, skip the check data, err := be.GetBucketCors(ctx.Context(), bucket) if err != nil { diff --git a/s3api/router.go b/s3api/router.go index e4b32432..57e4352b 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -185,6 +185,7 @@ func (sa *S3ApiRouter) Init() { bucketRouter := sa.app.Group("/:bucket") objectRouter := sa.app.Group("/:bucket/*") + applyBucketCORS := middlewares.ApplyBucketCORS(sa.be, middlewares.BucketFromPath, sa.corsAllowOrigin) // PUT bucket operations bucketRouter.Put("", @@ -194,7 +195,7 @@ func (sa *S3ApiRouter) Init() { metrics.ActionPutBucketTagging, services, middlewares.BucketObjectNameValidator(), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionPutBucketTagging, auth.PutBucketTaggingAction, auth.PermissionWrite, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), @@ -212,7 +213,7 @@ func (sa *S3ApiRouter) Init() { middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), middlewares.VerifyChecksums(false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Put("", @@ -226,7 +227,7 @@ func (sa *S3ApiRouter) Init() { middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), middlewares.VerifyChecksums(false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Put("", @@ -240,7 +241,7 @@ func (sa *S3ApiRouter) Init() { middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), middlewares.VerifyChecksums(false, true, true), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Put("", @@ -254,7 +255,7 @@ func (sa *S3ApiRouter) Init() { middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), middlewares.VerifyChecksums(false, true, true), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Put("", @@ -268,7 +269,7 @@ func (sa *S3ApiRouter) Init() { middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), middlewares.VerifyChecksums(false, false, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Put("", @@ -282,7 +283,7 @@ func (sa *S3ApiRouter) Init() { middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), middlewares.VerifyChecksums(false, false, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Put("", @@ -452,7 +453,7 @@ func (sa *S3ApiRouter) Init() { middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), middlewares.VerifyChecksums(false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), ), ) @@ -466,7 +467,7 @@ func (sa *S3ApiRouter) Init() { middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), middlewares.VerifyChecksums(false, false, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, )) // HeadBucket action @@ -491,7 +492,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionHeadBucket, auth.ListBucketAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, false, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) @@ -518,7 +519,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionDeleteBucketTagging, auth.PutBucketTaggingAction, auth.PermissionWrite, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Delete("", @@ -531,7 +532,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionDeleteBucketOwnershipControls, auth.PutBucketOwnershipControlsAction, auth.PermissionWrite, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Delete("", @@ -544,7 +545,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionDeleteBucketPolicy, auth.PutBucketPolicyAction, auth.PermissionWrite, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Delete("", @@ -557,7 +558,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionDeleteBucketCors, auth.PutBucketCorsAction, auth.PermissionWrite, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Delete("", @@ -674,7 +675,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionDeleteBucketWebsite, auth.DeleteBucketWebsiteAction, auth.PermissionWrite, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), ), ) @@ -687,7 +688,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionDeleteBucket, auth.DeleteBucketAction, auth.PermissionWrite, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) @@ -714,7 +715,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionGetBucketLocation, auth.GetBucketLocationAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), ), ) @@ -728,7 +729,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionGetBucketTagging, auth.GetBucketTaggingAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Get("", @@ -741,7 +742,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionGetBucketOwnershipControls, auth.GetBucketOwnershipControlsAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Get("", @@ -754,7 +755,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionGetBucketVersioning, auth.GetBucketVersioningAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Get("", @@ -767,7 +768,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionGetBucketPolicy, auth.GetBucketPolicyAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Get("", @@ -780,7 +781,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionGetBucketCors, auth.GetBucketCorsAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Get("", @@ -793,7 +794,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionGetObjectLockConfiguration, auth.GetBucketObjectLockConfigurationAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Get("", @@ -806,7 +807,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionGetBucketAcl, auth.GetBucketAclAction, auth.PermissionReadAcp, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, false, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Get("", @@ -819,7 +820,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionListMultipartUploads, auth.ListBucketMultipartUploadsAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Get("", @@ -832,7 +833,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionListObjectVersions, auth.ListBucketVersionsAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Get("", @@ -845,7 +846,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionGetBucketPolicyStatus, auth.GetBucketPolicyStatusAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Get("", @@ -1066,7 +1067,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionGetBucketWebsite, auth.GetBucketWebsiteAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), ), ) @@ -1080,7 +1081,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionListObjectsV2, auth.ListBucketAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) bucketRouter.Get("", @@ -1092,7 +1093,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionListObjects, auth.ListBucketAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) @@ -1121,7 +1122,7 @@ func (sa *S3ApiRouter) Init() { middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), middlewares.VerifyChecksums(false, true, true), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) @@ -1133,7 +1134,7 @@ func (sa *S3ApiRouter) Init() { middlewares.BucketObjectNameValidator(), middlewares.AuthorizePostObject(sa.root, sa.iam, sa.region), middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionPostObject, auth.PutObjectAction, auth.PermissionWrite, sa.region, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) @@ -1159,7 +1160,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionHeadObject, auth.GetObjectAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, false, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) @@ -1199,7 +1200,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionGetObjectTagging, auth.GetObjectTaggingAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) objectRouter.Get("", @@ -1212,7 +1213,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionGetObjectRetention, auth.GetObjectRetentionAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) objectRouter.Get("", @@ -1225,7 +1226,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionGetObjectLegalHold, auth.GetObjectLegalHoldAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) objectRouter.Get("", @@ -1238,7 +1239,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionGetObjectAcl, auth.GetObjectAclAction, auth.PermissionReadAcp, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) objectRouter.Get("", @@ -1251,7 +1252,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionGetObjectAttributes, auth.GetObjectAttributesAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) objectRouter.Get("", @@ -1264,7 +1265,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionListParts, auth.ListMultipartUploadPartsAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) objectRouter.Get("", @@ -1276,7 +1277,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionGetObject, auth.GetObjectAction, auth.PermissionRead, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) @@ -1304,7 +1305,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionDeleteObjectTagging, auth.DeleteObjectTaggingAction, auth.PermissionWrite, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) objectRouter.Delete("", @@ -1317,7 +1318,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionAbortMultipartUpload, auth.AbortMultipartUploadAction, auth.PermissionWrite, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) objectRouter.Delete("", @@ -1329,7 +1330,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionDeleteObject, auth.DeleteObjectAction, auth.PermissionWrite, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) @@ -1359,7 +1360,7 @@ func (sa *S3ApiRouter) Init() { middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), middlewares.VerifyChecksums(false, false, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) objectRouter.Post("", @@ -1374,7 +1375,7 @@ func (sa *S3ApiRouter) Init() { middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), middlewares.VerifyChecksums(false, false, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) objectRouter.Post("", @@ -1387,7 +1388,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionCompleteMultipartUpload, auth.PutObjectAction, auth.PermissionWrite, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) objectRouter.Post("", @@ -1400,7 +1401,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionCreateMultipartUpload, auth.PutObjectAction, auth.PermissionWrite, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) @@ -1412,7 +1413,7 @@ func (sa *S3ApiRouter) Init() { metrics.ActionPutObjectTagging, services, middlewares.BucketObjectNameValidator(), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionPutObjectTagging, auth.PutObjectTaggingAction, auth.PermissionWrite, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), @@ -1430,7 +1431,7 @@ func (sa *S3ApiRouter) Init() { middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), middlewares.VerifyChecksums(false, false, true), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) objectRouter.Put("", @@ -1444,7 +1445,7 @@ func (sa *S3ApiRouter) Init() { middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), middlewares.VerifyChecksums(false, false, true), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) objectRouter.Put("", @@ -1458,7 +1459,7 @@ func (sa *S3ApiRouter) Init() { middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), middlewares.VerifyChecksums(false, false, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) objectRouter.Put("", @@ -1472,7 +1473,7 @@ func (sa *S3ApiRouter) Init() { middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionUploadPartCopy, auth.PutObjectAction, auth.PermissionWrite, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) objectRouter.Put("", @@ -1486,7 +1487,7 @@ func (sa *S3ApiRouter) Init() { middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, true), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, true, true, false), middlewares.VerifyChecksums(true, false, false), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.ParseAcl(sa.be), )) @@ -1523,7 +1524,7 @@ func (sa *S3ApiRouter) Init() { metrics.ActionCopyObject, services, middlewares.BucketObjectNameValidator(), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionCopyObject, auth.PutObjectAction, auth.PermissionWrite, sa.region, false), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, false), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, false, true, false), @@ -1535,7 +1536,7 @@ func (sa *S3ApiRouter) Init() { metrics.ActionPutObject, services, middlewares.BucketObjectNameValidator(), - middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + applyBucketCORS, middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionPutObject, auth.PutObjectAction, auth.PermissionWrite, sa.region, true), middlewares.VerifyPresignedV4Signature(sa.root, sa.iam, sa.region, true), middlewares.VerifyV4Signature(sa.root, sa.iam, sa.region, true, true, false), diff --git a/s3err/access-forbidden-error.go b/s3err/access-forbidden-error.go index 085c3d36..6abee327 100644 --- a/s3err/access-forbidden-error.go +++ b/s3err/access-forbidden-error.go @@ -43,6 +43,13 @@ func (e AccessForbiddenError) XMLBody(requestID, hostID string) []byte { }) } +func (e AccessForbiddenError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "Method", Value: e.Method}, + ErrorField{Name: "ResourceType", Value: e.ResourceType}, + ) +} + func (e AccessForbiddenError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/bad-digest-error.go b/s3err/bad-digest-error.go index 1b209a4d..c095bd5a 100644 --- a/s3err/bad-digest-error.go +++ b/s3err/bad-digest-error.go @@ -43,6 +43,13 @@ func (e BadDigestError) XMLBody(requestID, hostID string) []byte { }) } +func (e BadDigestError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "CalculatedDigest", Value: e.CalculatedDigest}, + ErrorField{Name: "ExpectedDigest", Value: e.ExpectedDigest}, + ) +} + func (e BadDigestError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/bucket-error.go b/s3err/bucket-error.go index 0bbff389..37b4fef6 100644 --- a/s3err/bucket-error.go +++ b/s3err/bucket-error.go @@ -40,6 +40,12 @@ func (e BucketError) XMLBody(requestID, hostID string) []byte { }) } +func (e BucketError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "BucketName", Value: e.BucketName}, + ) +} + func (e BucketError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/content-sha256-mismatch-error.go b/s3err/content-sha256-mismatch-error.go index 22e9738f..1a3079ff 100644 --- a/s3err/content-sha256-mismatch-error.go +++ b/s3err/content-sha256-mismatch-error.go @@ -44,6 +44,13 @@ func (e ContentSHA256MismatchError) XMLBody(requestID, hostID string) []byte { }) } +func (e ContentSHA256MismatchError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "ClientComputedContentSHA256", Value: e.ClientComputedContentSHA256}, + ErrorField{Name: "S3ComputedContentSHA256", Value: e.S3ComputedContentSHA256}, + ) +} + func (e ContentSHA256MismatchError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/entity-too-large-error.go b/s3err/entity-too-large-error.go index 24f40868..a6a4a96c 100644 --- a/s3err/entity-too-large-error.go +++ b/s3err/entity-too-large-error.go @@ -43,6 +43,13 @@ func (e EntityTooLargeError) XMLBody(requestID, hostID string) []byte { }) } +func (e EntityTooLargeError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "ProposedSize", Value: e.ProposedSize}, + ErrorField{Name: "MaxSizeAllowed", Value: e.MaxSizeAllowed}, + ) +} + func (e EntityTooLargeError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/entity-too-small-error.go b/s3err/entity-too-small-error.go index d1c522b0..7a230bb9 100644 --- a/s3err/entity-too-small-error.go +++ b/s3err/entity-too-small-error.go @@ -43,6 +43,13 @@ func (e EntityTooSmallError) XMLBody(requestID, hostID string) []byte { }) } +func (e EntityTooSmallError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "ProposedSize", Value: e.ProposedSize}, + ErrorField{Name: "MinSizeAllowed", Value: e.MinSizeAllowed}, + ) +} + func (e EntityTooSmallError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/expired-presigned-url-error.go b/s3err/expired-presigned-url-error.go index f4406c86..95c4f815 100644 --- a/s3err/expired-presigned-url-error.go +++ b/s3err/expired-presigned-url-error.go @@ -46,6 +46,14 @@ func (e ExpiredPresignedURLError) XMLBody(requestID, hostID string) []byte { }) } +func (e ExpiredPresignedURLError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "ServerTime", Value: e.ServerTime}, + ErrorField{Name: "X-Amz-Expires", Value: e.XAmzExpires}, + ErrorField{Name: "Expires", Value: e.Expires}, + ) +} + func (e ExpiredPresignedURLError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/invalid-access-key-id-error.go b/s3err/invalid-access-key-id-error.go index b70efa3c..8f910db7 100644 --- a/s3err/invalid-access-key-id-error.go +++ b/s3err/invalid-access-key-id-error.go @@ -40,6 +40,12 @@ func (e InvalidAccessKeyIdError) XMLBody(requestID, hostID string) []byte { }) } +func (e InvalidAccessKeyIdError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "AWSAccessKeyId", Value: e.AWSAccessKeyId}, + ) +} + func (e InvalidAccessKeyIdError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/invalid-argument.go b/s3err/invalid-argument.go index ae14ca81..5e8d65b5 100644 --- a/s3err/invalid-argument.go +++ b/s3err/invalid-argument.go @@ -56,6 +56,8 @@ const ( InvalidArgCannedAcl InvalidArgOnlyAws4HmacSha256 InvalidArgDateHeader + InvalidArgIndexDocumentSuffix + InvalidArgErrorDocumentKey ) var invalidArgErrResponses = map[InvalidArgErrorCode]InvalidArgumentError{ @@ -187,6 +189,14 @@ var invalidArgErrResponses = map[InvalidArgErrorCode]InvalidArgumentError{ Description: "X-Amz-Date must be formated via ISO8601 Long format", ArgumentName: "X-Amz-Date", }, + InvalidArgIndexDocumentSuffix: { + Description: "The IndexDocument Suffix is not well formed", + ArgumentName: "IndexDocument", + }, + InvalidArgErrorDocumentKey: { + Description: "The ErrorDocument Key is not well formed", + ArgumentName: "ErrorDocument", + }, } // InvalidArgumentError is returned when a request argument is invalid. @@ -235,6 +245,13 @@ func (e InvalidArgumentError) XMLBody(requestID, hostID string) []byte { }) } +func (e InvalidArgumentError) HTMLBody(requestID, hostID string) []byte { + return e.BaseError().encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "ArgumentName", Value: e.ArgumentName}, + ErrorField{Name: "ArgumentValue", Value: e.ArgumentValue}, + ) +} + func GetInvalidArgumentErr(code InvalidArgErrorCode, value string) InvalidArgumentError { err := invalidArgErrResponses[code] err.ArgumentValue = value diff --git a/s3err/invalid-chunk-size-error.go b/s3err/invalid-chunk-size-error.go index 86eff473..7c147a4d 100644 --- a/s3err/invalid-chunk-size-error.go +++ b/s3err/invalid-chunk-size-error.go @@ -43,6 +43,13 @@ func (e InvalidChunkSizeError) XMLBody(requestID, hostID string) []byte { }) } +func (e InvalidChunkSizeError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "Chunk", Value: e.Chunk}, + ErrorField{Name: "BadChunkSize", Value: e.BadChunkSize}, + ) +} + func (e InvalidChunkSizeError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/invalid-digest-error.go b/s3err/invalid-digest-error.go index 537d0906..e7ca3113 100644 --- a/s3err/invalid-digest-error.go +++ b/s3err/invalid-digest-error.go @@ -40,6 +40,12 @@ func (e InvalidDigestError) XMLBody(requestID, hostID string) []byte { }) } +func (e InvalidDigestError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "Content-MD5", Value: e.ContentMD5}, + ) +} + func (e InvalidDigestError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/invalid-location-constraint-error.go b/s3err/invalid-location-constraint-error.go index 290d6428..bf27a5d7 100644 --- a/s3err/invalid-location-constraint-error.go +++ b/s3err/invalid-location-constraint-error.go @@ -40,6 +40,12 @@ func (e InvalidLocationConstraintError) XMLBody(requestID, hostID string) []byte }) } +func (e InvalidLocationConstraintError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "LocationConstraint", Value: e.LocationConstraint}, + ) +} + func (e InvalidLocationConstraintError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/invalid-part-error.go b/s3err/invalid-part-error.go index ed47cdcb..4cc7fc11 100644 --- a/s3err/invalid-part-error.go +++ b/s3err/invalid-part-error.go @@ -48,6 +48,14 @@ func (e InvalidPartError) XMLBody(requestID, hostID string) []byte { }) } +func (e InvalidPartError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "UploadId", Value: e.UploadId}, + ErrorField{Name: "PartNumber", Value: e.PartNumber}, + ErrorField{Name: "ETag", Value: e.ETag}, + ) +} + func (e InvalidPartError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/invalid-part-number-range-error.go b/s3err/invalid-part-number-range-error.go index 5edd9fea..4cf7dbfc 100644 --- a/s3err/invalid-part-number-range-error.go +++ b/s3err/invalid-part-number-range-error.go @@ -44,6 +44,13 @@ func (e InvalidPartNumberRangeError) XMLBody(requestID, hostID string) []byte { }) } +func (e InvalidPartNumberRangeError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "ActualPartCount", Value: e.ActualPartCount}, + ErrorField{Name: "PartNumberRequested", Value: e.PartNumberRequested}, + ) +} + func (e InvalidPartNumberRangeError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/invalid-range-error.go b/s3err/invalid-range-error.go index 37773768..bc23711d 100644 --- a/s3err/invalid-range-error.go +++ b/s3err/invalid-range-error.go @@ -43,6 +43,13 @@ func (e InvalidRangeError) XMLBody(requestID, hostID string) []byte { }) } +func (e InvalidRangeError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "RangeRequested", Value: e.RangeRequested}, + ErrorField{Name: "ActualObjectSize", Value: e.ActualObjectSize}, + ) +} + func (e InvalidRangeError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/invalid-tag-error.go b/s3err/invalid-tag-error.go index 981c5183..2c481028 100644 --- a/s3err/invalid-tag-error.go +++ b/s3err/invalid-tag-error.go @@ -43,6 +43,13 @@ func (e InvalidTagError) XMLBody(requestID, hostID string) []byte { }) } +func (e InvalidTagError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "TagKey", Value: e.TagKey}, + ErrorField{Name: "TagValue", Value: e.TagValue}, + ) +} + func (e InvalidTagError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/key-too-long-error.go b/s3err/key-too-long-error.go index d95fdd8a..073d56f7 100644 --- a/s3err/key-too-long-error.go +++ b/s3err/key-too-long-error.go @@ -43,6 +43,13 @@ func (e KeyTooLongError) XMLBody(requestID, hostID string) []byte { }) } +func (e KeyTooLongError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "Size", Value: e.Size}, + ErrorField{Name: "MaxSizeAllowed", Value: e.MaxSizeAllowed}, + ) +} + func (e KeyTooLongError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/max-message-length-exceeded-error.go b/s3err/max-message-length-exceeded-error.go new file mode 100644 index 00000000..1c9447d4 --- /dev/null +++ b/s3err/max-message-length-exceeded-error.go @@ -0,0 +1,59 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package s3err + +import "encoding/xml" + +// MaxMessageLengthExceeded is returned when the request size exceeds the maximum limit +// Produces the field in the XML response. +type MaxMessageLengthExceeded struct { + APIError + MaxMessageLengthBytes int64 +} + +func (e MaxMessageLengthExceeded) XMLBody(requestID, hostID string) []byte { + return encodeResponse(struct { + XMLName xml.Name `xml:"Error"` + Code string + Message string + MaxMessageLengthBytes int64 `xml:",omitempty"` + RequestID string `xml:"RequestId,omitempty"` + HostID string `xml:"HostId,omitempty"` + }{ + Code: e.Code, + Message: e.Description, + MaxMessageLengthBytes: e.MaxMessageLengthBytes, + RequestID: requestID, + HostID: hostID, + }) +} + +func (e MaxMessageLengthExceeded) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "MaxMessageLengthBytes", Value: e.MaxMessageLengthBytes}, + ) +} + +func (e MaxMessageLengthExceeded) Is(target error) bool { + t, ok := target.(APIError) + return ok && e.APIError == t +} + +func GetMaxMessageLengthExceeded(maxMessageLengthBytes int64) MaxMessageLengthExceeded { + return MaxMessageLengthExceeded{ + APIError: GetAPIError(ErrMaxMessageLengthExceeded), + MaxMessageLengthBytes: maxMessageLengthBytes, + } +} diff --git a/s3err/metadata-too-large-error.go b/s3err/metadata-too-large-error.go index 02bc2692..59cd6e87 100644 --- a/s3err/metadata-too-large-error.go +++ b/s3err/metadata-too-large-error.go @@ -43,6 +43,13 @@ func (e MetadataTooLargeError) XMLBody(requestID, hostID string) []byte { }) } +func (e MetadataTooLargeError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "Size", Value: e.Size}, + ErrorField{Name: "MaxSizeAllowed", Value: e.MaxSizeAllowed}, + ) +} + func (e MetadataTooLargeError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/method-not-allowed-error.go b/s3err/method-not-allowed-error.go index 881c9acf..a0009f28 100644 --- a/s3err/method-not-allowed-error.go +++ b/s3err/method-not-allowed-error.go @@ -52,6 +52,13 @@ func (e MethodNotAllowedError) XMLBody(requestID, hostID string) []byte { }) } +func (e MethodNotAllowedError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "Method", Value: e.Method}, + ErrorField{Name: "ResourceType", Value: e.ResourceType}, + ) +} + func (e MethodNotAllowedError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/no-such-upload-error.go b/s3err/no-such-upload-error.go index a1576a43..2d6d8452 100644 --- a/s3err/no-such-upload-error.go +++ b/s3err/no-such-upload-error.go @@ -40,6 +40,12 @@ func (e NoSuchUploadError) XMLBody(requestID, hostID string) []byte { }) } +func (e NoSuchUploadError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "UploadId", Value: e.UploadId}, + ) +} + func (e NoSuchUploadError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/no-such-version-error.go b/s3err/no-such-version-error.go index 8cd36e60..b1ed81c5 100644 --- a/s3err/no-such-version-error.go +++ b/s3err/no-such-version-error.go @@ -43,6 +43,13 @@ func (e NoSuchVersionError) XMLBody(requestID, hostID string) []byte { }) } +func (e NoSuchVersionError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "Key", Value: e.Key}, + ErrorField{Name: "VersionId", Value: e.VersionId}, + ) +} + func (e NoSuchVersionError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/not-implemented-error.go b/s3err/not-implemented-error.go index 64569a69..f4618238 100644 --- a/s3err/not-implemented-error.go +++ b/s3err/not-implemented-error.go @@ -52,6 +52,13 @@ func (e NotImplementedError) XMLBody(requestID, hostID string) []byte { }) } +func (e NotImplementedError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "Header", Value: e.Header}, + ErrorField{Name: "additionalMessage", Value: e.AdditionalMessage}, + ) +} + func (e NotImplementedError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/precondition-failed-error.go b/s3err/precondition-failed-error.go index 0e8fe3fb..f54b79d6 100644 --- a/s3err/precondition-failed-error.go +++ b/s3err/precondition-failed-error.go @@ -52,6 +52,12 @@ func (e PreconditionFailedError) XMLBody(requestID, hostID string) []byte { }) } +func (e PreconditionFailedError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "Condition", Value: e.Condition}, + ) +} + func (e PreconditionFailedError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/request-time-too-skewed-error.go b/s3err/request-time-too-skewed-error.go index b40eb109..d5ba3785 100644 --- a/s3err/request-time-too-skewed-error.go +++ b/s3err/request-time-too-skewed-error.go @@ -48,6 +48,14 @@ func (e RequestTimeTooSkewedError) XMLBody(requestID, hostID string) []byte { }) } +func (e RequestTimeTooSkewedError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "RequestTime", Value: e.RequestTime}, + ErrorField{Name: "ServerTime", Value: e.ServerTime}, + ErrorField{Name: "MaxAllowedSkewMilliseconds", Value: e.MaxAllowedSkewMilliseconds}, + ) +} + func (e RequestTimeTooSkewedError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/s3err.go b/s3err/s3err.go index ac3fe57c..8aa15e20 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -18,6 +18,7 @@ import ( "bytes" "encoding/xml" "fmt" + "html" "net/http" "strings" @@ -31,6 +32,7 @@ type S3Error interface { StatusCode() int BaseError() APIError XMLBody(requestID, hostID string) []byte + HTMLBody(requestID, hostID string) []byte } // APIError structure @@ -69,6 +71,10 @@ func (e APIError) XMLBody(requestID, hostID string) []byte { }) } +func (e APIError) HTMLBody(requestID, hostID string) []byte { + return e.encodeHTMLResponse(requestID, hostID) +} + // ErrorCode type of error status. type ErrorCode int @@ -166,9 +172,9 @@ const ( ErrMissingCORSOrigin ErrCORSIsNotEnabled ErrNoSuchWebsiteConfiguration - ErrInvalidWebsiteConfiguration - ErrInvalidWebsiteSuffix - ErrInvalidWebsiteRedirectCode + ErrInvalidWebsiteRedirectProtocol + ErrBothReplaceKeyAndPrefix + ErrMaxMessageLengthExceeded ErrNotModified ErrInvalidLocationConstraint ErrMalformedTrailer @@ -176,6 +182,7 @@ const ( ErrSlowDown ErrMetadataTooLarge ErrUnsupportedAuthorizationMechanism + ErrNoBucketInRequest // Non-AWS errors ErrExistingObjectIsDirectory @@ -648,19 +655,19 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "The specified bucket does not have a website configuration", HTTPStatusCode: http.StatusNotFound, }, - ErrInvalidWebsiteConfiguration: { - Code: "MalformedXML", - Description: "The XML you provided was not well-formed or did not validate against our published schema.", + ErrInvalidWebsiteRedirectProtocol: { + Code: "InvalidRequest", + Description: "Invalid protocol, protocol can be http or https. If not defined the protocol will be selected automatically.", HTTPStatusCode: http.StatusBadRequest, }, - ErrInvalidWebsiteSuffix: { - Code: "InvalidArgument", - Description: "The IndexDocument Suffix is not well formed", + ErrBothReplaceKeyAndPrefix: { + Code: "InvalidRequest", + Description: "You can only define ReplaceKeyPrefix or ReplaceKey but not both.", HTTPStatusCode: http.StatusBadRequest, }, - ErrInvalidWebsiteRedirectCode: { - Code: "InvalidArgument", - Description: "The website redirect code is not valid. Valid codes are 3XX.", + ErrMaxMessageLengthExceeded: { + Code: "MaxMessageLengthExceeded", + Description: "Your request was too big.", HTTPStatusCode: http.StatusBadRequest, }, ErrNotModified: { @@ -698,6 +705,11 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.", HTTPStatusCode: http.StatusBadRequest, }, + ErrNoBucketInRequest: { + Code: "WebsiteRedirect", + Description: "Request does not contain a bucket name.", + HTTPStatusCode: http.StatusMovedPermanently, + }, // non aws errors ErrExistingObjectIsDirectory: { @@ -793,6 +805,42 @@ func encodeResponse(response any) []byte { return bytesBuffer.Bytes() } +type ErrorField struct { + Name string + Value any +} + +func (e APIError) encodeHTMLResponse(requestID, hostID string, fields ...ErrorField) []byte { + status := fmt.Sprintf("%d %s", e.HTTPStatusCode, http.StatusText(e.HTTPStatusCode)) + + builder := &strings.Builder{} + builder.WriteString("\n") + builder.WriteString("") + builder.WriteString(html.EscapeString(status)) + builder.WriteString("\n\n

    ") + builder.WriteString(html.EscapeString(status)) + builder.WriteString("

    \n
      \n") + + writeHTMLErrorField(builder, "Code", e.Code) + writeHTMLErrorField(builder, "Message", e.Description) + for _, field := range fields { + writeHTMLErrorField(builder, field.Name, field.Value) + } + writeHTMLErrorField(builder, "RequestId", requestID) + writeHTMLErrorField(builder, "HostId", hostID) + + builder.WriteString("
    \n
    \n\n\n") + return []byte(builder.String()) +} + +func writeHTMLErrorField(builder *strings.Builder, name string, value any) { + builder.WriteString("
  • ") + builder.WriteString(html.EscapeString(name)) + builder.WriteString(": ") + builder.WriteString(html.EscapeString(fmt.Sprint(value))) + builder.WriteString("
    • \n") +} + // Returns invalid checksum error with the provided header in the error description func GetInvalidChecksumHeaderErr(header string) APIError { return APIError{ @@ -918,6 +966,30 @@ func GetCopySourceObjectTooLargeErr(limit int64) APIError { } } +func GetInvalidRedirectCodeErr(input int) APIError { + return APIError{ + Code: "InvalidRequest", + Description: fmt.Sprintf("The provided HTTP redirect code (%d) is not valid. Valid codes are 3XX except 300.", input), + HTTPStatusCode: http.StatusBadRequest, + } +} + +func GetInvalidHTTPErrorCodeErr(input int) APIError { + return APIError{ + Code: "InvalidRequest", + Description: fmt.Sprintf("The provided HTTP error code (%d) is not valid. Valid codes are 4XX or 5XX.", input), + HTTPStatusCode: http.StatusBadRequest, + } +} + +func GetWebsiteRoutingRulesLimitedErr(rules int) APIError { + return APIError{ + Code: "InvalidRequest", + Description: fmt.Sprintf("%d routing rules provided, the number of routing rules in a website configuration is limited to 50.", rules), + HTTPStatusCode: http.StatusBadRequest, + } +} + type ResourceType string const ( diff --git a/s3err/signature-does-not-match-error.go b/s3err/signature-does-not-match-error.go index 97cfeb96..51ec315f 100644 --- a/s3err/signature-does-not-match-error.go +++ b/s3err/signature-does-not-match-error.go @@ -55,6 +55,17 @@ func (e SignatureDoesNotMatchError) XMLBody(requestID, hostID string) []byte { }) } +func (e SignatureDoesNotMatchError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "AWSAccessKeyId", Value: e.AWSAccessKeyId}, + ErrorField{Name: "StringToSign", Value: e.StringToSign}, + ErrorField{Name: "SignatureProvided", Value: e.SignatureProvided}, + ErrorField{Name: "StringToSignBytes", Value: e.StringToSignBytes}, + ErrorField{Name: "CanonicalRequest", Value: e.CanonicalRequest}, + ErrorField{Name: "CanonicalRequestBytes", Value: e.CanonicalRequestBytes}, + ) +} + func (e SignatureDoesNotMatchError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3err/sigv4.go b/s3err/sigv4.go index a868089d..ebb25be3 100644 --- a/s3err/sigv4.go +++ b/s3err/sigv4.go @@ -44,6 +44,12 @@ func (e MalformedAuthError) XMLBody(requestID, hostID string) []byte { }) } +func (e MalformedAuthError) HTMLBody(requestID, hostID string) []byte { + return e.APIError.encodeHTMLResponse(requestID, hostID, + ErrorField{Name: "Region", Value: e.Region}, + ) +} + func (e MalformedAuthError) Is(target error) bool { t, ok := target.(APIError) return ok && e.APIError == t diff --git a/s3response/website.go b/s3response/website.go index 16884e38..976a0769 100644 --- a/s3response/website.go +++ b/s3response/website.go @@ -17,8 +17,10 @@ package s3response import ( "encoding/xml" "fmt" + "strconv" "strings" + "github.com/versity/versitygw/debuglogger" "github.com/versity/versitygw/s3err" ) @@ -74,10 +76,12 @@ type Redirect struct { func (c *WebsiteConfiguration) Validate() error { if c.RedirectAllRequestsTo != nil { if c.IndexDocument != nil || c.ErrorDocument != nil || len(c.RoutingRules) > 0 { - return s3err.GetAPIError(s3err.ErrInvalidWebsiteConfiguration) + debuglogger.Logf("website redirect conflicts with config") + return s3err.GetAPIError(s3err.ErrMalformedXML) } if c.RedirectAllRequestsTo.HostName == "" { - return s3err.GetAPIError(s3err.ErrInvalidWebsiteConfiguration) + debuglogger.Logf("website redirect hostname is empty") + return s3err.GetAPIError(s3err.ErrMalformedXML) } if err := validateProtocol(c.RedirectAllRequestsTo.Protocol); err != nil { return err @@ -86,26 +90,31 @@ func (c *WebsiteConfiguration) Validate() error { } if c.IndexDocument == nil { - return s3err.GetAPIError(s3err.ErrInvalidWebsiteConfiguration) + debuglogger.Logf("website index document is missing") + return s3err.GetAPIError(s3err.ErrMalformedXML) } if c.IndexDocument.Suffix == "" { - return s3err.GetAPIError(s3err.ErrInvalidWebsiteSuffix) + debuglogger.Logf("website index suffix is empty") + return s3err.GetInvalidArgumentErr(s3err.InvalidArgIndexDocumentSuffix, c.IndexDocument.Suffix) } if strings.Contains(c.IndexDocument.Suffix, "/") { - return s3err.GetAPIError(s3err.ErrInvalidWebsiteSuffix) + debuglogger.Logf("website index suffix contains slash") + return s3err.GetInvalidArgumentErr(s3err.InvalidArgIndexDocumentSuffix, c.IndexDocument.Suffix) } if c.ErrorDocument != nil && c.ErrorDocument.Key == "" { - return s3err.GetAPIError(s3err.ErrInvalidWebsiteConfiguration) + debuglogger.Logf("website error document key is empty") + return s3err.GetInvalidArgumentErr(s3err.InvalidArgErrorDocumentKey, "") } if len(c.RoutingRules) > maxRoutingRules { - return s3err.GetAPIError(s3err.ErrInvalidWebsiteConfiguration) + debuglogger.Logf("too many website routing rules: %d", len(c.RoutingRules)) + return s3err.GetWebsiteRoutingRulesLimitedErr(len(c.RoutingRules)) } - for i, rule := range c.RoutingRules { + for _, rule := range c.RoutingRules { if err := rule.Validate(); err != nil { - return fmt.Errorf("routing rule %d: %w", i, err) + return err } } @@ -114,27 +123,84 @@ func (c *WebsiteConfiguration) Validate() error { // Validate checks a single routing rule for validity. func (r *RoutingRule) Validate() error { - if r.Redirect.ReplaceKeyWith != "" && r.Redirect.ReplaceKeyPrefixWith != "" { - return s3err.GetAPIError(s3err.ErrInvalidWebsiteConfiguration) - } - - if err := validateProtocol(r.Redirect.Protocol); err != nil { + if err := r.Redirect.Validate(); err != nil { return err } - if r.Redirect.HttpRedirectCode != "" { - code := r.Redirect.HttpRedirectCode - if len(code) != 3 || code[0] != '3' { - return s3err.GetAPIError(s3err.ErrInvalidWebsiteRedirectCode) - } + if err := r.Condition.Validate(); err != nil { + return err } return nil } +func (c *RoutingRuleCondition) Validate() error { + if c == nil { + return nil + } + + return isValidHTTPCode(c.HttpErrorCodeReturnedEquals, validateErrorCode) +} + +func (r *Redirect) Validate() error { + if r.ReplaceKeyWith != "" && r.ReplaceKeyPrefixWith != "" { + debuglogger.Logf("website redirect has both key replacements") + return s3err.GetAPIError(s3err.ErrBothReplaceKeyAndPrefix) + } + + if err := validateProtocol(r.Protocol); err != nil { + return err + } + + if err := isValidHTTPCode(r.HttpRedirectCode, validateRedirectCode); err != nil { + return err + } + + return nil +} + +type httpCodeValidator func(code int) error + +func isValidHTTPCode(input string, validateCode httpCodeValidator) error { + if input == "" { + return nil + } + + code, err := strconv.Atoi(input) + if err != nil { + return s3err.GetAPIError(s3err.ErrMalformedXML) + } + + return validateCode(code) +} + +// isValidErrorCode checks if the provided code is a valid +// HTTP error code: S3 considers 400-417 and 500-505 as valid +func validateErrorCode(code int) error { + if (code >= 400 && code <= 417) || (code >= 500 && code <= 505) { + return nil + } + + debuglogger.Logf("invalid website error code: %d", code) + return s3err.GetInvalidHTTPErrorCodeErr(code) +} + +// validateRedirectCode check if the provided code +// is a valid HTTP redirect code +func validateRedirectCode(code int) error { + switch code { + case 301, 302, 303, 304, 305, 307, 308: + return nil + } + + debuglogger.Logf("invalid website redirect code: %d", code) + return s3err.GetInvalidRedirectCodeErr(code) +} + func validateProtocol(protocol string) error { if protocol != "" && protocol != "http" && protocol != "https" { - return s3err.GetAPIError(s3err.ErrInvalidWebsiteConfiguration) + debuglogger.Logf("invalid website redirect protocol: %q", protocol) + return s3err.GetAPIError(s3err.ErrInvalidWebsiteRedirectProtocol) } return nil } @@ -144,26 +210,27 @@ func ParseWebsiteConfigOutput(data []byte) (*WebsiteConfiguration, error) { var config WebsiteConfiguration err := xml.Unmarshal(data, &config) if err != nil { + debuglogger.Logf("failed to parse website config: %v", err) return nil, fmt.Errorf("failed to parse website config: %w", err) } return &config, nil } -// MatchPreRequestRule returns the first routing rule that matches based only -// on KeyPrefixEquals (i.e. rules without HttpErrorCodeReturnedEquals). These -// rules can be evaluated before the backend request is made. A rule with no -// condition at all is treated as an unconditional match. -func (c *WebsiteConfiguration) MatchPreRequestRule(key string) *RoutingRule { +// MatchPrefetchRoutingRule returns the first rule that can be evaluated before +// attempting an object read. Only prefix-only conditions participate in this +// phase. +func (c *WebsiteConfiguration) MatchPrefetchRoutingRule(key string) *RoutingRule { for i := range c.RoutingRules { rule := &c.RoutingRules[i] - - if rule.Condition != nil && rule.Condition.HttpErrorCodeReturnedEquals != "" { - // This is a post-request rule, skip it + condition := rule.Condition + if condition == nil || + condition.KeyPrefixEquals == "" || + condition.HttpErrorCodeReturnedEquals != "" { continue } - if rule.Condition == nil || strings.HasPrefix(key, rule.Condition.KeyPrefixEquals) { + if condition.KeyPrefixEquals != "" && strings.HasPrefix(key, condition.KeyPrefixEquals) { return rule } } @@ -171,28 +238,39 @@ func (c *WebsiteConfiguration) MatchPreRequestRule(key string) *RoutingRule { return nil } -// MatchPostRequestRule returns the first routing rule that matches based on -// HttpErrorCodeReturnedEquals (and optionally KeyPrefixEquals). These rules -// are evaluated after the backend returns an error. -func (c *WebsiteConfiguration) MatchPostRequestRule(key, httpErrorCode string) *RoutingRule { +// MatchPostErrorRoutingRule returns the first rule that matches after a 4xx +// object-read error. Prefix-only rules are skipped because they have already +// been evaluated in the pre-fetch phase. +func (c *WebsiteConfiguration) MatchPostErrorRoutingRule(key string, statusCode int) *RoutingRule { for i := range c.RoutingRules { rule := &c.RoutingRules[i] - - if rule.Condition == nil || rule.Condition.HttpErrorCodeReturnedEquals == "" { - // Not a post-request rule + condition := rule.Condition + if condition != nil && condition.HttpErrorCodeReturnedEquals == "" { continue } - if rule.Condition.HttpErrorCodeReturnedEquals != httpErrorCode { - continue + if condition.Matches(key, statusCode) { + return rule } - - if rule.Condition.KeyPrefixEquals != "" && !strings.HasPrefix(key, rule.Condition.KeyPrefixEquals) { - continue - } - - return rule } return nil } + +// Matches reports whether all configured condition fields match. +func (c *RoutingRuleCondition) Matches(key string, statusCode int) bool { + if c == nil { + return true + } + + if c.KeyPrefixEquals != "" && !strings.HasPrefix(key, c.KeyPrefixEquals) { + return false + } + + if c.HttpErrorCodeReturnedEquals != "" && + strconv.Itoa(statusCode) != c.HttpErrorCodeReturnedEquals { + return false + } + + return true +} diff --git a/s3response/website_test.go b/s3response/website_test.go index b644a154..a5867e7f 100644 --- a/s3response/website_test.go +++ b/s3response/website_test.go @@ -15,7 +15,7 @@ package s3response import ( - "encoding/xml" + "errors" "testing" "github.com/versity/versitygw/s3err" @@ -26,7 +26,7 @@ func TestWebsiteConfiguration_Validate(t *testing.T) { name string config WebsiteConfiguration wantErr bool - errCode s3err.ErrorCode + errCode string }{ { name: "valid index document only", @@ -70,7 +70,7 @@ func TestWebsiteConfiguration_Validate(t *testing.T) { name: "missing index document", config: WebsiteConfiguration{}, wantErr: true, - errCode: s3err.ErrInvalidWebsiteConfiguration, + errCode: "MalformedXML", }, { name: "empty index suffix", @@ -78,7 +78,7 @@ func TestWebsiteConfiguration_Validate(t *testing.T) { IndexDocument: &IndexDocument{Suffix: ""}, }, wantErr: true, - errCode: s3err.ErrInvalidWebsiteSuffix, + errCode: "InvalidArgument", }, { name: "index suffix with slash", @@ -86,7 +86,7 @@ func TestWebsiteConfiguration_Validate(t *testing.T) { IndexDocument: &IndexDocument{Suffix: "dir/index.html"}, }, wantErr: true, - errCode: s3err.ErrInvalidWebsiteSuffix, + errCode: "InvalidArgument", }, { name: "redirect all with index document", @@ -95,7 +95,7 @@ func TestWebsiteConfiguration_Validate(t *testing.T) { IndexDocument: &IndexDocument{Suffix: "index.html"}, }, wantErr: true, - errCode: s3err.ErrInvalidWebsiteConfiguration, + errCode: "MalformedXML", }, { name: "redirect all with empty hostname", @@ -103,7 +103,7 @@ func TestWebsiteConfiguration_Validate(t *testing.T) { RedirectAllRequestsTo: &RedirectAllRequestsTo{HostName: ""}, }, wantErr: true, - errCode: s3err.ErrInvalidWebsiteConfiguration, + errCode: "MalformedXML", }, { name: "redirect all with invalid protocol", @@ -114,7 +114,7 @@ func TestWebsiteConfiguration_Validate(t *testing.T) { }, }, wantErr: true, - errCode: s3err.ErrInvalidWebsiteConfiguration, + errCode: "InvalidRequest", }, { name: "routing rule with both replace key fields", @@ -130,7 +130,7 @@ func TestWebsiteConfiguration_Validate(t *testing.T) { }, }, wantErr: true, - errCode: s3err.ErrInvalidWebsiteConfiguration, + errCode: "InvalidRequest", }, { name: "routing rule with invalid redirect code", @@ -145,7 +145,7 @@ func TestWebsiteConfiguration_Validate(t *testing.T) { }, }, wantErr: true, - errCode: s3err.ErrInvalidWebsiteRedirectCode, + errCode: "InvalidRequest", }, { name: "routing rule with valid redirect code", @@ -168,7 +168,7 @@ func TestWebsiteConfiguration_Validate(t *testing.T) { ErrorDocument: &ErrorDocument{Key: ""}, }, wantErr: true, - errCode: s3err.ErrInvalidWebsiteConfiguration, + errCode: "InvalidArgument", }, } @@ -179,14 +179,12 @@ func TestWebsiteConfiguration_Validate(t *testing.T) { if err == nil { t.Fatal("expected error, got nil") } - apiErr, ok := err.(s3err.APIError) - if !ok { - // wrapped error from routing rule validation - return + var apiErr s3err.S3Error + if !errors.As(err, &apiErr) { + t.Fatalf("expected S3 error, got %T: %v", err, err) } - expectedErr := s3err.GetAPIError(tt.errCode) - if apiErr.Code != expectedErr.Code { - t.Errorf("expected error code %q, got %q", expectedErr.Code, apiErr.Code) + if apiErr.BaseError().Code != tt.errCode { + t.Errorf("expected error code %q, got %q", tt.errCode, apiErr.BaseError().Code) } } else { if err != nil { @@ -197,390 +195,83 @@ func TestWebsiteConfiguration_Validate(t *testing.T) { } } -func TestWebsiteConfiguration_XMLRoundTrip(t *testing.T) { - original := WebsiteConfiguration{ +func TestWebsiteConfiguration_MatchPrefetchRoutingRuleUsesPrefixOnlyRules(t *testing.T) { + config := WebsiteConfiguration{ IndexDocument: &IndexDocument{Suffix: "index.html"}, - ErrorDocument: &ErrorDocument{Key: "error.html"}, RoutingRules: []RoutingRule{ { Condition: &RoutingRuleCondition{ - KeyPrefixEquals: "docs/", HttpErrorCodeReturnedEquals: "404", }, Redirect: Redirect{ - HostName: "example.com", - Protocol: "https", - HttpRedirectCode: "301", - ReplaceKeyPrefixWith: "documents/", + HostName: "error.example.com", + }, + }, + { + Condition: &RoutingRuleCondition{ + KeyPrefixEquals: "old/", + HttpErrorCodeReturnedEquals: "404", + }, + Redirect: Redirect{ + HostName: "both.example.com", + }, + }, + { + Condition: &RoutingRuleCondition{ + KeyPrefixEquals: "old/", + }, + Redirect: Redirect{ + HostName: "prefix.example.com", }, }, }, } - data, err := xml.Marshal(original) - if err != nil { - t.Fatalf("marshal: %v", err) + rule := config.MatchPrefetchRoutingRule("old/page.html") + if rule == nil { + t.Fatal("expected a matching rule, got nil") } - - var parsed WebsiteConfiguration - if err := xml.Unmarshal(data, &parsed); err != nil { - t.Fatalf("unmarshal: %v", err) - } - - if parsed.IndexDocument == nil || parsed.IndexDocument.Suffix != "index.html" { - t.Error("IndexDocument.Suffix mismatch") - } - if parsed.ErrorDocument == nil || parsed.ErrorDocument.Key != "error.html" { - t.Error("ErrorDocument.Key mismatch") - } - if len(parsed.RoutingRules) != 1 { - t.Fatalf("expected 1 routing rule, got %d", len(parsed.RoutingRules)) - } - rule := parsed.RoutingRules[0] - if rule.Condition == nil || rule.Condition.KeyPrefixEquals != "docs/" { - t.Error("RoutingRule Condition.KeyPrefixEquals mismatch") - } - if rule.Redirect.HostName != "example.com" { - t.Error("RoutingRule Redirect.HostName mismatch") - } - if rule.Redirect.ReplaceKeyPrefixWith != "documents/" { - t.Error("RoutingRule Redirect.ReplaceKeyPrefixWith mismatch") + if rule.Redirect.HostName != "prefix.example.com" { + t.Fatalf("expected prefix-only rule to match, got %q", rule.Redirect.HostName) } } -func TestParseWebsiteConfigOutput(t *testing.T) { - xmlData := ` - index.html - error.html - ` +func TestRoutingRuleCondition_MatchesUsesAndLogic(t *testing.T) { + condition := RoutingRuleCondition{ + KeyPrefixEquals: "old/", + HttpErrorCodeReturnedEquals: "404", + } - config, err := ParseWebsiteConfigOutput([]byte(xmlData)) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if config.IndexDocument == nil || config.IndexDocument.Suffix != "index.html" { - t.Error("IndexDocument.Suffix mismatch") - } - if config.ErrorDocument == nil || config.ErrorDocument.Key != "error.html" { - t.Error("ErrorDocument.Key mismatch") - } -} - -func TestParseWebsiteConfigOutput_InvalidXML(t *testing.T) { - _, err := ParseWebsiteConfigOutput([]byte("not xml")) - if err == nil { - t.Fatal("expected error for invalid XML") - } -} - -func TestWebsiteConfiguration_MatchPreRequestRule(t *testing.T) { tests := []struct { - name string - config WebsiteConfiguration - key string - wantNil bool - wantHost string // expected redirect HostName if matched + name string + key string + statusCode int + want bool }{ { - name: "no routing rules", - config: WebsiteConfiguration{ - IndexDocument: &IndexDocument{Suffix: "index.html"}, - }, - key: "docs/page.html", - wantNil: true, + name: "both match", + key: "old/missing.html", + statusCode: 404, + want: true, }, { - name: "key prefix match", - config: WebsiteConfiguration{ - IndexDocument: &IndexDocument{Suffix: "index.html"}, - RoutingRules: []RoutingRule{ - { - Condition: &RoutingRuleCondition{ - KeyPrefixEquals: "docs/", - }, - Redirect: Redirect{ - HostName: "docs.example.com", - }, - }, - }, - }, - key: "docs/page.html", - wantHost: "docs.example.com", + name: "prefix only", + key: "old/existing.html", + statusCode: 200, + want: false, }, { - name: "key prefix does not match", - config: WebsiteConfiguration{ - IndexDocument: &IndexDocument{Suffix: "index.html"}, - RoutingRules: []RoutingRule{ - { - Condition: &RoutingRuleCondition{ - KeyPrefixEquals: "docs/", - }, - Redirect: Redirect{ - HostName: "docs.example.com", - }, - }, - }, - }, - key: "images/photo.jpg", - wantNil: true, - }, - { - name: "unconditional rule (no condition)", - config: WebsiteConfiguration{ - IndexDocument: &IndexDocument{Suffix: "index.html"}, - RoutingRules: []RoutingRule{ - { - Redirect: Redirect{ - HostName: "redirect.example.com", - }, - }, - }, - }, - key: "anything", - wantHost: "redirect.example.com", - }, - { - name: "skips post-request rules", - config: WebsiteConfiguration{ - IndexDocument: &IndexDocument{Suffix: "index.html"}, - RoutingRules: []RoutingRule{ - { - Condition: &RoutingRuleCondition{ - HttpErrorCodeReturnedEquals: "404", - KeyPrefixEquals: "docs/", - }, - Redirect: Redirect{ - HostName: "error.example.com", - }, - }, - }, - }, - key: "docs/page.html", - wantNil: true, - }, - { - name: "first matching rule wins", - config: WebsiteConfiguration{ - IndexDocument: &IndexDocument{Suffix: "index.html"}, - RoutingRules: []RoutingRule{ - { - Condition: &RoutingRuleCondition{ - KeyPrefixEquals: "docs/", - }, - Redirect: Redirect{ - HostName: "first.example.com", - }, - }, - { - Condition: &RoutingRuleCondition{ - KeyPrefixEquals: "docs/api/", - }, - Redirect: Redirect{ - HostName: "second.example.com", - }, - }, - }, - }, - key: "docs/api/endpoint", - wantHost: "first.example.com", + name: "status only", + key: "other/missing.html", + statusCode: 404, + want: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - rule := tt.config.MatchPreRequestRule(tt.key) - if tt.wantNil { - if rule != nil { - t.Fatalf("expected nil, got rule with redirect to %q", rule.Redirect.HostName) - } - return - } - if rule == nil { - t.Fatal("expected a matching rule, got nil") - } - if rule.Redirect.HostName != tt.wantHost { - t.Errorf("expected redirect host %q, got %q", tt.wantHost, rule.Redirect.HostName) - } - }) - } -} - -func TestWebsiteConfiguration_MatchPostRequestRule(t *testing.T) { - tests := []struct { - name string - config WebsiteConfiguration - key string - httpErrorCode string - wantNil bool - wantHost string - }{ - { - name: "no routing rules", - config: WebsiteConfiguration{ - IndexDocument: &IndexDocument{Suffix: "index.html"}, - }, - key: "page.html", - httpErrorCode: "404", - wantNil: true, - }, - { - name: "error code match", - config: WebsiteConfiguration{ - IndexDocument: &IndexDocument{Suffix: "index.html"}, - RoutingRules: []RoutingRule{ - { - Condition: &RoutingRuleCondition{ - HttpErrorCodeReturnedEquals: "404", - }, - Redirect: Redirect{ - HostName: "notfound.example.com", - }, - }, - }, - }, - key: "page.html", - httpErrorCode: "404", - wantHost: "notfound.example.com", - }, - { - name: "error code does not match", - config: WebsiteConfiguration{ - IndexDocument: &IndexDocument{Suffix: "index.html"}, - RoutingRules: []RoutingRule{ - { - Condition: &RoutingRuleCondition{ - HttpErrorCodeReturnedEquals: "404", - }, - Redirect: Redirect{ - HostName: "notfound.example.com", - }, - }, - }, - }, - key: "page.html", - httpErrorCode: "403", - wantNil: true, - }, - { - name: "error code and key prefix both match", - config: WebsiteConfiguration{ - IndexDocument: &IndexDocument{Suffix: "index.html"}, - RoutingRules: []RoutingRule{ - { - Condition: &RoutingRuleCondition{ - HttpErrorCodeReturnedEquals: "404", - KeyPrefixEquals: "docs/", - }, - Redirect: Redirect{ - HostName: "docs-error.example.com", - }, - }, - }, - }, - key: "docs/missing.html", - httpErrorCode: "404", - wantHost: "docs-error.example.com", - }, - { - name: "error code matches but key prefix does not", - config: WebsiteConfiguration{ - IndexDocument: &IndexDocument{Suffix: "index.html"}, - RoutingRules: []RoutingRule{ - { - Condition: &RoutingRuleCondition{ - HttpErrorCodeReturnedEquals: "404", - KeyPrefixEquals: "docs/", - }, - Redirect: Redirect{ - HostName: "docs-error.example.com", - }, - }, - }, - }, - key: "images/missing.jpg", - httpErrorCode: "404", - wantNil: true, - }, - { - name: "skips pre-request rules (no error code condition)", - config: WebsiteConfiguration{ - IndexDocument: &IndexDocument{Suffix: "index.html"}, - RoutingRules: []RoutingRule{ - { - Condition: &RoutingRuleCondition{ - KeyPrefixEquals: "docs/", - }, - Redirect: Redirect{ - HostName: "pre-request.example.com", - }, - }, - }, - }, - key: "docs/page.html", - httpErrorCode: "404", - wantNil: true, - }, - { - name: "skips rules with no condition", - config: WebsiteConfiguration{ - IndexDocument: &IndexDocument{Suffix: "index.html"}, - RoutingRules: []RoutingRule{ - { - Redirect: Redirect{ - HostName: "unconditional.example.com", - }, - }, - }, - }, - key: "page.html", - httpErrorCode: "404", - wantNil: true, - }, - { - name: "first matching rule wins", - config: WebsiteConfiguration{ - IndexDocument: &IndexDocument{Suffix: "index.html"}, - RoutingRules: []RoutingRule{ - { - Condition: &RoutingRuleCondition{ - HttpErrorCodeReturnedEquals: "404", - }, - Redirect: Redirect{ - HostName: "first.example.com", - }, - }, - { - Condition: &RoutingRuleCondition{ - HttpErrorCodeReturnedEquals: "404", - KeyPrefixEquals: "docs/", - }, - Redirect: Redirect{ - HostName: "second.example.com", - }, - }, - }, - }, - key: "docs/page.html", - httpErrorCode: "404", - wantHost: "first.example.com", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - rule := tt.config.MatchPostRequestRule(tt.key, tt.httpErrorCode) - if tt.wantNil { - if rule != nil { - t.Fatalf("expected nil, got rule with redirect to %q", rule.Redirect.HostName) - } - return - } - if rule == nil { - t.Fatal("expected a matching rule, got nil") - } - if rule.Redirect.HostName != tt.wantHost { - t.Errorf("expected redirect host %q, got %q", tt.wantHost, rule.Redirect.HostName) + if got := condition.Matches(tt.key, tt.statusCode); got != tt.want { + t.Fatalf("Matches() = %v, want %v", got, tt.want) } }) } diff --git a/tests/integration/PutBucketWebsite.go b/tests/integration/PutBucketWebsite.go index 11e97f38..ed0a09d4 100644 --- a/tests/integration/PutBucketWebsite.go +++ b/tests/integration/PutBucketWebsite.go @@ -16,12 +16,16 @@ package integration import ( "context" + "fmt" + "strings" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/versity/versitygw/s3err" ) +const maxWebsiteConfigSize = 131072 + func PutBucketWebsite_non_existing_bucket(s *S3Conf) error { testName := "PutBucketWebsite_non_existing_bucket" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -52,7 +56,7 @@ func PutBucketWebsite_empty_suffix(s *S3Conf) error { }, }) cancel() - return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidWebsiteSuffix)) + return checkApiErr(err, s3err.GetInvalidArgumentErr(s3err.InvalidArgIndexDocumentSuffix, "")) }) } @@ -69,7 +73,7 @@ func PutBucketWebsite_suffix_with_slash(s *S3Conf) error { }, }) cancel() - return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidWebsiteSuffix)) + return checkApiErr(err, s3err.GetInvalidArgumentErr(s3err.InvalidArgIndexDocumentSuffix, "/index.html")) }) } @@ -87,27 +91,284 @@ func PutBucketWebsite_invalid_redirect_protocol(s *S3Conf) error { }, }) cancel() - return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidWebsiteConfiguration)) + return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidWebsiteRedirectProtocol)) }) } -func PutBucketWebsite_redirect_and_index(s *S3Conf) error { - testName := "PutBucketWebsite_redirect_and_index" +func PutBucketWebsite_redirectAll_index_error_routingRules(s *S3Conf) error { + testName := "PutBucketWebsite_redirectAll_index_error_routingRules" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + for _, test := range []struct { + name string + config *types.WebsiteConfiguration + }{ + { + name: "index document", + config: &types.WebsiteConfiguration{ + RedirectAllRequestsTo: &types.RedirectAllRequestsTo{ + HostName: getPtr("example.com"), + }, + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + }, + }, + { + name: "error document", + config: &types.WebsiteConfiguration{ + RedirectAllRequestsTo: &types.RedirectAllRequestsTo{ + HostName: getPtr("example.com"), + }, + ErrorDocument: &types.ErrorDocument{ + Key: getPtr("error.html"), + }, + }, + }, + { + name: "routing rules", + config: &types.WebsiteConfiguration{ + RedirectAllRequestsTo: &types.RedirectAllRequestsTo{ + HostName: getPtr("example.com"), + }, + RoutingRules: []types.RoutingRule{ + { + Redirect: &types.Redirect{ + HostName: getPtr("redirect.example.com"), + }, + }, + }, + }, + }, + } { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: test.config, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMalformedXML)); err != nil { + return fmt.Errorf("%s: %w", test.name, err) + } + } + + return nil + }) +} + +func PutBucketWebsite_invalid_routing_rule_protocol(s *S3Conf) error { + testName := "PutBucketWebsite_invalid_routing_rule_protocol" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ Bucket: &bucket, WebsiteConfiguration: &types.WebsiteConfiguration{ - RedirectAllRequestsTo: &types.RedirectAllRequestsTo{ - HostName: getPtr("example.com"), - }, IndexDocument: &types.IndexDocument{ Suffix: getPtr("index.html"), }, + RoutingRules: []types.RoutingRule{ + { + Redirect: &types.Redirect{ + HostName: getPtr("example.com"), + Protocol: types.Protocol("ftp"), + }, + }, + }, }, }) cancel() - return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidWebsiteConfiguration)) + return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidWebsiteRedirectProtocol)) + }) +} + +func PutBucketWebsite_empty_error_document_key(s *S3Conf) error { + testName := "PutBucketWebsite_empty_error_document_key" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + ErrorDocument: &types.ErrorDocument{ + Key: getPtr(""), + }, + }, + }) + cancel() + return checkApiErr(err, s3err.GetInvalidArgumentErr(s3err.InvalidArgErrorDocumentKey, "")) + }) +} + +func PutBucketWebsite_too_many_routing_rules(s *S3Conf) error { + testName := "PutBucketWebsite_too_many_routing_rules" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + routingRules := make([]types.RoutingRule, 51) + for i := range routingRules { + routingRules[i] = types.RoutingRule{ + Condition: &types.Condition{ + KeyPrefixEquals: getPtr(fmt.Sprintf("prefix-%d/", i)), + }, + Redirect: &types.Redirect{ + ReplaceKeyPrefixWith: getPtr(fmt.Sprintf("replacement-%d/", i)), + }, + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + RoutingRules: routingRules, + }, + }) + cancel() + return checkApiErr(err, s3err.GetWebsiteRoutingRulesLimitedErr(51)) + }) +} + +func PutBucketWebsite_routing_rule_replace_key_and_prefix(s *S3Conf) error { + testName := "PutBucketWebsite_routing_rule_replace_key_and_prefix" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + RoutingRules: []types.RoutingRule{ + { + Redirect: &types.Redirect{ + ReplaceKeyWith: getPtr("replacement.html"), + ReplaceKeyPrefixWith: getPtr("replacement-prefix/"), + }, + }, + }, + }, + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrBothReplaceKeyAndPrefix)) + }) +} + +func PutBucketWebsite_invalid_http_redirect_code(s *S3Conf) error { + testName := "PutBucketWebsite_invalid_http_redirect_code" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + for _, test := range []struct { + code string + expectedErr s3err.S3Error + }{ + {code: "300", expectedErr: s3err.GetInvalidRedirectCodeErr(300)}, + {code: "306", expectedErr: s3err.GetInvalidRedirectCodeErr(306)}, + {code: "309", expectedErr: s3err.GetInvalidRedirectCodeErr(309)}, + {code: "399", expectedErr: s3err.GetInvalidRedirectCodeErr(399)}, + {code: "jibberish", expectedErr: s3err.GetAPIError(s3err.ErrMalformedXML)}, + {code: "3xx", expectedErr: s3err.GetAPIError(s3err.ErrMalformedXML)}, + } { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + RoutingRules: []types.RoutingRule{ + { + Redirect: &types.Redirect{ + HostName: getPtr("example.com"), + HttpRedirectCode: getPtr(test.code), + }, + }, + }, + }, + }) + cancel() + if err := checkApiErr(err, test.expectedErr); err != nil { + return fmt.Errorf("code %q: %w", test.code, err) + } + } + + return nil + }) +} + +func PutBucketWebsite_invalid_http_error_code(s *S3Conf) error { + testName := "PutBucketWebsite_invalid_http_error_code" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + for _, test := range []struct { + code string + expectedErr s3err.S3Error + }{ + {code: "399", expectedErr: s3err.GetInvalidHTTPErrorCodeErr(399)}, + {code: "418", expectedErr: s3err.GetInvalidHTTPErrorCodeErr(418)}, + {code: "499", expectedErr: s3err.GetInvalidHTTPErrorCodeErr(499)}, + {code: "506", expectedErr: s3err.GetInvalidHTTPErrorCodeErr(506)}, + {code: "jibberish", expectedErr: s3err.GetAPIError(s3err.ErrMalformedXML)}, + {code: "4xx", expectedErr: s3err.GetAPIError(s3err.ErrMalformedXML)}, + } { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + RoutingRules: []types.RoutingRule{ + { + Condition: &types.Condition{ + HttpErrorCodeReturnedEquals: getPtr(test.code), + }, + Redirect: &types.Redirect{ + HostName: getPtr("example.com"), + }, + }, + }, + }, + }) + cancel() + if err := checkApiErr(err, test.expectedErr); err != nil { + return fmt.Errorf("code %q: %w", test.code, err) + } + } + + return nil + }) +} + +func PutBucketWebsite_request_too_large(s *S3Conf) error { + testName := "PutBucketWebsite_request_too_large" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + longValue := strings.Repeat("a", 2048) + routingRules := make([]types.RoutingRule, 50) + for i := range routingRules { + routingRules[i] = types.RoutingRule{ + Condition: &types.Condition{ + KeyPrefixEquals: getPtr(fmt.Sprintf("prefix-%d-%s", i, longValue)), + }, + Redirect: &types.Redirect{ + HostName: getPtr("example.com"), + ReplaceKeyWith: getPtr(fmt.Sprintf("replacement-%d-%s", i, longValue)), + HttpRedirectCode: getPtr("301"), + }, + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + RoutingRules: routingRules, + }, + }) + cancel() + return checkApiErr(err, s3err.GetMaxMessageLengthExceeded(maxWebsiteConfigSize)) }) } diff --git a/tests/integration/WebsiteHosting.go b/tests/integration/WebsiteHosting.go index 453a02b2..9d970024 100644 --- a/tests/integration/WebsiteHosting.go +++ b/tests/integration/WebsiteHosting.go @@ -15,399 +15,770 @@ package integration import ( - "bytes" - "context" - "crypto/tls" "fmt" - "io" "net/http" "strings" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/versity/versitygw/s3err" ) -// websiteHTTPClient returns an HTTP client suitable for website endpoint -// requests. It does not follow redirects and skips TLS verification -// (matching the behaviour of the S3Conf http client for self-signed certs). -func websiteHTTPClient() *http.Client { - return &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }, - } -} - -// websiteGet issues a plain HTTP GET to the dedicated website endpoint. -// The bucket is resolved from the Host header. No S3 signing is applied. -func websiteGet(websiteEndpoint, host, path string) (*http.Response, error) { - url := fmt.Sprintf("%s/%s", strings.TrimRight(websiteEndpoint, "/"), strings.TrimLeft(path, "/")) - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - req.Host = host - return websiteHTTPClient().Do(req) -} - -// WebsiteHosting_error_document_served tests that when a website-enabled -// bucket has an error document configured, requesting a non-existing key -// returns the error document content with the original 404 status code. +// WebsiteHosting_error_document_served tests that a missing website object +// serves the configured error document while preserving the original 404 status. func WebsiteHosting_error_document_served(s *S3Conf) error { testName := "WebsiteHosting_error_document_served" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - // Configure website with error document - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ - Bucket: &bucket, - WebsiteConfiguration: &types.WebsiteConfiguration{ - IndexDocument: &types.IndexDocument{ - Suffix: getPtr("index.html"), - }, - ErrorDocument: &types.ErrorDocument{ - Key: getPtr("error.html"), - }, + err := putBucketWebsiteConfig(s3client, bucket, &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + ErrorDocument: &types.ErrorDocument{ + Key: getPtr("error.html"), }, }) - cancel() if err != nil { return err } + if err := grantPublicBucketPolicy(s3client, bucket, policyTypeObject); err != nil { + return err + } - // Upload the error document errorContent := "Custom Error Page" - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ + _, err = putObjectWithData(int64(len(errorContent)), &s3.PutObjectInput{ Bucket: &bucket, Key: getPtr("error.html"), Body: strings.NewReader(errorContent), ContentType: getPtr("text/html"), - }) - cancel() + }, s3client) if err != nil { return err } - // Request a non-existing key via plain HTTP on the website endpoint - resp, err := websiteGet(s.websiteEndpoint, bucket, "nonexistent-key") - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusNotFound { - return fmt.Errorf("expected status 404, got %v", resp.StatusCode) - } - - body, err := io.ReadAll(resp.Body) + resp, err := websiteGet(s, bucket, "nonexistent-key", nil) if err != nil { return err } - if string(body) != errorContent { - return fmt.Errorf("expected error document content %q, got %q", errorContent, string(body)) + if got := resp.Header.Get("Content-Type"); got != "text/html" { + return fmt.Errorf("expected text/html Content-Type, got %q", got) } - - return nil + return checkWebsiteResponse(resp, http.StatusNotFound, []byte(errorContent)) }) } -// WebsiteHosting_error_document_not_found tests that when the configured -// error document itself does not exist, a 404 error page is returned. +// WebsiteHosting_error_document_not_found tests that a missing configured +// error document returns the complete website NoSuchKey error response. func WebsiteHosting_error_document_not_found(s *S3Conf) error { testName := "WebsiteHosting_error_document_not_found" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - // Configure website with error document (but don't upload it) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ - Bucket: &bucket, - WebsiteConfiguration: &types.WebsiteConfiguration{ - IndexDocument: &types.IndexDocument{ - Suffix: getPtr("index.html"), - }, - ErrorDocument: &types.ErrorDocument{ - Key: getPtr("error.html"), - }, + err := putBucketWebsiteConfig(s3client, bucket, &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + ErrorDocument: &types.ErrorDocument{ + Key: getPtr("error.html"), }, }) - cancel() + if err != nil { + return err + } + if err := grantPublicBucketPolicy(s3client, bucket, policyTypeObject); err != nil { + return err + } + + resp, err := websiteGet(s, bucket, "nonexistent-key", nil) if err != nil { return err } - // Request a non-existing key - should get 404 since error doc doesn't exist either - resp, err := websiteGet(s.websiteEndpoint, bucket, "nonexistent-key") - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusNotFound { - return fmt.Errorf("expected status 404, got %v", resp.StatusCode) - } - - return nil + return checkWebsiteErrorResponse(resp, s3err.GetAPIError(s3err.ErrNoSuchKey)) }) } -// WebsiteHosting_no_error_document tests that when website is enabled -// but no error document is configured, a 404 error page is returned. +// WebsiteHosting_no_error_document tests that a website bucket without an +// error document returns the complete website NoSuchKey error response. func WebsiteHosting_no_error_document(s *S3Conf) error { testName := "WebsiteHosting_no_error_document" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - // Configure website without error document - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ - Bucket: &bucket, - WebsiteConfiguration: &types.WebsiteConfiguration{ - IndexDocument: &types.IndexDocument{ - Suffix: getPtr("index.html"), - }, + err := putBucketWebsiteConfig(s3client, bucket, &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), }, }) - cancel() + if err != nil { + return err + } + if err := grantPublicBucketPolicy(s3client, bucket, policyTypeObject); err != nil { + return err + } + + resp, err := websiteGet(s, bucket, "nonexistent-key", nil) if err != nil { return err } - // Request a non-existing key - should get 404 - resp, err := websiteGet(s.websiteEndpoint, bucket, "nonexistent-key") + return checkWebsiteErrorResponse(resp, s3err.GetAPIError(s3err.ErrNoSuchKey)) + }) +} + +// WebsiteHosting_private_object_and_error_document tests that website hosting +// does not serve either the requested object or the configured error document +// unless public object access has been granted. +func WebsiteHosting_private_object_and_error_document(s *S3Conf) error { + testName := "WebsiteHosting_private_object_and_error_document" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putBucketWebsiteConfig(s3client, bucket, &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + ErrorDocument: &types.ErrorDocument{ + Key: getPtr("error.html"), + }, + }) if err != nil { return err } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusNotFound { - return fmt.Errorf("expected status 404, got %v", resp.StatusCode) + privateError := "private error" + _, err = putObjectWithData(int64(len(privateError)), &s3.PutObjectInput{ + Bucket: &bucket, + Key: getPtr("error.html"), + Body: strings.NewReader(privateError), + ContentType: getPtr("text/html"), + }, s3client) + if err != nil { + return err } - return nil + resp, err := websiteGet(s, bucket, "private.html", nil) + if err != nil { + return err + } + + return checkWebsiteErrorResponse(resp, s3err.GetAPIError(s3err.ErrAccessDenied)) }) } // WebsiteHosting_routing_rule_post_request_redirect tests that a post-request -// routing rule (matching on error code) issues a redirect instead of serving -// the error or error document. +// routing rule matching a 404 issues a redirect instead of serving an error. func WebsiteHosting_routing_rule_post_request_redirect(s *S3Conf) error { testName := "WebsiteHosting_routing_rule_post_request_redirect" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - // Configure website with a post-request routing rule for 404 - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ - Bucket: &bucket, - WebsiteConfiguration: &types.WebsiteConfiguration{ - IndexDocument: &types.IndexDocument{ - Suffix: getPtr("index.html"), - }, - ErrorDocument: &types.ErrorDocument{ - Key: getPtr("error.html"), - }, - RoutingRules: []types.RoutingRule{ - { - Condition: &types.Condition{ - HttpErrorCodeReturnedEquals: getPtr("404"), - }, - Redirect: &types.Redirect{ - HostName: getPtr("fallback.example.com"), - ReplaceKeyWith: getPtr("not-found"), - HttpRedirectCode: getPtr("302"), - }, + err := putBucketWebsiteConfig(s3client, bucket, &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + ErrorDocument: &types.ErrorDocument{ + Key: getPtr("error.html"), + }, + RoutingRules: []types.RoutingRule{ + { + Condition: &types.Condition{ + HttpErrorCodeReturnedEquals: getPtr("404"), + }, + Redirect: &types.Redirect{ + HostName: getPtr("fallback.example.com"), + ReplaceKeyWith: getPtr("not-found"), + HttpRedirectCode: getPtr("302"), }, }, }, }) - cancel() + if err != nil { + return err + } + if err := grantPublicBucketPolicy(s3client, bucket, policyTypeObject); err != nil { + return err + } + + resp, err := websiteGet(s, bucket, "missing-page", nil) if err != nil { return err } - // Request a non-existing key via the website endpoint - resp, err := websiteGet(s.websiteEndpoint, bucket, "missing-page") + wantLocation, err := websiteAbsoluteURL(s, "fallback.example.com", "not-found") if err != nil { return err } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusFound { - return fmt.Errorf("expected status 302, got %v", resp.StatusCode) + if got := resp.Header.Get("Location"); got != wantLocation { + return fmt.Errorf("expected Location %q, got %q", wantLocation, got) } - - location := resp.Header.Get("Location") - if location == "" { - return fmt.Errorf("expected Location header, got none") - } - - // The redirect should point to fallback.example.com/not-found - if !strings.Contains(location, "fallback.example.com") || !strings.Contains(location, "not-found") { - return fmt.Errorf("expected redirect to fallback.example.com/not-found, got %q", location) - } - - return nil + return checkWebsiteResponse(resp, http.StatusFound, []byte(http.StatusText(http.StatusFound))) }) } -// WebsiteHosting_routing_rule_pre_request_redirect tests that a pre-request -// routing rule (matching on key prefix only) issues a redirect before the -// object is fetched. +// WebsiteHosting_routing_rule_pre_request_redirect tests that a key-prefix +// routing rule redirects before public access or object existence is checked. func WebsiteHosting_routing_rule_pre_request_redirect(s *S3Conf) error { testName := "WebsiteHosting_routing_rule_pre_request_redirect" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - // Configure website with a pre-request routing rule - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ - Bucket: &bucket, - WebsiteConfiguration: &types.WebsiteConfiguration{ - IndexDocument: &types.IndexDocument{ - Suffix: getPtr("index.html"), - }, - RoutingRules: []types.RoutingRule{ - { - Condition: &types.Condition{ - KeyPrefixEquals: getPtr("old-docs/"), - }, - Redirect: &types.Redirect{ - ReplaceKeyPrefixWith: getPtr("new-docs/"), - HttpRedirectCode: getPtr("301"), - }, + err := putBucketWebsiteConfig(s3client, bucket, &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + RoutingRules: []types.RoutingRule{ + { + Condition: &types.Condition{ + KeyPrefixEquals: getPtr("old-docs/"), + }, + Redirect: &types.Redirect{ + ReplaceKeyPrefixWith: getPtr("new-docs/"), + HttpRedirectCode: getPtr("301"), }, }, }, }) - cancel() if err != nil { return err } - // Request old-docs/page.html via the website endpoint - resp, err := websiteGet(s.websiteEndpoint, bucket, "old-docs/page.html") + resp, err := websiteGet(s, bucket, "old-docs/page.html", nil) if err != nil { return err } defer resp.Body.Close() - if resp.StatusCode != http.StatusMovedPermanently { - return fmt.Errorf("expected status 301, got %v", resp.StatusCode) + wantLocation, err := websiteURL(s, bucket, "new-docs/page.html") + if err != nil { + return err } - - location := resp.Header.Get("Location") - if location == "" { - return fmt.Errorf("expected Location header, got none") + if got := resp.Header.Get("Location"); got != wantLocation { + return fmt.Errorf("expected Location %q, got %q", wantLocation, got) } - - // The redirect should rewrite old-docs/ -> new-docs/ - if !strings.Contains(location, "new-docs/page.html") { - return fmt.Errorf("expected redirect to contain new-docs/page.html, got %q", location) - } - - return nil + return checkWebsiteResponse(resp, http.StatusMovedPermanently, []byte(http.StatusText(http.StatusMovedPermanently))) }) } -// WebsiteHosting_redirect_all_requests tests the RedirectAllRequestsTo -// configuration, which should redirect any request to the specified host. +// WebsiteHosting_routing_rule_prefix_and_error_redirect tests a routing rule +// with both KeyPrefixEquals and HttpErrorCodeReturnedEquals conditions. +func WebsiteHosting_routing_rule_prefix_and_error_redirect(s *S3Conf) error { + testName := "WebsiteHosting_routing_rule_prefix_and_error_redirect" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putBucketWebsiteConfig(s3client, bucket, &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + ErrorDocument: &types.ErrorDocument{ + Key: getPtr("error.html"), + }, + RoutingRules: []types.RoutingRule{ + { + Condition: &types.Condition{ + KeyPrefixEquals: getPtr("old/"), + HttpErrorCodeReturnedEquals: getPtr("404"), + }, + Redirect: &types.Redirect{ + ReplaceKeyPrefixWith: getPtr("archived/"), + HttpRedirectCode: getPtr("307"), + }, + }, + }, + }) + if err != nil { + return err + } + if err := grantPublicBucketPolicy(s3client, bucket, policyTypeObject); err != nil { + return err + } + + resp, err := websiteGet(s, bucket, "old/missing.html?ref=1", nil) + if err != nil { + return err + } + defer resp.Body.Close() + + wantLocation, err := websiteURL(s, bucket, "archived/missing.html?ref=1") + if err != nil { + return err + } + if got := resp.Header.Get("Location"); got != wantLocation { + return fmt.Errorf("expected Location %q, got %q", wantLocation, got) + } + return checkWebsiteResponse(resp, http.StatusTemporaryRedirect, []byte(http.StatusText(http.StatusTemporaryRedirect))) + }) +} + +// WebsiteHosting_routing_rule_no_match_serves_error_document tests that routing +// rules which do not match fall back to the configured error document. +func WebsiteHosting_routing_rule_no_match_serves_error_document(s *S3Conf) error { + testName := "WebsiteHosting_routing_rule_no_match_serves_error_document" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putBucketWebsiteConfig(s3client, bucket, &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + ErrorDocument: &types.ErrorDocument{ + Key: getPtr("error.html"), + }, + RoutingRules: []types.RoutingRule{ + { + Condition: &types.Condition{ + KeyPrefixEquals: getPtr("docs/"), + HttpErrorCodeReturnedEquals: getPtr("404"), + }, + Redirect: &types.Redirect{ + ReplaceKeyPrefixWith: getPtr("archive/"), + }, + }, + }, + }) + if err != nil { + return err + } + if err := grantPublicBucketPolicy(s3client, bucket, policyTypeObject); err != nil { + return err + } + errorContent := "fallback error" + _, err = putObjectWithData(int64(len(errorContent)), &s3.PutObjectInput{ + Bucket: &bucket, + Key: getPtr("error.html"), + Body: strings.NewReader(errorContent), + ContentType: getPtr("text/html"), + }, s3client) + if err != nil { + return err + } + + resp, err := websiteGet(s, bucket, "images/missing.png", nil) + if err != nil { + return err + } + defer resp.Body.Close() + + return checkWebsiteResponse(resp, http.StatusNotFound, []byte(errorContent)) + }) +} + +// WebsiteHosting_redirect_all_requests tests RedirectAllRequestsTo, including +// path and query preservation, without requiring public object access. func WebsiteHosting_redirect_all_requests(s *S3Conf) error { testName := "WebsiteHosting_redirect_all_requests" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - // Configure redirect-all - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ - Bucket: &bucket, - WebsiteConfiguration: &types.WebsiteConfiguration{ - RedirectAllRequestsTo: &types.RedirectAllRequestsTo{ - HostName: getPtr("www.example.com"), - Protocol: types.ProtocolHttps, - }, + err := putBucketWebsiteConfig(s3client, bucket, &types.WebsiteConfiguration{ + RedirectAllRequestsTo: &types.RedirectAllRequestsTo{ + HostName: getPtr("www.example.com"), + Protocol: types.ProtocolHttps, }, }) - cancel() if err != nil { return err } - // Request any path via the website endpoint - resp, err := websiteGet(s.websiteEndpoint, bucket, "any/path/here") + resp, err := websiteGet(s, bucket, "any/path/here?tracking=1", nil) if err != nil { return err } defer resp.Body.Close() - if resp.StatusCode != http.StatusMovedPermanently { - return fmt.Errorf("expected status 301, got %v", resp.StatusCode) + if got, want := resp.Header.Get("Location"), "https://www.example.com/any/path/here?tracking=1"; got != want { + return fmt.Errorf("expected Location %q, got %q", want, got) } - - location := resp.Header.Get("Location") - if !strings.HasPrefix(location, "https://www.example.com/") { - return fmt.Errorf("expected redirect to https://www.example.com/, got %q", location) - } - - if !strings.Contains(location, "any/path/here") { - return fmt.Errorf("expected redirect to preserve path, got %q", location) - } - - return nil + return checkWebsiteResponse(resp, http.StatusMovedPermanently, []byte(http.StatusText(http.StatusMovedPermanently))) }) } -// WebsiteHosting_index_document tests that requesting a directory-like -// path on a website-enabled bucket serves the index document. +// WebsiteHosting_index_document tests root and directory-style index document +// resolution through the website endpoint. func WebsiteHosting_index_document(s *S3Conf) error { testName := "WebsiteHosting_index_document" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - // Configure website - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ - Bucket: &bucket, - WebsiteConfiguration: &types.WebsiteConfiguration{ - IndexDocument: &types.IndexDocument{ - Suffix: getPtr("index.html"), - }, + err := putBucketWebsiteConfig(s3client, bucket, &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), }, }) - cancel() if err != nil { return err } + if err := grantPublicBucketPolicy(s3client, bucket, policyTypeObject); err != nil { + return err + } - // Upload index document at root indexContent := "Welcome" - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ + _, err = putObjectWithData(int64(len(indexContent)), &s3.PutObjectInput{ Bucket: &bucket, Key: getPtr("index.html"), Body: strings.NewReader(indexContent), ContentType: getPtr("text/html"), - }) - cancel() + }, s3client) + if err != nil { + return err + } + docsContent := "Docs Home" + _, err = putObjectWithData(int64(len(docsContent)), &s3.PutObjectInput{ + Bucket: &bucket, + Key: getPtr("docs/index.html"), + Body: strings.NewReader(docsContent), + ContentType: getPtr("text/html"), + }, s3client) if err != nil { return err } - // Request the root path via the website endpoint - resp, err := websiteGet(s.websiteEndpoint, bucket, "/") - if err != nil { - return err - } - defer resp.Body.Close() + for _, test := range []struct { + path string + body string + }{ + {"/", indexContent}, + {"docs/", docsContent}, + } { + resp, err := websiteGet(s, bucket, test.path, nil) + if err != nil { + return err + } - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return fmt.Errorf("expected status 200, got %v; body: %s", resp.StatusCode, body) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - - if !bytes.Equal(body, []byte(indexContent)) { - return fmt.Errorf("expected index document content %q, got %q", indexContent, string(body)) + err = checkWebsiteResponse(resp, http.StatusOK, []byte(test.body)) + resp.Body.Close() + if err != nil { + return fmt.Errorf("%s: %w", test.path, err) + } } return nil }) } + +// WebsiteHosting_index_error_document_and_routing_rules covers a combined +// website configuration with index, error document, pre-rule, and post-rule. +func WebsiteHosting_index_error_document_and_routing_rules(s *S3Conf) error { + testName := "WebsiteHosting_index_error_document_and_routing_rules" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putBucketWebsiteConfig(s3client, bucket, &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + ErrorDocument: &types.ErrorDocument{ + Key: getPtr("error.html"), + }, + RoutingRules: []types.RoutingRule{ + { + Condition: &types.Condition{ + KeyPrefixEquals: getPtr("legacy/"), + }, + Redirect: &types.Redirect{ + ReplaceKeyPrefixWith: getPtr("docs/"), + HttpRedirectCode: getPtr("301"), + }, + }, + { + Condition: &types.Condition{ + HttpErrorCodeReturnedEquals: getPtr("404"), + }, + Redirect: &types.Redirect{ + HostName: getPtr("fallback.example.com"), + ReplaceKeyWith: getPtr("missing"), + HttpRedirectCode: getPtr("302"), + }, + }, + }, + }) + if err != nil { + return err + } + if err := grantPublicBucketPolicy(s3client, bucket, policyTypeObject); err != nil { + return err + } + indexContent := "combined index" + _, err = putObjectWithData(int64(len(indexContent)), &s3.PutObjectInput{ + Bucket: &bucket, + Key: getPtr("index.html"), + Body: strings.NewReader(indexContent), + ContentType: getPtr("text/html"), + }, s3client) + if err != nil { + return err + } + combinedError := "combined error" + _, err = putObjectWithData(int64(len(combinedError)), &s3.PutObjectInput{ + Bucket: &bucket, + Key: getPtr("error.html"), + Body: strings.NewReader(combinedError), + ContentType: getPtr("text/html"), + }, s3client) + if err != nil { + return err + } + + indexResp, err := websiteGet(s, bucket, "/", nil) + if err != nil { + return err + } + if err := checkWebsiteResponse(indexResp, http.StatusOK, []byte(indexContent)); err != nil { + return err + } + indexResp.Body.Close() + + preResp, err := websiteGet(s, bucket, "legacy/page.html", nil) + if err != nil { + return err + } + wantPreLocation, err := websiteURL(s, bucket, "docs/page.html") + if err != nil { + preResp.Body.Close() + return err + } + if got := preResp.Header.Get("Location"); got != wantPreLocation { + preResp.Body.Close() + return fmt.Errorf("expected pre-rule Location %q, got %q", wantPreLocation, got) + } + if err := checkWebsiteResponse(preResp, http.StatusMovedPermanently, []byte(http.StatusText(http.StatusMovedPermanently))); err != nil { + return err + } + + postResp, err := websiteGet(s, bucket, "unknown.html", nil) + if err != nil { + return err + } + wantPostLocation, err := websiteAbsoluteURL(s, "fallback.example.com", "missing") + if err != nil { + postResp.Body.Close() + return err + } + if got := postResp.Header.Get("Location"); got != wantPostLocation { + postResp.Body.Close() + return fmt.Errorf("expected post-rule Location %q, got %q", wantPostLocation, got) + } + if err := checkWebsiteResponse(postResp, http.StatusFound, []byte(http.StatusText(http.StatusFound))); err != nil { + return err + } + + return nil + }) +} + +func WebsiteHosting_options_preflight_access_granted(s *S3Conf) error { + testName := "WebsiteHosting_options_preflight_access_granted" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putBucketCors(s3client, &s3.PutBucketCorsInput{ + Bucket: &bucket, + CORSConfiguration: &types.CORSConfiguration{ + CORSRules: []types.CORSRule{ + { + AllowedOrigins: []string{"https://client.example"}, + AllowedMethods: []string{http.MethodGet, http.MethodHead}, + AllowedHeaders: []string{"Content-Type", "X-Amz-Date"}, + ExposeHeaders: []string{"Content-Length"}, + MaxAgeSeconds: getPtr(int32(42)), + }, + }, + }, + }) + if err != nil { + return err + } + + resp, err := websiteOptions(s, bucket, "index.html", map[string]string{ + "Origin": "https://client.example", + "Access-Control-Request-Method": http.MethodGet, + "Access-Control-Request-Headers": "content-type, X-Amz-Date", + }) + if err != nil { + return err + } + + corsHeaders, err := extractCORSHeaders(resp) + if err != nil { + return err + } + if err := comparePreflightResult(&PreflightResult{ + Origin: "https://client.example", + Methods: "GET, HEAD", + AllowHeaders: "content-type, x-amz-date", + ExposeHeaders: "Content-Length, ETag", + MaxAge: "42", + AllowCredentials: "true", + Vary: "Origin, Access-Control-Request-Headers, Access-Control-Request-Method", + }, corsHeaders); err != nil { + return err + } + + return checkWebsiteResponse(resp, http.StatusOK, nil) + }) +} + +func WebsiteHosting_get_cors_headers(s *S3Conf) error { + testName := "WebsiteHosting_get_cors_headers" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putBucketWebsiteConfig(s3client, bucket, &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + }) + if err != nil { + return err + } + if err := grantPublicBucketPolicy(s3client, bucket, policyTypeObject); err != nil { + return err + } + + indexContent := "CORS GET" + _, err = putObjectWithData(int64(len(indexContent)), &s3.PutObjectInput{ + Bucket: &bucket, + Key: getPtr("index.html"), + Body: strings.NewReader(indexContent), + ContentType: getPtr("text/html"), + }, s3client) + if err != nil { + return err + } + + maxAge := int32(42) + err = putBucketCors(s3client, &s3.PutBucketCorsInput{ + Bucket: &bucket, + CORSConfiguration: &types.CORSConfiguration{ + CORSRules: []types.CORSRule{ + { + AllowedOrigins: []string{"https://client.example"}, + AllowedMethods: []string{http.MethodGet, http.MethodHead}, + ExposeHeaders: []string{"Content-Length"}, + MaxAgeSeconds: &maxAge, + }, + }, + }, + }) + if err != nil { + return err + } + + resp, err := websiteGet(s, bucket, "/", map[string]string{ + "Origin": "https://client.example", + }) + if err != nil { + return err + } + + corsHeaders, err := extractCORSHeaders(resp) + if err != nil { + resp.Body.Close() + return err + } + if err := comparePreflightResult(&PreflightResult{ + Origin: "https://client.example", + Methods: "GET, HEAD", + ExposeHeaders: "Content-Length, ETag, x-amz-storage-class", + MaxAge: "42", + AllowCredentials: "true", + Vary: "Origin, Access-Control-Request-Headers, Access-Control-Request-Method", + }, corsHeaders); err != nil { + resp.Body.Close() + return err + } + + return checkWebsiteResponse(resp, http.StatusOK, []byte(indexContent)) + }) +} + +func WebsiteHosting_head_cors_headers(s *S3Conf) error { + testName := "WebsiteHosting_head_cors_headers" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putBucketWebsiteConfig(s3client, bucket, &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + }) + if err != nil { + return err + } + if err := grantPublicBucketPolicy(s3client, bucket, policyTypeObject); err != nil { + return err + } + + headContent := "CORS HEAD" + _, err = putObjectWithData(int64(len(headContent)), &s3.PutObjectInput{ + Bucket: &bucket, + Key: getPtr("head.html"), + Body: strings.NewReader(headContent), + ContentType: getPtr("text/html"), + }, s3client) + if err != nil { + return err + } + + err = putBucketCors(s3client, &s3.PutBucketCorsInput{ + Bucket: &bucket, + CORSConfiguration: &types.CORSConfiguration{ + CORSRules: []types.CORSRule{ + { + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{http.MethodHead}, + }, + }, + }, + }) + if err != nil { + return err + } + + resp, err := websiteHead(s, bucket, "head.html", map[string]string{ + "Origin": "https://client.example", + }) + if err != nil { + return err + } + + corsHeaders, err := extractCORSHeaders(resp) + if err != nil { + resp.Body.Close() + return err + } + if err := comparePreflightResult(&PreflightResult{ + Origin: "*", + Methods: "HEAD", + ExposeHeaders: "ETag, x-amz-storage-class", + AllowCredentials: "false", + Vary: "Origin, Access-Control-Request-Headers, Access-Control-Request-Method", + }, corsHeaders); err != nil { + resp.Body.Close() + return err + } + + return checkWebsiteResponse(resp, http.StatusOK, nil) + }) +} + +func WebsiteHosting_options_preflight_access_forbidden(s *S3Conf) error { + testName := "WebsiteHosting_options_preflight_access_forbidden" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putBucketCors(s3client, &s3.PutBucketCorsInput{ + Bucket: &bucket, + CORSConfiguration: &types.CORSConfiguration{ + CORSRules: []types.CORSRule{ + { + AllowedOrigins: []string{"https://client.example"}, + AllowedMethods: []string{http.MethodHead}, + }, + }, + }, + }) + if err != nil { + return err + } + + resp, err := websiteOptions(s, bucket, "index.html", map[string]string{ + "Origin": "https://client.example", + "Access-Control-Request-Method": http.MethodGet, + }) + if err != nil { + return err + } + defer resp.Body.Close() + + return checkWebsiteErrorResponse(resp, + s3err.GetAccessForbiddenErr(s3err.ErrCORSForbidden, http.MethodOptions, s3err.ResourceTypeObject)) + }) +} + +func WebsiteHosting_options_preflight_missing_origin(s *S3Conf) error { + testName := "WebsiteHosting_options_preflight_missing_origin" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + resp, err := websiteOptions(s, bucket, "index.html", map[string]string{ + "Access-Control-Request-Method": http.MethodGet, + }) + if err != nil { + return err + } + defer resp.Body.Close() + + return checkWebsiteErrorResponse(resp, s3err.GetAPIError(s3err.ErrMissingCORSOrigin)) + }) +} diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 323b20fe..b93e2e94 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -14,8 +14,6 @@ package integration -import "fmt" - func TestAuthentication(ts *TestState) { ts.Run(Authentication_invalid_auth_header) ts.Run(Authentication_unsupported_signature_version) @@ -670,7 +668,14 @@ func TestPutBucketWebsite(ts *TestState) { ts.Run(PutBucketWebsite_empty_suffix) ts.Run(PutBucketWebsite_suffix_with_slash) ts.Run(PutBucketWebsite_invalid_redirect_protocol) - ts.Run(PutBucketWebsite_redirect_and_index) + ts.Run(PutBucketWebsite_redirectAll_index_error_routingRules) + ts.Run(PutBucketWebsite_invalid_routing_rule_protocol) + ts.Run(PutBucketWebsite_empty_error_document_key) + ts.Run(PutBucketWebsite_too_many_routing_rules) + ts.Run(PutBucketWebsite_routing_rule_replace_key_and_prefix) + ts.Run(PutBucketWebsite_invalid_http_redirect_code) + ts.Run(PutBucketWebsite_invalid_http_error_code) + ts.Run(PutBucketWebsite_request_too_large) ts.Run(PutBucketWebsite_success) ts.Run(PutBucketWebsite_success_redirect_all) } @@ -688,17 +693,22 @@ func TestDeleteBucketWebsite(ts *TestState) { } func TestWebsiteHosting(ts *TestState) { - if ts.conf.websiteEndpoint == "" { - fmt.Println("skipping TestWebsiteHosting: no website endpoint configured") - return - } ts.Run(WebsiteHosting_error_document_served) ts.Run(WebsiteHosting_error_document_not_found) ts.Run(WebsiteHosting_no_error_document) + ts.Run(WebsiteHosting_private_object_and_error_document) ts.Run(WebsiteHosting_routing_rule_post_request_redirect) ts.Run(WebsiteHosting_routing_rule_pre_request_redirect) + ts.Run(WebsiteHosting_routing_rule_prefix_and_error_redirect) + ts.Run(WebsiteHosting_routing_rule_no_match_serves_error_document) ts.Run(WebsiteHosting_redirect_all_requests) ts.Run(WebsiteHosting_index_document) + ts.Run(WebsiteHosting_index_error_document_and_routing_rules) + ts.Run(WebsiteHosting_get_cors_headers) + ts.Run(WebsiteHosting_head_cors_headers) + ts.Run(WebsiteHosting_options_preflight_access_granted) + ts.Run(WebsiteHosting_options_preflight_access_forbidden) + ts.Run(WebsiteHosting_options_preflight_missing_origin) } func TestPreflightOPTIONSEndpoint(ts *TestState) { @@ -899,7 +909,6 @@ func TestFullFlow(ts *TestState) { TestPutBucketWebsite(ts) TestGetBucketWebsite(ts) TestDeleteBucketWebsite(ts) - TestWebsiteHosting(ts) TestPreflightOPTIONSEndpoint(ts) TestPutObjectLockConfiguration(ts) TestGetObjectLockConfiguration(ts) @@ -1802,7 +1811,14 @@ func GetIntTests() IntTests { "PutBucketWebsite_empty_suffix": PutBucketWebsite_empty_suffix, "PutBucketWebsite_suffix_with_slash": PutBucketWebsite_suffix_with_slash, "PutBucketWebsite_invalid_redirect_protocol": PutBucketWebsite_invalid_redirect_protocol, - "PutBucketWebsite_redirect_and_index": PutBucketWebsite_redirect_and_index, + "PutBucketWebsite_redirectAll_index_error_routingRules": PutBucketWebsite_redirectAll_index_error_routingRules, + "PutBucketWebsite_invalid_routing_rule_protocol": PutBucketWebsite_invalid_routing_rule_protocol, + "PutBucketWebsite_empty_error_document_key": PutBucketWebsite_empty_error_document_key, + "PutBucketWebsite_too_many_routing_rules": PutBucketWebsite_too_many_routing_rules, + "PutBucketWebsite_routing_rule_replace_key_and_prefix": PutBucketWebsite_routing_rule_replace_key_and_prefix, + "PutBucketWebsite_invalid_http_redirect_code": PutBucketWebsite_invalid_http_redirect_code, + "PutBucketWebsite_invalid_http_error_code": PutBucketWebsite_invalid_http_error_code, + "PutBucketWebsite_request_too_large": PutBucketWebsite_request_too_large, "PutBucketWebsite_success": PutBucketWebsite_success, "PutBucketWebsite_success_redirect_all": PutBucketWebsite_success_redirect_all, "GetBucketWebsite_non_existing_bucket": GetBucketWebsite_non_existing_bucket, @@ -1814,10 +1830,19 @@ func GetIntTests() IntTests { "WebsiteHosting_error_document_served": WebsiteHosting_error_document_served, "WebsiteHosting_error_document_not_found": WebsiteHosting_error_document_not_found, "WebsiteHosting_no_error_document": WebsiteHosting_no_error_document, + "WebsiteHosting_private_object_and_error_document": WebsiteHosting_private_object_and_error_document, "WebsiteHosting_routing_rule_post_request_redirect": WebsiteHosting_routing_rule_post_request_redirect, "WebsiteHosting_routing_rule_pre_request_redirect": WebsiteHosting_routing_rule_pre_request_redirect, + "WebsiteHosting_routing_rule_prefix_and_error_redirect": WebsiteHosting_routing_rule_prefix_and_error_redirect, + "WebsiteHosting_routing_rule_no_match_serves_error_document": WebsiteHosting_routing_rule_no_match_serves_error_document, "WebsiteHosting_redirect_all_requests": WebsiteHosting_redirect_all_requests, "WebsiteHosting_index_document": WebsiteHosting_index_document, + "WebsiteHosting_index_error_document_and_routing_rules": WebsiteHosting_index_error_document_and_routing_rules, + "WebsiteHosting_get_cors_headers": WebsiteHosting_get_cors_headers, + "WebsiteHosting_head_cors_headers": WebsiteHosting_head_cors_headers, + "WebsiteHosting_options_preflight_access_granted": WebsiteHosting_options_preflight_access_granted, + "WebsiteHosting_options_preflight_access_forbidden": WebsiteHosting_options_preflight_access_forbidden, + "WebsiteHosting_options_preflight_missing_origin": WebsiteHosting_options_preflight_missing_origin, "PreflightOPTIONS_non_existing_bucket": PreflightOPTIONS_non_existing_bucket, "PreflightOPTIONS_missing_origin": PreflightOPTIONS_missing_origin, "PreflightOPTIONS_invalid_request_method": PreflightOPTIONS_invalid_request_method, diff --git a/tests/integration/s3conf.go b/tests/integration/s3conf.go index 6382f07a..9b7ad8e0 100644 --- a/tests/integration/s3conf.go +++ b/tests/integration/s3conf.go @@ -36,7 +36,9 @@ type S3Conf struct { awsSecret string awsRegion string endpoint string - websiteEndpoint string + websiteScheme string + websiteDomain string + websitePort string hostStyle bool checksumDisable bool PartSize int64 @@ -65,6 +67,9 @@ func NewS3Conf(opts ...Option) *S3Conf { customHTTPClient := &http.Client{ Transport: customTransport, Timeout: shortTimeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, } s.httpClient = customHTTPClient @@ -86,8 +91,14 @@ func WithRegion(r string) Option { func WithEndpoint(e string) Option { return func(s *S3Conf) { s.endpoint = e } } -func WithWebsiteEndpoint(e string) Option { - return func(s *S3Conf) { s.websiteEndpoint = e } +func WithWebsiteScheme(scheme string) Option { + return func(s *S3Conf) { s.websiteScheme = scheme } +} +func WithWebsiteDomain(d string) Option { + return func(s *S3Conf) { s.websiteDomain = d } +} +func WithWebsitePort(p string) Option { + return func(s *S3Conf) { s.websitePort = p } } func WithDisableChecksum() Option { return func(s *S3Conf) { s.checksumDisable = true } diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 09a29b44..83b34c29 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -452,6 +452,147 @@ func checkHTTPResponseApiErr(resp *http.Response, expected s3err.S3Error) error return compareS3ApiError(expected, &errResp) } +// websiteGet issues a plain HTTP GET to the dedicated website endpoint. +// The bucket is resolved from the request URL host. No S3 signing is applied. +func websiteGet(s *S3Conf, bucket, path string, headers map[string]string) (*http.Response, error) { + return websiteRequest(s, http.MethodGet, bucket, path, headers) +} + +func websiteHead(s *S3Conf, bucket, path string, headers map[string]string) (*http.Response, error) { + return websiteRequest(s, http.MethodHead, bucket, path, headers) +} + +func websiteOptions(s *S3Conf, bucket, path string, headers map[string]string) (*http.Response, error) { + return websiteRequest(s, http.MethodOptions, bucket, path, headers) +} + +func websiteRequest(s *S3Conf, method, bucket, path string, headers map[string]string) (*http.Response, error) { + reqURL, err := websiteURL(s, bucket, path) + if err != nil { + return nil, err + } + req, err := http.NewRequest(method, reqURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create website request: %w", err) + } + for key, val := range headers { + req.Header.Set(key, val) + } + + return s.httpClient.Do(req) +} + +func websiteHost(s *S3Conf, bucket string) string { + _, domain, port := websiteEndpointParts(s) + + host := fmt.Sprintf("%s.%s", bucket, domain) + if port != "" { + host = fmt.Sprintf("%s:%s", host, port) + } + return host +} + +func websiteURL(s *S3Conf, bucket, path string) (string, error) { + return websiteAbsoluteURL(s, websiteHost(s, bucket), path) +} + +func websiteAbsoluteURL(s *S3Conf, host, path string) (string, error) { + scheme, _, _ := websiteEndpointParts(s) + + rel, err := url.Parse("/" + strings.TrimLeft(path, "/")) + if err != nil { + return "", fmt.Errorf("parse website request path: %w", err) + } + + return (&url.URL{ + Scheme: scheme, + Host: host, + Path: rel.Path, + RawQuery: rel.RawQuery, + }).String(), nil +} + +func websiteEndpointParts(s *S3Conf) (scheme, domain, port string) { + scheme = strings.ToLower(strings.TrimSpace(s.websiteScheme)) + domain = strings.TrimSpace(s.websiteDomain) + port = strings.TrimPrefix(strings.TrimSpace(s.websitePort), ":") + + return scheme, domain, port +} + +func putBucketWebsiteConfig(client *s3.Client, bucket string, config *types.WebsiteConfiguration) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: config, + }) + cancel() + return err +} + +func checkWebsiteResponse(resp *http.Response, expectedStatus int, expectedBody []byte) error { + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("read website response body: %w", err) + } + + if resp.StatusCode != expectedStatus { + return fmt.Errorf("expected status %v, got %v; body: %s", expectedStatus, resp.StatusCode, body) + } + + return compareBodySHA256(expectedBody, body) +} + +func checkWebsiteErrorResponse(resp *http.Response, expected s3err.S3Error) error { + apiErr := expected.BaseError() + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("read website error body: %w", err) + } + + if resp.StatusCode != apiErr.HTTPStatusCode { + return fmt.Errorf("expected status %v, got %v; body: %s", apiErr.HTTPStatusCode, resp.StatusCode, body) + } + if got := resp.Header.Get("x-amz-error-code"); got != apiErr.Code { + return fmt.Errorf("expected x-amz-error-code %q, got %q", apiErr.Code, got) + } + if got := resp.Header.Get("x-amz-error-message"); got != apiErr.Description { + return fmt.Errorf("expected x-amz-error-message %q, got %q", apiErr.Description, got) + } + requestID := resp.Header.Get("x-amz-request-id") + if requestID == "" { + return fmt.Errorf("expected x-amz-request-id header") + } + hostID := resp.Header.Get("x-amz-id-2") + if hostID == "" { + return fmt.Errorf("expected x-amz-id-2 header") + } + if got := resp.Header.Get("Content-Type"); !strings.Contains(got, "text/html") { + return fmt.Errorf("expected html Content-Type, got %q", got) + } + if methodErr, ok := expected.(s3err.MethodNotAllowedError); ok && len(methodErr.AllowedMethods) != 0 { + if got, want := resp.Header.Get("Allow"), methodErr.AllowedMethodsString(); got != want { + return fmt.Errorf("expected Allow header %q, got %q", want, got) + } + } + + expectedBody := expected.HTMLBody(requestID, hostID) + return compareBodySHA256(expectedBody, body) +} + +func compareBodySHA256(expected, actual []byte) error { + expectedSum := sha256.Sum256(expected) + actualSum := sha256.Sum256(actual) + if expectedSum != actualSum { + return fmt.Errorf("body checksum mismatch: expected sha256 %x, got %x; expected body %q, got %q", + expectedSum, actualSum, string(expected), string(actual)) + } + + return nil +} + func compareS3ApiError(expected s3err.S3Error, received *APIErrorResponse) error { apiErr := expected.BaseError() if received == nil { diff --git a/tests/test_rest_not_implemented.sh b/tests/test_rest_not_implemented.sh index 2604add5..3366bc44 100755 --- a/tests/test_rest_not_implemented.sh +++ b/tests/test_rest_not_implemented.sh @@ -162,21 +162,6 @@ source ./tests/setup.sh assert_success } -@test "REST - GetBucketWebsite" { - run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "website=" "GET" - assert_success -} - -@test "REST - PutBucketWebsite" { - run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "website=" "PUT" - assert_success -} - -@test "REST - DeleteBucketWebsite" { - run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "website=" "DELETE" - assert_success -} - @test "REST - GetPublicAccessBlock" { run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "publicAccessBlock=" "GET" assert_success diff --git a/tests/website-hosting-tests/dnsmasq.conf b/tests/website-hosting-tests/dnsmasq.conf new file mode 100644 index 00000000..5d7054b9 --- /dev/null +++ b/tests/website-hosting-tests/dnsmasq.conf @@ -0,0 +1,2 @@ +address=/.dev/10.89.0.10 +no-resolv diff --git a/tests/website-hosting-tests/docker-compose.yml b/tests/website-hosting-tests/docker-compose.yml new file mode 100644 index 00000000..637cb207 --- /dev/null +++ b/tests/website-hosting-tests/docker-compose.yml @@ -0,0 +1,58 @@ +services: + dnsmasq: + image: strm/dnsmasq + container_name: website-dns-resolver + restart: on-failure + volumes: + - './dnsmasq.conf:/etc/dnsmasq.conf' + cap_add: + - NET_ADMIN + healthcheck: + test: 'if [ -z "$(netstat -nltu |grep \:53)" ]; then exit 1;else exit 0;fi' + interval: 2s + timeout: 2s + retries: 20 + networks: + devnet: + ipv4_address: 10.89.0.53 + + server: + build: + context: ../.. + dockerfile: tests/host-style-tests/Dockerfile + depends_on: + dnsmasq: + condition: service_healthy + command: ["-a", "user", "-s", "pass", "--health", "/health", "--iam-dir", "/tmp/vgw", "--website", ":8080", "--website-domain", "dev", "--website-no-tls", "posix", "/tmp/vgw"] + dns: + - 10.89.0.53 + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:7070/health"] + interval: 2s + timeout: 2s + retries: 20 + networks: + devnet: + ipv4_address: 10.89.0.10 + + test: + build: + context: ../.. + dockerfile: tests/host-style-tests/Dockerfile + depends_on: + server: + condition: service_healthy + dnsmasq: + condition: service_healthy + command: ["test", "-a", "user", "-s", "pass", "-e", "http://10.89.0.10:7070", "website-hosting", "--scheme", "http", "--domain", "dev", "--port", "8080"] + dns: + - 10.89.0.53 + networks: + devnet: + +networks: + devnet: + driver: bridge + ipam: + config: + - subnet: 10.89.0.0/16 diff --git a/website/handler.go b/website/handler.go index 463ceec7..506a6f2f 100644 --- a/website/handler.go +++ b/website/handler.go @@ -15,9 +15,8 @@ package website import ( - "encoding/xml" + "errors" "fmt" - "html" "io" "net/http" "strconv" @@ -25,11 +24,25 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/debuglogger" + "github.com/versity/versitygw/s3api/middlewares" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3response" ) -// newHandler returns a fiber handler that serves static website content. +var websiteAllowedMethods = []string{fiber.MethodGet, fiber.MethodHead, fiber.MethodOptions} + +type websiteController struct { + be backend.Backend + domain string + domainSuffix string + applyCORS fiber.Handler +} + +// newWebsiteController returns a controller that serves static website content. // It resolves the bucket name from the Host header using the configured domain, // fetches the website configuration, and serves objects accordingly. // @@ -40,95 +53,202 @@ import ( // Catch-all mode (--website-domain omitted or empty): // - Host "blog.example.com" -> bucket "blog.example.com" // - Host "mysite.org" -> bucket "mysite.org" -func newHandler(be backend.Backend, domain string) fiber.Handler { - // Pre-compute the domain suffix for subdomain extraction. - // Given domain "example.com", we look for ".example.com" suffix. - domainSuffix := "." + domain +func newWebsiteController(be backend.Backend, domain string) *websiteController { + controller := &websiteController{ + be: be, + domain: domain, + domainSuffix: "." + domain, + } + controller.applyCORS = middlewares.ApplyBucketCORS(be, controller.resolveBucket, "") + return controller +} - return func(ctx *fiber.Ctx) error { - host := ctx.Hostname() - if host == "" { - return sendError(ctx, http.StatusBadRequest, "Bad Request", "Missing Host header") +func (c *websiteController) Get(ctx *fiber.Ctx) error { + return c.serve(ctx, c.getObject) +} + +func (c *websiteController) Head(ctx *fiber.Ctx) error { + return c.serve(ctx, c.headObject) +} + +func (c *websiteController) Options(ctx *fiber.Ctx) error { + bucket, err := c.resolveBucket(ctx) + if err != nil { + return sendError(ctx, err) + } + + origin := ctx.Get("Origin") + method := auth.CORSHTTPMethod(ctx.Get("Access-Control-Request-Method")) + headers := ctx.Get("Access-Control-Request-Headers") + + if origin == "" { + debuglogger.Logf("origin is missing: %v", origin) + return sendError(ctx, s3err.GetAPIError(s3err.ErrMissingCORSOrigin)) + } + + if !method.IsValid() { + debuglogger.Logf("invalid cors method: %s", method) + return sendError(ctx, s3err.GetInvalidCORSMethodErr(method.String())) + } + + parsedHeaders, err := auth.ParseCORSHeaders(headers) + if err != nil { + return sendError(ctx, err) + } + + cors, err := c.be.GetBucketCors(ctx.Context(), bucket) + if err != nil { + debuglogger.Logf("failed to get bucket cors: %v", err) + if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchCORSConfiguration)) { + err = s3err.GetAccessForbiddenErr(s3err.ErrCORSIsNotEnabled, http.MethodOptions, s3err.ResourceTypeBucket) + debuglogger.Logf("bucket cors is not set: %v", err) } + return sendError(ctx, err) + } - // Strip port from host if present - if idx := strings.LastIndex(host, ":"); idx != -1 { - // Be careful with IPv6: only strip if it's not inside brackets - if !strings.Contains(host[idx:], "]") { - host = host[:idx] - } - } + corsConfig, err := auth.ParseCORSOutput(cors) + if err != nil { + return sendError(ctx, err) + } - // Resolve bucket name from host - bucket := resolveBucket(host, domain, domainSuffix) - if bucket == "" { - return sendError(ctx, http.StatusForbidden, "Forbidden", - fmt.Sprintf("No bucket could be resolved from host %q", html.EscapeString(ctx.Hostname()))) - } + allowConfig, err := corsConfig.IsAllowed(origin, method, parsedHeaders, s3err.ResourceTypeObject) + if err != nil { + debuglogger.Logf("cors access forbidden: %v", err) + return sendError(ctx, err) + } - // Fetch website configuration - data, err := be.GetBucketWebsite(ctx.Context(), bucket) - if err != nil { - return sendError(ctx, http.StatusNotFound, "Not Found", - fmt.Sprintf("No website configuration for bucket %q", bucket)) - } + setCORSPreflightHeaders(ctx, allowConfig) + ctx.Status(http.StatusOK) + return nil +} - var config s3response.WebsiteConfiguration - if xmlErr := xml.Unmarshal(data, &config); xmlErr != nil { - return sendError(ctx, http.StatusInternalServerError, "Internal Server Error", - "Invalid website configuration") - } +func registerWebsiteRoutes(app *fiber.App, be backend.Backend, domain string) { + controller := newWebsiteController(be, domain) - key := strings.TrimPrefix(ctx.Path(), "/") + app.Head("*", controller.Head) + app.Get("*", controller.Get) + app.Options("*", controller.Options) + app.All("*", controller.MethodNotAllowed) +} - // Handle RedirectAllRequestsTo - if config.RedirectAllRequestsTo != nil { - return handleRedirectAll(ctx, config.RedirectAllRequestsTo, key) - } - - // Evaluate pre-request routing rules - if rule := config.MatchPreRequestRule(key); rule != nil { - return applyRedirect(ctx, &rule.Redirect, rule.Condition, key) - } - - // Rewrite directory-like keys to include index document suffix - if config.IndexDocument != nil && config.IndexDocument.Suffix != "" { - if key == "" || strings.HasSuffix(key, "/") { - key = key + config.IndexDocument.Suffix - } - } - - // Fetch the object - emptyRange := "" - result, getErr := be.GetObject(ctx.Context(), &s3.GetObjectInput{ - Bucket: &bucket, - Key: &key, - Range: &emptyRange, - }) - if getErr == nil && result.Body != nil { - defer result.Body.Close() - return serveObject(ctx, result, key) - } - - // Object not found (or other error) — evaluate post-request routing rules - httpErrCode := http.StatusNotFound - errorCode := strconv.Itoa(httpErrCode) - - if rule := config.MatchPostRequestRule(key, errorCode); rule != nil { - return applyRedirect(ctx, &rule.Redirect, rule.Condition, key) - } - - // Serve error document if configured - if config.ErrorDocument != nil && config.ErrorDocument.Key != "" { - return serveErrorDocument(ctx, be, bucket, config.ErrorDocument.Key, httpErrCode) - } - - return sendError(ctx, http.StatusNotFound, "Not Found", - fmt.Sprintf("The specified key %q does not exist", key)) +func setCORSPreflightHeaders(ctx *fiber.Ctx, allowConfig *auth.CORSAllowanceConfig) { + ctx.Set("Access-Control-Allow-Origin", allowConfig.Origin) + ctx.Set("Access-Control-Allow-Methods", allowConfig.Methods) + ctx.Set("Access-Control-Expose-Headers", corsExposeHeaders(allowConfig.ExposedHeaders)) + ctx.Set("Access-Control-Allow-Credentials", allowConfig.AllowCredentials) + ctx.Set("Access-Control-Allow-Headers", allowConfig.AllowHeaders) + ctx.Set("Vary", middlewares.VaryHdr) + if allowConfig.MaxAge != nil { + ctx.Set("Access-Control-Max-Age", strconv.Itoa(int(*allowConfig.MaxAge))) } } -// resolveBucket extracts the bucket name from the host header. +func corsExposeHeaders(exposed string) string { + exposed = strings.TrimSpace(exposed) + if exposed == "" { + return "ETag" + } + if exposed == "*" { + return exposed + } + + for part := range strings.SplitSeq(exposed, ",") { + if strings.EqualFold(strings.TrimSpace(part), "ETag") { + return exposed + } + } + + return exposed + ", ETag" +} + +func (c *websiteController) MethodNotAllowed(ctx *fiber.Ctx) error { + return sendError(ctx, s3err.GetMethodNotAllowedErr(ctx.Method(), s3err.ResourceTypeObject, websiteAllowedMethods)) +} + +type websiteRequestInfo struct { + bucket string + config *s3response.WebsiteConfiguration + key string +} + +type websiteObjectReader func(ctx *fiber.Ctx, bucket, key string) websiteResult + +func (c *websiteController) serve(ctx *fiber.Ctx, readObject websiteObjectReader) error { + req, err := c.resolveRequest(ctx) + if err != nil { + return sendError(ctx, err) + } + + if err := c.applyCORS(ctx); err != nil { + return sendError(ctx, err) + } + + if req.config.RedirectAllRequestsTo != nil { + return handleRedirectAll(ctx, req.config.RedirectAllRequestsTo, req.key) + } + + if rule := req.config.MatchPrefetchRoutingRule(req.key); rule != nil { + return applyRedirect(ctx, &rule.Redirect, rule.Condition, req.key) + } + + resolvedKey := resolveIndexKey(req.key, req.config) + result := readObject(ctx, req.bucket, resolvedKey) + if result.Err == nil { + return serveWebsiteResult(ctx, req.bucket, req.config, result, readObject) + } + if result.StatusCode >= http.StatusInternalServerError { + return sendError(ctx, result.Err) + } + + if rule := req.config.MatchPostErrorRoutingRule(req.key, result.StatusCode); rule != nil { + return applyRedirect(ctx, &rule.Redirect, rule.Condition, req.key) + } + + return serveWebsiteResult(ctx, req.bucket, req.config, result, readObject) +} + +func (c *websiteController) resolveRequest(ctx *fiber.Ctx) (*websiteRequestInfo, error) { + bucket, err := c.resolveBucket(ctx) + if err != nil { + return nil, err + } + + key := strings.TrimPrefix(ctx.Path(), "/") + if err := validateWebsiteNames(bucket, key); err != nil { + return nil, err + } + + data, err := c.be.GetBucketWebsite(ctx.Context(), bucket) + if err != nil { + return nil, err + } + + config, err := s3response.ParseWebsiteConfigOutput(data) + if err != nil { + return nil, err + } + + return &websiteRequestInfo{ + bucket: bucket, + config: config, + key: key, + }, nil +} + +func validateWebsiteNames(bucket, key string) error { + if !utils.IsValidBucketName(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + if key != "" && !utils.IsObjectNameValid(key) { + return s3err.GetAPIError(s3err.ErrBadRequest) + } + + return nil +} + +// resolveBucket extracts the bucket name from the request host header. +// +// It strips the port when present before applying website endpoint routing. // // When domain is set: // - If host equals the domain exactly, the bucket IS the domain (apex). @@ -137,36 +257,146 @@ func newHandler(be backend.Backend, domain string) fiber.Handler { // // When domain is empty (catch-all mode): // - The full hostname is used as the bucket name. -func resolveBucket(host, domain, domainSuffix string) string { - if domain == "" { - // Catch-all: the full hostname is the bucket name - return host +func (c *websiteController) resolveBucket(ctx *fiber.Ctx) (string, error) { + host := ctx.Hostname() + if host == "" { + return "", s3err.GetAPIError(s3err.ErrNoBucketInRequest) } - if strings.EqualFold(host, domain) { - return domain + // Strip port from host if present. Be careful with IPv6: only strip if the + // last colon is not inside brackets. + if idx := strings.LastIndex(host, ":"); idx != -1 && !strings.Contains(host[idx:], "]") { + host = host[:idx] } - lower := strings.ToLower(host) - if strings.HasSuffix(lower, strings.ToLower(domainSuffix)) { - sub := host[:len(host)-len(domainSuffix)] - if sub != "" && !strings.Contains(sub, ".") { - return sub + if c.domain == "" { + return host, nil + } + + if strings.EqualFold(host, c.domain) { + return c.domain, nil + } + + lowerHost := strings.ToLower(host) + lowerDomainSuffix := strings.ToLower(c.domainSuffix) + if strings.HasSuffix(lowerHost, lowerDomainSuffix) { + bucket := host[:len(host)-len(c.domainSuffix)] + if bucket != "" && !strings.Contains(bucket, ".") { + return bucket, nil } } - return "" + return "", s3err.GetAPIError(s3err.ErrNoBucketInRequest) +} + +type websiteResult struct { + Key string + StatusCode int + Object websiteObject + Err error +} + +type websiteObject struct { + Body io.ReadCloser + Headers map[string]*string + Metadata map[string]string +} + +func resolveIndexKey(key string, config *s3response.WebsiteConfiguration) string { + if config.IndexDocument != nil && config.IndexDocument.Suffix != "" { + if key == "" || strings.HasSuffix(key, "/") { + return key + config.IndexDocument.Suffix + } + } + + return key +} + +func (c *websiteController) getObject(ctx *fiber.Ctx, bucket, key string) websiteResult { + if err := auth.VerifyPublicAccess(ctx.Context(), c.be, auth.GetObjectAction, auth.PermissionRead, bucket, key); err != nil { + return websiteResult{ + Key: key, + StatusCode: statusCodeFromError(err), + Err: err, + } + } + + result, err := c.be.GetObject(ctx.Context(), &s3.GetObjectInput{ + Bucket: &bucket, + Key: &key, + }) + if err != nil { + return websiteResult{ + Key: key, + StatusCode: statusCodeFromError(err), + Err: err, + } + } + + return websiteResult{ + Key: key, + StatusCode: http.StatusOK, + Object: websiteObject{ + Body: result.Body, + Headers: getObjectHeaders(result), + Metadata: result.Metadata, + }, + } +} + +func (c *websiteController) headObject(ctx *fiber.Ctx, bucket, key string) websiteResult { + if err := auth.VerifyPublicAccess(ctx.Context(), c.be, auth.ListBucketAction, auth.PermissionRead, bucket, key); err != nil { + return websiteResult{ + Key: key, + StatusCode: statusCodeFromError(err), + Err: err, + } + } + + result, err := c.be.HeadObject(ctx.Context(), &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &key, + }) + if err != nil { + return websiteResult{ + Key: key, + StatusCode: statusCodeFromError(err), + Err: err, + } + } + + return websiteResult{ + Key: key, + StatusCode: http.StatusOK, + Object: websiteObject{ + Headers: headObjectHeaders(result), + Metadata: result.Metadata, + }, + } +} + +func statusCodeFromError(err error) int { + var serr s3err.S3Error + if errors.As(err, &serr) { + return serr.StatusCode() + } + + return http.StatusInternalServerError } // handleRedirectAll sends a 301 redirect for RedirectAllRequestsTo configuration. func handleRedirectAll(ctx *fiber.Ctx, redirect *s3response.RedirectAllRequestsTo, key string) error { protocol := redirect.Protocol if protocol == "" { - protocol = "https" + protocol = "http" } location := fmt.Sprintf("%s://%s/%s", protocol, redirect.HostName, key) + if query := string(ctx.Request().URI().QueryString()); query != "" { + location += "?" + query + } ctx.Set("Location", location) + _, _ = utils.EnsureRequestIDs(ctx) return ctx.SendStatus(http.StatusMovedPermanently) } @@ -189,7 +419,7 @@ func applyRedirect(ctx *fiber.Ctx, redirect *s3response.Redirect, condition *s3r key = redirect.ReplaceKeyPrefixWith + strings.TrimPrefix(originalKey, condition.KeyPrefixEquals) } - httpCode := http.StatusFound // 302 default + httpCode := http.StatusMovedPermanently if redirect.HttpRedirectCode != "" { if code, err := strconv.Atoi(redirect.HttpRedirectCode); err == nil { httpCode = code @@ -197,118 +427,112 @@ func applyRedirect(ctx *fiber.Ctx, redirect *s3response.Redirect, condition *s3r } location := fmt.Sprintf("%s://%s/%s", protocol, host, key) + if query := string(ctx.Request().URI().QueryString()); query != "" { + location += "?" + query + } ctx.Set("Location", location) return ctx.SendStatus(httpCode) } -// serveObject writes the S3 object content to the response. -func serveObject(ctx *fiber.Ctx, result *s3.GetObjectOutput, key string) error { - contentType := guessContentType(result, key) - ctx.Set("Content-Type", contentType) +func getObjectHeaders(result *s3.GetObjectOutput) map[string]*string { + return map[string]*string{ + "ETag": result.ETag, + "accept-ranges": result.AcceptRanges, + "Cache-Control": result.CacheControl, + "Content-Disposition": result.ContentDisposition, + "Content-Encoding": result.ContentEncoding, + "Content-Language": result.ContentLanguage, + "Content-Length": utils.ConvertPtrToStringPtr(result.ContentLength), + "Content-Range": result.ContentRange, + "Content-Type": result.ContentType, + "Expires": result.ExpiresString, + "Last-Modified": utils.FormatDatePtrToString(result.LastModified, http.TimeFormat), + "x-amz-restore": result.Restore, + "x-amz-version-id": result.VersionId, + } +} - if result.ETag != nil { - ctx.Set("ETag", *result.ETag) +func headObjectHeaders(result *s3.HeadObjectOutput) map[string]*string { + return map[string]*string{ + "ETag": result.ETag, + "accept-ranges": result.AcceptRanges, + "Cache-Control": result.CacheControl, + "Content-Disposition": result.ContentDisposition, + "Content-Encoding": result.ContentEncoding, + "Content-Language": result.ContentLanguage, + "Content-Length": utils.ConvertPtrToStringPtr(result.ContentLength), + "Content-Range": result.ContentRange, + "Content-Type": result.ContentType, + "Expires": result.ExpiresString, + "Last-Modified": utils.FormatDatePtrToString(result.LastModified, http.TimeFormat), + "x-amz-restore": result.Restore, + "x-amz-version-id": result.VersionId, } - if result.CacheControl != nil { - ctx.Set("Cache-Control", *result.CacheControl) - } - if result.ContentEncoding != nil { - ctx.Set("Content-Encoding", *result.ContentEncoding) - } - if result.ContentLanguage != nil { - ctx.Set("Content-Language", *result.ContentLanguage) - } - if result.ContentLength != nil { - ctx.Set("Content-Length", strconv.FormatInt(*result.ContentLength, 10)) - } - if result.LastModified != nil { - ctx.Set("Last-Modified", result.LastModified.UTC().Format(http.TimeFormat)) +} + +func serveWebsiteResult(ctx *fiber.Ctx, bucket string, config *s3response.WebsiteConfiguration, result websiteResult, readObject websiteObjectReader) error { + if result.Err == nil { + return serveObject(ctx, result.Object, http.StatusOK) } - _, err := io.Copy(ctx.Response().BodyWriter(), result.Body) + if config.ErrorDocument != nil && config.ErrorDocument.Key != "" { + return serveErrorDocument(ctx, readObject, bucket, config.ErrorDocument.Key, result.StatusCode) + } + + return sendError(ctx, result.Err) +} + +func serveObject(ctx *fiber.Ctx, object websiteObject, statusCode int) error { + ctx.Status(statusCode) + setWebsiteObjectHeaders(ctx, object) + + if object.Body == nil { + return nil + } + defer object.Body.Close() + + _, err := io.Copy(ctx.Response().BodyWriter(), object.Body) if err != nil { - return sendError(ctx, http.StatusInternalServerError, "Internal Server Error", - "Failed to read object") + return sendError(ctx, err) } return nil } +func setWebsiteObjectHeaders(ctx *fiber.Ctx, object websiteObject) { + utils.SetMetaHeaders(ctx, object.Metadata) + for key, value := range object.Headers { + if value != nil && *value != "" { + ctx.Set(key, *value) + } + } +} + // serveErrorDocument fetches and serves the configured error document. -func serveErrorDocument(ctx *fiber.Ctx, be backend.Backend, bucket, errorDocKey string, statusCode int) error { - emptyRange := "" - result, err := be.GetObject(ctx.Context(), &s3.GetObjectInput{ - Bucket: &bucket, - Key: &errorDocKey, - Range: &emptyRange, - }) - if err != nil { - return sendError(ctx, statusCode, "Not Found", "The specified key does not exist") - } - if result.Body == nil { - return sendError(ctx, statusCode, "Not Found", "The specified key does not exist") - } - defer result.Body.Close() - - contentType := guessContentType(result, errorDocKey) - ctx.Set("Content-Type", contentType) - - ctx.Status(statusCode) - _, writeErr := io.Copy(ctx.Response().BodyWriter(), result.Body) - if writeErr != nil { - return sendError(ctx, statusCode, "Not Found", "The specified key does not exist") +func serveErrorDocument(ctx *fiber.Ctx, readObject websiteObjectReader, bucket, errorDocKey string, statusCode int) error { + result := readObject(ctx, bucket, errorDocKey) + if result.Err != nil { + return sendError(ctx, result.Err) } - return nil -} - -// guessContentType returns the content type from the GetObject result, or -// infers it from the key extension, defaulting to text/html. -func guessContentType(result *s3.GetObjectOutput, key string) string { - if result.ContentType != nil && *result.ContentType != "" { - return *result.ContentType - } - - // Simple extension-based inference for common web types - switch { - case strings.HasSuffix(key, ".html"), strings.HasSuffix(key, ".htm"): - return "text/html; charset=utf-8" - case strings.HasSuffix(key, ".css"): - return "text/css; charset=utf-8" - case strings.HasSuffix(key, ".js"): - return "application/javascript" - case strings.HasSuffix(key, ".json"): - return "application/json" - case strings.HasSuffix(key, ".xml"): - return "application/xml" - case strings.HasSuffix(key, ".svg"): - return "image/svg+xml" - case strings.HasSuffix(key, ".png"): - return "image/png" - case strings.HasSuffix(key, ".jpg"), strings.HasSuffix(key, ".jpeg"): - return "image/jpeg" - case strings.HasSuffix(key, ".gif"): - return "image/gif" - case strings.HasSuffix(key, ".ico"): - return "image/x-icon" - case strings.HasSuffix(key, ".txt"): - return "text/plain; charset=utf-8" - default: - return "text/html; charset=utf-8" - } + return serveObject(ctx, result.Object, statusCode) } // sendError sends a simple HTML error page. -func sendError(ctx *fiber.Ctx, statusCode int, title, message string) error { - ctx.Set("Content-Type", "text/html; charset=utf-8") - ctx.Status(statusCode) - body := fmt.Sprintf(` - -%d %s - -

    %d %s

    -

    %s

    - -`, statusCode, title, statusCode, title, message) - return ctx.SendString(body) +func sendError(ctx *fiber.Ctx, err error) error { + requestId, hostId := utils.EnsureRequestIDs(ctx) + serr, ok := err.(s3err.S3Error) + if !ok { + debuglogger.InternalError(err) + serr = s3err.GetAPIError(s3err.ErrInternalError) + } + + ctx.Response().Header.Set("x-amz-error-code", serr.BaseError().Code) + ctx.Response().Header.Set("x-amz-error-message", serr.BaseError().Description) + if methodErr, ok := serr.(s3err.MethodNotAllowedError); ok && len(methodErr.AllowedMethods) != 0 { + ctx.Response().Header.Set("Allow", methodErr.AllowedMethodsString()) + } + + ctx.Response().Header.SetContentType(fiber.MIMETextHTMLCharsetUTF8) + return ctx.Status(serr.StatusCode()).Send(serr.HTMLBody(requestId, hostId)) } diff --git a/website/handler_test.go b/website/handler_test.go new file mode 100644 index 00000000..486de5cc --- /dev/null +++ b/website/handler_test.go @@ -0,0 +1,972 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package website + +import ( + "context" + "encoding/json" + "encoding/xml" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +type websiteTestBackend struct { + backend.BackendUnsupported + + websiteConfig []byte + corsConfig []byte + corsErr error + objects map[string]string + objectErrors map[string]error + public bool + calls []string +} + +func (b *websiteTestBackend) record(call string) { + b.calls = append(b.calls, call) +} + +func (b *websiteTestBackend) GetBucketWebsite(_ context.Context, _ string) ([]byte, error) { + b.record("GetBucketWebsite") + return b.websiteConfig, nil +} + +func (b *websiteTestBackend) GetBucketCors(_ context.Context, _ string) ([]byte, error) { + b.record("GetBucketCors") + if b.corsErr != nil { + return nil, b.corsErr + } + if b.corsConfig == nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchCORSConfiguration) + } + return b.corsConfig, nil +} + +func (b *websiteTestBackend) GetBucketPolicy(_ context.Context, _ string) ([]byte, error) { + b.record("GetBucketPolicy") + return nil, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy) +} + +func (b *websiteTestBackend) GetBucketAcl(_ context.Context, _ *s3.GetBucketAclInput) ([]byte, error) { + b.record("GetBucketAcl") + acl := auth.ACL{Owner: "owner"} + if b.public { + acl.Grantees = []auth.Grantee{ + { + Permission: auth.PermissionRead, + Access: "all-users", + Type: types.TypeGroup, + }, + } + } + + data, err := json.Marshal(acl) + if err != nil { + return nil, err + } + return data, nil +} + +func (b *websiteTestBackend) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { + b.record("HeadObject") + if input == nil || input.Key == nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err, ok := b.objectErrors[*input.Key]; ok { + return nil, err + } + body, ok := b.objects[*input.Key] + if !ok { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + + length := int64(len(body)) + contentType := "text/html" + return &s3.HeadObjectOutput{ + ContentLength: &length, + ContentType: &contentType, + }, nil +} + +func (b *websiteTestBackend) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { + b.record("GetObject") + if input == nil || input.Key == nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err, ok := b.objectErrors[*input.Key]; ok { + return nil, err + } + body, ok := b.objects[*input.Key] + if !ok { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + + length := int64(len(body)) + contentType := "text/html" + return &s3.GetObjectOutput{ + Body: io.NopCloser(strings.NewReader(body)), + ContentLength: &length, + ContentType: &contentType, + }, nil +} + +func TestWebsiteHandlerRoutingRuleOrder(t *testing.T) { + tests := []struct { + name string + rules []s3response.RoutingRule + wantStatus int + wantLocation string + }{ + { + name: "key prefix rule before 404 rule wins", + rules: []s3response.RoutingRule{ + { + Condition: &s3response.RoutingRuleCondition{ + KeyPrefixEquals: "old/", + }, + Redirect: s3response.Redirect{ + ReplaceKeyPrefixWith: "new/", + HttpRedirectCode: "301", + }, + }, + { + Condition: &s3response.RoutingRuleCondition{ + HttpErrorCodeReturnedEquals: "404", + }, + Redirect: s3response.Redirect{ + ReplaceKeyWith: "error.html", + HttpRedirectCode: "302", + }, + }, + }, + wantStatus: http.StatusMovedPermanently, + wantLocation: "http://site.test/new/missing.html", + }, + { + name: "key prefix rule wins pre-fetch even when 404 rule comes first", + rules: []s3response.RoutingRule{ + { + Condition: &s3response.RoutingRuleCondition{ + HttpErrorCodeReturnedEquals: "404", + }, + Redirect: s3response.Redirect{ + ReplaceKeyWith: "error.html", + HttpRedirectCode: "302", + }, + }, + { + Condition: &s3response.RoutingRuleCondition{ + KeyPrefixEquals: "old/", + }, + Redirect: s3response.Redirect{ + ReplaceKeyPrefixWith: "new/", + HttpRedirectCode: "301", + }, + }, + }, + wantStatus: http.StatusMovedPermanently, + wantLocation: "http://site.test/new/missing.html", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + RoutingRules: tt.rules, + }, nil, true) + + resp := websiteRequest(t, be, "/old/missing.html") + defer resp.Body.Close() + + if resp.StatusCode != tt.wantStatus { + t.Fatalf("status = %d, want %d", resp.StatusCode, tt.wantStatus) + } + if got := resp.Header.Get("Location"); got != tt.wantLocation { + t.Fatalf("Location = %q, want %q", got, tt.wantLocation) + } + if containsCall(be.calls, "GetObject") { + t.Fatal("GetObject was called for a redirect response") + } + }) + } +} + +func TestWebsiteHandlerRoutingRuleBothConditions(t *testing.T) { + config := s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + RoutingRules: []s3response.RoutingRule{ + { + Condition: &s3response.RoutingRuleCondition{ + KeyPrefixEquals: "old/", + HttpErrorCodeReturnedEquals: "404", + }, + Redirect: s3response.Redirect{ + ReplaceKeyPrefixWith: "new/", + HttpRedirectCode: "302", + }, + }, + }, + } + + t.Run("missing object with matching prefix redirects", func(t *testing.T) { + be := newWebsiteTestBackend(t, config, nil, true) + resp := websiteRequest(t, be, "/old/missing.html") + defer resp.Body.Close() + + if resp.StatusCode != http.StatusFound { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusFound) + } + if got := resp.Header.Get("Location"); got != "http://site.test/new/missing.html" { + t.Fatalf("Location = %q", got) + } + }) + + t.Run("existing object with matching prefix does not redirect", func(t *testing.T) { + be := newWebsiteTestBackend(t, config, map[string]string{ + "old/existing.html": "served", + }, true) + resp := websiteRequest(t, be, "/old/existing.html") + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + if got := readBody(t, resp); got != "served" { + t.Fatalf("body = %q, want %q", got, "served") + } + if got := resp.Header.Get("Location"); got != "" { + t.Fatalf("unexpected Location header %q", got) + } + }) + + t.Run("missing object with wrong prefix does not redirect", func(t *testing.T) { + be := newWebsiteTestBackend(t, config, nil, true) + resp := websiteRequest(t, be, "/other/missing.html") + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusNotFound) + } + if got := resp.Header.Get("Location"); got != "" { + t.Fatalf("unexpected Location header %q", got) + } + }) +} + +func TestWebsiteHandlerRedirectConstruction(t *testing.T) { + tests := []struct { + name string + rule s3response.RoutingRule + path string + wantLocation string + }{ + { + name: "ReplaceKeyWith replaces full key", + rule: s3response.RoutingRule{ + Condition: &s3response.RoutingRuleCondition{ + HttpErrorCodeReturnedEquals: "404", + }, + Redirect: s3response.Redirect{ + ReplaceKeyWith: "error.html", + }, + }, + path: "/a/b/c.html", + wantLocation: "http://site.test/error.html", + }, + { + name: "ReplaceKeyPrefixWith replaces matching prefix", + rule: s3response.RoutingRule{ + Condition: &s3response.RoutingRuleCondition{ + KeyPrefixEquals: "old/", + }, + Redirect: s3response.Redirect{ + ReplaceKeyPrefixWith: "new/", + }, + }, + path: "/old/a/b.html", + wantLocation: "http://site.test/new/a/b.html", + }, + { + name: "HostName Protocol and query string are preserved", + rule: s3response.RoutingRule{ + Condition: &s3response.RoutingRuleCondition{ + KeyPrefixEquals: "old/", + }, + Redirect: s3response.Redirect{ + HostName: "example.com", + Protocol: "https", + ReplaceKeyPrefixWith: "new/", + }, + }, + path: "/old/page.html?x=1&y=2", + wantLocation: "https://example.com/new/page.html?x=1&y=2", + }, + { + name: "query string is preserved with current endpoint host", + rule: s3response.RoutingRule{ + Condition: &s3response.RoutingRuleCondition{ + KeyPrefixEquals: "old/", + }, + Redirect: s3response.Redirect{ + ReplaceKeyPrefixWith: "new/", + }, + }, + path: "/old/page.html?x=1&y=2", + wantLocation: "http://site.test/new/page.html?x=1&y=2", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + RoutingRules: []s3response.RoutingRule{tt.rule}, + }, nil, true) + + resp := websiteRequest(t, be, tt.path) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusMovedPermanently { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusMovedPermanently) + } + if got := resp.Header.Get("Location"); got != tt.wantLocation { + t.Fatalf("Location = %q, want %q", got, tt.wantLocation) + } + }) + } +} + +func TestWebsiteHandlerPostErrorRoutingUsesOriginalKeyBeforeIndexExpansion(t *testing.T) { + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + RoutingRules: []s3response.RoutingRule{ + { + Condition: &s3response.RoutingRuleCondition{ + KeyPrefixEquals: "blog/", + HttpErrorCodeReturnedEquals: "404", + }, + Redirect: s3response.Redirect{ + ReplaceKeyPrefixWith: "archive/", + HttpRedirectCode: "302", + }, + }, + }, + }, nil, true) + + resp := websiteRequest(t, be, "/blog/") + defer resp.Body.Close() + + if resp.StatusCode != http.StatusFound { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusFound) + } + if got := resp.Header.Get("Location"); got != "http://site.test/archive/" { + t.Fatalf("Location = %q, want %q", got, "http://site.test/archive/") + } + if countCalls(be.calls, "GetObject") != 1 { + t.Fatalf("GetObject calls = %d, want 1; calls: %v", countCalls(be.calls, "GetObject"), be.calls) + } +} + +func TestWebsiteHandlerObjectStore5xxBypassesRoutingAndErrorDocument(t *testing.T) { + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + ErrorDocument: &s3response.ErrorDocument{Key: "error.html"}, + RoutingRules: []s3response.RoutingRule{ + { + Condition: &s3response.RoutingRuleCondition{ + HttpErrorCodeReturnedEquals: "500", + }, + Redirect: s3response.Redirect{ + ReplaceKeyWith: "elsewhere.html", + HttpRedirectCode: "302", + }, + }, + }, + }, map[string]string{ + "error.html": "custom error document", + }, true) + be.objectErrors = map[string]error{ + "boom.html": s3err.GetAPIError(s3err.ErrInternalError), + } + + resp := websiteRequest(t, be, "/boom.html") + defer resp.Body.Close() + + if resp.StatusCode != http.StatusInternalServerError { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusInternalServerError) + } + if got := resp.Header.Get("Location"); got != "" { + t.Fatalf("unexpected Location header %q", got) + } + if got := resp.Header.Get("x-amz-error-code"); got != "InternalError" { + t.Fatalf("x-amz-error-code = %q, want %q", got, "InternalError") + } + if got := countCalls(be.calls, "GetObject"); got != 1 { + t.Fatalf("GetObject calls = %d, want 1; calls: %v", got, be.calls) + } +} + +func TestWebsiteHandlerPublicAccessDeniedPreventsObjectReadAndCanRoute(t *testing.T) { + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + RoutingRules: []s3response.RoutingRule{ + { + Condition: &s3response.RoutingRuleCondition{ + HttpErrorCodeReturnedEquals: "403", + }, + Redirect: s3response.Redirect{ + ReplaceKeyWith: "denied.html", + HttpRedirectCode: "302", + }, + }, + }, + }, map[string]string{ + "private.html": "secret", + }, false) + + resp := websiteRequest(t, be, "/private.html") + defer resp.Body.Close() + + if resp.StatusCode != http.StatusFound { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusFound) + } + if got := resp.Header.Get("Location"); got != "http://site.test/denied.html" { + t.Fatalf("Location = %q", got) + } + if containsCall(be.calls, "HeadObject") { + t.Fatal("HeadObject was called after public access was denied") + } + if containsCall(be.calls, "GetObject") { + t.Fatal("GetObject was called after public access was denied") + } +} + +func TestWebsiteHandlerVerifiesPublicAccessBeforeGetObject(t *testing.T) { + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + }, map[string]string{ + "index.html": "home", + }, true) + + resp := websiteRequest(t, be, "/") + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + if got := readBody(t, resp); got != "home" { + t.Fatalf("body = %q, want %q", got, "home") + } + + verifyIdx := firstCallIndex(be.calls, "GetBucketAcl") + getObjectIdx := firstCallIndex(be.calls, "GetObject") + if verifyIdx == -1 { + t.Fatal("expected public access verification to read bucket ACL") + } + if getObjectIdx == -1 { + t.Fatal("expected GetObject call") + } + if verifyIdx > getObjectIdx { + t.Fatalf("GetObject happened before public access verification: %v", be.calls) + } +} + +func TestWebsiteHandlerHeadUsesHeadObjectAndReturnsHeadersOnly(t *testing.T) { + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + }, map[string]string{ + "index.html": "home", + }, true) + + resp := websiteRequestWithMethod(t, be, http.MethodHead, "/") + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + if got := resp.Header.Get("Content-Length"); got != "4" { + t.Fatalf("Content-Length = %q, want %q", got, "4") + } + if got := resp.Header.Get("Content-Type"); got != "text/html" { + t.Fatalf("Content-Type = %q, want %q", got, "text/html") + } + if got := readBody(t, resp); got != "" { + t.Fatalf("body = %q, want empty body", got) + } + if containsCall(be.calls, "GetObject") { + t.Fatalf("GetObject was called for HEAD request: %v", be.calls) + } + if !containsCall(be.calls, "HeadObject") { + t.Fatalf("HeadObject was not called for HEAD request: %v", be.calls) + } +} + +func TestWebsiteHandlerGetValidatesBucketName(t *testing.T) { + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + }, nil, true) + + resp := websiteRequestWithHostAndHeaders(t, be, http.MethodGet, "bad_bucket", "/", nil) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusBadRequest) + } + if got := resp.Header.Get("x-amz-error-code"); got != "InvalidBucketName" { + t.Fatalf("x-amz-error-code = %q", got) + } + if len(be.calls) != 0 { + t.Fatalf("invalid bucket should not call backend, got calls: %v", be.calls) + } +} + +func TestWebsiteHandlerHeadValidatesObjectName(t *testing.T) { + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + }, nil, true) + + resp := websiteRequestWithHeaders(t, be, http.MethodHead, "/../../private.html", nil) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusBadRequest) + } + if got := resp.Header.Get("x-amz-error-code"); got != "400" { + t.Fatalf("x-amz-error-code = %q", got) + } + if len(be.calls) != 0 { + t.Fatalf("invalid object should not call backend, got calls: %v", be.calls) + } +} + +func TestWebsiteHandlerGetAppliesBucketCORS(t *testing.T) { + corsConfig, err := xml.Marshal(auth.CORSConfiguration{ + Rules: []auth.CORSRule{ + { + AllowedOrigins: []auth.CORSOrigin{"https://client.example"}, + AllowedMethods: []auth.CORSHTTPMethod{http.MethodGet, http.MethodHead}, + ExposeHeaders: []auth.CORSHeader{"Content-Length"}, + }, + }, + }) + if err != nil { + t.Fatalf("marshal cors config: %v", err) + } + + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + }, map[string]string{ + "index.html": "home", + }, true) + be.corsConfig = corsConfig + + resp := websiteRequestWithHeaders(t, be, http.MethodGet, "/", map[string]string{ + "Origin": "https://client.example", + }) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + if got := resp.Header.Get("Access-Control-Allow-Origin"); got != "https://client.example" { + t.Fatalf("Access-Control-Allow-Origin = %q", got) + } + if got := resp.Header.Get("Access-Control-Allow-Methods"); got != "GET, HEAD" { + t.Fatalf("Access-Control-Allow-Methods = %q", got) + } + if got := resp.Header.Get("Access-Control-Expose-Headers"); got != "Content-Length, ETag, x-amz-storage-class" { + t.Fatalf("Access-Control-Expose-Headers = %q", got) + } + if got := resp.Header.Get("Access-Control-Allow-Credentials"); got != "true" { + t.Fatalf("Access-Control-Allow-Credentials = %q", got) + } + if got := resp.Header.Get("Vary"); got != "Origin, Access-Control-Request-Headers, Access-Control-Request-Method" { + t.Fatalf("Vary = %q", got) + } + if got := readBody(t, resp); got != "home" { + t.Fatalf("body = %q, want %q", got, "home") + } + if !containsCall(be.calls, "GetBucketCors") { + t.Fatalf("GetBucketCors was not called: %v", be.calls) + } +} + +func TestWebsiteHandlerHeadAppliesBucketCORS(t *testing.T) { + corsConfig, err := xml.Marshal(auth.CORSConfiguration{ + Rules: []auth.CORSRule{ + { + AllowedOrigins: []auth.CORSOrigin{"https://client.example"}, + AllowedMethods: []auth.CORSHTTPMethod{http.MethodHead}, + ExposeHeaders: []auth.CORSHeader{"Content-Length"}, + }, + }, + }) + if err != nil { + t.Fatalf("marshal cors config: %v", err) + } + + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + }, map[string]string{ + "index.html": "home", + }, true) + be.corsConfig = corsConfig + + resp := websiteRequestWithHeaders(t, be, http.MethodHead, "/", map[string]string{ + "Origin": "https://client.example", + }) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + if got := resp.Header.Get("Access-Control-Allow-Origin"); got != "https://client.example" { + t.Fatalf("Access-Control-Allow-Origin = %q", got) + } + if got := resp.Header.Get("Access-Control-Allow-Methods"); got != "HEAD" { + t.Fatalf("Access-Control-Allow-Methods = %q", got) + } + if got := resp.Header.Get("Access-Control-Expose-Headers"); got != "Content-Length, ETag, x-amz-storage-class" { + t.Fatalf("Access-Control-Expose-Headers = %q", got) + } + if got := readBody(t, resp); got != "" { + t.Fatalf("body = %q, want empty body", got) + } + if !containsCall(be.calls, "GetBucketCors") { + t.Fatalf("GetBucketCors was not called: %v", be.calls) + } + if containsCall(be.calls, "GetObject") { + t.Fatalf("GetObject was called for HEAD request: %v", be.calls) + } +} + +func TestWebsiteHandlerOptionsAccessGranted(t *testing.T) { + maxAge := int32(42) + corsConfig, err := xml.Marshal(auth.CORSConfiguration{ + Rules: []auth.CORSRule{ + { + AllowedOrigins: []auth.CORSOrigin{"https://client.example"}, + AllowedMethods: []auth.CORSHTTPMethod{http.MethodGet, http.MethodHead}, + AllowedHeaders: []auth.CORSHeader{"Content-Type", "X-Amz-Date"}, + ExposeHeaders: []auth.CORSHeader{"Content-Length"}, + MaxAgeSeconds: &maxAge, + }, + }, + }) + if err != nil { + t.Fatalf("marshal cors config: %v", err) + } + + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + }, map[string]string{ + "index.html": "home", + }, true) + be.corsConfig = corsConfig + + resp := websiteRequestWithHeaders(t, be, http.MethodOptions, "/index.html", map[string]string{ + "Origin": "https://client.example", + "Access-Control-Request-Method": http.MethodGet, + "Access-Control-Request-Headers": "content-type, X-Amz-Date", + }) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + if got := resp.Header.Get("Access-Control-Allow-Origin"); got != "https://client.example" { + t.Fatalf("Access-Control-Allow-Origin = %q", got) + } + if got := resp.Header.Get("Access-Control-Allow-Methods"); got != "GET, HEAD" { + t.Fatalf("Access-Control-Allow-Methods = %q", got) + } + if got := resp.Header.Get("Access-Control-Allow-Headers"); got != "content-type, x-amz-date" { + t.Fatalf("Access-Control-Allow-Headers = %q", got) + } + if got := resp.Header.Get("Access-Control-Expose-Headers"); got != "Content-Length, ETag" { + t.Fatalf("Access-Control-Expose-Headers = %q", got) + } + if got := resp.Header.Get("Access-Control-Max-Age"); got != "42" { + t.Fatalf("Access-Control-Max-Age = %q", got) + } + if got := resp.Header.Get("Access-Control-Allow-Credentials"); got != "true" { + t.Fatalf("Access-Control-Allow-Credentials = %q", got) + } + if got := resp.Header.Get("Vary"); got != "Origin, Access-Control-Request-Headers, Access-Control-Request-Method" { + t.Fatalf("Vary = %q", got) + } + if got := readBody(t, resp); got != "" { + t.Fatalf("body = %q, want empty body", got) + } + if !containsCall(be.calls, "GetBucketCors") { + t.Fatalf("GetBucketCors was not called: %v", be.calls) + } + for _, unexpected := range []string{"GetBucketWebsite", "GetObject", "HeadObject", "GetBucketAcl"} { + if containsCall(be.calls, unexpected) { + t.Fatalf("%s was called for OPTIONS request: %v", unexpected, be.calls) + } + } +} + +func TestWebsiteHandlerOptionsMissingOrigin(t *testing.T) { + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + }, nil, true) + + resp := websiteRequestWithHeaders(t, be, http.MethodOptions, "/", map[string]string{ + "Access-Control-Request-Method": http.MethodGet, + }) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusBadRequest) + } + if got := resp.Header.Get("x-amz-error-code"); got != "BadRequest" { + t.Fatalf("x-amz-error-code = %q", got) + } + if containsCall(be.calls, "GetBucketCors") { + t.Fatalf("GetBucketCors was called despite missing origin: %v", be.calls) + } +} + +func TestWebsiteHandlerOptionsInvalidRequestMethod(t *testing.T) { + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + }, nil, true) + + resp := websiteRequestWithHeaders(t, be, http.MethodOptions, "/", map[string]string{ + "Origin": "https://client.example", + "Access-Control-Request-Method": http.MethodOptions, + }) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusBadRequest) + } + if got := resp.Header.Get("x-amz-error-code"); got != "BadRequest" { + t.Fatalf("x-amz-error-code = %q", got) + } + if containsCall(be.calls, "GetBucketCors") { + t.Fatalf("GetBucketCors was called despite invalid request method: %v", be.calls) + } +} + +func TestWebsiteHandlerOptionsUnsetBucketCORS(t *testing.T) { + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + }, nil, true) + be.corsErr = s3err.GetAPIError(s3err.ErrNoSuchCORSConfiguration) + + resp := websiteRequestWithHeaders(t, be, http.MethodOptions, "/", map[string]string{ + "Origin": "https://client.example", + "Access-Control-Request-Method": http.MethodGet, + }) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusForbidden) + } + if got := resp.Header.Get("x-amz-error-code"); got != "AccessForbidden" { + t.Fatalf("x-amz-error-code = %q", got) + } + body := readBody(t, resp) + for _, want := range []string{ + "
  • Method: OPTIONS
    • ", + "
  • ResourceType: BUCKET
    • ", + } { + if !strings.Contains(body, want) { + t.Fatalf("body missing %q: %s", want, body) + } + } +} + +func TestWebsiteHandlerOptionsAccessForbidden(t *testing.T) { + corsConfig, err := xml.Marshal(auth.CORSConfiguration{ + Rules: []auth.CORSRule{ + { + AllowedOrigins: []auth.CORSOrigin{"https://client.example"}, + AllowedMethods: []auth.CORSHTTPMethod{http.MethodHead}, + }, + }, + }) + if err != nil { + t.Fatalf("marshal cors config: %v", err) + } + + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + }, nil, true) + be.corsConfig = corsConfig + + resp := websiteRequestWithHeaders(t, be, http.MethodOptions, "/index.html", map[string]string{ + "Origin": "https://client.example", + "Access-Control-Request-Method": http.MethodGet, + }) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusForbidden) + } + if got := resp.Header.Get("x-amz-error-code"); got != "AccessForbidden" { + t.Fatalf("x-amz-error-code = %q", got) + } + body := readBody(t, resp) + for _, want := range []string{ + "
  • Method: OPTIONS
    • ", + "
  • ResourceType: OBJECT
    • ", + } { + if !strings.Contains(body, want) { + t.Fatalf("body missing %q: %s", want, body) + } + } +} + +func TestWebsiteHandlerMethodNotAllowed(t *testing.T) { + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + }, nil, true) + + resp := websiteRequestWithMethod(t, be, http.MethodPut, "/some-key") + defer resp.Body.Close() + + if resp.StatusCode != http.StatusMethodNotAllowed { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusMethodNotAllowed) + } + if got := resp.Header.Get("Allow"); got != "GET, HEAD, OPTIONS" { + t.Fatalf("Allow = %q, want %q", got, "GET, HEAD, OPTIONS") + } + if got := resp.Header.Get("Content-Type"); !strings.HasPrefix(got, "text/html") { + t.Fatalf("Content-Type = %q, want text/html", got) + } + if got := resp.Header.Get("Server"); got != "VERSITYGW" { + t.Fatalf("Server = %q, want %q", got, "VERSITYGW") + } + + body := readBody(t, resp) + for _, want := range []string{ + "
  • Code: MethodNotAllowed
    • ", + "
  • Method: PUT
    • ", + "
  • ResourceType: OBJECT
    • ", + } { + if !strings.Contains(body, want) { + t.Fatalf("method not allowed body missing %q: %s", want, body) + } + } + if containsCall(be.calls, "GetBucketWebsite") { + t.Fatalf("unmatched method should not load website config: %v", be.calls) + } +} + +func newWebsiteTestBackend(t *testing.T, config s3response.WebsiteConfiguration, objects map[string]string, public bool) *websiteTestBackend { + t.Helper() + + data, err := xml.Marshal(config) + if err != nil { + t.Fatalf("marshal website config: %v", err) + } + if objects == nil { + objects = map[string]string{} + } + + return &websiteTestBackend{ + websiteConfig: data, + objects: objects, + public: public, + } +} + +func websiteRequest(t *testing.T, be backend.Backend, path string) *http.Response { + t.Helper() + + return websiteRequestWithMethod(t, be, http.MethodGet, path) +} + +func websiteRequestWithMethod(t *testing.T, be backend.Backend, method, path string) *http.Response { + t.Helper() + + return websiteRequestWithHeaders(t, be, method, path, nil) +} + +func websiteRequestWithHeaders(t *testing.T, be backend.Backend, method, path string, headers map[string]string) *http.Response { + t.Helper() + + return websiteRequestWithHostAndHeaders(t, be, method, "site.test", path, headers) +} + +func websiteRequestWithHostAndHeaders(t *testing.T, be backend.Backend, method, host, path string, headers map[string]string) *http.Response { + t.Helper() + + app := fiber.New(fiber.Config{ + ServerHeader: "VERSITYGW", + }) + registerWebsiteRoutes(app, be, "") + + req := httptest.NewRequest(method, path, nil) + req.Host = host + req.Header.Set("Host", host) + for key, value := range headers { + req.Header.Set(key, value) + } + resp, err := app.Test(req, -1) + if err != nil { + t.Fatalf("website request failed: %v", err) + } + return resp +} + +func readBody(t *testing.T, resp *http.Response) string { + t.Helper() + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read response body: %v", err) + } + return string(body) +} + +func containsCall(calls []string, want string) bool { + return firstCallIndex(calls, want) != -1 +} + +func countCalls(calls []string, want string) int { + var count int + for _, call := range calls { + if call == want { + count++ + } + } + return count +} + +func firstCallIndex(calls []string, want string) int { + for i, call := range calls { + if call == want { + return i + } + } + return -1 +} diff --git a/website/server.go b/website/server.go index 398d1d72..c9c18d88 100644 --- a/website/server.go +++ b/website/server.go @@ -17,11 +17,14 @@ package website import ( "fmt" "net" + "os" "github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2/middleware/logger" "github.com/gofiber/fiber/v2/middleware/recover" "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/debuglogger" + "github.com/versity/versitygw/s3api/middlewares" "github.com/versity/versitygw/s3api/utils" ) @@ -31,6 +34,7 @@ type Server struct { CertStorage *utils.CertStorage domain string quiet bool + socketPerm os.FileMode } // Option sets various options for NewServer(). @@ -46,6 +50,13 @@ func WithTLS(cs *utils.CertStorage) Option { return func(s *Server) { s.CertStorage = cs } } +// WithSocketPerm sets the file-mode permissions applied to file-backed UNIX +// domain sockets after binding. It has no effect on TCP/IP or abstract +// namespace sockets. +func WithSocketPerm(perm os.FileMode) Option { + return func(s *Server) { s.socketPerm = perm } +} + // NewServer creates a new static website hosting server. // The domain parameter is the base domain for virtual-host routing: // - Host "blog." resolves to bucket "blog" @@ -83,8 +94,12 @@ func NewServer(be backend.Backend, domain string, opts ...Option) *Server { })) } - // All requests go through the website handler - app.Use(newHandler(be, domain)) + // initialize the debug logger in debug mode + if debuglogger.IsDebugEnabled() { + app.Use(middlewares.DebugLogger()) + } + + registerWebsiteRoutes(app, be, domain) return server } @@ -103,9 +118,9 @@ func (s *Server) ServeMultiPort(ports []string) error { var err error if s.CertStorage != nil { - ln, err = utils.NewMultiAddrTLSListener(s.app.Config().Network, addrSpec, s.CertStorage.GetCertificate) + ln, err = utils.NewMultiAddrTLSListener(s.app.Config().Network, addrSpec, s.CertStorage.GetCertificate, utils.ListenerOptions{SocketPerm: s.socketPerm}) } else { - ln, err = utils.NewMultiAddrListener(s.app.Config().Network, addrSpec) + ln, err = utils.NewMultiAddrListener(s.app.Config().Network, addrSpec, utils.ListenerOptions{SocketPerm: s.socketPerm}) } if err != nil { From f08f76fea4846d6770604a5321941f467477e7d8 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Wed, 10 Jun 2026 12:26:28 +0400 Subject: [PATCH 4/4] feat: support x-amz-website-redirect-location Integrate x-amz-website-redirect-location across object metadata flows so uploads, copies, multipart creation, HEAD, and GET preserve and return redirect locations, and website hosting applies object-level redirects from the stored value. --- backend/azure/azure.go | 105 ++++++----- backend/common.go | 2 +- backend/posix/posix.go | 200 ++++++++++++--------- s3api/controllers/bucket-post.go | 55 +++--- s3api/controllers/object-get.go | 1 + s3api/controllers/object-get_test.go | 2 + s3api/controllers/object-head.go | 1 + s3api/controllers/object-head_test.go | 2 + s3api/controllers/object-post.go | 11 ++ s3api/controllers/object-put.go | 22 +++ s3api/utils/utils.go | 10 ++ s3err/s3err.go | 6 + s3response/website.go | 20 ++- s3response/website_test.go | 46 ++++- tests/integration/CopyObject.go | 105 ++++++++--- tests/integration/CreateMultipartUpload.go | 38 +++- tests/integration/GetObject.go | 26 +-- tests/integration/HeadObject.go | 26 +-- tests/integration/PostObject.go | 47 ++++- tests/integration/PutBucketWebsite.go | 50 ++++++ tests/integration/PutObject.go | 13 ++ tests/integration/WebsiteHosting.go | 94 +++++++++- tests/integration/group-tests.go | 16 ++ tests/integration/utils.go | 20 ++- website/handler.go | 108 +++++++---- website/handler_test.go | 154 +++++++++++++--- 26 files changed, 877 insertions(+), 303 deletions(-) diff --git a/backend/azure/azure.go b/backend/azure/azure.go index ea5439f7..cdcb7a45 100644 --- a/backend/azure/azure.go +++ b/backend/azure/azure.go @@ -68,6 +68,7 @@ const ( keyObjRetention key = "Objectretention" keyObjLegalHold key = "Objectlegalhold" keyExpires key = "Vgwexpires" + keyWebsiteRedirect key = "Vgwwebsiteredirect" onameAttr key = "Objname" onameAttrLower key = "objname" metaTmpMultipartPrefix key = ".sgwtmp" + "/multipart" @@ -84,18 +85,19 @@ const ( func (key) Table() map[string]struct{} { return map[string]struct{}{ - "acl": {}, - "ownership": {}, - "tags": {}, - "policy": {}, - "bucketlock": {}, - "website": {}, - "objectretention": {}, - "vgwexpires": {}, - "objectlegalhold": {}, - "objname": {}, - ".sgwtmp/multipart": {}, - "mpmetadata": {}, + "acl": {}, + "ownership": {}, + "tags": {}, + "policy": {}, + "bucketlock": {}, + "website": {}, + "objectretention": {}, + "vgwexpires": {}, + "vgwwebsiteredirect": {}, + "objectlegalhold": {}, + "objname": {}, + ".sgwtmp/multipart": {}, + "mpmetadata": {}, } } @@ -372,6 +374,15 @@ func (az *Azure) PutObject(ctx context.Context, po s3response.PutObjectInput) (s metadata[string(keyExpires)] = po.Expires } } + if getString(po.WebsiteRedirectLocation) != "" { + if metadata == nil { + metadata = map[string]*string{ + string(keyWebsiteRedirect): po.WebsiteRedirectLocation, + } + } else { + metadata[string(keyWebsiteRedirect)] = po.WebsiteRedirectLocation + } + } opts := &blockblob.UploadStreamOptions{ Metadata: metadata, @@ -571,22 +582,23 @@ func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.G } return &s3.GetObjectOutput{ - AcceptRanges: backend.GetPtrFromString("bytes"), - ContentLength: blobDownloadResponse.ContentLength, - ContentEncoding: blobDownloadResponse.ContentEncoding, - ContentType: blobDownloadResponse.ContentType, - ContentDisposition: blobDownloadResponse.ContentDisposition, - ContentLanguage: blobDownloadResponse.ContentLanguage, - CacheControl: blobDownloadResponse.CacheControl, - ExpiresString: blobDownloadResponse.Metadata[string(keyExpires)], - ETag: backend.GetPtrFromString(convertAzureEtag(blobDownloadResponse.ETag)), - LastModified: blobDownloadResponse.LastModified, - Metadata: parseAndFilterAzMetadata(blobDownloadResponse.Metadata), - TagCount: &tagcount, - ContentRange: contentRange, - Body: blobDownloadResponse.Body, - StorageClass: types.StorageClassStandard, - PartsCount: partsCount, + AcceptRanges: backend.GetPtrFromString("bytes"), + ContentLength: blobDownloadResponse.ContentLength, + ContentEncoding: blobDownloadResponse.ContentEncoding, + ContentType: blobDownloadResponse.ContentType, + ContentDisposition: blobDownloadResponse.ContentDisposition, + ContentLanguage: blobDownloadResponse.ContentLanguage, + CacheControl: blobDownloadResponse.CacheControl, + ExpiresString: blobDownloadResponse.Metadata[string(keyExpires)], + WebsiteRedirectLocation: blobDownloadResponse.Metadata[string(keyWebsiteRedirect)], + ETag: backend.GetPtrFromString(convertAzureEtag(blobDownloadResponse.ETag)), + LastModified: blobDownloadResponse.LastModified, + Metadata: parseAndFilterAzMetadata(blobDownloadResponse.Metadata), + TagCount: &tagcount, + ContentRange: contentRange, + Body: blobDownloadResponse.Body, + StorageClass: types.StorageClassStandard, + PartsCount: partsCount, }, nil } @@ -670,20 +682,21 @@ func (az *Azure) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3 } result := &s3.HeadObjectOutput{ - ContentRange: contentRange, - AcceptRanges: backend.GetPtrFromString("bytes"), - ContentLength: &length, - PartsCount: partsCount, - ContentType: resp.ContentType, - ContentEncoding: resp.ContentEncoding, - ContentLanguage: resp.ContentLanguage, - ContentDisposition: resp.ContentDisposition, - CacheControl: resp.CacheControl, - ExpiresString: resp.Metadata[string(keyExpires)], - ETag: backend.GetPtrFromString(convertAzureEtag(resp.ETag)), - LastModified: resp.LastModified, - Metadata: parseAndFilterAzMetadata(resp.Metadata), - StorageClass: types.StorageClassStandard, + ContentRange: contentRange, + AcceptRanges: backend.GetPtrFromString("bytes"), + ContentLength: &length, + PartsCount: partsCount, + ContentType: resp.ContentType, + ContentEncoding: resp.ContentEncoding, + ContentLanguage: resp.ContentLanguage, + ContentDisposition: resp.ContentDisposition, + CacheControl: resp.CacheControl, + ExpiresString: resp.Metadata[string(keyExpires)], + WebsiteRedirectLocation: resp.Metadata[string(keyWebsiteRedirect)], + ETag: backend.GetPtrFromString(convertAzureEtag(resp.ETag)), + LastModified: resp.LastModified, + Metadata: parseAndFilterAzMetadata(resp.Metadata), + StorageClass: types.StorageClassStandard, } status, ok := resp.Metadata[string(keyObjLegalHold)] @@ -1193,6 +1206,9 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu if getString(input.Expires) != "" { meta[string(keyExpires)] = *input.Expires } + if getString(input.WebsiteRedirectLocation) != "" { + meta[string(keyWebsiteRedirect)] = *input.WebsiteRedirectLocation + } // Set object metadata _, err = dstClient.SetMetadata(ctx, parseMetadata(meta), nil) if err != nil { @@ -1273,6 +1289,7 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu ContentLanguage: input.ContentLanguage, CacheControl: input.CacheControl, Expires: input.Expires, + WebsiteRedirectLocation: input.WebsiteRedirectLocation, Metadata: input.Metadata, ObjectLockRetainUntilDate: input.ObjectLockRetainUntilDate, ObjectLockMode: input.ObjectLockMode, @@ -1288,6 +1305,7 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu pInput.ContentLanguage = downloadResp.ContentLanguage pInput.ContentType = downloadResp.ContentType pInput.Metadata = parseAzMetadata(downloadResp.Metadata) + delete(pInput.Metadata, string(keyWebsiteRedirect)) } if input.TaggingDirective == types.TaggingDirectiveReplace { @@ -1397,6 +1415,9 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input s3response.Cre if getString(input.Expires) != "" { meta[string(keyExpires)] = input.Expires } + if getString(input.WebsiteRedirectLocation) != "" { + meta[string(keyWebsiteRedirect)] = input.WebsiteRedirectLocation + } // parse object tags tags, err := backend.ParseObjectTags(getString(input.Tagging)) diff --git a/backend/common.go b/backend/common.go index 536f7a4d..01dfd40a 100644 --- a/backend/common.go +++ b/backend/common.go @@ -531,7 +531,7 @@ func DecompressData(data []byte) ([]byte, error) { return nil, err } if closeErr != nil { - return nil, err + return nil, closeErr } return decompressed, nil diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 8494ec64..b8073463 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -122,6 +122,7 @@ const ( contentDispHdr = "content-disposition" cacheCtrlHdr = "cache-control" expiresHdr = "expires" + websiteRedirectHdr = "website-redirect-location" emptyMD5 = "\"d41d8cd98f00b204e9800998ecf8427e\"" aclkey = "acl" ownershipkey = "ownership" @@ -1549,13 +1550,14 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu s3response.Create err = p.storeObjectMetaProperties(nil, bucket, filepath.Join(objdir, uploadID), metaProperties{ - ContentType: mpu.ContentType, - ContentEncoding: mpu.ContentEncoding, - ContentDisposition: mpu.ContentDisposition, - ContentLanguage: mpu.ContentLanguage, - CacheControl: mpu.CacheControl, - Expires: mpu.Expires, - Metadata: mpu.Metadata, + ContentType: mpu.ContentType, + ContentEncoding: mpu.ContentEncoding, + ContentDisposition: mpu.ContentDisposition, + ContentLanguage: mpu.ContentLanguage, + CacheControl: mpu.CacheControl, + Expires: mpu.Expires, + WebsiteRedirectLocation: mpu.WebsiteRedirectLocation, + Metadata: mpu.Metadata, }) if err != nil { // cleanup object if returning error @@ -2403,13 +2405,14 @@ func (p *Posix) checkUploadIDExists(bucket, object, uploadID string) ([32]byte, } type metaProperties struct { - ContentType *string - ContentEncoding *string - ContentDisposition *string - ContentLanguage *string - CacheControl *string - Expires *string - Metadata map[string]string + ContentType *string + ContentEncoding *string + ContentDisposition *string + ContentLanguage *string + CacheControl *string + Expires *string + WebsiteRedirectLocation *string + Metadata map[string]string } // loadObjectMetadata loads the given object metadata, if it fails to load @@ -2508,6 +2511,11 @@ func (p *Posix) loadObjectMetaProperties(f *os.File, bucket, object string, fi * result.Expires = backend.GetPtrFromString(string(b)) } + b, err = p.meta.RetrieveAttribute(f, bucket, object, websiteRedirectHdr) + if err == nil { + result.WebsiteRedirectLocation = backend.GetPtrFromString(string(b)) + } + result.Metadata = p.loadObjectMetadata(f, bucket, object) return result @@ -2587,6 +2595,12 @@ func (p *Posix) storeObjectMetaProperties(f *os.File, bucket, object string, m m return fmt.Errorf("set expires: %w", err) } } + if getString(m.WebsiteRedirectLocation) != "" { + err := p.meta.StoreAttribute(f, bucket, object, websiteRedirectHdr, []byte(*m.WebsiteRedirectLocation)) + if err != nil { + return fmt.Errorf("set website-redirect-location: %w", err) + } + } if m.Metadata != nil { err := p.storeObjectMetadata(f, bucket, object, m.Metadata) if err != nil { @@ -3682,6 +3696,14 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje return s3response.PutObjectOutput{}, fmt.Errorf("set content-type attr: %w", err) } + if getString(po.WebsiteRedirectLocation) != "" { + err = p.meta.StoreAttribute(nil, *po.Bucket, *po.Key, websiteRedirectHdr, + []byte(*po.WebsiteRedirectLocation)) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("set website-redirect-location attr: %w", err) + } + } + expectedSum := getEmptyChecksumValue(checksumAlgorithm) if checksumValue != "" && expectedSum != checksumValue { return s3response.PutObjectOutput{}, s3err.GetChecksumBadDigestErr(checksumAlgorithm) @@ -3879,13 +3901,14 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje err = p.storeObjectMetaProperties(f.File(), *po.Bucket, *po.Key, metaProperties{ - ContentType: po.ContentType, - ContentEncoding: po.ContentEncoding, - ContentLanguage: po.ContentLanguage, - ContentDisposition: po.ContentDisposition, - CacheControl: po.CacheControl, - Expires: po.Expires, - Metadata: po.Metadata, + ContentType: po.ContentType, + ContentEncoding: po.ContentEncoding, + ContentLanguage: po.ContentLanguage, + ContentDisposition: po.ContentDisposition, + CacheControl: po.CacheControl, + Expires: po.Expires, + WebsiteRedirectLocation: po.WebsiteRedirectLocation, + Metadata: po.Metadata, }) if err != nil { return s3response.PutObjectOutput{}, err @@ -4583,32 +4606,33 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge var length int64 = 0 return &s3.GetObjectOutput{ - ChecksumCRC32: checksums.CRC32, - ChecksumCRC32C: checksums.CRC32C, - ChecksumSHA1: checksums.SHA1, - ChecksumSHA256: checksums.SHA256, - ChecksumCRC64NVME: checksums.CRC64NVME, - ChecksumSHA512: checksums.SHA512, - ChecksumMD5: checksums.MD5, - ChecksumXXHASH64: checksums.XXHASH64, - ChecksumXXHASH3: checksums.XXHASH3, - ChecksumXXHASH128: checksums.XXHASH128, - ChecksumType: checksums.Type, - AcceptRanges: backend.GetPtrFromString("bytes"), - ContentLength: &length, - ContentEncoding: objMeta.ContentEncoding, - ContentType: objMeta.ContentType, - ContentLanguage: objMeta.ContentLanguage, - ContentDisposition: objMeta.ContentDisposition, - CacheControl: objMeta.CacheControl, - ExpiresString: objMeta.Expires, - ETag: &etag, - LastModified: backend.GetTimePtr(fid.ModTime()), - Metadata: objMeta.Metadata, - TagCount: tagCount, - ContentRange: nil, - StorageClass: types.StorageClassStandard, - VersionId: &versionId, + ChecksumCRC32: checksums.CRC32, + ChecksumCRC32C: checksums.CRC32C, + ChecksumSHA1: checksums.SHA1, + ChecksumSHA256: checksums.SHA256, + ChecksumCRC64NVME: checksums.CRC64NVME, + ChecksumSHA512: checksums.SHA512, + ChecksumMD5: checksums.MD5, + ChecksumXXHASH64: checksums.XXHASH64, + ChecksumXXHASH3: checksums.XXHASH3, + ChecksumXXHASH128: checksums.XXHASH128, + ChecksumType: checksums.Type, + AcceptRanges: backend.GetPtrFromString("bytes"), + ContentLength: &length, + ContentEncoding: objMeta.ContentEncoding, + ContentType: objMeta.ContentType, + ContentLanguage: objMeta.ContentLanguage, + ContentDisposition: objMeta.ContentDisposition, + CacheControl: objMeta.CacheControl, + ExpiresString: objMeta.Expires, + WebsiteRedirectLocation: objMeta.WebsiteRedirectLocation, + ETag: &etag, + LastModified: backend.GetTimePtr(fid.ModTime()), + Metadata: objMeta.Metadata, + TagCount: tagCount, + ContentRange: nil, + StorageClass: types.StorageClassStandard, + VersionId: &versionId, }, nil } @@ -4735,34 +4759,35 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge } return &s3.GetObjectOutput{ - AcceptRanges: backend.GetPtrFromString("bytes"), - ContentLength: &length, - ContentEncoding: objMeta.ContentEncoding, - ContentType: objMeta.ContentType, - ContentDisposition: objMeta.ContentDisposition, - ContentLanguage: objMeta.ContentLanguage, - CacheControl: objMeta.CacheControl, - ExpiresString: objMeta.Expires, - ETag: &etag, - LastModified: backend.GetTimePtr(fi.ModTime()), - Metadata: objMeta.Metadata, - TagCount: tagCount, - ContentRange: contentRange, - StorageClass: types.StorageClassStandard, - VersionId: &versionId, - Body: body, - ChecksumCRC32: checksums.CRC32, - ChecksumCRC32C: checksums.CRC32C, - ChecksumSHA1: checksums.SHA1, - ChecksumSHA256: checksums.SHA256, - ChecksumCRC64NVME: checksums.CRC64NVME, - ChecksumSHA512: checksums.SHA512, - ChecksumMD5: checksums.MD5, - ChecksumXXHASH64: checksums.XXHASH64, - ChecksumXXHASH3: checksums.XXHASH3, - ChecksumXXHASH128: checksums.XXHASH128, - ChecksumType: checksums.Type, - PartsCount: partsCount, + AcceptRanges: backend.GetPtrFromString("bytes"), + ContentLength: &length, + ContentEncoding: objMeta.ContentEncoding, + ContentType: objMeta.ContentType, + ContentDisposition: objMeta.ContentDisposition, + ContentLanguage: objMeta.ContentLanguage, + CacheControl: objMeta.CacheControl, + ExpiresString: objMeta.Expires, + WebsiteRedirectLocation: objMeta.WebsiteRedirectLocation, + ETag: &etag, + LastModified: backend.GetTimePtr(fi.ModTime()), + Metadata: objMeta.Metadata, + TagCount: tagCount, + ContentRange: contentRange, + StorageClass: types.StorageClassStandard, + VersionId: &versionId, + Body: body, + ChecksumCRC32: checksums.CRC32, + ChecksumCRC32C: checksums.CRC32C, + ChecksumSHA1: checksums.SHA1, + ChecksumSHA256: checksums.SHA256, + ChecksumCRC64NVME: checksums.CRC64NVME, + ChecksumSHA512: checksums.SHA512, + ChecksumMD5: checksums.MD5, + ChecksumXXHASH64: checksums.XXHASH64, + ChecksumXXHASH3: checksums.XXHASH3, + ChecksumXXHASH128: checksums.XXHASH128, + ChecksumType: checksums.Type, + PartsCount: partsCount, }, nil } @@ -5012,6 +5037,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. ContentLanguage: objMeta.ContentLanguage, CacheControl: objMeta.CacheControl, ExpiresString: objMeta.Expires, + WebsiteRedirectLocation: objMeta.WebsiteRedirectLocation, ETag: &etag, LastModified: backend.GetTimePtr(fi.ModTime()), Metadata: objMeta.Metadata, @@ -5326,17 +5352,26 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput // Store the provided object meta properties err = p.storeObjectMetaProperties(nil, dstBucket, dstObject, metaProperties{ - ContentType: input.ContentType, - ContentEncoding: input.ContentEncoding, - ContentLanguage: input.ContentLanguage, - ContentDisposition: input.ContentDisposition, - CacheControl: input.CacheControl, - Expires: input.Expires, - Metadata: input.Metadata, + ContentType: input.ContentType, + ContentEncoding: input.ContentEncoding, + ContentLanguage: input.ContentLanguage, + ContentDisposition: input.ContentDisposition, + CacheControl: input.CacheControl, + Expires: input.Expires, + WebsiteRedirectLocation: input.WebsiteRedirectLocation, + Metadata: input.Metadata, }) if err != nil { return s3response.CopyObjectOutput{}, err } + // explicitly delete the website redirect location, as if it's not + // provided as CopyObject input, it should not be copied + if getString(input.WebsiteRedirectLocation) == "" { + err := p.meta.DeleteAttribute(dstBucket, dstObject, websiteRedirectHdr) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.CopyObjectOutput{}, fmt.Errorf("delete website-redirect-location: %w", err) + } + } if input.TaggingDirective == types.TaggingDirectiveReplace { tags, err := backend.ParseObjectTags(getString(input.Tagging)) @@ -5375,6 +5410,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput ContentLanguage: input.ContentLanguage, CacheControl: input.CacheControl, Expires: input.Expires, + WebsiteRedirectLocation: input.WebsiteRedirectLocation, Metadata: input.Metadata, ObjectLockRetainUntilDate: input.ObjectLockRetainUntilDate, ObjectLockMode: input.ObjectLockMode, diff --git a/s3api/controllers/bucket-post.go b/s3api/controllers/bucket-post.go index 1fd0a825..1015d8dd 100644 --- a/s3api/controllers/bucket-post.go +++ b/s3api/controllers/bucket-post.go @@ -111,6 +111,7 @@ func (c S3ApiController) POSTObject(ctx *fiber.Ctx) (*Response, error) { contentLanguage := parsed.Fields["content-language"] cacheControl := parsed.Fields["cache-control"] expires := parsed.Fields["expires"] + websiteRedirectLocation := parsed.Fields["x-amz-website-redirect-location"] key := parsed.Fields["key"] @@ -196,29 +197,39 @@ func (c S3ApiController) POSTObject(ctx *fiber.Ctx) (*Response, error) { }, err } + err = utils.ValidateWebsiteRedirectLocation(websiteRedirectLocation) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + res, err := c.be.PutObject(ctx.Context(), s3response.PutObjectInput{ - Bucket: &bucket, - Key: &key, - ContentType: &contentType, - ContentEncoding: &contentEncoding, - ContentDisposition: &contentDisposition, - ContentLanguage: &contentLanguage, - CacheControl: &cacheControl, - Expires: &expires, - Body: parsed.FileRdr, - ContentLength: &parsed.ContentLength, - Tagging: &tagging, - Metadata: metadata, - ChecksumCRC32: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32]), - ChecksumCRC32C: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32c]), - ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]), - ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]), - ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]), - ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]), - ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]), - ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]), - ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]), - ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]), + Bucket: &bucket, + Key: &key, + ContentType: &contentType, + ContentEncoding: &contentEncoding, + ContentDisposition: &contentDisposition, + ContentLanguage: &contentLanguage, + CacheControl: &cacheControl, + Expires: &expires, + WebsiteRedirectLocation: &websiteRedirectLocation, + Body: parsed.FileRdr, + ContentLength: &parsed.ContentLength, + Tagging: &tagging, + Metadata: metadata, + ChecksumCRC32: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32]), + ChecksumCRC32C: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32c]), + ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]), + ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]), + ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]), + ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]), + ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]), + ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]), + ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]), + ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]), }) if err != nil { return &Response{ diff --git a/s3api/controllers/object-get.go b/s3api/controllers/object-get.go index a7575807..07c312ab 100644 --- a/s3api/controllers/object-get.go +++ b/s3api/controllers/object-get.go @@ -551,6 +551,7 @@ func (c S3ApiController) GetObject(ctx *fiber.Ctx) (*Response, error) { "Content-Language": utils.ApplyOverride(res.ContentLanguage, responseOverrides["Content-Language"]), "Cache-Control": utils.ApplyOverride(res.CacheControl, responseOverrides["Cache-Control"]), "Expires": utils.ApplyOverride(res.ExpiresString, responseOverrides["Expires"]), + "x-amz-website-redirect-location": res.WebsiteRedirectLocation, "x-amz-checksum-crc32": res.ChecksumCRC32, "x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME, "x-amz-checksum-crc32c": res.ChecksumCRC32C, diff --git a/s3api/controllers/object-get_test.go b/s3api/controllers/object-get_test.go index 2fdc417f..0808bbde 100644 --- a/s3api/controllers/object-get_test.go +++ b/s3api/controllers/object-get_test.go @@ -794,6 +794,7 @@ func TestS3ApiController_GetObject(t *testing.T) { "Content-Language": nil, "Cache-Control": nil, "Expires": nil, + "x-amz-website-redirect-location": nil, "x-amz-checksum-crc32": nil, "x-amz-checksum-crc64nvme": nil, "x-amz-checksum-crc32c": nil, @@ -844,6 +845,7 @@ func TestS3ApiController_GetObject(t *testing.T) { "Content-Language": nil, "Cache-Control": nil, "Expires": nil, + "x-amz-website-redirect-location": nil, "x-amz-checksum-crc32": nil, "x-amz-checksum-crc64nvme": nil, "x-amz-checksum-crc32c": nil, diff --git a/s3api/controllers/object-head.go b/s3api/controllers/object-head.go index 789c8e8f..de949adc 100644 --- a/s3api/controllers/object-head.go +++ b/s3api/controllers/object-head.go @@ -179,6 +179,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) (*Response, error) { "Content-Length": utils.ConvertPtrToStringPtr(res.ContentLength), "Content-Type": utils.ApplyOverride(res.ContentType, responseOverrides["Content-Type"]), "Expires": utils.ApplyOverride(res.ExpiresString, responseOverrides["Expires"]), + "x-amz-website-redirect-location": res.WebsiteRedirectLocation, "ETag": res.ETag, "Last-Modified": utils.FormatDatePtrToString(res.LastModified, timefmt), "x-amz-restore": res.Restore, diff --git a/s3api/controllers/object-head_test.go b/s3api/controllers/object-head_test.go index ade3d4c4..067b51ec 100644 --- a/s3api/controllers/object-head_test.go +++ b/s3api/controllers/object-head_test.go @@ -184,6 +184,7 @@ func TestS3ApiController_HeadObject(t *testing.T) { "Content-Language": nil, "Cache-Control": nil, "Expires": nil, + "x-amz-website-redirect-location": nil, "x-amz-checksum-crc32": nil, "x-amz-checksum-crc64nvme": nil, "x-amz-checksum-crc32c": nil, @@ -233,6 +234,7 @@ func TestS3ApiController_HeadObject(t *testing.T) { "Content-Language": nil, "Cache-Control": nil, "Expires": nil, + "x-amz-website-redirect-location": nil, "x-amz-checksum-crc32": nil, "x-amz-checksum-crc64nvme": nil, "x-amz-checksum-crc32c": nil, diff --git a/s3api/controllers/object-post.go b/s3api/controllers/object-post.go index 9e767b7b..03180c96 100644 --- a/s3api/controllers/object-post.go +++ b/s3api/controllers/object-post.go @@ -154,6 +154,7 @@ func (c S3ApiController) CreateMultipartUpload(ctx *fiber.Ctx) (*Response, error contentEncoding := ctx.Get("Content-Encoding") tagging := ctx.Get("X-Amz-Tagging") expires := ctx.Get("Expires") + websiteRedirectLocation := ctx.Get("X-Amz-Website-Redirect-Location") legalHoldHdr := ctx.Get("X-Amz-Object-Lock-Legal-Hold") lockModeHdr := ctx.Get("X-Amz-Object-Lock-Mode") objLockDate := ctx.Get("X-Amz-Object-Lock-Retain-Until-Date") @@ -203,6 +204,15 @@ func (c S3ApiController) CreateMultipartUpload(ctx *fiber.Ctx) (*Response, error }, err } + err = utils.ValidateWebsiteRedirectLocation(websiteRedirectLocation) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + objLockState, err := utils.ParsObjectLockHdrs(ctx) if err != nil { return &Response{ @@ -232,6 +242,7 @@ func (c S3ApiController) CreateMultipartUpload(ctx *fiber.Ctx) (*Response, error ContentLanguage: &contentLanguage, CacheControl: &cacheControl, Expires: &expires, + WebsiteRedirectLocation: &websiteRedirectLocation, ObjectLockRetainUntilDate: &objLockState.RetainUntilDate, ObjectLockMode: objLockState.ObjectLockMode, ObjectLockLegalHoldStatus: objLockState.LegalHoldStatus, diff --git a/s3api/controllers/object-put.go b/s3api/controllers/object-put.go index d2335267..a66032a3 100644 --- a/s3api/controllers/object-put.go +++ b/s3api/controllers/object-put.go @@ -494,6 +494,7 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { contentLanguage := ctx.Get("Content-Language") cacheControl := ctx.Get("Cache-Control") expires := ctx.Get("Expires") + websiteRedirectLocation := ctx.Get("X-Amz-Website-Redirect-Location") tagging := ctx.Get("x-amz-tagging") storageClass := ctx.Get("X-Amz-Storage-Class") legalHoldHdr := ctx.Get("X-Amz-Object-Lock-Legal-Hold") @@ -578,6 +579,15 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { }, s3err.GetInvalidArgumentErr(s3err.InvalidArgTaggingDirective, string(taggingDirective)) } + err = utils.ValidateWebsiteRedirectLocation(websiteRedirectLocation) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + checksumAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm")) err = utils.IsChecksumAlgorithmValid(checksumAlgorithm) if err != nil { @@ -618,6 +628,7 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { ContentLanguage: &contentLanguage, CacheControl: &cacheControl, Expires: &expires, + WebsiteRedirectLocation: &websiteRedirectLocation, Tagging: &tagging, TaggingDirective: taggingDirective, CopySource: ©Source, @@ -665,6 +676,7 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) { contentLanguage := ctx.Get("Content-Language") cacheControl := ctx.Get("Cache-Control") expires := ctx.Get("Expires") + websiteRedirectLocation := ctx.Get("X-Amz-Website-Redirect-Location") tagging := ctx.Get("x-amz-tagging") legalHoldHdr := ctx.Get("X-Amz-Object-Lock-Legal-Hold") lockModeHdr := ctx.Get("X-Amz-Object-Lock-Mode") @@ -729,6 +741,15 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) { }, err } + err = utils.ValidateWebsiteRedirectLocation(websiteRedirectLocation) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []types.ObjectIdentifier{{Key: &key}}, true, IsBucketPublic, c.be, true) if err != nil { return &Response{ @@ -787,6 +808,7 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) { ContentLanguage: &contentLanguage, CacheControl: &cacheControl, Expires: &expires, + WebsiteRedirectLocation: &websiteRedirectLocation, Metadata: metadata, Body: body, Tagging: &tagging, diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index 2516a3cb..150b12e9 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -59,6 +59,16 @@ func SetBucketNameValidationStrict(strict bool) { // object metadata combined, excluded the 'x-amz-meta-' prefix const maxMetadataSize = 2048 +func ValidateWebsiteRedirectLocation(location string) error { + if location == "" || strings.HasPrefix(location, "http://") || + strings.HasPrefix(location, "https://") || strings.HasPrefix(location, "/") { + return nil + } + + debuglogger.Logf("invalid website redirect location: %q", location) + return s3err.GetAPIError(s3err.ErrInvalidRedirectLocation) +} + // GetUserMetaData extracts user metadata from headers with the "x-amz-meta-" prefix. // Keys are normalized to lowercase and duplicate headers are merged as // comma-separated values. The total metadata size is validated against the diff --git a/s3err/s3err.go b/s3err/s3err.go index 8aa15e20..b994d23c 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -173,6 +173,7 @@ const ( ErrCORSIsNotEnabled ErrNoSuchWebsiteConfiguration ErrInvalidWebsiteRedirectProtocol + ErrInvalidRedirectLocation ErrBothReplaceKeyAndPrefix ErrMaxMessageLengthExceeded ErrNotModified @@ -660,6 +661,11 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "Invalid protocol, protocol can be http or https. If not defined the protocol will be selected automatically.", HTTPStatusCode: http.StatusBadRequest, }, + ErrInvalidRedirectLocation: { + Code: "InvalidRedirectLocation", + Description: "The website redirect location must have a prefix of 'http://' or 'https://' or '/'.", + HTTPStatusCode: http.StatusBadRequest, + }, ErrBothReplaceKeyAndPrefix: { Code: "InvalidRequest", Description: "You can only define ReplaceKeyPrefix or ReplaceKey but not both.", diff --git a/s3response/website.go b/s3response/website.go index 976a0769..153de9a7 100644 --- a/s3response/website.go +++ b/s3response/website.go @@ -54,7 +54,7 @@ type RedirectAllRequestsTo struct { // RoutingRule specifies a redirect rule with an optional condition. type RoutingRule struct { Condition *RoutingRuleCondition `xml:"Condition,omitempty"` - Redirect Redirect `xml:"Redirect"` + Redirect *Redirect `xml:"Redirect"` } // RoutingRuleCondition specifies when a routing rule applies. @@ -139,10 +139,28 @@ func (c *RoutingRuleCondition) Validate() error { return nil } + if c.HttpErrorCodeReturnedEquals == "" && c.KeyPrefixEquals == "" { + debuglogger.Logf("website routing rule condition is empty") + return s3err.GetAPIError(s3err.ErrMalformedXML) + } + return isValidHTTPCode(c.HttpErrorCodeReturnedEquals, validateErrorCode) } func (r *Redirect) Validate() error { + if r == nil { + return nil + } + + if r.HostName == "" && + r.HttpRedirectCode == "" && + r.Protocol == "" && + r.ReplaceKeyPrefixWith == "" && + r.ReplaceKeyWith == "" { + debuglogger.Logf("website routing rule redirect is empty") + return s3err.GetAPIError(s3err.ErrMalformedXML) + } + if r.ReplaceKeyWith != "" && r.ReplaceKeyPrefixWith != "" { debuglogger.Logf("website redirect has both key replacements") return s3err.GetAPIError(s3err.ErrBothReplaceKeyAndPrefix) diff --git a/s3response/website_test.go b/s3response/website_test.go index a5867e7f..1f3c7aaa 100644 --- a/s3response/website_test.go +++ b/s3response/website_test.go @@ -59,7 +59,7 @@ func TestWebsiteConfiguration_Validate(t *testing.T) { Condition: &RoutingRuleCondition{ KeyPrefixEquals: "docs/", }, - Redirect: Redirect{ + Redirect: &Redirect{ ReplaceKeyPrefixWith: "documents/", }, }, @@ -122,7 +122,7 @@ func TestWebsiteConfiguration_Validate(t *testing.T) { IndexDocument: &IndexDocument{Suffix: "index.html"}, RoutingRules: []RoutingRule{ { - Redirect: Redirect{ + Redirect: &Redirect{ ReplaceKeyWith: "newkey", ReplaceKeyPrefixWith: "newprefix/", }, @@ -132,13 +132,45 @@ func TestWebsiteConfiguration_Validate(t *testing.T) { wantErr: true, errCode: "InvalidRequest", }, + { + name: "routing rule with empty condition", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + RoutingRules: []RoutingRule{ + { + Condition: &RoutingRuleCondition{}, + Redirect: &Redirect{ + HostName: "example.com", + }, + }, + }, + }, + wantErr: true, + errCode: "MalformedXML", + }, + { + name: "routing rule with empty redirect", + config: WebsiteConfiguration{ + IndexDocument: &IndexDocument{Suffix: "index.html"}, + RoutingRules: []RoutingRule{ + { + Condition: &RoutingRuleCondition{ + KeyPrefixEquals: "docs/", + }, + Redirect: &Redirect{}, + }, + }, + }, + wantErr: true, + errCode: "MalformedXML", + }, { name: "routing rule with invalid redirect code", config: WebsiteConfiguration{ IndexDocument: &IndexDocument{Suffix: "index.html"}, RoutingRules: []RoutingRule{ { - Redirect: Redirect{ + Redirect: &Redirect{ HttpRedirectCode: "200", }, }, @@ -153,7 +185,7 @@ func TestWebsiteConfiguration_Validate(t *testing.T) { IndexDocument: &IndexDocument{Suffix: "index.html"}, RoutingRules: []RoutingRule{ { - Redirect: Redirect{ + Redirect: &Redirect{ HttpRedirectCode: "301", HostName: "example.com", }, @@ -203,7 +235,7 @@ func TestWebsiteConfiguration_MatchPrefetchRoutingRuleUsesPrefixOnlyRules(t *tes Condition: &RoutingRuleCondition{ HttpErrorCodeReturnedEquals: "404", }, - Redirect: Redirect{ + Redirect: &Redirect{ HostName: "error.example.com", }, }, @@ -212,7 +244,7 @@ func TestWebsiteConfiguration_MatchPrefetchRoutingRuleUsesPrefixOnlyRules(t *tes KeyPrefixEquals: "old/", HttpErrorCodeReturnedEquals: "404", }, - Redirect: Redirect{ + Redirect: &Redirect{ HostName: "both.example.com", }, }, @@ -220,7 +252,7 @@ func TestWebsiteConfiguration_MatchPrefetchRoutingRuleUsesPrefixOnlyRules(t *tes Condition: &RoutingRuleCondition{ KeyPrefixEquals: "old/", }, - Redirect: Redirect{ + Redirect: &Redirect{ HostName: "prefix.example.com", }, }, diff --git a/tests/integration/CopyObject.go b/tests/integration/CopyObject.go index 0907b29f..90b781a7 100644 --- a/tests/integration/CopyObject.go +++ b/tests/integration/CopyObject.go @@ -666,21 +666,23 @@ func CopyObject_should_copy_meta_props(s *S3Conf) error { cType, cEnc, cDesp, cLang, cLength := "application/json", "base64", "test-desp", "us", int64(100) cacheControl, expires := "no-cache", time.Now().Add(time.Hour*10) + redirectLocation := "/source-redirect" meta := map[string]string{ "foo": "bar", "baz": "quxx", } _, err := putObjectWithData(cLength, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &srcObj, - ContentDisposition: &cDesp, - ContentEncoding: &cEnc, - ContentLanguage: &cLang, - ContentType: &cType, - CacheControl: &cacheControl, - Expires: &expires, - Metadata: meta, + Bucket: &bucket, + Key: &srcObj, + ContentDisposition: &cDesp, + ContentEncoding: &cEnc, + ContentLanguage: &cLang, + ContentType: &cType, + CacheControl: &cacheControl, + Expires: &expires, + WebsiteRedirectLocation: &redirectLocation, + Metadata: meta, }, s3client) if err != nil { return err @@ -697,7 +699,7 @@ func CopyObject_should_copy_meta_props(s *S3Conf) error { return err } - return checkObjectMetaProps(s3client, bucket, dstObj, ObjectMetaProps{ + if err := checkObjectMetaProps(s3client, bucket, dstObj, ObjectMetaProps{ ContentLength: cLength, ContentType: cType, ContentEncoding: cEnc, @@ -706,7 +708,24 @@ func CopyObject_should_copy_meta_props(s *S3Conf) error { CacheControl: cacheControl, ExpiresString: expires.UTC().Format(timefmt), Metadata: meta, + }); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dstObj, }) + cancel() + if err != nil { + return err + } + if got := getString(out.WebsiteRedirectLocation); got != "" { + return fmt.Errorf("expected WebsiteRedirectLocation not to be copied, got %v", got) + } + + return nil }) } @@ -736,6 +755,7 @@ func CopyObject_should_replace_meta_props(s *S3Conf) error { cType, cEnc, cDesp, cLang := "application/binary", "hex", "desp", "mex" cacheControl, expires := "no-cache", time.Now().Add(time.Hour*10) + redirectLocation := "https://example.com/replaced" meta := map[string]string{ "foo": "bar", "baz": "quxx", @@ -744,17 +764,18 @@ func CopyObject_should_replace_meta_props(s *S3Conf) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ - Bucket: &bucket, - Key: &dstObj, - CopySource: getPtr(bucket + "/" + srcObj), - MetadataDirective: types.MetadataDirectiveReplace, - ContentDisposition: &cDesp, - ContentEncoding: &cEnc, - ContentLanguage: &cLang, - ContentType: &cType, - CacheControl: &cacheControl, - Expires: &expires, - Metadata: meta, + Bucket: &bucket, + Key: &dstObj, + CopySource: getPtr(bucket + "/" + srcObj), + MetadataDirective: types.MetadataDirectiveReplace, + ContentDisposition: &cDesp, + ContentEncoding: &cEnc, + ContentLanguage: &cLang, + ContentType: &cType, + CacheControl: &cacheControl, + Expires: &expires, + WebsiteRedirectLocation: &redirectLocation, + Metadata: meta, }) cancel() if err != nil { @@ -762,18 +783,44 @@ func CopyObject_should_replace_meta_props(s *S3Conf) error { } return checkObjectMetaProps(s3client, bucket, dstObj, ObjectMetaProps{ - ContentLength: contentLength, - ContentType: cType, - ContentEncoding: cEnc, - ContentDisposition: cDesp, - ContentLanguage: cLang, - CacheControl: cacheControl, - ExpiresString: expires.UTC().Format(timefmt), - Metadata: meta, + ContentLength: contentLength, + ContentType: cType, + ContentEncoding: cEnc, + ContentDisposition: cDesp, + ContentLanguage: cLang, + CacheControl: cacheControl, + ExpiresString: expires.UTC().Format(timefmt), + WebsiteRedirectLocation: redirectLocation, + Metadata: meta, }) }) } +func CopyObject_invalid_website_redirect_location(s *S3Conf) error { + testName := "CopyObject_invalid_website_redirect_location" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + srcObj, dstObj := "source-object", "dest-object" + + _, err := putObjectWithData(100, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &dstObj, + CopySource: getPtr(bucket + "/" + srcObj), + WebsiteRedirectLocation: getPtr("ftp://example.com"), + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRedirectLocation)) + }) +} + func CopyObject_default_content_type_with_replace_metadata(s *S3Conf) error { testName := "CopyObject_default_content_type_with_replace_metadata" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { diff --git a/tests/integration/CreateMultipartUpload.go b/tests/integration/CreateMultipartUpload.go index e95e2b37..b4771dc0 100644 --- a/tests/integration/CreateMultipartUpload.go +++ b/tests/integration/CreateMultipartUpload.go @@ -67,18 +67,20 @@ func CreateMultipartUpload_with_metadata(s *S3Conf) error { } cType, cEnc, cDesp, cLang := "application/text", "testenc", "testdesp", "sp" cacheControl, expires := "no-cache", time.Now().Add(time.Hour*5) + redirectLocation := "/multipart-redirect" ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - Metadata: meta, - ContentType: &cType, - ContentEncoding: &cEnc, - ContentDisposition: &cDesp, - ContentLanguage: &cLang, - CacheControl: &cacheControl, - Expires: &expires, + Bucket: &bucket, + Key: &obj, + Metadata: meta, + ContentType: &cType, + ContentEncoding: &cEnc, + ContentDisposition: &cDesp, + ContentLanguage: &cLang, + CacheControl: &cacheControl, + Expires: &expires, + WebsiteRedirectLocation: &redirectLocation, }) cancel() if err != nil { @@ -151,11 +153,29 @@ func CreateMultipartUpload_with_metadata(s *S3Conf) error { return fmt.Errorf("expected uploaded object content-encoding to be %v, instead got %v", expires.UTC().Format(timefmt), getString(resp.ExpiresString)) } + if getString(resp.WebsiteRedirectLocation) != redirectLocation { + return fmt.Errorf("expected uploaded object website redirect location to be %v, instead got %v", + redirectLocation, getString(resp.WebsiteRedirectLocation)) + } return nil }) } +func CreateMultipartUpload_invalid_website_redirect_location(s *S3Conf) error { + testName := "CreateMultipartUpload_invalid_website_redirect_location" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + WebsiteRedirectLocation: getPtr("example.com"), + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRedirectLocation)) + }) +} + func CreateMultipartUpload_with_object_lock(s *S3Conf) error { testName := "CreateMultipartUpload_with_object_lock" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { diff --git a/tests/integration/GetObject.go b/tests/integration/GetObject.go index eb6def24..3a3ca7c0 100644 --- a/tests/integration/GetObject.go +++ b/tests/integration/GetObject.go @@ -574,22 +574,24 @@ func GetObject_success(s *S3Conf) error { dataLength, obj := int64(1234567), "my-obj" ctype, cDisp, cEnc, cLang := defaultContentType, "cont-desp", "json", "eng" cacheControl, expires := "cache-ctrl", time.Now().Add(time.Hour*2) + redirectLocation := "/get-object-redirect" meta := map[string]string{ "foo": "bar", "baz": "quxx", } r, err := putObjectWithData(dataLength, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - ContentType: &ctype, - ContentDisposition: &cDisp, - ContentEncoding: &cEnc, - ContentLanguage: &cLang, - Expires: &expires, - CacheControl: &cacheControl, - Metadata: meta, - Tagging: getPtr("key=value&key1=val1"), + Bucket: &bucket, + Key: &obj, + ContentType: &ctype, + ContentDisposition: &cDisp, + ContentEncoding: &cEnc, + ContentLanguage: &cLang, + Expires: &expires, + CacheControl: &cacheControl, + WebsiteRedirectLocation: &redirectLocation, + Metadata: meta, + Tagging: getPtr("key=value&key1=val1"), }, s3client) if err != nil { return err @@ -632,6 +634,10 @@ func GetObject_success(s *S3Conf) error { return fmt.Errorf("expected Cache-Control %v, instead got %v", cacheControl, getString(out.CacheControl)) } + if getString(out.WebsiteRedirectLocation) != redirectLocation { + return fmt.Errorf("expected WebsiteRedirectLocation %v, instead got %v", + redirectLocation, getString(out.WebsiteRedirectLocation)) + } if out.StorageClass != types.StorageClassStandard { return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, out.StorageClass) diff --git a/tests/integration/HeadObject.go b/tests/integration/HeadObject.go index df30d5a4..6fa68af4 100644 --- a/tests/integration/HeadObject.go +++ b/tests/integration/HeadObject.go @@ -569,18 +569,20 @@ func HeadObject_success(s *S3Conf) error { } ctype, cDisp, cEnc, cLang := defaultContentType, "cont-desp", "json", "eng" cacheControl, expires := "cache-ctrl", time.Now().Add(time.Hour*2) + redirectLocation := "/head-object-redirect" _, err := putObjectWithData(dataLen, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - Metadata: meta, - ContentType: &ctype, - ContentDisposition: &cDisp, - ContentEncoding: &cEnc, - ContentLanguage: &cLang, - CacheControl: &cacheControl, - Expires: &expires, - Tagging: getPtr("key=value"), + Bucket: &bucket, + Key: &obj, + Metadata: meta, + ContentType: &ctype, + ContentDisposition: &cDisp, + ContentEncoding: &cEnc, + ContentLanguage: &cLang, + CacheControl: &cacheControl, + Expires: &expires, + WebsiteRedirectLocation: &redirectLocation, + Tagging: getPtr("key=value"), }, s3client) if err != nil { return err @@ -631,6 +633,10 @@ func HeadObject_success(s *S3Conf) error { return fmt.Errorf("expected Cache-Control %v, instead got %v", cacheControl, getString(out.CacheControl)) } + if getString(out.WebsiteRedirectLocation) != redirectLocation { + return fmt.Errorf("expected WebsiteRedirectLocation %v, instead got %v", + redirectLocation, getString(out.WebsiteRedirectLocation)) + } if out.StorageClass != types.StorageClassStandard { return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, out.StorageClass) diff --git a/tests/integration/PostObject.go b/tests/integration/PostObject.go index d020ed6b..fcef5c0b 100644 --- a/tests/integration/PostObject.go +++ b/tests/integration/PostObject.go @@ -887,6 +887,7 @@ func PostObject_success_with_meta_properties(s *S3Conf) error { cLanguage := "en-US" cDisposition := "inline" cEncoding := "gzip" + redirectLocation := "/post-object-redirect" resp, err := sendPostObject(PostRequestConfig{ bucket: bucket, @@ -900,18 +901,20 @@ func PostObject_success_with_meta_properties(s *S3Conf) error { []any{"eq", "$Content-Encoding", cEncoding}, []any{"eq", "$Cache-Control", cacheControl}, []any{"eq", "$Expires", expires}, + []any{"eq", "$x-amz-website-redirect-location", redirectLocation}, []any{"eq", "$x-amz-meta-foo", "bar"}, []any{"eq", "$x-amz-meta-baz", "quxx"}, }, extraFields: map[string]string{ - "Content-Type": cType, - "Cache-Control": cacheControl, - "Expires": expires, - "Content-Language": cLanguage, - "Content-Disposition": cDisposition, - "Content-Encoding": cEncoding, - "x-amz-meta-foo": "bar", - "x-amz-meta-baz": "quxx", + "Content-Type": cType, + "Cache-Control": cacheControl, + "Expires": expires, + "Content-Language": cLanguage, + "Content-Disposition": cDisposition, + "Content-Encoding": cEncoding, + "x-amz-website-redirect-location": redirectLocation, + "x-amz-meta-foo": "bar", + "x-amz-meta-baz": "quxx", }, }) if err != nil { @@ -955,6 +958,10 @@ func PostObject_success_with_meta_properties(s *S3Conf) error { return fmt.Errorf("expected Cache-Control %s, instead got %s", cacheControl, getString(out.CacheControl)) } + if getString(out.WebsiteRedirectLocation) != redirectLocation { + return fmt.Errorf("expected WebsiteRedirectLocation %s, instead got %s", + redirectLocation, getString(out.WebsiteRedirectLocation)) + } expectedMeta := map[string]string{ "foo": "bar", @@ -969,6 +976,30 @@ func PostObject_success_with_meta_properties(s *S3Conf) error { }) } +func PostObject_invalid_website_redirect_location(s *S3Conf) error { + testName := "PostObject_invalid_website_redirect_location" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + redirectLocation := "ftp://example.com" + resp, err := sendPostObject(PostRequestConfig{ + bucket: bucket, + key: "test-object", + s3Conf: s, + fileContent: []byte("data"), + policyConditions: []any{ + []any{"eq", "$x-amz-website-redirect-location", redirectLocation}, + }, + extraFields: map[string]string{ + "x-amz-website-redirect-location": redirectLocation, + }, + }) + if err != nil { + return err + } + + return checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrInvalidRedirectLocation)) + }) +} + func PostObject_invalid_tagging(s *S3Conf) error { testName := "PostObject_invalid_tagging" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { diff --git a/tests/integration/PutBucketWebsite.go b/tests/integration/PutBucketWebsite.go index ed0a09d4..1612866e 100644 --- a/tests/integration/PutBucketWebsite.go +++ b/tests/integration/PutBucketWebsite.go @@ -180,6 +180,56 @@ func PutBucketWebsite_invalid_routing_rule_protocol(s *S3Conf) error { }) } +func PutBucketWebsite_empty_routing_rule_condition(s *S3Conf) error { + testName := "PutBucketWebsite_empty_routing_rule_condition" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + RoutingRules: []types.RoutingRule{ + { + Condition: &types.Condition{}, + Redirect: &types.Redirect{ + HostName: getPtr("example.com"), + }, + }, + }, + }, + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrMalformedXML)) + }) +} + +func PutBucketWebsite_empty_routing_rule_redirect(s *S3Conf) error { + testName := "PutBucketWebsite_empty_routing_rule_redirect" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + RoutingRules: []types.RoutingRule{ + { + Condition: &types.Condition{ + KeyPrefixEquals: getPtr("docs/"), + }, + Redirect: &types.Redirect{}, + }, + }, + }, + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrMalformedXML)) + }) +} + func PutBucketWebsite_empty_error_document_key(s *S3Conf) error { testName := "PutBucketWebsite_empty_error_document_key" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { diff --git a/tests/integration/PutObject.go b/tests/integration/PutObject.go index c0eb48da..f27db548 100644 --- a/tests/integration/PutObject.go +++ b/tests/integration/PutObject.go @@ -629,6 +629,19 @@ func PutObject_with_metadata(s *S3Conf) error { }) } +func PutObject_invalid_website_redirect_location(s *S3Conf) error { + testName := "PutObject_invalid_website_redirect_location" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + _, err := putObjectWithData(10, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + WebsiteRedirectLocation: getPtr("ftp://example.com"), + }, s3client) + return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRedirectLocation)) + }) +} + func PutObject_checksum_algorithm_and_header_mismatch(s *S3Conf) error { testName := "PutObject_checksum_algorithm_and_header_mismatch" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { diff --git a/tests/integration/WebsiteHosting.go b/tests/integration/WebsiteHosting.go index 9d970024..3fde8464 100644 --- a/tests/integration/WebsiteHosting.go +++ b/tests/integration/WebsiteHosting.go @@ -122,6 +122,44 @@ func WebsiteHosting_no_error_document(s *S3Conf) error { }) } +// WebsiteHosting_no_bucket_in_request_location tests that website endpoint +// requests that cannot resolve a bucket still include a useful Location header. +func WebsiteHosting_no_bucket_in_request_location(s *S3Conf) error { + testName := "WebsiteHosting_no_bucket_in_request_location" + return actionHandlerNoSetup(s, testName, func(_ *s3.Client, _ string) error { + _, domain, port := websiteEndpointParts(s) + badHost := "nested.bucket." + domain + baseHost := domain + if port != "" { + badHost = fmt.Sprintf("%s:%s", badHost, port) + baseHost = fmt.Sprintf("%s:%s", baseHost, port) + } + + reqURL, err := websiteAbsoluteURL(s, badHost, "/") + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodGet, reqURL, nil) + if err != nil { + return fmt.Errorf("failed to create website request: %w", err) + } + resp, err := s.httpClient.Do(req) + if err != nil { + return err + } + + wantLocation, err := websiteAbsoluteURL(s, baseHost, "/") + if err != nil { + return err + } + if got := resp.Header.Get("Location"); got != wantLocation { + return fmt.Errorf("expected Location %q, got %q", wantLocation, got) + } + + return checkWebsiteErrorResponse(resp, s3err.GetAPIError(s3err.ErrNoBucketInRequest)) + }) +} + // WebsiteHosting_private_object_and_error_document tests that website hosting // does not serve either the requested object or the configured error document // unless public object access has been granted. @@ -203,7 +241,7 @@ func WebsiteHosting_routing_rule_post_request_redirect(s *S3Conf) error { if got := resp.Header.Get("Location"); got != wantLocation { return fmt.Errorf("expected Location %q, got %q", wantLocation, got) } - return checkWebsiteResponse(resp, http.StatusFound, []byte(http.StatusText(http.StatusFound))) + return checkWebsiteResponse(resp, http.StatusFound, nil) }) } @@ -245,7 +283,7 @@ func WebsiteHosting_routing_rule_pre_request_redirect(s *S3Conf) error { if got := resp.Header.Get("Location"); got != wantLocation { return fmt.Errorf("expected Location %q, got %q", wantLocation, got) } - return checkWebsiteResponse(resp, http.StatusMovedPermanently, []byte(http.StatusText(http.StatusMovedPermanently))) + return checkWebsiteResponse(resp, http.StatusMovedPermanently, nil) }) } @@ -294,7 +332,7 @@ func WebsiteHosting_routing_rule_prefix_and_error_redirect(s *S3Conf) error { if got := resp.Header.Get("Location"); got != wantLocation { return fmt.Errorf("expected Location %q, got %q", wantLocation, got) } - return checkWebsiteResponse(resp, http.StatusTemporaryRedirect, []byte(http.StatusText(http.StatusTemporaryRedirect))) + return checkWebsiteResponse(resp, http.StatusTemporaryRedirect, nil) }) } @@ -373,7 +411,49 @@ func WebsiteHosting_redirect_all_requests(s *S3Conf) error { if got, want := resp.Header.Get("Location"), "https://www.example.com/any/path/here?tracking=1"; got != want { return fmt.Errorf("expected Location %q, got %q", want, got) } - return checkWebsiteResponse(resp, http.StatusMovedPermanently, []byte(http.StatusText(http.StatusMovedPermanently))) + return checkWebsiteResponse(resp, http.StatusMovedPermanently, nil) + }) +} + +// WebsiteHosting_object_redirect_location tests that an object-level website +// redirect emits a 301 with the stored Location after object fetch succeeds. +func WebsiteHosting_object_redirect_location(s *S3Conf) error { + testName := "WebsiteHosting_object_redirect_location" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putBucketWebsiteConfig(s3client, bucket, &types.WebsiteConfiguration{ + IndexDocument: &types.IndexDocument{ + Suffix: getPtr("index.html"), + }, + }) + if err != nil { + return err + } + if err := grantPublicBucketPolicy(s3client, bucket, policyTypeObject); err != nil { + return err + } + + redirectLocation := "/new-page.html" + objectBody := "old" + _, err = putObjectWithData(int64(len(objectBody)), &s3.PutObjectInput{ + Bucket: &bucket, + Key: getPtr("old-page.html"), + Body: strings.NewReader(objectBody), + ContentType: getPtr("text/html"), + WebsiteRedirectLocation: &redirectLocation, + }, s3client) + if err != nil { + return err + } + + resp, err := websiteGet(s, bucket, "old-page.html", nil) + if err != nil { + return err + } + + if got := resp.Header.Get("Location"); got != redirectLocation { + return fmt.Errorf("expected Location %q, got %q", redirectLocation, got) + } + return checkWebsiteResponse(resp, http.StatusMovedPermanently, nil) }) } @@ -521,7 +601,7 @@ func WebsiteHosting_index_error_document_and_routing_rules(s *S3Conf) error { preResp.Body.Close() return fmt.Errorf("expected pre-rule Location %q, got %q", wantPreLocation, got) } - if err := checkWebsiteResponse(preResp, http.StatusMovedPermanently, []byte(http.StatusText(http.StatusMovedPermanently))); err != nil { + if err := checkWebsiteResponse(preResp, http.StatusMovedPermanently, nil); err != nil { return err } @@ -538,7 +618,7 @@ func WebsiteHosting_index_error_document_and_routing_rules(s *S3Conf) error { postResp.Body.Close() return fmt.Errorf("expected post-rule Location %q, got %q", wantPostLocation, got) } - if err := checkWebsiteResponse(postResp, http.StatusFound, []byte(http.StatusText(http.StatusFound))); err != nil { + if err := checkWebsiteResponse(postResp, http.StatusFound, nil); err != nil { return err } @@ -584,7 +664,7 @@ func WebsiteHosting_options_preflight_access_granted(s *S3Conf) error { Origin: "https://client.example", Methods: "GET, HEAD", AllowHeaders: "content-type, x-amz-date", - ExposeHeaders: "Content-Length, ETag", + ExposeHeaders: "Content-Length", MaxAge: "42", AllowCredentials: "true", Vary: "Origin, Access-Control-Request-Headers, Access-Control-Request-Method", diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index b93e2e94..9dc93c73 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -203,6 +203,7 @@ func TestPutObject(ts *TestState) { ts.Run(PutObject_invalid_object_names) ts.Run(PutObject_object_acl_not_supported) ts.Run(PutObject_long_metadata) + ts.Run(PutObject_invalid_website_redirect_location) } func TestHeadObject(ts *TestState) { @@ -370,6 +371,7 @@ func TestCopyObject(ts *TestState) { ts.Run(CopyObject_non_existing_dir_object) ts.Run(CopyObject_should_copy_meta_props) ts.Run(CopyObject_should_replace_meta_props) + ts.Run(CopyObject_invalid_website_redirect_location) ts.Run(CopyObject_default_content_type_with_replace_metadata) ts.Run(CopyObject_missing_bucket_lock) ts.Run(CopyObject_invalid_legal_hold) @@ -420,6 +422,7 @@ func TestCreateMultipartUpload(ts *TestState) { ts.Run(CreateMultipartUpload_non_existing_bucket) ts.Run(CreateMultipartUpload_long_metadata) ts.Run(CreateMultipartUpload_with_metadata) + ts.Run(CreateMultipartUpload_invalid_website_redirect_location) ts.Run(CreateMultipartUpload_with_tagging) ts.Run(CreateMultipartUpload_with_object_lock) ts.Run(CreateMultipartUpload_with_object_lock_not_enabled) @@ -670,6 +673,8 @@ func TestPutBucketWebsite(ts *TestState) { ts.Run(PutBucketWebsite_invalid_redirect_protocol) ts.Run(PutBucketWebsite_redirectAll_index_error_routingRules) ts.Run(PutBucketWebsite_invalid_routing_rule_protocol) + ts.Run(PutBucketWebsite_empty_routing_rule_condition) + ts.Run(PutBucketWebsite_empty_routing_rule_redirect) ts.Run(PutBucketWebsite_empty_error_document_key) ts.Run(PutBucketWebsite_too_many_routing_rules) ts.Run(PutBucketWebsite_routing_rule_replace_key_and_prefix) @@ -696,12 +701,14 @@ func TestWebsiteHosting(ts *TestState) { ts.Run(WebsiteHosting_error_document_served) ts.Run(WebsiteHosting_error_document_not_found) ts.Run(WebsiteHosting_no_error_document) + ts.Run(WebsiteHosting_no_bucket_in_request_location) ts.Run(WebsiteHosting_private_object_and_error_document) ts.Run(WebsiteHosting_routing_rule_post_request_redirect) ts.Run(WebsiteHosting_routing_rule_pre_request_redirect) ts.Run(WebsiteHosting_routing_rule_prefix_and_error_redirect) ts.Run(WebsiteHosting_routing_rule_no_match_serves_error_document) ts.Run(WebsiteHosting_redirect_all_requests) + ts.Run(WebsiteHosting_object_redirect_location) ts.Run(WebsiteHosting_index_document) ts.Run(WebsiteHosting_index_error_document_and_routing_rules) ts.Run(WebsiteHosting_get_cors_headers) @@ -1265,6 +1272,7 @@ func TestPostObject(ts *TestState) { ts.Run(PostObject_success_status_201) ts.Run(PostObject_should_ignore_anything_after_file) ts.Run(PostObject_success_with_meta_properties) + ts.Run(PostObject_invalid_website_redirect_location) ts.Run(PostObject_invalid_tagging) ts.Run(PostObject_success_with_tagging) ts.Run(PostObject_success_double_dash_boundary) @@ -1396,6 +1404,7 @@ func GetIntTests() IntTests { "PutObject_should_combine_metadata": PutObject_should_combine_metadata, "PutObject_long_metadata": PutObject_long_metadata, "PutObject_with_metadata": PutObject_with_metadata, + "PutObject_invalid_website_redirect_location": PutObject_invalid_website_redirect_location, "PutObject_invalid_credentials": PutObject_invalid_credentials, "PutObject_checksum_algorithm_and_header_mismatch": PutObject_checksum_algorithm_and_header_mismatch, "PutObject_multiple_checksum_headers": PutObject_multiple_checksum_headers, @@ -1600,6 +1609,7 @@ func GetIntTests() IntTests { "CopyObject_non_existing_dir_object": CopyObject_non_existing_dir_object, "CopyObject_should_copy_meta_props": CopyObject_should_copy_meta_props, "CopyObject_should_replace_meta_props": CopyObject_should_replace_meta_props, + "CopyObject_invalid_website_redirect_location": CopyObject_invalid_website_redirect_location, "CopyObject_default_content_type_with_replace_metadata": CopyObject_default_content_type_with_replace_metadata, "CopyObject_missing_bucket_lock": CopyObject_missing_bucket_lock, "CopyObject_invalid_legal_hold": CopyObject_invalid_legal_hold, @@ -1634,6 +1644,7 @@ func GetIntTests() IntTests { "CreateMultipartUpload_non_existing_bucket": CreateMultipartUpload_non_existing_bucket, "CreateMultipartUpload_long_metadata": CreateMultipartUpload_long_metadata, "CreateMultipartUpload_with_metadata": CreateMultipartUpload_with_metadata, + "CreateMultipartUpload_invalid_website_redirect_location": CreateMultipartUpload_invalid_website_redirect_location, "CreateMultipartUpload_with_tagging": CreateMultipartUpload_with_tagging, "CreateMultipartUpload_with_object_lock": CreateMultipartUpload_with_object_lock, "CreateMultipartUpload_with_object_lock_not_enabled": CreateMultipartUpload_with_object_lock_not_enabled, @@ -1813,6 +1824,8 @@ func GetIntTests() IntTests { "PutBucketWebsite_invalid_redirect_protocol": PutBucketWebsite_invalid_redirect_protocol, "PutBucketWebsite_redirectAll_index_error_routingRules": PutBucketWebsite_redirectAll_index_error_routingRules, "PutBucketWebsite_invalid_routing_rule_protocol": PutBucketWebsite_invalid_routing_rule_protocol, + "PutBucketWebsite_empty_routing_rule_condition": PutBucketWebsite_empty_routing_rule_condition, + "PutBucketWebsite_empty_routing_rule_redirect": PutBucketWebsite_empty_routing_rule_redirect, "PutBucketWebsite_empty_error_document_key": PutBucketWebsite_empty_error_document_key, "PutBucketWebsite_too_many_routing_rules": PutBucketWebsite_too_many_routing_rules, "PutBucketWebsite_routing_rule_replace_key_and_prefix": PutBucketWebsite_routing_rule_replace_key_and_prefix, @@ -1830,12 +1843,14 @@ func GetIntTests() IntTests { "WebsiteHosting_error_document_served": WebsiteHosting_error_document_served, "WebsiteHosting_error_document_not_found": WebsiteHosting_error_document_not_found, "WebsiteHosting_no_error_document": WebsiteHosting_no_error_document, + "WebsiteHosting_no_bucket_in_request_location": WebsiteHosting_no_bucket_in_request_location, "WebsiteHosting_private_object_and_error_document": WebsiteHosting_private_object_and_error_document, "WebsiteHosting_routing_rule_post_request_redirect": WebsiteHosting_routing_rule_post_request_redirect, "WebsiteHosting_routing_rule_pre_request_redirect": WebsiteHosting_routing_rule_pre_request_redirect, "WebsiteHosting_routing_rule_prefix_and_error_redirect": WebsiteHosting_routing_rule_prefix_and_error_redirect, "WebsiteHosting_routing_rule_no_match_serves_error_document": WebsiteHosting_routing_rule_no_match_serves_error_document, "WebsiteHosting_redirect_all_requests": WebsiteHosting_redirect_all_requests, + "WebsiteHosting_object_redirect_location": WebsiteHosting_object_redirect_location, "WebsiteHosting_index_document": WebsiteHosting_index_document, "WebsiteHosting_index_error_document_and_routing_rules": WebsiteHosting_index_error_document_and_routing_rules, "WebsiteHosting_get_cors_headers": WebsiteHosting_get_cors_headers, @@ -2147,6 +2162,7 @@ func GetIntTests() IntTests { "PostObject_success_status_201": PostObject_success_status_201, "PostObject_should_ignore_anything_after_file": PostObject_should_ignore_anything_after_file, "PostObject_success_with_meta_properties": PostObject_success_with_meta_properties, + "PostObject_invalid_website_redirect_location": PostObject_invalid_website_redirect_location, "PostObject_invalid_tagging": PostObject_invalid_tagging, "PostObject_success_with_tagging": PostObject_success_with_tagging, "PostObject_invalid_checksum_value": PostObject_invalid_checksum_value, diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 83b34c29..2a224342 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -2136,14 +2136,15 @@ func compareDelMarkers(d1, d2 []types.DeleteMarkerEntry) bool { } type ObjectMetaProps struct { - ContentLength int64 - ContentType string - ContentEncoding string - ContentDisposition string - ContentLanguage string - CacheControl string - ExpiresString string - Metadata map[string]string + ContentLength int64 + ContentType string + ContentEncoding string + ContentDisposition string + ContentLanguage string + CacheControl string + ExpiresString string + WebsiteRedirectLocation string + Metadata map[string]string } func checkObjectMetaProps(client *s3.Client, bucket, object string, o ObjectMetaProps) error { @@ -2186,6 +2187,9 @@ func checkObjectMetaProps(client *s3.Client, bucket, object string, o ObjectMeta if o.ExpiresString != "" && getString(out.ExpiresString) != o.ExpiresString { return fmt.Errorf("expected Expires %v, instead got %v", o.ExpiresString, getString(out.ExpiresString)) } + if o.WebsiteRedirectLocation != "" && getString(out.WebsiteRedirectLocation) != o.WebsiteRedirectLocation { + return fmt.Errorf("expected WebsiteRedirectLocation %v, instead got %v", o.WebsiteRedirectLocation, getString(out.WebsiteRedirectLocation)) + } if out.StorageClass != types.StorageClassStandard { return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, out.StorageClass) } diff --git a/website/handler.go b/website/handler.go index 506a6f2f..98c69141 100644 --- a/website/handler.go +++ b/website/handler.go @@ -134,7 +134,7 @@ func registerWebsiteRoutes(app *fiber.App, be backend.Backend, domain string) { func setCORSPreflightHeaders(ctx *fiber.Ctx, allowConfig *auth.CORSAllowanceConfig) { ctx.Set("Access-Control-Allow-Origin", allowConfig.Origin) ctx.Set("Access-Control-Allow-Methods", allowConfig.Methods) - ctx.Set("Access-Control-Expose-Headers", corsExposeHeaders(allowConfig.ExposedHeaders)) + ctx.Set("Access-Control-Expose-Headers", allowConfig.ExposedHeaders) ctx.Set("Access-Control-Allow-Credentials", allowConfig.AllowCredentials) ctx.Set("Access-Control-Allow-Headers", allowConfig.AllowHeaders) ctx.Set("Vary", middlewares.VaryHdr) @@ -143,24 +143,6 @@ func setCORSPreflightHeaders(ctx *fiber.Ctx, allowConfig *auth.CORSAllowanceConf } } -func corsExposeHeaders(exposed string) string { - exposed = strings.TrimSpace(exposed) - if exposed == "" { - return "ETag" - } - if exposed == "*" { - return exposed - } - - for part := range strings.SplitSeq(exposed, ",") { - if strings.EqualFold(strings.TrimSpace(part), "ETag") { - return exposed - } - } - - return exposed + ", ETag" -} - func (c *websiteController) MethodNotAllowed(ctx *fiber.Ctx) error { return sendError(ctx, s3err.GetMethodNotAllowedErr(ctx.Method(), s3err.ResourceTypeObject, websiteAllowedMethods)) } @@ -188,7 +170,7 @@ func (c *websiteController) serve(ctx *fiber.Ctx, readObject websiteObjectReader } if rule := req.config.MatchPrefetchRoutingRule(req.key); rule != nil { - return applyRedirect(ctx, &rule.Redirect, rule.Condition, req.key) + return applyRedirect(ctx, rule.Redirect, rule.Condition, req.key) } resolvedKey := resolveIndexKey(req.key, req.config) @@ -201,7 +183,7 @@ func (c *websiteController) serve(ctx *fiber.Ctx, readObject websiteObjectReader } if rule := req.config.MatchPostErrorRoutingRule(req.key, result.StatusCode); rule != nil { - return applyRedirect(ctx, &rule.Redirect, rule.Condition, req.key) + return applyRedirect(ctx, rule.Redirect, rule.Condition, req.key) } return serveWebsiteResult(ctx, req.bucket, req.config, result, readObject) @@ -213,6 +195,8 @@ func (c *websiteController) resolveRequest(ctx *fiber.Ctx) (*websiteRequestInfo, return nil, err } + fmt.Println(bucket) + key := strings.TrimPrefix(ctx.Path(), "/") if err := validateWebsiteNames(bucket, key); err != nil { return nil, err @@ -260,14 +244,13 @@ func validateWebsiteNames(bucket, key string) error { func (c *websiteController) resolveBucket(ctx *fiber.Ctx) (string, error) { host := ctx.Hostname() if host == "" { + ctx.Set("Location", c.noBucketLocation(ctx, host)) return "", s3err.GetAPIError(s3err.ErrNoBucketInRequest) } // Strip port from host if present. Be careful with IPv6: only strip if the // last colon is not inside brackets. - if idx := strings.LastIndex(host, ":"); idx != -1 && !strings.Contains(host[idx:], "]") { - host = host[:idx] - } + host = stripHostPort(host) if c.domain == "" { return host, nil @@ -286,9 +269,43 @@ func (c *websiteController) resolveBucket(ctx *fiber.Ctx) (string, error) { } } + ctx.Set("Location", c.noBucketLocation(ctx, ctx.Hostname())) return "", s3err.GetAPIError(s3err.ErrNoBucketInRequest) } +func (c *websiteController) noBucketLocation(ctx *fiber.Ctx, host string) string { + locationHost := c.domain + if locationHost == "" { + locationHost = stripHostPort(host) + } + if locationHost == "" { + return "/" + } + if c.domain != "" { + if port := hostPort(host); port != "" { + locationHost += ":" + port + } + } + + return fmt.Sprintf("%s://%s/", ctx.Protocol(), locationHost) +} + +func stripHostPort(host string) string { + if idx := strings.LastIndex(host, ":"); idx != -1 && !strings.Contains(host[idx:], "]") { + return host[:idx] + } + + return host +} + +func hostPort(host string) string { + if idx := strings.LastIndex(host, ":"); idx != -1 && !strings.Contains(host[idx:], "]") { + return host[idx+1:] + } + + return "" +} + type websiteResult struct { Key string StatusCode int @@ -297,9 +314,10 @@ type websiteResult struct { } type websiteObject struct { - Body io.ReadCloser - Headers map[string]*string - Metadata map[string]string + Body io.ReadCloser + Headers map[string]*string + Metadata map[string]string + WebsiteRedirectLocation *string } func resolveIndexKey(key string, config *s3response.WebsiteConfiguration) string { @@ -337,15 +355,16 @@ func (c *websiteController) getObject(ctx *fiber.Ctx, bucket, key string) websit Key: key, StatusCode: http.StatusOK, Object: websiteObject{ - Body: result.Body, - Headers: getObjectHeaders(result), - Metadata: result.Metadata, + Body: result.Body, + Headers: getObjectHeaders(result), + Metadata: result.Metadata, + WebsiteRedirectLocation: result.WebsiteRedirectLocation, }, } } func (c *websiteController) headObject(ctx *fiber.Ctx, bucket, key string) websiteResult { - if err := auth.VerifyPublicAccess(ctx.Context(), c.be, auth.ListBucketAction, auth.PermissionRead, bucket, key); err != nil { + if err := auth.VerifyPublicAccess(ctx.Context(), c.be, auth.GetObjectAction, auth.PermissionRead, bucket, key); err != nil { return websiteResult{ Key: key, StatusCode: statusCodeFromError(err), @@ -369,8 +388,9 @@ func (c *websiteController) headObject(ctx *fiber.Ctx, bucket, key string) websi Key: key, StatusCode: http.StatusOK, Object: websiteObject{ - Headers: headObjectHeaders(result), - Metadata: result.Metadata, + Headers: headObjectHeaders(result), + Metadata: result.Metadata, + WebsiteRedirectLocation: result.WebsiteRedirectLocation, }, } } @@ -388,16 +408,14 @@ func statusCodeFromError(err error) int { func handleRedirectAll(ctx *fiber.Ctx, redirect *s3response.RedirectAllRequestsTo, key string) error { protocol := redirect.Protocol if protocol == "" { - protocol = "http" + protocol = ctx.Protocol() } location := fmt.Sprintf("%s://%s/%s", protocol, redirect.HostName, key) if query := string(ctx.Request().URI().QueryString()); query != "" { location += "?" + query } - ctx.Set("Location", location) - _, _ = utils.EnsureRequestIDs(ctx) - return ctx.SendStatus(http.StatusMovedPermanently) + return sendRedirect(ctx, http.StatusMovedPermanently, location) } // applyRedirect constructs and sends a redirect response from a routing rule. @@ -430,8 +448,14 @@ func applyRedirect(ctx *fiber.Ctx, redirect *s3response.Redirect, condition *s3r if query := string(ctx.Request().URI().QueryString()); query != "" { location += "?" + query } + return sendRedirect(ctx, httpCode, location) +} + +func sendRedirect(ctx *fiber.Ctx, statusCode int, location string) error { ctx.Set("Location", location) - return ctx.SendStatus(httpCode) + _, _ = utils.EnsureRequestIDs(ctx) + ctx.Status(statusCode) + return nil } func getObjectHeaders(result *s3.GetObjectOutput) map[string]*string { @@ -472,6 +496,14 @@ func headObjectHeaders(result *s3.HeadObjectOutput) map[string]*string { func serveWebsiteResult(ctx *fiber.Ctx, bucket string, config *s3response.WebsiteConfiguration, result websiteResult, readObject websiteObjectReader) error { if result.Err == nil { + // Precedence: RedirectAllRequestsTo, pre-fetch routing rules, object + // redirect metadata, then post-error routing/error documents. + if location := backend.GetStringFromPtr(result.Object.WebsiteRedirectLocation); location != "" { + if result.Object.Body != nil { + _ = result.Object.Body.Close() + } + return sendRedirect(ctx, http.StatusMovedPermanently, location) + } return serveObject(ctx, result.Object, http.StatusOK) } diff --git a/website/handler_test.go b/website/handler_test.go index 486de5cc..85a519b0 100644 --- a/website/handler_test.go +++ b/website/handler_test.go @@ -36,13 +36,14 @@ import ( type websiteTestBackend struct { backend.BackendUnsupported - websiteConfig []byte - corsConfig []byte - corsErr error - objects map[string]string - objectErrors map[string]error - public bool - calls []string + websiteConfig []byte + corsConfig []byte + corsErr error + objects map[string]string + objectRedirects map[string]string + objectErrors map[string]error + public bool + calls []string } func (b *websiteTestBackend) record(call string) { @@ -105,9 +106,11 @@ func (b *websiteTestBackend) HeadObject(_ context.Context, input *s3.HeadObjectI length := int64(len(body)) contentType := "text/html" + redirectLocation := redirectPtr(b.objectRedirects[*input.Key]) return &s3.HeadObjectOutput{ - ContentLength: &length, - ContentType: &contentType, + ContentLength: &length, + ContentType: &contentType, + WebsiteRedirectLocation: redirectLocation, }, nil } @@ -126,13 +129,22 @@ func (b *websiteTestBackend) GetObject(_ context.Context, input *s3.GetObjectInp length := int64(len(body)) contentType := "text/html" + redirectLocation := redirectPtr(b.objectRedirects[*input.Key]) return &s3.GetObjectOutput{ - Body: io.NopCloser(strings.NewReader(body)), - ContentLength: &length, - ContentType: &contentType, + Body: io.NopCloser(strings.NewReader(body)), + ContentLength: &length, + ContentType: &contentType, + WebsiteRedirectLocation: redirectLocation, }, nil } +func redirectPtr(location string) *string { + if location == "" { + return nil + } + return &location +} + func TestWebsiteHandlerRoutingRuleOrder(t *testing.T) { tests := []struct { name string @@ -147,7 +159,7 @@ func TestWebsiteHandlerRoutingRuleOrder(t *testing.T) { Condition: &s3response.RoutingRuleCondition{ KeyPrefixEquals: "old/", }, - Redirect: s3response.Redirect{ + Redirect: &s3response.Redirect{ ReplaceKeyPrefixWith: "new/", HttpRedirectCode: "301", }, @@ -156,7 +168,7 @@ func TestWebsiteHandlerRoutingRuleOrder(t *testing.T) { Condition: &s3response.RoutingRuleCondition{ HttpErrorCodeReturnedEquals: "404", }, - Redirect: s3response.Redirect{ + Redirect: &s3response.Redirect{ ReplaceKeyWith: "error.html", HttpRedirectCode: "302", }, @@ -172,7 +184,7 @@ func TestWebsiteHandlerRoutingRuleOrder(t *testing.T) { Condition: &s3response.RoutingRuleCondition{ HttpErrorCodeReturnedEquals: "404", }, - Redirect: s3response.Redirect{ + Redirect: &s3response.Redirect{ ReplaceKeyWith: "error.html", HttpRedirectCode: "302", }, @@ -181,7 +193,7 @@ func TestWebsiteHandlerRoutingRuleOrder(t *testing.T) { Condition: &s3response.RoutingRuleCondition{ KeyPrefixEquals: "old/", }, - Redirect: s3response.Redirect{ + Redirect: &s3response.Redirect{ ReplaceKeyPrefixWith: "new/", HttpRedirectCode: "301", }, @@ -224,7 +236,7 @@ func TestWebsiteHandlerRoutingRuleBothConditions(t *testing.T) { KeyPrefixEquals: "old/", HttpErrorCodeReturnedEquals: "404", }, - Redirect: s3response.Redirect{ + Redirect: &s3response.Redirect{ ReplaceKeyPrefixWith: "new/", HttpRedirectCode: "302", }, @@ -277,6 +289,61 @@ func TestWebsiteHandlerRoutingRuleBothConditions(t *testing.T) { }) } +func TestWebsiteHandlerObjectRedirectLocation(t *testing.T) { + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + }, map[string]string{ + "old.html": "old", + }, true) + be.objectRedirects["old.html"] = "/new.html" + + resp := websiteRequest(t, be, "/old.html") + defer resp.Body.Close() + + if resp.StatusCode != http.StatusMovedPermanently { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusMovedPermanently) + } + if got := resp.Header.Get("Location"); got != "/new.html" { + t.Fatalf("Location = %q, want %q", got, "/new.html") + } + if got := readBody(t, resp); got != "" { + t.Fatalf("body = %q, want empty body", got) + } +} + +func TestWebsiteHandlerPrefetchRoutingPrecedesObjectRedirect(t *testing.T) { + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + RoutingRules: []s3response.RoutingRule{ + { + Condition: &s3response.RoutingRuleCondition{ + KeyPrefixEquals: "old/", + }, + Redirect: &s3response.Redirect{ + ReplaceKeyPrefixWith: "new/", + HttpRedirectCode: "302", + }, + }, + }, + }, map[string]string{ + "old/page.html": "old", + }, true) + be.objectRedirects["old/page.html"] = "/object-redirect.html" + + resp := websiteRequest(t, be, "/old/page.html") + defer resp.Body.Close() + + if resp.StatusCode != http.StatusFound { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusFound) + } + if got := resp.Header.Get("Location"); got != "http://site.test/new/page.html" { + t.Fatalf("Location = %q", got) + } + if containsCall(be.calls, "GetObject") { + t.Fatal("GetObject was called before pre-fetch routing completed") + } +} + func TestWebsiteHandlerRedirectConstruction(t *testing.T) { tests := []struct { name string @@ -290,7 +357,7 @@ func TestWebsiteHandlerRedirectConstruction(t *testing.T) { Condition: &s3response.RoutingRuleCondition{ HttpErrorCodeReturnedEquals: "404", }, - Redirect: s3response.Redirect{ + Redirect: &s3response.Redirect{ ReplaceKeyWith: "error.html", }, }, @@ -303,7 +370,7 @@ func TestWebsiteHandlerRedirectConstruction(t *testing.T) { Condition: &s3response.RoutingRuleCondition{ KeyPrefixEquals: "old/", }, - Redirect: s3response.Redirect{ + Redirect: &s3response.Redirect{ ReplaceKeyPrefixWith: "new/", }, }, @@ -316,7 +383,7 @@ func TestWebsiteHandlerRedirectConstruction(t *testing.T) { Condition: &s3response.RoutingRuleCondition{ KeyPrefixEquals: "old/", }, - Redirect: s3response.Redirect{ + Redirect: &s3response.Redirect{ HostName: "example.com", Protocol: "https", ReplaceKeyPrefixWith: "new/", @@ -331,7 +398,7 @@ func TestWebsiteHandlerRedirectConstruction(t *testing.T) { Condition: &s3response.RoutingRuleCondition{ KeyPrefixEquals: "old/", }, - Redirect: s3response.Redirect{ + Redirect: &s3response.Redirect{ ReplaceKeyPrefixWith: "new/", }, }, @@ -369,7 +436,7 @@ func TestWebsiteHandlerPostErrorRoutingUsesOriginalKeyBeforeIndexExpansion(t *te KeyPrefixEquals: "blog/", HttpErrorCodeReturnedEquals: "404", }, - Redirect: s3response.Redirect{ + Redirect: &s3response.Redirect{ ReplaceKeyPrefixWith: "archive/", HttpRedirectCode: "302", }, @@ -400,7 +467,7 @@ func TestWebsiteHandlerObjectStore5xxBypassesRoutingAndErrorDocument(t *testing. Condition: &s3response.RoutingRuleCondition{ HttpErrorCodeReturnedEquals: "500", }, - Redirect: s3response.Redirect{ + Redirect: &s3response.Redirect{ ReplaceKeyWith: "elsewhere.html", HttpRedirectCode: "302", }, @@ -438,7 +505,7 @@ func TestWebsiteHandlerPublicAccessDeniedPreventsObjectReadAndCanRoute(t *testin Condition: &s3response.RoutingRuleCondition{ HttpErrorCodeReturnedEquals: "403", }, - Redirect: s3response.Redirect{ + Redirect: &s3response.Redirect{ ReplaceKeyWith: "denied.html", HttpRedirectCode: "302", }, @@ -544,6 +611,28 @@ func TestWebsiteHandlerGetValidatesBucketName(t *testing.T) { } } +func TestWebsiteHandlerNoBucketInRequestSetsLocation(t *testing.T) { + be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ + IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, + }, nil, true) + + resp := websiteRequestWithDomainHostAndHeaders(t, be, "site.test", http.MethodGet, "wrong.test:8080", "/", nil) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusMovedPermanently { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusMovedPermanently) + } + if got := resp.Header.Get("Location"); got != "http://site.test:8080/" { + t.Fatalf("Location = %q, want %q", got, "http://site.test:8080/") + } + if got := resp.Header.Get("x-amz-error-code"); got != "WebsiteRedirect" { + t.Fatalf("x-amz-error-code = %q, want %q", got, "WebsiteRedirect") + } + if len(be.calls) != 0 { + t.Fatalf("request without bucket should not call backend, got calls: %v", be.calls) + } +} + func TestWebsiteHandlerHeadValidatesObjectName(t *testing.T) { be := newWebsiteTestBackend(t, s3response.WebsiteConfiguration{ IndexDocument: &s3response.IndexDocument{Suffix: "index.html"}, @@ -707,7 +796,7 @@ func TestWebsiteHandlerOptionsAccessGranted(t *testing.T) { if got := resp.Header.Get("Access-Control-Allow-Headers"); got != "content-type, x-amz-date" { t.Fatalf("Access-Control-Allow-Headers = %q", got) } - if got := resp.Header.Get("Access-Control-Expose-Headers"); got != "Content-Length, ETag" { + if got := resp.Header.Get("Access-Control-Expose-Headers"); got != "Content-Length" { t.Fatalf("Access-Control-Expose-Headers = %q", got) } if got := resp.Header.Get("Access-Control-Max-Age"); got != "42" { @@ -893,9 +982,10 @@ func newWebsiteTestBackend(t *testing.T, config s3response.WebsiteConfiguration, } return &websiteTestBackend{ - websiteConfig: data, - objects: objects, - public: public, + websiteConfig: data, + objects: objects, + objectRedirects: map[string]string{}, + public: public, } } @@ -920,10 +1010,16 @@ func websiteRequestWithHeaders(t *testing.T, be backend.Backend, method, path st func websiteRequestWithHostAndHeaders(t *testing.T, be backend.Backend, method, host, path string, headers map[string]string) *http.Response { t.Helper() + return websiteRequestWithDomainHostAndHeaders(t, be, "", method, host, path, headers) +} + +func websiteRequestWithDomainHostAndHeaders(t *testing.T, be backend.Backend, domain, method, host, path string, headers map[string]string) *http.Response { + t.Helper() + app := fiber.New(fiber.Config{ ServerHeader: "VERSITYGW", }) - registerWebsiteRoutes(app, be, "") + registerWebsiteRoutes(app, be, domain) req := httptest.NewRequest(method, path, nil) req.Host = host