versitygw/s3api/controllers/base.go

// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package controllers

import (
	"encoding/xml"
	"fmt"
	"net/http"
	"sort"
	"strings"

	"github.com/aws/aws-sdk-go-v2/service/s3/types"
	"github.com/gofiber/fiber/v3"
	"github.com/versity/versitygw/auth"
	"github.com/versity/versitygw/backend"
	"github.com/versity/versitygw/debuglogger"
	"github.com/versity/versitygw/metrics"
	"github.com/versity/versitygw/s3api/utils"
	"github.com/versity/versitygw/s3err"
	"github.com/versity/versitygw/s3event"
	"github.com/versity/versitygw/s3log"
)

type S3ApiController struct {
	be             backend.Backend
	iam            auth.IAMService
	logger         s3log.AuditLogger
	evSender       s3event.S3EventSender
	mm             metrics.Manager
	mpMaxParts     int
	readonly       bool
	disableACL     bool
	virtualDomain  string
	disableSSECTLS bool
}

const (
	// time constants
	iso8601TimeFormatExtended = "Mon Jan _2 15:04:05 2006"
	timefmt                   = "Mon, 02 Jan 2006 15:04:05 GMT"

	maxXMLBodyLen                = 4 * 1024 * 1024
	minPartNumber                = 1
	maxPartNumber                = 10000
	maxWebsiteConfigurationBytes = 131072
	maxDeleteObjects             = 1000

	defaultRegion      = "us-east-1"
	defaultContentType = "binary/octet-stream"
)

var (
	xmlhdr = []byte(`<?xml version="1.0" encoding="UTF-8"?>` + "\n")
)

func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, mm metrics.Manager, readonly, disableACL bool, virtualDomain string, mpMaxParts int, disableSSECTLS ...bool) S3ApiController {
	disableSSECTransportEnforcement := len(disableSSECTLS) > 0 && disableSSECTLS[0]
	return S3ApiController{
		be:             be,
		iam:            iam,
		logger:         logger,
		evSender:       evs,
		readonly:       readonly,
		mm:             mm,
		disableACL:     disableACL,
		virtualDomain:  virtualDomain,
		mpMaxParts:     mpMaxParts,
		disableSSECTLS: disableSSECTransportEnforcement,
	}
}

func (c S3ApiController) parseSSECHeaders(ctx fiber.Ctx) (utils.SSECHeaders, error) {
	h, parseErr := utils.ParseSSECHeaders(ctx)
	if err := utils.ValidateSSEHeaders(h, ctx.Get("X-Amz-Server-Side-Encryption")); err != nil {
		return h, err
	}
	if parseErr != nil {
		return h, parseErr
	}
	if err := c.validateSSECTransport(ctx, h); err != nil {
		return h, err
	}
	return h, nil
}

func (c S3ApiController) parseCopySourceSSECHeaders(ctx fiber.Ctx) (utils.SSECHeaders, error) {
	h, err := utils.ParseCopySourceSSECHeaders(ctx)
	if err != nil {
		return h, err
	}
	if err := c.validateSSECTransport(ctx, h); err != nil {
		return h, err
	}
	return h, nil
}

func (c S3ApiController) parseSSECFields(ctx fiber.Ctx, fields map[string]string) (utils.SSECHeaders, error) {
	h, parseErr := utils.ParseSSECFields(fields)
	if err := utils.ValidateSSEHeaders(h, fields["x-amz-server-side-encryption"]); err != nil {
		return h, err
	}
	if parseErr != nil {
		return h, parseErr
	}
	if err := c.validateSSECTransport(ctx, h); err != nil {
		return h, err
	}
	return h, nil
}

func (c S3ApiController) validateSSECTransport(ctx fiber.Ctx, h utils.SSECHeaders) error {
	if !c.disableSSECTLS && !ctx.Secure() && backend.HasSSEC(h.Algorithm, h.Key, h.KeyMD5) {
		return s3err.GetInvalidArgumentErr(s3err.InvalidArgSSECRequiresTLS, "")
	}
	return nil
}

// verifyAccess wraps auth.VerifyAccess, always injecting the controller's
// own configured IAM backend, readonly mode, and disableACL setting into opts
func (c S3ApiController) verifyAccess(ctx fiber.Ctx, opts auth.AccessOptions) error {
	opts.Iam = c.iam
	opts.Readonly = c.readonly
	opts.DisableACL = c.disableACL
	return auth.VerifyAccess(ctx, c.be, opts)
}

// verifyObjectsAccess wraps auth.VerifyObjectsAccess, for the one request
// shape (DeleteObjects) that names several objects at once. The returned
// slice has one entry per object (nil where it may proceed); the error
// return is a whole-request failure, not about any one object.
func (c S3ApiController) verifyObjectsAccess(ctx fiber.Ctx, opts auth.AccessOptions, objects []types.ObjectIdentifier, bypass auth.BypassMode) ([]error, error) {
	opts.Iam = c.iam
	opts.Readonly = c.readonly
	opts.DisableACL = c.disableACL
	return auth.VerifyObjectsAccess(ctx, c.be, opts, objects, bypass)
}

// verifyObjectCopyAccess wraps auth.VerifyObjectCopyAccess
func (c S3ApiController) verifyObjectCopyAccess(ctx fiber.Ctx, copySource string, opts auth.AccessOptions) error {
	opts.Iam = c.iam
	opts.Readonly = c.readonly
	opts.DisableACL = c.disableACL
	return auth.VerifyObjectCopyAccess(ctx, c.be, copySource, opts)
}

func (c S3ApiController) getAclHeaderValue(ctx fiber.Ctx, key string, defaultValues ...string) string {
	if c.disableACL {
		return ""
	}

	return ctx.Get(key, defaultValues...)
}

func (c S3ApiController) effectiveMpMaxParts() int {
	if c.mpMaxParts > 0 {
		return c.mpMaxParts
	}

	return maxPartNumber
}

// Returns MethodNotAllowed for unmatched routes
func (c S3ApiController) HandleErrorRoute(err error) Controller {
	return func(ctx fiber.Ctx) (*Response, error) {
		return &Response{}, err
	}
}

// MetaOptions holds the metadata for metrics, audit logs and s3 events
type MetaOptions struct {
	ContentLength int64
	BucketOwner   string
	ObjectSize    int64
	ObjectCount   int64
	EventName     s3event.EventType
	ObjectETag    *string
	VersionId     *string
	Status        int
}

// Response is the type definition for a controller response
// Data - Response body
// Headers - Resposne headers
// MetaOpts - Meta options for metrics, audit logs and s3 events
type Response struct {
	Data     any
	Headers  map[string]*string
	MetaOpts *MetaOptions
}

// Services groups the metrics manager, s3 event sender and audit logger
type Services struct {
	Logger         s3log.AuditLogger
	EventSender    s3event.S3EventSender
	MetricsManager metrics.Manager
}

// Controller is the type definition for an s3api controller
type Controller func(ctx fiber.Ctx) (*Response, error)

// ProcessHandlers groups a controller and multiple middlewares into a single fiber handler
func ProcessHandlers(controller Controller, s3action string, svc *Services, handlers ...fiber.Handler) fiber.Handler {
	return func(ctx fiber.Ctx) error {
		// if skip locals is set, skip to the next rout handler
		if utils.ContextKeySkip.IsSet(ctx) {
			utils.ContextKeySkip.Delete(ctx)
			return ctx.Next()
		}

		for _, handler := range handlers {
			err := handler(ctx)
			if err != nil {
				return ProcessController(ctx, func(ctx fiber.Ctx) (*Response, error) {
					return &Response{
						MetaOpts: &MetaOptions{},
					}, err
				}, s3action, svc)
			}
		}

		return ProcessController(ctx, controller, s3action, svc)
	}
}

// WrapMiddleware executes the given middleware and handles sending the audit logs
// and metrics. It also handles the error parsing
func WrapMiddleware(handler fiber.Handler, logger s3log.AuditLogger, mm metrics.Manager) fiber.Handler {
	return func(ctx fiber.Ctx) error {
		requestID, hostID := utils.EnsureRequestIDs(ctx)

		err := handler(ctx)
		if err != nil {
			if mm != nil {
				mm.Send(ctx, err, metrics.ActionUndetected, 0, 0)
			}
			if logger != nil {
				logger.Log(ctx, err, ctx.BodyRaw(), s3log.LogMeta{
					Action: metrics.ActionUndetected,
				})
			}

			ctx.Response().Header.SetContentType(fiber.MIMEApplicationXML)

			if serr, ok := err.(s3err.S3Error); ok {
				if mnaErr, ok := serr.(s3err.MethodNotAllowedError); ok && len(mnaErr.AllowedMethods) != 0 {
					// for MethodNotAllowed errors, set the 'Allow' header
					ctx.Response().Header.Set("Allow", mnaErr.AllowedMethodsString())
				}
				// for signing region mismatches, set the
				// 'x-amz-bucket-region' header
				utils.SetRegionMismatchHeader(ctx, serr)
				return ctx.Status(serr.StatusCode()).Send(serr.XMLBody(requestID, hostID))
			}

			debuglogger.InternalError(err)
			ctx.Status(http.StatusInternalServerError)

			// If the error is not 's3err.S3Error' return 'InternalError'
			return ctx.Send(s3err.GetAPIError(s3err.ErrInternalError).XMLBody(requestID, hostID))
		}

		return ctx.Next()
	}
}

// ProcessController executes the given s3api controller and handles the metrics
// access logs and s3 events
func ProcessController(ctx fiber.Ctx, controller Controller, s3action string, svc *Services) error {
	response, err := controller(ctx)

	// Set the response headers
	SetResponseHeaders(ctx, response.Headers)
	requestID, hostID := utils.EnsureRequestIDs(ctx)
	ensureExposeMetaHeaders(ctx)

	opts := response.MetaOpts
	if opts == nil {
		opts = &MetaOptions{}
	}
	// Send the metrics
	if svc.MetricsManager != nil {
		if opts.ObjectCount > 0 {
			svc.MetricsManager.Send(ctx, err, s3action, opts.ObjectCount, opts.Status)
		} else {
			svc.MetricsManager.Send(ctx, err, s3action, opts.ContentLength, opts.Status)
		}
	}
	// Handle the error case
	if err != nil {
		// Audit the error log
		if svc.Logger != nil {
			svc.Logger.Log(ctx, err, nil, s3log.LogMeta{
				Action:      s3action,
				BucketOwner: opts.BucketOwner,
				ObjectSize:  opts.ObjectSize,
			})
		}

		// set content type to application/xml
		ctx.Response().Header.SetContentType(fiber.MIMEApplicationXML)

		if serr, ok := err.(s3err.S3Error); ok {
			if mnaErr, ok := serr.(s3err.MethodNotAllowedError); ok && len(mnaErr.AllowedMethods) != 0 {
				ctx.Response().Header.Set("Allow", mnaErr.AllowedMethodsString())
			}
			utils.SetRegionMismatchHeader(ctx, serr)
			return ctx.Status(serr.StatusCode()).Send(serr.XMLBody(requestID, hostID))
		}

		debuglogger.InternalError(err)

		// If the error is not 's3err.S3Error' return 'InternalError'
		return ctx.Status(http.StatusInternalServerError).Send(s3err.GetAPIError(s3err.ErrInternalError).XMLBody(requestID, hostID))
	}

	// At this point, the S3 action has succeeded in the backend and
	// the event has already occurred. This means the S3 event must be sent,
	// even if unexpected issues arise while further parsing the response payload.
	if svc.EventSender != nil && opts.EventName != "" {
		svc.EventSender.SendEvent(ctx, s3event.EventMeta{
			BucketOwner: opts.BucketOwner,
			ObjectSize:  opts.ObjectSize,
			ObjectETag:  opts.ObjectETag,
			VersionId:   opts.VersionId,
			EventName:   opts.EventName,
		})
	}

	if opts.Status == 0 {
		opts.Status = http.StatusOK
	}

	// if no data payload is provided, send the response status
	if response.Data == nil {
		if svc.Logger != nil {
			svc.Logger.Log(ctx, nil, []byte{}, s3log.LogMeta{
				Action:      s3action,
				BucketOwner: opts.BucketOwner,
				ObjectSize:  opts.ObjectSize,
			})
		}
		ctx.Status(opts.Status)
		return nil
	}

	var responseBytes []byte

	// Handle already encoded responses(text, json...)
	encodedResp, ok := response.Data.([]byte)
	if ok {
		responseBytes = encodedResp
	} else {
		if responseBytes, err = xml.Marshal(response.Data); err != nil {
			debuglogger.InternalError(err)
			if svc.Logger != nil {
				svc.Logger.Log(ctx, err, nil, s3log.LogMeta{
					Action:      s3action,
					BucketOwner: opts.BucketOwner,
					ObjectSize:  opts.ObjectSize,
				})
			}

			err := s3err.GetAPIError(s3err.ErrInternalError)
			return ctx.Status(err.HTTPStatusCode).Send(err.XMLBody(requestID, hostID))
		}

		if len(responseBytes) > 0 {
			ctx.Response().Header.SetContentType(fiber.MIMEApplicationXML)
		}
	}

	if ok {
		if len(responseBytes) > 0 {
			ctx.Response().Header.Set("Content-Length", fmt.Sprint(len(responseBytes)))
		}

		if svc.Logger != nil {
			svc.Logger.Log(ctx, nil, responseBytes, s3log.LogMeta{
				Action:      s3action,
				BucketOwner: opts.BucketOwner,
				ObjectSize:  opts.ObjectSize,
			})
		}

		return ctx.Status(opts.Status).Send(responseBytes)
	}

	msglen := len(xmlhdr) + len(responseBytes)
	if msglen > maxXMLBodyLen {
		debuglogger.Logf("XML encoded body len %v exceeds max len %v",
			msglen, maxXMLBodyLen)
		if svc.Logger != nil {
			svc.Logger.Log(ctx, err, []byte{}, s3log.LogMeta{
				Action:      s3action,
				BucketOwner: opts.BucketOwner,
				ObjectSize:  opts.ObjectSize,
			})
		}

		// set content type to application/xml
		ctx.Response().Header.SetContentType(fiber.MIMEApplicationXML)

		err := s3err.GetAPIError(s3err.ErrInternalError)
		return ctx.Status(err.HTTPStatusCode).Send(err.XMLBody(requestID, hostID))
	}
	res := make([]byte, 0, msglen)
	res = append(res, xmlhdr...)
	res = append(res, responseBytes...)

	// Set the Content-Length header
	ctx.Response().Header.SetContentLength(msglen)

	if svc.Logger != nil {
		svc.Logger.Log(ctx, nil, responseBytes, s3log.LogMeta{
			Action:      s3action,
			BucketOwner: opts.BucketOwner,
			ObjectSize:  opts.ObjectSize,
		})
	}

	return ctx.Status(opts.Status).Send(res)
}

func ensureExposeMetaHeaders(ctx fiber.Ctx) {
	// Only attempt to modify expose headers when CORS is actually in use.
	if len(ctx.Response().Header.Peek("Access-Control-Allow-Origin")) == 0 {
		return
	}

	existing := strings.TrimSpace(string(ctx.Response().Header.Peek("Access-Control-Expose-Headers")))
	if existing == "*" {
		return
	}

	lowerExisting := map[string]struct{}{}
	if existing != "" {
		for part := range strings.SplitSeq(existing, ",") {
			p := strings.ToLower(strings.TrimSpace(part))
			if p != "" {
				lowerExisting[p] = struct{}{}
			}
		}
	}

	metaNames := map[string]struct{}{}
	for k := range ctx.Response().Header.All() {
		key := string(k)
		if strings.HasPrefix(strings.ToLower(key), "x-amz-meta-") {
			metaNames[key] = struct{}{}
		}
	}
	if len(metaNames) == 0 {
		// Still ensure ETag is present if any expose headers exist/are needed.
		if _, ok := lowerExisting["etag"]; ok {
			return
		}
		if existing == "" {
			ctx.Response().Header.Set("Access-Control-Expose-Headers", "ETag")
			return
		}
		ctx.Response().Header.Set("Access-Control-Expose-Headers", existing+", ETag")
		return
	}

	metaList := make([]string, 0, len(metaNames))
	for k := range metaNames {
		metaList = append(metaList, k)
	}
	sort.Strings(metaList)

	toAdd := make([]string, 0, 1+len(metaList))
	if _, ok := lowerExisting["etag"]; !ok {
		toAdd = append(toAdd, "ETag")
		lowerExisting["etag"] = struct{}{}
	}
	for _, h := range metaList {
		lh := strings.ToLower(h)
		if _, ok := lowerExisting[lh]; ok {
			continue
		}
		toAdd = append(toAdd, h)
		lowerExisting[lh] = struct{}{}
	}
	if len(toAdd) == 0 {
		return
	}

	if existing == "" {
		ctx.Response().Header.Set("Access-Control-Expose-Headers", strings.Join(toAdd, ", "))
		return
	}
	ctx.Response().Header.Set("Access-Control-Expose-Headers", existing+", "+strings.Join(toAdd, ", "))
}

// Sets the response headers
func SetResponseHeaders(ctx fiber.Ctx, headers map[string]*string) {
	if headers == nil {
		return
	}

	ctx.Response().Header.DisableNormalizing()
	for key, val := range headers {
		if val == nil || *val == "" {
			continue
		}
		ctx.Response().Header.Add(key, *val)
	}
}