feat: add AWS-compatible standalone IAM service

Closes #1640 Add a standalone AWS IAM Query API implementation for managing IAM users through standard AWS SDKs and the AWS CLI. Server usage Start the IAM server with internal file-backed storage: mkdir -p /tmp/versitygw-iam ./versitygw --port 127.0.0.1:7070 --access user --secret pass iam --dir /tmp/versitygw-iam Start the IAM server with Vault KV v2 storage using AppRole: VGW_IAM_VAULT_ROLE_SECRET=<role-secret> ./versitygw --port 127.0.0.1:7070 --access user --secret pass iam --vault-endpoint-url http://127.0.0.1:8200 --vault-auth-method approle --vault-role-id <role-id> --vault-mount-path kv --vault-secret-storage-path iam Vault authentication also supports root tokens, separate authentication and secret-storage namespaces, custom mount paths, server certificate validation, and mutual TLS client certificates. Configure the AWS CLI credentials used by the IAM server: export AWS_ACCESS_KEY_ID=user export AWS_SECRET_ACCESS_KEY=pass export AWS_DEFAULT_REGION=us-east-1 Implemented IAM actions CreateUser creates an IAM user with an AWS-compatible ARN, generated AIDA user ID, creation timestamp, optional path, and tags. It validates usernames, paths, tag limits, reserved tag prefixes, duplicate tag keys, and existing users. aws --endpoint-url http://127.0.0.1:7070 iam create-user --user-name bob aws --endpoint-url http://127.0.0.1:7070 iam create-user --user-name bob --path /engineering/ --tags Key=team,Value=storage GetUser returns a stored user or the root identity when requested without a username through the IAM Query API. aws --endpoint-url http://127.0.0.1:7070 iam get-user --user-name bob ListUsers returns users in deterministic username order and supports path filtering, marker-based pagination, and MaxItems limits. aws --endpoint-url http://127.0.0.1:7070 iam list-users aws --endpoint-url http://127.0.0.1:7070 iam list-users --path-prefix /engineering/ --max-items 100 UpdateUser updates the username and/or path, recalculates the user ARN, and rejects conflicts with existing users. aws --endpoint-url http://127.0.0.1:7070 iam update-user --user-name bob --new-user-name robert --new-path /platform/ DeleteUser permanently removes an IAM user and returns AWS-compatible errors for missing users. aws --endpoint-url http://127.0.0.1:7070 iam delete-user --user-name robert IAM protocol and authentication - Support the AWS IAM Query protocol version 2010-05-08 over GET and POST form requests. - Return AWS-compatible XML responses, error documents, status codes, request IDs, user metadata, and pagination fields. - Authenticate root credentials with AWS Signature Version 4 for the IAM service in us-east-1. - Support both Authorization-header and query-string SigV4 authentication. - Validate credential scope, signed headers, timestamps, clock skew, content length, signatures, and unsupported signature or session-token modes. - Add IAM-specific validation and error mapping for malformed requests, invalid actions, duplicate entities, missing users, throttling, and internal failures. Storage implementations - Add an internal JSON-backed store using iam.json and iam.json.backup with atomic temporary-file replacement, concurrent access protection, stable ordering, pagination, and persistence across restarts. - Add a Vault KV v2 store with one secret per user, CAS-based duplicate protection, permanent deletion, AppRole reauthentication, namespace support, configurable authentication and KV mounts, root-token authentication, and TLS/mTLS configuration. - Introduce a common Storer interface and require exactly one storage backend to be configured. Server and embedding support - Register the new `versitygw iam` command with environment-variable and CLI configuration for both storage backends. - Add `embedgw.RunIAMAPI` and `IAMConfig` for embedding the IAM service in Go applications. Gateway-level internal packages - Add `internal/iamstore` as a reusable generic file-backed IAM persistence engine and migrate the existing gateway internal IAM service to it. - Add `internal/sigv4auth` for shared SigV4 header and presigned-query parsing, canonical request generation, signature verification, and structured authentication errors. - Refactor the S3 authentication paths to use the shared SigV4 implementation while preserving S3-specific error responses. - Add `internal/httpctx` for shared Fiber context keys and AWS-style request ID handling. - Add `internal/routekit` for shared query, form, and header route matchers. - Add `internal/netutil` for reusable certificate storage, hostname-aware listeners, multi-address serving, TLS listeners, and UNIX socket handling. - Update the custom SigV4 signer to honor an explicitly supplied signed-header list so unrelated headers do not alter IAM signatures. Testing and CI - Add AWS IAM SDK-based integration coverage for all supported user actions, header authentication, query authentication, validation, errors, filtering, and pagination. - Split standalone IAM tests into `versitygw test iam` and retain existing gateway IAM tests under `versitygw test gw-iam`. - Add unit coverage for controllers, authentication, routing, storage, embedding, listeners, request matching, persistence, and signing behavior. - Add `runiamtests.sh` to exercise internal storage over HTTP and HTTPS plus Vault storage through AppRole. - Add a dedicated IAM functional-test workflow with a Vault service and merged runtime coverage reporting. - Include the IAM test runner in shellcheck and add the AWS IAM SDK dependency.
2026-09-29 11:15:36 +00:00 · 2026-08-25 01:03:12 +04:00
parent 762e244b43
commit e012a6fd01
60 changed files with 9108 additions and 629 deletions
@@ -0,0 +1,406 @@
+// Copyright 2026 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sigv4auth
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/smithy-go/logging"
+	"github.com/gofiber/fiber/v3"
+	"github.com/versity/versitygw/aws/signer/v4"
+	"github.com/versity/versitygw/debuglogger"
+)
+
+const (
+	AlgorithmECDSAP256SHA256 = "AWS4-ECDSA-P256-SHA256"
+
+	QueryAlgorithm     = "X-Amz-Algorithm"
+	QueryCredential    = "X-Amz-Credential"
+	QueryDate          = "X-Amz-Date"
+	QueryExpires       = "X-Amz-Expires"
+	QuerySignedHeaders = "X-Amz-SignedHeaders"
+	QuerySignature     = "X-Amz-Signature"
+	QuerySecurityToken = "X-Amz-Security-Token"
+
+	maxQueryExpirationSeconds = 604800
+)
+
+type QueryErrorKind string
+
+const (
+	ErrQueryMissingRequiredParams QueryErrorKind = "missing_required_query_parameters"
+	ErrQueryUnsupportedAlgorithm  QueryErrorKind = "unsupported_query_algorithm"
+	ErrQueryUnsupportedECDSA      QueryErrorKind = "unsupported_query_ecdsa"
+	ErrQueryInvalidDateFormat     QueryErrorKind = "invalid_query_date_format"
+	ErrQueryDateMismatch          QueryErrorKind = "query_date_mismatch"
+	ErrQueryIncorrectRegion       QueryErrorKind = "query_incorrect_region"
+	ErrQueryExpiresNumber         QueryErrorKind = "query_expires_number"
+	ErrQueryExpiresNegative       QueryErrorKind = "query_expires_negative"
+	ErrQueryExpiresTooLarge       QueryErrorKind = "query_expires_too_large"
+	ErrQueryExpired               QueryErrorKind = "query_expired"
+	ErrQuerySecurityToken         QueryErrorKind = "query_security_token"
+)
+
+type QueryError struct {
+	Kind       QueryErrorKind
+	Value      string
+	Expected   string
+	Actual     string
+	Expires    int
+	ExpiresAt  time.Time
+	ServerTime time.Time
+}
+
+func (e *QueryError) Error() string {
+	if e == nil {
+		return ""
+	}
+	switch e.Kind {
+	case ErrQueryIncorrectRegion:
+		return fmt.Sprintf("sigv4 query %s: expected %q, got %q", e.Kind, e.Expected, e.Actual)
+	case ErrQueryDateMismatch:
+		return fmt.Sprintf("sigv4 query %s: expected %q, got %q", e.Kind, e.Expected, e.Actual)
+	case ErrQueryExpired:
+		return fmt.Sprintf("sigv4 query %s: expired at %s", e.Kind, e.ExpiresAt.Format(time.RFC3339))
+	case ErrQueryUnsupportedAlgorithm, ErrQueryUnsupportedECDSA, ErrQueryExpiresNumber:
+		return fmt.Sprintf("sigv4 query %s: %q", e.Kind, e.Value)
+	default:
+		return string(e.Kind)
+	}
+}
+
+type QueryAuthOptions struct {
+	Service string
+	Region  string
+	// RequireExpiration enables the X-Amz-Expires validation required by S3
+	// presigned URLs. Other SigV4 query-auth services, including IAM, leave it
+	// disabled.
+	RequireExpiration bool
+	Now               func() time.Time
+}
+
+type QueryAuthDetails struct {
+	SigningTime time.Time
+	Expires     int
+	ExpiresAt   time.Time
+	ServerTime  time.Time
+}
+
+// ParseQueryAuthorization parses and validates AWS SigV4 query-string
+// authentication parameters. The credential scope service must match
+// opts.Service. If opts.Region is set, the credential scope region must match
+// it as well.
+func ParseQueryAuthorization(ctx fiber.Ctx, opts QueryAuthOptions) (AuthData, QueryAuthDetails, error) {
+	a := AuthData{}
+	details := QueryAuthDetails{}
+
+	if err := ValidateQueryAlgorithm(ctx.Query(QueryAlgorithm)); err != nil {
+		return a, details, err
+	}
+
+	credsQuery := ctx.Query(QueryCredential)
+	if credsQuery == "" {
+		return a, details, missingQueryParameterError(QueryCredential)
+	}
+
+	creds, err := ParseCredentials(credsQuery, opts.Service)
+	if err != nil {
+		return a, details, err
+	}
+
+	if opts.Region != "" && creds.Region != opts.Region {
+		return a, details, &QueryError{
+			Kind:     ErrQueryIncorrectRegion,
+			Expected: opts.Region,
+			Actual:   creds.Region,
+		}
+	}
+
+	date := ctx.Query(QueryDate)
+	if date == "" {
+		return a, details, missingQueryParameterError(QueryDate)
+	}
+
+	tdate, err := time.Parse(ISO8601Format, date)
+	if err != nil {
+		return a, details, &QueryError{Kind: ErrQueryInvalidDateFormat, Value: date}
+	}
+
+	if date[:8] != creds.Date {
+		return a, details, &QueryError{
+			Kind:     ErrQueryDateMismatch,
+			Expected: creds.Date,
+			Actual:   date[:8],
+		}
+	}
+
+	signature := ctx.Query(QuerySignature)
+	if signature == "" {
+		return a, details, missingQueryParameterError(QuerySignature)
+	}
+
+	signedHdrs := ctx.Query(QuerySignedHeaders)
+	if signedHdrs == "" {
+		return a, details, missingQueryParameterError(QuerySignedHeaders)
+	}
+
+	expiration := QueryExpiration{}
+	if opts.RequireExpiration {
+		now := time.Now().UTC()
+		if opts.Now != nil {
+			now = opts.Now().UTC()
+		}
+		expiration, err = ValidateQueryExpiration(ctx.Query(QueryExpires), tdate, now)
+		if err != nil {
+			return a, details, err
+		}
+	}
+
+	a = AuthData{
+		Algorithm:     ctx.Query(QueryAlgorithm),
+		Access:        creds.Access,
+		Region:        creds.Region,
+		Service:       creds.Service,
+		SignedHeaders: signedHdrs,
+		Signature:     signature,
+		Date:          date,
+	}
+	details = QueryAuthDetails{
+		SigningTime: tdate,
+		Expires:     expiration.Expires,
+		ExpiresAt:   expiration.ExpiresAt,
+		ServerTime:  expiration.ServerTime,
+	}
+
+	return a, details, nil
+}
+
+func ValidateQueryAlgorithm(algo string) error {
+	switch algo {
+	case "":
+		return missingQueryParameterError(QueryAlgorithm)
+	case AlgorithmHMACSHA256:
+		return nil
+	case AlgorithmECDSAP256SHA256:
+		return &QueryError{Kind: ErrQueryUnsupportedECDSA, Value: algo}
+	default:
+		return &QueryError{Kind: ErrQueryUnsupportedAlgorithm, Value: algo}
+	}
+}
+
+type QueryExpiration struct {
+	Expires    int
+	ExpiresAt  time.Time
+	ServerTime time.Time
+}
+
+func ValidateQueryExpiration(str string, date, now time.Time) (QueryExpiration, error) {
+	if str == "" {
+		return QueryExpiration{}, missingQueryParameterError(QueryExpires)
+	}
+
+	exp, err := strconv.Atoi(str)
+	if err != nil {
+		return QueryExpiration{}, &QueryError{Kind: ErrQueryExpiresNumber, Value: str}
+	}
+
+	if exp < 0 {
+		return QueryExpiration{}, &QueryError{Kind: ErrQueryExpiresNegative, Value: str}
+	}
+
+	if exp > maxQueryExpirationSeconds {
+		return QueryExpiration{}, &QueryError{Kind: ErrQueryExpiresTooLarge, Value: str}
+	}
+
+	now = now.UTC()
+	expiresAt := date.Add(time.Duration(exp) * time.Second)
+	expiration := QueryExpiration{
+		Expires:    exp,
+		ExpiresAt:  expiresAt,
+		ServerTime: now,
+	}
+
+	if expiresAt.Before(now) {
+		return expiration, &QueryError{
+			Kind:       ErrQueryExpired,
+			Expires:    exp,
+			ExpiresAt:  expiresAt,
+			ServerTime: now,
+		}
+	}
+
+	return expiration, nil
+}
+
+func missingQueryParameterError(parameter string) *QueryError {
+	return &QueryError{Kind: ErrQueryMissingRequiredParams, Value: parameter}
+}
+
+// CheckQuerySignature rebuilds a SigV4 query-auth request and compares the
+// generated query signature to the signature presented by the client.
+func CheckQuerySignature(ctx fiber.Ctx, auth AuthData, secret, payloadHash string, tdate time.Time, contentLen int64, opts CheckOptions) (*CheckResult, error) {
+	service := opts.Service
+	if service == "" {
+		service = auth.Service
+	}
+	signedHdrs := strings.Split(auth.SignedHeaders, ";")
+
+	req, err := createPresignedHTTPRequestFromCtx(ctx, signedHdrs, contentLen, opts.RequiredSignedHeaders)
+	if err != nil {
+		return nil, err
+	}
+
+	signer := v4.NewSigner()
+	uri, _, signMeta, err := signer.PresignHTTP(ctx.RequestCtx(),
+		aws.Credentials{
+			AccessKeyID:     auth.Access,
+			SecretAccessKey: secret,
+		},
+		req, payloadHash, service, auth.Region, tdate, signedHdrs,
+		func(options *v4.SignerOptions) {
+			options.DisableURIPathEscaping = opts.DisableURIPathEscaping
+			if debuglogger.IsDebugEnabled() {
+				options.LogSigning = true
+				options.Logger = logging.NewStandardLogger(os.Stderr)
+			}
+		})
+	if err != nil {
+		return nil, fmt.Errorf("presign generated http request: %w", err)
+	}
+
+	urlParts, err := url.Parse(uri)
+	if err != nil {
+		return nil, fmt.Errorf("parse presigned url: %w", err)
+	}
+
+	signature := urlParts.Query().Get(QuerySignature)
+	if signature != auth.Signature {
+		return nil, &SignatureMismatchError{
+			AccessKeyID:           auth.Access,
+			StringToSign:          signMeta.StringToSign,
+			SignatureProvided:     auth.Signature,
+			StringToSignBytes:     HexBytes(signMeta.StringToSign),
+			CanonicalRequest:      signMeta.CanonicalString,
+			CanonicalRequestBytes: HexBytes(signMeta.CanonicalString),
+		}
+	}
+
+	return &CheckResult{
+		CanonicalString: signMeta.CanonicalString,
+		StringToSign:    signMeta.StringToSign,
+	}, nil
+}
+
+var generatedQueryAuthParams = map[string]struct{}{
+	QueryAlgorithm:     {},
+	QueryCredential:    {},
+	QueryDate:          {},
+	QuerySignedHeaders: {},
+	QuerySignature:     {},
+}
+
+func createPresignedHTTPRequestFromCtx(ctx fiber.Ctx, signedHdrs []string, contentLength int64, requiredSignedHdrs []string) (*http.Request, error) {
+	req := ctx.Request()
+	if err := validateRequiredSignedHeaders(signedHdrs, requiredSignedHdrs); err != nil {
+		return nil, err
+	}
+
+	uri, _, _ := strings.Cut(ctx.OriginalURL(), "?")
+	query := strings.Builder{}
+
+	for key, value := range ctx.Request().URI().QueryArgs().All() {
+		keyStr := string(key)
+		if _, ok := generatedQueryAuthParams[keyStr]; ok {
+			continue
+		}
+
+		if query.Len() > 0 {
+			query.WriteByte('&')
+		}
+		query.WriteString(url.QueryEscape(keyStr))
+		query.WriteByte('=')
+		query.WriteString(url.QueryEscape(string(value)))
+	}
+
+	if query.Len() > 0 {
+		uri += "?" + query.String()
+	}
+
+	httpReq, err := http.NewRequest(string(req.Header.Method()), uri, nil)
+	if err != nil {
+		return nil, errors.New("error in creating an http request")
+	}
+	if err := addRequestHeadersFromCtx(ctx, httpReq, signedHdrs, requiredSignedHdrs); err != nil {
+		return nil, err
+	}
+
+	if !includeHeader("Content-Length", signedHdrs) {
+		httpReq.ContentLength = 0
+	} else {
+		httpReq.ContentLength = contentLength
+	}
+
+	httpReq.Host = string(req.Header.Host())
+
+	return httpReq, nil
+}
+
+// IsQueryAuth determines if a request uses SigV4 query-string auth.
+func IsQueryAuth(ctx fiber.Ctx) bool {
+	algo := ctx.Query(QueryAlgorithm)
+	creds := ctx.Query(QueryCredential)
+	date := ctx.Query(QueryDate)
+	signature := ctx.Query(QuerySignature)
+	signedHeaders := ctx.Query(QuerySignedHeaders)
+
+	return !allEmpty(algo, creds, date, signature, signedHeaders)
+}
+
+// IsQueryAuthV2 determines if a request is query-string signed with the legacy
+// AWS Signature Version 2 signer.
+func IsQueryAuthV2(ctx fiber.Ctx) bool {
+	expires := ctx.Query("Expires")
+	access := ctx.Query("AWSAccessKeyId")
+	signature := ctx.Query("Signature")
+
+	return anyNonEmpty(expires, access, signature)
+}
+
+func allEmpty(args ...string) bool {
+	for _, a := range args {
+		if a != "" {
+			return false
+		}
+	}
+
+	return true
+}
+
+func anyNonEmpty(args ...string) bool {
+	for _, a := range args {
+		if a != "" {
+			return true
+		}
+	}
+
+	return false
+}