From 160a99cbbd4ea829d5c3d227f65fc2f0a6bb2a4d Mon Sep 17 00:00:00 2001 From: Jon Austin <62040422+jonaustin09@users.noreply.github.com> Date: Fri, 9 Jun 2023 10:30:20 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20Added=20admin=20CLI,=20created=20api=20?= =?UTF-8?q?endpoint=20for=20creating=20new=20user,=20cr=E2=80=A6=20(#68)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Added admin CLI, created api endpoint for creating new user, created action for admin CLI to create a new user, changed the authentication middleware to verify the users from db * feat: Added both single and multi user support, added caching layer for getting IAM users * fix: Added all the files --- backend/auth/iam.go | 108 ++++++++++++++++++++- cmd/versitygw/admin.go | 145 ++++++++++++++++++++++++++++ cmd/versitygw/main.go | 31 +++--- s3api/controllers/admin.go | 49 ++++++++++ s3api/middlewares/authentication.go | 51 +++++----- s3api/router.go | 7 +- s3api/router_test.go | 5 +- s3api/server.go | 6 +- s3api/server_test.go | 20 ++-- s3api/utils/utils_test.go | 3 - 10 files changed, 363 insertions(+), 62 deletions(-) create mode 100644 cmd/versitygw/admin.go create mode 100644 s3api/controllers/admin.go diff --git a/backend/auth/iam.go b/backend/auth/iam.go index 835f11e..3d9b909 100644 --- a/backend/auth/iam.go +++ b/backend/auth/iam.go @@ -14,24 +14,124 @@ package auth -import "github.com/versity/versitygw/s3err" +import ( + "encoding/json" + "fmt" + "os" + "sync" + + "github.com/versity/versitygw/s3err" +) + +type Account struct { + Secret string `json:"secret"` + Role string `json:"role"` + Region string `json:"region"` +} type IAMConfig struct { - AccessAccounts map[string]string + AccessAccounts map[string]Account `json:"accessAccounts"` +} + +type AccountsCache struct { + mu sync.Mutex + Accounts map[string]Account +} + +func (c *AccountsCache) getAccount(access string) *Account { + c.mu.Lock() + defer c.mu.Unlock() + + acc, ok := c.Accounts[access] + if !ok { + return nil + } + + return &acc +} + +func (c *AccountsCache) updateAccounts() error { + c.mu.Lock() + defer c.mu.Unlock() + + var data IAMConfig + + file, err := os.ReadFile("users.json") + if err != nil { + return fmt.Errorf("error reading config file: %w", err) + } + + if err := json.Unmarshal(file, &data); err != nil { + return fmt.Errorf("error parsing the data: %w", err) + } + + c.Accounts = data.AccessAccounts + + return nil } type IAMService interface { GetIAMConfig() (*IAMConfig, error) + CreateAccount(access string, account *Account) error + GetUserAccount(access string) *Account } -type IAMServiceUnsupported struct{} +type IAMServiceUnsupported struct { + accCache *AccountsCache +} var _ IAMService = &IAMServiceUnsupported{} func New() IAMService { - return &IAMServiceUnsupported{} + return &IAMServiceUnsupported{accCache: &AccountsCache{Accounts: map[string]Account{}}} } func (IAMServiceUnsupported) GetIAMConfig() (*IAMConfig, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } + +func (s IAMServiceUnsupported) CreateAccount(access string, account *Account) error { + var data IAMConfig + + file, err := os.ReadFile("users.json") + if err != nil { + data = IAMConfig{AccessAccounts: map[string]Account{ + access: *account, + }} + } else { + if err := json.Unmarshal(file, &data); err != nil { + return err + } + + _, ok := data.AccessAccounts[access] + if ok { + return fmt.Errorf("user with the given access already exists") + } + + data.AccessAccounts[access] = *account + } + + updatedJSON, err := json.MarshalIndent(data, "", " ") + if err != nil { + return err + } + + if err := os.WriteFile("users.json", updatedJSON, 0644); err != nil { + return err + } + return nil +} + +func (s IAMServiceUnsupported) GetUserAccount(access string) *Account { + acc := s.accCache.getAccount(access) + if acc == nil { + err := s.accCache.updateAccounts() + if err != nil { + return nil + } + + return s.accCache.getAccount(access) + } + + return acc +} diff --git a/cmd/versitygw/admin.go b/cmd/versitygw/admin.go new file mode 100644 index 0000000..4a699cb --- /dev/null +++ b/cmd/versitygw/admin.go @@ -0,0 +1,145 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/urfave/cli/v2" +) + +var ( + adminAccess string + adminSecret string + adminRegion string +) + +func adminCommand() *cli.Command { + return &cli.Command{ + Name: "admin", + Usage: "admin CLI tool", + Description: `admin CLI tool for interacting with admin api. + Here is the available api list: + create-user + `, + Subcommands: []*cli.Command{ + { + Name: "create-user", + Usage: "Create a new user", + Action: createUser, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "access", + Usage: "access value for the new user", + Required: true, + Aliases: []string{"a"}, + }, + &cli.StringFlag{ + Name: "secret", + Usage: "secret value for the new user", + Required: true, + Aliases: []string{"s"}, + }, + &cli.StringFlag{ + Name: "role", + Usage: "role for the new user", + Required: true, + Aliases: []string{"r"}, + }, + &cli.StringFlag{ + Name: "region", + Usage: "s3 region string for the user", + Value: "us-east-1", + Aliases: []string{"rg"}, + }, + }, + }, + }, + Flags: []cli.Flag{ + // TODO: create a configuration file for this + &cli.StringFlag{ + Name: "adminAccess", + Usage: "admin access account", + EnvVars: []string{"ADMIN_ACCESS_KEY_ID", "ADMIN_ACCESS_KEY"}, + Aliases: []string{"aa"}, + Destination: &adminAccess, + }, + &cli.StringFlag{ + Name: "adminSecret", + Usage: "admin secret access key", + EnvVars: []string{"ADMIN_SECRET_ACCESS_KEY", "ADMIN_SECRET_KEY"}, + Aliases: []string{"as"}, + Destination: &adminSecret, + }, + &cli.StringFlag{ + Name: "adminRegion", + Usage: "s3 region string", + Value: "us-east-1", + Destination: &adminRegion, + Aliases: []string{"ar"}, + }, + }, + } +} + +func createUser(ctx *cli.Context) error { + access, secret, role, region := ctx.String("access"), ctx.String("secret"), ctx.String("role"), ctx.String("region") + if access == "" || secret == "" || region == "" { + return fmt.Errorf("invalid input parameters for the new user") + } + if role != "admin" && role != "user" { + return fmt.Errorf("invalid input parameter for role") + } + + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:7070/create-user?access=%v&secret=%v&role=%v®ion=%v", access, secret, role, region), nil) + if err != nil { + return fmt.Errorf("failed to send the request: %w", err) + } + + signer := v4.NewSigner() + + hashedPayload := sha256.Sum256([]byte{}) + hexPayload := hex.EncodeToString(hashedPayload[:]) + + req.Header.Set("X-Amz-Content-Sha256", hexPayload) + + signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now()) + if signErr != nil { + return fmt.Errorf("failed to sign the request: %w", err) + } + + client := http.Client{} + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to send the request: %w", err) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + fmt.Printf("%s", body) + + return nil +} diff --git a/cmd/versitygw/main.go b/cmd/versitygw/main.go index 28ef013..95e2814 100644 --- a/cmd/versitygw/main.go +++ b/cmd/versitygw/main.go @@ -30,8 +30,8 @@ import ( var ( port string - adminAccess string - adminSecret string + rootUserAccess string + rootUserSecret string region string certFile, keyFile string debug bool @@ -51,6 +51,7 @@ func main() { app.Commands = []*cli.Command{ posixCommand(), + adminCommand(), } if err := app.Run(os.Args); err != nil { @@ -94,21 +95,24 @@ func initFlags() []cli.Flag { }, &cli.StringFlag{ Name: "access", - Usage: "admin access account", - Destination: &adminAccess, - EnvVars: []string{"ADMIN_ACCESS_KEY_ID", "ADMIN_ACCESS_KEY"}, + Usage: "root user access key", + EnvVars: []string{"ROOT_ACCESS_KEY_ID", "ROOT_ACCESS_KEY"}, + Aliases: []string{"a"}, + Destination: &rootUserAccess, }, &cli.StringFlag{ Name: "secret", - Usage: "admin secret access key", - Destination: &adminSecret, - EnvVars: []string{"ADMIN_SECRET_ACCESS_KEY", "ADMIN_SECRET_KEY"}, + Usage: "root user secret access key", + EnvVars: []string{"ROOT_SECRET_ACCESS_KEY", "ROOT_SECRET_KEY"}, + Aliases: []string{"s"}, + Destination: &rootUserSecret, }, &cli.StringFlag{ Name: "region", Usage: "s3 region string", Value: "us-east-1", Destination: ®ion, + Aliases: []string{"r"}, }, &cli.StringFlag{ Name: "cert", @@ -155,12 +159,11 @@ func runGateway(be backend.Backend) error { opts = append(opts, s3api.WithDebug()) } - srv, err := s3api.New(app, be, port, - middlewares.AdminConfig{ - AdminAccess: adminAccess, - AdminSecret: adminSecret, - Region: region, - }, auth.IAMServiceUnsupported{}, opts...) + srv, err := s3api.New(app, be, middlewares.RootUserConfig{ + Access: rootUserAccess, + Secret: rootUserSecret, + Region: region, + }, port, auth.New(), opts...) if err != nil { return fmt.Errorf("init gateway: %v", err) } diff --git a/s3api/controllers/admin.go b/s3api/controllers/admin.go new file mode 100644 index 0000000..0abbabf --- /dev/null +++ b/s3api/controllers/admin.go @@ -0,0 +1,49 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "fmt" + + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/backend/auth" +) + +type AdminController struct { + IAMService auth.IAMService +} + +func NewAdminController() AdminController { + return AdminController{IAMService: auth.New()} +} + +func (c AdminController) CreateUser(ctx *fiber.Ctx) error { + access, secret, role, region := ctx.Query("access"), ctx.Query("secret"), ctx.Query("role"), ctx.Query("region") + requesterRole := ctx.Locals("role") + + if requesterRole != "admin" { + return fmt.Errorf("access denied: only admin users have access to this resource") + } + + user := auth.Account{Secret: secret, Role: role, Region: region} + + err := c.IAMService.CreateAccount(access, &user) + if err != nil { + return fmt.Errorf("failed to create a user: %w", err) + } + + ctx.SendString("The user has been created successfully") + return nil +} diff --git a/s3api/middlewares/authentication.go b/s3api/middlewares/authentication.go index 10ede7f..4fcf629 100644 --- a/s3api/middlewares/authentication.go +++ b/s3api/middlewares/authentication.go @@ -35,17 +35,14 @@ const ( iso8601Format = "20060102T150405Z" ) -type AdminConfig struct { - AdminAccess string - AdminSecret string - Region string +type RootUserConfig struct { + Access string + Secret string + Region string } -func VerifyV4Signature(config AdminConfig, iam auth.IAMService, debug bool) fiber.Handler { - acct := accounts{ - admin: config, - iam: iam, - } +func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, debug bool) fiber.Handler { + acct := accounts{root: root, iam: iam} return func(ctx *fiber.Ctx) error { authorization := ctx.Get("Authorization") @@ -77,8 +74,8 @@ func VerifyV4Signature(config AdminConfig, iam auth.IAMService, debug bool) fibe } signedHdrs := strings.Split(signHdrKv[1], ";") - secret, ok := acct.getAcctSecret(creds[0]) - if !ok { + account := acct.getAccount(creds[0]) + if account == nil { return controllers.Responce[any](ctx, nil, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID)) } @@ -115,8 +112,8 @@ func VerifyV4Signature(config AdminConfig, iam auth.IAMService, debug bool) fibe signErr := signer.SignHTTP(req.Context(), aws.Credentials{ AccessKeyID: creds[0], - SecretAccessKey: secret, - }, req, hexPayload, creds[3], config.Region, tdate, func(options *v4.SignerOptions) { + SecretAccessKey: account.Secret, + }, req, hexPayload, creds[3], account.Region, tdate, func(options *v4.SignerOptions) { if debug { options.LogSigning = true options.Logger = logging.NewStandardLogger(os.Stderr) @@ -137,25 +134,27 @@ func VerifyV4Signature(config AdminConfig, iam auth.IAMService, debug bool) fibe return controllers.Responce[any](ctx, nil, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)) } + ctx.Locals("role", account.Role) + return ctx.Next() } } type accounts struct { - admin AdminConfig - iam auth.IAMService + root RootUserConfig + iam auth.IAMService } -func (a accounts) getAcctSecret(access string) (string, bool) { - if a.admin.AdminAccess == access { - return a.admin.AdminSecret, true +func (a accounts) getAccount(access string) *auth.Account { + var account *auth.Account + if access == a.root.Access { + account = &auth.Account{ + Secret: a.root.Secret, + Role: "admin", + Region: a.root.Region, + } + } else { + account = a.iam.GetUserAccount(access) } - - conf, err := a.iam.GetIAMConfig() - if err != nil { - return "", false - } - - secret, ok := conf.AccessAccounts[access] - return secret, ok + return account } diff --git a/s3api/router.go b/s3api/router.go index d8485ab..637052f 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -17,13 +17,18 @@ package s3api import ( "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/backend/auth" "github.com/versity/versitygw/s3api/controllers" ) type S3ApiRouter struct{} -func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend) { +func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) { s3ApiController := controllers.New(be) + adminController := controllers.AdminController{IAMService: iam} + + // TODO: think of better routing system + app.Post("/create-user", adminController.CreateUser) // ListBuckets action app.Get("/", s3ApiController.ListBuckets) diff --git a/s3api/router_test.go b/s3api/router_test.go index 4d2d22e..2b14dd9 100644 --- a/s3api/router_test.go +++ b/s3api/router_test.go @@ -19,12 +19,14 @@ import ( "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/backend/auth" ) func TestS3ApiRouter_Init(t *testing.T) { type args struct { app *fiber.App be backend.Backend + iam auth.IAMService } tests := []struct { name string @@ -37,12 +39,13 @@ func TestS3ApiRouter_Init(t *testing.T) { args: args{ app: fiber.New(), be: backend.BackendUnsupported{}, + iam: auth.IAMServiceUnsupported{}, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - tt.sa.Init(tt.args.app, tt.args.be) + tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam) }) } } diff --git a/s3api/server.go b/s3api/server.go index d1dcbfb..1588dab 100644 --- a/s3api/server.go +++ b/s3api/server.go @@ -33,7 +33,7 @@ type S3ApiServer struct { debug bool } -func New(app *fiber.App, be backend.Backend, port string, adminUser middlewares.AdminConfig, iam auth.IAMService, opts ...Option) (*S3ApiServer, error) { +func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port string, iam auth.IAMService, opts ...Option) (*S3ApiServer, error) { server := &S3ApiServer{ app: app, backend: be, @@ -45,10 +45,10 @@ func New(app *fiber.App, be backend.Backend, port string, adminUser middlewares. opt(server) } - app.Use(middlewares.VerifyV4Signature(adminUser, iam, server.debug)) + app.Use(middlewares.VerifyV4Signature(root, iam, server.debug)) app.Use(logger.New()) app.Use(middlewares.VerifyMD5Body()) - server.router.Init(app, be) + server.router.Init(app, be, iam) return server, nil } diff --git a/s3api/server_test.go b/s3api/server_test.go index b1686fc..ea6aa1c 100644 --- a/s3api/server_test.go +++ b/s3api/server_test.go @@ -26,10 +26,10 @@ import ( func TestNew(t *testing.T) { type args struct { - app *fiber.App - be backend.Backend - port string - adminUser middlewares.AdminConfig + app *fiber.App + be backend.Backend + port string + root middlewares.RootUserConfig } app := fiber.New() @@ -46,10 +46,10 @@ func TestNew(t *testing.T) { { name: "Create S3 api server", args: args{ - app: app, - be: be, - port: port, - adminUser: middlewares.AdminConfig{}, + app: app, + be: be, + port: port, + root: middlewares.RootUserConfig{}, }, wantS3ApiServer: &S3ApiServer{ app: app, @@ -62,8 +62,8 @@ func TestNew(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotS3ApiServer, err := New(tt.args.app, tt.args.be, - tt.args.port, tt.args.adminUser, auth.IAMServiceUnsupported{}) + gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.root, + tt.args.port, auth.IAMServiceUnsupported{}) if (err != nil) != tt.wantErr { t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/s3api/utils/utils_test.go b/s3api/utils/utils_test.go index 701dca9..08df618 100644 --- a/s3api/utils/utils_test.go +++ b/s3api/utils/utils_test.go @@ -2,7 +2,6 @@ package utils import ( "bytes" - "fmt" "net/http" "reflect" "testing" @@ -62,8 +61,6 @@ func TestCreateHttpRequestFromCtx(t *testing.T) { return } - fmt.Println(got.Header, tt.want.Header) - if !reflect.DeepEqual(got.Header, tt.want.Header) { t.Errorf("CreateHttpRequestFromCtx() got = %v, want %v", got, tt.want) }