From 8b79fb24de9ba19b35c19f379f5d0151581a798c Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Sun, 28 May 2023 12:10:12 -0700 Subject: [PATCH] update module/import paths to new name, add cli framework --- .gitignore | 1 + backend/backend.go | 2 +- backend/backend_test.go | 2 +- backend/posix/posix.go | 4 +- backend/scoutfs/scoutfs.go | 4 +- backend/walk_test.go | 2 +- cmd/scoutgw/main.go | 18 ---- cmd/versitygw/main.go | 118 ++++++++++++++++++++++++++ cmd/versitygw/posix.go | 39 +++++++++ go.mod | 6 +- go.sum | 8 ++ s3api/controllers/backend_moq_test.go | 2 +- s3api/controllers/base.go | 6 +- s3api/controllers/base_test.go | 4 +- s3api/router.go | 4 +- s3api/router_test.go | 2 +- s3api/server.go | 33 +++++-- s3api/server_test.go | 2 +- 18 files changed, 216 insertions(+), 41 deletions(-) delete mode 100644 cmd/scoutgw/main.go create mode 100644 cmd/versitygw/main.go create mode 100644 cmd/versitygw/posix.go diff --git a/.gitignore b/.gitignore index 6b98c69..20fde38 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ *.dll *.so *.dylib +cmd/versitygw/versitygw # Test binary, built with `go test -c` *.test diff --git a/backend/backend.go b/backend/backend.go index 8cdcec1..02b4f26 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -6,7 +6,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/versity/scoutgw/s3err" + "github.com/versity/versitygw/s3err" ) //go:generate moq -out backend_moq_test.go . Backend diff --git a/backend/backend_test.go b/backend/backend_test.go index ae12be8..dd68d82 100644 --- a/backend/backend_test.go +++ b/backend/backend_test.go @@ -7,7 +7,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/versity/scoutgw/s3err" + "github.com/versity/versitygw/s3err" ) func TestBackend_ListBuckets(t *testing.T) { diff --git a/backend/posix/posix.go b/backend/posix/posix.go index e23f022..807c30e 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -20,8 +20,8 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/google/uuid" "github.com/pkg/xattr" - "github.com/versity/scoutgw/backend" - "github.com/versity/scoutgw/s3err" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3err" ) type Posix struct { diff --git a/backend/scoutfs/scoutfs.go b/backend/scoutfs/scoutfs.go index 43909ff..f32c5dc 100644 --- a/backend/scoutfs/scoutfs.go +++ b/backend/scoutfs/scoutfs.go @@ -1,8 +1,8 @@ package scoutfs import ( - "github.com/versity/scoutgw/backend" - "github.com/versity/scoutgw/backend/posix" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/backend/posix" ) type ScoutFS struct { diff --git a/backend/walk_test.go b/backend/walk_test.go index 075b73c..a25ce4b 100644 --- a/backend/walk_test.go +++ b/backend/walk_test.go @@ -6,7 +6,7 @@ import ( "testing/fstest" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/versity/scoutgw/backend" + "github.com/versity/versitygw/backend" ) type walkTest struct { diff --git a/cmd/scoutgw/main.go b/cmd/scoutgw/main.go deleted file mode 100644 index 5b42a5d..0000000 --- a/cmd/scoutgw/main.go +++ /dev/null @@ -1,18 +0,0 @@ -package main - -import ( - "github.com/gofiber/fiber/v2" - "github.com/versity/scoutgw/backend" - "github.com/versity/scoutgw/s3api" - "log" -) - -func main() { - app := fiber.New(fiber.Config{}) - back := backend.New() - if api, err := s3api.New(app, back, ":7070"); err != nil { - log.Fatalln(err) - } else if err = api.Serve(); err != nil { - log.Fatalln(err) - } -} diff --git a/cmd/versitygw/main.go b/cmd/versitygw/main.go new file mode 100644 index 0000000..c44ea9a --- /dev/null +++ b/cmd/versitygw/main.go @@ -0,0 +1,118 @@ +package main + +import ( + "crypto/tls" + "fmt" + "log" + "os" + + "github.com/gofiber/fiber/v2" + "github.com/urfave/cli/v2" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3api" +) + +var ( + port string + adminAccess string + adminSecret string + region string + certFile, keyFile string +) + +func main() { + app := initApp() + + app.Commands = []*cli.Command{ + posixCommand(), + } + + if err := app.Run(os.Args); err != nil { + log.Fatal(err) + } +} + +func initApp() *cli.App { + return &cli.App{ + Name: "versitygw", + Usage: "Start S3 gateway service with specified backend storage.", + Description: `The S3 gateway is an S3 protocol translator that allows an S3 client +to access the supported backend storage as if it was a native S3 service.`, + Action: func(ctx *cli.Context) error { + return ctx.App.Command("help").Run(ctx) + }, + Flags: initFlags(), + } +} + +func initFlags() []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: "port", + Usage: "gateway listen address : or :", + Value: ":7070", + Destination: &port, + Aliases: []string{"p"}, + }, + &cli.StringFlag{ + Name: "access", + Usage: "admin access account", + Destination: &adminAccess, + EnvVars: []string{"ADMIN_ACCESS_KEY_ID", "ADMIN_ACCESS_KEY"}, + }, + &cli.StringFlag{ + Name: "secret", + Usage: "admin secret access key", + Destination: &adminSecret, + EnvVars: []string{"ADMIN_SECRET_ACCESS_KEY", "ADMIN_SECRET_KEY"}, + }, + &cli.StringFlag{ + Name: "region", + Usage: "s3 region string", + Value: "us-east-1", + Destination: ®ion, + EnvVars: []string{"ADMIN_SECRET_ACCESS_KEY", "ADMIN_SECRET_KEY"}, + }, + &cli.StringFlag{ + Name: "cert", + Usage: "TLS cert file", + Destination: &certFile, + }, + &cli.StringFlag{ + Name: "key", + Usage: "TLS key file", + Destination: &keyFile, + }, + } +} + +func runGateway(be backend.Backend) error { + app := fiber.New(fiber.Config{ + AppName: "versitygw", + ServerHeader: "VERSITYGW", + }) + + var opts []s3api.Option + + if certFile != "" || keyFile != "" { + if certFile == "" { + return fmt.Errorf("TLS key specified without cert file") + } + if keyFile == "" { + return fmt.Errorf("TLS cert specified without key file") + } + + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return fmt.Errorf("tls: load certs: %v", err) + } + opts = append(opts, s3api.WithTLS(cert)) + } + + srv, err := s3api.New(app, be, port, opts...) + if err != nil { + return fmt.Errorf("init gateway: %v", err) + } + + return srv.Serve() +} diff --git a/cmd/versitygw/posix.go b/cmd/versitygw/posix.go new file mode 100644 index 0000000..f604455 --- /dev/null +++ b/cmd/versitygw/posix.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + + "github.com/urfave/cli/v2" + "github.com/versity/versitygw/backend/posix" +) + +func posixCommand() *cli.Command { + return &cli.Command{ + Name: "posix", + Usage: "posix filesystem storage backend", + Description: `Any posix filesystem that supports extended attributes. The top level +directory for the gateway must be provided. All sub directories of the +top level directory are treated as buckets, and all files/directories +below the "bucket directory" are treated as the objects. The object +name is split on "/" separator to translate to posix storage. +For example: +top level: /mnt/fs/gwroot +bucket: mybucket +object: a/b/c/myobject +will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`, + Action: runPosix, + } +} + +func runPosix(ctx *cli.Context) error { + if ctx.NArg() == 0 { + return fmt.Errorf("no directory provided for operation") + } + + be, err := posix.New(ctx.Args().Get(0)) + if err != nil { + return fmt.Errorf("init posix: %v", err) + } + + return runGateway(be) +} diff --git a/go.mod b/go.mod index 3a665e9..d79b4f2 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/versity/scoutgw +module github.com/versity/versitygw go 1.20 @@ -8,6 +8,7 @@ require ( github.com/gofiber/fiber/v2 v2.46.0 github.com/google/uuid v1.3.0 github.com/pkg/xattr v0.4.9 + github.com/urfave/cli/v2 v2.25.4 github.com/valyala/fasthttp v1.47.0 golang.org/x/sys v0.8.0 ) @@ -23,15 +24,18 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2 // indirect github.com/aws/smithy-go v1.13.5 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/klauspost/compress v1.16.5 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/philhofer/fwd v1.1.2 // indirect github.com/rivo/uniseg v0.4.4 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect github.com/tinylib/msgp v1.1.8 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect ) diff --git a/go.sum b/go.sum index ffd65b5..15e448c 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,8 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1 h1:O+9nAy9Bb6bJFTpeNFtd9UfHbgxO1 github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1/go.mod h1:J9kLNzEiHSeGMyN7238EjJmBpCniVzFda75Gxl/NqB8= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gofiber/fiber/v2 v2.46.0 h1:wkkWotblsGVlLjXj2dpgKQAYHtXumsK/HyFugQM68Ns= github.com/gofiber/fiber/v2 v2.46.0/go.mod h1:DNl0/c37WLe0g92U6lx1VMQuxGUQY5V7EIaVoEsUffc= @@ -49,6 +51,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 h1:rmMl4fXJhKMNWl+K+r/fq4FbbKI+Ia2m9hYBLm2h4G4= github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94/go.mod h1:90zrgN3D/WJsDd1iXHT96alCoN2KJo6/4x1DZC3wZs8= github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4= @@ -58,12 +62,16 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw= github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= +github.com/urfave/cli/v2 v2.25.4 h1:HyYwPrTO3im9rYhUff/ZNs78eolxt0nJ4LN+9yJKSH4= +github.com/urfave/cli/v2 v2.25.4/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.47.0 h1:y7moDoxYzMooFpT5aHgNgVOQDrS3qlkfiP9mDtGGK9c= github.com/valyala/fasthttp v1.47.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/s3api/controllers/backend_moq_test.go b/s3api/controllers/backend_moq_test.go index 73d60c1..5dccd89 100644 --- a/s3api/controllers/backend_moq_test.go +++ b/s3api/controllers/backend_moq_test.go @@ -6,7 +6,7 @@ package controllers import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/versity/scoutgw/backend" + "github.com/versity/versitygw/backend" "io" "sync" ) diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index f9007b1..88bf6af 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -13,9 +13,9 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gofiber/fiber/v2" - "github.com/versity/scoutgw/backend" - "github.com/versity/scoutgw/s3api/utils" - "github.com/versity/scoutgw/s3err" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" ) type S3ApiController struct { diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index d3f1012..a045a61 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -10,8 +10,8 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gofiber/fiber/v2" "github.com/valyala/fasthttp" - "github.com/versity/scoutgw/backend" - "github.com/versity/scoutgw/s3err" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3err" ) func TestNew(t *testing.T) { diff --git a/s3api/router.go b/s3api/router.go index a9ac81a..7794135 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -2,8 +2,8 @@ package s3api import ( "github.com/gofiber/fiber/v2" - "github.com/versity/scoutgw/backend" - "github.com/versity/scoutgw/s3api/controllers" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3api/controllers" ) type S3ApiRouter struct{} diff --git a/s3api/router_test.go b/s3api/router_test.go index 5d1262e..d5cb033 100644 --- a/s3api/router_test.go +++ b/s3api/router_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/gofiber/fiber/v2" - "github.com/versity/scoutgw/backend" + "github.com/versity/versitygw/backend" ) func TestS3ApiRouter_Init(t *testing.T) { diff --git a/s3api/server.go b/s3api/server.go index 7ded07c..25b83ab 100644 --- a/s3api/server.go +++ b/s3api/server.go @@ -1,9 +1,11 @@ package s3api import ( + "crypto/tls" + "github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2/middleware/logger" - "github.com/versity/scoutgw/backend" + "github.com/versity/versitygw/backend" ) type S3ApiServer struct { @@ -11,16 +13,37 @@ type S3ApiServer struct { backend backend.Backend router *S3ApiRouter port string + cert *tls.Certificate } -func New(app *fiber.App, be backend.Backend, port string) (s3ApiServer *S3ApiServer, err error) { - s3ApiServer = &S3ApiServer{app, be, new(S3ApiRouter), port} +func New(app *fiber.App, be backend.Backend, port string, opts ...Option) (*S3ApiServer, error) { + server := &S3ApiServer{ + app: app, + backend: be, + router: new(S3ApiRouter), + port: port, + } + + for _, opt := range opts { + opt(server) + } app.Use(logger.New()) - s3ApiServer.router.Init(app, be) - return + server.router.Init(app, be) + return server, nil +} + +// Option sets various options for New() +type Option func(*S3ApiServer) + +// WithTLS sets TLS Credentials +func WithTLS(cert tls.Certificate) Option { + return func(s *S3ApiServer) { s.cert = &cert } } func (sa *S3ApiServer) Serve() (err error) { + if sa.cert != nil { + return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert) + } return sa.app.Listen(sa.port) } diff --git a/s3api/server_test.go b/s3api/server_test.go index a9dd686..e6010e8 100644 --- a/s3api/server_test.go +++ b/s3api/server_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/gofiber/fiber/v2" - "github.com/versity/scoutgw/backend" + "github.com/versity/versitygw/backend" ) func TestNew(t *testing.T) {