Merge pull request #36 from versity/ben/cli

update module/import paths to new name, add cli framework

.gitignore

@@ -7,6 +7,7 @@
 *.dll
 *.so
 *.dylib
+cmd/versitygw/versitygw
 
 # Test binary, built with `go test -c`
 *.test

backend/backend.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/versity/scoutgw/s3err"
+	"github.com/versity/versitygw/s3err"
 )
 
 //go:generate moq -out backend_moq_test.go . Backend

backend/backend_test.go

@@ -7,7 +7,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/versity/scoutgw/s3err"
+	"github.com/versity/versitygw/s3err"
 )
 
 func TestBackend_ListBuckets(t *testing.T) {

backend/posix/posix.go

@@ -20,8 +20,8 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/google/uuid"
 	"github.com/pkg/xattr"
-	"github.com/versity/scoutgw/backend"
-	"github.com/versity/scoutgw/s3err"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
 )
 
 type Posix struct {

backend/scoutfs/scoutfs.go

@@ -1,8 +1,8 @@
 package scoutfs
 
 import (
-	"github.com/versity/scoutgw/backend"
-	"github.com/versity/scoutgw/backend/posix"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/backend/posix"
 )
 
 type ScoutFS struct {

backend/walk_test.go

@@ -6,7 +6,7 @@ import (
 	"testing/fstest"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/versity/scoutgw/backend"
+	"github.com/versity/versitygw/backend"
 )
 
 type walkTest struct {

cmd/scoutgw/main.go

@@ -1,18 +0,0 @@
-package main
-
-import (
-	"github.com/gofiber/fiber/v2"
-	"github.com/versity/scoutgw/backend"
-	"github.com/versity/scoutgw/s3api"
-	"log"
-)
-
-func main() {
-	app := fiber.New(fiber.Config{})
-	back := backend.New()
-	if api, err := s3api.New(app, back, ":7070"); err != nil {
-		log.Fatalln(err)
-	} else if err = api.Serve(); err != nil {
-		log.Fatalln(err)
-	}
-}

cmd/versitygw/main.go

@@ -0,0 +1,118 @@
+package main
+
+import (
+	"crypto/tls"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api"
+)
+
+var (
+	port              string
+	adminAccess       string
+	adminSecret       string
+	region            string
+	certFile, keyFile string
+)
+
+func main() {
+	app := initApp()
+
+	app.Commands = []*cli.Command{
+		posixCommand(),
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func initApp() *cli.App {
+	return &cli.App{
+		Name:  "versitygw",
+		Usage: "Start S3 gateway service with specified backend storage.",
+		Description: `The S3 gateway is an S3 protocol translator that allows an S3 client
+to access the supported backend storage as if it was a native S3 service.`,
+		Action: func(ctx *cli.Context) error {
+			return ctx.App.Command("help").Run(ctx)
+		},
+		Flags: initFlags(),
+	}
+}
+
+func initFlags() []cli.Flag {
+	return []cli.Flag{
+		&cli.StringFlag{
+			Name:        "port",
+			Usage:       "gateway listen address <ip>:<port> or :<port>",
+			Value:       ":7070",
+			Destination: &port,
+			Aliases:     []string{"p"},
+		},
+		&cli.StringFlag{
+			Name:        "access",
+			Usage:       "admin access account",
+			Destination: &adminAccess,
+			EnvVars:     []string{"ADMIN_ACCESS_KEY_ID", "ADMIN_ACCESS_KEY"},
+		},
+		&cli.StringFlag{
+			Name:        "secret",
+			Usage:       "admin secret access key",
+			Destination: &adminSecret,
+			EnvVars:     []string{"ADMIN_SECRET_ACCESS_KEY", "ADMIN_SECRET_KEY"},
+		},
+		&cli.StringFlag{
+			Name:        "region",
+			Usage:       "s3 region string",
+			Value:       "us-east-1",
+			Destination: &region,
+			EnvVars:     []string{"ADMIN_SECRET_ACCESS_KEY", "ADMIN_SECRET_KEY"},
+		},
+		&cli.StringFlag{
+			Name:        "cert",
+			Usage:       "TLS cert file",
+			Destination: &certFile,
+		},
+		&cli.StringFlag{
+			Name:        "key",
+			Usage:       "TLS key file",
+			Destination: &keyFile,
+		},
+	}
+}
+
+func runGateway(be backend.Backend) error {
+	app := fiber.New(fiber.Config{
+		AppName:      "versitygw",
+		ServerHeader: "VERSITYGW",
+	})
+
+	var opts []s3api.Option
+
+	if certFile != "" || keyFile != "" {
+		if certFile == "" {
+			return fmt.Errorf("TLS key specified without cert file")
+		}
+		if keyFile == "" {
+			return fmt.Errorf("TLS cert specified without key file")
+		}
+
+		cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+		if err != nil {
+			return fmt.Errorf("tls: load certs: %v", err)
+		}
+		opts = append(opts, s3api.WithTLS(cert))
+	}
+
+	srv, err := s3api.New(app, be, port, opts...)
+	if err != nil {
+		return fmt.Errorf("init gateway: %v", err)
+	}
+
+	return srv.Serve()
+}

cmd/versitygw/posix.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend/posix"
+)
+
+func posixCommand() *cli.Command {
+	return &cli.Command{
+		Name:  "posix",
+		Usage: "posix filesystem storage backend",
+		Description: `Any posix filesystem that supports extended attributes. The top level
+directory for the gateway must be provided. All sub directories of the
+top level directory are treated as buckets, and all files/directories
+below the "bucket directory" are treated as the objects. The object
+name is split on "/" separator to translate to posix storage.
+For example:
+top level: /mnt/fs/gwroot
+bucket: mybucket
+object: a/b/c/myobject
+will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
+		Action: runPosix,
+	}
+}
+
+func runPosix(ctx *cli.Context) error {
+	if ctx.NArg() == 0 {
+		return fmt.Errorf("no directory provided for operation")
+	}
+
+	be, err := posix.New(ctx.Args().Get(0))
+	if err != nil {
+		return fmt.Errorf("init posix: %v", err)
+	}
+
+	return runGateway(be)
+}

go.mod

@@ -1,4 +1,4 @@
-module github.com/versity/scoutgw
+module github.com/versity/versitygw
 
 go 1.20
 
@@ -8,6 +8,7 @@ require (
 	github.com/gofiber/fiber/v2 v2.46.0
 	github.com/google/uuid v1.3.0
 	github.com/pkg/xattr v0.4.9
+	github.com/urfave/cli/v2 v2.25.4
 	github.com/valyala/fasthttp v1.47.0
 	golang.org/x/sys v0.8.0
 )
@@ -23,15 +24,18 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2 // indirect
 	github.com/aws/smithy-go v1.13.5 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/klauspost/compress v1.16.5 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/philhofer/fwd v1.1.2 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect
 	github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect
 	github.com/tinylib/msgp v1.1.8 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 )

go.sum

@@ -22,6 +22,8 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1 h1:O+9nAy9Bb6bJFTpeNFtd9UfHbgxO1
 github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1/go.mod h1:J9kLNzEiHSeGMyN7238EjJmBpCniVzFda75Gxl/NqB8=
 github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
 github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
+github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/gofiber/fiber/v2 v2.46.0 h1:wkkWotblsGVlLjXj2dpgKQAYHtXumsK/HyFugQM68Ns=
 github.com/gofiber/fiber/v2 v2.46.0/go.mod h1:DNl0/c37WLe0g92U6lx1VMQuxGUQY5V7EIaVoEsUffc=
@@ -49,6 +51,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 h1:rmMl4fXJhKMNWl+K+r/fq4FbbKI+Ia2m9hYBLm2h4G4=
 github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94/go.mod h1:90zrgN3D/WJsDd1iXHT96alCoN2KJo6/4x1DZC3wZs8=
 github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4=
@@ -58,12 +62,16 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
 github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0=
 github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
+github.com/urfave/cli/v2 v2.25.4 h1:HyYwPrTO3im9rYhUff/ZNs78eolxt0nJ4LN+9yJKSH4=
+github.com/urfave/cli/v2 v2.25.4/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.47.0 h1:y7moDoxYzMooFpT5aHgNgVOQDrS3qlkfiP9mDtGGK9c=
 github.com/valyala/fasthttp v1.47.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
+github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
+github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

s3api/controllers/backend_moq_test.go

@@ -6,7 +6,7 @@ package controllers
 import (
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/versity/scoutgw/backend"
+	"github.com/versity/versitygw/backend"
 	"io"
 	"sync"
 )

s3api/controllers/base.go

@@ -13,9 +13,9 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/gofiber/fiber/v2"
-	"github.com/versity/scoutgw/backend"
-	"github.com/versity/scoutgw/s3api/utils"
-	"github.com/versity/scoutgw/s3err"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api/utils"
+	"github.com/versity/versitygw/s3err"
 )
 
 type S3ApiController struct {

s3api/controllers/base_test.go

@@ -10,8 +10,8 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
-	"github.com/versity/scoutgw/backend"
-	"github.com/versity/scoutgw/s3err"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
 )
 
 func TestNew(t *testing.T) {

s3api/router.go

@@ -2,8 +2,8 @@ package s3api
 
 import (
 	"github.com/gofiber/fiber/v2"
-	"github.com/versity/scoutgw/backend"
-	"github.com/versity/scoutgw/s3api/controllers"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api/controllers"
 )
 
 type S3ApiRouter struct{}

s3api/router_test.go

@@ -4,7 +4,7 @@ import (
 	"testing"
 
 	"github.com/gofiber/fiber/v2"
-	"github.com/versity/scoutgw/backend"
+	"github.com/versity/versitygw/backend"
 )
 
 func TestS3ApiRouter_Init(t *testing.T) {

s3api/server.go

@@ -1,9 +1,11 @@
 package s3api
 
 import (
+	"crypto/tls"
+
 	"github.com/gofiber/fiber/v2"
 	"github.com/gofiber/fiber/v2/middleware/logger"
-	"github.com/versity/scoutgw/backend"
+	"github.com/versity/versitygw/backend"
 )
 
 type S3ApiServer struct {
@@ -11,16 +13,37 @@ type S3ApiServer struct {
 	backend backend.Backend
 	router  *S3ApiRouter
 	port    string
+	cert    *tls.Certificate
 }
 
-func New(app *fiber.App, be backend.Backend, port string) (s3ApiServer *S3ApiServer, err error) {
-	s3ApiServer = &S3ApiServer{app, be, new(S3ApiRouter), port}
+func New(app *fiber.App, be backend.Backend, port string, opts ...Option) (*S3ApiServer, error) {
+	server := &S3ApiServer{
+		app:     app,
+		backend: be,
+		router:  new(S3ApiRouter),
+		port:    port,
+	}
+
+	for _, opt := range opts {
+		opt(server)
+	}
 
 	app.Use(logger.New())
-	s3ApiServer.router.Init(app, be)
-	return
+	server.router.Init(app, be)
+	return server, nil
+}
+
+// Option sets various options for New()
+type Option func(*S3ApiServer)
+
+// WithTLS sets TLS Credentials
+func WithTLS(cert tls.Certificate) Option {
+	return func(s *S3ApiServer) { s.cert = &cert }
 }
 
 func (sa *S3ApiServer) Serve() (err error) {
+	if sa.cert != nil {
+		return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)
+	}
 	return sa.app.Listen(sa.port)
 }

s3api/server_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 
 	"github.com/gofiber/fiber/v2"
-	"github.com/versity/scoutgw/backend"
+	"github.com/versity/versitygw/backend"
 )
 
 func TestNew(t *testing.T) {