diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index cc0092a..249c746 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -23,5 +23,16 @@ jobs: run: | go get -v -t -d ./... + - name: Build + run: go build -o versitygw cmd/versitygw/*.go + - name: Test run: go test -v -timeout 30s -tags=github ./... + + - name: Install govulncheck + run: go install golang.org/x/vuln/cmd/govulncheck@latest + shell: bash + + - name: Run govulncheck + run: govulncheck ./... + shell: bash \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6b98c69..33586a4 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ *.dll *.so *.dylib +cmd/versitygw/versitygw +/versitygw # Test binary, built with `go test -c` *.test @@ -22,3 +24,11 @@ go.work # ignore IntelliJ directories .idea + +# auto generated VERSION file +VERSION + +# build output +/versitygw.spec +*.tar +*.tar.gz diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3100315 --- /dev/null +++ b/Makefile @@ -0,0 +1,73 @@ +# Copyright 2023 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Go parameters +GOCMD=go +GOBUILD=$(GOCMD) build +GOCLEAN=$(GOCMD) clean +GOTEST=$(GOCMD) test + +BIN=versitygw + +VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi) +BUILD := $(shell git rev-parse --short HEAD || echo release-rpm) +TIME := `date -u '+%Y-%m-%d_%I:%M:%S%p'` + +LDFLAGS=-ldflags "-X=main.Build=$(BUILD) -X=main.BuildTime=$(TIME) -X=main.Version=$(VERSION)" + +all: build + +build: $(BIN) + +.PHONY: $(BIN) +$(BIN): + $(GOBUILD) $(LDFLAGS) -o $(BIN) cmd/$(BIN)/*.go + +.PHONY: test +test: + $(GOTEST) ./... + +.PHONY: check +check: +# note this requires staticcheck be in your PATH: +# export PATH=$PATH:~/go/bin +# go install honnef.co/go/tools/cmd/staticcheck@latest + staticcheck ./... + golint ./... + gofmt -s -l . + +.PHONY: clean +clean: + $(GOCLEAN) + +.PHONY: cleanall +cleanall: clean + rm -f $(BIN) + rm -f versitygw-*.tar + rm -f versitygw-*.tar.gz + rm -f versitygw.spec + +%.spec: %.spec.in + sed -e 's/@@VERSION@@/$(VERSION)/g' < $< > $@+ + mv $@+ $@ + +TARFILE = $(BIN)-$(VERSION).tar + +dist: $(BIN).spec + echo $(VERSION) >VERSION + git archive --format=tar --prefix $(BIN)-$(VERSION)/ HEAD > $(TARFILE) + @ tar rf $(TARFILE) --transform="s@\(.*\)@$(BIN)-$(VERSION)/\1@" $(BIN).spec VERSION + rm -f VERSION + rm -f $(BIN).spec + gzip -f $(TARFILE) diff --git a/README.md b/README.md new file mode 100644 index 0000000..33e47a1 --- /dev/null +++ b/README.md @@ -0,0 +1,35 @@ +# Versity S3 Gateway + +[![Versity Logo](https://www.versity.com/wp-content/themes/versity-theme/assets/img/svg/logo.svg)](https://www.versity.com) + + [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE) + +The Versity S3 Gateway provides an S3 server that translates S3 client access to a modular backend service. The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage. + +The Versity S3 Gateway is focused on performance, simplicity, and expandability. New backend types can be added to support new storage systems. The initial backend is a posix filesystem. The posix backend allows standing up an S3 compatible server from an existing filesystem mount with a simple command. + +The gateway is completely stateless. Mutliple gateways can host the same backend service and clients can load balance across the gateways. + +The S3 HTTP(S) server and routing is implemented using the [Fiber](https://gofiber.io) web framework. This framework is actively developed with a focus on performance. S3 API compatibility leverages the official [aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) whenever possible for maximum service compatibility with AWS S3. + +## Getting Started + +###Run the gateway with posix backend: + +``` +ADMIN_ACCESS_KEY="testuser" ADMIN_SECRET_KEY="secret" ./versitygw --port :10000 posix /tmp/vgw +``` +This will enable an S3 server on the current host listening on port 10000 and hosting the directory `/tmp/vgw`. + +To get the usage output, run the following: + +``` +./versitygw --help +``` + +The command format is + +``` +versitygw [global options] command [command options] [arguments...] +``` +The global options are specified before the backend type and the backend options are speficied after. \ No newline at end of file diff --git a/backend/auth/iam.go b/backend/auth/iam.go index 0e5bc4f..835f11e 100644 --- a/backend/auth/iam.go +++ b/backend/auth/iam.go @@ -1,6 +1,20 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package auth -import "github.com/versity/scoutgw/s3err" +import "github.com/versity/versitygw/s3err" type IAMConfig struct { AccessAccounts map[string]string diff --git a/backend/backend.go b/backend/backend.go index 70db993..002d5cb 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -1,3 +1,17 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package backend import ( @@ -6,7 +20,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/versity/scoutgw/s3err" + "github.com/versity/versitygw/s3err" ) //go:generate moq -out backend_moq_test.go . Backend @@ -31,7 +45,7 @@ type Backend interface { PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (etag string, err error) PutObject(*s3.PutObjectInput) (string, error) - HeadObject(bucket, object string, etag string) (*s3.HeadObjectOutput, error) + HeadObject(bucket, object string) (*s3.HeadObjectOutput, error) GetObject(bucket, object, acceptRange string, startOffset, length int64, writer io.Writer) (*s3.GetObjectOutput, error) GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error) GetObjectAttributes(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, error) @@ -126,7 +140,7 @@ func (BackendUnsupported) DeleteObjects(bucket string, objects *s3.DeleteObjects func (BackendUnsupported) GetObject(bucket, object, acceptRange string, startOffset, length int64, writer io.Writer) (*s3.GetObjectOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) HeadObject(bucket, object string, etag string) (*s3.HeadObjectOutput, error) { +func (BackendUnsupported) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error) { diff --git a/backend/backend_moq_test.go b/backend/backend_moq_test.go index 5dc73a0..c4c41ec 100644 --- a/backend/backend_moq_test.go +++ b/backend/backend_moq_test.go @@ -65,7 +65,7 @@ var _ Backend = &BackendMock{} // HeadBucketFunc: func(bucket string) (*s3.HeadBucketOutput, error) { // panic("mock out the HeadBucket method") // }, -// HeadObjectFunc: func(bucket string, object string, etag string) (*s3.HeadObjectOutput, error) { +// HeadObjectFunc: func(bucket string, object string) (*s3.HeadObjectOutput, error) { // panic("mock out the HeadObject method") // }, // ListBucketsFunc: func() (*s3.ListBucketsOutput, error) { @@ -172,7 +172,7 @@ type BackendMock struct { HeadBucketFunc func(bucket string) (*s3.HeadBucketOutput, error) // HeadObjectFunc mocks the HeadObject method. - HeadObjectFunc func(bucket string, object string, etag string) (*s3.HeadObjectOutput, error) + HeadObjectFunc func(bucket string, object string) (*s3.HeadObjectOutput, error) // ListBucketsFunc mocks the ListBuckets method. ListBucketsFunc func() (*s3.ListBucketsOutput, error) @@ -350,8 +350,6 @@ type BackendMock struct { Bucket string // Object is the object argument value. Object string - // Etag is the etag argument value. - Etag string } // ListBuckets holds details about calls to the ListBuckets method. ListBuckets []struct { @@ -1082,23 +1080,21 @@ func (mock *BackendMock) HeadBucketCalls() []struct { } // HeadObject calls HeadObjectFunc. -func (mock *BackendMock) HeadObject(bucket string, object string, etag string) (*s3.HeadObjectOutput, error) { +func (mock *BackendMock) HeadObject(bucket string, object string) (*s3.HeadObjectOutput, error) { if mock.HeadObjectFunc == nil { panic("BackendMock.HeadObjectFunc: method is nil but Backend.HeadObject was just called") } callInfo := struct { Bucket string Object string - Etag string }{ Bucket: bucket, Object: object, - Etag: etag, } mock.lockHeadObject.Lock() mock.calls.HeadObject = append(mock.calls.HeadObject, callInfo) mock.lockHeadObject.Unlock() - return mock.HeadObjectFunc(bucket, object, etag) + return mock.HeadObjectFunc(bucket, object) } // HeadObjectCalls gets all the calls that were made to HeadObject. @@ -1108,12 +1104,10 @@ func (mock *BackendMock) HeadObject(bucket string, object string, etag string) ( func (mock *BackendMock) HeadObjectCalls() []struct { Bucket string Object string - Etag string } { var calls []struct { Bucket string Object string - Etag string } mock.lockHeadObject.RLock() calls = mock.calls.HeadObject diff --git a/backend/backend_test.go b/backend/backend_test.go index ae12be8..41c6dc0 100644 --- a/backend/backend_test.go +++ b/backend/backend_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package backend import ( @@ -7,7 +21,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/versity/scoutgw/s3err" + "github.com/versity/versitygw/s3err" ) func TestBackend_ListBuckets(t *testing.T) { diff --git a/backend/common.go b/backend/common.go index 2afc2b6..63873be 100644 --- a/backend/common.go +++ b/backend/common.go @@ -1,3 +1,17 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package backend import ( diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 50f4456..0c4aa9b 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -1,11 +1,28 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package posix import ( "crypto/md5" "crypto/sha256" "encoding/hex" + "encoding/json" + "errors" "fmt" "io" + "io/fs" "os" "path/filepath" "sort" @@ -17,8 +34,8 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/google/uuid" "github.com/pkg/xattr" - "github.com/versity/scoutgw/backend" - "github.com/versity/scoutgw/s3err" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3err" ) type Posix struct { @@ -98,7 +115,7 @@ func (p *Posix) ListBuckets() (*s3.ListBucketsOutput, error) { func (p *Posix) HeadBucket(bucket string) (*s3.HeadBucketOutput, error) { _, err := os.Lstat(bucket) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { @@ -122,7 +139,7 @@ func (p *Posix) PutBucket(bucket string) error { func (p *Posix) DeleteBucket(bucket string) error { names, err := os.ReadDir(bucket) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { @@ -133,7 +150,7 @@ func (p *Posix) DeleteBucket(bucket string) error { // if .sgwtmp is only item in directory // then clean this up before trying to remove the bucket err = os.RemoveAll(filepath.Join(bucket, metaTmpDir)) - if err != nil && !os.IsNotExist(err) { + if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("remove temp dir: %w", err) } } @@ -154,7 +171,7 @@ func (p *Posix) CreateMultipartUpload(mpu *s3.CreateMultipartUploadInput) (*s3.C object := *mpu.Key _, err := os.Stat(bucket) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { @@ -202,7 +219,7 @@ func (p *Posix) CreateMultipartUpload(mpu *s3.CreateMultipartUploadInput) (*s3.C func (p *Posix) CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) { _, err := os.Stat(bucket) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { @@ -234,7 +251,7 @@ func (p *Posix) CompleteMultipartUpload(bucket, object, uploadID string, parts [ } } - f, err := openTmpFile(metaTmpDir, object, 0) + f, err := openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, 0) if err != nil { return nil, fmt.Errorf("open temp file: %w", err) } @@ -319,7 +336,7 @@ func (p *Posix) checkUploadIDExists(bucket, object, uploadID string) ([32]byte, objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum)) _, err := os.Stat(filepath.Join(objdir, uploadID)) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return [32]byte{}, s3err.GetAPIError(s3err.ErrNoSuchUpload) } if err != nil { @@ -328,7 +345,7 @@ func (p *Posix) checkUploadIDExists(bucket, object, uploadID string) ([32]byte, return sum, nil } -func loadUserMetaData(path string, m map[string]string) (tag, contentType, contentEncoding string) { +func loadUserMetaData(path string, m map[string]string) (contentType, contentEncoding string) { ents, err := xattr.List(path) if err != nil || len(ents) == 0 { return @@ -348,16 +365,7 @@ func loadUserMetaData(path string, m map[string]string) (tag, contentType, conte m[strings.TrimPrefix(e, "user.")] = string(b) } - b, err := xattr.Get(path, "user."+tagHdr) - tag = string(b) - if err != nil { - tag = "" - } - if tag != "" { - m[tagHdr] = tag - } - - b, err = xattr.Get(path, "user.content-type") + b, err := xattr.Get(path, "user.content-type") contentType = string(b) if err != nil { contentType = "" @@ -467,7 +475,7 @@ func (p *Posix) AbortMultipartUpload(mpu *s3.AbortMultipartUploadInput) error { uploadID := *mpu.UploadId _, err := os.Stat(bucket) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { @@ -495,7 +503,7 @@ func (p *Posix) ListMultipartUploads(mpu *s3.ListMultipartUploadsInput) (*s3.Lis bucket := *mpu.Bucket _, err := os.Stat(bucket) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { @@ -585,7 +593,7 @@ func (p *Posix) ListMultipartUploads(mpu *s3.ListMultipartUploadsInput) (*s3.Lis func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) { _, err := os.Stat(bucket) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { @@ -600,7 +608,7 @@ func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarke objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum)) ents, err := os.ReadDir(filepath.Join(objdir, uploadID)) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload) } if err != nil { @@ -670,7 +678,7 @@ func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarke func (p *Posix) PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (string, error) { _, err := os.Stat(bucket) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return "", s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { @@ -678,10 +686,11 @@ func (p *Posix) PutObjectPart(bucket, object, uploadID string, part int, length } sum := sha256.Sum256([]byte(object)) - objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum)) + objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum)) partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", part)) - f, err := openTmpFile(objdir, partPath, length) + f, err := openTmpFile(filepath.Join(bucket, objdir), + bucket, partPath, length) if err != nil { return "", fmt.Errorf("open temp file: %w", err) } @@ -708,7 +717,7 @@ func (p *Posix) PutObjectPart(bucket, object, uploadID string, part int, length func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) { _, err := os.Stat(*po.Bucket) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return "", s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { @@ -717,8 +726,6 @@ func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) { name := filepath.Join(*po.Bucket, *po.Key) - etag := "" - if strings.HasSuffix(*po.Key, "/") { // object is directory err = mkdirAll(name, os.FileMode(0755), *po.Bucket, *po.Key) @@ -730,50 +737,54 @@ func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) { xattr.Set(name, "user."+k, []byte(v)) } - // set our tag that this dir was specifically put + // set our attribute that this dir was specifically put xattr.Set(name, dirObjKey, nil) - } else { - // object is file - f, err := openTmpFile(metaTmpDir, *po.Key, po.ContentLength) + + // TODO: what etag should be returned here + // and we should set etag xattr to identify dir was + // specifically uploaded + return "", nil + } + + // object is file + f, err := openTmpFile(filepath.Join(*po.Bucket, metaTmpDir), + *po.Bucket, *po.Key, po.ContentLength) + if err != nil { + return "", fmt.Errorf("open temp file: %w", err) + } + defer f.cleanup() + + hash := md5.New() + rdr := io.TeeReader(po.Body, hash) + _, err = io.Copy(f, rdr) + if err != nil { + return "", fmt.Errorf("write object data: %w", err) + } + dir := filepath.Dir(name) + if dir != "" { + err = mkdirAll(dir, os.FileMode(0755), *po.Bucket, *po.Key) if err != nil { - return "", fmt.Errorf("open temp file: %w", err) + return "", fmt.Errorf("make object parent directories: %w", err) } - defer f.cleanup() + } - // TODO: fallocate based on content length + err = f.link() + if err != nil { + return "", fmt.Errorf("link object in namespace: %w", err) + } - hash := md5.New() - rdr := io.TeeReader(po.Body, hash) - _, err = io.Copy(f, rdr) + for k, v := range po.Metadata { + xattr.Set(name, "user."+k, []byte(v)) + } + + dataSum := hash.Sum(nil) + etag := hex.EncodeToString(dataSum[:]) + xattr.Set(name, "user.etag", []byte(etag)) + + if newObjUID != 0 || newObjGID != 0 { + err = os.Chown(name, newObjUID, newObjGID) if err != nil { - return "", fmt.Errorf("write object data: %w", err) - } - dir := filepath.Dir(name) - if dir != "" { - err = mkdirAll(dir, os.FileMode(0755), *po.Bucket, *po.Key) - if err != nil { - return "", fmt.Errorf("make object parent directories: %w", err) - } - } - - err = f.link() - if err != nil { - return "", fmt.Errorf("link object in namespace: %w", err) - } - - for k, v := range po.Metadata { - xattr.Set(name, "user."+k, []byte(v)) - } - - dataSum := hash.Sum(nil) - etag := hex.EncodeToString(dataSum[:]) - xattr.Set(name, "user.etag", []byte(etag)) - - if newObjUID != 0 || newObjGID != 0 { - err = os.Chown(name, newObjUID, newObjGID) - if err != nil { - return "", fmt.Errorf("set object uid/gid: %v", err) - } + return "", fmt.Errorf("set object uid/gid: %v", err) } } @@ -782,7 +793,7 @@ func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) { func (p *Posix) DeleteObject(bucket, object string) error { _, err := os.Stat(bucket) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { @@ -790,7 +801,7 @@ func (p *Posix) DeleteObject(bucket, object string) error { } os.Remove(filepath.Join(bucket, object)) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil { @@ -844,7 +855,7 @@ func (p *Posix) DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) err func (p *Posix) GetObject(bucket, object, acceptRange string, startOffset, length int64, writer io.Writer) (*s3.GetObjectOutput, error) { _, err := os.Stat(bucket) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { @@ -853,7 +864,7 @@ func (p *Posix) GetObject(bucket, object, acceptRange string, startOffset, lengt objPath := filepath.Join(bucket, object) fi, err := os.Stat(objPath) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil { @@ -866,7 +877,7 @@ func (p *Posix) GetObject(bucket, object, acceptRange string, startOffset, lengt } f, err := os.Open(objPath) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil { @@ -882,7 +893,7 @@ func (p *Posix) GetObject(bucket, object, acceptRange string, startOffset, lengt userMetaData := make(map[string]string) - _, contentType, contentEncoding := loadUserMetaData(objPath, userMetaData) + contentType, contentEncoding := loadUserMetaData(objPath, userMetaData) b, err := xattr.Get(objPath, "user.etag") etag := string(b) @@ -890,8 +901,11 @@ func (p *Posix) GetObject(bucket, object, acceptRange string, startOffset, lengt etag = "" } - // TODO: fill range request header? - // TODO: parse tags for tag count? + tags, err := p.getXattrTags(bucket, object) + if err != nil { + return nil, fmt.Errorf("get object tags: %w", err) + } + return &s3.GetObjectOutput{ AcceptRanges: &acceptRange, ContentLength: length, @@ -900,12 +914,13 @@ func (p *Posix) GetObject(bucket, object, acceptRange string, startOffset, lengt ETag: &etag, LastModified: backend.GetTimePtr(fi.ModTime()), Metadata: userMetaData, + TagCount: int32(len(tags)), }, nil } -func (p *Posix) HeadObject(bucket, object string, etag string) (*s3.HeadObjectOutput, error) { +func (p *Posix) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error) { _, err := os.Stat(bucket) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { @@ -914,7 +929,7 @@ func (p *Posix) HeadObject(bucket, object string, etag string) (*s3.HeadObjectOu objPath := filepath.Join(bucket, object) fi, err := os.Stat(objPath) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil { @@ -922,10 +937,14 @@ func (p *Posix) HeadObject(bucket, object string, etag string) (*s3.HeadObjectOu } userMetaData := make(map[string]string) - _, contentType, contentEncoding := loadUserMetaData(filepath.Join(bucket, objPath), userMetaData) + contentType, contentEncoding := loadUserMetaData(objPath, userMetaData) + + b, err := xattr.Get(objPath, "user.etag") + etag := string(b) + if err != nil { + etag = "" + } - // TODO: fill accept ranges request header? - // TODO: do we need to get etag from xattr? return &s3.HeadObjectOutput{ ContentLength: fi.Size(), ContentType: &contentType, @@ -938,7 +957,7 @@ func (p *Posix) HeadObject(bucket, object string, etag string) (*s3.HeadObjectOu func (p *Posix) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error) { _, err := os.Stat(srcBucket) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { @@ -946,7 +965,7 @@ func (p *Posix) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (* } _, err = os.Stat(DstBucket) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { @@ -955,7 +974,7 @@ func (p *Posix) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (* objPath := filepath.Join(srcBucket, srcObject) f, err := os.Open(objPath) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil { @@ -983,7 +1002,7 @@ func (p *Posix) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (* func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) { _, err := os.Stat(bucket) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { @@ -1008,9 +1027,10 @@ func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) ( Prefix: &prefix, }, nil } + func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) { _, err := os.Stat(bucket) - if err != nil && os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { @@ -1035,3 +1055,90 @@ func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) Prefix: &prefix, }, nil } + +func (p *Posix) GetTags(bucket, object string) (map[string]string, error) { + _, err := os.Stat(bucket) + if errors.Is(err, fs.ErrNotExist) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) + } + if err != nil { + return nil, fmt.Errorf("stat bucket: %w", err) + } + + return p.getXattrTags(bucket, object) +} + +func (p *Posix) getXattrTags(bucket, object string) (map[string]string, error) { + tags := make(map[string]string) + b, err := xattr.Get(filepath.Join(bucket, object), "user."+tagHdr) + if errors.Is(err, fs.ErrNotExist) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if isNoAttr(err) { + return tags, nil + } + if err != nil { + return nil, fmt.Errorf("get tags: %w", err) + } + + err = json.Unmarshal(b, &tags) + if err != nil { + return nil, fmt.Errorf("unmarshal tags: %w", err) + } + + return tags, nil +} + +func (p *Posix) SetTags(bucket, object string, tags map[string]string) error { + _, err := os.Stat(bucket) + if errors.Is(err, fs.ErrNotExist) { + return s3err.GetAPIError(s3err.ErrNoSuchBucket) + } + if err != nil { + return fmt.Errorf("stat bucket: %w", err) + } + + if tags == nil { + err = xattr.Remove(filepath.Join(bucket, object), "user."+tagHdr) + if errors.Is(err, fs.ErrNotExist) { + return s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err != nil { + return fmt.Errorf("remove tags: %w", err) + } + return nil + } + + b, err := json.Marshal(tags) + if err != nil { + return fmt.Errorf("marshal tags: %w", err) + } + + err = xattr.Set(filepath.Join(bucket, object), "user."+tagHdr, b) + if errors.Is(err, fs.ErrNotExist) { + return s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err != nil { + return fmt.Errorf("set tags: %w", err) + } + + return nil +} + +func (p *Posix) RemoveTags(bucket, object string) error { + return p.SetTags(bucket, object, nil) +} + +func isNoAttr(err error) bool { + if err == nil { + return false + } + xerr, ok := err.(*xattr.Error) + if ok && xerr.Err == xattr.ENOATTR { + return true + } + if err == syscall.ENODATA { + return true + } + return false +} diff --git a/backend/posix/posix_darwin.go b/backend/posix/posix_darwin.go index ee7103a..e67f93b 100644 --- a/backend/posix/posix_darwin.go +++ b/backend/posix/posix_darwin.go @@ -1,3 +1,17 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package posix import ( @@ -6,30 +20,43 @@ import ( "fmt" "io/fs" "os" + "path/filepath" ) type tmpfile struct { f *os.File + bucket string objname string + size int64 } -func openTmpFile(dir, obj string, size int64) (*tmpfile, error) { +func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) { // Create a temp file for upload while in progress (see link comments below). + err := os.MkdirAll(dir, 0700) + if err != nil { + return nil, fmt.Errorf("make temp dir: %w", err) + } f, err := os.CreateTemp(dir, - fmt.Sprintf("%x\n", sha256.Sum256([]byte(obj)))) + fmt.Sprintf("%x.", sha256.Sum256([]byte(obj)))) if err != nil { return nil, err } - return &tmpfile{f: f, objname: obj}, nil + return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil } func (tmp *tmpfile) link() error { + tempname := tmp.f.Name() + // cleanup in case anything goes wrong, if rename succeeds then + // this will no longer exist + defer os.Remove(tempname) + // We use Rename as the atomic operation for object puts. The upload is // written to a temp file to not conflict with any other simultaneous // uploads. The final operation is to move the temp file into place for // the object. This ensures the object semantics of last upload completed // wins and is not some combination of writes from simultaneous uploads. - err := os.Remove(tmp.objname) + objPath := filepath.Join(tmp.bucket, tmp.objname) + err := os.Remove(objPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("remove stale path: %w", err) } @@ -39,7 +66,7 @@ func (tmp *tmpfile) link() error { return fmt.Errorf("close tmpfile: %w", err) } - err = os.Rename(tmp.f.Name(), tmp.objname) + err = os.Rename(tempname, objPath) if err != nil { return fmt.Errorf("rename tmpfile: %w", err) } @@ -48,7 +75,13 @@ func (tmp *tmpfile) link() error { } func (tmp *tmpfile) Write(b []byte) (int, error) { - return tmp.f.Write(b) + if int64(len(b)) > tmp.size { + return 0, fmt.Errorf("write exceeds content length") + } + + n, err := tmp.f.Write(b) + tmp.size -= int64(n) + return n, err } func (tmp *tmpfile) cleanup() { diff --git a/backend/posix/posix_linux.go b/backend/posix/posix_linux.go index 6135372..9ce49d3 100644 --- a/backend/posix/posix_linux.go +++ b/backend/posix/posix_linux.go @@ -1,3 +1,17 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package posix import ( @@ -17,11 +31,13 @@ const procfddir = "/proc/self/fd" type tmpfile struct { f *os.File + bucket string objname string isOTmp bool + size int64 } -func openTmpFile(dir, obj string, size int64) (*tmpfile, error) { +func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) { // O_TMPFILE allows for a file handle to an unnamed file in the filesystem. // This can help reduce contention within the namespace (parent directories), // etc. And will auto cleanup the inode on close if we never link this @@ -31,30 +47,37 @@ func openTmpFile(dir, obj string, size int64) (*tmpfile, error) { fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666) if err != nil { // O_TMPFILE not supported, try fallback + err := os.MkdirAll(dir, 0700) + if err != nil { + return nil, fmt.Errorf("make temp dir: %w", err) + } f, err := os.CreateTemp(dir, - fmt.Sprintf("%x\n", sha256.Sum256([]byte(obj)))) + fmt.Sprintf("%x.", sha256.Sum256([]byte(obj)))) if err != nil { return nil, err } - tmp := &tmpfile{f: f} + tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size} // falloc is best effort, its fine if this fails if size > 0 { - tmp.falloc(size) + tmp.falloc() } return tmp, nil } + // for O_TMPFILE, filename is /proc/self/fd/ to be used + // later to link file into namespace f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd))) - tmp := &tmpfile{f: f, isOTmp: true} + + tmp := &tmpfile{f: f, isOTmp: true, size: size} // falloc is best effort, its fine if this fails if size > 0 { - tmp.falloc(size) + tmp.falloc() } return tmp, nil } -func (tmp *tmpfile) falloc(size int64) error { - err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, size) +func (tmp *tmpfile) falloc() error { + err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size) if err != nil { return fmt.Errorf("fallocate: %v", err) } @@ -68,36 +91,33 @@ func (tmp *tmpfile) link() error { // temp file into place for the object. This ensures the object semantics // of last upload completed wins and is not some combination of writes // from simultaneous uploads. - err := os.Remove(tmp.objname) + objPath := filepath.Join(tmp.bucket, tmp.objname) + err := os.Remove(objPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("remove stale path: %w", err) } - if tmp.isOTmp { - procdir, err := os.Open(procfddir) - if err != nil { - return fmt.Errorf("open proc dir: %w", err) - } - defer procdir.Close() + if !tmp.isOTmp { + // O_TMPFILE not suported, use fallback + return tmp.fallbackLink() + } - dir, err := os.Open(filepath.Dir(tmp.objname)) - if err != nil { - return fmt.Errorf("open parent dir: %w", err) - } - defer dir.Close() + procdir, err := os.Open(procfddir) + if err != nil { + return fmt.Errorf("open proc dir: %w", err) + } + defer procdir.Close() - err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()), - int(dir.Fd()), filepath.Base(tmp.objname), unix.AT_SYMLINK_FOLLOW) - if err != nil { - return fmt.Errorf("link tmpfile: %w", err) - } + dir, err := os.Open(filepath.Dir(objPath)) + if err != nil { + return fmt.Errorf("open parent dir: %w", err) + } + defer dir.Close() - err = tmp.f.Close() - if err != nil { - return fmt.Errorf("close tmpfile: %w", err) - } - - return nil + err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()), + int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW) + if err != nil { + return fmt.Errorf("link tmpfile: %w", err) } err = tmp.f.Close() @@ -105,7 +125,22 @@ func (tmp *tmpfile) link() error { return fmt.Errorf("close tmpfile: %w", err) } - err = os.Rename(tmp.f.Name(), tmp.objname) + return nil +} + +func (tmp *tmpfile) fallbackLink() error { + tempname := tmp.f.Name() + // cleanup in case anything goes wrong, if rename succeeds then + // this will no longer exist + defer os.Remove(tempname) + + err := tmp.f.Close() + if err != nil { + return fmt.Errorf("close tmpfile: %w", err) + } + + objPath := filepath.Join(tmp.bucket, tmp.objname) + err = os.Rename(tempname, objPath) if err != nil { return fmt.Errorf("rename tmpfile: %w", err) } @@ -114,7 +149,13 @@ func (tmp *tmpfile) link() error { } func (tmp *tmpfile) Write(b []byte) (int, error) { - return tmp.f.Write(b) + if int64(len(b)) > tmp.size { + return 0, fmt.Errorf("write exceeds content length") + } + + n, err := tmp.f.Write(b) + tmp.size -= int64(n) + return n, err } func (tmp *tmpfile) cleanup() { diff --git a/backend/scoutfs/scoutfs.go b/backend/scoutfs/scoutfs.go index 43909ff..aa89c10 100644 --- a/backend/scoutfs/scoutfs.go +++ b/backend/scoutfs/scoutfs.go @@ -1,8 +1,22 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package scoutfs import ( - "github.com/versity/scoutgw/backend" - "github.com/versity/scoutgw/backend/posix" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/backend/posix" ) type ScoutFS struct { diff --git a/backend/walk.go b/backend/walk.go index de78033..3d7cce6 100644 --- a/backend/walk.go +++ b/backend/walk.go @@ -1,3 +1,17 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package backend import ( diff --git a/backend/walk_test.go b/backend/walk_test.go index 075b73c..8cabde4 100644 --- a/backend/walk_test.go +++ b/backend/walk_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package backend_test import ( @@ -6,7 +20,7 @@ import ( "testing/fstest" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/versity/scoutgw/backend" + "github.com/versity/versitygw/backend" ) type walkTest struct { diff --git a/cmd/scoutgw/main.go b/cmd/scoutgw/main.go deleted file mode 100644 index 8b3b1a3..0000000 --- a/cmd/scoutgw/main.go +++ /dev/null @@ -1,21 +0,0 @@ -package main - -import ( - "log" - - "github.com/gofiber/fiber/v2" - "github.com/versity/scoutgw/backend" - "github.com/versity/scoutgw/s3api" - "github.com/versity/scoutgw/s3api/utils" -) - -func main() { - app := fiber.New(fiber.Config{}) - back := backend.New() - rootUser := utils.GetRootUserCreds() - if api, err := s3api.New(app, back, ":7070", rootUser); err != nil { - log.Fatalln(err) - } else if err = api.Serve(); err != nil { - log.Fatalln(err) - } -} diff --git a/cmd/versitygw/main.go b/cmd/versitygw/main.go new file mode 100644 index 0000000..9c99d1b --- /dev/null +++ b/cmd/versitygw/main.go @@ -0,0 +1,155 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package main + +import ( + "crypto/tls" + "fmt" + "log" + "os" + + "github.com/gofiber/fiber/v2" + "github.com/urfave/cli/v2" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3api" + "github.com/versity/versitygw/s3api/utils" +) + +var ( + port string + adminAccess string + adminSecret string + region string + certFile, keyFile string +) + +var ( + // Version is the latest tag (set within Makefile) + Version = "git" + // Build is the commit hash (set within Makefile) + Build = "norev" + // BuildTime is the date/time of build (set within Makefile) + BuildTime = "none" +) + +func main() { + app := initApp() + + app.Commands = []*cli.Command{ + posixCommand(), + } + + if err := app.Run(os.Args); err != nil { + log.Fatal(err) + } +} + +func initApp() *cli.App { + return &cli.App{ + Name: "versitygw", + Usage: "Start S3 gateway service with specified backend storage.", + Description: `The S3 gateway is an S3 protocol translator that allows an S3 client +to access the supported backend storage as if it was a native S3 service.`, + Action: func(ctx *cli.Context) error { + return ctx.App.Command("help").Run(ctx) + }, + Flags: initFlags(), + } +} + +func initFlags() []cli.Flag { + return []cli.Flag{ + &cli.BoolFlag{ + Name: "version", + Usage: "list versitygw version", + Aliases: []string{"v"}, + Action: func(*cli.Context, bool) error { + fmt.Println("Version :", Version) + fmt.Println("Build :", Build) + fmt.Println("BuildTime:", BuildTime) + os.Exit(0) + return nil + }, + }, + &cli.StringFlag{ + Name: "port", + Usage: "gateway listen address : or :", + Value: ":7070", + Destination: &port, + Aliases: []string{"p"}, + }, + &cli.StringFlag{ + Name: "access", + Usage: "admin access account", + Destination: &adminAccess, + EnvVars: []string{"ADMIN_ACCESS_KEY_ID", "ADMIN_ACCESS_KEY"}, + }, + &cli.StringFlag{ + Name: "secret", + Usage: "admin secret access key", + Destination: &adminSecret, + EnvVars: []string{"ADMIN_SECRET_ACCESS_KEY", "ADMIN_SECRET_KEY"}, + }, + &cli.StringFlag{ + Name: "region", + Usage: "s3 region string", + Value: "us-east-1", + Destination: ®ion, + }, + &cli.StringFlag{ + Name: "cert", + Usage: "TLS cert file", + Destination: &certFile, + }, + &cli.StringFlag{ + Name: "key", + Usage: "TLS key file", + Destination: &keyFile, + }, + } +} + +func runGateway(be backend.Backend) error { + app := fiber.New(fiber.Config{ + AppName: "versitygw", + ServerHeader: "VERSITYGW", + }) + + var opts []s3api.Option + + if certFile != "" || keyFile != "" { + if certFile == "" { + return fmt.Errorf("TLS key specified without cert file") + } + if keyFile == "" { + return fmt.Errorf("TLS cert specified without key file") + } + + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return fmt.Errorf("tls: load certs: %v", err) + } + opts = append(opts, s3api.WithTLS(cert)) + } + + rootUser := utils.GetRootUserCreds() + + srv, err := s3api.New(app, be, port, rootUser, opts...) + if err != nil { + return fmt.Errorf("init gateway: %v", err) + } + + return srv.Serve() +} diff --git a/cmd/versitygw/posix.go b/cmd/versitygw/posix.go new file mode 100644 index 0000000..a7de618 --- /dev/null +++ b/cmd/versitygw/posix.go @@ -0,0 +1,53 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package main + +import ( + "fmt" + + "github.com/urfave/cli/v2" + "github.com/versity/versitygw/backend/posix" +) + +func posixCommand() *cli.Command { + return &cli.Command{ + Name: "posix", + Usage: "posix filesystem storage backend", + Description: `Any posix filesystem that supports extended attributes. The top level +directory for the gateway must be provided. All sub directories of the +top level directory are treated as buckets, and all files/directories +below the "bucket directory" are treated as the objects. The object +name is split on "/" separator to translate to posix storage. +For example: +top level: /mnt/fs/gwroot +bucket: mybucket +object: a/b/c/myobject +will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`, + Action: runPosix, + } +} + +func runPosix(ctx *cli.Context) error { + if ctx.NArg() == 0 { + return fmt.Errorf("no directory provided for operation") + } + + be, err := posix.New(ctx.Args().Get(0)) + if err != nil { + return fmt.Errorf("init posix: %v", err) + } + + return runGateway(be) +} diff --git a/go.mod b/go.mod index 88c311b..bbc91b2 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/versity/scoutgw +module github.com/versity/versitygw go 1.20 @@ -8,6 +8,10 @@ require ( github.com/gofiber/fiber/v2 v2.45.0 github.com/google/uuid v1.3.0 github.com/pkg/xattr v0.4.9 + github.com/gofiber/fiber/v2 v2.46.0 + github.com/google/uuid v1.3.0 + github.com/pkg/xattr v0.4.9 + github.com/urfave/cli/v2 v2.25.4 github.com/valyala/fasthttp v1.47.0 golang.org/x/sys v0.8.0 ) @@ -23,15 +27,18 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2 // indirect github.com/aws/smithy-go v1.13.5 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/klauspost/compress v1.16.5 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.18 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/philhofer/fwd v1.1.2 // indirect github.com/rivo/uniseg v0.4.4 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect github.com/tinylib/msgp v1.1.8 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect ) diff --git a/go.sum b/go.sum index 6ab0e03..15e448c 100644 --- a/go.sum +++ b/go.sum @@ -22,9 +22,11 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1 h1:O+9nAy9Bb6bJFTpeNFtd9UfHbgxO1 github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1/go.mod h1:J9kLNzEiHSeGMyN7238EjJmBpCniVzFda75Gxl/NqB8= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gofiber/fiber/v2 v2.45.0 h1:p4RpkJT9GAW6parBSbcNFH2ApnAuW3OzaQzbOCoDu+s= -github.com/gofiber/fiber/v2 v2.45.0/go.mod h1:DNl0/c37WLe0g92U6lx1VMQuxGUQY5V7EIaVoEsUffc= +github.com/gofiber/fiber/v2 v2.46.0 h1:wkkWotblsGVlLjXj2dpgKQAYHtXumsK/HyFugQM68Ns= +github.com/gofiber/fiber/v2 v2.46.0/go.mod h1:DNl0/c37WLe0g92U6lx1VMQuxGUQY5V7EIaVoEsUffc= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -36,8 +38,8 @@ github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQs github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= -github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= @@ -49,6 +51,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 h1:rmMl4fXJhKMNWl+K+r/fq4FbbKI+Ia2m9hYBLm2h4G4= github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94/go.mod h1:90zrgN3D/WJsDd1iXHT96alCoN2KJo6/4x1DZC3wZs8= github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4= @@ -58,12 +62,16 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw= github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= +github.com/urfave/cli/v2 v2.25.4 h1:HyYwPrTO3im9rYhUff/ZNs78eolxt0nJ4LN+9yJKSH4= +github.com/urfave/cli/v2 v2.25.4/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.47.0 h1:y7moDoxYzMooFpT5aHgNgVOQDrS3qlkfiP9mDtGGK9c= github.com/valyala/fasthttp v1.47.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/s3api/controllers/backend_moq_test.go b/s3api/controllers/backend_moq_test.go index 2986503..5dccd89 100644 --- a/s3api/controllers/backend_moq_test.go +++ b/s3api/controllers/backend_moq_test.go @@ -6,7 +6,7 @@ package controllers import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/versity/scoutgw/backend" + "github.com/versity/versitygw/backend" "io" "sync" ) @@ -66,7 +66,7 @@ var _ backend.Backend = &BackendMock{} // HeadBucketFunc: func(bucket string) (*s3.HeadBucketOutput, error) { // panic("mock out the HeadBucket method") // }, -// HeadObjectFunc: func(bucket string, object string, etag string) (*s3.HeadObjectOutput, error) { +// HeadObjectFunc: func(bucket string, object string) (*s3.HeadObjectOutput, error) { // panic("mock out the HeadObject method") // }, // ListBucketsFunc: func() (*s3.ListBucketsOutput, error) { @@ -173,7 +173,7 @@ type BackendMock struct { HeadBucketFunc func(bucket string) (*s3.HeadBucketOutput, error) // HeadObjectFunc mocks the HeadObject method. - HeadObjectFunc func(bucket string, object string, etag string) (*s3.HeadObjectOutput, error) + HeadObjectFunc func(bucket string, object string) (*s3.HeadObjectOutput, error) // ListBucketsFunc mocks the ListBuckets method. ListBucketsFunc func() (*s3.ListBucketsOutput, error) @@ -351,8 +351,6 @@ type BackendMock struct { Bucket string // Object is the object argument value. Object string - // Etag is the etag argument value. - Etag string } // ListBuckets holds details about calls to the ListBuckets method. ListBuckets []struct { @@ -1083,23 +1081,21 @@ func (mock *BackendMock) HeadBucketCalls() []struct { } // HeadObject calls HeadObjectFunc. -func (mock *BackendMock) HeadObject(bucket string, object string, etag string) (*s3.HeadObjectOutput, error) { +func (mock *BackendMock) HeadObject(bucket string, object string) (*s3.HeadObjectOutput, error) { if mock.HeadObjectFunc == nil { panic("BackendMock.HeadObjectFunc: method is nil but Backend.HeadObject was just called") } callInfo := struct { Bucket string Object string - Etag string }{ Bucket: bucket, Object: object, - Etag: etag, } mock.lockHeadObject.Lock() mock.calls.HeadObject = append(mock.calls.HeadObject, callInfo) mock.lockHeadObject.Unlock() - return mock.HeadObjectFunc(bucket, object, etag) + return mock.HeadObjectFunc(bucket, object) } // HeadObjectCalls gets all the calls that were made to HeadObject. @@ -1109,12 +1105,10 @@ func (mock *BackendMock) HeadObject(bucket string, object string, etag string) ( func (mock *BackendMock) HeadObjectCalls() []struct { Bucket string Object string - Etag string } { var calls []struct { Bucket string Object string - Etag string } mock.lockHeadObject.RLock() calls = mock.calls.HeadObject diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 601a802..222b7ad 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -1,3 +1,17 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package controllers import ( @@ -13,9 +27,9 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gofiber/fiber/v2" - "github.com/versity/scoutgw/backend" - "github.com/versity/scoutgw/s3api/utils" - "github.com/versity/scoutgw/s3err" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" ) type S3ApiController struct { @@ -299,7 +313,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { key = strings.Join([]string{key, keyEnd}, "/") } - res, err := c.be.HeadObject(bucket, key, "") + res, err := c.be.HeadObject(bucket, key) return Responce(ctx, res, err) } diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index 7963d50..310ffdb 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package controllers import ( @@ -10,8 +24,8 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gofiber/fiber/v2" "github.com/valyala/fasthttp" - "github.com/versity/scoutgw/backend" - "github.com/versity/scoutgw/s3err" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3err" ) func TestNew(t *testing.T) { diff --git a/s3api/middlewares/authentication.go b/s3api/middlewares/authentication.go index 131a5ab..a69e290 100644 --- a/s3api/middlewares/authentication.go +++ b/s3api/middlewares/authentication.go @@ -1,3 +1,17 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package middlewares import ( @@ -9,9 +23,9 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" "github.com/gofiber/fiber/v2" - "github.com/versity/scoutgw/s3api/controllers" - "github.com/versity/scoutgw/s3api/utils" - "github.com/versity/scoutgw/s3err" + "github.com/versity/versitygw/s3api/controllers" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" ) const ( diff --git a/s3api/router.go b/s3api/router.go index a9ac81a..d8485ab 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -1,9 +1,23 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package s3api import ( "github.com/gofiber/fiber/v2" - "github.com/versity/scoutgw/backend" - "github.com/versity/scoutgw/s3api/controllers" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3api/controllers" ) type S3ApiRouter struct{} diff --git a/s3api/router_test.go b/s3api/router_test.go index 5d1262e..4d2d22e 100644 --- a/s3api/router_test.go +++ b/s3api/router_test.go @@ -1,10 +1,24 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package s3api import ( "testing" "github.com/gofiber/fiber/v2" - "github.com/versity/scoutgw/backend" + "github.com/versity/versitygw/backend" ) func TestS3ApiRouter_Init(t *testing.T) { diff --git a/s3api/server.go b/s3api/server.go index fe8d9d6..b78ae79 100644 --- a/s3api/server.go +++ b/s3api/server.go @@ -1,11 +1,27 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package s3api import ( + "crypto/tls" + "github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2/middleware/logger" - "github.com/versity/scoutgw/backend" - "github.com/versity/scoutgw/s3api/middlewares" - "github.com/versity/scoutgw/s3api/utils" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3api/middlewares" + "github.com/versity/versitygw/s3api/utils" ) type S3ApiServer struct { @@ -13,17 +29,38 @@ type S3ApiServer struct { backend backend.Backend router *S3ApiRouter port string + cert *tls.Certificate } -func New(app *fiber.App, be backend.Backend, port string, rootUser utils.RootUser) (s3ApiServer *S3ApiServer, err error) { - s3ApiServer = &S3ApiServer{app, be, new(S3ApiRouter), port} +func New(app *fiber.App, be backend.Backend, port string, rootUser utils.RootUser, opts ...Option) (*S3ApiServer, error) { + server := &S3ApiServer{ + app: app, + backend: be, + router: new(S3ApiRouter), + port: port, + } + + for _, opt := range opts { + opt(server) + } app.Use(middlewares.VerifyV4Signature(rootUser)) app.Use(logger.New()) - s3ApiServer.router.Init(app, be) - return + server.router.Init(app, be) + return server, nil +} + +// Option sets various options for New() +type Option func(*S3ApiServer) + +// WithTLS sets TLS Credentials +func WithTLS(cert tls.Certificate) Option { + return func(s *S3ApiServer) { s.cert = &cert } } func (sa *S3ApiServer) Serve() (err error) { + if sa.cert != nil { + return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert) + } return sa.app.Listen(sa.port) } diff --git a/s3api/server_test.go b/s3api/server_test.go index 8d5313e..b4280bf 100644 --- a/s3api/server_test.go +++ b/s3api/server_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package s3api import ( @@ -5,8 +19,8 @@ import ( "testing" "github.com/gofiber/fiber/v2" - "github.com/versity/scoutgw/backend" - "github.com/versity/scoutgw/s3api/utils" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3api/utils" ) func TestNew(t *testing.T) { diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index 7e3f358..a7ae3e9 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -1,3 +1,17 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package utils import ( diff --git a/s3err/s3err.go b/s3err/s3err.go index 7ac899b..0f623de 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -1,3 +1,17 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package s3err import ( diff --git a/versitygw.spec.in b/versitygw.spec.in new file mode 100644 index 0000000..f939e76 --- /dev/null +++ b/versitygw.spec.in @@ -0,0 +1,31 @@ +%global debug_package %{nil} +%define pkg_version @@VERSION@@ + +Name: versitygw +Version: %{pkg_version} +Release: 1%{?dist} +Summary: Versity S3 Gateway + +License: Apache-2.0 +URL: https://github.com/versity/versitygw +Source0: %{name}-%{version}.tar.gz +%description +The S3 gateway is an S3 protocol translator that allows an S3 client +to access the supported backend storage as if it was a native S3 service. + +BuildRequires: golang >= 1.20 + +%prep +%setup + +%build +make + +%install +mkdir -p %{buildroot}%{_bindir} +install -m 0755 %{name} %{buildroot}%{_bindir}/ + +%post + +%files +%{_bindir}/%{name}