From 145c2dd4e306ccf23325f715a673a6f3eb583f4b Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Fri, 8 Sep 2023 22:13:02 -0700 Subject: [PATCH] fix: builds for non 64 bit linux arch The scoutfs backend is only supported on 64bit linux. This corrects the build constraints to only supported linux arch, and prevents the incompatible import for unspported arch. We also need to adjust the body limit setting on 32 bit since this is an int, and our default limit will overfow on 32 bit. --- backend/scoutfs/scoutfs.go | 9 ++--- .../{scoutfs_linux.go => scoutfs_compat.go} | 25 ++++++++++++ ...{scoutfs_darwin.go => scoutfs_incompat.go} | 10 +++++ .../scoutfs/{scoutfs_windows.go => stat.go} | 39 ++++--------------- cmd/versitygw/main.go | 10 ++++- 5 files changed, 56 insertions(+), 37 deletions(-) rename backend/scoutfs/{scoutfs_linux.go => scoutfs_compat.go} (90%) rename backend/scoutfs/{scoutfs_darwin.go => scoutfs_incompat.go} (86%) rename backend/scoutfs/{scoutfs_windows.go => stat.go} (54%) diff --git a/backend/scoutfs/scoutfs.go b/backend/scoutfs/scoutfs.go index 0935ebb..47f30de 100644 --- a/backend/scoutfs/scoutfs.go +++ b/backend/scoutfs/scoutfs.go @@ -30,7 +30,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/pkg/xattr" - "github.com/versity/scoutfs-go" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/backend/posix" "github.com/versity/versitygw/s3err" @@ -188,7 +187,7 @@ func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteM // scoutfs move data is a metadata only operation that moves the data // extent references from the source, appeding to the destination. // this needs to be 4k aligned. - err = scoutfs.MoveData(pf, f.f) + err = moveData(pf, f.f) pf.Close() if err != nil { return nil, fmt.Errorf("move blocks part %v: %v", p.PartNumber, err) @@ -392,7 +391,7 @@ func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3. // Check if there are any offline exents associated with this file. // If so, we will set storage class to glacier. - st, err := scoutfs.StatMore(objPath) + st, err := statMore(objPath) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } @@ -466,7 +465,7 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer if s.glaciermode { // Check if there are any offline exents associated with this file. // If so, we will return the InvalidObjectState error. - st, err := scoutfs.StatMore(objPath) + st, err := statMore(objPath) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } @@ -666,7 +665,7 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc { if s.glaciermode { // Check if there are any offline exents associated with this file. // If so, we will return the InvalidObjectState error. - st, err := scoutfs.StatMore(objPath) + st, err := statMore(objPath) if errors.Is(err, fs.ErrNotExist) { return types.Object{}, backend.ErrSkipObj } diff --git a/backend/scoutfs/scoutfs_linux.go b/backend/scoutfs/scoutfs_compat.go similarity index 90% rename from backend/scoutfs/scoutfs_linux.go rename to backend/scoutfs/scoutfs_compat.go index b1f06fe..92652eb 100644 --- a/backend/scoutfs/scoutfs_linux.go +++ b/backend/scoutfs/scoutfs_compat.go @@ -12,6 +12,8 @@ // specific language governing permissions and limitations // under the License. +//go:build linux && amd64 + package scoutfs import ( @@ -26,6 +28,7 @@ import ( "golang.org/x/sys/unix" + "github.com/versity/scoutfs-go" "github.com/versity/versitygw/backend/posix" ) @@ -182,3 +185,25 @@ func (tmp *tmpfile) Write(b []byte) (int, error) { func (tmp *tmpfile) cleanup() { tmp.f.Close() } + +func moveData(from *os.File, to *os.File) error { + return scoutfs.MoveData(from, to) +} + +func statMore(path string) (stat, error) { + st, err := scoutfs.StatMore(path) + if err != nil { + return stat{}, err + } + var s stat + + s.Meta_seq = st.Meta_seq + s.Data_seq = st.Data_seq + s.Data_version = st.Data_version + s.Online_blocks = st.Online_blocks + s.Offline_blocks = st.Offline_blocks + s.Crtime_sec = st.Crtime_sec + s.Crtime_nsec = st.Crtime_nsec + + return s, nil +} diff --git a/backend/scoutfs/scoutfs_darwin.go b/backend/scoutfs/scoutfs_incompat.go similarity index 86% rename from backend/scoutfs/scoutfs_darwin.go rename to backend/scoutfs/scoutfs_incompat.go index 927b47b..3776b0b 100644 --- a/backend/scoutfs/scoutfs_darwin.go +++ b/backend/scoutfs/scoutfs_incompat.go @@ -12,6 +12,8 @@ // specific language governing permissions and limitations // under the License. +//go:build !(linux && amd64) + package scoutfs import ( @@ -46,3 +48,11 @@ func (tmp *tmpfile) Write(b []byte) (int, error) { func (tmp *tmpfile) cleanup() { } + +func moveData(from *os.File, to *os.File) error { + return errNotSupported +} + +func statMore(path string) (stat, error) { + return stat{}, errNotSupported +} diff --git a/backend/scoutfs/scoutfs_windows.go b/backend/scoutfs/stat.go similarity index 54% rename from backend/scoutfs/scoutfs_windows.go rename to backend/scoutfs/stat.go index 927b47b..7094c10 100644 --- a/backend/scoutfs/scoutfs_windows.go +++ b/backend/scoutfs/stat.go @@ -14,35 +14,12 @@ package scoutfs -import ( - "errors" - "fmt" - "os" -) - -func New(rootdir string, opts ...Option) (*ScoutFS, error) { - return nil, fmt.Errorf("scoutfs only available on linux") -} - -type tmpfile struct { - f *os.File -} - -var ( - errNotSupported = errors.New("not supported") -) - -func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) { - return nil, errNotSupported -} - -func (tmp *tmpfile) link() error { - return errNotSupported -} - -func (tmp *tmpfile) Write(b []byte) (int, error) { - return 0, errNotSupported -} - -func (tmp *tmpfile) cleanup() { +type stat struct { + Meta_seq uint64 + Data_seq uint64 + Data_version uint64 + Online_blocks uint64 + Offline_blocks uint64 + Crtime_sec uint64 + Crtime_nsec uint32 } diff --git a/cmd/versitygw/main.go b/cmd/versitygw/main.go index b6e62b7..a2af5b9 100644 --- a/cmd/versitygw/main.go +++ b/cmd/versitygw/main.go @@ -20,6 +20,7 @@ import ( "fmt" "log" "os" + "strconv" "github.com/gofiber/fiber/v2" "github.com/urfave/cli/v2" @@ -193,10 +194,17 @@ func initFlags() []cli.Flag { } func runGateway(ctx *cli.Context, be backend.Backend, s auth.Storer) error { + // int32 max for 32 bit arch + blimit := int64(2*1024*1024*1024 - 1) + if strconv.IntSize > 32 { + // 5GB max for 64 bit arch + blimit = int64(5 * 1024 * 1024 * 1024) + } + app := fiber.New(fiber.Config{ AppName: "versitygw", ServerHeader: "VERSITYGW", - BodyLimit: 5 * 1024 * 1024 * 1024, + BodyLimit: int(blimit), }) var opts []s3api.Option