From c4f5f958ebec225df8a4631488c34b717a168dfb Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Fri, 23 Jun 2023 18:32:34 -0700 Subject: [PATCH 1/4] add functional tests to github actions --- .github/workflows/functional.yml | 31 +++++++++++++++++++++++++++++++ .gitignore | 1 + cmd/versitygw/test.go | 3 +++ 3 files changed, 35 insertions(+) create mode 100644 .github/workflows/functional.yml diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml new file mode 100644 index 0000000..4ffdf7e --- /dev/null +++ b/.github/workflows/functional.yml @@ -0,0 +1,31 @@ +name: functional tests +on: pull_request +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: 'stable' + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Get dependencies + run: | + go get -v -t -d ./... + + - name: Build + run: make + + - name: Test + run: | + mkdir /tmp/gw + ./versitygw -a user -s pass posix /tmp/gw & + sleep 1 + ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow diff --git a/.gitignore b/.gitignore index 33586a4..e581c1c 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ VERSION /versitygw.spec *.tar *.tar.gz +/rand.data diff --git a/cmd/versitygw/test.go b/cmd/versitygw/test.go index 92b06b9..83073e6 100644 --- a/cmd/versitygw/test.go +++ b/cmd/versitygw/test.go @@ -285,6 +285,9 @@ func getAction(tf testFunc) func(*cli.Context) error { fmt.Println() fmt.Println("RAN:", integration.RunCount, "PASS:", integration.PassCount, "FAIL:", integration.FailCount) + if integration.FailCount > 0 { + return fmt.Errorf("test failed with %v errors", integration.FailCount) + } return nil } } From 1d476c6d4d582cd9ba0d0500a98f97b756fae090 Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Sun, 25 Jun 2023 10:01:54 -0700 Subject: [PATCH 2/4] add signal handler for clean shutdown --- Makefile | 3 +++ cmd/versitygw/main.go | 26 ++++++++++++++++++++++--- cmd/versitygw/posix.go | 2 +- cmd/versitygw/scoutfs.go | 2 +- cmd/versitygw/singal.go | 42 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 70 insertions(+), 5 deletions(-) create mode 100644 cmd/versitygw/singal.go diff --git a/Makefile b/Makefile index 3100315..81509c9 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,9 @@ build: $(BIN) $(BIN): $(GOBUILD) $(LDFLAGS) -o $(BIN) cmd/$(BIN)/*.go +testbin: + $(GOBUILD) $(LDFLAGS) -o $(BIN) -cover -race cmd/$(BIN)/*.go + .PHONY: test test: $(GOTEST) ./... diff --git a/cmd/versitygw/main.go b/cmd/versitygw/main.go index b307ffa..0f9bb06 100644 --- a/cmd/versitygw/main.go +++ b/cmd/versitygw/main.go @@ -15,6 +15,7 @@ package main import ( + "context" "crypto/tls" "fmt" "log" @@ -47,6 +48,8 @@ var ( ) func main() { + setupSignalHandler() + app := initApp() app.Commands = []*cli.Command{ @@ -56,7 +59,14 @@ func main() { testCommand(), } - if err := app.Run(os.Args); err != nil { + ctx, cancel := context.WithCancel(context.Background()) + go func() { + <-sigDone + fmt.Fprintf(os.Stderr, "terminating signal caught, shutting down\n") + cancel() + }() + + if err := app.RunContext(ctx, os.Args); err != nil { log.Fatal(err) } } @@ -134,7 +144,7 @@ func initFlags() []cli.Flag { } } -func runGateway(be backend.Backend, s auth.Storer) error { +func runGateway(ctx *cli.Context, be backend.Backend, s auth.Storer) error { app := fiber.New(fiber.Config{ AppName: "versitygw", ServerHeader: "VERSITYGW", @@ -180,5 +190,15 @@ func runGateway(be backend.Backend, s auth.Storer) error { return fmt.Errorf("init gateway: %v", err) } - return srv.Serve() + c := make(chan error, 1) + go func() { c <- srv.Serve() }() + + select { + case <-ctx.Done(): + be.Shutdown() + return ctx.Err() + case err := <-c: + be.Shutdown() + return err + } } diff --git a/cmd/versitygw/posix.go b/cmd/versitygw/posix.go index ebd8204..8928b8f 100644 --- a/cmd/versitygw/posix.go +++ b/cmd/versitygw/posix.go @@ -49,5 +49,5 @@ func runPosix(ctx *cli.Context) error { return fmt.Errorf("init posix: %v", err) } - return runGateway(be, be) + return runGateway(ctx, be, be) } diff --git a/cmd/versitygw/scoutfs.go b/cmd/versitygw/scoutfs.go index 047b137..7327d4c 100644 --- a/cmd/versitygw/scoutfs.go +++ b/cmd/versitygw/scoutfs.go @@ -69,5 +69,5 @@ func runScoutfs(ctx *cli.Context) error { return fmt.Errorf("init scoutfs: %v", err) } - return runGateway(be, be) + return runGateway(ctx, be, be) } diff --git a/cmd/versitygw/singal.go b/cmd/versitygw/singal.go new file mode 100644 index 0000000..147e55e --- /dev/null +++ b/cmd/versitygw/singal.go @@ -0,0 +1,42 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package main + +import ( + "fmt" + "os" + "os/signal" + "syscall" +) + +var ( + sigDone = make(chan bool, 1) +) + +func setupSignalHandler() { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) + + go func() { + for sig := range sigs { + fmt.Fprintf(os.Stderr, "caught signal %v\n", sig) + switch sig { + case syscall.SIGINT, syscall.SIGTERM: + sigDone <- true + case syscall.SIGHUP: + } + } + }() +} From 1da0c1ceba1e1126f3dc6f42802c50843c6b527e Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Sun, 25 Jun 2023 10:30:04 -0700 Subject: [PATCH 3/4] add coverage report for actions tests --- .github/workflows/functional.yml | 19 ++++++++-------- .github/workflows/go.yml | 4 ++-- .gitignore | 1 + runtests.sh | 37 ++++++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 12 deletions(-) create mode 100755 runtests.sh diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml index 4ffdf7e..b97ffe0 100644 --- a/.github/workflows/functional.yml +++ b/.github/workflows/functional.yml @@ -3,7 +3,7 @@ on: pull_request jobs: build: - name: Build + name: RunTests runs-on: ubuntu-latest steps: @@ -16,16 +16,15 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Get dependencies + - name: Get Dependencies run: | go get -v -t -d ./... - - name: Build - run: make - - - name: Test + - name: Build and Run run: | - mkdir /tmp/gw - ./versitygw -a user -s pass posix /tmp/gw & - sleep 1 - ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow + make testbin + ./runtests.sh + + - name: Coverage Report + run: | + go tool covdata percent -i=/tmp/covdata diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index ab37927..f49bb49 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -24,10 +24,10 @@ jobs: go get -v -t -d ./... - name: Build - run: go build -o versitygw cmd/versitygw/*.go + run: make - name: Test - run: go test -v -timeout 30s -tags=github ./... + run: go test -coverprofile profile.txt -race -v -timeout 30s -tags=github ./... - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest diff --git a/.gitignore b/.gitignore index e581c1c..d236503 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ VERSION *.tar *.tar.gz /rand.data +/profile.txt diff --git a/runtests.sh b/runtests.sh new file mode 100755 index 0000000..a186f36 --- /dev/null +++ b/runtests.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# make temp dirs +mkdir /tmp/gw +rm -rf /tmp/covdata +mkdir /tmp/covdata + +# run server in background +GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass posix /tmp/gw & +GW_PID=$! + +# wait a second for server to start up +sleep 1 + +# check if server is still running +if ! kill -0 $GW_PID; then + echo "server no longer running" + exit 1 +fi + +# run tests +if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow; then + echo "tests failed" + kill $GW_PID + exit 1 +fi + +# kill off server +kill $GW_PID +exit 0 + +# if the above binary was built with -cover enabled (make testbin), +# then the following can be used for code coverage reports: +# go tool covdata percent -i=/tmp/covdata +# go tool covdata textfmt -i=/tmp/covdata -o profile.txt +# go tool cover -html=profile.txt + From 77b0759f86a91a880b912a741e3e368773be3077 Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Sun, 25 Jun 2023 11:00:54 -0700 Subject: [PATCH 4/4] fix full flow mising TestRangeGet test --- integration/tests.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/tests.go b/integration/tests.go index d12347d..a11ebe4 100644 --- a/integration/tests.go +++ b/integration/tests.go @@ -1218,6 +1218,6 @@ func TestFullFlow(s *S3Conf) { TestIncompleteMultiParts(s) TestIncorrectMultiParts(s) TestListAbortMultiPartObject(s) - TestListAbortMultiPartObject(s) + TestRangeGet(s) TestInvalidMultiParts(s) }