From 3e2617526594e1583f98b2947d6683d96a885d46 Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Wed, 11 Feb 2026 09:58:15 -0800 Subject: [PATCH] feat: replace aws-sdk-go-v2 s3 manager with transfermanager The latest version of the go sdk has deprecate the s3 manager in favor of the new transfermanager. This updates to the new transfermanager functionality for the two places we were using the manager, iam s3 and integration tests. This also removes the pinning of nats nkeys since they have fixed the tags in their repo now. And general cleanup of the go.mod. --- .github/dependabot.yml | 2 -- auth/iam_s3_object.go | 8 ++++---- go.mod | 11 ++++------- go.sum | 8 ++++---- tests/integration/s3conf.go | 36 +++++++++++++++++++++++------------- 5 files changed, 35 insertions(+), 30 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4af7646c..d2540176 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,8 +11,6 @@ updates: allow: # Allow both direct and indirect updates for all packages - dependency-type: "all" - ignore: - - dependency-name: "github.com/nats-io/nkeys" - package-ecosystem: "github-actions" directory: "/" diff --git a/auth/iam_s3_object.go b/auth/iam_s3_object.go index 2d152b72..f8dafa09 100644 --- a/auth/iam_s3_object.go +++ b/auth/iam_s3_object.go @@ -29,7 +29,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/smithy-go" @@ -290,13 +290,13 @@ func (s *IAMServiceS3) storeAccts(conf iAMConfig) error { } obj := iamFile - uploader := manager.NewUploader(s.client) - upinfo := &s3.PutObjectInput{ + uploader := transfermanager.New(s.client) + upinfo := &transfermanager.UploadObjectInput{ Body: bytes.NewReader(b), Bucket: &s.bucket, Key: &obj, } - _, err = uploader.Upload(context.Background(), upinfo) + _, err = uploader.UploadObject(context.Background(), upinfo) if err != nil { return fmt.Errorf("store accounts in %v: %w", iamFile, err) } diff --git a/go.mod b/go.mod index 84909a34..2bf16667 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,9 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 github.com/DataDog/datadog-go/v5 v5.8.3 github.com/aws/aws-sdk-go-v2 v1.41.1 + github.com/aws/aws-sdk-go-v2/config v1.32.7 + github.com/aws/aws-sdk-go-v2/credentials v1.19.7 + github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.2 github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 github.com/aws/smithy-go v1.24.0 github.com/davecgh/go-spew v1.1.1 @@ -40,10 +43,7 @@ require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/andybalholm/brotli v1.2.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect - github.com/aws/aws-sdk-go-v2/config v1.32.7 - github.com/aws/aws-sdk-go-v2/credentials v1.19.7 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.0 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect @@ -71,6 +71,7 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/nats-io/nkeys v0.4.15 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.25 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect @@ -85,7 +86,3 @@ require ( golang.org/x/time v0.14.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) - -// pinned in dependabot.yml since newer versions in the go mod cache no longer -// exist. remove the ignore once this repo creates a newer tag. -require github.com/nats-io/nkeys v0.4.12 // indirect diff --git a/go.sum b/go.sum index faea6d88..f7d7624c 100644 --- a/go.sum +++ b/go.sum @@ -35,8 +35,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUT github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.0 h1:MpkX8EjkwuvyuX9B7+Zgk5M4URb2WQ84Y6jM81n5imw= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.0/go.mod h1:4V9Pv5sFfMPWQF0Q0zYN6BlV/504dFGaTeogallRqQw= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.2 h1:1q8/WwEqZnM/vO4q1gx2g7lHYmyN+o4P7G6EW4zKbRQ= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.2/go.mod h1:owKRexW+Ir5ACD2UTesmjkQ+w7mcmknLNfwOiKfVLTg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik= @@ -137,8 +137,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/nats-io/nats.go v1.48.0 h1:pSFyXApG+yWU/TgbKCjmm5K4wrHu86231/w84qRVR+U= github.com/nats-io/nats.go v1.48.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= -github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc= -github.com/nats-io/nkeys v0.4.12/go.mod h1:MT59A1HYcjIcyQDJStTfaOY6vhy9XTUjOFo+SVsvpBg= +github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= +github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s= diff --git a/tests/integration/s3conf.go b/tests/integration/s3conf.go index 8600d91b..879214d5 100644 --- a/tests/integration/s3conf.go +++ b/tests/integration/s3conf.go @@ -26,7 +26,7 @@ import ( v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/smithy-go/middleware" ) @@ -189,31 +189,41 @@ func (c *S3Conf) Config() aws.Config { } func (c *S3Conf) UploadData(r io.Reader, bucket, object string) error { - uploader := manager.NewUploader(c.GetClient()) - uploader.PartSize = c.PartSize - uploader.Concurrency = c.Concurrency + uploader := transfermanager.New(c.GetClient(), + func(options *transfermanager.Options) { + options.PartSizeBytes = c.PartSize + options.Concurrency = c.Concurrency + }) - upinfo := &s3.PutObjectInput{ + upinfo := &transfermanager.UploadObjectInput{ Body: r, Bucket: &bucket, Key: &object, } - _, err := uploader.Upload(context.Background(), upinfo) + _, err := uploader.UploadObject(context.Background(), upinfo) return err } func (c *S3Conf) DownloadData(w io.WriterAt, bucket, object string) (int64, error) { - downloader := manager.NewDownloader(c.GetClient()) - downloader.PartSize = c.PartSize - downloader.Concurrency = c.Concurrency + downloader := transfermanager.New(c.GetClient(), + func(options *transfermanager.Options) { + options.PartSizeBytes = c.PartSize + options.Concurrency = c.Concurrency + }) - downinfo := &s3.GetObjectInput{ - Bucket: &bucket, - Key: &object, + downinfo := &transfermanager.DownloadObjectInput{ + Bucket: &bucket, + Key: &object, + WriterAt: w, } - return downloader.Download(context.Background(), w, downinfo) + out, err := downloader.DownloadObject(context.Background(), downinfo) + if err != nil { + return 0, err + } + + return *out.ContentLength, nil } func (c *S3Conf) getAdminCommand(args ...string) []string {