mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-09-20 15:04:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: replace aws-sdk-go-v2 s3 manager with transfermanager

Browse Source 
The latest version of the go sdk has deprecate the s3 manager in
favor of the new transfermanager. This updates to the new
transfermanager functionality for the two places we were using
the manager, iam s3 and integration tests.

This also removes the pinning of nats nkeys since they have fixed
the tags in their repo now. And general cleanup of the go.mod.
This commit is contained in:
Ben McClelland
2026-02-11 09:58:15 -08:00
parent 09544283c7
commit 3e26175265
5 changed files with 35 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/dependabot.yml
-2
View File
@@ -11,8 +11,6 @@ updates:
allow:
# Allow both direct and indirect updates for all packages
- dependency-type: "all"
ignore:
- dependency-name: "github.com/nats-io/nkeys"
- package-ecosystem: "github-actions"
directory: "/"
auth/iam_s3_object.go
+4 -4
View File
@@ -29,7 +29,7 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
@@ -290,13 +290,13 @@ func (s *IAMServiceS3) storeAccts(conf iAMConfig) error {
}
obj := iamFile
uploader := manager.NewUploader(s.client)
upinfo := &s3.PutObjectInput{
uploader := transfermanager.New(s.client)
upinfo := &transfermanager.UploadObjectInput{
Body: bytes.NewReader(b),
Bucket: &s.bucket,
Key: &obj,
}
_, err = uploader.Upload(context.Background(), upinfo)
_, err = uploader.UploadObject(context.Background(), upinfo)
if err != nil {
return fmt.Errorf("store accounts in %v: %w", iamFile, err)
}
go.mod
+4 -7
View File
@@ -10,6 +10,9 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
github.com/DataDog/datadog-go/v5 v5.8.3
github.com/aws/aws-sdk-go-v2 v1.41.1
github.com/aws/aws-sdk-go-v2/config v1.32.7
github.com/aws/aws-sdk-go-v2/credentials v1.19.7
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.2
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0
github.com/aws/smithy-go v1.24.0
github.com/davecgh/go-spew v1.1.1
@@ -40,10 +43,7 @@ require (
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/andybalholm/brotli v1.2.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
github.com/aws/aws-sdk-go-v2/config v1.32.7
github.com/aws/aws-sdk-go-v2/credentials v1.19.7
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.0
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
@@ -71,6 +71,7 @@ require (
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.19 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/nats-io/nkeys v0.4.15 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pierrec/lz4/v4 v4.1.25 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
@@ -85,7 +86,3 @@ require (
golang.org/x/time v0.14.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
// pinned in dependabot.yml since newer versions in the go mod cache no longer
// exist. remove the ignore once this repo creates a newer tag.
require github.com/nats-io/nkeys v0.4.12 // indirect
go.sum
+4 -4
View File
@@ -35,8 +35,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUT
github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.0 h1:MpkX8EjkwuvyuX9B7+Zgk5M4URb2WQ84Y6jM81n5imw=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.0/go.mod h1:4V9Pv5sFfMPWQF0Q0zYN6BlV/504dFGaTeogallRqQw=
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.2 h1:1q8/WwEqZnM/vO4q1gx2g7lHYmyN+o4P7G6EW4zKbRQ=
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.2/go.mod h1:owKRexW+Ir5ACD2UTesmjkQ+w7mcmknLNfwOiKfVLTg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik=
@@ -137,8 +137,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/nats-io/nats.go v1.48.0 h1:pSFyXApG+yWU/TgbKCjmm5K4wrHu86231/w84qRVR+U=
github.com/nats-io/nats.go v1.48.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc=
github.com/nats-io/nkeys v0.4.12/go.mod h1:MT59A1HYcjIcyQDJStTfaOY6vhy9XTUjOFo+SVsvpBg=
github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s=
tests/integration/s3conf.go
+23 -13
View File
@@ -26,7 +26,7 @@ import (
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/smithy-go/middleware"
)
@@ -189,31 +189,41 @@ func (c *S3Conf) Config() aws.Config {
}
func (c *S3Conf) UploadData(r io.Reader, bucket, object string) error {
uploader := manager.NewUploader(c.GetClient())
uploader.PartSize = c.PartSize
uploader.Concurrency = c.Concurrency
uploader := transfermanager.New(c.GetClient(),
func(options *transfermanager.Options) {
options.PartSizeBytes = c.PartSize
options.Concurrency = c.Concurrency
})
upinfo := &s3.PutObjectInput{
upinfo := &transfermanager.UploadObjectInput{
Body: r,
Bucket: &bucket,
Key: &object,
}
_, err := uploader.Upload(context.Background(), upinfo)
_, err := uploader.UploadObject(context.Background(), upinfo)
return err
}
func (c *S3Conf) DownloadData(w io.WriterAt, bucket, object string) (int64, error) {
downloader := manager.NewDownloader(c.GetClient())
downloader.PartSize = c.PartSize
downloader.Concurrency = c.Concurrency
downloader := transfermanager.New(c.GetClient(),
func(options *transfermanager.Options) {
options.PartSizeBytes = c.PartSize
options.Concurrency = c.Concurrency
})
downinfo := &s3.GetObjectInput{
Bucket: &bucket,
Key: &object,
downinfo := &transfermanager.DownloadObjectInput{
Bucket: &bucket,
Key: &object,
WriterAt: w,
}
return downloader.Download(context.Background(), w, downinfo)
out, err := downloader.DownloadObject(context.Background(), downinfo)
if err != nil {
return 0, err
}
return *out.ContentLength, nil
}
func (c *S3Conf) getAdminCommand(args ...string) []string {