From a8d3322fb197891f9badafe1c1057cf51267e058 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Nov 2023 21:36:02 +0000 Subject: [PATCH 1/2] chore(deps): bump the dev-dependencies group with 7 updates Bumps the dev-dependencies group with 7 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.22.2` | `1.23.1` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.42.1` | `1.43.1` | | [github.com/gofiber/fiber/v2](https://github.com/gofiber/fiber) | `2.50.0` | `2.51.0` | | [github.com/segmentio/kafka-go](https://github.com/segmentio/kafka-go) | `0.4.44` | `0.4.45` | | [github.com/valyala/fasthttp](https://github.com/valyala/fasthttp) | `1.50.0` | `1.51.0` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.24.0` | `1.25.4` | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.13.6` | `1.14.1` | Updates `github.com/aws/aws-sdk-go-v2` from 1.22.2 to 1.23.1 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.22.2...v1.23.1) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.42.1 to 1.43.1 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.42.1...service/s3/v1.43.1) Updates `github.com/gofiber/fiber/v2` from 2.50.0 to 2.51.0 - [Release notes](https://github.com/gofiber/fiber/releases) - [Commits](https://github.com/gofiber/fiber/compare/v2.50.0...v2.51.0) Updates `github.com/segmentio/kafka-go` from 0.4.44 to 0.4.45 - [Release notes](https://github.com/segmentio/kafka-go/releases) - [Commits](https://github.com/segmentio/kafka-go/compare/v0.4.44...v0.4.45) Updates `github.com/valyala/fasthttp` from 1.50.0 to 1.51.0 - [Release notes](https://github.com/valyala/fasthttp/releases) - [Commits](https://github.com/valyala/fasthttp/compare/v1.50.0...v1.51.0) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.24.0 to 1.25.4 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.24.0...config/v1.25.4) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.13.6 to 1.14.1 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/service/mq/v1.14.1/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/mq/v1.13.6...service/mq/v1.14.1) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: dev-dependencies - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: dev-dependencies - dependency-name: github.com/gofiber/fiber/v2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: dev-dependencies - dependency-name: github.com/segmentio/kafka-go dependency-type: direct:production update-type: version-update:semver-patch dependency-group: dev-dependencies - dependency-name: github.com/valyala/fasthttp dependency-type: direct:production update-type: version-update:semver-minor dependency-group: dev-dependencies - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-minor dependency-group: dev-dependencies - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-type: direct:production update-type: version-update:semver-minor dependency-group: dev-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 46 ++++++++++++++--------------- go.sum | 92 +++++++++++++++++++++++++++++----------------------------- 2 files changed, 69 insertions(+), 69 deletions(-) diff --git a/go.mod b/go.mod index fe75bcc0..17a40a92 100644 --- a/go.mod +++ b/go.mod @@ -3,28 +3,28 @@ module github.com/versity/versitygw go 1.20 require ( - github.com/aws/aws-sdk-go-v2 v1.22.2 - github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1 - github.com/aws/smithy-go v1.16.0 + github.com/aws/aws-sdk-go-v2 v1.23.1 + github.com/aws/aws-sdk-go-v2/service/s3 v1.43.1 + github.com/aws/smithy-go v1.17.0 github.com/go-ldap/ldap/v3 v3.4.6 - github.com/gofiber/fiber/v2 v2.50.0 + github.com/gofiber/fiber/v2 v2.51.0 github.com/google/uuid v1.4.0 github.com/nats-io/nats.go v1.31.0 github.com/pkg/xattr v0.4.9 - github.com/segmentio/kafka-go v0.4.44 + github.com/segmentio/kafka-go v0.4.45 github.com/urfave/cli/v2 v2.25.7 - github.com/valyala/fasthttp v1.50.0 + github.com/valyala/fasthttp v1.51.0 github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 golang.org/x/sys v0.14.0 ) require ( github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.17.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.25.4 // indirect github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/nats-io/nkeys v0.4.6 // indirect @@ -36,21 +36,21 @@ require ( require ( github.com/andybalholm/brotli v1.0.5 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 // indirect - github.com/aws/aws-sdk-go-v2/config v1.24.0 - github.com/aws/aws-sdk-go-v2/credentials v1.15.2 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6 - github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 // indirect + github.com/aws/aws-sdk-go-v2/config v1.25.4 + github.com/aws/aws-sdk-go-v2/credentials v1.16.3 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.14.1 + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.4 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/klauspost/compress v1.17.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect diff --git a/go.sum b/go.sum index b1ff1ea6..19c49380 100644 --- a/go.sum +++ b/go.sum @@ -4,44 +4,44 @@ github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3Uu github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/aws/aws-sdk-go-v2 v1.22.2 h1:lV0U8fnhAnPz8YcdmZVV60+tr6CakHzqA6P8T46ExJI= -github.com/aws/aws-sdk-go-v2 v1.22.2/go.mod h1:Kd0OJtkW3Q0M0lUWGszapWjEvrXDzRW+D21JNsroB+c= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 h1:hHgLiIrTRtddC0AKcJr5s7i/hLgcpTt+q/FKxf1Zayk= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0/go.mod h1:w4I/v3NOWgD+qvs1NPEwhd++1h3XPHFaVxasfY6HlYQ= -github.com/aws/aws-sdk-go-v2/config v1.24.0 h1:4LEk29JO3w+y9dEo/5Tq5QTP7uIEw+KQrKiHOs4xlu4= -github.com/aws/aws-sdk-go-v2/config v1.24.0/go.mod h1:11nNDAuK86kOUHeuEQo8f3CkcV5xuUxvPwFjTZE/PnQ= -github.com/aws/aws-sdk-go-v2/credentials v1.15.2 h1:rKH7khRMxPdD0u3dHecd0Q7NOVw3EUe7AqdkUOkiOGI= -github.com/aws/aws-sdk-go-v2/credentials v1.15.2/go.mod h1:tXM8wmaeAhfC7nZoCxb0FzM/aRaB1m1WQ7x0qlBLq80= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 h1:G5KawTAkyHH6WyKQCdHiW4h3PmAXNJpOgwKg3H7sDRE= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3/go.mod h1:hugKmSFnZB+HgNI1sYGT14BUPZkO6alC/e0AWu+0IAQ= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6 h1:IpQbitxCZeC64C1ALz9QZu6AHHWundnU2evQ9xbp5k8= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6/go.mod h1:27jIVQK+al9s0yTo3pkMdahRinbscqSC6zNGfNWXPZc= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 h1:AaQsr5vvGR7rmeSWBtTCcw16tT9r51mWijuCQhzLnq8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2/go.mod h1:o1IiRn7CWocIFTXJjGKJDOwxv1ibL53NpcvcqGWyRBA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 h1:UZx8SXZ0YtzRiALzYAWcjb9Y9hZUR7MBKaBQ5ouOjPs= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2/go.mod h1:ipuRpcSaklmxR6C39G187TpBAO132gUfleTGccUPs8c= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 h1:usgqiJtamuGIBj+OvYmMq89+Z1hIKkMJToz1WpoeNUY= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 h1:pyVrNAf7Hwz0u39dLKN5t+n0+K/3rMYKuiOoIum3AsU= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2/go.mod h1:mydrfOb9uiOYCxuCPR8YHQNQyGQwUQ7gPMZGBKbH8NY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 h1:CJxo7ZBbaIzmXfV3hjcx36n9V87gJsIUPJflwqEHl3Q= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0/go.mod h1:yjVfjuY4nD1EW9i387Kau+I6V5cBA5YnC/mWNopjZrI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 h1:f2LhPofnjcdOQKRtumKjMvIHkfSQ8aH/rwKUDEQ/SB4= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2/go.mod h1:q+xX0H4OfuWDuBy7y/LDi4v8IBOWuF+vtp8Z6ex+lw4= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 h1:h7j73yuAVVjic8pqswh+L/7r2IHP43QwRyOu6zcCDDE= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2/go.mod h1:H07AHdK5LSy8F7EJUQhoxyiCNkePoHj2D8P2yGTWafo= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 h1:gbIaOzpXixUpoPK+js/bCBK1QBDXM22SigsnzGZio0U= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2/go.mod h1:p+S7RNbdGN8qgHDSg2SCQJ9FeMAmvcETQiVpeGhYnNM= -github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1 h1:o6MCcX1rJW8Y3g+hvg2xpjF6JR6DftuYhfl3Nc1WV9Q= -github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1/go.mod h1:UDtxEWbREX6y4KREapT+jjtjoH0TiVSS6f5nfaY1UaM= -github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 h1:km+ZNjtLtpXYf42RdaDZnNHm9s7SYAuDGTafy6nd89A= -github.com/aws/aws-sdk-go-v2/service/sso v1.17.1/go.mod h1:aHBr3pvBSD5MbzOvQtYutyPLLRPbl/y9x86XyJJnUXQ= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 h1:iRFNqZH4a67IqPvK8xxtyQYnyrlsvwmpHOe9r55ggBA= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1/go.mod h1:pTy5WM+6sNv2tB24JNKFtn6EvciQ5k40ZJ0pq/Iaxj0= -github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 h1:txgVXIXWPXyqdiVn92BV6a/rgtpX31HYdsOYj0sVQQQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.25.1/go.mod h1:VAiJiNaoP1L89STFlEMgmHX1bKixY+FaP+TpRFrmyZ4= -github.com/aws/smithy-go v1.16.0 h1:gJZEH/Fqh+RsvlJ1Zt4tVAtV6bKkp3cC+R6FCZMNzik= -github.com/aws/smithy-go v1.16.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= +github.com/aws/aws-sdk-go-v2 v1.23.1 h1:qXaFsOOMA+HsZtX8WoCa+gJnbyW7qyFFBlPqvTSzbaI= +github.com/aws/aws-sdk-go-v2 v1.23.1/go.mod h1:i1XDttT4rnf6vxc9AuskLc6s7XBee8rlLilKlc03uAA= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 h1:ZY3108YtBNq96jNZTICHxN1gSBSbnvIdYwwqnvCV4Mc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1/go.mod h1:t8PYl/6LzdAqsU4/9tz28V/kU+asFePvpOMkdul0gEQ= +github.com/aws/aws-sdk-go-v2/config v1.25.4 h1:r+X1x8QI6FEPdJDWCNBDZHyAcyFwSjHN8q8uuus+Axs= +github.com/aws/aws-sdk-go-v2/config v1.25.4/go.mod h1:8GTjImECskr7D88P/Nn9uM4M4rLY9i77hLJZgkZEWV8= +github.com/aws/aws-sdk-go-v2/credentials v1.16.3 h1:8PeI2krzzjDJ5etmgaMiD1JswsrLrWvKKu/uBUtNy1g= +github.com/aws/aws-sdk-go-v2/credentials v1.16.3/go.mod h1:Kdh/okh+//vQ/AjEt81CjvkTo64+/zIE4OewP7RpfXk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5 h1:KehRNiVzIfAcj6gw98zotVbb/K67taJE0fkfgM6vzqU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5/go.mod h1:VhnExhw6uXy9QzetvpXDolo1/hjhx4u9qukBGkuUwjs= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.14.1 h1:AozIaYZ7g0qxdaA5CbJwtcZKKsopcZ/Xea8XdLPsHu8= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.14.1/go.mod h1:WgrpsWiiOlz+gpdPuhHylSItxh3gXMgW03Qfj3a21uA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4 h1:LAm3Ycm9HJfbSCd5I+wqC2S9Ej7FPrgr5CQoOljJZcE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4/go.mod h1:xEhvbJcyUf/31yfGSQBe01fukXwXJ0gxDp7rLfymWE0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4 h1:4GV0kKZzUxiWxSVpn/9gwR0g21NF1Jsyduzo9rHgC/Q= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4/go.mod h1:dYvTNAggxDZy6y1AF7YDwXsPuHFy/VNEpEI/2dWK9IU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.4 h1:40Q4X5ebZruRtknEZH/bg91sT5pR853F7/1X9QRbI54= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.4/go.mod h1:u77N7eEECzUv7F0xl2gcfK/vzc8wcjWobpy+DcrLJ5E= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1 h1:rpkF4n0CyFcrJUG/rNNohoTmhtWlFTRI4BsZOh9PvLs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1/go.mod h1:l9ymW25HOqymeU2m1gbUQ3rUIsTwKs8gYHXkqDQUhiI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.4 h1:6DRKQc+9cChgzL5gplRGusI5dBGeiEod4m/pmGbcX48= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.4/go.mod h1:s8ORvrW4g4v7IvYKIAoBg17w3GQ+XuwXDXYrQ5SkzU0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4 h1:rdovz3rEu0vZKbzoMYPTehp0E8veoE9AyfzqCr5Eeao= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4/go.mod h1:aYCGNjyUCUelhofxlZyj63srdxWUSsBSGg5l6MCuXuE= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.4 h1:o3DcfCxGDIT20pTbVKVhp3vWXOj/VvgazNJvumWeYW0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.4/go.mod h1:Uy0KVOxuTK2ne+/PKQ+VvEeWmjMMksE17k/2RK/r5oM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.43.1 h1:1w11lfXOa8HoHoSlNtt4mqv/N3HmDOa+OnUH3Y9DHm8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.43.1/go.mod h1:dqJ5JBL0clzgHriH35Amx3LRFY6wNIPUX7QO/BerSBo= +github.com/aws/aws-sdk-go-v2/service/sso v1.17.3 h1:CdsSOGlFF3Pn+koXOIpTtvX7st0IuGsZ8kJqcWMlX54= +github.com/aws/aws-sdk-go-v2/service/sso v1.17.3/go.mod h1:oA6VjNsLll2eVuUoF2D+CMyORgNzPEW/3PyUdq6WQjI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1 h1:cbRqFTVnJV+KRpwFl76GJdIZJKKCdTPnjUZ7uWh3pIU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1/go.mod h1:hHL974p5auvXlZPIjJTblXJpbkfK4klBczlsEaMCGVY= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.4 h1:yEvZ4neOQ/KpUqyR+X0ycUTW/kVRNR4nDZ38wStHGAA= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.4/go.mod h1:feTnm2Tk/pJxdX+eooEsxvlvTWBvDm6CasRZ+JOs2IY= +github.com/aws/smithy-go v1.17.0 h1:wWJD7LX6PBV6etBUwO0zElG0nWN9rUhp0WdYeHSHAaI= +github.com/aws/smithy-go v1.17.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -51,8 +51,8 @@ github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A= github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc= -github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw= -github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw= +github.com/gofiber/fiber/v2 v2.51.0 h1:JNACcZy5e2tGApWB2QrRpenTWn0fq0hkFm6k0C86gKQ= +github.com/gofiber/fiber/v2 v2.51.0/go.mod h1:xaQRZQJGqnKOQnbQw+ltvku3/h8QxvNi8o6JiJ7Ll0U= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= @@ -67,8 +67,8 @@ github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQs github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E= @@ -89,8 +89,8 @@ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/segmentio/kafka-go v0.4.44 h1:Vjjksniy0WSTZ7CuVJrz1k04UoZeTc77UV6Yyk6tLY4= -github.com/segmentio/kafka-go v0.4.44/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= +github.com/segmentio/kafka-go v0.4.45 h1:prqrZp1mMId4kI6pyPolkLsH6sWOUmDxmmucbL4WS6E= +github.com/segmentio/kafka-go v0.4.45/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -102,8 +102,8 @@ github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M= -github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= +github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA= +github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM= From cd8ad7d4823e9ec4d54de2bda4aed0615e135ce9 Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Tue, 28 Nov 2023 09:45:28 -0800 Subject: [PATCH 2/2] fix: breaking changes with aws sdk updates --- backend/posix/posix.go | 259 ++++++++++++++++++++++++++++----- backend/s3proxy/s3.go | 12 +- backend/scoutfs/scoutfs.go | 68 +++++++-- backend/walk_test.go | 4 +- integration/tests.go | 113 ++++++++------ integration/utils.go | 12 +- s3api/controllers/base.go | 47 +++--- s3api/controllers/base_test.go | 6 +- 8 files changed, 394 insertions(+), 127 deletions(-) diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 02d6867a..a5c88da7 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -146,6 +146,10 @@ func (p *Posix) ListBuckets(_ context.Context, owner string, isAdmin bool) (s3re } func (p *Posix) HeadBucket(_ context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + _, err := os.Lstat(*input.Bucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -158,6 +162,10 @@ func (p *Posix) HeadBucket(_ context.Context, input *s3.HeadBucketInput) (*s3.He } func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput) error { + if input.Bucket == nil { + return s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + bucket := *input.Bucket owner := string(input.ObjectOwnership) @@ -183,6 +191,10 @@ func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput) err } func (p *Posix) DeleteBucket(_ context.Context, input *s3.DeleteBucketInput) error { + if input.Bucket == nil { + return s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + names, err := os.ReadDir(*input.Bucket) if errors.Is(err, fs.ErrNotExist) { return s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -212,6 +224,13 @@ func (p *Posix) DeleteBucket(_ context.Context, input *s3.DeleteBucketInput) err } func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) { + if mpu.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if mpu.Key == nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + bucket := *mpu.Bucket object := *mpu.Key @@ -269,6 +288,19 @@ func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipart } func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if input.Key == nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if input.UploadId == nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload) + } + if input.MultipartUpload == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + bucket := *input.Bucket object := *input.Key uploadID := *input.UploadId @@ -294,7 +326,7 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul partsize := int64(0) var totalsize int64 for i, p := range parts { - partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber)) + partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber)) fi, err := os.Lstat(partPath) if err != nil { return nil, s3err.GetAPIError(s3err.ErrInvalidPart) @@ -326,7 +358,7 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul defer f.cleanup() for _, p := range parts { - pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber))) + pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber))) if err != nil { return nil, fmt.Errorf("open part %v: %v", p.PartNumber, err) } @@ -512,6 +544,16 @@ func mkdirAll(path string, perm os.FileMode, bucket, object string) error { } func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUploadInput) error { + if mpu.Bucket == nil { + return s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if mpu.Key == nil { + return s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if mpu.UploadId == nil { + return s3err.GetAPIError(s3err.ErrNoSuchUpload) + } + bucket := *mpu.Bucket object := *mpu.Key uploadID := *mpu.UploadId @@ -542,6 +584,12 @@ func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUp } func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) { + var lmu s3response.ListMultipartUploadsResult + + if mpu.Bucket == nil { + return lmu, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + bucket := *mpu.Bucket var delimiter string if mpu.Delimiter != nil { @@ -552,8 +600,6 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl prefix = *mpu.Prefix } - var lmu s3response.ListMultipartUploadsResult - _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { return lmu, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -626,12 +672,16 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl } } + maxUploads := 0 + if mpu.MaxUploads != nil { + maxUploads = int(*mpu.MaxUploads) + } if (uploadIDMarker != "" && !uploadIdMarkerFound) || (keyMarker != "" && keyMarkerInd == -1) { return s3response.ListMultipartUploadsResult{ Bucket: bucket, Delimiter: delimiter, KeyMarker: keyMarker, - MaxUploads: int(mpu.MaxUploads), + MaxUploads: maxUploads, Prefix: prefix, UploadIDMarker: uploadIDMarker, Uploads: []s3response.Upload{}, @@ -643,18 +693,18 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl }) for i := keyMarkerInd + 1; i < len(uploads); i++ { - if mpu.MaxUploads == 0 { + if maxUploads == 0 { break } if keyMarker != "" && uploadIDMarker != "" && uploads[i].UploadID < uploadIDMarker { continue } - if i != len(uploads)-1 && len(resultUpds) == int(mpu.MaxUploads) { + if i != len(uploads)-1 && len(resultUpds) == maxUploads { return s3response.ListMultipartUploadsResult{ Bucket: bucket, Delimiter: delimiter, KeyMarker: keyMarker, - MaxUploads: int(mpu.MaxUploads), + MaxUploads: maxUploads, NextKeyMarker: resultUpds[i-1].Key, NextUploadIDMarker: resultUpds[i-1].UploadID, IsTruncated: true, @@ -671,7 +721,7 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl Bucket: bucket, Delimiter: delimiter, KeyMarker: keyMarker, - MaxUploads: int(mpu.MaxUploads), + MaxUploads: maxUploads, Prefix: prefix, UploadIDMarker: uploadIDMarker, Uploads: resultUpds, @@ -679,13 +729,29 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl } func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) { + var lpr s3response.ListPartsResult + + if input.Bucket == nil { + return lpr, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if input.Key == nil { + return lpr, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if input.UploadId == nil { + return lpr, s3err.GetAPIError(s3err.ErrNoSuchUpload) + } + bucket := *input.Bucket object := *input.Key uploadID := *input.UploadId - stringMarker := *input.PartNumberMarker - maxParts := int(input.MaxParts) - - var lpr s3response.ListPartsResult + stringMarker := "" + if input.PartNumberMarker != nil { + stringMarker = *input.PartNumberMarker + } + maxParts := 0 + if input.MaxParts != nil { + maxParts = int(*input.MaxParts) + } var partNumberMarker int if stringMarker != "" { @@ -777,11 +843,21 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon } func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string, error) { + if input.Bucket == nil { + return "", s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if input.Key == nil { + return "", s3err.GetAPIError(s3err.ErrNoSuchKey) + } + bucket := *input.Bucket object := *input.Key uploadID := *input.UploadId part := input.PartNumber - length := input.ContentLength + length := int64(0) + if input.ContentLength != nil { + length = *input.ContentLength + } r := input.Body _, err := os.Stat(bucket) @@ -803,7 +879,7 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string return "", fmt.Errorf("stat uploadid: %w", err) } - partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", part)) + partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part)) f, err := openTmpFile(filepath.Join(bucket, objdir), bucket, partPath, length) @@ -833,6 +909,13 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string } func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { + if upi.Bucket == nil { + return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if upi.Key == nil { + return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + _, err := os.Stat(*upi.Bucket) if errors.Is(err, fs.ErrNotExist) { return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -852,7 +935,7 @@ func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) ( return s3response.CopyObjectResult{}, fmt.Errorf("stat uploadid: %w", err) } - partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", upi.PartNumber)) + partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", *upi.PartNumber)) substrs := strings.SplitN(*upi.CopySource, "/", 2) if len(substrs) != 2 { @@ -938,6 +1021,13 @@ func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) ( } func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, error) { + if po.Bucket == nil { + return "", s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if po.Key == nil { + return "", s3err.GetAPIError(s3err.ErrNoSuchKey) + } + tagsStr := getString(po.Tagging) tags := make(map[string]string) _, err := os.Stat(*po.Bucket) @@ -964,9 +1054,13 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e name := filepath.Join(*po.Bucket, *po.Key) + contentLength := int64(0) + if po.ContentLength != nil { + contentLength = *po.ContentLength + } if strings.HasSuffix(*po.Key, "/") { // object is directory - if po.ContentLength != 0 { + if contentLength != 0 { // posix directories can't contain data, send error // if reuests has a data payload associated with a // directory object @@ -995,7 +1089,7 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e } f, err := openTmpFile(filepath.Join(*po.Bucket, metaTmpDir), - *po.Bucket, *po.Key, po.ContentLength) + *po.Bucket, *po.Key, contentLength) if err != nil { return "", fmt.Errorf("open temp file: %w", err) } @@ -1039,6 +1133,13 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e } func (p *Posix) DeleteObject(_ context.Context, input *s3.DeleteObjectInput) error { + if input.Bucket == nil { + return s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if input.Key == nil { + return s3err.GetAPIError(s3err.ErrNoSuchKey) + } + bucket := *input.Bucket object := *input.Key @@ -1131,6 +1232,16 @@ func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) } func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if input.Key == nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if input.Range == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidRange) + } + bucket := *input.Bucket _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { @@ -1192,15 +1303,17 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io return nil, fmt.Errorf("get object tags: %w", err) } + tagCount := int32(len(tags)) + return &s3.GetObjectOutput{ AcceptRanges: &acceptRange, - ContentLength: length, + ContentLength: &length, ContentEncoding: &contentEncoding, ContentType: &contentType, ETag: &etag, LastModified: backend.GetTimePtr(fi.ModTime()), Metadata: userMetaData, - TagCount: int32(len(tags)), + TagCount: &tagCount, ContentRange: &contentRange, }, nil } @@ -1235,20 +1348,28 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io return nil, fmt.Errorf("get object tags: %w", err) } + tagCount := int32(len(tags)) + return &s3.GetObjectOutput{ AcceptRanges: &acceptRange, - ContentLength: length, + ContentLength: &length, ContentEncoding: &contentEncoding, ContentType: &contentType, ETag: &etag, LastModified: backend.GetTimePtr(fi.ModTime()), Metadata: userMetaData, - TagCount: int32(len(tags)), + TagCount: &tagCount, ContentRange: &contentRange, }, nil } func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if input.Key == nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } bucket := *input.Bucket object := *input.Key @@ -1278,8 +1399,10 @@ func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.He etag = "" } + size := fi.Size() + return &s3.HeadObjectOutput{ - ContentLength: fi.Size(), + ContentLength: &size, ContentType: &contentType, ContentEncoding: &contentEncoding, ETag: &etag, @@ -1289,6 +1412,18 @@ func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.He } func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if input.Key == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidCopyDest) + } + if input.CopySource == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidCopySource) + } + if input.ExpectedBucketOwner == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidRequest) + } srcBucket, srcObject, ok := strings.Cut(*input.CopySource, "/") if !ok { return nil, s3err.GetAPIError(s3err.ErrInvalidCopySource) @@ -1361,7 +1496,16 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. } } - etag, err := p.PutObject(ctx, &s3.PutObjectInput{Bucket: &dstBucket, Key: &dstObject, Body: f, ContentLength: fInfo.Size(), Metadata: meta}) + contentLength := fInfo.Size() + + etag, err := p.PutObject(ctx, + &s3.PutObjectInput{ + Bucket: &dstBucket, + Key: &dstObject, + Body: f, + ContentLength: &contentLength, + Metadata: meta, + }) if err != nil { return nil, err } @@ -1380,11 +1524,26 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. } func (p *Posix) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } bucket := *input.Bucket - prefix := *input.Prefix - marker := *input.Marker - delim := *input.Delimiter - maxkeys := input.MaxKeys + prefix := "" + if input.Prefix != nil { + prefix = *input.Prefix + } + marker := "" + if input.Marker != nil { + marker = *input.Marker + } + delim := "" + if input.Delimiter != nil { + delim = *input.Delimiter + } + maxkeys := int32(0) + if input.MaxKeys != nil { + maxkeys = *input.MaxKeys + } _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { @@ -1405,9 +1564,9 @@ func (p *Posix) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3. CommonPrefixes: results.CommonPrefixes, Contents: results.Objects, Delimiter: &delim, - IsTruncated: results.Truncated, + IsTruncated: &results.Truncated, Marker: &marker, - MaxKeys: maxkeys, + MaxKeys: &maxkeys, Name: &bucket, NextMarker: &results.NextMarker, Prefix: &prefix, @@ -1466,21 +1625,38 @@ func fileToObj(bucket string) backend.GetObjFunc { return types.Object{}, fmt.Errorf("get fileinfo: %w", err) } + size := fi.Size() + return types.Object{ ETag: &etag, Key: &path, LastModified: backend.GetTimePtr(fi.ModTime()), - Size: fi.Size(), + Size: &size, }, nil } } func (p *Posix) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } bucket := *input.Bucket - prefix := *input.Prefix - marker := *input.ContinuationToken - delim := *input.Delimiter - maxkeys := input.MaxKeys + prefix := "" + if input.Prefix != nil { + prefix = *input.Prefix + } + marker := "" + if input.ContinuationToken != nil { + marker = *input.ContinuationToken + } + delim := "" + if input.Delimiter != nil { + delim = *input.Delimiter + } + maxkeys := int32(0) + if input.MaxKeys != nil { + maxkeys = *input.MaxKeys + } _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { @@ -1491,23 +1667,25 @@ func (p *Posix) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) ( } fileSystem := os.DirFS(bucket) - results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys), + results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys, fileToObj(bucket), []string{metaTmpDir}) if err != nil { return nil, fmt.Errorf("walk %v: %w", bucket, err) } + count := int32(len(results.Objects)) + return &s3.ListObjectsV2Output{ CommonPrefixes: results.CommonPrefixes, Contents: results.Objects, Delimiter: &delim, - IsTruncated: results.Truncated, + IsTruncated: &results.Truncated, ContinuationToken: &marker, - MaxKeys: int32(maxkeys), + MaxKeys: &maxkeys, Name: &bucket, NextContinuationToken: &results.NextMarker, Prefix: &prefix, - KeyCount: int32(len(results.Objects)), + KeyCount: &count, }, nil } @@ -1528,6 +1706,9 @@ func (p *Posix) PutBucketAcl(_ context.Context, bucket string, data []byte) erro } func (p *Posix) GetBucketAcl(_ context.Context, input *s3.GetBucketAclInput) ([]byte, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } _, err := os.Stat(*input.Bucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) diff --git a/backend/s3proxy/s3.go b/backend/s3proxy/s3.go index 19e43750..b02ddbe3 100644 --- a/backend/s3proxy/s3.go +++ b/backend/s3proxy/s3.go @@ -184,8 +184,8 @@ func (s *S3be) ListMultipartUploads(ctx context.Context, input *s3.ListMultipart Delimiter: *output.Delimiter, Prefix: *output.Prefix, EncodingType: string(output.EncodingType), - MaxUploads: int(output.MaxUploads), - IsTruncated: output.IsTruncated, + MaxUploads: int(*output.MaxUploads), + IsTruncated: *output.IsTruncated, Uploads: uploads, CommonPrefixes: cps, }, nil @@ -205,10 +205,10 @@ func (s *S3be) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3respo var parts []s3response.Part for _, p := range output.Parts { parts = append(parts, s3response.Part{ - PartNumber: int(p.PartNumber), + PartNumber: int(*p.PartNumber), LastModified: p.LastModified.Format(iso8601Format), ETag: *p.ETag, - Size: p.Size, + Size: *p.Size, }) } pnm, err := strconv.Atoi(*output.PartNumberMarker) @@ -238,8 +238,8 @@ func (s *S3be) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3respo StorageClass: string(output.StorageClass), PartNumberMarker: pnm, NextPartNumberMarker: npmn, - MaxParts: int(output.MaxParts), - IsTruncated: output.IsTruncated, + MaxParts: int(*output.MaxParts), + IsTruncated: *output.IsTruncated, Parts: parts, }, nil } diff --git a/backend/scoutfs/scoutfs.go b/backend/scoutfs/scoutfs.go index 47f30de5..35a6ecb7 100644 --- a/backend/scoutfs/scoutfs.go +++ b/backend/scoutfs/scoutfs.go @@ -415,8 +415,10 @@ func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3. } } + contentLength := fi.Size() + return &s3.HeadObjectOutput{ - ContentLength: fi.Size(), + ContentLength: &contentLength, ContentType: &contentType, ContentEncoding: &contentEncoding, ETag: &etag, @@ -507,15 +509,17 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer return nil, fmt.Errorf("get object tags: %w", err) } + tagCount := int32(len(tags)) + return &s3.GetObjectOutput{ AcceptRanges: &acceptRange, - ContentLength: length, + ContentLength: &length, ContentEncoding: &contentEncoding, ContentType: &contentType, ETag: &etag, LastModified: backend.GetTimePtr(fi.ModTime()), Metadata: userMetaData, - TagCount: int32(len(tags)), + TagCount: &tagCount, StorageClass: types.StorageClassStandard, }, nil } @@ -542,11 +546,26 @@ func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error) } func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } bucket := *input.Bucket - prefix := *input.Prefix - marker := *input.Marker - delim := *input.Delimiter - maxkeys := input.MaxKeys + prefix := "" + if input.Prefix != nil { + prefix = *input.Prefix + } + marker := "" + if input.Marker != nil { + marker = *input.Marker + } + delim := "" + if input.Delimiter != nil { + delim = *input.Delimiter + } + maxkeys := int32(0) + if input.MaxKeys != nil { + maxkeys = *input.MaxKeys + } _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { @@ -567,9 +586,9 @@ func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s CommonPrefixes: results.CommonPrefixes, Contents: results.Objects, Delimiter: &delim, - IsTruncated: results.Truncated, + IsTruncated: &results.Truncated, Marker: &marker, - MaxKeys: maxkeys, + MaxKeys: &maxkeys, Name: &bucket, NextMarker: &results.NextMarker, Prefix: &prefix, @@ -577,11 +596,26 @@ func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s } func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } bucket := *input.Bucket - prefix := *input.Prefix - marker := *input.ContinuationToken - delim := *input.Delimiter - maxkeys := input.MaxKeys + prefix := "" + if input.Prefix != nil { + prefix = *input.Prefix + } + marker := "" + if input.ContinuationToken != nil { + marker = *input.ContinuationToken + } + delim := "" + if input.Delimiter != nil { + delim = *input.Delimiter + } + maxkeys := int32(0) + if input.MaxKeys != nil { + maxkeys = *input.MaxKeys + } _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { @@ -602,9 +636,9 @@ func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) CommonPrefixes: results.CommonPrefixes, Contents: results.Objects, Delimiter: &delim, - IsTruncated: results.Truncated, + IsTruncated: &results.Truncated, ContinuationToken: &marker, - MaxKeys: int32(maxkeys), + MaxKeys: &maxkeys, Name: &bucket, NextContinuationToken: &results.NextMarker, Prefix: &prefix, @@ -677,11 +711,13 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc { } } + size := fi.Size() + return types.Object{ ETag: &etag, Key: &path, LastModified: backend.GetTimePtr(fi.ModTime()), - Size: fi.Size(), + Size: &size, StorageClass: sc, }, nil } diff --git a/backend/walk_test.go b/backend/walk_test.go index a7b31013..a7b4b826 100644 --- a/backend/walk_test.go +++ b/backend/walk_test.go @@ -55,11 +55,13 @@ func getObj(path string, d fs.DirEntry) (types.Object, error) { return types.Object{}, fmt.Errorf("get fileinfo: %w", err) } + size := fi.Size() + return types.Object{ ETag: &etag, Key: &path, LastModified: backend.GetTimePtr(fi.ModTime()), - Size: fi.Size(), + Size: &size, }, nil } diff --git a/integration/tests.go b/integration/tests.go index 7aa4d523..043782aa 100644 --- a/integration/tests.go +++ b/integration/tests.go @@ -1123,8 +1123,12 @@ func HeadObject_success(s *S3Conf) { if !areMapsSame(out.Metadata, meta) { return fmt.Errorf("incorrect object metadata") } - if out.ContentLength != dataLen { - return fmt.Errorf("expected data length %v, instead got %v", dataLen, out.ContentLength) + contentLength := int64(0) + if out.ContentLength != nil { + contentLength = *out.ContentLength + } + if contentLength != dataLen { + return fmt.Errorf("expected data length %v, instead got %v", dataLen, contentLength) } return nil @@ -1251,7 +1255,7 @@ func GetObject_success(s *S3Conf) { if err != nil { return err } - if out.ContentLength != dataLength { + if *out.ContentLength != dataLength { return fmt.Errorf("expected content-length %v, instead got %v", dataLength, out.ContentLength) } @@ -1396,18 +1400,18 @@ func ListObject_truncated(s *S3Conf) { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out1, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ Bucket: &bucket, - MaxKeys: maxKeys, + MaxKeys: &maxKeys, }) cancel() if err != nil { return err } - if !out1.IsTruncated { + if !*out1.IsTruncated { return fmt.Errorf("expected out1put to be truncated") } - if out1.MaxKeys != maxKeys { + if *out1.MaxKeys != maxKeys { return fmt.Errorf("expected max-keys to be %v, instead got %v", maxKeys, out1.MaxKeys) } @@ -1429,7 +1433,7 @@ func ListObject_truncated(s *S3Conf) { return err } - if out2.IsTruncated { + if *out2.IsTruncated { return fmt.Errorf("expected output not to be truncated") } @@ -1446,11 +1450,12 @@ func ListObject_truncated(s *S3Conf) { func ListObjects_invalid_max_keys(s *S3Conf) { testName := "ListObjects_invalid_max_keys" + maxKeys := int32(-5) actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ Bucket: &bucket, - MaxKeys: -5, + MaxKeys: &maxKeys, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidMaxKeys)); err != nil { @@ -1470,16 +1475,17 @@ func ListObjects_max_keys_0(s *S3Conf) { return err } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + maxKeys := int32(0) out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ Bucket: &bucket, - MaxKeys: 0, + MaxKeys: &maxKeys, }) cancel() if err != nil { return nil } - if !compareObjects(objects, out.Contents) { + if len(out.Contents) > 0 { return fmt.Errorf("unexpected output for list objects with max-keys 0") } @@ -1537,7 +1543,7 @@ func ListObjects_max_keys_none(s *S3Conf) { return err } - if out.MaxKeys != 1000 { + if *out.MaxKeys != 1000 { return fmt.Errorf("expected max-keys to be 1000, instead got %v", out.MaxKeys) } @@ -1944,7 +1950,7 @@ func CopyObject_success(s *S3Conf) { if err != nil { return err } - if out.ContentLength != dataLength { + if *out.ContentLength != dataLength { return fmt.Errorf("expected content-length %v, instead got %v", dataLength, out.ContentLength) } @@ -2252,11 +2258,13 @@ func UploadPart_non_existing_bucket(s *S3Conf) { testName := "UploadPart_non_existing_bucket" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { bucketName := getBucketName() + partNumber := int32(1) ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucketName, - Key: getPtr("my-obj"), - UploadId: getPtr("uploadId"), + Bucket: &bucketName, + Key: getPtr("my-obj"), + UploadId: getPtr("uploadId"), + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { @@ -2269,13 +2277,14 @@ func UploadPart_non_existing_bucket(s *S3Conf) { func UploadPart_invalid_part_number(s *S3Conf) { testName := "UploadPart_invalid_part_number" + partNumber := int32(-10) actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ Bucket: &bucket, Key: getPtr("my-obj"), UploadId: getPtr("uploadId"), - PartNumber: -10, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidPart)); err != nil { @@ -2287,13 +2296,14 @@ func UploadPart_invalid_part_number(s *S3Conf) { func UploadPart_non_existing_mp_upload(s *S3Conf) { testName := "UploadPart_non_existing_mp_upload" + partNumber := int32(1) actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ Bucket: &bucket, Key: getPtr("my-obj"), UploadId: getPtr("uploadId"), - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { @@ -2305,6 +2315,7 @@ func UploadPart_non_existing_mp_upload(s *S3Conf) { func UploadPart_non_existing_key(s *S3Conf) { testName := "UploadPart_non_existing_key" + partNumber := int32(1) actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" out, err := createMp(s3client, bucket, obj) @@ -2317,7 +2328,7 @@ func UploadPart_non_existing_key(s *S3Conf) { Bucket: &bucket, Key: getPtr("non-existing-object-key"), UploadId: out.UploadId, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { @@ -2329,6 +2340,7 @@ func UploadPart_non_existing_key(s *S3Conf) { func UploadPart_success(s *S3Conf) { testName := "UploadPart_success" + partNumber := int32(1) actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" out, err := createMp(s3client, bucket, obj) @@ -2340,7 +2352,7 @@ func UploadPart_success(s *S3Conf) { Bucket: &bucket, Key: &obj, UploadId: out.UploadId, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err != nil { @@ -2357,6 +2369,7 @@ func UploadPartCopy_non_existing_bucket(s *S3Conf) { testName := "UploadPartCopy_non_existing_bucket" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { bucketName := getBucketName() + partNumber := int32(1) ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ @@ -2364,6 +2377,7 @@ func UploadPartCopy_non_existing_bucket(s *S3Conf) { CopySource: getPtr("Copy-Source"), UploadId: getPtr("uploadId"), Key: getPtr("my-obj"), + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { @@ -2392,12 +2406,13 @@ func UploadPartCopy_incorrect_uploadId(s *S3Conf) { } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/" + srcObj), UploadId: getPtr("incorrect-upload-id"), Key: &obj, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { @@ -2432,12 +2447,13 @@ func UploadPartCopy_incorrect_object_key(s *S3Conf) { } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/" + srcObj), UploadId: out.UploadId, Key: getPtr("non-existing-object-key"), - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { @@ -2457,12 +2473,13 @@ func UploadPartCopy_invalid_part_number(s *S3Conf) { testName := "UploadPartCopy_invalid_part_number" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(-10) _, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr("Copy-Source"), UploadId: getPtr("uploadId"), Key: getPtr("non-existing-object-key"), - PartNumber: -10, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidPart)); err != nil { @@ -2484,12 +2501,13 @@ func UploadPartCopy_invalid_copy_source(s *S3Conf) { } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr("invalid-copy-source"), UploadId: out.UploadId, Key: &obj, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidCopySource)); err != nil { @@ -2511,12 +2529,13 @@ func UploadPartCopy_non_existing_source_bucket(s *S3Conf) { } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr("src/bucket/src/obj"), UploadId: out.UploadId, Key: &obj, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { @@ -2543,12 +2562,13 @@ func UploadPartCopy_non_existing_source_object_key(s *S3Conf) { } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/non/existing/obj/key"), UploadId: out.UploadId, Key: &obj, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchKey)); err != nil { @@ -2587,12 +2607,13 @@ func UploadPartCopy_success(s *S3Conf) { } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) copyOut, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/" + srcObj), UploadId: out.UploadId, Key: &obj, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err != nil { @@ -2613,10 +2634,10 @@ func UploadPartCopy_success(s *S3Conf) { if len(res.Parts) != 1 { return fmt.Errorf("expected parts to be 1, instead got %v", len(res.Parts)) } - if res.Parts[0].PartNumber != 1 { + if *res.Parts[0].PartNumber != 1 { return fmt.Errorf("expected part-number to be 1, instead got %v", res.Parts[0].PartNumber) } - if res.Parts[0].Size != int64(objSize) { + if *res.Parts[0].Size != int64(objSize) { return fmt.Errorf("expected part size to be %v, instead got %v", objSize, res.Parts[0].Size) } if *res.Parts[0].ETag != *copyOut.CopyPartResult.ETag { @@ -2655,12 +2676,13 @@ func UploadPartCopy_by_range_invalid_range(s *S3Conf) { } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/" + srcObj), UploadId: out.UploadId, Key: &obj, - PartNumber: 1, + PartNumber: &partNumber, CopySourceRange: getPtr("invalid-range"), }) cancel() @@ -2700,13 +2722,14 @@ func UploadPartCopy_greater_range_than_obj_size(s *S3Conf) { } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/" + srcObj), UploadId: out.UploadId, Key: &obj, CopySourceRange: getPtr(fmt.Sprintf("bytes=0-%v", srcObjSize+50)), // The specified range is greater than the actual object size - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRange)); err != nil { @@ -2745,13 +2768,14 @@ func UploadPartCopy_by_range_success(s *S3Conf) { } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) copyOut, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/" + srcObj), CopySourceRange: getPtr("bytes=100-200"), UploadId: out.UploadId, Key: &obj, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err != nil { @@ -2772,10 +2796,10 @@ func UploadPartCopy_by_range_success(s *S3Conf) { if len(res.Parts) != 1 { return fmt.Errorf("expected parts to be 1, instead got %v", len(res.Parts)) } - if res.Parts[0].PartNumber != 1 { + if *res.Parts[0].PartNumber != 1 { return fmt.Errorf("expected part-number to be 1, instead got %v", res.Parts[0].PartNumber) } - if res.Parts[0].Size != 101 { + if *res.Parts[0].Size != 101 { return fmt.Errorf("expected part size to be %v, instead got %v", 101, res.Parts[0].Size) } if *res.Parts[0].ETag != *copyOut.CopyPartResult.ETag { @@ -2904,11 +2928,12 @@ func ListMultipartUploads_empty_result(s *S3Conf) { func ListMultipartUploads_invalid_max_uploads(s *S3Conf) { testName := "ListMultipartUploads_invalid_max_uploads" + maxUploads := int32(-3) actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ Bucket: &bucket, - MaxUploads: -3, + MaxUploads: &maxUploads, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidMaxKeys)); err != nil { @@ -2931,18 +2956,19 @@ func ListMultipartUploads_max_uploads(s *S3Conf) { uploads = append(uploads, types.MultipartUpload{UploadId: out.UploadId, Key: out.Key}) } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + maxUploads := int32(2) out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ Bucket: &bucket, - MaxUploads: 2, + MaxUploads: &maxUploads, }) cancel() if err != nil { return err } - if !out.IsTruncated { + if !*out.IsTruncated { return fmt.Errorf("expected the output to be truncated") } - if out.MaxUploads != 2 { + if *out.MaxUploads != 2 { return fmt.Errorf("expected max-uploads to be 2, instead got %v", out.MaxUploads) } if ok := compareMultipartUploads(out.Uploads, uploads[:2]); !ok { @@ -3227,11 +3253,12 @@ func CompleteMultipartUpload_invalid_part_number(s *S3Conf) { return err } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) res, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ Bucket: &bucket, Key: &obj, UploadId: out.UploadId, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err != nil { @@ -3239,6 +3266,7 @@ func CompleteMultipartUpload_invalid_part_number(s *S3Conf) { } ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + partNumber = int32(5) _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ Bucket: &bucket, Key: &obj, @@ -3247,7 +3275,7 @@ func CompleteMultipartUpload_invalid_part_number(s *S3Conf) { Parts: []types.CompletedPart{ { ETag: res.ETag, - PartNumber: 5, + PartNumber: &partNumber, }, }, }, @@ -3270,11 +3298,12 @@ func CompleteMultipartUpload_invalid_ETag(s *S3Conf) { return err } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ Bucket: &bucket, Key: &obj, UploadId: out.UploadId, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err != nil { @@ -3290,7 +3319,7 @@ func CompleteMultipartUpload_invalid_ETag(s *S3Conf) { Parts: []types.CompletedPart{ { ETag: getPtr("invalidETag"), - PartNumber: 1, + PartNumber: &partNumber, }, }, }, @@ -3358,7 +3387,7 @@ func CompleteMultipartUpload_success(s *S3Conf) { if *resp.ETag != *res.ETag { return fmt.Errorf("expected the uploaded object etag to be %v, instead got %v", *res.ETag, *resp.ETag) } - if resp.ContentLength != int64(objSize) { + if *resp.ContentLength != int64(objSize) { return fmt.Errorf("expected the uploaded object size to be %v, instead got %v", objSize, resp.ContentLength) } diff --git a/integration/utils.go b/integration/utils.go index 402a2490..2d4f2812 100644 --- a/integration/utils.go +++ b/integration/utils.go @@ -81,7 +81,7 @@ func teardown(s *S3Conf, bucket string) error { } } - if out.IsTruncated { + if *out.IsTruncated { in.ContinuationToken = out.ContinuationToken } else { break @@ -293,7 +293,7 @@ func compareParts(parts1, parts2 []types.Part) bool { } for i, prt := range parts1 { - if prt.PartNumber != parts2[i].PartNumber { + if *prt.PartNumber != *parts2[i].PartNumber { return false } if *prt.ETag != *parts2[i].ETag { @@ -484,18 +484,22 @@ func uploadParts(client *s3.Client, size, partCount int, bucket, key, uploadId s return parts, err } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + pn := int32(partNumber) out, err := client.UploadPart(ctx, &s3.UploadPartInput{ Bucket: &bucket, Key: &key, UploadId: &uploadId, Body: bytes.NewReader(partBuffer), - PartNumber: int32(partNumber), + PartNumber: &pn, }) cancel() if err != nil { return parts, err } else { - parts = append(parts, types.Part{ETag: out.ETag, PartNumber: int32(partNumber)}) + parts = append(parts, types.Part{ + ETag: out.ETag, + PartNumber: &pn, + }) offset += partSize } } diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 8df77309..02c15458 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -63,7 +63,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { key := ctx.Params("key") keyEnd := ctx.Params("*1") uploadId := ctx.Query("uploadId") - maxParts := ctx.QueryInt("max-parts", 0) + maxParts := int32(ctx.QueryInt("max-parts", 0)) partNumberMarker := ctx.Query("part-number-marker") acceptRange := ctx.Get("Range") acct := ctx.Locals("account").(auth.Account) @@ -111,7 +111,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { Key: &key, UploadId: &uploadId, PartNumberMarker: &partNumberMarker, - MaxParts: int32(maxParts), + MaxParts: &maxParts, }) return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner}) } @@ -169,7 +169,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { utils.SetResponseHeaders(ctx, []utils.CustomHeader{ { Key: "Content-Length", - Value: fmt.Sprint(res.ContentLength), + Value: fmt.Sprint(getint64(res.ContentLength)), }, { Key: "Content-Type", @@ -199,11 +199,17 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { Key: "accept-ranges", Value: getstring(res.AcceptRanges), }, - { - Key: "x-amz-tagging-count", - Value: fmt.Sprint(res.TagCount), - }, }) + + if res.TagCount != nil { + utils.SetResponseHeaders(ctx, []utils.CustomHeader{ + { + Key: "x-amz-tagging-count", + Value: fmt.Sprint(*res.TagCount), + }, + }) + } + return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "GetObject", BucketOwner: parsedAcl.Owner}) } @@ -214,6 +220,13 @@ func getstring(s *string) string { return *s } +func getint64(i *int64) int64 { + if i == nil { + return 0 + } + return *i +} + func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { bucket := ctx.Params("bucket") prefix := ctx.Query("prefix") @@ -259,7 +272,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { Delimiter: &delimiter, Prefix: &prefix, UploadIdMarker: &uploadIdMarker, - MaxUploads: maxUploads, + MaxUploads: &maxUploads, KeyMarker: &keyMarker, }) return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListMultipartUploads", BucketOwner: parsedAcl.Owner}) @@ -282,7 +295,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { Prefix: &prefix, ContinuationToken: &cToken, Delimiter: &delimiter, - MaxKeys: maxkeys, + MaxKeys: &maxkeys, }) return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListObjectsV2", BucketOwner: parsedAcl.Owner}) } @@ -305,7 +318,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { Prefix: &prefix, Marker: &marker, Delimiter: &delimiter, - MaxKeys: maxkeys, + MaxKeys: &maxkeys, }) return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListObjects", BucketOwner: parsedAcl.Owner}) } @@ -467,7 +480,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { } if ctx.Request().URI().QueryArgs().Has("uploadId") && ctx.Request().URI().QueryArgs().Has("partNumber") && copySource != "" { - partNumber := ctx.QueryInt("partNumber", -1) + partNumber := int32(ctx.QueryInt("partNumber", -1)) if partNumber < 1 || partNumber > 10000 { return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidPart), &MetaOpts{Logger: c.logger, Action: "UploadPartCopy", BucketOwner: parsedAcl.Owner}) } @@ -476,7 +489,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { Bucket: &bucket, Key: &keyStart, CopySource: ©Source, - PartNumber: int32(partNumber), + PartNumber: &partNumber, UploadId: &uploadId, ExpectedBucketOwner: &bucketOwner, CopySourceRange: ©SrcRange, @@ -485,7 +498,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { } if ctx.Request().URI().QueryArgs().Has("uploadId") && ctx.Request().URI().QueryArgs().Has("partNumber") { - partNumber := ctx.QueryInt("partNumber", -1) + partNumber := int32(ctx.QueryInt("partNumber", -1)) if partNumber < 1 || partNumber > 10000 { return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart), &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner}) } @@ -505,8 +518,8 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { Bucket: &bucket, Key: &keyStart, UploadId: &uploadId, - PartNumber: int32(partNumber), - ContentLength: contentLength, + PartNumber: &partNumber, + ContentLength: &contentLength, Body: body, }) ctx.Response().Header.Set("Etag", etag) @@ -641,7 +654,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { etag, err := c.be.PutObject(ctx.Context(), &s3.PutObjectInput{ Bucket: &bucket, Key: &keyStart, - ContentLength: contentLength, + ContentLength: &contentLength, Metadata: metadata, Body: bytes.NewReader(ctx.Request().Body()), Tagging: &tagging, @@ -805,7 +818,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { utils.SetResponseHeaders(ctx, []utils.CustomHeader{ { Key: "Content-Length", - Value: fmt.Sprint(res.ContentLength), + Value: fmt.Sprint(getint64(res.ContentLength)), }, { Key: "Content-Type", diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index 11e7f92a..74d71c91 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -174,6 +174,7 @@ func TestS3ApiController_GetActions(t *testing.T) { now := time.Now() app := fiber.New() + contentLength := int64(1000) s3ApiController := S3ApiController{ be: &BackendMock{ GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { @@ -194,7 +195,7 @@ func TestS3ApiController_GetActions(t *testing.T) { ContentType: getPtr("application/xml"), ContentEncoding: getPtr("gzip"), ETag: getPtr("98sda7f97sa9df798sd79f8as9df"), - ContentLength: 1000, + ContentLength: &contentLength, LastModified: &now, StorageClass: "storage class", }, nil @@ -1198,6 +1199,7 @@ func TestS3ApiController_HeadObject(t *testing.T) { contentType := "application/xml" eTag := "Valid etag" lastModifie := time.Now() + contentLength := int64(64) s3ApiController := S3ApiController{ be: &BackendMock{ @@ -1207,7 +1209,7 @@ func TestS3ApiController_HeadObject(t *testing.T) { HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { return &s3.HeadObjectOutput{ ContentEncoding: &contentEncoding, - ContentLength: 64, + ContentLength: &contentLength, ContentType: &contentType, LastModified: &lastModifie, ETag: &eTag,