diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index b40f8d3ea..35ab1c177 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -312,6 +312,7 @@ jobs: s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \ s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \ s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \ + s3tests/functional/test_s3.py::test_get_checksum_object_attributes \ s3tests/functional/test_s3.py::test_bucket_head \ s3tests/functional/test_s3.py::test_bucket_head_notexist \ s3tests/functional/test_s3.py::test_object_raw_authenticated \ @@ -1173,6 +1174,7 @@ jobs: s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \ s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \ s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \ + s3tests/functional/test_s3.py::test_get_checksum_object_attributes \ s3tests/functional/test_s3.py::test_bucket_head \ s3tests/functional/test_s3.py::test_bucket_head_notexist \ s3tests/functional/test_s3.py::test_object_raw_authenticated \ diff --git a/weed/s3api/s3api_object_handlers_attributes.go b/weed/s3api/s3api_object_handlers_attributes.go index 4f372e256..901409a26 100644 --- a/weed/s3api/s3api_object_handlers_attributes.go +++ b/weed/s3api/s3api_object_handlers_attributes.go @@ -29,10 +29,8 @@ type GetObjectAttributesResponse struct { // ObjectAttributesChecksum holds checksum info for GetObjectAttributes. type ObjectAttributesChecksum struct { - ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"` - ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"` - ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"` - ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"` + ChecksumResult + ChecksumType string `xml:"ChecksumType,omitempty"` } // ObjectAttributesParts holds parts info for GetObjectAttributes. @@ -77,6 +75,24 @@ func validateObjectAttributes(attrs map[string]struct{}) bool { return true } +// objectAttributesChecksum returns the additional checksum that PutObject or +// CompleteMultipartUpload stored with the object, or nil if it has none. +func objectAttributesChecksum(entry *filer_pb.Entry) *ObjectAttributesChecksum { + if entry == nil || entry.Extended == nil { + return nil + } + value := string(entry.Extended[s3_constants.ExtChecksumValue]) + if value == "" { + return nil + } + checksum := &ObjectAttributesChecksum{ChecksumType: string(entry.Extended[s3_constants.ExtChecksumType])} + checksum.SetChecksum(string(entry.Extended[s3_constants.ExtChecksumAlgorithm]), value) + if checksum.ChecksumResult == (ChecksumResult{}) { + return nil + } + return checksum +} + func (s3a *S3ApiServer) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Request) { bucket, object := s3_constants.GetBucketAndObject(r) glog.V(3).Infof("GetObjectAttributesHandler %s %s", bucket, object) @@ -245,10 +261,9 @@ func (s3a *S3ApiServer) GetObjectAttributesHandler(w http.ResponseWriter, r *htt resp.StorageClass = storageClass } - // Checksum: accepted in validation so clients don't get a 400, but SeaweedFS - // does not yet store S3 checksums (CRC32, CRC32C, SHA1, SHA256), so - // resp.Checksum is intentionally left nil. When checksum storage is added, - // populate resp.Checksum here. + if _, ok := requestedAttrs["Checksum"]; ok { + resp.Checksum = objectAttributesChecksum(entry) + } if _, ok := requestedAttrs["ObjectSize"]; ok { var size int64 diff --git a/weed/s3api/s3api_object_handlers_attributes_test.go b/weed/s3api/s3api_object_handlers_attributes_test.go new file mode 100644 index 000000000..6138c267e --- /dev/null +++ b/weed/s3api/s3api_object_handlers_attributes_test.go @@ -0,0 +1,88 @@ +package s3api + +import ( + "encoding/xml" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestObjectAttributesChecksum verifies that GetObjectAttributes reports the +// checksum that PutObject or CompleteMultipartUpload stored with the object +func TestObjectAttributesChecksum(t *testing.T) { + testCases := []struct { + name string + extended map[string][]byte + want *ObjectAttributesChecksum + }{ + { + name: "PutObject with SHA256", + extended: map[string][]byte{ + s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumSHA256), + s3_constants.ExtChecksumValue: []byte("arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0="), + }, + want: &ObjectAttributesChecksum{ + ChecksumResult: ChecksumResult{ChecksumSHA256: "arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0="}, + }, + }, + { + name: "multipart upload with a composite CRC32C", + extended: map[string][]byte{ + s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumCRC32C), + s3_constants.ExtChecksumValue: []byte("x3Y2bw==-3"), + s3_constants.ExtChecksumType: []byte("COMPOSITE"), + }, + want: &ObjectAttributesChecksum{ + ChecksumResult: ChecksumResult{ChecksumCRC32C: "x3Y2bw==-3"}, + ChecksumType: "COMPOSITE", + }, + }, + { + name: "multipart upload with a full object CRC64NVME", + extended: map[string][]byte{ + s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumCRC64NVME), + s3_constants.ExtChecksumValue: []byte("AAAAAAAAAAA="), + s3_constants.ExtChecksumType: []byte("FULL_OBJECT"), + }, + want: &ObjectAttributesChecksum{ + ChecksumResult: ChecksumResult{ChecksumCRC64NVME: "AAAAAAAAAAA="}, + ChecksumType: "FULL_OBJECT", + }, + }, + { + name: "no checksum", + extended: map[string][]byte{s3_constants.ExtETagKey: []byte("d41d8cd98f00b204e9800998ecf8427e")}, + }, + { + name: "unknown algorithm", + extended: map[string][]byte{ + s3_constants.ExtChecksumAlgorithm: []byte("X-Amz-Checksum-Md5"), + s3_constants.ExtChecksumValue: []byte("1B2M2Y8AsgTpgAmY7PhCfg=="), + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, objectAttributesChecksum(&filer_pb.Entry{Extended: tc.extended})) + }) + } + assert.Nil(t, objectAttributesChecksum(&filer_pb.Entry{})) +} + +// TestGetObjectAttributesChecksumXML verifies the Checksum element's layout +func TestGetObjectAttributesChecksumXML(t *testing.T) { + resp := GetObjectAttributesResponse{ + Checksum: &ObjectAttributesChecksum{ + ChecksumResult: ChecksumResult{ChecksumCRC32: "NhCmhg=="}, + ChecksumType: "FULL_OBJECT", + }, + } + out, err := xml.Marshal(resp) + require.NoError(t, err) + assert.Equal(t, "NhCmhg=="+ + "FULL_OBJECT", string(out)) +}