s3api: report the stored checksum in GetObjectAttributes (#11529)

GetObjectAttributes accepted the Checksum attribute but never filled it
in, as its comment said SeaweedFS did not store S3 checksums. PutObject
and CompleteMultipartUpload store them now, and HeadObject returns them.
Fill in Checksum from the same entry fields, with the ChecksumType and
ChecksumCRC64NVME members the response did not have.

Also run ceph/s3-tests' test_get_checksum_object_attributes in CI.

.github/workflows/s3tests.yml

@@ -312,6 +312,7 @@ jobs:
           s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
           s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
           s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
+          s3tests/functional/test_s3.py::test_get_checksum_object_attributes \
           s3tests/functional/test_s3.py::test_bucket_head \
           s3tests/functional/test_s3.py::test_bucket_head_notexist \
           s3tests/functional/test_s3.py::test_object_raw_authenticated \
@@ -1173,6 +1174,7 @@ jobs:
           s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
           s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
           s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
+          s3tests/functional/test_s3.py::test_get_checksum_object_attributes \
           s3tests/functional/test_s3.py::test_bucket_head \
           s3tests/functional/test_s3.py::test_bucket_head_notexist \
           s3tests/functional/test_s3.py::test_object_raw_authenticated \

weed/s3api/s3api_object_handlers_attributes.go

@@ -29,10 +29,8 @@ type GetObjectAttributesResponse struct {
 
 // ObjectAttributesChecksum holds checksum info for GetObjectAttributes.
 type ObjectAttributesChecksum struct {
-	ChecksumCRC32  string `xml:"ChecksumCRC32,omitempty"`
-	ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
-	ChecksumSHA1   string `xml:"ChecksumSHA1,omitempty"`
-	ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
+	ChecksumResult
+	ChecksumType string `xml:"ChecksumType,omitempty"`
 }
 
 // ObjectAttributesParts holds parts info for GetObjectAttributes.
@@ -77,6 +75,24 @@ func validateObjectAttributes(attrs map[string]struct{}) bool {
 	return true
 }
 
+// objectAttributesChecksum returns the additional checksum that PutObject or
+// CompleteMultipartUpload stored with the object, or nil if it has none.
+func objectAttributesChecksum(entry *filer_pb.Entry) *ObjectAttributesChecksum {
+	if entry == nil || entry.Extended == nil {
+		return nil
+	}
+	value := string(entry.Extended[s3_constants.ExtChecksumValue])
+	if value == "" {
+		return nil
+	}
+	checksum := &ObjectAttributesChecksum{ChecksumType: string(entry.Extended[s3_constants.ExtChecksumType])}
+	checksum.SetChecksum(string(entry.Extended[s3_constants.ExtChecksumAlgorithm]), value)
+	if checksum.ChecksumResult == (ChecksumResult{}) {
+		return nil
+	}
+	return checksum
+}
+
 func (s3a *S3ApiServer) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Request) {
 	bucket, object := s3_constants.GetBucketAndObject(r)
 	glog.V(3).Infof("GetObjectAttributesHandler %s %s", bucket, object)
@@ -245,10 +261,9 @@ func (s3a *S3ApiServer) GetObjectAttributesHandler(w http.ResponseWriter, r *htt
 		resp.StorageClass = storageClass
 	}
 
-	// Checksum: accepted in validation so clients don't get a 400, but SeaweedFS
-	// does not yet store S3 checksums (CRC32, CRC32C, SHA1, SHA256), so
-	// resp.Checksum is intentionally left nil. When checksum storage is added,
-	// populate resp.Checksum here.
+	if _, ok := requestedAttrs["Checksum"]; ok {
+		resp.Checksum = objectAttributesChecksum(entry)
+	}
 
 	if _, ok := requestedAttrs["ObjectSize"]; ok {
 		var size int64

weed/s3api/s3api_object_handlers_attributes_test.go

@@ -0,0 +1,88 @@
+package s3api
+
+import (
+	"encoding/xml"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestObjectAttributesChecksum verifies that GetObjectAttributes reports the
+// checksum that PutObject or CompleteMultipartUpload stored with the object
+func TestObjectAttributesChecksum(t *testing.T) {
+	testCases := []struct {
+		name     string
+		extended map[string][]byte
+		want     *ObjectAttributesChecksum
+	}{
+		{
+			name: "PutObject with SHA256",
+			extended: map[string][]byte{
+				s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumSHA256),
+				s3_constants.ExtChecksumValue:     []byte("arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0="),
+			},
+			want: &ObjectAttributesChecksum{
+				ChecksumResult: ChecksumResult{ChecksumSHA256: "arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0="},
+			},
+		},
+		{
+			name: "multipart upload with a composite CRC32C",
+			extended: map[string][]byte{
+				s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumCRC32C),
+				s3_constants.ExtChecksumValue:     []byte("x3Y2bw==-3"),
+				s3_constants.ExtChecksumType:      []byte("COMPOSITE"),
+			},
+			want: &ObjectAttributesChecksum{
+				ChecksumResult: ChecksumResult{ChecksumCRC32C: "x3Y2bw==-3"},
+				ChecksumType:   "COMPOSITE",
+			},
+		},
+		{
+			name: "multipart upload with a full object CRC64NVME",
+			extended: map[string][]byte{
+				s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumCRC64NVME),
+				s3_constants.ExtChecksumValue:     []byte("AAAAAAAAAAA="),
+				s3_constants.ExtChecksumType:      []byte("FULL_OBJECT"),
+			},
+			want: &ObjectAttributesChecksum{
+				ChecksumResult: ChecksumResult{ChecksumCRC64NVME: "AAAAAAAAAAA="},
+				ChecksumType:   "FULL_OBJECT",
+			},
+		},
+		{
+			name:     "no checksum",
+			extended: map[string][]byte{s3_constants.ExtETagKey: []byte("d41d8cd98f00b204e9800998ecf8427e")},
+		},
+		{
+			name: "unknown algorithm",
+			extended: map[string][]byte{
+				s3_constants.ExtChecksumAlgorithm: []byte("X-Amz-Checksum-Md5"),
+				s3_constants.ExtChecksumValue:     []byte("1B2M2Y8AsgTpgAmY7PhCfg=="),
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.want, objectAttributesChecksum(&filer_pb.Entry{Extended: tc.extended}))
+		})
+	}
+	assert.Nil(t, objectAttributesChecksum(&filer_pb.Entry{}))
+}
+
+// TestGetObjectAttributesChecksumXML verifies the Checksum element's layout
+func TestGetObjectAttributesChecksumXML(t *testing.T) {
+	resp := GetObjectAttributesResponse{
+		Checksum: &ObjectAttributesChecksum{
+			ChecksumResult: ChecksumResult{ChecksumCRC32: "NhCmhg=="},
+			ChecksumType:   "FULL_OBJECT",
+		},
+	}
+	out, err := xml.Marshal(resp)
+	require.NoError(t, err)
+	assert.Equal(t, "<GetObjectAttributesResponse><Checksum><ChecksumCRC32>NhCmhg==</ChecksumCRC32>"+
+		"<ChecksumType>FULL_OBJECT</ChecksumType></Checksum></GetObjectAttributesResponse>", string(out))
+}