From da81c6cc278fd2cb02c5e933b3cbdf16149c3a78 Mon Sep 17 00:00:00 2001 From: Krishnan Parthasarathi Date: Thu, 21 Mar 2024 10:21:35 -0700 Subject: [PATCH] Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`. --- Makefile | 2 +- cmd/batch-expire.go | 2 +- cmd/bucket-replication-metrics_gen.go | 2 + cmd/bucket-replication-utils_gen.go | 4 ++ cmd/data-scanner.go | 6 ++- cmd/data-usage-cache_gen.go | 32 +++++++++------- cmd/erasure-server-pool-rebalance_gen.go | 2 + cmd/local-locker_gen.go | 4 ++ cmd/storage-datatypes_gen.go | 12 ++++++ cmd/tier-last-day-stats_gen.go | 8 ++-- cmd/xl-storage-format-v1_gen.go | 12 +++--- cmd/xl-storage-format-v2_gen.go | 48 ++++++++++++------------ internal/bucket/bandwidth/monitor_gen.go | 2 + internal/event/name.go | 6 +++ internal/event/name_test.go | 2 +- 15 files changed, 91 insertions(+), 53 deletions(-) diff --git a/Makefile b/Makefile index 485e1b1eb..35fb50131 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ help: ## print this help getdeps: ## fetch necessary dependencies @mkdir -p ${GOPATH}/bin @echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR) - @echo "Installing msgp" && go install -v github.com/tinylib/msgp@6ac204f0b4d48d17ab4fa442134c7fba13127a4e + @echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.10-0.20240227114326-6d6f813fff1b @echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest crosscompile: ## cross compile minio diff --git a/cmd/batch-expire.go b/cmd/batch-expire.go index b454f4744..e6869d620 100644 --- a/cmd/batch-expire.go +++ b/cmd/batch-expire.go @@ -427,7 +427,7 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo default: } stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts) - _, err := api.DeleteObject(ctx, exp.Bucket, exp.Name, ObjectOptions{ + _, err := api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{ DeletePrefix: true, }) if err != nil { diff --git a/cmd/bucket-replication-metrics_gen.go b/cmd/bucket-replication-metrics_gen.go index f0443ab8a..d59688beb 100644 --- a/cmd/bucket-replication-metrics_gen.go +++ b/cmd/bucket-replication-metrics_gen.go @@ -584,6 +584,7 @@ func (z *InQueueStats) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z InQueueStats) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 0 + _ = z err = en.Append(0x80) if err != nil { return @@ -595,6 +596,7 @@ func (z InQueueStats) EncodeMsg(en *msgp.Writer) (err error) { func (z InQueueStats) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 0 + _ = z o = append(o, 0x80) return } diff --git a/cmd/bucket-replication-utils_gen.go b/cmd/bucket-replication-utils_gen.go index 4a1078fd6..5ed010d9b 100644 --- a/cmd/bucket-replication-utils_gen.go +++ b/cmd/bucket-replication-utils_gen.go @@ -749,6 +749,7 @@ func (z *ReplicateDecision) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z ReplicateDecision) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 0 + _ = z err = en.Append(0x80) if err != nil { return @@ -760,6 +761,7 @@ func (z ReplicateDecision) EncodeMsg(en *msgp.Writer) (err error) { func (z ReplicateDecision) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 0 + _ = z o = append(o, 0x80) return } @@ -1388,6 +1390,7 @@ func (z *ResyncDecision) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z ResyncDecision) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 0 + _ = z err = en.Append(0x80) if err != nil { return @@ -1399,6 +1402,7 @@ func (z ResyncDecision) EncodeMsg(en *msgp.Writer) (err error) { func (z ResyncDecision) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 0 + _ = z o = append(o, 0x80) return } diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go index 4ba4a7c4a..b96df6a54 100644 --- a/cmd/data-scanner.go +++ b/cmd/data-scanner.go @@ -1240,7 +1240,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay } }() - dobj, err = objLayer.DeleteObject(ctx, obj.Bucket, obj.Name, opts) + dobj, err = objLayer.DeleteObject(ctx, obj.Bucket, encodeDirObject(obj.Name), opts) if err != nil { if isErrObjectNotFound(err) || isErrVersionNotFound(err) { return false @@ -1261,7 +1261,9 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay if obj.DeleteMarker { eventName = event.ObjectRemovedDeleteMarkerCreated } - + if lcEvent.Action.DeleteAll() { + eventName = event.ObjectRemovedDeleteAllVersions + } // Notify object deleted event. sendEvent(eventArgs{ EventName: eventName, diff --git a/cmd/data-usage-cache_gen.go b/cmd/data-usage-cache_gen.go index 0be4fd0fa..69e0cc6c8 100644 --- a/cmd/data-usage-cache_gen.go +++ b/cmd/data-usage-cache_gen.go @@ -1762,7 +1762,7 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { - // omitempty: check for empty values + // check for omitted fields zb0001Len := uint32(10) var zb0001Mask uint16 /* 10 bits */ _ = zb0001Mask @@ -1866,7 +1866,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { return } } - if (zb0001Mask & 0x80) == 0 { // if not empty + if (zb0001Mask & 0x80) == 0 { // if not omitted // write "rs" err = en.Append(0xa2, 0x72, 0x73) if err != nil { @@ -1885,7 +1885,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { } } } - if (zb0001Mask & 0x100) == 0 { // if not empty + if (zb0001Mask & 0x100) == 0 { // if not omitted // write "ats" err = en.Append(0xa3, 0x61, 0x74, 0x73) if err != nil { @@ -1920,7 +1920,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { // MarshalMsg implements msgp.Marshaler func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // omitempty: check for empty values + // check for omitted fields zb0001Len := uint32(10) var zb0001Mask uint16 /* 10 bits */ _ = zb0001Mask @@ -1968,7 +1968,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) { for za0002 := range z.ObjVersions { o = msgp.AppendUint64(o, z.ObjVersions[za0002]) } - if (zb0001Mask & 0x80) == 0 { // if not empty + if (zb0001Mask & 0x80) == 0 { // if not omitted // string "rs" o = append(o, 0xa2, 0x72, 0x73) if z.ReplicationStats == nil { @@ -1981,7 +1981,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) { } } } - if (zb0001Mask & 0x100) == 0 { // if not empty + if (zb0001Mask & 0x100) == 0 { // if not omitted // string "ats" o = append(o, 0xa3, 0x61, 0x74, 0x73) if z.AllTierStats == nil { @@ -3243,7 +3243,7 @@ func (z *replicationAllStats) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) { - // omitempty: check for empty values + // check for omitted fields zb0001Len := uint32(3) var zb0001Mask uint8 /* 3 bits */ _ = zb0001Mask @@ -3267,7 +3267,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) { if zb0001Len == 0 { return } - if (zb0001Mask & 0x1) == 0 { // if not empty + if (zb0001Mask & 0x1) == 0 { // if not omitted // write "t" err = en.Append(0xa1, 0x74) if err != nil { @@ -3291,7 +3291,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) { } } } - if (zb0001Mask & 0x2) == 0 { // if not empty + if (zb0001Mask & 0x2) == 0 { // if not omitted // write "r" err = en.Append(0xa1, 0x72) if err != nil { @@ -3303,7 +3303,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) { return } } - if (zb0001Mask & 0x4) == 0 { // if not empty + if (zb0001Mask & 0x4) == 0 { // if not omitted // write "rc" err = en.Append(0xa2, 0x72, 0x63) if err != nil { @@ -3321,7 +3321,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) { // MarshalMsg implements msgp.Marshaler func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // omitempty: check for empty values + // check for omitted fields zb0001Len := uint32(3) var zb0001Mask uint8 /* 3 bits */ _ = zb0001Mask @@ -3342,7 +3342,7 @@ func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) { if zb0001Len == 0 { return } - if (zb0001Mask & 0x1) == 0 { // if not empty + if (zb0001Mask & 0x1) == 0 { // if not omitted // string "t" o = append(o, 0xa1, 0x74) o = msgp.AppendMapHeader(o, uint32(len(z.Targets))) @@ -3355,12 +3355,12 @@ func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) { } } } - if (zb0001Mask & 0x2) == 0 { // if not empty + if (zb0001Mask & 0x2) == 0 { // if not omitted // string "r" o = append(o, 0xa1, 0x72) o = msgp.AppendUint64(o, z.ReplicaSize) } - if (zb0001Mask & 0x4) == 0 { // if not empty + if (zb0001Mask & 0x4) == 0 { // if not omitted // string "rc" o = append(o, 0xa2, 0x72, 0x63) o = msgp.AppendUint64(o, z.ReplicaCount) @@ -3478,6 +3478,8 @@ func (z *replicationAllStatsV1) DecodeMsg(dc *msgp.Reader) (err error) { delete(z.Targets, key) } } + var field []byte + _ = field for zb0002 > 0 { zb0002-- var za0001 string @@ -3588,6 +3590,8 @@ func (z *replicationAllStatsV1) UnmarshalMsg(bts []byte) (o []byte, err error) { delete(z.Targets, key) } } + var field []byte + _ = field for zb0002 > 0 { var za0001 string var za0002 replicationStats diff --git a/cmd/erasure-server-pool-rebalance_gen.go b/cmd/erasure-server-pool-rebalance_gen.go index a0cbd56f4..0787e2600 100644 --- a/cmd/erasure-server-pool-rebalance_gen.go +++ b/cmd/erasure-server-pool-rebalance_gen.go @@ -614,6 +614,7 @@ func (z *rebalanceMetrics) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z rebalanceMetrics) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 0 + _ = z err = en.Append(0x80) if err != nil { return @@ -625,6 +626,7 @@ func (z rebalanceMetrics) EncodeMsg(en *msgp.Writer) (err error) { func (z rebalanceMetrics) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 0 + _ = z o = append(o, 0x80) return } diff --git a/cmd/local-locker_gen.go b/cmd/local-locker_gen.go index 6ba4a893e..33851561d 100644 --- a/cmd/local-locker_gen.go +++ b/cmd/local-locker_gen.go @@ -21,6 +21,8 @@ func (z *localLockMap) DecodeMsg(dc *msgp.Reader) (err error) { delete((*z), key) } } + var field []byte + _ = field for zb0004 > 0 { zb0004-- var zb0001 string @@ -115,6 +117,8 @@ func (z *localLockMap) UnmarshalMsg(bts []byte) (o []byte, err error) { delete((*z), key) } } + var field []byte + _ = field for zb0004 > 0 { var zb0001 string var zb0002 []lockRequesterInfo diff --git a/cmd/storage-datatypes_gen.go b/cmd/storage-datatypes_gen.go index d383a8cfe..abf90e4e8 100644 --- a/cmd/storage-datatypes_gen.go +++ b/cmd/storage-datatypes_gen.go @@ -38,6 +38,7 @@ func (z *BaseOptions) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z BaseOptions) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 0 + _ = z err = en.Append(0x80) if err != nil { return @@ -49,6 +50,7 @@ func (z BaseOptions) EncodeMsg(en *msgp.Writer) (err error) { func (z BaseOptions) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 0 + _ = z o = append(o, 0x80) return } @@ -532,6 +534,7 @@ func (z *DeleteOptions) EncodeMsg(en *msgp.Writer) (err error) { return } // map header, size 0 + _ = z.BaseOptions err = en.Append(0x80) if err != nil { return @@ -576,6 +579,7 @@ func (z *DeleteOptions) MarshalMsg(b []byte) (o []byte, err error) { // string "BaseOptions" o = append(o, 0x84, 0xab, 0x42, 0x61, 0x73, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73) // map header, size 0 + _ = z.BaseOptions o = append(o, 0x80) // string "r" o = append(o, 0xa1, 0x72) @@ -1906,6 +1910,8 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) { delete(z.Metadata, key) } } + var field []byte + _ = field for zb0002 > 0 { zb0002-- var za0001 string @@ -2362,6 +2368,8 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { delete(z.Metadata, key) } } + var field []byte + _ = field for zb0002 > 0 { var za0001 string var za0002 string @@ -4455,6 +4463,7 @@ func (z *RenameDataHandlerParams) EncodeMsg(en *msgp.Writer) (err error) { return } // map header, size 0 + _ = z.Opts.BaseOptions err = en.Append(0x80) if err != nil { return @@ -4494,6 +4503,7 @@ func (z *RenameDataHandlerParams) MarshalMsg(b []byte) (o []byte, err error) { // string "BaseOptions" o = append(o, 0x81, 0xab, 0x42, 0x61, 0x73, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73) // map header, size 0 + _ = z.Opts.BaseOptions o = append(o, 0x80) return } @@ -4983,6 +4993,7 @@ func (z *RenameOptions) EncodeMsg(en *msgp.Writer) (err error) { return } // map header, size 0 + _ = z.BaseOptions err = en.Append(0x80) if err != nil { return @@ -4997,6 +5008,7 @@ func (z *RenameOptions) MarshalMsg(b []byte) (o []byte, err error) { // string "BaseOptions" o = append(o, 0x81, 0xab, 0x42, 0x61, 0x73, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73) // map header, size 0 + _ = z.BaseOptions o = append(o, 0x80) return } diff --git a/cmd/tier-last-day-stats_gen.go b/cmd/tier-last-day-stats_gen.go index b996a7e11..c85857325 100644 --- a/cmd/tier-last-day-stats_gen.go +++ b/cmd/tier-last-day-stats_gen.go @@ -21,6 +21,8 @@ func (z *DailyAllTierStats) DecodeMsg(dc *msgp.Reader) (err error) { delete((*z), key) } } + var field []byte + _ = field for zb0004 > 0 { zb0004-- var zb0001 string @@ -30,8 +32,6 @@ func (z *DailyAllTierStats) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err) return } - var field []byte - _ = field var zb0005 uint32 zb0005, err = dc.ReadMapHeader() if err != nil { @@ -167,6 +167,8 @@ func (z *DailyAllTierStats) UnmarshalMsg(bts []byte) (o []byte, err error) { delete((*z), key) } } + var field []byte + _ = field for zb0004 > 0 { var zb0001 string var zb0002 lastDayTierStats @@ -176,8 +178,6 @@ func (z *DailyAllTierStats) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err) return } - var field []byte - _ = field var zb0005 uint32 zb0005, bts, err = msgp.ReadMapHeaderBytes(bts) if err != nil { diff --git a/cmd/xl-storage-format-v1_gen.go b/cmd/xl-storage-format-v1_gen.go index c25d01263..444db638b 100644 --- a/cmd/xl-storage-format-v1_gen.go +++ b/cmd/xl-storage-format-v1_gen.go @@ -648,7 +648,7 @@ func (z *ObjectPartInfo) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *ObjectPartInfo) EncodeMsg(en *msgp.Writer) (err error) { - // omitempty: check for empty values + // check for omitted fields zb0001Len := uint32(7) var zb0001Mask uint8 /* 7 bits */ _ = zb0001Mask @@ -718,7 +718,7 @@ func (z *ObjectPartInfo) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "ModTime") return } - if (zb0001Mask & 0x20) == 0 { // if not empty + if (zb0001Mask & 0x20) == 0 { // if not omitted // write "index" err = en.Append(0xa5, 0x69, 0x6e, 0x64, 0x65, 0x78) if err != nil { @@ -730,7 +730,7 @@ func (z *ObjectPartInfo) EncodeMsg(en *msgp.Writer) (err error) { return } } - if (zb0001Mask & 0x40) == 0 { // if not empty + if (zb0001Mask & 0x40) == 0 { // if not omitted // write "crc" err = en.Append(0xa3, 0x63, 0x72, 0x63) if err != nil { @@ -760,7 +760,7 @@ func (z *ObjectPartInfo) EncodeMsg(en *msgp.Writer) (err error) { // MarshalMsg implements msgp.Marshaler func (z *ObjectPartInfo) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // omitempty: check for empty values + // check for omitted fields zb0001Len := uint32(7) var zb0001Mask uint8 /* 7 bits */ _ = zb0001Mask @@ -792,12 +792,12 @@ func (z *ObjectPartInfo) MarshalMsg(b []byte) (o []byte, err error) { // string "ModTime" o = append(o, 0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65) o = msgp.AppendTime(o, z.ModTime) - if (zb0001Mask & 0x20) == 0 { // if not empty + if (zb0001Mask & 0x20) == 0 { // if not omitted // string "index" o = append(o, 0xa5, 0x69, 0x6e, 0x64, 0x65, 0x78) o = msgp.AppendBytes(o, z.Index) } - if (zb0001Mask & 0x40) == 0 { // if not empty + if (zb0001Mask & 0x40) == 0 { // if not omitted // string "crc" o = append(o, 0xa3, 0x63, 0x72, 0x63) o = msgp.AppendMapHeader(o, uint32(len(z.Checksums))) diff --git a/cmd/xl-storage-format-v2_gen.go b/cmd/xl-storage-format-v2_gen.go index 1813fde66..8ddbda7ff 100644 --- a/cmd/xl-storage-format-v2_gen.go +++ b/cmd/xl-storage-format-v2_gen.go @@ -340,7 +340,7 @@ func (z *xlMetaDataDirDecoder) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *xlMetaDataDirDecoder) EncodeMsg(en *msgp.Writer) (err error) { - // omitempty: check for empty values + // check for omitted fields zb0001Len := uint32(1) var zb0001Mask uint8 /* 1 bits */ _ = zb0001Mask @@ -356,7 +356,7 @@ func (z *xlMetaDataDirDecoder) EncodeMsg(en *msgp.Writer) (err error) { if zb0001Len == 0 { return } - if (zb0001Mask & 0x1) == 0 { // if not empty + if (zb0001Mask & 0x1) == 0 { // if not omitted // write "V2Obj" err = en.Append(0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a) if err != nil { @@ -387,7 +387,7 @@ func (z *xlMetaDataDirDecoder) EncodeMsg(en *msgp.Writer) (err error) { // MarshalMsg implements msgp.Marshaler func (z *xlMetaDataDirDecoder) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // omitempty: check for empty values + // check for omitted fields zb0001Len := uint32(1) var zb0001Mask uint8 /* 1 bits */ _ = zb0001Mask @@ -400,7 +400,7 @@ func (z *xlMetaDataDirDecoder) MarshalMsg(b []byte) (o []byte, err error) { if zb0001Len == 0 { return } - if (zb0001Mask & 0x1) == 0 { // if not empty + if (zb0001Mask & 0x1) == 0 { // if not omitted // string "V2Obj" o = append(o, 0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a) if z.ObjectV2 == nil { @@ -571,7 +571,7 @@ func (z *xlMetaV2DeleteMarker) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *xlMetaV2DeleteMarker) EncodeMsg(en *msgp.Writer) (err error) { - // omitempty: check for empty values + // check for omitted fields zb0001Len := uint32(3) var zb0001Mask uint8 /* 3 bits */ _ = zb0001Mask @@ -607,7 +607,7 @@ func (z *xlMetaV2DeleteMarker) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "ModTime") return } - if (zb0001Mask & 0x4) == 0 { // if not empty + if (zb0001Mask & 0x4) == 0 { // if not omitted // write "MetaSys" err = en.Append(0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73) if err != nil { @@ -637,7 +637,7 @@ func (z *xlMetaV2DeleteMarker) EncodeMsg(en *msgp.Writer) (err error) { // MarshalMsg implements msgp.Marshaler func (z *xlMetaV2DeleteMarker) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // omitempty: check for empty values + // check for omitted fields zb0001Len := uint32(3) var zb0001Mask uint8 /* 3 bits */ _ = zb0001Mask @@ -656,7 +656,7 @@ func (z *xlMetaV2DeleteMarker) MarshalMsg(b []byte) (o []byte, err error) { // string "MTime" o = append(o, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65) o = msgp.AppendInt64(o, z.ModTime) - if (zb0001Mask & 0x4) == 0 { // if not empty + if (zb0001Mask & 0x4) == 0 { // if not omitted // string "MetaSys" o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73) o = msgp.AppendMapHeader(o, uint32(len(z.MetaSys))) @@ -879,7 +879,7 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err, "PartETags") return } - if cap(z.PartETags) >= int(zb0006) { + if z.PartETags != nil && cap(z.PartETags) >= int(zb0006) { z.PartETags = (z.PartETags)[:zb0006] } else { z.PartETags = make([]string, zb0006) @@ -926,7 +926,7 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err, "PartActualSizes") return } - if cap(z.PartActualSizes) >= int(zb0008) { + if z.PartActualSizes != nil && cap(z.PartActualSizes) >= int(zb0008) { z.PartActualSizes = (z.PartActualSizes)[:zb0008] } else { z.PartActualSizes = make([]int64, zb0008) @@ -1061,7 +1061,7 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) { - // omitempty: check for empty values + // check for omitted fields zb0001Len := uint32(18) var zb0001Mask uint32 /* 18 bits */ _ = zb0001Mask @@ -1256,7 +1256,7 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) { } } } - if (zb0001Mask & 0x2000) == 0 { // if not empty + if (zb0001Mask & 0x2000) == 0 { // if not omitted // write "PartIdx" err = en.Append(0xa7, 0x50, 0x61, 0x72, 0x74, 0x49, 0x64, 0x78) if err != nil { @@ -1359,7 +1359,7 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) { // MarshalMsg implements msgp.Marshaler func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // omitempty: check for empty values + // check for omitted fields zb0001Len := uint32(18) var zb0001Mask uint32 /* 18 bits */ _ = zb0001Mask @@ -1434,7 +1434,7 @@ func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.AppendInt64(o, z.PartActualSizes[za0007]) } } - if (zb0001Mask & 0x2000) == 0 { // if not empty + if (zb0001Mask & 0x2000) == 0 { // if not omitted // string "PartIdx" o = append(o, 0xa7, 0x50, 0x61, 0x72, 0x74, 0x49, 0x64, 0x78) o = msgp.AppendArrayHeader(o, uint32(len(z.PartIndices))) @@ -1596,7 +1596,7 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "PartETags") return } - if cap(z.PartETags) >= int(zb0006) { + if z.PartETags != nil && cap(z.PartETags) >= int(zb0006) { z.PartETags = (z.PartETags)[:zb0006] } else { z.PartETags = make([]string, zb0006) @@ -1639,7 +1639,7 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "PartActualSizes") return } - if cap(z.PartActualSizes) >= int(zb0008) { + if z.PartActualSizes != nil && cap(z.PartActualSizes) >= int(zb0008) { z.PartActualSizes = (z.PartActualSizes)[:zb0008] } else { z.PartActualSizes = make([]int64, zb0008) @@ -1893,7 +1893,7 @@ func (z *xlMetaV2Version) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *xlMetaV2Version) EncodeMsg(en *msgp.Writer) (err error) { - // omitempty: check for empty values + // check for omitted fields zb0001Len := uint32(5) var zb0001Mask uint8 /* 5 bits */ _ = zb0001Mask @@ -1927,7 +1927,7 @@ func (z *xlMetaV2Version) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "Type") return } - if (zb0001Mask & 0x2) == 0 { // if not empty + if (zb0001Mask & 0x2) == 0 { // if not omitted // write "V1Obj" err = en.Append(0xa5, 0x56, 0x31, 0x4f, 0x62, 0x6a) if err != nil { @@ -1946,7 +1946,7 @@ func (z *xlMetaV2Version) EncodeMsg(en *msgp.Writer) (err error) { } } } - if (zb0001Mask & 0x4) == 0 { // if not empty + if (zb0001Mask & 0x4) == 0 { // if not omitted // write "V2Obj" err = en.Append(0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a) if err != nil { @@ -1965,7 +1965,7 @@ func (z *xlMetaV2Version) EncodeMsg(en *msgp.Writer) (err error) { } } } - if (zb0001Mask & 0x8) == 0 { // if not empty + if (zb0001Mask & 0x8) == 0 { // if not omitted // write "DelObj" err = en.Append(0xa6, 0x44, 0x65, 0x6c, 0x4f, 0x62, 0x6a) if err != nil { @@ -2000,7 +2000,7 @@ func (z *xlMetaV2Version) EncodeMsg(en *msgp.Writer) (err error) { // MarshalMsg implements msgp.Marshaler func (z *xlMetaV2Version) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // omitempty: check for empty values + // check for omitted fields zb0001Len := uint32(5) var zb0001Mask uint8 /* 5 bits */ _ = zb0001Mask @@ -2024,7 +2024,7 @@ func (z *xlMetaV2Version) MarshalMsg(b []byte) (o []byte, err error) { // string "Type" o = append(o, 0xa4, 0x54, 0x79, 0x70, 0x65) o = msgp.AppendUint8(o, uint8(z.Type)) - if (zb0001Mask & 0x2) == 0 { // if not empty + if (zb0001Mask & 0x2) == 0 { // if not omitted // string "V1Obj" o = append(o, 0xa5, 0x56, 0x31, 0x4f, 0x62, 0x6a) if z.ObjectV1 == nil { @@ -2037,7 +2037,7 @@ func (z *xlMetaV2Version) MarshalMsg(b []byte) (o []byte, err error) { } } } - if (zb0001Mask & 0x4) == 0 { // if not empty + if (zb0001Mask & 0x4) == 0 { // if not omitted // string "V2Obj" o = append(o, 0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a) if z.ObjectV2 == nil { @@ -2050,7 +2050,7 @@ func (z *xlMetaV2Version) MarshalMsg(b []byte) (o []byte, err error) { } } } - if (zb0001Mask & 0x8) == 0 { // if not empty + if (zb0001Mask & 0x8) == 0 { // if not omitted // string "DelObj" o = append(o, 0xa6, 0x44, 0x65, 0x6c, 0x4f, 0x62, 0x6a) if z.DeleteMarker == nil { diff --git a/internal/bucket/bandwidth/monitor_gen.go b/internal/bucket/bandwidth/monitor_gen.go index b0852289f..40898ef39 100644 --- a/internal/bucket/bandwidth/monitor_gen.go +++ b/internal/bucket/bandwidth/monitor_gen.go @@ -38,6 +38,7 @@ func (z *BucketBandwidthReport) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z BucketBandwidthReport) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 0 + _ = z err = en.Append(0x80) if err != nil { return @@ -49,6 +50,7 @@ func (z BucketBandwidthReport) EncodeMsg(en *msgp.Writer) (err error) { func (z BucketBandwidthReport) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 0 + _ = z o = append(o, 0x80) return } diff --git a/internal/event/name.go b/internal/event/name.go index 3f13cad82..df0096740 100644 --- a/internal/event/name.go +++ b/internal/event/name.go @@ -47,6 +47,7 @@ const ( ObjectCreatedDeleteTagging ObjectRemovedDelete ObjectRemovedDeleteMarkerCreated + ObjectRemovedDeleteAllVersions ObjectRemovedNoOP BucketCreated BucketRemoved @@ -100,6 +101,7 @@ func (name Name) Expand() []Name { ObjectRemovedDelete, ObjectRemovedDeleteMarkerCreated, ObjectRemovedNoOP, + ObjectRemovedDeleteAllVersions, } case ObjectReplicationAll: return []Name{ @@ -193,6 +195,8 @@ func (name Name) String() string { return "s3:ObjectRemoved:DeleteMarkerCreated" case ObjectRemovedNoOP: return "s3:ObjectRemoved:NoOP" + case ObjectRemovedDeleteAllVersions: + return "s3:ObjectRemoved:DeleteAllVersions" case ObjectReplicationAll: return "s3:Replication:*" case ObjectReplicationFailed: @@ -313,6 +317,8 @@ func ParseName(s string) (Name, error) { return ObjectRemovedDeleteMarkerCreated, nil case "s3:ObjectRemoved:NoOP": return ObjectRemovedNoOP, nil + case "s3:ObjectRemoved:DeleteAllVersions": + return ObjectRemovedDeleteAllVersions, nil case "s3:Replication:*": return ObjectReplicationAll, nil case "s3:Replication:OperationFailedReplication": diff --git a/internal/event/name_test.go b/internal/event/name_test.go index 130ff7036..81a32a4a8 100644 --- a/internal/event/name_test.go +++ b/internal/event/name_test.go @@ -36,7 +36,7 @@ func TestNameExpand(t *testing.T) { ObjectCreatedCompleteMultipartUpload, ObjectCreatedCopy, ObjectCreatedPost, ObjectCreatedPut, ObjectCreatedPutRetention, ObjectCreatedPutLegalHold, ObjectCreatedPutTagging, ObjectCreatedDeleteTagging, }}, - {ObjectRemovedAll, []Name{ObjectRemovedDelete, ObjectRemovedDeleteMarkerCreated, ObjectRemovedNoOP}}, + {ObjectRemovedAll, []Name{ObjectRemovedDelete, ObjectRemovedDeleteMarkerCreated, ObjectRemovedNoOP, ObjectRemovedDeleteAllVersions}}, {ObjectAccessedHead, []Name{ObjectAccessedHead}}, }