do delete expired entries on s3 list request

https://github.com/seaweedfs/seaweedfs/issues/6837

weed/command/s3.go

@@ -51,6 +51,7 @@ type S3Options struct {
 	metricsHttpIp             *string
 	allowEmptyFolder          *bool
 	allowDeleteBucketNotEmpty *bool
+	allowDeleteObjectsByTTL   *bool
 	auditLogConfig            *string
 	localFilerSocket          *string
 	dataCenter                *string
@@ -80,6 +81,7 @@ func init() {
 	s3StandaloneOptions.metricsHttpIp = cmdS3.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
 	s3StandaloneOptions.allowEmptyFolder = cmdS3.Flag.Bool("allowEmptyFolder", true, "allow empty folders")
 	s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
+	s3StandaloneOptions.allowDeleteObjectsByTTL = cmdS3.Flag.Bool("allowDeleteObjectsByTTL", false, "allow deleting all expired entries")
 	s3StandaloneOptions.localFilerSocket = cmdS3.Flag.String("localFilerSocket", "", "local filer socket path")
 	s3StandaloneOptions.localSocket = cmdS3.Flag.String("localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
 	s3StandaloneOptions.idleTimeout = cmdS3.Flag.Int("idleTimeout", 10, "connection idle seconds")
@@ -261,6 +263,7 @@ func (s3opt *S3Options) startS3Server() bool {
 		GrpcDialOption:            grpcDialOption,
 		AllowEmptyFolder:          *s3opt.allowEmptyFolder,
 		AllowDeleteBucketNotEmpty: *s3opt.allowDeleteBucketNotEmpty,
+		AllowDeleteObjectsByTTL:   *s3opt.allowDeleteObjectsByTTL,
 		LocalFilerSocket:          localFilerSocket,
 		DataCenter:                *s3opt.dataCenter,
 		FilerGroup:                filerGroup,

weed/s3api/s3api_object_handlers_list.go

@@ -9,6 +9,7 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -308,6 +309,12 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 						cursor.maxKeys--
 						lastEntryWasCommonPrefix = false
 					}
+					if s3a.option.AllowDeleteObjectsByTTL && entry.Attributes != nil && entry.Attributes.TtlSec > 0 &&
+						(entry.Attributes.GetMtime()+int64(entry.Attributes.TtlSec)) >= time.Now().Unix() {
+						if delErr := doDeleteEntry(client, dir, entry.Name, true, false); delErr != nil {
+							glog.Errorf("delete expired entries %s/%s: %v", dir, entry.Name, delErr)
+						}
+					}
 				}
 			})
 			if doErr != nil {

weed/s3api/s3api_server.go

@@ -41,6 +41,7 @@ type S3ApiServerOption struct {
 	GrpcDialOption            grpc.DialOption
 	AllowEmptyFolder          bool
 	AllowDeleteBucketNotEmpty bool
+	AllowDeleteObjectsByTTL   bool
 	LocalFilerSocket          string
 	DataCenter                string
 	FilerGroup                string