Revert "s3: listObjectParts return ErrNoSuchUpload if does not exist"

This reverts commit 6cf2e7d493.

docker/Dockerfile.rocksdb_large

@@ -9,7 +9,7 @@ ENV ROCKSDB_VERSION v6.22.1
 RUN cd /tmp && \
     git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \
     cd rocksdb && \
-    make static_lib && \
+    PORTABLE=1 make static_lib && \
     make install-static
 
 ENV CGO_CFLAGS "-I/tmp/rocksdb/include"
@@ -29,7 +29,7 @@ FROM alpine AS final
 LABEL author="Chris Lu"
 COPY --from=builder /go/bin/weed /usr/bin/
 RUN mkdir -p /etc/seaweedfs
-COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
+COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml
 COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
 RUN apk add fuse snappy gflags
 
@@ -50,7 +50,7 @@ EXPOSE 8333
 # webdav server http port
 EXPOSE 7333
 
-RUN mkdir -p /data/filerldb2
+RUN mkdir -p /data/filer_rocksdb
 
 VOLUME /data
 

docker/filer_rocksdb.toml

@@ -0,0 +1,3 @@
+[rocksdb]
+enabled = true
+dir = "/data/filer_rocksdb"

go.mod

@@ -162,7 +162,10 @@ require (
 	modernc.org/token v1.0.0 // indirect
 )
 
-require github.com/fluent/fluent-logger-golang v1.8.0
+require (
+	github.com/fluent/fluent-logger-golang v1.8.0
+	github.com/hanwen/go-fuse/v2 v2.1.0
+)
 
 require (
 	cloud.google.com/go/kms v1.0.0 // indirect

go.sum

@@ -415,6 +415,10 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
+github.com/hanwen/go-fuse v1.0.0 h1:GxS9Zrn6c35/BnfiVsZVWmsG803xwE7eVRDvcf/BEVc=
+github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok=
+github.com/hanwen/go-fuse/v2 v2.1.0 h1:+32ffteETaLYClUj0a3aHjZ1hOPxxaNEHiZiujuDaek=
+github.com/hanwen/go-fuse/v2 v2.1.0/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc=
 github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
 github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
@@ -512,6 +516,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kurin/blazer v0.5.3 h1:SAgYv0TKU0kN/ETfO5ExjNAPyMt2FocO2s/UlCHfjAk=
 github.com/kurin/blazer v0.5.3/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E=
 github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=

k8s/helm_charts2/templates/volume-statefulset.yaml

@@ -40,7 +40,7 @@ spec:
       imagePullSecrets:
         - name: {{ .Values.global.imagePullSecrets }}
       {{- end }}
-      terminationGracePeriodSeconds: 10
+      terminationGracePeriodSeconds: 150
       {{- if .Values.volume.priorityClassName }}
       priorityClassName: {{ .Values.volume.priorityClassName | quote }}
       {{- end }}

other/java/examples/pom.xml

@@ -23,7 +23,7 @@
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
-            <version>2.9.2</version>
+            <version>2.10.1</version>
             <scope>compile</scope>
         </dependency>
     </dependencies>

other/java/hdfs-over-ftp/pom.xml

@@ -36,7 +36,7 @@
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
-            <version>3.2.1</version>
+            <version>3.2.2</version>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>

other/java/hdfs2/pom.xml

@@ -6,7 +6,7 @@
 
     <properties>
         <seaweedfs.client.version>2.85</seaweedfs.client.version>
-        <hadoop.version>2.9.2</hadoop.version>
+        <hadoop.version>2.10.1</hadoop.version>
     </properties>
 
     <groupId>com.github.chrislusf</groupId>

other/java/hdfs3/pom.xml

@@ -6,7 +6,7 @@
 
     <properties>
         <seaweedfs.client.version>2.85</seaweedfs.client.version>
-        <hadoop.version>3.1.1</hadoop.version>
+        <hadoop.version>3.1.4</hadoop.version>
     </properties>
 
     <groupId>com.github.chrislusf</groupId>

weed/Makefile

@@ -21,6 +21,10 @@ debug_mount:
 	go build -gcflags="all=-N -l"
 	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 mount -dir=~/tmp/mm -cacheCapacityMB=0 -filer.path=/ -umask=000
 
+debug_mount2:
+	go build -gcflags="all=-N -l"
+	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 mount2 -dir=~/tmp/mm -cacheCapacityMB=0 -filer.path=/ -umask=000
+
 debug_server:
 	go build -gcflags="all=-N -l"
 	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- server -dir=~/tmp/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1

weed/command/command.go

@@ -30,6 +30,7 @@ var Commands = []*Command{
 	cmdMaster,
 	cmdMasterFollower,
 	cmdMount,
+	cmdMount2,
 	cmdS3,
 	cmdIam,
 	cmdMsgBroker,

weed/command/filer_cat.go

@@ -8,7 +8,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 	"google.golang.org/grpc"
-	"math"
 	"net/url"
 	"os"
 	"strings"
@@ -115,7 +114,7 @@ func runFilerCat(cmd *Command, args []string) bool {
 
 		filerCat.filerClient = client
 
-		return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
+		return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, int64(filer.FileSize(respLookupEntry.Entry)))
 
 	})
 

weed/command/mount2.go

@@ -0,0 +1,83 @@
+package command
+
+import (
+	"os"
+	"time"
+)
+
+type Mount2Options struct {
+	filer              *string
+	filerMountRootPath *string
+	dir                *string
+	dirAutoCreate      *bool
+	collection         *string
+	replication        *string
+	diskType           *string
+	ttlSec             *int
+	chunkSizeLimitMB   *int
+	concurrentWriters  *int
+	cacheDir           *string
+	cacheSizeMB        *int64
+	dataCenter         *string
+	allowOthers        *bool
+	umaskString        *string
+	nonempty           *bool
+	volumeServerAccess *string
+	uidMap             *string
+	gidMap             *string
+	readOnly           *bool
+	debug              *bool
+	debugPort          *int
+}
+
+var (
+	mount2Options Mount2Options
+)
+
+func init() {
+	cmdMount2.Run = runMount2 // break init cycle
+	mount2Options.filer = cmdMount2.Flag.String("filer", "localhost:8888", "comma-separated weed filer location")
+	mount2Options.filerMountRootPath = cmdMount2.Flag.String("filer.path", "/", "mount this remote path from filer server")
+	mount2Options.dir = cmdMount2.Flag.String("dir", ".", "mount weed filer to this directory")
+	mount2Options.dirAutoCreate = cmdMount2.Flag.Bool("dirAutoCreate", false, "auto create the directory to mount to")
+	mount2Options.collection = cmdMount2.Flag.String("collection", "", "collection to create the files")
+	mount2Options.replication = cmdMount2.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.")
+	mount2Options.diskType = cmdMount2.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
+	mount2Options.ttlSec = cmdMount2.Flag.Int("ttl", 0, "file ttl in seconds")
+	mount2Options.chunkSizeLimitMB = cmdMount2.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files")
+	mount2Options.concurrentWriters = cmdMount2.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers if not 0")
+	mount2Options.cacheDir = cmdMount2.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data")
+	mount2Options.cacheSizeMB = cmdMount2.Flag.Int64("cacheCapacityMB", 1000, "local file chunk cache capacity in MB (0 will disable cache)")
+	mount2Options.dataCenter = cmdMount2.Flag.String("dataCenter", "", "prefer to write to the data center")
+	mount2Options.allowOthers = cmdMount2.Flag.Bool("allowOthers", true, "allows other users to access the file system")
+	mount2Options.umaskString = cmdMount2.Flag.String("umask", "022", "octal umask, e.g., 022, 0111")
+	mount2Options.nonempty = cmdMount2.Flag.Bool("nonempty", false, "allows the mounting over a non-empty directory")
+	mount2Options.volumeServerAccess = cmdMount2.Flag.String("volumeServerAccess", "direct", "access volume servers by [direct|publicUrl|filerProxy]")
+	mount2Options.uidMap = cmdMount2.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated <local_uid>:<filer_uid>")
+	mount2Options.gidMap = cmdMount2.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated <local_gid>:<filer_gid>")
+	mount2Options.readOnly = cmdMount2.Flag.Bool("readOnly", false, "read only")
+	mount2Options.debug = cmdMount2.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:<debug.port>/debug/pprof/goroutine?debug=2")
+	mount2Options.debugPort = cmdMount2.Flag.Int("debug.port", 6061, "http port for debugging")
+
+	mountCpuProfile = cmdMount2.Flag.String("cpuprofile", "", "cpu profile output file")
+	mountMemProfile = cmdMount2.Flag.String("memprofile", "", "memory profile output file")
+	mountReadRetryTime = cmdMount2.Flag.Duration("readRetryTime", 6*time.Second, "maximum read retry wait time")
+}
+
+var cmdMount2 = &Command{
+	UsageLine: "mount2 -filer=localhost:8888 -dir=/some/dir",
+	Short:     "<WIP> mount weed filer to a directory as file system in userspace(FUSE)",
+	Long: `mount weed filer to userspace.
+
+  Pre-requisites:
+  1) have SeaweedFS master and volume servers running
+  2) have a "weed filer" running
+  These 2 requirements can be achieved with one command "weed server -filer=true"
+
+  This uses github.com/seaweedfs/fuse, which enables writing FUSE file systems on
+  Linux, and OS X.
+
+  On OS X, it requires OSXFUSE (http://osxfuse.github.com/).
+
+  `,
+}

weed/command/mount2_std.go

@@ -0,0 +1,210 @@
+package command
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/mount"
+	"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
+	"github.com/chrislusf/seaweedfs/weed/mount/unmount"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/storage/types"
+	"github.com/hanwen/go-fuse/v2/fuse"
+	"net/http"
+	"os"
+	"os/user"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/util/grace"
+)
+
+func runMount2(cmd *Command, args []string) bool {
+
+	if *mountOptions.debug {
+		go http.ListenAndServe(fmt.Sprintf(":%d", *mountOptions.debugPort), nil)
+	}
+
+	grace.SetupProfiling(*mountCpuProfile, *mountMemProfile)
+	if *mountReadRetryTime < time.Second {
+		*mountReadRetryTime = time.Second
+	}
+	util.RetryWaitTime = *mountReadRetryTime
+
+	umask, umaskErr := strconv.ParseUint(*mountOptions.umaskString, 8, 64)
+	if umaskErr != nil {
+		fmt.Printf("can not parse umask %s", *mountOptions.umaskString)
+		return false
+	}
+
+	if len(args) > 0 {
+		return false
+	}
+
+	return RunMount2(&mount2Options, os.FileMode(umask))
+}
+
+func RunMount2(option *Mount2Options, umask os.FileMode) bool {
+
+	// basic checks
+	chunkSizeLimitMB := *mountOptions.chunkSizeLimitMB
+	if chunkSizeLimitMB <= 0 {
+		fmt.Printf("Please specify a reasonable buffer size.")
+		return false
+	}
+
+	// try to connect to filer
+	filerAddresses := pb.ServerAddresses(*option.filer).ToAddresses()
+	util.LoadConfiguration("security", false)
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+	var cipher bool
+	var err error
+	for i := 0; i < 10; i++ {
+		err = pb.WithOneOfGrpcFilerClients(false, filerAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+			if err != nil {
+				return fmt.Errorf("get filer grpc address %v configuration: %v", filerAddresses, err)
+			}
+			cipher = resp.Cipher
+			return nil
+		})
+		if err != nil {
+			glog.V(0).Infof("failed to talk to filer %v: %v", filerAddresses, err)
+			glog.V(0).Infof("wait for %d seconds ...", i+1)
+			time.Sleep(time.Duration(i+1) * time.Second)
+		}
+	}
+	if err != nil {
+		glog.Errorf("failed to talk to filer %v: %v", filerAddresses, err)
+		return true
+	}
+
+	filerMountRootPath := *option.filerMountRootPath
+
+	// clean up mount point
+	dir := util.ResolvePath(*option.dir)
+	if dir == "" {
+		fmt.Printf("Please specify the mount directory via \"-dir\"")
+		return false
+	}
+
+	unmount.Unmount(dir)
+
+	// detect mount folder mode
+	if *option.dirAutoCreate {
+		os.MkdirAll(dir, os.FileMode(0777)&^umask)
+	}
+	fileInfo, err := os.Stat(dir)
+
+	// collect uid, gid
+	uid, gid := uint32(0), uint32(0)
+	mountMode := os.ModeDir | 0777
+	if err == nil {
+		mountMode = os.ModeDir | os.FileMode(0777)&^umask
+		uid, gid = util.GetFileUidGid(fileInfo)
+		fmt.Printf("mount point owner uid=%d gid=%d mode=%s\n", uid, gid, mountMode)
+	} else {
+		fmt.Printf("can not stat %s\n", dir)
+		return false
+	}
+
+	// detect uid, gid
+	if uid == 0 {
+		if u, err := user.Current(); err == nil {
+			if parsedId, pe := strconv.ParseUint(u.Uid, 10, 32); pe == nil {
+				uid = uint32(parsedId)
+			}
+			if parsedId, pe := strconv.ParseUint(u.Gid, 10, 32); pe == nil {
+				gid = uint32(parsedId)
+			}
+			fmt.Printf("current uid=%d gid=%d\n", uid, gid)
+		}
+	}
+
+	// mapping uid, gid
+	uidGidMapper, err := meta_cache.NewUidGidMapper(*option.uidMap, *option.gidMap)
+	if err != nil {
+		fmt.Printf("failed to parse %s %s: %v\n", *option.uidMap, *option.gidMap, err)
+		return false
+	}
+
+	// Ensure target mount point availability
+	if isValid := checkMountPointAvailable(dir); !isValid {
+		glog.Fatalf("Expected mount to still be active, target mount point: %s, please check!", dir)
+		return true
+	}
+
+	// mount fuse
+	fuseMountOptions := &fuse.MountOptions{
+		AllowOther:               *option.allowOthers,
+		Options:                  nil,
+		MaxBackground:            128,
+		MaxWrite:                 1024 * 1024 * 2,
+		MaxReadAhead:             1024 * 1024 * 2,
+		IgnoreSecurityLabels:     false,
+		RememberInodes:           false,
+		FsName:                   *option.filer + ":" + filerMountRootPath,
+		Name:                     "seaweedfs",
+		SingleThreaded:           false,
+		DisableXAttrs:            false,
+		Debug:                    true,
+		EnableLocks:              false,
+		ExplicitDataCacheControl: false,
+		// SyncRead:                 false, // set to false to enable the FUSE_CAP_ASYNC_READ capability
+		DirectMount:      true,
+		DirectMountFlags: 0,
+		// EnableAcl:                false,
+	}
+
+	// find mount point
+	mountRoot := filerMountRootPath
+	if mountRoot != "/" && strings.HasSuffix(mountRoot, "/") {
+		mountRoot = mountRoot[0 : len(mountRoot)-1]
+	}
+
+	seaweedFileSystem := mount.NewSeaweedFileSystem(&mount.Option{
+		MountDirectory:     dir,
+		FilerAddresses:     filerAddresses,
+		GrpcDialOption:     grpcDialOption,
+		FilerMountRootPath: mountRoot,
+		Collection:         *option.collection,
+		Replication:        *option.replication,
+		TtlSec:             int32(*option.ttlSec),
+		DiskType:           types.ToDiskType(*option.diskType),
+		ChunkSizeLimit:     int64(chunkSizeLimitMB) * 1024 * 1024,
+		ConcurrentWriters:  *option.concurrentWriters,
+		CacheDir:           *option.cacheDir,
+		CacheSizeMB:        *option.cacheSizeMB,
+		DataCenter:         *option.dataCenter,
+		MountUid:           uid,
+		MountGid:           gid,
+		MountMode:          mountMode,
+		MountCtime:         fileInfo.ModTime(),
+		MountMtime:         time.Now(),
+		Umask:              umask,
+		VolumeServerAccess: *mountOptions.volumeServerAccess,
+		Cipher:             cipher,
+		UidGidMapper:       uidGidMapper,
+	})
+
+	server, err := fuse.NewServer(seaweedFileSystem, dir, fuseMountOptions)
+	if err != nil {
+		glog.Fatalf("Mount fail: %v", err)
+	}
+	grace.OnInterrupt(func() {
+		unmount.Unmount(dir)
+	})
+
+	seaweedFileSystem.StartBackgroundTasks()
+
+	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH)
+
+	server.Serve()
+
+	return true
+}

weed/command/scaffold.go

@@ -2,7 +2,7 @@ package command
 
 import (
 	"fmt"
-	"os"
+	"github.com/chrislusf/seaweedfs/weed/util"
 	"path/filepath"
 
 	"github.com/chrislusf/seaweedfs/weed/command/scaffold"
@@ -56,7 +56,7 @@ func runScaffold(cmd *Command, args []string) bool {
 	}
 
 	if *outputPath != "" {
-		os.WriteFile(filepath.Join(*outputPath, *config+".toml"), []byte(content), 0644)
+		util.WriteFile(filepath.Join(*outputPath, *config+".toml"), []byte(content), 0644)
 	} else {
 		fmt.Println(content)
 	}

weed/filer/filechunks.go

@@ -101,6 +101,21 @@ func DoMinusChunks(as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk) {
 	return
 }
 
+func DoMinusChunksBySourceFileId(as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk) {
+
+	fileIds := make(map[string]bool)
+	for _, interval := range bs {
+		fileIds[interval.GetFileIdString()] = true
+	}
+	for _, chunk := range as {
+		if _, found := fileIds[chunk.GetSourceFileId()]; !found {
+			delta = append(delta, chunk)
+		}
+	}
+
+	return
+}
+
 type ChunkView struct {
 	FileId      string
 	Offset      int64

weed/filer/filer_conf.go

@@ -75,12 +75,12 @@ func (fc *FilerConf) loadFromFiler(filer *Filer) (err error) {
 		return fc.LoadFromBytes(entry.Content)
 	}
 
-	return fc.loadFromChunks(filer, entry.Content, entry.Chunks)
+	return fc.loadFromChunks(filer, entry.Content, entry.Chunks, entry.Size())
 }
 
-func (fc *FilerConf) loadFromChunks(filer *Filer, content []byte, chunks []*filer_pb.FileChunk) (err error) {
+func (fc *FilerConf) loadFromChunks(filer *Filer, content []byte, chunks []*filer_pb.FileChunk, size uint64) (err error) {
 	if len(content) == 0 {
-		content, err = filer.readEntry(chunks)
+		content, err = filer.readEntry(chunks, size)
 		if err != nil {
 			glog.Errorf("read filer conf content: %v", err)
 			return

weed/filer/filer_on_meta_event.go

@@ -2,8 +2,6 @@ package filer
 
 import (
 	"bytes"
-	"math"
-
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
@@ -55,9 +53,9 @@ func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataR
 	}
 }
 
-func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) {
+func (f *Filer) readEntry(chunks []*filer_pb.FileChunk, size uint64) ([]byte, error) {
 	var buf bytes.Buffer
-	err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64)
+	err := StreamContent(f.MasterClient, &buf, chunks, 0, int64(size))
 	if err != nil {
 		return nil, err
 	}
@@ -66,7 +64,7 @@ func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) {
 
 func (f *Filer) reloadFilerConfiguration(entry *filer_pb.Entry) {
 	fc := NewFilerConf()
-	err := fc.loadFromChunks(f, entry.Content, entry.Chunks)
+	err := fc.loadFromChunks(f, entry.Content, entry.Chunks, FileSize(entry))
 	if err != nil {
 		glog.Errorf("read filer conf chunks: %v", err)
 		return

weed/filer/read_write.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
-	"math"
 	"time"
 )
 
@@ -23,7 +22,7 @@ func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.Seaweed
 		return err
 	}
 
-	return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
+	return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, int64(FileSize(respLookupEntry.Entry)))
 
 }
 

weed/filesys/file.go

@@ -72,8 +72,8 @@ func (file *File) Attr(ctx context.Context, attr *fuse.Attr) (err error) {
 	attr.Mtime = time.Unix(entry.Attributes.Mtime, 0)
 	attr.Gid = entry.Attributes.Gid
 	attr.Uid = entry.Attributes.Uid
-	attr.Blocks = attr.Size/blockSize + 1
-	attr.BlockSize = uint32(file.wfs.option.ChunkSizeLimit)
+	attr.BlockSize = blockSize
+	attr.Blocks = (attr.Size + blockSize - 1) / blockSize
 	if entry.HardLinkCounter > 0 {
 		attr.Nlink = uint32(entry.HardLinkCounter)
 	}

weed/images/orientation_test.go

@@ -1,6 +1,7 @@
 package images
 
 import (
+	"github.com/chrislusf/seaweedfs/weed/util"
 	"os"
 	"testing"
 )
@@ -12,7 +13,7 @@ func TestXYZ(t *testing.T) {
 
 	fixed_data := FixJpgOrientation(dat)
 
-	os.WriteFile("fixed1.jpg", fixed_data, 0644)
+	util.WriteFile("fixed1.jpg", fixed_data, 0644)
 
 	os.Remove("fixed1.jpg")
 

weed/images/resizing_test.go

@@ -2,6 +2,7 @@ package images
 
 import (
 	"bytes"
+	"github.com/chrislusf/seaweedfs/weed/util"
 	"os"
 	"testing"
 )
@@ -15,7 +16,7 @@ func TestResizing(t *testing.T) {
 	buf := new(bytes.Buffer)
 	buf.ReadFrom(resized)
 
-	os.WriteFile("resized1.png", buf.Bytes(), 0644)
+	util.WriteFile("resized1.png", buf.Bytes(), 0644)
 
 	os.Remove("resized1.png")
 

weed/mount/directory.go

@@ -0,0 +1,42 @@
+package mount
+
+import (
+	"bytes"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/hanwen/go-fuse/v2/fs"
+	"strings"
+)
+
+type Directory struct {
+	fs.Inode
+
+	name   string
+	wfs    *WFS
+	entry  *filer_pb.Entry
+	parent *Directory
+	id     uint64
+}
+
+func (dir *Directory) FullPath() string {
+	var parts []string
+	for p := dir; p != nil; p = p.parent {
+		if strings.HasPrefix(p.name, "/") {
+			if len(p.name) > 1 {
+				parts = append(parts, p.name[1:])
+			}
+		} else {
+			parts = append(parts, p.name)
+		}
+	}
+
+	if len(parts) == 0 {
+		return "/"
+	}
+
+	var buf bytes.Buffer
+	for i := len(parts) - 1; i >= 0; i-- {
+		buf.WriteString("/")
+		buf.WriteString(parts[i])
+	}
+	return buf.String()
+}

weed/mount/directory_read.go

@@ -0,0 +1,84 @@
+package mount
+
+import (
+	"context"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/hanwen/go-fuse/v2/fs"
+	"github.com/hanwen/go-fuse/v2/fuse"
+	"math"
+	"os"
+	"syscall"
+)
+
+var _ = fs.NodeReaddirer(&Directory{})
+var _ = fs.NodeGetattrer(&Directory{})
+
+func (dir *Directory) Getattr(ctx context.Context, fh fs.FileHandle, out *fuse.AttrOut) syscall.Errno {
+	out.Mode = 0755
+	return 0
+}
+
+func (dir *Directory) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) {
+
+	dirPath := util.FullPath(dir.FullPath())
+	glog.V(4).Infof("Readdir %s", dirPath)
+
+	sourceChan := make(chan fuse.DirEntry, 64)
+
+	stream := newDirectoryListStream(sourceChan)
+
+	processEachEntryFn := func(entry *filer.Entry, isLast bool) {
+		sourceChan <- fuse.DirEntry{
+			Mode: uint32(entry.Mode),
+			Name: entry.Name(),
+			Ino:  dirPath.Child(entry.Name()).AsInode(os.ModeDir),
+		}
+	}
+
+	if err := meta_cache.EnsureVisited(dir.wfs.metaCache, dir.wfs, dirPath); err != nil {
+		glog.Errorf("dir ReadDirAll %s: %v", dirPath, err)
+		return nil, fs.ToErrno(os.ErrInvalid)
+	}
+	go func() {
+		dir.wfs.metaCache.ListDirectoryEntries(context.Background(), dirPath, "", false, int64(math.MaxInt32), func(entry *filer.Entry) bool {
+			processEachEntryFn(entry, false)
+			return true
+		})
+		close(sourceChan)
+	}()
+
+	return stream, fs.OK
+}
+
+var _ = fs.DirStream(&DirectoryListStream{})
+
+type DirectoryListStream struct {
+	next       fuse.DirEntry
+	sourceChan chan fuse.DirEntry
+	isStarted  bool
+	hasNext    bool
+}
+
+func newDirectoryListStream(ch chan fuse.DirEntry) *DirectoryListStream {
+	return &DirectoryListStream{
+		sourceChan: ch,
+	}
+}
+
+func (i *DirectoryListStream) HasNext() bool {
+	if !i.isStarted {
+		i.next, i.hasNext = <-i.sourceChan
+		i.isStarted = true
+	}
+	return i.hasNext
+}
+func (i *DirectoryListStream) Next() (fuse.DirEntry, syscall.Errno) {
+	t := i.next
+	i.next, i.hasNext = <-i.sourceChan
+	return t, fs.OK
+}
+func (i *DirectoryListStream) Close() {
+}

weed/mount/dirty_pages_chunked.go

@@ -0,0 +1,99 @@
+package mount
+
+import (
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filesys/page_writer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"io"
+	"sync"
+	"time"
+)
+
+type ChunkedDirtyPages struct {
+	fh             *FileHandle
+	writeWaitGroup sync.WaitGroup
+	lastErr        error
+	collection     string
+	replication    string
+	uploadPipeline *page_writer.UploadPipeline
+	hasWrites      bool
+}
+
+var (
+	_ = page_writer.DirtyPages(&ChunkedDirtyPages{})
+)
+
+func newMemoryChunkPages(fh *FileHandle, chunkSize int64) *ChunkedDirtyPages {
+
+	dirtyPages := &ChunkedDirtyPages{
+		fh: fh,
+	}
+
+	dirtyPages.uploadPipeline = page_writer.NewUploadPipeline(fh.wfs.concurrentWriters, chunkSize, dirtyPages.saveChunkedFileIntevalToStorage, fh.wfs.option.ConcurrentWriters)
+
+	return dirtyPages
+}
+
+func (pages *ChunkedDirtyPages) AddPage(offset int64, data []byte) {
+	pages.hasWrites = true
+
+	glog.V(4).Infof("%v memory AddPage [%d, %d)", pages.fh, offset, offset+int64(len(data)))
+	pages.uploadPipeline.SaveDataAt(data, offset)
+
+	return
+}
+
+func (pages *ChunkedDirtyPages) FlushData() error {
+	if !pages.hasWrites {
+		return nil
+	}
+	pages.uploadPipeline.FlushAll()
+	if pages.lastErr != nil {
+		return fmt.Errorf("flush data: %v", pages.lastErr)
+	}
+	return nil
+}
+
+func (pages *ChunkedDirtyPages) ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) {
+	if !pages.hasWrites {
+		return
+	}
+	return pages.uploadPipeline.MaybeReadDataAt(data, startOffset)
+}
+
+func (pages *ChunkedDirtyPages) GetStorageOptions() (collection, replication string) {
+	return pages.collection, pages.replication
+}
+
+func (pages *ChunkedDirtyPages) saveChunkedFileIntevalToStorage(reader io.Reader, offset int64, size int64, cleanupFn func()) {
+
+	mtime := time.Now().UnixNano()
+	defer cleanupFn()
+
+	fileFullPath := pages.fh.FullPath()
+	fileName := fileFullPath.Name()
+	chunk, collection, replication, err := pages.fh.wfs.saveDataAsChunk(fileFullPath)(reader, fileName, offset)
+	if err != nil {
+		glog.V(0).Infof("%v saveToStorage [%d,%d): %v", fileFullPath, offset, offset+size, err)
+		pages.lastErr = err
+		return
+	}
+	chunk.Mtime = mtime
+	pages.collection, pages.replication = collection, replication
+	pages.fh.addChunks([]*filer_pb.FileChunk{chunk})
+	pages.fh.entryViewCache = nil
+	glog.V(3).Infof("%v saveToStorage %s [%d,%d)", fileFullPath, chunk.FileId, offset, offset+size)
+
+}
+
+func (pages ChunkedDirtyPages) Destroy() {
+	pages.uploadPipeline.Shutdown()
+}
+
+func (pages *ChunkedDirtyPages) LockForRead(startOffset, stopOffset int64) {
+	pages.uploadPipeline.LockForRead(startOffset, stopOffset)
+}
+func (pages *ChunkedDirtyPages) UnlockForRead(startOffset, stopOffset int64) {
+	pages.uploadPipeline.UnlockForRead(startOffset, stopOffset)
+}

weed/mount/filehandle.go

@@ -0,0 +1,95 @@
+package mount
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"io"
+	"sort"
+	"sync"
+)
+
+type FileHandleId uint64
+
+type FileHandle struct {
+	fh           FileHandleId
+	counter      int64
+	entry        *filer_pb.Entry
+	chunkAddLock sync.Mutex
+	inode        uint64
+	wfs          *WFS
+
+	// cache file has been written to
+	dirtyMetadata  bool
+	dirtyPages     *PageWriter
+	entryViewCache []filer.VisibleInterval
+	reader         io.ReaderAt
+	contentType    string
+	handle         uint64
+	sync.Mutex
+
+	isDeleted bool
+}
+
+func newFileHandle(wfs *WFS, handleId FileHandleId, inode uint64, entry *filer_pb.Entry) *FileHandle {
+	fh := &FileHandle{
+		fh:      handleId,
+		counter: 1,
+		inode:   inode,
+		wfs:     wfs,
+	}
+	// dirtyPages: newContinuousDirtyPages(file, writeOnly),
+	fh.dirtyPages = newPageWriter(fh, wfs.option.ChunkSizeLimit)
+	if entry != nil {
+		entry.Attributes.FileSize = filer.FileSize(entry)
+	}
+
+	return fh
+}
+
+func (fh *FileHandle) FullPath() util.FullPath {
+	return fh.wfs.inodeToPath.GetPath(fh.inode)
+}
+
+func (fh *FileHandle) addChunks(chunks []*filer_pb.FileChunk) {
+
+	// find the earliest incoming chunk
+	newChunks := chunks
+	earliestChunk := newChunks[0]
+	for i := 1; i < len(newChunks); i++ {
+		if lessThan(earliestChunk, newChunks[i]) {
+			earliestChunk = newChunks[i]
+		}
+	}
+
+	if fh.entry == nil {
+		return
+	}
+
+	// pick out-of-order chunks from existing chunks
+	for _, chunk := range fh.entry.Chunks {
+		if lessThan(earliestChunk, chunk) {
+			chunks = append(chunks, chunk)
+		}
+	}
+
+	// sort incoming chunks
+	sort.Slice(chunks, func(i, j int) bool {
+		return lessThan(chunks[i], chunks[j])
+	})
+
+	glog.V(4).Infof("%s existing %d chunks adds %d more", fh.FullPath(), len(fh.entry.Chunks), len(chunks))
+
+	fh.chunkAddLock.Lock()
+	fh.entry.Chunks = append(fh.entry.Chunks, newChunks...)
+	fh.entryViewCache = nil
+	fh.chunkAddLock.Unlock()
+}
+
+func lessThan(a, b *filer_pb.FileChunk) bool {
+	if a.Mtime == b.Mtime {
+		return a.Fid.FileKey < b.Fid.FileKey
+	}
+	return a.Mtime < b.Mtime
+}

weed/mount/filehandle_map.go

@@ -0,0 +1,84 @@
+package mount
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"sync"
+)
+
+type FileHandleToInode struct {
+	sync.RWMutex
+	nextFh   FileHandleId
+	inode2fh map[uint64]*FileHandle
+	fh2inode map[FileHandleId]uint64
+}
+
+func NewFileHandleToInode() *FileHandleToInode {
+	return &FileHandleToInode{
+		inode2fh: make(map[uint64]*FileHandle),
+		fh2inode: make(map[FileHandleId]uint64),
+		nextFh:   0,
+	}
+}
+
+func (i *FileHandleToInode) GetFileHandle(fh FileHandleId) *FileHandle {
+	i.RLock()
+	defer i.RUnlock()
+	inode, found := i.fh2inode[fh]
+	if found {
+		return i.inode2fh[inode]
+	}
+	return nil
+}
+
+func (i *FileHandleToInode) FindFileHandle(inode uint64) (fh *FileHandle, found bool) {
+	i.RLock()
+	defer i.RUnlock()
+	fh, found = i.inode2fh[inode]
+	return
+}
+
+func (i *FileHandleToInode) AcquireFileHandle(wfs *WFS, inode uint64, entry *filer_pb.Entry) *FileHandle {
+	i.Lock()
+	defer i.Unlock()
+	fh, found := i.inode2fh[inode]
+	if !found {
+		fh = newFileHandle(wfs, i.nextFh, inode, entry)
+		i.nextFh++
+		i.inode2fh[inode] = fh
+		i.fh2inode[fh.fh] = inode
+	} else {
+		fh.counter++
+	}
+	return fh
+}
+
+func (i *FileHandleToInode) ReleaseByInode(inode uint64) {
+	i.Lock()
+	defer i.Unlock()
+	fh, found := i.inode2fh[inode]
+	if found {
+		fh.counter--
+		if fh.counter <= 0 {
+			delete(i.inode2fh, inode)
+			delete(i.fh2inode, fh.fh)
+		}
+	}
+}
+func (i *FileHandleToInode) ReleaseByHandle(fh FileHandleId) {
+	i.Lock()
+	defer i.Unlock()
+	inode, found := i.fh2inode[fh]
+	if found {
+		fhHandle, fhFound := i.inode2fh[inode]
+		if !fhFound {
+			delete(i.fh2inode, fh)
+		} else {
+			fhHandle.counter--
+			if fhHandle.counter <= 0 {
+				delete(i.inode2fh, inode)
+				delete(i.fh2inode, fhHandle.fh)
+			}
+		}
+
+	}
+}

weed/mount/filehandle_read.go

@@ -0,0 +1,114 @@
+package mount
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"io"
+	"math"
+)
+
+func (fh *FileHandle) lockForRead(startOffset int64, size int) {
+	fh.dirtyPages.LockForRead(startOffset, startOffset+int64(size))
+}
+func (fh *FileHandle) unlockForRead(startOffset int64, size int) {
+	fh.dirtyPages.UnlockForRead(startOffset, startOffset+int64(size))
+}
+
+func (fh *FileHandle) readFromDirtyPages(buff []byte, startOffset int64) (maxStop int64) {
+	maxStop = fh.dirtyPages.ReadDirtyDataAt(buff, startOffset)
+	return
+}
+
+func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {
+
+	fileFullPath := fh.FullPath()
+
+	entry := fh.entry
+	if entry == nil {
+		return 0, io.EOF
+	}
+
+	if entry.IsInRemoteOnly() {
+		glog.V(4).Infof("download remote entry %s", fileFullPath)
+		newEntry, err := fh.downloadRemoteEntry(entry)
+		if err != nil {
+			glog.V(1).Infof("download remote entry %s: %v", fileFullPath, err)
+			return 0, err
+		}
+		entry = newEntry
+	}
+
+	fileSize := int64(filer.FileSize(entry))
+
+	if fileSize == 0 {
+		glog.V(1).Infof("empty fh %v", fileFullPath)
+		return 0, io.EOF
+	}
+
+	if offset+int64(len(buff)) <= int64(len(entry.Content)) {
+		totalRead := copy(buff, entry.Content[offset:])
+		glog.V(4).Infof("file handle read cached %s [%d,%d] %d", fileFullPath, offset, offset+int64(totalRead), totalRead)
+		return int64(totalRead), nil
+	}
+
+	var chunkResolveErr error
+	if fh.entryViewCache == nil {
+		fh.entryViewCache, chunkResolveErr = filer.NonOverlappingVisibleIntervals(fh.wfs.LookupFn(), entry.Chunks, 0, math.MaxInt64)
+		if chunkResolveErr != nil {
+			return 0, fmt.Errorf("fail to resolve chunk manifest: %v", chunkResolveErr)
+		}
+		fh.reader = nil
+	}
+
+	reader := fh.reader
+	if reader == nil {
+		chunkViews := filer.ViewFromVisibleIntervals(fh.entryViewCache, 0, math.MaxInt64)
+		glog.V(4).Infof("file handle read %s [%d,%d) from %d views", fileFullPath, offset, offset+int64(len(buff)), len(chunkViews))
+		for _, chunkView := range chunkViews {
+			glog.V(4).Infof("  read %s [%d,%d) from chunk %+v", fileFullPath, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size), chunkView.FileId)
+		}
+		reader = filer.NewChunkReaderAtFromClient(fh.wfs.LookupFn(), chunkViews, fh.wfs.chunkCache, fileSize)
+	}
+	fh.reader = reader
+
+	totalRead, err := reader.ReadAt(buff, offset)
+
+	if err != nil && err != io.EOF {
+		glog.Errorf("file handle read %s: %v", fileFullPath, err)
+	}
+
+	glog.V(4).Infof("file handle read %s [%d,%d] %d : %v", fileFullPath, offset, offset+int64(totalRead), totalRead, err)
+
+	return int64(totalRead), err
+}
+
+func (fh *FileHandle) downloadRemoteEntry(entry *filer_pb.Entry) (*filer_pb.Entry, error) {
+
+	fileFullPath := fh.FullPath()
+	dir, _ := fileFullPath.DirAndName()
+
+	err := fh.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.CacheRemoteObjectToLocalClusterRequest{
+			Directory: string(dir),
+			Name:      entry.Name,
+		}
+
+		glog.V(4).Infof("download entry: %v", request)
+		resp, err := client.CacheRemoteObjectToLocalCluster(context.Background(), request)
+		if err != nil {
+			return fmt.Errorf("CacheRemoteObjectToLocalCluster file %s: %v", fileFullPath, err)
+		}
+
+		entry = resp.Entry
+
+		fh.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, resp.Entry))
+
+		return nil
+	})
+
+	return entry, err
+}

weed/mount/inode_to_path.go

@@ -0,0 +1,161 @@
+package mount
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"sync"
+)
+
+type InodeToPath struct {
+	sync.RWMutex
+	nextInodeId uint64
+	inode2path  map[uint64]*InodeEntry
+	path2inode  map[util.FullPath]uint64
+}
+type InodeEntry struct {
+	util.FullPath
+	nlookup          uint64
+	isDirectory      bool
+	isChildrenCached bool
+}
+
+func NewInodeToPath() *InodeToPath {
+	t := &InodeToPath{
+		inode2path:  make(map[uint64]*InodeEntry),
+		path2inode:  make(map[util.FullPath]uint64),
+		nextInodeId: 2, // the root inode id is 1
+	}
+	t.inode2path[1] = &InodeEntry{"/", 1, true, false}
+	t.path2inode["/"] = 1
+	return t
+}
+
+func (i *InodeToPath) Lookup(path util.FullPath, isDirectory bool) uint64 {
+	i.Lock()
+	defer i.Unlock()
+	inode, found := i.path2inode[path]
+	if !found {
+		inode = i.nextInodeId
+		i.nextInodeId++
+		i.path2inode[path] = inode
+		i.inode2path[inode] = &InodeEntry{path, 1, isDirectory, false}
+	} else {
+		i.inode2path[inode].nlookup++
+	}
+	return inode
+}
+
+func (i *InodeToPath) GetInode(path util.FullPath) uint64 {
+	if path == "/" {
+		return 1
+	}
+	i.Lock()
+	defer i.Unlock()
+	inode, found := i.path2inode[path]
+	if !found {
+		// glog.Fatalf("GetInode unknown inode for %s", path)
+		// this could be the parent for mount point
+	}
+	return inode
+}
+
+func (i *InodeToPath) GetPath(inode uint64) util.FullPath {
+	i.RLock()
+	defer i.RUnlock()
+	path, found := i.inode2path[inode]
+	if !found {
+		glog.Fatalf("not found inode %d", inode)
+	}
+	return path.FullPath
+}
+
+func (i *InodeToPath) HasPath(path util.FullPath) bool {
+	i.RLock()
+	defer i.RUnlock()
+	_, found := i.path2inode[path]
+	return found
+}
+
+func (i *InodeToPath) MarkChildrenCached(fullpath util.FullPath) {
+	i.RLock()
+	defer i.RUnlock()
+	inode, found := i.path2inode[fullpath]
+	if !found {
+		glog.Fatalf("MarkChildrenCached not found inode %v", fullpath)
+	}
+	path, found := i.inode2path[inode]
+	path.isChildrenCached = true
+}
+
+func (i *InodeToPath) IsChildrenCached(fullpath util.FullPath) bool {
+	i.RLock()
+	defer i.RUnlock()
+	inode, found := i.path2inode[fullpath]
+	if !found {
+		return false
+	}
+	path, found := i.inode2path[inode]
+	if found {
+		return path.isChildrenCached
+	}
+	return false
+}
+
+func (i *InodeToPath) HasInode(inode uint64) bool {
+	if inode == 1 {
+		return true
+	}
+	i.RLock()
+	defer i.RUnlock()
+	_, found := i.inode2path[inode]
+	return found
+}
+
+func (i *InodeToPath) RemovePath(path util.FullPath) {
+	i.Lock()
+	defer i.Unlock()
+	inode, found := i.path2inode[path]
+	if found {
+		delete(i.path2inode, path)
+		delete(i.inode2path, inode)
+	}
+}
+
+func (i *InodeToPath) MovePath(sourcePath, targetPath util.FullPath) {
+	i.Lock()
+	defer i.Unlock()
+	sourceInode, sourceFound := i.path2inode[sourcePath]
+	targetInode, targetFound := i.path2inode[targetPath]
+	if sourceFound {
+		delete(i.path2inode, sourcePath)
+		i.path2inode[targetPath] = sourceInode
+	} else {
+		// it is possible some source folder items has not been visited before
+		// so no need to worry about their source inodes
+		return
+	}
+	i.inode2path[sourceInode].FullPath = targetPath
+	if targetFound {
+		delete(i.inode2path, targetInode)
+	} else {
+		i.inode2path[sourceInode].nlookup++
+	}
+}
+
+func (i *InodeToPath) Forget(inode, nlookup uint64, onForgetDir func(dir util.FullPath)) {
+	i.Lock()
+	path, found := i.inode2path[inode]
+	if found {
+		path.nlookup -= nlookup
+		if path.nlookup <= 0 {
+			delete(i.path2inode, path.FullPath)
+			delete(i.inode2path, inode)
+		}
+	}
+	i.Unlock()
+	if found {
+		if path.isDirectory && onForgetDir != nil {
+			onForgetDir(path.FullPath)
+		}
+	}
+}

weed/mount/meta_cache/cache_config.go

@@ -0,0 +1,32 @@
+package meta_cache
+
+import "github.com/chrislusf/seaweedfs/weed/util"
+
+var (
+	_ = util.Configuration(&cacheConfig{})
+)
+
+// implementing util.Configuraion
+type cacheConfig struct {
+	dir string
+}
+
+func (c cacheConfig) GetString(key string) string {
+	return c.dir
+}
+
+func (c cacheConfig) GetBool(key string) bool {
+	panic("implement me")
+}
+
+func (c cacheConfig) GetInt(key string) int {
+	panic("implement me")
+}
+
+func (c cacheConfig) GetStringSlice(key string) []string {
+	panic("implement me")
+}
+
+func (c cacheConfig) SetDefault(key string, value interface{}) {
+	panic("implement me")
+}

weed/mount/meta_cache/id_mapper.go

@@ -0,0 +1,101 @@
+package meta_cache
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+type UidGidMapper struct {
+	uidMapper *IdMapper
+	gidMapper *IdMapper
+}
+
+type IdMapper struct {
+	localToFiler map[uint32]uint32
+	filerToLocal map[uint32]uint32
+}
+
+// UidGidMapper translates local uid/gid to filer uid/gid
+// The local storage always persists the same as the filer.
+// The local->filer translation happens when updating the filer first and later saving to meta_cache.
+// And filer->local happens when reading from the meta_cache.
+func NewUidGidMapper(uidPairsStr, gidPairStr string) (*UidGidMapper, error) {
+	uidMapper, err := newIdMapper(uidPairsStr)
+	if err != nil {
+		return nil, err
+	}
+	gidMapper, err := newIdMapper(gidPairStr)
+	if err != nil {
+		return nil, err
+	}
+
+	return &UidGidMapper{
+		uidMapper: uidMapper,
+		gidMapper: gidMapper,
+	}, nil
+}
+
+func (m *UidGidMapper) LocalToFiler(uid, gid uint32) (uint32, uint32) {
+	return m.uidMapper.LocalToFiler(uid), m.gidMapper.LocalToFiler(gid)
+}
+func (m *UidGidMapper) FilerToLocal(uid, gid uint32) (uint32, uint32) {
+	return m.uidMapper.FilerToLocal(uid), m.gidMapper.FilerToLocal(gid)
+}
+
+func (m *IdMapper) LocalToFiler(id uint32) uint32 {
+	value, found := m.localToFiler[id]
+	if found {
+		return value
+	}
+	return id
+}
+func (m *IdMapper) FilerToLocal(id uint32) uint32 {
+	value, found := m.filerToLocal[id]
+	if found {
+		return value
+	}
+	return id
+}
+
+func newIdMapper(pairsStr string) (*IdMapper, error) {
+
+	localToFiler, filerToLocal, err := parseUint32Pairs(pairsStr)
+	if err != nil {
+		return nil, err
+	}
+
+	return &IdMapper{
+		localToFiler: localToFiler,
+		filerToLocal: filerToLocal,
+	}, nil
+
+}
+
+func parseUint32Pairs(pairsStr string) (localToFiler, filerToLocal map[uint32]uint32, err error) {
+
+	if pairsStr == "" {
+		return
+	}
+
+	localToFiler = make(map[uint32]uint32)
+	filerToLocal = make(map[uint32]uint32)
+	for _, pairStr := range strings.Split(pairsStr, ",") {
+		pair := strings.Split(pairStr, ":")
+		localUidStr, filerUidStr := pair[0], pair[1]
+		localUid, localUidErr := strconv.Atoi(localUidStr)
+		if localUidErr != nil {
+			err = fmt.Errorf("failed to parse local %s: %v", localUidStr, localUidErr)
+			return
+		}
+		filerUid, filerUidErr := strconv.Atoi(filerUidStr)
+		if filerUidErr != nil {
+			err = fmt.Errorf("failed to parse remote %s: %v", filerUidStr, filerUidErr)
+			return
+		}
+		localToFiler[uint32(localUid)] = uint32(filerUid)
+		filerToLocal[uint32(filerUid)] = uint32(localUid)
+	}
+
+	return
+}

weed/mount/meta_cache/meta_cache.go

@@ -0,0 +1,160 @@
+package meta_cache
+
+import (
+	"context"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/filer/leveldb"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"os"
+)
+
+// need to have logic similar to FilerStoreWrapper
+// e.g. fill fileId field for chunks
+
+type MetaCache struct {
+	localStore filer.VirtualFilerStore
+	// sync.RWMutex
+	uidGidMapper   *UidGidMapper
+	markCachedFn   func(fullpath util.FullPath)
+	isCachedFn     func(fullpath util.FullPath) bool
+	invalidateFunc func(fullpath util.FullPath, entry *filer_pb.Entry)
+}
+
+func NewMetaCache(dbFolder string, uidGidMapper *UidGidMapper, markCachedFn func(path util.FullPath), isCachedFn func(path util.FullPath) bool, invalidateFunc func(util.FullPath, *filer_pb.Entry)) *MetaCache {
+	return &MetaCache{
+		localStore:   openMetaStore(dbFolder),
+		markCachedFn: markCachedFn,
+		isCachedFn:   isCachedFn,
+		uidGidMapper: uidGidMapper,
+		invalidateFunc: func(fullpath util.FullPath, entry *filer_pb.Entry) {
+			invalidateFunc(fullpath, entry)
+		},
+	}
+}
+
+func openMetaStore(dbFolder string) filer.VirtualFilerStore {
+
+	os.RemoveAll(dbFolder)
+	os.MkdirAll(dbFolder, 0755)
+
+	store := &leveldb.LevelDBStore{}
+	config := &cacheConfig{
+		dir: dbFolder,
+	}
+
+	if err := store.Initialize(config, ""); err != nil {
+		glog.Fatalf("Failed to initialize metadata cache store for %s: %+v", store.GetName(), err)
+	}
+
+	return filer.NewFilerStoreWrapper(store)
+
+}
+
+func (mc *MetaCache) InsertEntry(ctx context.Context, entry *filer.Entry) error {
+	//mc.Lock()
+	//defer mc.Unlock()
+	return mc.doInsertEntry(ctx, entry)
+}
+
+func (mc *MetaCache) doInsertEntry(ctx context.Context, entry *filer.Entry) error {
+	return mc.localStore.InsertEntry(ctx, entry)
+}
+
+func (mc *MetaCache) AtomicUpdateEntryFromFiler(ctx context.Context, oldPath util.FullPath, newEntry *filer.Entry) error {
+	//mc.Lock()
+	//defer mc.Unlock()
+
+	oldDir, _ := oldPath.DirAndName()
+	if mc.isCachedFn(util.FullPath(oldDir)) {
+		if oldPath != "" {
+			if newEntry != nil && oldPath == newEntry.FullPath {
+				// skip the unnecessary deletion
+				// leave the update to the following InsertEntry operation
+			} else {
+				glog.V(3).Infof("DeleteEntry %s", oldPath)
+				if err := mc.localStore.DeleteEntry(ctx, oldPath); err != nil {
+					return err
+				}
+			}
+		}
+	} else {
+		// println("unknown old directory:", oldDir)
+	}
+
+	if newEntry != nil {
+		newDir, _ := newEntry.DirAndName()
+		if mc.isCachedFn(util.FullPath(newDir)) {
+			glog.V(3).Infof("InsertEntry %s/%s", newDir, newEntry.Name())
+			if err := mc.localStore.InsertEntry(ctx, newEntry); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (mc *MetaCache) UpdateEntry(ctx context.Context, entry *filer.Entry) error {
+	//mc.Lock()
+	//defer mc.Unlock()
+	return mc.localStore.UpdateEntry(ctx, entry)
+}
+
+func (mc *MetaCache) FindEntry(ctx context.Context, fp util.FullPath) (entry *filer.Entry, err error) {
+	//mc.RLock()
+	//defer mc.RUnlock()
+	entry, err = mc.localStore.FindEntry(ctx, fp)
+	if err != nil {
+		return nil, err
+	}
+	mc.mapIdFromFilerToLocal(entry)
+	return
+}
+
+func (mc *MetaCache) DeleteEntry(ctx context.Context, fp util.FullPath) (err error) {
+	//mc.Lock()
+	//defer mc.Unlock()
+	return mc.localStore.DeleteEntry(ctx, fp)
+}
+func (mc *MetaCache) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) {
+	//mc.Lock()
+	//defer mc.Unlock()
+	return mc.localStore.DeleteFolderChildren(ctx, fp)
+}
+
+func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) error {
+	//mc.RLock()
+	//defer mc.RUnlock()
+
+	if !mc.isCachedFn(dirPath) {
+		// if this request comes after renaming, it should be fine
+		glog.Warningf("unsynchronized dir: %v", dirPath)
+	}
+
+	_, err := mc.localStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *filer.Entry) bool {
+		mc.mapIdFromFilerToLocal(entry)
+		return eachEntryFunc(entry)
+	})
+	if err != nil {
+		return err
+	}
+	return err
+}
+
+func (mc *MetaCache) Shutdown() {
+	//mc.Lock()
+	//defer mc.Unlock()
+	mc.localStore.Shutdown()
+}
+
+func (mc *MetaCache) mapIdFromFilerToLocal(entry *filer.Entry) {
+	entry.Attr.Uid, entry.Attr.Gid = mc.uidGidMapper.FilerToLocal(entry.Attr.Uid, entry.Attr.Gid)
+}
+
+func (mc *MetaCache) Debug() {
+	if debuggable, ok := mc.localStore.(filer.Debuggable); ok {
+		println("start debugging")
+		debuggable.Debug(os.Stderr)
+	}
+}

weed/mount/meta_cache/meta_cache_init.go

@@ -0,0 +1,67 @@
+package meta_cache
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+func EnsureVisited(mc *MetaCache, client filer_pb.FilerClient, dirPath util.FullPath) error {
+
+	for {
+
+		// the directory children are already cached
+		// so no need for this and upper directories
+		if mc.isCachedFn(dirPath) {
+			return nil
+		}
+
+		if err := doEnsureVisited(mc, client, dirPath); err != nil {
+			return err
+		}
+
+		// continue to parent directory
+		if dirPath != "/" {
+			parent, _ := dirPath.DirAndName()
+			dirPath = util.FullPath(parent)
+		} else {
+			break
+		}
+	}
+
+	return nil
+
+}
+
+func doEnsureVisited(mc *MetaCache, client filer_pb.FilerClient, path util.FullPath) error {
+
+	glog.V(4).Infof("ReadDirAllEntries %s ...", path)
+
+	err := util.Retry("ReadDirAllEntries", func() error {
+		return filer_pb.ReadDirAllEntries(client, path, "", func(pbEntry *filer_pb.Entry, isLast bool) error {
+			entry := filer.FromPbEntry(string(path), pbEntry)
+			if IsHiddenSystemEntry(string(path), entry.Name()) {
+				return nil
+			}
+			if err := mc.doInsertEntry(context.Background(), entry); err != nil {
+				glog.V(0).Infof("read %s: %v", entry.FullPath, err)
+				return err
+			}
+			return nil
+		})
+	})
+
+	if err != nil {
+		err = fmt.Errorf("list %s: %v", path, err)
+	}
+	mc.markCachedFn(path)
+	return err
+}
+
+func IsHiddenSystemEntry(dir, name string) bool {
+	return dir == "/" && (name == "topics" || name == "etc")
+}

weed/mount/meta_cache/meta_cache_subscribe.go

@@ -0,0 +1,68 @@
+package meta_cache
+
+import (
+	"context"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.FilerClient, dir string, lastTsNs int64) error {
+
+	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
+		message := resp.EventNotification
+
+		for _, sig := range message.Signatures {
+			if sig == selfSignature && selfSignature != 0 {
+				return nil
+			}
+		}
+
+		dir := resp.Directory
+		var oldPath util.FullPath
+		var newEntry *filer.Entry
+		if message.OldEntry != nil {
+			oldPath = util.NewFullPath(dir, message.OldEntry.Name)
+			glog.V(4).Infof("deleting %v", oldPath)
+		}
+
+		if message.NewEntry != nil {
+			if message.NewParentPath != "" {
+				dir = message.NewParentPath
+			}
+			key := util.NewFullPath(dir, message.NewEntry.Name)
+			glog.V(4).Infof("creating %v", key)
+			newEntry = filer.FromPbEntry(dir, message.NewEntry)
+		}
+		err := mc.AtomicUpdateEntryFromFiler(context.Background(), oldPath, newEntry)
+		if err == nil {
+			if message.OldEntry != nil && message.NewEntry != nil {
+				oldKey := util.NewFullPath(resp.Directory, message.OldEntry.Name)
+				mc.invalidateFunc(oldKey, message.OldEntry)
+				if message.OldEntry.Name != message.NewEntry.Name {
+					newKey := util.NewFullPath(dir, message.NewEntry.Name)
+					mc.invalidateFunc(newKey, message.NewEntry)
+				}
+			} else if message.OldEntry == nil && message.NewEntry != nil {
+				// no need to invaalidate
+			} else if message.OldEntry != nil && message.NewEntry == nil {
+				oldKey := util.NewFullPath(resp.Directory, message.OldEntry.Name)
+				mc.invalidateFunc(oldKey, message.OldEntry)
+			}
+		}
+
+		return err
+
+	}
+
+	util.RetryForever("followMetaUpdates", func() error {
+		return pb.WithFilerClientFollowMetadata(client, "mount", selfSignature, dir, &lastTsNs, selfSignature, processEventFn, true)
+	}, func(err error) bool {
+		glog.Errorf("follow metadata updates: %v", err)
+		return true
+	})
+
+	return nil
+}

weed/mount/page_writer.go

@@ -0,0 +1,95 @@
+package mount
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/filesys/page_writer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+)
+
+type PageWriter struct {
+	fh          *FileHandle
+	collection  string
+	replication string
+	chunkSize   int64
+
+	randomWriter page_writer.DirtyPages
+}
+
+var (
+	_ = page_writer.DirtyPages(&PageWriter{})
+)
+
+func newPageWriter(fh *FileHandle, chunkSize int64) *PageWriter {
+	pw := &PageWriter{
+		fh:           fh,
+		chunkSize:    chunkSize,
+		randomWriter: newMemoryChunkPages(fh, chunkSize),
+		// randomWriter: newTempFileDirtyPages(fh.f, chunkSize),
+	}
+	return pw
+}
+
+func (pw *PageWriter) AddPage(offset int64, data []byte) {
+
+	glog.V(4).Infof("%v AddPage [%d, %d)", pw.fh.fh, offset, offset+int64(len(data)))
+
+	chunkIndex := offset / pw.chunkSize
+	for i := chunkIndex; len(data) > 0; i++ {
+		writeSize := min(int64(len(data)), (i+1)*pw.chunkSize-offset)
+		pw.addToOneChunk(i, offset, data[:writeSize])
+		offset += writeSize
+		data = data[writeSize:]
+	}
+}
+
+func (pw *PageWriter) addToOneChunk(chunkIndex, offset int64, data []byte) {
+	pw.randomWriter.AddPage(offset, data)
+}
+
+func (pw *PageWriter) FlushData() error {
+	return pw.randomWriter.FlushData()
+}
+
+func (pw *PageWriter) ReadDirtyDataAt(data []byte, offset int64) (maxStop int64) {
+	glog.V(4).Infof("ReadDirtyDataAt %v [%d, %d)", pw.fh.fh, offset, offset+int64(len(data)))
+
+	chunkIndex := offset / pw.chunkSize
+	for i := chunkIndex; len(data) > 0; i++ {
+		readSize := min(int64(len(data)), (i+1)*pw.chunkSize-offset)
+
+		maxStop = pw.randomWriter.ReadDirtyDataAt(data[:readSize], offset)
+
+		offset += readSize
+		data = data[readSize:]
+	}
+
+	return
+}
+
+func (pw *PageWriter) GetStorageOptions() (collection, replication string) {
+	return pw.randomWriter.GetStorageOptions()
+}
+
+func (pw *PageWriter) LockForRead(startOffset, stopOffset int64) {
+	pw.randomWriter.LockForRead(startOffset, stopOffset)
+}
+
+func (pw *PageWriter) UnlockForRead(startOffset, stopOffset int64) {
+	pw.randomWriter.UnlockForRead(startOffset, stopOffset)
+}
+
+func (pw *PageWriter) Destroy() {
+	pw.randomWriter.Destroy()
+}
+
+func max(x, y int64) int64 {
+	if x > y {
+		return x
+	}
+	return y
+}
+func min(x, y int64) int64 {
+	if x < y {
+		return x
+	}
+	return y
+}

weed/mount/page_writer/chunk_interval_list.go

@@ -0,0 +1,115 @@
+package page_writer
+
+import "math"
+
+// ChunkWrittenInterval mark one written interval within one page chunk
+type ChunkWrittenInterval struct {
+	StartOffset int64
+	stopOffset  int64
+	prev        *ChunkWrittenInterval
+	next        *ChunkWrittenInterval
+}
+
+func (interval *ChunkWrittenInterval) Size() int64 {
+	return interval.stopOffset - interval.StartOffset
+}
+
+func (interval *ChunkWrittenInterval) isComplete(chunkSize int64) bool {
+	return interval.stopOffset-interval.StartOffset == chunkSize
+}
+
+// ChunkWrittenIntervalList mark written intervals within one page chunk
+type ChunkWrittenIntervalList struct {
+	head *ChunkWrittenInterval
+	tail *ChunkWrittenInterval
+}
+
+func newChunkWrittenIntervalList() *ChunkWrittenIntervalList {
+	list := &ChunkWrittenIntervalList{
+		head: &ChunkWrittenInterval{
+			StartOffset: -1,
+			stopOffset:  -1,
+		},
+		tail: &ChunkWrittenInterval{
+			StartOffset: math.MaxInt64,
+			stopOffset:  math.MaxInt64,
+		},
+	}
+	list.head.next = list.tail
+	list.tail.prev = list.head
+	return list
+}
+
+func (list *ChunkWrittenIntervalList) MarkWritten(startOffset, stopOffset int64) {
+	interval := &ChunkWrittenInterval{
+		StartOffset: startOffset,
+		stopOffset:  stopOffset,
+	}
+	list.addInterval(interval)
+}
+
+func (list *ChunkWrittenIntervalList) IsComplete(chunkSize int64) bool {
+	return list.size() == 1 && list.head.next.isComplete(chunkSize)
+}
+func (list *ChunkWrittenIntervalList) WrittenSize() (writtenByteCount int64) {
+	for t := list.head; t != nil; t = t.next {
+		writtenByteCount += t.Size()
+	}
+	return
+}
+
+func (list *ChunkWrittenIntervalList) addInterval(interval *ChunkWrittenInterval) {
+
+	p := list.head
+	for ; p.next != nil && p.next.StartOffset <= interval.StartOffset; p = p.next {
+	}
+	q := list.tail
+	for ; q.prev != nil && q.prev.stopOffset >= interval.stopOffset; q = q.prev {
+	}
+
+	if interval.StartOffset <= p.stopOffset && q.StartOffset <= interval.stopOffset {
+		// merge p and q together
+		p.stopOffset = q.stopOffset
+		unlinkNodesBetween(p, q.next)
+		return
+	}
+	if interval.StartOffset <= p.stopOffset {
+		// merge new interval into p
+		p.stopOffset = interval.stopOffset
+		unlinkNodesBetween(p, q)
+		return
+	}
+	if q.StartOffset <= interval.stopOffset {
+		// merge new interval into q
+		q.StartOffset = interval.StartOffset
+		unlinkNodesBetween(p, q)
+		return
+	}
+
+	// add the new interval between p and q
+	unlinkNodesBetween(p, q)
+	p.next = interval
+	interval.prev = p
+	q.prev = interval
+	interval.next = q
+
+}
+
+// unlinkNodesBetween remove all nodes after start and before stop, exclusive
+func unlinkNodesBetween(start *ChunkWrittenInterval, stop *ChunkWrittenInterval) {
+	if start.next == stop {
+		return
+	}
+	start.next.prev = nil
+	start.next = stop
+	stop.prev.next = nil
+	stop.prev = start
+}
+
+func (list *ChunkWrittenIntervalList) size() int {
+	var count int
+	for t := list.head; t != nil; t = t.next {
+		count++
+	}
+	return count - 2
+}

weed/mount/page_writer/chunk_interval_list_test.go

@@ -0,0 +1,49 @@
+package page_writer
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func Test_PageChunkWrittenIntervalList(t *testing.T) {
+	list := newChunkWrittenIntervalList()
+
+	assert.Equal(t, 0, list.size(), "empty list")
+
+	list.MarkWritten(0, 5)
+	assert.Equal(t, 1, list.size(), "one interval")
+
+	list.MarkWritten(0, 5)
+	assert.Equal(t, 1, list.size(), "duplicated interval2")
+
+	list.MarkWritten(95, 100)
+	assert.Equal(t, 2, list.size(), "two intervals")
+
+	list.MarkWritten(50, 60)
+	assert.Equal(t, 3, list.size(), "three intervals")
+
+	list.MarkWritten(50, 55)
+	assert.Equal(t, 3, list.size(), "three intervals merge")
+
+	list.MarkWritten(40, 50)
+	assert.Equal(t, 3, list.size(), "three intervals grow forward")
+
+	list.MarkWritten(50, 65)
+	assert.Equal(t, 3, list.size(), "three intervals grow backward")
+
+	list.MarkWritten(70, 80)
+	assert.Equal(t, 4, list.size(), "four intervals")
+
+	list.MarkWritten(60, 70)
+	assert.Equal(t, 3, list.size(), "three intervals merged")
+
+	list.MarkWritten(59, 71)
+	assert.Equal(t, 3, list.size(), "covered three intervals")
+
+	list.MarkWritten(5, 59)
+	assert.Equal(t, 2, list.size(), "covered two intervals")
+
+	list.MarkWritten(70, 99)
+	assert.Equal(t, 1, list.size(), "covered one intervals")
+
+}

weed/mount/page_writer/dirty_pages.go

@@ -0,0 +1,30 @@
+package page_writer
+
+type DirtyPages interface {
+	AddPage(offset int64, data []byte)
+	FlushData() error
+	ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64)
+	GetStorageOptions() (collection, replication string)
+	Destroy()
+	LockForRead(startOffset, stopOffset int64)
+	UnlockForRead(startOffset, stopOffset int64)
+}
+
+func max(x, y int64) int64 {
+	if x > y {
+		return x
+	}
+	return y
+}
+func min(x, y int64) int64 {
+	if x < y {
+		return x
+	}
+	return y
+}
+func minInt(x, y int) int {
+	if x < y {
+		return x
+	}
+	return y
+}

weed/mount/page_writer/page_chunk.go

@@ -0,0 +1,16 @@
+package page_writer
+
+import (
+	"io"
+)
+
+type SaveToStorageFunc func(reader io.Reader, offset int64, size int64, cleanupFn func())
+
+type PageChunk interface {
+	FreeResource()
+	WriteDataAt(src []byte, offset int64) (n int)
+	ReadDataAt(p []byte, off int64) (maxStop int64)
+	IsComplete() bool
+	WrittenSize() int64
+	SaveContent(saveFn SaveToStorageFunc)
+}

weed/mount/page_writer/page_chunk_mem.go

@@ -0,0 +1,69 @@
+package page_writer
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/util/mem"
+)
+
+var (
+	_ = PageChunk(&MemChunk{})
+)
+
+type MemChunk struct {
+	buf             []byte
+	usage           *ChunkWrittenIntervalList
+	chunkSize       int64
+	logicChunkIndex LogicChunkIndex
+}
+
+func NewMemChunk(logicChunkIndex LogicChunkIndex, chunkSize int64) *MemChunk {
+	return &MemChunk{
+		logicChunkIndex: logicChunkIndex,
+		chunkSize:       chunkSize,
+		buf:             mem.Allocate(int(chunkSize)),
+		usage:           newChunkWrittenIntervalList(),
+	}
+}
+
+func (mc *MemChunk) FreeResource() {
+	mem.Free(mc.buf)
+}
+
+func (mc *MemChunk) WriteDataAt(src []byte, offset int64) (n int) {
+	innerOffset := offset % mc.chunkSize
+	n = copy(mc.buf[innerOffset:], src)
+	mc.usage.MarkWritten(innerOffset, innerOffset+int64(n))
+	return
+}
+
+func (mc *MemChunk) ReadDataAt(p []byte, off int64) (maxStop int64) {
+	memChunkBaseOffset := int64(mc.logicChunkIndex) * mc.chunkSize
+	for t := mc.usage.head.next; t != mc.usage.tail; t = t.next {
+		logicStart := max(off, int64(mc.logicChunkIndex)*mc.chunkSize+t.StartOffset)
+		logicStop := min(off+int64(len(p)), memChunkBaseOffset+t.stopOffset)
+		if logicStart < logicStop {
+			copy(p[logicStart-off:logicStop-off], mc.buf[logicStart-memChunkBaseOffset:logicStop-memChunkBaseOffset])
+			maxStop = max(maxStop, logicStop)
+		}
+	}
+	return
+}
+
+func (mc *MemChunk) IsComplete() bool {
+	return mc.usage.IsComplete(mc.chunkSize)
+}
+
+func (mc *MemChunk) WrittenSize() int64 {
+	return mc.usage.WrittenSize()
+}
+
+func (mc *MemChunk) SaveContent(saveFn SaveToStorageFunc) {
+	if saveFn == nil {
+		return
+	}
+	for t := mc.usage.head.next; t != mc.usage.tail; t = t.next {
+		reader := util.NewBytesReader(mc.buf[t.StartOffset:t.stopOffset])
+		saveFn(reader, int64(mc.logicChunkIndex)*mc.chunkSize+t.StartOffset, t.Size(), func() {
+		})
+	}
+}

weed/mount/page_writer/page_chunk_swapfile.go

@@ -0,0 +1,121 @@
+package page_writer
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/util/mem"
+	"os"
+)
+
+var (
+	_ = PageChunk(&SwapFileChunk{})
+)
+
+type ActualChunkIndex int
+
+type SwapFile struct {
+	dir                     string
+	file                    *os.File
+	logicToActualChunkIndex map[LogicChunkIndex]ActualChunkIndex
+	chunkSize               int64
+}
+
+type SwapFileChunk struct {
+	swapfile         *SwapFile
+	usage            *ChunkWrittenIntervalList
+	logicChunkIndex  LogicChunkIndex
+	actualChunkIndex ActualChunkIndex
+}
+
+func NewSwapFile(dir string, chunkSize int64) *SwapFile {
+	return &SwapFile{
+		dir:                     dir,
+		file:                    nil,
+		logicToActualChunkIndex: make(map[LogicChunkIndex]ActualChunkIndex),
+		chunkSize:               chunkSize,
+	}
+}
+func (sf *SwapFile) FreeResource() {
+	if sf.file != nil {
+		sf.file.Close()
+		os.Remove(sf.file.Name())
+	}
+}
+
+func (sf *SwapFile) NewTempFileChunk(logicChunkIndex LogicChunkIndex) (tc *SwapFileChunk) {
+	if sf.file == nil {
+		var err error
+		sf.file, err = os.CreateTemp(sf.dir, "")
+		if err != nil {
+			glog.Errorf("create swap file: %v", err)
+			return nil
+		}
+	}
+	actualChunkIndex, found := sf.logicToActualChunkIndex[logicChunkIndex]
+	if !found {
+		actualChunkIndex = ActualChunkIndex(len(sf.logicToActualChunkIndex))
+		sf.logicToActualChunkIndex[logicChunkIndex] = actualChunkIndex
+	}
+
+	return &SwapFileChunk{
+		swapfile:         sf,
+		usage:            newChunkWrittenIntervalList(),
+		logicChunkIndex:  logicChunkIndex,
+		actualChunkIndex: actualChunkIndex,
+	}
+}
+
+func (sc *SwapFileChunk) FreeResource() {
+}
+
+func (sc *SwapFileChunk) WriteDataAt(src []byte, offset int64) (n int) {
+	innerOffset := offset % sc.swapfile.chunkSize
+	var err error
+	n, err = sc.swapfile.file.WriteAt(src, int64(sc.actualChunkIndex)*sc.swapfile.chunkSize+innerOffset)
+	if err == nil {
+		sc.usage.MarkWritten(innerOffset, innerOffset+int64(n))
+	} else {
+		glog.Errorf("failed to write swap file %s: %v", sc.swapfile.file.Name(), err)
+	}
+	return
+}
+
+func (sc *SwapFileChunk) ReadDataAt(p []byte, off int64) (maxStop int64) {
+	chunkStartOffset := int64(sc.logicChunkIndex) * sc.swapfile.chunkSize
+	for t := sc.usage.head.next; t != sc.usage.tail; t = t.next {
+		logicStart := max(off, chunkStartOffset+t.StartOffset)
+		logicStop := min(off+int64(len(p)), chunkStartOffset+t.stopOffset)
+		if logicStart < logicStop {
+			actualStart := logicStart - chunkStartOffset + int64(sc.actualChunkIndex)*sc.swapfile.chunkSize
+			if _, err := sc.swapfile.file.ReadAt(p[logicStart-off:logicStop-off], actualStart); err != nil {
+				glog.Errorf("failed to reading swap file %s: %v", sc.swapfile.file.Name(), err)
+				break
+			}
+			maxStop = max(maxStop, logicStop)
+		}
+	}
+	return
+}
+
+func (sc *SwapFileChunk) IsComplete() bool {
+	return sc.usage.IsComplete(sc.swapfile.chunkSize)
+}
+
+func (sc *SwapFileChunk) WrittenSize() int64 {
+	return sc.usage.WrittenSize()
+}
+
+func (sc *SwapFileChunk) SaveContent(saveFn SaveToStorageFunc) {
+	if saveFn == nil {
+		return
+	}
+	for t := sc.usage.head.next; t != sc.usage.tail; t = t.next {
+		data := mem.Allocate(int(t.Size()))
+		sc.swapfile.file.ReadAt(data, t.StartOffset+int64(sc.actualChunkIndex)*sc.swapfile.chunkSize)
+		reader := util.NewBytesReader(data)
+		saveFn(reader, int64(sc.logicChunkIndex)*sc.swapfile.chunkSize+t.StartOffset, t.Size(), func() {
+		})
+		mem.Free(data)
+	}
+	sc.usage = newChunkWrittenIntervalList()
+}

weed/mount/page_writer/upload_pipeline.go

@@ -0,0 +1,182 @@
+package page_writer
+
+import (
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+type LogicChunkIndex int
+
+type UploadPipeline struct {
+	filepath             util.FullPath
+	ChunkSize            int64
+	writableChunks       map[LogicChunkIndex]PageChunk
+	writableChunksLock   sync.Mutex
+	sealedChunks         map[LogicChunkIndex]*SealedChunk
+	sealedChunksLock     sync.Mutex
+	uploaders            *util.LimitedConcurrentExecutor
+	uploaderCount        int32
+	uploaderCountCond    *sync.Cond
+	saveToStorageFn      SaveToStorageFunc
+	activeReadChunks     map[LogicChunkIndex]int
+	activeReadChunksLock sync.Mutex
+	bufferChunkLimit     int
+}
+
+type SealedChunk struct {
+	chunk            PageChunk
+	referenceCounter int // track uploading or reading processes
+}
+
+func (sc *SealedChunk) FreeReference(messageOnFree string) {
+	sc.referenceCounter--
+	if sc.referenceCounter == 0 {
+		glog.V(4).Infof("Free sealed chunk: %s", messageOnFree)
+		sc.chunk.FreeResource()
+	}
+}
+
+func NewUploadPipeline(writers *util.LimitedConcurrentExecutor, chunkSize int64, saveToStorageFn SaveToStorageFunc, bufferChunkLimit int) *UploadPipeline {
+	return &UploadPipeline{
+		ChunkSize:         chunkSize,
+		writableChunks:    make(map[LogicChunkIndex]PageChunk),
+		sealedChunks:      make(map[LogicChunkIndex]*SealedChunk),
+		uploaders:         writers,
+		uploaderCountCond: sync.NewCond(&sync.Mutex{}),
+		saveToStorageFn:   saveToStorageFn,
+		activeReadChunks:  make(map[LogicChunkIndex]int),
+		bufferChunkLimit:  bufferChunkLimit,
+	}
+}
+
+func (up *UploadPipeline) SaveDataAt(p []byte, off int64) (n int) {
+	up.writableChunksLock.Lock()
+	defer up.writableChunksLock.Unlock()
+
+	logicChunkIndex := LogicChunkIndex(off / up.ChunkSize)
+
+	memChunk, found := up.writableChunks[logicChunkIndex]
+	if !found {
+		if len(up.writableChunks) < up.bufferChunkLimit {
+			memChunk = NewMemChunk(logicChunkIndex, up.ChunkSize)
+		} else {
+			fullestChunkIndex, fullness := LogicChunkIndex(-1), int64(0)
+			for lci, mc := range up.writableChunks {
+				chunkFullness := mc.WrittenSize()
+				if fullness < chunkFullness {
+					fullestChunkIndex = lci
+					fullness = chunkFullness
+				}
+			}
+			up.moveToSealed(up.writableChunks[fullestChunkIndex], fullestChunkIndex)
+			delete(up.writableChunks, fullestChunkIndex)
+			fmt.Printf("flush chunk %d with %d bytes written", logicChunkIndex, fullness)
+			memChunk = NewMemChunk(logicChunkIndex, up.ChunkSize)
+		}
+		up.writableChunks[logicChunkIndex] = memChunk
+	}
+	n = memChunk.WriteDataAt(p, off)
+	up.maybeMoveToSealed(memChunk, logicChunkIndex)
+
+	return
+}
+
+func (up *UploadPipeline) MaybeReadDataAt(p []byte, off int64) (maxStop int64) {
+	logicChunkIndex := LogicChunkIndex(off / up.ChunkSize)
+
+	// read from sealed chunks first
+	up.sealedChunksLock.Lock()
+	sealedChunk, found := up.sealedChunks[logicChunkIndex]
+	if found {
+		sealedChunk.referenceCounter++
+	}
+	up.sealedChunksLock.Unlock()
+	if found {
+		maxStop = sealedChunk.chunk.ReadDataAt(p, off)
+		glog.V(4).Infof("%s read sealed memchunk [%d,%d)", up.filepath, off, maxStop)
+		sealedChunk.FreeReference(fmt.Sprintf("%s finish reading chunk %d", up.filepath, logicChunkIndex))
+	}
+
+	// read from writable chunks last
+	up.writableChunksLock.Lock()
+	defer up.writableChunksLock.Unlock()
+	writableChunk, found := up.writableChunks[logicChunkIndex]
+	if !found {
+		return
+	}
+	writableMaxStop := writableChunk.ReadDataAt(p, off)
+	glog.V(4).Infof("%s read writable memchunk [%d,%d)", up.filepath, off, writableMaxStop)
+	maxStop = max(maxStop, writableMaxStop)
+
+	return
+}
+
+func (up *UploadPipeline) FlushAll() {
+	up.writableChunksLock.Lock()
+	defer up.writableChunksLock.Unlock()
+
+	for logicChunkIndex, memChunk := range up.writableChunks {
+		up.moveToSealed(memChunk, logicChunkIndex)
+	}
+
+	up.waitForCurrentWritersToComplete()
+}
+
+func (up *UploadPipeline) maybeMoveToSealed(memChunk PageChunk, logicChunkIndex LogicChunkIndex) {
+	if memChunk.IsComplete() {
+		up.moveToSealed(memChunk, logicChunkIndex)
+	}
+}
+
+func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex LogicChunkIndex) {
+	atomic.AddInt32(&up.uploaderCount, 1)
+	glog.V(4).Infof("%s uploaderCount %d ++> %d", up.filepath, up.uploaderCount-1, up.uploaderCount)
+
+	up.sealedChunksLock.Lock()
+
+	if oldMemChunk, found := up.sealedChunks[logicChunkIndex]; found {
+		oldMemChunk.FreeReference(fmt.Sprintf("%s replace chunk %d", up.filepath, logicChunkIndex))
+	}
+	sealedChunk := &SealedChunk{
+		chunk:            memChunk,
+		referenceCounter: 1, // default 1 is for uploading process
+	}
+	up.sealedChunks[logicChunkIndex] = sealedChunk
+	delete(up.writableChunks, logicChunkIndex)
+
+	up.sealedChunksLock.Unlock()
+
+	up.uploaders.Execute(func() {
+		// first add to the file chunks
+		sealedChunk.chunk.SaveContent(up.saveToStorageFn)
+
+		// notify waiting process
+		atomic.AddInt32(&up.uploaderCount, -1)
+		glog.V(4).Infof("%s uploaderCount %d --> %d", up.filepath, up.uploaderCount+1, up.uploaderCount)
+		// Lock and Unlock are not required,
+		// but it may signal multiple times during one wakeup,
+		// and the waiting goroutine may miss some of them!
+		up.uploaderCountCond.L.Lock()
+		up.uploaderCountCond.Broadcast()
+		up.uploaderCountCond.L.Unlock()
+
+		// wait for readers
+		for up.IsLocked(logicChunkIndex) {
+			time.Sleep(59 * time.Millisecond)
+		}
+
+		// then remove from sealed chunks
+		up.sealedChunksLock.Lock()
+		defer up.sealedChunksLock.Unlock()
+		delete(up.sealedChunks, logicChunkIndex)
+		sealedChunk.FreeReference(fmt.Sprintf("%s finished uploading chunk %d", up.filepath, logicChunkIndex))
+
+	})
+}
+
+func (up *UploadPipeline) Shutdown() {
+}

weed/mount/page_writer/upload_pipeline_lock.go

@@ -0,0 +1,63 @@
+package page_writer
+
+import (
+	"sync/atomic"
+)
+
+func (up *UploadPipeline) LockForRead(startOffset, stopOffset int64) {
+	startLogicChunkIndex := LogicChunkIndex(startOffset / up.ChunkSize)
+	stopLogicChunkIndex := LogicChunkIndex(stopOffset / up.ChunkSize)
+	if stopOffset%up.ChunkSize > 0 {
+		stopLogicChunkIndex += 1
+	}
+	up.activeReadChunksLock.Lock()
+	defer up.activeReadChunksLock.Unlock()
+	for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ {
+		if count, found := up.activeReadChunks[i]; found {
+			up.activeReadChunks[i] = count + 1
+		} else {
+			up.activeReadChunks[i] = 1
+		}
+	}
+}
+
+func (up *UploadPipeline) UnlockForRead(startOffset, stopOffset int64) {
+	startLogicChunkIndex := LogicChunkIndex(startOffset / up.ChunkSize)
+	stopLogicChunkIndex := LogicChunkIndex(stopOffset / up.ChunkSize)
+	if stopOffset%up.ChunkSize > 0 {
+		stopLogicChunkIndex += 1
+	}
+	up.activeReadChunksLock.Lock()
+	defer up.activeReadChunksLock.Unlock()
+	for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ {
+		if count, found := up.activeReadChunks[i]; found {
+			if count == 1 {
+				delete(up.activeReadChunks, i)
+			} else {
+				up.activeReadChunks[i] = count - 1
+			}
+		}
+	}
+}
+
+func (up *UploadPipeline) IsLocked(logicChunkIndex LogicChunkIndex) bool {
+	up.activeReadChunksLock.Lock()
+	defer up.activeReadChunksLock.Unlock()
+	if count, found := up.activeReadChunks[logicChunkIndex]; found {
+		return count > 0
+	}
+	return false
+}
+
+func (up *UploadPipeline) waitForCurrentWritersToComplete() {
+	up.uploaderCountCond.L.Lock()
+	t := int32(100)
+	for {
+		t = atomic.LoadInt32(&up.uploaderCount)
+		if t <= 0 {
+			break
+		}
+		up.uploaderCountCond.Wait()
+	}
+	up.uploaderCountCond.L.Unlock()
+}

weed/mount/page_writer/upload_pipeline_test.go

@@ -0,0 +1,47 @@
+package page_writer
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"testing"
+)
+
+func TestUploadPipeline(t *testing.T) {
+
+	uploadPipeline := NewUploadPipeline(nil, 2*1024*1024, nil, 16)
+
+	writeRange(uploadPipeline, 0, 131072)
+	writeRange(uploadPipeline, 131072, 262144)
+	writeRange(uploadPipeline, 262144, 1025536)
+
+	confirmRange(t, uploadPipeline, 0, 1025536)
+
+	writeRange(uploadPipeline, 1025536, 1296896)
+
+	confirmRange(t, uploadPipeline, 1025536, 1296896)
+
+	writeRange(uploadPipeline, 1296896, 2162688)
+
+	confirmRange(t, uploadPipeline, 1296896, 2162688)
+
+	confirmRange(t, uploadPipeline, 1296896, 2162688)
+}
+
+// startOff and stopOff must be divided by 4
+func writeRange(uploadPipeline *UploadPipeline, startOff, stopOff int64) {
+	p := make([]byte, 4)
+	for i := startOff / 4; i < stopOff/4; i += 4 {
+		util.Uint32toBytes(p, uint32(i))
+		uploadPipeline.SaveDataAt(p, i)
+	}
+}
+
+func confirmRange(t *testing.T, uploadPipeline *UploadPipeline, startOff, stopOff int64) {
+	p := make([]byte, 4)
+	for i := startOff; i < stopOff/4; i += 4 {
+		uploadPipeline.MaybeReadDataAt(p, i)
+		x := util.BytesToUint32(p)
+		if x != uint32(i) {
+			t.Errorf("expecting %d found %d at offset [%d,%d)", i, x, i, i+4)
+		}
+	}
+}

weed/mount/unmount/unmount.go

@@ -0,0 +1,6 @@
+package unmount
+
+// Unmount tries to unmount the filesystem mounted at dir.
+func Unmount(dir string) error {
+	return unmount(dir)
+}

weed/mount/unmount/unmount_linux.go

@@ -0,0 +1,21 @@
+package unmount
+
+import (
+	"bytes"
+	"errors"
+	"os/exec"
+)
+
+func unmount(dir string) error {
+	cmd := exec.Command("fusermount", "-u", dir)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		if len(output) > 0 {
+			output = bytes.TrimRight(output, "\n")
+			msg := err.Error() + ": " + string(output)
+			err = errors.New(msg)
+		}
+		return err
+	}
+	return nil
+}

weed/mount/unmount/unmount_std.go

@@ -0,0 +1,18 @@
+//go:build !linux
+// +build !linux
+
+package unmount
+
+import (
+	"os"
+	"syscall"
+)
+
+func unmount(dir string) error {
+	err := syscall.Unmount(dir, 0)
+	if err != nil {
+		err = &os.PathError{Op: "unmount", Path: dir, Err: err}
+		return err
+	}
+	return nil
+}

weed/mount/weedfs.go

@@ -0,0 +1,187 @@
+package mount
+
+import (
+	"context"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/storage/types"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/util/chunk_cache"
+	"github.com/chrislusf/seaweedfs/weed/util/grace"
+	"github.com/chrislusf/seaweedfs/weed/wdclient"
+	"github.com/hanwen/go-fuse/v2/fuse"
+	"google.golang.org/grpc"
+	"math/rand"
+	"os"
+	"path"
+	"path/filepath"
+	"time"
+
+	"github.com/hanwen/go-fuse/v2/fs"
+)
+
+type Option struct {
+	MountDirectory     string
+	FilerAddresses     []pb.ServerAddress
+	filerIndex         int
+	GrpcDialOption     grpc.DialOption
+	FilerMountRootPath string
+	Collection         string
+	Replication        string
+	TtlSec             int32
+	DiskType           types.DiskType
+	ChunkSizeLimit     int64
+	ConcurrentWriters  int
+	CacheDir           string
+	CacheSizeMB        int64
+	DataCenter         string
+	Umask              os.FileMode
+
+	MountUid         uint32
+	MountGid         uint32
+	MountMode        os.FileMode
+	MountCtime       time.Time
+	MountMtime       time.Time
+	MountParentInode uint64
+
+	VolumeServerAccess string // how to access volume servers
+	Cipher             bool   // whether encrypt data on volume server
+	UidGidMapper       *meta_cache.UidGidMapper
+
+	uniqueCacheDir         string
+	uniqueCacheTempPageDir string
+}
+
+type WFS struct {
+	// follow https://github.com/hanwen/go-fuse/blob/master/fuse/api.go
+	fuse.RawFileSystem
+	fs.Inode
+	option            *Option
+	metaCache         *meta_cache.MetaCache
+	stats             statsCache
+	root              Directory
+	chunkCache        *chunk_cache.TieredChunkCache
+	signature         int32
+	concurrentWriters *util.LimitedConcurrentExecutor
+	inodeToPath       *InodeToPath
+	fhmap             *FileHandleToInode
+}
+
+func NewSeaweedFileSystem(option *Option) *WFS {
+	wfs := &WFS{
+		RawFileSystem: fuse.NewDefaultRawFileSystem(),
+		option:        option,
+		signature:     util.RandomInt32(),
+		inodeToPath:   NewInodeToPath(),
+		fhmap:         NewFileHandleToInode(),
+	}
+
+	wfs.root = Directory{
+		name:   "/",
+		wfs:    wfs,
+		entry:  nil,
+		parent: nil,
+	}
+
+	wfs.option.filerIndex = rand.Intn(len(option.FilerAddresses))
+	wfs.option.setupUniqueCacheDirectory()
+	if option.CacheSizeMB > 0 {
+		wfs.chunkCache = chunk_cache.NewTieredChunkCache(256, option.getUniqueCacheDir(), option.CacheSizeMB, 1024*1024)
+	}
+
+	wfs.metaCache = meta_cache.NewMetaCache(path.Join(option.getUniqueCacheDir(), "meta"), option.UidGidMapper, func(path util.FullPath) {
+		wfs.inodeToPath.MarkChildrenCached(path)
+	}, func(path util.FullPath) bool {
+		return wfs.inodeToPath.IsChildrenCached(path)
+	}, func(filePath util.FullPath, entry *filer_pb.Entry) {
+	})
+	grace.OnInterrupt(func() {
+		wfs.metaCache.Shutdown()
+	})
+
+	if wfs.option.ConcurrentWriters > 0 {
+		wfs.concurrentWriters = util.NewLimitedConcurrentExecutor(wfs.option.ConcurrentWriters)
+	}
+	return wfs
+}
+
+func (wfs *WFS) StartBackgroundTasks() {
+	startTime := time.Now()
+	go meta_cache.SubscribeMetaEvents(wfs.metaCache, wfs.signature, wfs, wfs.option.FilerMountRootPath, startTime.UnixNano())
+}
+
+func (wfs *WFS) Root() *Directory {
+	return &wfs.root
+}
+
+func (wfs *WFS) String() string {
+	return "seaweedfs"
+}
+
+func (wfs *WFS) maybeReadEntry(inode uint64) (path util.FullPath, fh *FileHandle, entry *filer_pb.Entry, status fuse.Status) {
+	path = wfs.inodeToPath.GetPath(inode)
+	var found bool
+	if fh, found = wfs.fhmap.FindFileHandle(inode); found {
+		return path, fh, fh.entry, fuse.OK
+	}
+	entry, status = wfs.maybeLoadEntry(path)
+	return
+}
+
+func (wfs *WFS) maybeLoadEntry(fullpath util.FullPath) (*filer_pb.Entry, fuse.Status) {
+
+	// glog.V(3).Infof("read entry cache miss %s", fullpath)
+	dir, name := fullpath.DirAndName()
+
+	// return a valid entry for the mount root
+	if string(fullpath) == wfs.option.FilerMountRootPath {
+		return &filer_pb.Entry{
+			Name:        name,
+			IsDirectory: true,
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime:    wfs.option.MountMtime.Unix(),
+				FileMode: uint32(wfs.option.MountMode),
+				Uid:      wfs.option.MountUid,
+				Gid:      wfs.option.MountGid,
+				Crtime:   wfs.option.MountCtime.Unix(),
+			},
+		}, fuse.OK
+	}
+
+	// read from async meta cache
+	meta_cache.EnsureVisited(wfs.metaCache, wfs, util.FullPath(dir))
+	cachedEntry, cacheErr := wfs.metaCache.FindEntry(context.Background(), fullpath)
+	if cacheErr == filer_pb.ErrNotFound {
+		return nil, fuse.ENOENT
+	}
+	return cachedEntry.ToProtoEntry(), fuse.OK
+}
+
+func (wfs *WFS) LookupFn() wdclient.LookupFileIdFunctionType {
+	if wfs.option.VolumeServerAccess == "filerProxy" {
+		return func(fileId string) (targetUrls []string, err error) {
+			return []string{"http://" + wfs.getCurrentFiler().ToHttpAddress() + "/?proxyChunkId=" + fileId}, nil
+		}
+	}
+	return filer.LookupFn(wfs)
+}
+
+func (wfs *WFS) getCurrentFiler() pb.ServerAddress {
+	return wfs.option.FilerAddresses[wfs.option.filerIndex]
+}
+
+func (option *Option) setupUniqueCacheDirectory() {
+	cacheUniqueId := util.Md5String([]byte(option.MountDirectory + string(option.FilerAddresses[0]) + option.FilerMountRootPath + util.Version()))[0:8]
+	option.uniqueCacheDir = path.Join(option.CacheDir, cacheUniqueId)
+	option.uniqueCacheTempPageDir = filepath.Join(option.uniqueCacheDir, "sw")
+	os.MkdirAll(option.uniqueCacheTempPageDir, os.FileMode(0777)&^option.Umask)
+}
+
+func (option *Option) getTempFilePageDir() string {
+	return option.uniqueCacheTempPageDir
+}
+func (option *Option) getUniqueCacheDir() string {
+	return option.uniqueCacheDir
+}

weed/mount/weedfs_attr.go

@@ -0,0 +1,208 @@
+package mount
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/hanwen/go-fuse/v2/fuse"
+	"os"
+	"syscall"
+	"time"
+)
+
+func (wfs *WFS) GetAttr(cancel <-chan struct{}, input *fuse.GetAttrIn, out *fuse.AttrOut) (code fuse.Status) {
+	if input.NodeId == 1 {
+		wfs.setRootAttr(out)
+		return fuse.OK
+	}
+
+	_, _, entry, status := wfs.maybeReadEntry(input.NodeId)
+	if status != fuse.OK {
+		return status
+	}
+	out.AttrValid = 1
+	wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry)
+
+	return fuse.OK
+}
+
+func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse.AttrOut) (code fuse.Status) {
+
+	path, fh, entry, status := wfs.maybeReadEntry(input.NodeId)
+	if status != fuse.OK {
+		return status
+	}
+
+	if size, ok := input.GetSize(); ok {
+		glog.V(4).Infof("%v setattr set size=%v chunks=%d", path, size, len(entry.Chunks))
+		if size < filer.FileSize(entry) {
+			// fmt.Printf("truncate %v \n", fullPath)
+			var chunks []*filer_pb.FileChunk
+			var truncatedChunks []*filer_pb.FileChunk
+			for _, chunk := range entry.Chunks {
+				int64Size := int64(chunk.Size)
+				if chunk.Offset+int64Size > int64(size) {
+					// this chunk is truncated
+					int64Size = int64(size) - chunk.Offset
+					if int64Size > 0 {
+						chunks = append(chunks, chunk)
+						glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk.GetFileIdString(), chunk.Size, int64Size)
+						chunk.Size = uint64(int64Size)
+					} else {
+						glog.V(4).Infof("truncated whole chunk %+v\n", chunk.GetFileIdString())
+						truncatedChunks = append(truncatedChunks, chunk)
+					}
+				}
+			}
+			// set the new chunks and reset entry cache
+			entry.Chunks = chunks
+			if fh != nil {
+				fh.entryViewCache = nil
+			}
+		}
+		entry.Attributes.Mtime = time.Now().Unix()
+		entry.Attributes.FileSize = size
+
+	}
+
+	if mode, ok := input.GetMode(); ok {
+		entry.Attributes.FileMode = uint32(mode)
+	}
+
+	if uid, ok := input.GetUID(); ok {
+		entry.Attributes.Uid = uid
+	}
+
+	if gid, ok := input.GetGID(); ok {
+		entry.Attributes.Gid = gid
+	}
+
+	if mtime, ok := input.GetMTime(); ok {
+		entry.Attributes.Mtime = mtime.Unix()
+	}
+
+	entry.Attributes.Mtime = time.Now().Unix()
+	out.AttrValid = 1
+	wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry)
+
+	if fh != nil {
+		fh.dirtyMetadata = true
+		return fuse.OK
+	}
+
+	return wfs.saveEntry(path, entry)
+
+}
+
+func (wfs *WFS) setRootAttr(out *fuse.AttrOut) {
+	now := uint64(time.Now().Unix())
+	out.AttrValid = 119
+	out.Ino = 1
+	setBlksize(&out.Attr, blockSize)
+	out.Uid = wfs.option.MountUid
+	out.Gid = wfs.option.MountGid
+	out.Mtime = now
+	out.Ctime = now
+	out.Atime = now
+	out.Mode = toSystemType(os.ModeDir) | uint32(wfs.option.MountMode)
+	out.Nlink = 1
+}
+
+func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.Entry) {
+	out.Ino = inode
+	out.Size = filer.FileSize(entry)
+	out.Blocks = (out.Size + blockSize - 1) / blockSize
+	setBlksize(out, blockSize)
+	out.Mtime = uint64(entry.Attributes.Mtime)
+	out.Ctime = uint64(entry.Attributes.Mtime)
+	out.Atime = uint64(entry.Attributes.Mtime)
+	out.Mode = toSystemMode(os.FileMode(entry.Attributes.FileMode))
+	if entry.HardLinkCounter > 0 {
+		out.Nlink = uint32(entry.HardLinkCounter)
+	} else {
+		out.Nlink = 1
+	}
+	out.Uid = entry.Attributes.Uid
+	out.Gid = entry.Attributes.Gid
+}
+
+func (wfs *WFS) setAttrByFilerEntry(out *fuse.Attr, inode uint64, entry *filer.Entry) {
+	out.Ino = inode
+	out.Size = entry.FileSize
+	out.Blocks = (out.Size + blockSize - 1) / blockSize
+	setBlksize(out, blockSize)
+	out.Atime = uint64(entry.Attr.Mtime.Unix())
+	out.Mtime = uint64(entry.Attr.Mtime.Unix())
+	out.Ctime = uint64(entry.Attr.Mtime.Unix())
+	out.Crtime_ = uint64(entry.Attr.Crtime.Unix())
+	out.Mode = toSystemMode(entry.Attr.Mode)
+	if entry.HardLinkCounter > 0 {
+		out.Nlink = uint32(entry.HardLinkCounter)
+	} else {
+		out.Nlink = 1
+	}
+	out.Uid = entry.Attr.Uid
+	out.Gid = entry.Attr.Gid
+}
+
+func (wfs *WFS) outputPbEntry(out *fuse.EntryOut, inode uint64, entry *filer_pb.Entry) {
+	out.NodeId = inode
+	out.Generation = 1
+	out.EntryValid = 1
+	out.AttrValid = 1
+	wfs.setAttrByPbEntry(&out.Attr, inode, entry)
+}
+
+func (wfs *WFS) outputFilerEntry(out *fuse.EntryOut, inode uint64, entry *filer.Entry) {
+	out.NodeId = inode
+	out.Generation = 1
+	out.EntryValid = 1
+	out.AttrValid = 1
+	wfs.setAttrByFilerEntry(&out.Attr, inode, entry)
+}
+
+func toSystemMode(mode os.FileMode) uint32 {
+	return toSystemType(mode) | uint32(mode)
+}
+
+func toSystemType(mode os.FileMode) uint32 {
+	switch mode & os.ModeType {
+	case os.ModeDir:
+		return syscall.S_IFDIR
+	case os.ModeSymlink:
+		return syscall.S_IFLNK
+	case os.ModeNamedPipe:
+		return syscall.S_IFIFO
+	case os.ModeSocket:
+		return syscall.S_IFSOCK
+	case os.ModeDevice:
+		return syscall.S_IFBLK
+	case os.ModeCharDevice:
+		return syscall.S_IFCHR
+	default:
+		return syscall.S_IFREG
+	}
+}
+
+func toFileType(mode uint32) os.FileMode {
+	switch mode & (syscall.S_IFMT & 0xffff) {
+	case syscall.S_IFDIR:
+		return os.ModeDir
+	case syscall.S_IFLNK:
+		return os.ModeSymlink
+	case syscall.S_IFIFO:
+		return os.ModeNamedPipe
+	case syscall.S_IFSOCK:
+		return os.ModeSocket
+	case syscall.S_IFBLK:
+		return os.ModeDevice
+	case syscall.S_IFCHR:
+		return os.ModeCharDevice
+	default:
+		return 0
+	}
+}
+
+func toFileMode(mode uint32) os.FileMode {
+	return toFileType(mode) | os.FileMode(mode&07777)
+}

weed/mount/weedfs_attr_darwin.go

@@ -0,0 +1,8 @@
+package mount
+
+import (
+	"github.com/hanwen/go-fuse/v2/fuse"
+)
+
+func setBlksize(out *fuse.Attr, size uint32) {
+}

weed/mount/weedfs_attr_linux.go

@@ -0,0 +1,9 @@
+package mount
+
+import (
+	"github.com/hanwen/go-fuse/v2/fuse"
+)
+
+func setBlksize(out *fuse.Attr, size uint32) {
+	out.Blksize = size
+}

weed/mount/weedfs_dir_lookup.go

@@ -0,0 +1,59 @@
+package mount
+
+import (
+	"context"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/hanwen/go-fuse/v2/fuse"
+)
+
+// Lookup is called by the kernel when the VFS wants to know
+// about a file inside a directory. Many lookup calls can
+// occur in parallel, but only one call happens for each (dir,
+// name) pair.
+
+func (wfs *WFS) Lookup(cancel <-chan struct{}, header *fuse.InHeader, name string, out *fuse.EntryOut) (code fuse.Status) {
+
+	if s := checkName(name); s != fuse.OK {
+		return s
+	}
+
+	dirPath := wfs.inodeToPath.GetPath(header.NodeId)
+
+	fullFilePath := dirPath.Child(name)
+
+	visitErr := meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath)
+	if visitErr != nil {
+		glog.Errorf("dir Lookup %s: %v", dirPath, visitErr)
+		return fuse.EIO
+	}
+	localEntry, cacheErr := wfs.metaCache.FindEntry(context.Background(), fullFilePath)
+	if cacheErr == filer_pb.ErrNotFound {
+		return fuse.ENOENT
+	}
+
+	if localEntry == nil {
+		// glog.V(3).Infof("dir Lookup cache miss %s", fullFilePath)
+		entry, err := filer_pb.GetEntry(wfs, fullFilePath)
+		if err != nil {
+			glog.V(1).Infof("dir GetEntry %s: %v", fullFilePath, err)
+			return fuse.ENOENT
+		}
+		localEntry = filer.FromPbEntry(string(dirPath), entry)
+	} else {
+		glog.V(4).Infof("dir Lookup cache hit %s", fullFilePath)
+	}
+
+	if localEntry == nil {
+		return fuse.ENOENT
+	}
+
+	inode := wfs.inodeToPath.Lookup(fullFilePath, localEntry.IsDirectory())
+
+	wfs.outputFilerEntry(out, inode, localEntry)
+
+	return fuse.OK
+
+}

weed/mount/weedfs_dir_mkrm.go

@@ -0,0 +1,111 @@
+package mount
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/hanwen/go-fuse/v2/fuse"
+	"os"
+	"strings"
+	"syscall"
+	"time"
+)
+
+/** Create a directory
+ *
+ * Note that the mode argument may not have the type specification
+ * bits set, i.e. S_ISDIR(mode) can be false.  To obtain the
+ * correct directory type bits use  mode|S_IFDIR
+ * */
+func (wfs *WFS) Mkdir(cancel <-chan struct{}, in *fuse.MkdirIn, name string, out *fuse.EntryOut) (code fuse.Status) {
+
+	if s := checkName(name); s != fuse.OK {
+		return s
+	}
+
+	newEntry := &filer_pb.Entry{
+		Name:        name,
+		IsDirectory: true,
+		Attributes: &filer_pb.FuseAttributes{
+			Mtime:    time.Now().Unix(),
+			Crtime:   time.Now().Unix(),
+			FileMode: uint32(os.ModeDir) | in.Mode&^uint32(wfs.option.Umask),
+			Uid:      in.Uid,
+			Gid:      in.Gid,
+		},
+	}
+
+	dirFullPath := wfs.inodeToPath.GetPath(in.NodeId)
+
+	entryFullPath := dirFullPath.Child(name)
+
+	err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		wfs.mapPbIdFromLocalToFiler(newEntry)
+		defer wfs.mapPbIdFromFilerToLocal(newEntry)
+
+		request := &filer_pb.CreateEntryRequest{
+			Directory:  string(dirFullPath),
+			Entry:      newEntry,
+			Signatures: []int32{wfs.signature},
+		}
+
+		glog.V(1).Infof("mkdir: %v", request)
+		if err := filer_pb.CreateEntry(client, request); err != nil {
+			glog.V(0).Infof("mkdir %s: %v", entryFullPath, err)
+			return err
+		}
+
+		if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil {
+			return fmt.Errorf("local mkdir dir %s: %v", entryFullPath, err)
+		}
+
+		return nil
+	})
+
+	glog.V(0).Infof("mkdir %s: %v", entryFullPath, err)
+
+	if err != nil {
+		return fuse.EIO
+	}
+
+	inode := wfs.inodeToPath.Lookup(entryFullPath, true)
+
+	wfs.outputPbEntry(out, inode, newEntry)
+
+	return fuse.OK
+
+}
+
+/** Remove a directory */
+func (wfs *WFS) Rmdir(cancel <-chan struct{}, header *fuse.InHeader, name string) (code fuse.Status) {
+
+	if name == "." {
+		return fuse.Status(syscall.EINVAL)
+	}
+	if name == ".." {
+		return fuse.Status(syscall.ENOTEMPTY)
+	}
+
+	dirFullPath := wfs.inodeToPath.GetPath(header.NodeId)
+	entryFullPath := dirFullPath.Child(name)
+
+	glog.V(3).Infof("remove directory: %v", entryFullPath)
+	ignoreRecursiveErr := true // ignore recursion error since the OS should manage it
+	err := filer_pb.Remove(wfs, string(dirFullPath), name, true, true, ignoreRecursiveErr, false, []int32{wfs.signature})
+	if err != nil {
+		glog.V(0).Infof("remove %s: %v", entryFullPath, err)
+		if strings.Contains(err.Error(), filer.MsgFailDelNonEmptyFolder) {
+			return fuse.Status(syscall.ENOTEMPTY)
+		}
+		return fuse.ENOENT
+	}
+
+	wfs.metaCache.DeleteEntry(context.Background(), entryFullPath)
+	wfs.inodeToPath.RemovePath(entryFullPath)
+
+	return fuse.OK
+
+}

weed/mount/weedfs_dir_read.go

@@ -0,0 +1,132 @@
+package mount
+
+import (
+	"context"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/hanwen/go-fuse/v2/fuse"
+	"math"
+	"os"
+)
+
+// Directory handling
+
+/** Open directory
+ *
+ * Unless the 'default_permissions' mount option is given,
+ * this method should check if opendir is permitted for this
+ * directory. Optionally opendir may also return an arbitrary
+ * filehandle in the fuse_file_info structure, which will be
+ * passed to readdir, releasedir and fsyncdir.
+ */
+func (wfs *WFS) OpenDir(cancel <-chan struct{}, input *fuse.OpenIn, out *fuse.OpenOut) (code fuse.Status) {
+	if !wfs.inodeToPath.HasInode(input.NodeId) {
+		return fuse.ENOENT
+	}
+	return fuse.OK
+}
+
+/** Release directory
+ *
+ * If the directory has been removed after the call to opendir, the
+ * path parameter will be NULL.
+ */
+func (wfs *WFS) ReleaseDir(input *fuse.ReleaseIn) {
+}
+
+/** Synchronize directory contents
+ *
+ * If the directory has been removed after the call to opendir, the
+ * path parameter will be NULL.
+ *
+ * If the datasync parameter is non-zero, then only the user data
+ * should be flushed, not the meta data
+ */
+func (wfs *WFS) FsyncDir(cancel <-chan struct{}, input *fuse.FsyncIn) (code fuse.Status) {
+	return fuse.OK
+}
+
+/** Read directory
+ *
+ * The filesystem may choose between two modes of operation:
+ *
+ * 1) The readdir implementation ignores the offset parameter, and
+ * passes zero to the filler function's offset.  The filler
+ * function will not return '1' (unless an error happens), so the
+ * whole directory is read in a single readdir operation.
+ *
+ * 2) The readdir implementation keeps track of the offsets of the
+ * directory entries.  It uses the offset parameter and always
+ * passes non-zero offset to the filler function.  When the buffer
+ * is full (or an error happens) the filler function will return
+ * '1'.
+ */
+func (wfs *WFS) ReadDir(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
+	return wfs.doReadDirectory(input, out, false)
+}
+
+func (wfs *WFS) ReadDirPlus(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
+	return wfs.doReadDirectory(input, out, true)
+}
+
+func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPlusMode bool) fuse.Status {
+	dirPath := wfs.inodeToPath.GetPath(input.NodeId)
+
+	var counter uint64
+	var dirEntry fuse.DirEntry
+	if input.Offset == 0 {
+		counter++
+		dirEntry.Ino = input.NodeId
+		dirEntry.Name = "."
+		dirEntry.Mode = toSystemMode(os.ModeDir)
+		out.AddDirEntry(dirEntry)
+
+		counter++
+		parentDir, _ := dirPath.DirAndName()
+		parentInode := wfs.inodeToPath.GetInode(util.FullPath(parentDir))
+		dirEntry.Ino = parentInode
+		dirEntry.Name = ".."
+		dirEntry.Mode = toSystemMode(os.ModeDir)
+		out.AddDirEntry(dirEntry)
+
+	}
+
+	processEachEntryFn := func(entry *filer.Entry, isLast bool) bool {
+		counter++
+		if counter <= input.Offset {
+			return true
+		}
+		dirEntry.Name = entry.Name()
+		inode := wfs.inodeToPath.GetInode(dirPath.Child(dirEntry.Name))
+		dirEntry.Ino = inode
+		dirEntry.Mode = toSystemMode(entry.Mode)
+		if !isPlusMode {
+			if !out.AddDirEntry(dirEntry) {
+				return false
+			}
+		} else {
+			entryOut := out.AddDirLookupEntry(dirEntry)
+			if entryOut == nil {
+				return false
+			}
+			wfs.outputFilerEntry(entryOut, inode, entry)
+		}
+		return true
+	}
+
+	if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath); err != nil {
+		glog.Errorf("dir ReadDirAll %s: %v", dirPath, err)
+		return fuse.EIO
+	}
+	listErr := wfs.metaCache.ListDirectoryEntries(context.Background(), dirPath, "", false, int64(math.MaxInt32), func(entry *filer.Entry) bool {
+		return processEachEntryFn(entry, false)
+	})
+	if listErr != nil {
+		glog.Errorf("list meta cache: %v", listErr)
+		return fuse.EIO
+	}
+
+	return fuse.OK
+}

weed/mount/weedfs_file_io.go

@@ -0,0 +1,98 @@
+package mount
+
+import (
+	"github.com/hanwen/go-fuse/v2/fuse"
+)
+
+/**
+	 * Open a file
+	 *
+	 * Open flags are available in fi->flags. The following rules
+	 * apply.
+	 *
+	 *  - Creation (O_CREAT, O_EXCL, O_NOCTTY) flags will be
+	 *    filtered out / handled by the kernel.
+	 *
+	 *  - Access modes (O_RDONLY, O_WRONLY, O_RDWR) should be used
+	 *    by the filesystem to check if the operation is
+	 *    permitted.  If the ``-o default_permissions`` mount
+	 *    option is given, this check is already done by the
+	 *    kernel before calling open() and may thus be omitted by
+	 *    the filesystem.
+	 *
+	 *  - When writeback caching is enabled, the kernel may send
+	 *    read requests even for files opened with O_WRONLY. The
+	 *    filesystem should be prepared to handle this.
+	 *
+	 *  - When writeback caching is disabled, the filesystem is
+	 *    expected to properly handle the O_APPEND flag and ensure
+	 *    that each write is appending to the end of the file.
+	 *
+         *  - When writeback caching is enabled, the kernel will
+	 *    handle O_APPEND. However, unless all changes to the file
+	 *    come through the kernel this will not work reliably. The
+	 *    filesystem should thus either ignore the O_APPEND flag
+	 *    (and let the kernel handle it), or return an error
+	 *    (indicating that reliably O_APPEND is not available).
+	 *
+	 * Filesystem may store an arbitrary file handle (pointer,
+	 * index, etc) in fi->fh, and use this in other all other file
+	 * operations (read, write, flush, release, fsync).
+	 *
+	 * Filesystem may also implement stateless file I/O and not store
+	 * anything in fi->fh.
+	 *
+	 * There are also some flags (direct_io, keep_cache) which the
+	 * filesystem may set in fi, to change the way the file is opened.
+	 * See fuse_file_info structure in <fuse_common.h> for more details.
+	 *
+	 * If this request is answered with an error code of ENOSYS
+	 * and FUSE_CAP_NO_OPEN_SUPPORT is set in
+	 * `fuse_conn_info.capable`, this is treated as success and
+	 * future calls to open and release will also succeed without being
+	 * sent to the filesystem process.
+	 *
+	 * Valid replies:
+	 *   fuse_reply_open
+	 *   fuse_reply_err
+	 *
+	 * @param req request handle
+	 * @param ino the inode number
+	 * @param fi file information
+*/
+func (wfs *WFS) Open(cancel <-chan struct{}, in *fuse.OpenIn, out *fuse.OpenOut) (status fuse.Status) {
+	fileHandle, code := wfs.AcquireHandle(in.NodeId, in.Uid, in.Gid)
+	if code == fuse.OK {
+		out.Fh = uint64(fileHandle.fh)
+	}
+	return code
+}
+
+/**
+ * Release an open file
+ *
+ * Release is called when there are no more references to an open
+ * file: all file descriptors are closed and all memory mappings
+ * are unmapped.
+ *
+ * For every open call there will be exactly one release call (unless
+ * the filesystem is force-unmounted).
+ *
+ * The filesystem may reply with an error, but error values are
+ * not returned to close() or munmap() which triggered the
+ * release.
+ *
+ * fi->fh will contain the value set by the open method, or will
+ * be undefined if the open method didn't set any value.
+ * fi->flags will contain the same flags as for open.
+ *
+ * Valid replies:
+ *   fuse_reply_err
+ *
+ * @param req request handle
+ * @param ino the inode number
+ * @param fi file information
+ */
+func (wfs *WFS) Release(cancel <-chan struct{}, in *fuse.ReleaseIn) {
+	wfs.ReleaseHandle(FileHandleId(in.Fh))
+}

weed/mount/weedfs_file_mkrm.go

@@ -0,0 +1,130 @@
+package mount
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/hanwen/go-fuse/v2/fuse"
+	"time"
+)
+
+/**
+ * Create and open a file
+ *
+ * If the file does not exist, first create it with the specified
+ * mode, and then open it.
+ *
+ * If this method is not implemented or under Linux kernel
+ * versions earlier than 2.6.15, the mknod() and open() methods
+ * will be called instead.
+ */
+func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, out *fuse.CreateOut) (code fuse.Status) {
+	// if implemented, need to use
+	// 	inode := wfs.inodeToPath.Lookup(entryFullPath)
+	// to ensure nlookup counter
+	return fuse.ENOSYS
+}
+
+/** Create a file node
+ *
+ * This is called for creation of all non-directory, non-symlink
+ * nodes.  If the filesystem defines a create() method, then for
+ * regular files that will be called instead.
+ */
+func (wfs *WFS) Mknod(cancel <-chan struct{}, in *fuse.MknodIn, name string, out *fuse.EntryOut) (code fuse.Status) {
+
+	if s := checkName(name); s != fuse.OK {
+		return s
+	}
+
+	newEntry := &filer_pb.Entry{
+		Name:        name,
+		IsDirectory: false,
+		Attributes: &filer_pb.FuseAttributes{
+			Mtime:       time.Now().Unix(),
+			Crtime:      time.Now().Unix(),
+			FileMode:    uint32(toFileMode(in.Mode) &^ wfs.option.Umask),
+			Uid:         in.Uid,
+			Gid:         in.Gid,
+			Collection:  wfs.option.Collection,
+			Replication: wfs.option.Replication,
+			TtlSec:      wfs.option.TtlSec,
+		},
+	}
+
+	dirFullPath := wfs.inodeToPath.GetPath(in.NodeId)
+
+	entryFullPath := dirFullPath.Child(name)
+
+	err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		wfs.mapPbIdFromLocalToFiler(newEntry)
+		defer wfs.mapPbIdFromFilerToLocal(newEntry)
+
+		request := &filer_pb.CreateEntryRequest{
+			Directory:  string(dirFullPath),
+			Entry:      newEntry,
+			Signatures: []int32{wfs.signature},
+		}
+
+		glog.V(1).Infof("mknod: %v", request)
+		if err := filer_pb.CreateEntry(client, request); err != nil {
+			glog.V(0).Infof("mknod %s: %v", entryFullPath, err)
+			return err
+		}
+
+		if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil {
+			return fmt.Errorf("local mknod %s: %v", entryFullPath, err)
+		}
+
+		return nil
+	})
+
+	glog.V(0).Infof("mknod %s: %v", entryFullPath, err)
+
+	if err != nil {
+		return fuse.EIO
+	}
+
+	inode := wfs.inodeToPath.Lookup(entryFullPath, false)
+
+	wfs.outputPbEntry(out, inode, newEntry)
+
+	return fuse.OK
+
+}
+
+/** Remove a file */
+func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name string) (code fuse.Status) {
+
+	dirFullPath := wfs.inodeToPath.GetPath(header.NodeId)
+	entryFullPath := dirFullPath.Child(name)
+
+	entry, status := wfs.maybeLoadEntry(entryFullPath)
+	if status != fuse.OK {
+		return status
+	}
+
+	// first, ensure the filer store can correctly delete
+	glog.V(3).Infof("remove file: %v", entryFullPath)
+	isDeleteData := entry != nil && entry.HardLinkCounter <= 1
+	err := filer_pb.Remove(wfs, string(dirFullPath), name, isDeleteData, false, false, false, []int32{wfs.signature})
+	if err != nil {
+		glog.V(0).Infof("remove %s: %v", entryFullPath, err)
+		return fuse.ENOENT
+	}
+
+	// then, delete meta cache
+	if err = wfs.metaCache.DeleteEntry(context.Background(), entryFullPath); err != nil {
+		glog.V(3).Infof("local DeleteEntry %s: %v", entryFullPath, err)
+		return fuse.EIO
+	}
+
+	wfs.metaCache.DeleteEntry(context.Background(), entryFullPath)
+	wfs.inodeToPath.RemovePath(entryFullPath)
+
+	return fuse.OK
+
+}

weed/mount/weedfs_file_read.go

@@ -0,0 +1,59 @@
+package mount
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/hanwen/go-fuse/v2/fuse"
+	"io"
+)
+
+/**
+ * Read data
+ *
+ * Read should send exactly the number of bytes requested except
+ * on EOF or error, otherwise the rest of the data will be
+ * substituted with zeroes.  An exception to this is when the file
+ * has been opened in 'direct_io' mode, in which case the return
+ * value of the read system call will reflect the return value of
+ * this operation.
+ *
+ * fi->fh will contain the value set by the open method, or will
+ * be undefined if the open method didn't set any value.
+ *
+ * Valid replies:
+ *   fuse_reply_buf
+ *   fuse_reply_iov
+ *   fuse_reply_data
+ *   fuse_reply_err
+ *
+ * @param req request handle
+ * @param ino the inode number
+ * @param size number of bytes to read
+ * @param off offset to read from
+ * @param fi file information
+ */
+func (wfs *WFS) Read(cancel <-chan struct{}, in *fuse.ReadIn, buff []byte) (fuse.ReadResult, fuse.Status) {
+	fh := wfs.GetHandle(FileHandleId(in.Fh))
+	if fh == nil {
+		return nil, fuse.ENOENT
+	}
+
+	offset := int64(in.Offset)
+	fh.lockForRead(offset, len(buff))
+	defer fh.unlockForRead(offset, len(buff))
+
+	totalRead, err := fh.readFromChunks(buff, offset)
+	if err == nil || err == io.EOF {
+		maxStop := fh.readFromDirtyPages(buff, offset)
+		totalRead = max(maxStop-offset, totalRead)
+	}
+	if err == io.EOF {
+		err = nil
+	}
+
+	if err != nil {
+		glog.Warningf("file handle read %s %d: %v", fh.FullPath(), totalRead, err)
+		return nil, fuse.EIO
+	}
+
+	return fuse.ReadResultData(buff[:totalRead]), fuse.OK
+}

weed/mount/weedfs_file_sync.go

@@ -0,0 +1,180 @@
+package mount
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/hanwen/go-fuse/v2/fuse"
+	"os"
+	"time"
+)
+
+/**
+ * Flush method
+ *
+ * This is called on each close() of the opened file.
+ *
+ * Since file descriptors can be duplicated (dup, dup2, fork), for
+ * one open call there may be many flush calls.
+ *
+ * Filesystems shouldn't assume that flush will always be called
+ * after some writes, or that if will be called at all.
+ *
+ * fi->fh will contain the value set by the open method, or will
+ * be undefined if the open method didn't set any value.
+ *
+ * NOTE: the name of the method is misleading, since (unlike
+ * fsync) the filesystem is not forced to flush pending writes.
+ * One reason to flush data is if the filesystem wants to return
+ * write errors during close.  However, such use is non-portable
+ * because POSIX does not require [close] to wait for delayed I/O to
+ * complete.
+ *
+ * If the filesystem supports file locking operations (setlk,
+ * getlk) it should remove all locks belonging to 'fi->owner'.
+ *
+ * If this request is answered with an error code of ENOSYS,
+ * this is treated as success and future calls to flush() will
+ * succeed automatically without being send to the filesystem
+ * process.
+ *
+ * Valid replies:
+ *   fuse_reply_err
+ *
+ * @param req request handle
+ * @param ino the inode number
+ * @param fi file information
+ *
+ * [close]: http://pubs.opengroup.org/onlinepubs/9699919799/functions/close.html
+ */
+func (wfs *WFS) Flush(cancel <-chan struct{}, in *fuse.FlushIn) fuse.Status {
+	fh := wfs.GetHandle(FileHandleId(in.Fh))
+	if fh == nil {
+		return fuse.ENOENT
+	}
+
+	fh.Lock()
+	defer fh.Unlock()
+
+	return wfs.doFlush(fh, in.Uid, in.Gid)
+}
+
+/**
+ * Synchronize file contents
+ *
+ * If the datasync parameter is non-zero, then only the user data
+ * should be flushed, not the meta data.
+ *
+ * If this request is answered with an error code of ENOSYS,
+ * this is treated as success and future calls to fsync() will
+ * succeed automatically without being send to the filesystem
+ * process.
+ *
+ * Valid replies:
+ *   fuse_reply_err
+ *
+ * @param req request handle
+ * @param ino the inode number
+ * @param datasync flag indicating if only data should be flushed
+ * @param fi file information
+ */
+func (wfs *WFS) Fsync(cancel <-chan struct{}, in *fuse.FsyncIn) (code fuse.Status) {
+
+	fh := wfs.GetHandle(FileHandleId(in.Fh))
+	if fh == nil {
+		return fuse.ENOENT
+	}
+
+	fh.Lock()
+	defer fh.Unlock()
+
+	return wfs.doFlush(fh, in.Uid, in.Gid)
+
+}
+
+func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32) fuse.Status {
+	// flush works at fh level
+	fileFullPath := fh.FullPath()
+	dir, _ := fileFullPath.DirAndName()
+	// send the data to the OS
+	glog.V(4).Infof("doFlush %s fh %d", fileFullPath, fh.handle)
+
+	if err := fh.dirtyPages.FlushData(); err != nil {
+		glog.Errorf("%v doFlush: %v", fileFullPath, err)
+		return fuse.EIO
+	}
+
+	if !fh.dirtyMetadata {
+		return fuse.OK
+	}
+
+	err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		entry := fh.entry
+		if entry == nil {
+			return nil
+		}
+
+		if entry.Attributes != nil {
+			entry.Attributes.Mime = fh.contentType
+			if entry.Attributes.Uid == 0 {
+				entry.Attributes.Uid = uid
+			}
+			if entry.Attributes.Gid == 0 {
+				entry.Attributes.Gid = gid
+			}
+			if entry.Attributes.Crtime == 0 {
+				entry.Attributes.Crtime = time.Now().Unix()
+			}
+			entry.Attributes.Mtime = time.Now().Unix()
+			entry.Attributes.FileMode = uint32(os.FileMode(entry.Attributes.FileMode) &^ wfs.option.Umask)
+			entry.Attributes.Collection, entry.Attributes.Replication = fh.dirtyPages.GetStorageOptions()
+		}
+
+		request := &filer_pb.CreateEntryRequest{
+			Directory:  string(dir),
+			Entry:      entry,
+			Signatures: []int32{wfs.signature},
+		}
+
+		glog.V(4).Infof("%s set chunks: %v", fileFullPath, len(entry.Chunks))
+		for i, chunk := range entry.Chunks {
+			glog.V(4).Infof("%s chunks %d: %v [%d,%d)", fileFullPath, i, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size))
+		}
+
+		manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(entry.Chunks)
+
+		chunks, _ := filer.CompactFileChunks(wfs.LookupFn(), nonManifestChunks)
+		chunks, manifestErr := filer.MaybeManifestize(wfs.saveDataAsChunk(fileFullPath), chunks)
+		if manifestErr != nil {
+			// not good, but should be ok
+			glog.V(0).Infof("MaybeManifestize: %v", manifestErr)
+		}
+		entry.Chunks = append(chunks, manifestChunks...)
+
+		wfs.mapPbIdFromLocalToFiler(request.Entry)
+		defer wfs.mapPbIdFromFilerToLocal(request.Entry)
+
+		if err := filer_pb.CreateEntry(client, request); err != nil {
+			glog.Errorf("fh flush create %s: %v", fileFullPath, err)
+			return fmt.Errorf("fh flush create %s: %v", fileFullPath, err)
+		}
+
+		wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry))
+
+		return nil
+	})
+
+	if err == nil {
+		fh.dirtyMetadata = false
+	}
+
+	if err != nil {
+		glog.Errorf("%v fh %d flush: %v", fileFullPath, fh.handle, err)
+		return fuse.EIO
+	}
+
+	return fuse.OK
+}

weed/mount/weedfs_file_write.go

@@ -0,0 +1,66 @@
+package mount
+
+import (
+	"github.com/hanwen/go-fuse/v2/fuse"
+	"net/http"
+)
+
+/**
+ * Write data
+ *
+ * Write should return exactly the number of bytes requested
+ * except on error.  An exception to this is when the file has
+ * been opened in 'direct_io' mode, in which case the return value
+ * of the write system call will reflect the return value of this
+ * operation.
+ *
+ * Unless FUSE_CAP_HANDLE_KILLPRIV is disabled, this method is
+ * expected to reset the setuid and setgid bits.
+ *
+ * fi->fh will contain the value set by the open method, or will
+ * be undefined if the open method didn't set any value.
+ *
+ * Valid replies:
+ *   fuse_reply_write
+ *   fuse_reply_err
+ *
+ * @param req request handle
+ * @param ino the inode number
+ * @param buf data to write
+ * @param size number of bytes to write
+ * @param off offset to write to
+ * @param fi file information
+ */
+func (wfs *WFS) Write(cancel <-chan struct{}, in *fuse.WriteIn, data []byte) (written uint32, code fuse.Status) {
+
+	fh := wfs.GetHandle(FileHandleId(in.Fh))
+	if fh == nil {
+		return 0, fuse.ENOENT
+	}
+
+	fh.Lock()
+	defer fh.Unlock()
+
+	entry := fh.entry
+	if entry == nil {
+		return 0, fuse.OK
+	}
+
+	entry.Content = nil
+	offset := int64(in.Offset)
+	entry.Attributes.FileSize = uint64(max(offset+int64(len(data)), int64(entry.Attributes.FileSize)))
+	// glog.V(4).Infof("%v write [%d,%d) %d", fh.f.fullpath(), req.Offset, req.Offset+int64(len(req.Data)), len(req.Data))
+
+	fh.dirtyPages.AddPage(offset, data)
+
+	written = uint32(len(data))
+
+	if offset == 0 {
+		// detect mime type
+		fh.contentType = http.DetectContentType(data)
+	}
+
+	fh.dirtyMetadata = true
+
+	return written, fuse.OK
+}

weed/mount/weedfs_filehandle.go

@@ -0,0 +1,20 @@
+package mount
+
+import "github.com/hanwen/go-fuse/v2/fuse"
+
+func (wfs *WFS) AcquireHandle(inode uint64, uid, gid uint32) (fileHandle *FileHandle, code fuse.Status) {
+	_, _, entry, status := wfs.maybeReadEntry(inode)
+	if status == fuse.OK {
+		fileHandle = wfs.fhmap.AcquireFileHandle(wfs, inode, entry)
+		fileHandle.entry = entry
+	}
+	return
+}
+
+func (wfs *WFS) ReleaseHandle(handleId FileHandleId) {
+	wfs.fhmap.ReleaseByHandle(handleId)
+}
+
+func (wfs *WFS) GetHandle(handleId FileHandleId) *FileHandle {
+	return wfs.fhmap.GetFileHandle(handleId)
+}

weed/mount/weedfs_forget.go

@@ -0,0 +1,68 @@
+package mount
+
+import (
+	"context"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+// Forget is called when the kernel discards entries from its
+// dentry cache. This happens on unmount, and when the kernel
+// is short on memory. Since it is not guaranteed to occur at
+// any moment, and since there is no return value, Forget
+// should not do I/O, as there is no channel to report back
+// I/O errors.
+// from https://github.com/libfuse/libfuse/blob/master/include/fuse_lowlevel.h
+/**
+ * Forget about an inode
+ *
+ * This function is called when the kernel removes an inode
+ * from its internal caches.
+ *
+ * The inode's lookup count increases by one for every call to
+ * fuse_reply_entry and fuse_reply_create. The nlookup parameter
+ * indicates by how much the lookup count should be decreased.
+ *
+ * Inodes with a non-zero lookup count may receive request from
+ * the kernel even after calls to unlink, rmdir or (when
+ * overwriting an existing file) rename. Filesystems must handle
+ * such requests properly and it is recommended to defer removal
+ * of the inode until the lookup count reaches zero. Calls to
+ * unlink, rmdir or rename will be followed closely by forget
+ * unless the file or directory is open, in which case the
+ * kernel issues forget only after the release or releasedir
+ * calls.
+ *
+ * Note that if a file system will be exported over NFS the
+ * inodes lifetime must extend even beyond forget. See the
+ * generation field in struct fuse_entry_param above.
+ *
+ * On unmount the lookup count for all inodes implicitly drops
+ * to zero. It is not guaranteed that the file system will
+ * receive corresponding forget messages for the affected
+ * inodes.
+ *
+ * Valid replies:
+ *   fuse_reply_none
+ *
+ * @param req request handle
+ * @param ino the inode number
+ * @param nlookup the number of lookups to forget
+ */
+/*
+https://libfuse.github.io/doxygen/include_2fuse__lowlevel_8h.html
+
+int fuse_reply_entry	(	fuse_req_t 	req,
+const struct fuse_entry_param * 	e
+)
+Reply with a directory entry
+
+Possible requests: lookup, mknod, mkdir, symlink, link
+
+Side effects: increments the lookup count on success
+
+*/
+func (wfs *WFS) Forget(nodeid, nlookup uint64) {
+	wfs.inodeToPath.Forget(nodeid, nlookup, func(dir util.FullPath) {
+		wfs.metaCache.DeleteFolderChildren(context.Background(), dir)
+	})
+}

weed/mount/weedfs_link.go

@@ -0,0 +1,93 @@
+package mount
+
+import (
+	"context"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/hanwen/go-fuse/v2/fuse"
+	"time"
+)
+
+const (
+	HARD_LINK_MARKER = '\x01'
+)
+
+/** Create a hard link to a file */
+func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *fuse.EntryOut) (code fuse.Status) {
+
+	if s := checkName(name); s != fuse.OK {
+		return s
+	}
+
+	newParentPath := wfs.inodeToPath.GetPath(in.NodeId)
+	oldEntryPath := wfs.inodeToPath.GetPath(in.Oldnodeid)
+	oldParentPath, _ := oldEntryPath.DirAndName()
+
+	oldEntry, status := wfs.maybeLoadEntry(oldEntryPath)
+	if status != fuse.OK {
+		return status
+	}
+
+	// update old file to hardlink mode
+	if len(oldEntry.HardLinkId) == 0 {
+		oldEntry.HardLinkId = append(util.RandomBytes(16), HARD_LINK_MARKER)
+		oldEntry.HardLinkCounter = 1
+	}
+	oldEntry.HardLinkCounter++
+	updateOldEntryRequest := &filer_pb.UpdateEntryRequest{
+		Directory:  oldParentPath,
+		Entry:      oldEntry,
+		Signatures: []int32{wfs.signature},
+	}
+
+	// CreateLink 1.2 : update new file to hardlink mode
+	oldEntry.Attributes.Mtime = time.Now().Unix()
+	request := &filer_pb.CreateEntryRequest{
+		Directory: string(newParentPath),
+		Entry: &filer_pb.Entry{
+			Name:            name,
+			IsDirectory:     false,
+			Attributes:      oldEntry.Attributes,
+			Chunks:          oldEntry.Chunks,
+			Extended:        oldEntry.Extended,
+			HardLinkId:      oldEntry.HardLinkId,
+			HardLinkCounter: oldEntry.HardLinkCounter,
+		},
+		Signatures: []int32{wfs.signature},
+	}
+
+	// apply changes to the filer, and also apply to local metaCache
+	err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		wfs.mapPbIdFromLocalToFiler(request.Entry)
+		defer wfs.mapPbIdFromFilerToLocal(request.Entry)
+
+		if err := filer_pb.UpdateEntry(client, updateOldEntryRequest); err != nil {
+			return err
+		}
+		wfs.metaCache.UpdateEntry(context.Background(), filer.FromPbEntry(updateOldEntryRequest.Directory, updateOldEntryRequest.Entry))
+
+		if err := filer_pb.CreateEntry(client, request); err != nil {
+			return err
+		}
+
+		wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry))
+
+		return nil
+	})
+
+	newEntryPath := newParentPath.Child(name)
+
+	if err != nil {
+		glog.V(0).Infof("Link %v -> %s: %v", oldEntryPath, newEntryPath, err)
+		return fuse.EIO
+	}
+
+	inode := wfs.inodeToPath.Lookup(newEntryPath, false)
+
+	wfs.outputPbEntry(out, inode, request.Entry)
+
+	return fuse.OK
+}

weed/mount/weedfs_rename.go

@@ -0,0 +1,235 @@
+package mount
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/hanwen/go-fuse/v2/fs"
+	"github.com/hanwen/go-fuse/v2/fuse"
+	"io"
+	"strings"
+	"syscall"
+)
+
+/** Rename a file
+ *
+ * If the target exists it should be atomically replaced. If
+ * the target's inode's lookup count is non-zero, the file
+ * system is expected to postpone any removal of the inode
+ * until the lookup count reaches zero (see description of the
+ * forget function).
+ *
+ * If this request is answered with an error code of ENOSYS, this is
+ * treated as a permanent failure with error code EINVAL, i.e. all
+ * future bmap requests will fail with EINVAL without being
+ * send to the filesystem process.
+ *
+ * *flags* may be `RENAME_EXCHANGE` or `RENAME_NOREPLACE`. If
+ * RENAME_NOREPLACE is specified, the filesystem must not
+ * overwrite *newname* if it exists and return an error
+ * instead. If `RENAME_EXCHANGE` is specified, the filesystem
+ * must atomically exchange the two files, i.e. both must
+ * exist and neither may be deleted.
+ *
+ * Valid replies:
+ *   fuse_reply_err
+ *
+ * @param req request handle
+ * @param parent inode number of the old parent directory
+ * @param name old name
+ * @param newparent inode number of the new parent directory
+ * @param newname new name
+ */
+/*
+renameat2()
+       renameat2() has an additional flags argument.  A renameat2() call
+       with a zero flags argument is equivalent to renameat().
+
+       The flags argument is a bit mask consisting of zero or more of
+       the following flags:
+
+       RENAME_EXCHANGE
+              Atomically exchange oldpath and newpath.  Both pathnames
+              must exist but may be of different types (e.g., one could
+              be a non-empty directory and the other a symbolic link).
+
+       RENAME_NOREPLACE
+              Don't overwrite newpath of the rename.  Return an error if
+              newpath already exists.
+
+              RENAME_NOREPLACE can't be employed together with
+              RENAME_EXCHANGE.
+
+              RENAME_NOREPLACE requires support from the underlying
+              filesystem.  Support for various filesystems was added as
+              follows:
+
+              *  ext4 (Linux 3.15);
+
+              *  btrfs, tmpfs, and cifs (Linux 3.17);
+
+              *  xfs (Linux 4.0);
+
+              *  Support for many other filesystems was added in Linux
+                 4.9, including ext2, minix, reiserfs, jfs, vfat, and
+                 bpf.
+
+       RENAME_WHITEOUT (since Linux 3.18)
+              This operation makes sense only for overlay/union
+              filesystem implementations.
+
+              Specifying RENAME_WHITEOUT creates a "whiteout" object at
+              the source of the rename at the same time as performing
+              the rename.  The whole operation is atomic, so that if the
+              rename succeeds then the whiteout will also have been
+              created.
+
+              A "whiteout" is an object that has special meaning in
+              union/overlay filesystem constructs.  In these constructs,
+              multiple layers exist and only the top one is ever
+              modified.  A whiteout on an upper layer will effectively
+              hide a matching file in the lower layer, making it appear
+              as if the file didn't exist.
+
+              When a file that exists on the lower layer is renamed, the
+              file is first copied up (if not already on the upper
+              layer) and then renamed on the upper, read-write layer.
+              At the same time, the source file needs to be "whiteouted"
+              (so that the version of the source file in the lower layer
+              is rendered invisible).  The whole operation needs to be
+              done atomically.
+
+              When not part of a union/overlay, the whiteout appears as
+              a character device with a {0,0} device number.  (Note that
+              other union/overlay implementations may employ different
+              methods for storing whiteout entries; specifically, BSD
+              union mount employs a separate inode type, DT_WHT, which,
+              while supported by some filesystems available in Linux,
+              such as CODA and XFS, is ignored by the kernel's whiteout
+              support code, as of Linux 4.19, at least.)
+
+              RENAME_WHITEOUT requires the same privileges as creating a
+              device node (i.e., the CAP_MKNOD capability).
+
+              RENAME_WHITEOUT can't be employed together with
+              RENAME_EXCHANGE.
+
+              RENAME_WHITEOUT requires support from the underlying
+              filesystem.  Among the filesystems that support it are
+              tmpfs (since Linux 3.18), ext4 (since Linux 3.18), XFS
+              (since Linux 4.1), f2fs (since Linux 4.2), btrfs (since
+              Linux 4.7), and ubifs (since Linux 4.9).
+*/
+const (
+	RenameEmptyFlag = 0
+	RenameNoReplace = 1
+	RenameExchange  = fs.RENAME_EXCHANGE
+	RenameWhiteout  = 3
+)
+
+func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string, newName string) (code fuse.Status) {
+	if s := checkName(newName); s != fuse.OK {
+		return s
+	}
+
+	switch in.Flags {
+	case RenameEmptyFlag:
+	case RenameNoReplace:
+	case RenameExchange:
+	case RenameWhiteout:
+		return fuse.ENOTSUP
+	default:
+		return fuse.EINVAL
+	}
+
+	oldDir := wfs.inodeToPath.GetPath(in.NodeId)
+	oldPath := oldDir.Child(oldName)
+	newDir := wfs.inodeToPath.GetPath(in.Newdir)
+	newPath := newDir.Child(newName)
+
+	glog.V(4).Infof("dir Rename %s => %s", oldPath, newPath)
+
+	// update remote filer
+	err := wfs.WithFilerClient(true, func(client filer_pb.SeaweedFilerClient) error {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		request := &filer_pb.StreamRenameEntryRequest{
+			OldDirectory: string(oldDir),
+			OldName:      oldName,
+			NewDirectory: string(newDir),
+			NewName:      newName,
+			Signatures:   []int32{wfs.signature},
+		}
+
+		stream, err := client.StreamRenameEntry(ctx, request)
+		if err != nil {
+			code = fuse.EIO
+			return fmt.Errorf("dir AtomicRenameEntry %s => %s : %v", oldPath, newPath, err)
+		}
+
+		for {
+			resp, recvErr := stream.Recv()
+			if recvErr != nil {
+				if recvErr == io.EOF {
+					break
+				} else {
+					if strings.Contains(recvErr.Error(), "not empty") {
+						code = fuse.Status(syscall.ENOTEMPTY)
+					} else if strings.Contains(recvErr.Error(), "not directory") {
+						code = fuse.ENOTDIR
+					}
+					return fmt.Errorf("dir Rename %s => %s receive: %v", oldPath, newPath, recvErr)
+				}
+			}
+
+			if err = wfs.handleRenameResponse(ctx, resp); err != nil {
+				glog.V(0).Infof("dir Rename %s => %s : %v", oldPath, newPath, err)
+				return err
+			}
+
+		}
+
+		return nil
+
+	})
+	if err != nil {
+		glog.V(0).Infof("Link: %v", err)
+		return
+	}
+
+	return fuse.OK
+
+}
+
+func (wfs *WFS) handleRenameResponse(ctx context.Context, resp *filer_pb.StreamRenameEntryResponse) error {
+	// comes from filer StreamRenameEntry, can only be create or delete entry
+
+	if resp.EventNotification.NewEntry != nil {
+		// with new entry, the old entry name also exists. This is the first step to create new entry
+		newEntry := filer.FromPbEntry(resp.EventNotification.NewParentPath, resp.EventNotification.NewEntry)
+		if err := wfs.metaCache.AtomicUpdateEntryFromFiler(ctx, "", newEntry); err != nil {
+			return err
+		}
+
+		oldParent, newParent := util.FullPath(resp.Directory), util.FullPath(resp.EventNotification.NewParentPath)
+		oldName, newName := resp.EventNotification.OldEntry.Name, resp.EventNotification.NewEntry.Name
+
+		oldPath := oldParent.Child(oldName)
+		newPath := newParent.Child(newName)
+
+		wfs.inodeToPath.MovePath(oldPath, newPath)
+
+	} else if resp.EventNotification.OldEntry != nil {
+		// without new entry, only old entry name exists. This is the second step to delete old entry
+		if err := wfs.metaCache.AtomicUpdateEntryFromFiler(ctx, util.NewFullPath(resp.Directory, resp.EventNotification.OldEntry.Name), nil); err != nil {
+			return err
+		}
+	}
+
+	return nil
+
+}

weed/mount/weedfs_stats.go

@@ -0,0 +1,80 @@
+package mount
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/hanwen/go-fuse/v2/fuse"
+	"math"
+	"time"
+)
+
+const blockSize = 512
+
+type statsCache struct {
+	filer_pb.StatisticsResponse
+	lastChecked int64 // unix time in seconds
+}
+
+func (wfs *WFS) StatFs(cancel <-chan struct{}, in *fuse.InHeader, out *fuse.StatfsOut) (code fuse.Status) {
+
+	glog.V(4).Infof("reading fs stats")
+
+	if wfs.stats.lastChecked < time.Now().Unix()-20 {
+
+		err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+			request := &filer_pb.StatisticsRequest{
+				Collection:  wfs.option.Collection,
+				Replication: wfs.option.Replication,
+				Ttl:         fmt.Sprintf("%ds", wfs.option.TtlSec),
+				DiskType:    string(wfs.option.DiskType),
+			}
+
+			glog.V(4).Infof("reading filer stats: %+v", request)
+			resp, err := client.Statistics(context.Background(), request)
+			if err != nil {
+				glog.V(0).Infof("reading filer stats %v: %v", request, err)
+				return err
+			}
+			glog.V(4).Infof("read filer stats: %+v", resp)
+
+			wfs.stats.TotalSize = resp.TotalSize
+			wfs.stats.UsedSize = resp.UsedSize
+			wfs.stats.FileCount = resp.FileCount
+			wfs.stats.lastChecked = time.Now().Unix()
+
+			return nil
+		})
+		if err != nil {
+			glog.V(0).Infof("filer Statistics: %v", err)
+			return fuse.OK
+		}
+	}
+
+	totalDiskSize := wfs.stats.TotalSize
+	usedDiskSize := wfs.stats.UsedSize
+	actualFileCount := wfs.stats.FileCount
+
+	// Compute the total number of available blocks
+	out.Blocks = totalDiskSize / blockSize
+
+	// Compute the number of used blocks
+	numBlocks := uint64(usedDiskSize / blockSize)
+
+	// Report the number of free and available blocks for the block size
+	out.Bfree = out.Blocks - numBlocks
+	out.Bavail = out.Blocks - numBlocks
+	out.Bsize = uint32(blockSize)
+
+	// Report the total number of possible files in the file system (and those free)
+	out.Files = math.MaxInt64
+	out.Ffree = math.MaxInt64 - actualFileCount
+
+	// Report the maximum length of a name and the minimum fragment size
+	out.NameLen = 1024
+	out.Frsize = uint32(blockSize)
+
+	return fuse.OK
+}

weed/mount/weedfs_symlink.go

@@ -0,0 +1,78 @@
+package mount
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/hanwen/go-fuse/v2/fuse"
+	"os"
+	"time"
+)
+
+/** Create a symbolic link */
+func (wfs *WFS) Symlink(cancel <-chan struct{}, header *fuse.InHeader, target string, name string, out *fuse.EntryOut) (code fuse.Status) {
+
+	if s := checkName(name); s != fuse.OK {
+		return s
+	}
+
+	dirPath := wfs.inodeToPath.GetPath(header.NodeId)
+	entryFullPath := dirPath.Child(name)
+
+	request := &filer_pb.CreateEntryRequest{
+		Directory: string(dirPath),
+		Entry: &filer_pb.Entry{
+			Name:        name,
+			IsDirectory: false,
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime:         time.Now().Unix(),
+				Crtime:        time.Now().Unix(),
+				FileMode:      uint32((os.FileMode(0777) | os.ModeSymlink) &^ wfs.option.Umask),
+				Uid:           header.Uid,
+				Gid:           header.Gid,
+				SymlinkTarget: target,
+			},
+		},
+		Signatures: []int32{wfs.signature},
+	}
+
+	err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		wfs.mapPbIdFromLocalToFiler(request.Entry)
+		defer wfs.mapPbIdFromFilerToLocal(request.Entry)
+
+		if err := filer_pb.CreateEntry(client, request); err != nil {
+			return fmt.Errorf("symlink %s: %v", entryFullPath, err)
+		}
+
+		wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry))
+
+		return nil
+	})
+	if err != nil {
+		glog.V(0).Infof("Symlink %s => %s: %v", entryFullPath, target, err)
+		return fuse.EIO
+	}
+
+	inode := wfs.inodeToPath.Lookup(entryFullPath, false)
+
+	wfs.outputPbEntry(out, inode, request.Entry)
+
+	return fuse.OK
+}
+
+func (wfs *WFS) Readlink(cancel <-chan struct{}, header *fuse.InHeader) (out []byte, code fuse.Status) {
+	entryFullPath := wfs.inodeToPath.GetPath(header.NodeId)
+
+	entry, status := wfs.maybeLoadEntry(entryFullPath)
+	if status != fuse.OK {
+		return nil, status
+	}
+	if os.FileMode(entry.Attributes.FileMode)&os.ModeSymlink == 0 {
+		return nil, fuse.EINVAL
+	}
+
+	return []byte(entry.Attributes.SymlinkTarget), fuse.OK
+}

weed/mount/weedfs_unsupported.go

@@ -0,0 +1,65 @@
+package mount
+
+import "github.com/hanwen/go-fuse/v2/fuse"
+
+// https://github.com/libfuse/libfuse/blob/48ae2e72b39b6a31cb2194f6f11786b7ca06aac6/include/fuse.h#L778
+
+/**
+ * Copy a range of data from one file to anotherNiels de Vos, 4 years ago: • libfuse: add copy_file_range() support
+ *
+ * Performs an optimized copy between two file descriptors without the
+ * additional cost of transferring data through the FUSE kernel module
+ * to user space (glibc) and then back into the FUSE filesystem again.
+ *
+ * In case this method is not implemented, applications are expected to
+ * fall back to a regular file copy.   (Some glibc versions did this
+ * emulation automatically, but the emulation has been removed from all
+ * glibc release branches.)
+ */
+func (wfs *WFS) CopyFileRange(cancel <-chan struct{}, in *fuse.CopyFileRangeIn) (written uint32, code fuse.Status) {
+	return 0, fuse.ENOSYS
+}
+
+/**
+ * Allocates space for an open file
+ *
+ * This function ensures that required space is allocated for specified
+ * file.  If this function returns success then any subsequent write
+ * request to specified range is guaranteed not to fail because of lack
+ * of space on the file system media.
+ */
+func (wfs *WFS) Fallocate(cancel <-chan struct{}, in *fuse.FallocateIn) (code fuse.Status) {
+	return fuse.ENOSYS
+}
+
+/**
+ * Find next data or hole after the specified offset
+ */
+func (wfs *WFS) Lseek(cancel <-chan struct{}, in *fuse.LseekIn, out *fuse.LseekOut) fuse.Status {
+	return fuse.ENOSYS
+}
+
+func (wfs *WFS) GetLk(cancel <-chan struct{}, in *fuse.LkIn, out *fuse.LkOut) (code fuse.Status) {
+	return fuse.ENOSYS
+}
+
+func (wfs *WFS) SetLk(cancel <-chan struct{}, in *fuse.LkIn) (code fuse.Status) {
+	return fuse.ENOSYS
+}
+
+func (wfs *WFS) SetLkw(cancel <-chan struct{}, in *fuse.LkIn) (code fuse.Status) {
+	return fuse.ENOSYS
+}
+
+/**
+ * Check file access permissions
+ *
+ * This will be called for the access() system call.  If the
+ * 'default_permissions' mount option is given, this method is not
+ * called.
+ *
+ * This method is not called under Linux kernel versions 2.4.x
+ */
+func (wfs *WFS) Access(cancel <-chan struct{}, input *fuse.AccessIn) (code fuse.Status) {
+	return fuse.ENOSYS
+}

weed/mount/weedfs_write.go

@@ -0,0 +1,84 @@
+package mount
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath) filer.SaveDataAsChunkFunctionType {
+
+	return func(reader io.Reader, filename string, offset int64) (chunk *filer_pb.FileChunk, collection, replication string, err error) {
+		var fileId, host string
+		var auth security.EncodedJwt
+
+		if err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+			return util.Retry("assignVolume", func() error {
+				request := &filer_pb.AssignVolumeRequest{
+					Count:       1,
+					Replication: wfs.option.Replication,
+					Collection:  wfs.option.Collection,
+					TtlSec:      wfs.option.TtlSec,
+					DiskType:    string(wfs.option.DiskType),
+					DataCenter:  wfs.option.DataCenter,
+					Path:        string(fullPath),
+				}
+
+				resp, err := client.AssignVolume(context.Background(), request)
+				if err != nil {
+					glog.V(0).Infof("assign volume failure %v: %v", request, err)
+					return err
+				}
+				if resp.Error != "" {
+					return fmt.Errorf("assign volume failure %v: %v", request, resp.Error)
+				}
+
+				fileId, auth = resp.FileId, security.EncodedJwt(resp.Auth)
+				loc := resp.Location
+				host = wfs.AdjustedUrl(loc)
+				collection, replication = resp.Collection, resp.Replication
+
+				return nil
+			})
+		}); err != nil {
+			return nil, "", "", fmt.Errorf("filerGrpcAddress assign volume: %v", err)
+		}
+
+		fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
+		if wfs.option.VolumeServerAccess == "filerProxy" {
+			fileUrl = fmt.Sprintf("http://%s/?proxyChunkId=%s", wfs.getCurrentFiler(), fileId)
+		}
+		uploadOption := &operation.UploadOption{
+			UploadUrl:         fileUrl,
+			Filename:          filename,
+			Cipher:            wfs.option.Cipher,
+			IsInputCompressed: false,
+			MimeType:          "",
+			PairMap:           nil,
+			Jwt:               auth,
+		}
+		uploadResult, err, data := operation.Upload(reader, uploadOption)
+		if err != nil {
+			glog.V(0).Infof("upload data %v to %s: %v", filename, fileUrl, err)
+			return nil, "", "", fmt.Errorf("upload data: %v", err)
+		}
+		if uploadResult.Error != "" {
+			glog.V(0).Infof("upload failure %v to %s: %v", filename, fileUrl, err)
+			return nil, "", "", fmt.Errorf("upload result: %v", uploadResult.Error)
+		}
+
+		if offset == 0 {
+			wfs.chunkCache.SetChunk(fileId, data)
+		}
+
+		chunk = uploadResult.ToPbFileChunk(fileId, offset)
+		return chunk, collection, replication, nil
+	}
+}

weed/mount/weedfs_xattr.go

@@ -0,0 +1,168 @@
+package mount
+
+import (
+	"github.com/hanwen/go-fuse/v2/fuse"
+	sys "golang.org/x/sys/unix"
+	"runtime"
+	"strings"
+	"syscall"
+)
+
+const (
+	// https://man7.org/linux/man-pages/man7/xattr.7.html#:~:text=The%20VFS%20imposes%20limitations%20that,in%20listxattr(2)).
+	MAX_XATTR_NAME_SIZE  = 255
+	MAX_XATTR_VALUE_SIZE = 65536
+	XATTR_PREFIX         = "xattr-" // same as filer
+)
+
+// GetXAttr reads an extended attribute, and should return the
+// number of bytes. If the buffer is too small, return ERANGE,
+// with the required buffer size.
+func (wfs *WFS) GetXAttr(cancel <-chan struct{}, header *fuse.InHeader, attr string, dest []byte) (size uint32, code fuse.Status) {
+
+	//validate attr name
+	if len(attr) > MAX_XATTR_NAME_SIZE {
+		if runtime.GOOS == "darwin" {
+			return 0, fuse.EPERM
+		} else {
+			return 0, fuse.ERANGE
+		}
+	}
+	if len(attr) == 0 {
+		return 0, fuse.EINVAL
+	}
+
+	_, _, entry, status := wfs.maybeReadEntry(header.NodeId)
+	if status != fuse.OK {
+		return 0, status
+	}
+	if entry == nil {
+		return 0, fuse.ENOENT
+	}
+	if entry.Extended == nil {
+		return 0, fuse.ENOATTR
+	}
+	data, found := entry.Extended[XATTR_PREFIX+attr]
+	if !found {
+		return 0, fuse.ENOATTR
+	}
+	if len(dest) < len(data) {
+		return uint32(len(data)), fuse.ERANGE
+	}
+	copy(dest, data)
+
+	return uint32(len(data)), fuse.OK
+}
+
+// SetXAttr writes an extended attribute.
+// https://man7.org/linux/man-pages/man2/setxattr.2.html
+//        By default (i.e., flags is zero), the extended attribute will be
+//       created if it does not exist, or the value will be replaced if
+//       the attribute already exists.  To modify these semantics, one of
+//       the following values can be specified in flags:
+//
+//       XATTR_CREATE
+//              Perform a pure create, which fails if the named attribute
+//              exists already.
+//
+//       XATTR_REPLACE
+//              Perform a pure replace operation, which fails if the named
+//              attribute does not already exist.
+func (wfs *WFS) SetXAttr(cancel <-chan struct{}, input *fuse.SetXAttrIn, attr string, data []byte) fuse.Status {
+	//validate attr name
+	if len(attr) > MAX_XATTR_NAME_SIZE {
+		if runtime.GOOS == "darwin" {
+			return fuse.EPERM
+		} else {
+			return fuse.ERANGE
+		}
+	}
+	if len(attr) == 0 {
+		return fuse.EINVAL
+	}
+	//validate attr value
+	if len(data) > MAX_XATTR_VALUE_SIZE {
+		if runtime.GOOS == "darwin" {
+			return fuse.Status(syscall.E2BIG)
+		} else {
+			return fuse.ERANGE
+		}
+	}
+
+	path, _, entry, status := wfs.maybeReadEntry(input.NodeId)
+	if status != fuse.OK {
+		return status
+	}
+	if entry.Extended == nil {
+		entry.Extended = make(map[string][]byte)
+	}
+	oldData, _ := entry.Extended[XATTR_PREFIX+attr]
+	switch input.Flags {
+	case sys.XATTR_CREATE:
+		if len(oldData) > 0 {
+			break
+		}
+		fallthrough
+	case sys.XATTR_REPLACE:
+		fallthrough
+	default:
+		entry.Extended[XATTR_PREFIX+attr] = data
+	}
+
+	return wfs.saveEntry(path, entry)
+
+}
+
+// ListXAttr lists extended attributes as '\0' delimited byte
+// slice, and return the number of bytes. If the buffer is too
+// small, return ERANGE, with the required buffer size.
+func (wfs *WFS) ListXAttr(cancel <-chan struct{}, header *fuse.InHeader, dest []byte) (n uint32, code fuse.Status) {
+	_, _, entry, status := wfs.maybeReadEntry(header.NodeId)
+	if status != fuse.OK {
+		return 0, status
+	}
+	if entry == nil {
+		return 0, fuse.ENOENT
+	}
+	if entry.Extended == nil {
+		return 0, fuse.ENOATTR
+	}
+
+	var data []byte
+	for k := range entry.Extended {
+		if strings.HasPrefix(k, XATTR_PREFIX) {
+			data = append(data, k[len(XATTR_PREFIX):]...)
+			data = append(data, 0)
+		}
+	}
+	if len(dest) < len(data) {
+		return uint32(len(data)), fuse.ERANGE
+	}
+
+	copy(dest, data)
+
+	return uint32(len(data)), fuse.OK
+}
+
+// RemoveXAttr removes an extended attribute.
+func (wfs *WFS) RemoveXAttr(cancel <-chan struct{}, header *fuse.InHeader, attr string) fuse.Status {
+	if len(attr) == 0 {
+		return fuse.EINVAL
+	}
+	path, _, entry, status := wfs.maybeReadEntry(header.NodeId)
+	if status != fuse.OK {
+		return status
+	}
+	if entry.Extended == nil {
+		return fuse.ENOATTR
+	}
+	_, found := entry.Extended[XATTR_PREFIX+attr]
+
+	if !found {
+		return fuse.ENOATTR
+	}
+
+	delete(entry.Extended, XATTR_PREFIX+attr)
+
+	return wfs.saveEntry(path, entry)
+}

weed/mount/wfs_filer_client.go

@@ -0,0 +1,51 @@
+package mount
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+)
+
+var _ = filer_pb.FilerClient(&WFS{})
+
+func (wfs *WFS) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) (err error) {
+
+	return util.Retry("filer grpc", func() error {
+
+		i := wfs.option.filerIndex
+		n := len(wfs.option.FilerAddresses)
+		for x := 0; x < n; x++ {
+
+			filerGrpcAddress := wfs.option.FilerAddresses[i].ToGrpcAddress()
+			err = pb.WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {
+				client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+				return fn(client)
+			}, filerGrpcAddress, wfs.option.GrpcDialOption)
+
+			if err != nil {
+				glog.V(0).Infof("WithFilerClient %d %v: %v", x, filerGrpcAddress, err)
+			} else {
+				wfs.option.filerIndex = i
+				return nil
+			}
+
+			i++
+			if i >= n {
+				i = 0
+			}
+
+		}
+		return err
+	})
+
+}
+
+func (wfs *WFS) AdjustedUrl(location *filer_pb.Location) string {
+	if wfs.option.VolumeServerAccess == "publicUrl" {
+		return location.PublicUrl
+	}
+	return location.Url
+}

weed/mount/wfs_save.go

@@ -0,0 +1,67 @@
+package mount
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/hanwen/go-fuse/v2/fuse"
+	"syscall"
+)
+
+func (wfs *WFS) saveEntry(path util.FullPath, entry *filer_pb.Entry) (code fuse.Status) {
+
+	parentDir, _ := path.DirAndName()
+
+	err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		wfs.mapPbIdFromLocalToFiler(entry)
+		defer wfs.mapPbIdFromFilerToLocal(entry)
+
+		request := &filer_pb.UpdateEntryRequest{
+			Directory:  parentDir,
+			Entry:      entry,
+			Signatures: []int32{wfs.signature},
+		}
+
+		glog.V(1).Infof("save entry: %v", request)
+		_, err := client.UpdateEntry(context.Background(), request)
+		if err != nil {
+			return fmt.Errorf("UpdateEntry dir %s: %v", path, err)
+		}
+
+		if err := wfs.metaCache.UpdateEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil {
+			return fmt.Errorf("UpdateEntry dir %s: %v", path, err)
+		}
+
+		return nil
+	})
+	if err != nil {
+		glog.Errorf("saveEntry %s: %v", path, err)
+		return fuse.EIO
+	}
+
+	return fuse.OK
+}
+
+func (wfs *WFS) mapPbIdFromFilerToLocal(entry *filer_pb.Entry) {
+	if entry.Attributes == nil {
+		return
+	}
+	entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.FilerToLocal(entry.Attributes.Uid, entry.Attributes.Gid)
+}
+func (wfs *WFS) mapPbIdFromLocalToFiler(entry *filer_pb.Entry) {
+	if entry.Attributes == nil {
+		return
+	}
+	entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.LocalToFiler(entry.Attributes.Uid, entry.Attributes.Gid)
+}
+
+func checkName(name string) fuse.Status {
+	if len(name) >= 256 {
+		return fuse.Status(syscall.ENAMETOOLONG)
+	}
+	return fuse.OK
+}

weed/replication/sink/filersink/filer_sink.go

@@ -199,7 +199,7 @@ func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParent
 		// delete the chunks that are deleted from the source
 		if deleteIncludeChunks {
 			// remove the deleted chunks. Actual data deletion happens in filer UpdateEntry FindUnusedFileChunks
-			existingEntry.Chunks = filer.DoMinusChunks(existingEntry.Chunks, deletedChunks)
+			existingEntry.Chunks = filer.DoMinusChunksBySourceFileId(existingEntry.Chunks, deletedChunks)
 		}
 
 		// replicate the chunks that are new in the source

weed/replication/sub/notification_kafka.go

@@ -137,7 +137,7 @@ func (progress *KafkaProgress) saveProgress() error {
 	if err != nil {
 		return fmt.Errorf("failed to marshal progress: %v", err)
 	}
-	err = os.WriteFile(progress.offsetFile, data, 0640)
+	err = util.WriteFile(progress.offsetFile, data, 0640)
 	if err != nil {
 		return fmt.Errorf("failed to save progress to %s: %v", progress.offsetFile, err)
 	}

weed/s3api/s3api_bucket_handlers.go

@@ -4,13 +4,14 @@ import (
 	"context"
 	"encoding/xml"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/filer"
-	"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
-	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"math"
 	"net/http"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+
 	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
 	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 
@@ -309,3 +310,15 @@ func (s3a *S3ApiServer) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *h
 	s3err.WriteEmptyResponse(w, r, http.StatusNoContent)
 
 }
+
+// GetBucketLocationHandler Get bucket location
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLocation.html
+func (s3a *S3ApiServer) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseXML(w, r, LocationConstraint{})
+}
+
+// GetBucketRequestPaymentHandler Get bucket location
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketRequestPayment.html
+func (s3a *S3ApiServer) GetBucketRequestPaymentHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseXML(w, r, RequestPaymentConfiguration{Payer: "BucketOwner"})
+}

weed/s3api/s3api_bucket_skip_handlers.go

@@ -0,0 +1,49 @@
+package s3api
+
+import (
+	"net/http"
+
+	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
+)
+
+// GetBucketCorsHandler Get bucket CORS
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketCors.html
+func (s3a *S3ApiServer) GetBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
+	s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchCORSConfiguration)
+}
+
+// PutBucketCorsHandler Put bucket CORS
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketCors.html
+func (s3a *S3ApiServer) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
+	s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
+}
+
+// DeleteBucketCorsHandler Delete bucket CORS
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketCors.html
+func (s3a *S3ApiServer) DeleteBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
+	s3err.WriteErrorResponse(w, r, http.StatusNoContent)
+}
+
+// GetBucketPolicyHandler Get bucket Policy
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketPolicy.html
+func (s3a *S3ApiServer) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+	s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucketPolicy)
+}
+
+// PutBucketPolicyHandler Put bucket Policy
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketPolicy.html
+func (s3a *S3ApiServer) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+	s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
+}
+
+// DeleteBucketPolicyHandler Delete bucket Policy
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketPolicy.html
+func (s3a *S3ApiServer) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+	s3err.WriteErrorResponse(w, r, http.StatusNoContent)
+}
+
+// PutBucketAclHandler Put bucket ACL
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAcl.html
+func (s3a *S3ApiServer) PutBucketAclHandler(w http.ResponseWriter, r *http.Request) {
+	s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
+}

weed/s3api/s3api_object_handlers.go

@@ -436,10 +436,14 @@ func setEtag(w http.ResponseWriter, etag string) {
 }
 
 func filerErrorToS3Error(errString string) s3err.ErrorCode {
-	if strings.HasPrefix(errString, "existing ") && strings.HasSuffix(errString, "is a directory") {
+	switch {
+	case strings.HasPrefix(errString, "existing ") && strings.HasSuffix(errString, "is a directory"):
 		return s3err.ErrExistingObjectIsDirectory
+	case strings.HasSuffix(errString, "is a file"):
+		return s3err.ErrExistingObjectIsFile
+	default:
+		return s3err.ErrInternalError
 	}
-	return s3err.ErrInternalError
 }
 
 func (s3a *S3ApiServer) maybeAddFilerJwtAuthorization(r *http.Request, isWrite bool) {

weed/s3api/s3api_object_multipart_handlers.go

@@ -163,13 +163,13 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re
 		UploadId:         aws.String(uploadID),
 	})
 
-	glog.V(2).Infof("ListObjectPartsHandler %s count=%d", string(s3err.EncodeXMLResponse(response)), len(response.Part))
-
 	if errCode != s3err.ErrNone {
 		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}
 
+	glog.V(2).Infof("ListObjectPartsHandler %s count=%d", string(s3err.EncodeXMLResponse(response)), len(response.Part))
+
 	writeSuccessResponseXML(w, r, response)
 
 }

weed/s3api/s3api_server.go

@@ -2,16 +2,16 @@ package s3api
 
 import (
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/pb"
-	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/chrislusf/seaweedfs/weed/util"
 	"net/http"
 	"strings"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	. "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
 	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/gorilla/mux"
 	"google.golang.org/grpc"
 )
@@ -77,10 +77,13 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
 
 	for _, bucket := range routers {
 
-		// HeadObject
-		bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.HeadObjectHandler, ACTION_READ), "GET"))
-		// HeadBucket
-		bucket.Methods("HEAD").HandlerFunc(track(s3a.iam.Auth(s3a.HeadBucketHandler, ACTION_READ), "GET"))
+		// each case should follow the next rule:
+		// - requesting object with query must precede any other methods
+		// - requesting object must precede any methods with buckets
+		// - requesting bucket with query must precede raw methods with buckets
+		// - requesting bucket must be processed in the end
+
+		// objects with query
 
 		// CopyObjectPart
 		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", `.*?(\/|%2F).*?`).HandlerFunc(track(s3a.iam.Auth(s3a.CopyObjectPartHandler, ACTION_WRITE), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
@@ -113,62 +116,87 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
 		// PutObjectLockConfiguration
 		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.PutObjectLockConfigurationHandler, ACTION_WRITE), "PUT")).Queries("object-lock", "")
 
+		// GetObjectACL
+		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.GetObjectAclHandler, ACTION_READ), "GET")).Queries("acl", "")
+
+		// objects with query
+
+		// raw objects
+
+		// HeadObject
+		bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.HeadObjectHandler, ACTION_READ), "GET"))
+
+		// GetObject, but directory listing is not supported
+		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.GetObjectHandler, ACTION_READ), "GET"))
+
 		// CopyObject
 		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(track(s3a.iam.Auth(s3a.CopyObjectHandler, ACTION_WRITE), "COPY"))
 		// PutObject
 		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.PutObjectHandler, ACTION_WRITE), "PUT"))
-
 		// DeleteObject
 		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteObjectHandler, ACTION_WRITE), "DELETE"))
 
-		// ListObjectsV2
-		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.ListObjectsV2Handler, ACTION_LIST), "LIST")).Queries("list-type", "2")
-		// GetObject, but directory listing is not supported
-		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.GetObjectHandler, ACTION_READ), "GET"))
+		// raw objects
 
-		// PostPolicy
-		bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(track(s3a.iam.Auth(s3a.PostPolicyBucketHandler, ACTION_WRITE), "POST"))
+		// buckets with query
 
 		// DeleteMultipleObjects
 		bucket.Methods("POST").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteMultipleObjectsHandler, ACTION_WRITE), "DELETE")).Queries("delete", "")
 
 		// GetBucketACL
-		bucket.Methods("GET").HandlerFunc(s3a.iam.Auth(s3a.GetBucketAclHandler, ACTION_READ)).Queries("acl", "")
+		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.GetBucketAclHandler, ACTION_READ), "GET")).Queries("acl", "")
+		// PutBucketACL
+		bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketAclHandler, ACTION_WRITE), "PUT")).Queries("acl", "")
 
-		// GetObjectACL
-		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(s3a.iam.Auth(s3a.GetObjectAclHandler, ACTION_READ)).Queries("acl", "")
+		// GetBucketPolicy
+		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.GetBucketPolicyHandler, ACTION_READ), "GET")).Queries("policy", "")
+		// PutBucketPolicy
+		bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketPolicyHandler, ACTION_WRITE), "PUT")).Queries("policy", "")
+		// DeleteBucketPolicy
+		bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketPolicyHandler, ACTION_WRITE), "DELETE")).Queries("policy", "")
+
+		// GetBucketCors
+		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.GetBucketCorsHandler, ACTION_READ), "GET")).Queries("cors", "")
+		// PutBucketCors
+		bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketCorsHandler, ACTION_WRITE), "PUT")).Queries("cors", "")
+		// DeleteBucketCors
+		bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketCorsHandler, ACTION_WRITE), "DELETE")).Queries("cors", "")
 
 		// GetBucketLifecycleConfiguration
-		bucket.Methods("GET").HandlerFunc(s3a.iam.Auth(s3a.GetBucketLifecycleConfigurationHandler, ACTION_READ)).Queries("lifecycle", "")
-
+		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.GetBucketLifecycleConfigurationHandler, ACTION_READ), "GET")).Queries("lifecycle", "")
 		// PutBucketLifecycleConfiguration
-		bucket.Methods("PUT").HandlerFunc(s3a.iam.Auth(s3a.PutBucketLifecycleConfigurationHandler, ACTION_WRITE)).Queries("lifecycle", "")
-
+		bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketLifecycleConfigurationHandler, ACTION_WRITE), "PUT")).Queries("lifecycle", "")
 		// DeleteBucketLifecycleConfiguration
-		bucket.Methods("DELETE").HandlerFunc(s3a.iam.Auth(s3a.DeleteBucketLifecycleHandler, ACTION_WRITE)).Queries("lifecycle", "")
-		// ListObjectsV1 (Legacy)
-		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.ListObjectsV1Handler, ACTION_LIST), "LIST"))
+		bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketLifecycleHandler, ACTION_WRITE), "DELETE")).Queries("lifecycle", "")
+
+		// GetBucketLocation
+		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.GetBucketLocationHandler, ACTION_READ), "GET")).Queries("location", "")
+
+		// GetBucketRequestPayment
+		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.GetBucketRequestPaymentHandler, ACTION_READ), "GET")).Queries("requestPayment", "")
+
+		// ListObjectsV2
+		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.ListObjectsV2Handler, ACTION_LIST), "LIST")).Queries("list-type", "2")
+
+		// buckets with query
+
+		// raw buckets
+
+		// PostPolicy
+		bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(track(s3a.iam.Auth(s3a.PostPolicyBucketHandler, ACTION_WRITE), "POST"))
+
+		// HeadBucket
+		bucket.Methods("HEAD").HandlerFunc(track(s3a.iam.Auth(s3a.HeadBucketHandler, ACTION_READ), "GET"))
 
 		// PutBucket
 		bucket.Methods("PUT").HandlerFunc(track(s3a.PutBucketHandler, "PUT"))
-
 		// DeleteBucket
 		bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketHandler, ACTION_WRITE), "DELETE"))
 
-		/*
+		// ListObjectsV1 (Legacy)
+		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.ListObjectsV1Handler, ACTION_LIST), "LIST"))
 
-			// not implemented
-			// GetBucketLocation
-			bucket.Methods("GET").HandlerFunc(s3a.GetBucketLocationHandler).Queries("location", "")
-			// GetBucketPolicy
-			bucket.Methods("GET").HandlerFunc(s3a.GetBucketPolicyHandler).Queries("policy", "")
-			// GetObjectACL
-			bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(s3a.GetObjectACLHandler).Queries("acl", "")
-			// PutBucketPolicy
-			bucket.Methods("PUT").HandlerFunc(s3a.PutBucketPolicyHandler).Queries("policy", "")
-			// DeleteBucketPolicy
-			bucket.Methods("DELETE").HandlerFunc(s3a.DeleteBucketPolicyHandler).Queries("policy", "")
-		*/
+		// raw buckets
 
 	}
 

weed/s3api/s3api_xsd_generated.go

@@ -646,6 +646,10 @@ type ListVersionsResult struct {
 	CommonPrefixes      []PrefixEntry     `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CommonPrefixes,omitempty"`
 }
 
+type LocationConstraint struct {
+	LocationConstraint string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint"`
+}
+
 type LoggingSettings struct {
 	TargetBucket string            `xml:"http://s3.amazonaws.com/doc/2006-03-01/ TargetBucket"`
 	TargetPrefix string            `xml:"http://s3.amazonaws.com/doc/2006-03-01/ TargetPrefix"`

weed/s3api/s3err/s3-error.go

@@ -58,4 +58,5 @@ var s3ErrorResponseMap = map[string]string{
 	"InvalidDuration":                   "Duration provided in the request is invalid.",
 	"XAmzContentSHA256Mismatch":         "The provided 'x-amz-content-sha256' header does not match what was computed.",
 	// Add new API errors here.
+	"NoSuchCORSConfiguration": "The CORS configuration does not exist",
 }

weed/s3api/s3err/s3api_errors.go

@@ -51,6 +51,8 @@ const (
 	ErrBucketAlreadyExists
 	ErrBucketAlreadyOwnedByYou
 	ErrNoSuchBucket
+	ErrNoSuchBucketPolicy
+	ErrNoSuchCORSConfiguration
 	ErrNoSuchLifecycleConfiguration
 	ErrNoSuchKey
 	ErrNoSuchUpload
@@ -99,6 +101,7 @@ const (
 	ErrPreconditionFailed
 
 	ErrExistingObjectIsDirectory
+	ErrExistingObjectIsFile
 )
 
 // error code to APIError structure, these fields carry respective
@@ -164,6 +167,16 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The specified bucket does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrNoSuchBucketPolicy: {
+		Code:           "NoSuchBucketPolicy",
+		Description:    "The bucket policy does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchCORSConfiguration: {
+		Code:           "NoSuchCORSConfiguration",
+		Description:    "The CORS configuration does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrNoSuchLifecycleConfiguration: {
 		Code:           "NoSuchLifecycleConfiguration",
 		Description:    "The lifecycle configuration does not exist",
@@ -371,6 +384,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Existing Object is a directory.",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrExistingObjectIsFile: {
+		Code:           "ExistingObjectIsFile",
+		Description:    "Existing Object is a file.",
+		HTTPStatusCode: http.StatusConflict,
+	},
 }
 
 // GetAPIError provides API Error for input API error code.

weed/server/filer_server_handlers_read.go

@@ -21,6 +21,64 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
+
+// Validates the preconditions. Returns true if GET/HEAD operation should not proceed.
+// Preconditions supported are:
+//  If-Modified-Since
+//  If-Unmodified-Since
+//  If-Match
+//  If-None-Match
+func checkPreconditions(w http.ResponseWriter, r *http.Request, entry *filer.Entry) bool {
+
+	etag := filer.ETagEntry(entry)
+	/// When more than one conditional request header field is present in a
+	/// request, the order in which the fields are evaluated becomes
+	/// important.  In practice, the fields defined in this document are
+	/// consistently implemented in a single, logical order, since "lost
+	/// update" preconditions have more strict requirements than cache
+	/// validation, a validated cache is more efficient than a partial
+	/// response, and entity tags are presumed to be more accurate than date
+	/// validators. https://tools.ietf.org/html/rfc7232#section-5
+	if entry.Attr.Mtime.IsZero() {
+		return false
+	}
+	w.Header().Set("Last-Modified", entry.Attr.Mtime.UTC().Format(http.TimeFormat))
+
+	ifMatchETagHeader := r.Header.Get("If-Match")
+	ifUnmodifiedSinceHeader := r.Header.Get("If-Unmodified-Since")
+	if ifMatchETagHeader != "" {
+		if util.CanonicalizeETag(etag) != util.CanonicalizeETag(ifMatchETagHeader) {
+			w.WriteHeader(http.StatusPreconditionFailed)
+			return true
+		}
+	} else if ifUnmodifiedSinceHeader != "" {
+		if t, parseError := time.Parse(http.TimeFormat, ifUnmodifiedSinceHeader); parseError == nil {
+			if t.Before(entry.Attr.Mtime) {
+				w.WriteHeader(http.StatusPreconditionFailed)
+				return true
+			}
+		}
+	}
+
+	ifNoneMatchETagHeader := r.Header.Get("If-None-Match")
+	ifModifiedSinceHeader := r.Header.Get("If-Modified-Since")
+	if ifNoneMatchETagHeader != "" {
+		if util.CanonicalizeETag(etag) == util.CanonicalizeETag(ifNoneMatchETagHeader) {
+			w.WriteHeader(http.StatusNotModified)
+			return true
+		}
+	} else if ifModifiedSinceHeader != "" {
+		if t, parseError := time.Parse(http.TimeFormat, ifModifiedSinceHeader); parseError == nil {
+			if t.After(entry.Attr.Mtime) {
+				w.WriteHeader(http.StatusNotModified)
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
 func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) {
 
 	path := r.URL.Path
@@ -37,11 +95,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 		}
 		if err == filer_pb.ErrNotFound {
 			glog.V(1).Infof("Not found %s: %v", path, err)
-			stats.FilerRequestCounter.WithLabelValues("read.notfound").Inc()
+			stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadNotFound).Inc()
 			w.WriteHeader(http.StatusNotFound)
 		} else {
 			glog.Errorf("Internal %s: %v", path, err)
-			stats.FilerRequestCounter.WithLabelValues("read.internalerror").Inc()
+			stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadInternal).Inc()
 			w.WriteHeader(http.StatusInternalServerError)
 		}
 		return
@@ -61,10 +119,8 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	// set etag
 	etag := filer.ETagEntry(entry)
-	if ifm := r.Header.Get("If-Match"); ifm != "" && (ifm != "\""+etag+"\"" && ifm != etag) {
-		w.WriteHeader(http.StatusPreconditionFailed)
+	if checkPreconditions(w, r, entry) {
 		return
 	}
 
@@ -81,19 +137,6 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 		w.Header().Set("Content-Type", mimeType)
 	}
 
-	// if modified since
-	if !entry.Attr.Mtime.IsZero() {
-		w.Header().Set("Last-Modified", entry.Attr.Mtime.UTC().Format(http.TimeFormat))
-		if r.Header.Get("If-Modified-Since") != "" {
-			if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil {
-				if !t.Before(entry.Attr.Mtime) {
-					w.WriteHeader(http.StatusNotModified)
-					return
-				}
-			}
-		}
-	}
-
 	// print out the header from extended properties
 	for k, v := range entry.Extended {
 		if !strings.HasPrefix(k, "xattr-") {
@@ -123,10 +166,6 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 		w.Header().Set(xhttp.AmzTagCount, strconv.Itoa(tagCount))
 	}
 
-	if inm := r.Header.Get("If-None-Match"); inm == "\""+etag+"\"" {
-		w.WriteHeader(http.StatusNotModified)
-		return
-	}
 	setEtag(w, etag)
 
 	filename := entry.Name()
@@ -162,6 +201,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 		if offset+size <= int64(len(entry.Content)) {
 			_, err := writer.Write(entry.Content[offset : offset+size])
 			if err != nil {
+				stats.FilerRequestCounter.WithLabelValues(stats.ErrorWriteEntry).Inc()
 				glog.Errorf("failed to write entry content: %v", err)
 			}
 			return err
@@ -173,6 +213,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 				Directory: dir,
 				Name:      name,
 			}); err != nil {
+				stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadCache).Inc()
 				glog.Errorf("CacheRemoteObjectToLocalCluster %s: %v", entry.FullPath, err)
 				return fmt.Errorf("cache %s: %v", entry.FullPath, err)
 			} else {
@@ -182,6 +223,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 
 		err = filer.StreamContent(fs.filer.MasterClient, writer, chunks, offset, size)
 		if err != nil {
+			stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadStream).Inc()
 			glog.Errorf("failed to stream content %s: %v", r.URL, err)
 		}
 		return err

weed/server/volume_grpc_copy.go

@@ -67,7 +67,7 @@ func (vs *VolumeServer) VolumeCopy(req *volume_server_pb.VolumeCopyRequest, stre
 		dataBaseFileName = storage.VolumeFileName(location.Directory, volFileInfoResp.Collection, int(req.VolumeId))
 		indexBaseFileName = storage.VolumeFileName(location.IdxDirectory, volFileInfoResp.Collection, int(req.VolumeId))
 
-		os.WriteFile(dataBaseFileName+".note", []byte(fmt.Sprintf("copying from %s", req.SourceDataNode)), 0755)
+		util.WriteFile(dataBaseFileName+".note", []byte(fmt.Sprintf("copying from %s", req.SourceDataNode)), 0755)
 
 		defer func() {
 			if err != nil {

weed/shell/command_collection_list.go

@@ -38,7 +38,7 @@ func (c *commandCollectionList) Do(args []string, commandEnv *CommandEnv, writer
 		return err
 	}
 
-	topologyInfo, _, err := collectTopologyInfo(commandEnv)
+	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
 	if err != nil {
 		return err
 	}

weed/shell/command_ec_common.go

@@ -172,7 +172,7 @@ func countFreeShardSlots(dn *master_pb.DataNodeInfo, diskType types.DiskType) (c
 	if diskInfo == nil {
 		return 0
 	}
-	return int(diskInfo.MaxVolumeCount-diskInfo.ActiveVolumeCount)*erasure_coding.DataShardsCount - countShards(diskInfo.EcShardInfos)
+	return int(diskInfo.MaxVolumeCount-diskInfo.VolumeCount)*erasure_coding.DataShardsCount - countShards(diskInfo.EcShardInfos)
 }
 
 type RackId string
@@ -206,7 +206,7 @@ func collectEcNodes(commandEnv *CommandEnv, selectedDataCenter string) (ecNodes
 
 	// list all possible locations
 	// collect topology information
-	topologyInfo, _, err := collectTopologyInfo(commandEnv)
+	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
 	if err != nil {
 		return
 	}

weed/shell/command_ec_decode.go

@@ -8,6 +8,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
 	"io"
+	"time"
 
 	"google.golang.org/grpc"
 
@@ -53,7 +54,7 @@ func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
 	vid := needle.VolumeId(*volumeId)
 
 	// collect topology information
-	topologyInfo, _, err := collectTopologyInfo(commandEnv)
+	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
 	if err != nil {
 		return err
 	}
@@ -233,7 +234,11 @@ func lookupVolumeIds(commandEnv *CommandEnv, volumeIds []string) (volumeIdLocati
 	return resp.VolumeIdLocations, nil
 }
 
-func collectTopologyInfo(commandEnv *CommandEnv) (topoInfo *master_pb.TopologyInfo, volumeSizeLimitMb uint64, err error) {
+func collectTopologyInfo(commandEnv *CommandEnv, delayBeforeCollecting time.Duration) (topoInfo *master_pb.TopologyInfo, volumeSizeLimitMb uint64, err error) {
+
+	if delayBeforeCollecting > 0 {
+		time.Sleep(delayBeforeCollecting)
+	}
 
 	var resp *master_pb.VolumeListResponse
 	err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {

weed/shell/command_ec_encode.go

@@ -267,7 +267,7 @@ func balancedEcDistribution(servers []*EcNode) (allocated [][]uint32) {
 func collectVolumeIdsForEcEncode(commandEnv *CommandEnv, selectedCollection string, fullPercentage float64, quietPeriod time.Duration) (vids []needle.VolumeId, err error) {
 
 	// collect topology information
-	topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv)
+	topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 0)
 	if err != nil {
 		return
 	}

weed/shell/command_ec_encode_test.go

@@ -0,0 +1,30 @@
+package shell
+
+import (
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
+	"testing"
+)
+
+func TestEcDistribution(t *testing.T) {
+
+	topologyInfo := parseOutput(topoData)
+
+	// find out all volume servers with one slot left.
+	ecNodes, totalFreeEcSlots := collectEcVolumeServersByDc(topologyInfo, "")
+
+	sortEcNodesByFreeslotsDecending(ecNodes)
+
+	if totalFreeEcSlots < erasure_coding.TotalShardsCount {
+		println("not enough free ec shard slots", totalFreeEcSlots)
+	}
+	allocatedDataNodes := ecNodes
+	if len(allocatedDataNodes) > erasure_coding.TotalShardsCount {
+		allocatedDataNodes = allocatedDataNodes[:erasure_coding.TotalShardsCount]
+	}
+
+	for _, dn := range allocatedDataNodes {
+		fmt.Printf("info %+v %+v\n", dn.info, dn)
+	}
+
+}

weed/shell/command_fs_cat.go

@@ -2,12 +2,10 @@ package shell
 
 import (
 	"fmt"
-	"io"
-	"math"
-
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"io"
 )
 
 func init() {
@@ -57,7 +55,7 @@ func (c *commandFsCat) Do(args []string, commandEnv *CommandEnv, writer io.Write
 			return err
 		}
 
-		return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
+		return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, int64(filer.FileSize(respLookupEntry.Entry)))
 
 	})
 

weed/shell/command_s3_bucket_list.go

@@ -35,7 +35,7 @@ func (c *commandS3BucketList) Do(args []string, commandEnv *CommandEnv, writer i
 	}
 
 	// collect collection information
-	topologyInfo, _, err := collectTopologyInfo(commandEnv)
+	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
 	if err != nil {
 		return err
 	}

weed/shell/command_s3_bucket_quota_check.go

@@ -38,7 +38,7 @@ func (c *commandS3BucketQuotaEnforce) Do(args []string, commandEnv *CommandEnv,
 	}
 
 	// collect collection information
-	topologyInfo, _, err := collectTopologyInfo(commandEnv)
+	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
 	if err != nil {
 		return err
 	}

weed/shell/command_volume_balance.go

@@ -76,7 +76,7 @@ func (c *commandVolumeBalance) Do(args []string, commandEnv *CommandEnv, writer
 	}
 
 	// collect topology information
-	topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv)
+	topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 15*time.Second)
 	if err != nil {
 		return err
 	}

weed/shell/command_volume_check_disk.go

@@ -58,7 +58,7 @@ func (c *commandVolumeCheckDisk) Do(args []string, commandEnv *CommandEnv, write
 	c.env = commandEnv
 
 	// collect topology information
-	topologyInfo, _, err := collectTopologyInfo(commandEnv)
+	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
 	if err != nil {
 		return err
 	}
@@ -138,9 +138,9 @@ func (c *commandVolumeCheckDisk) doVolumeCheckDisk(minuend, subtrahend *needle_m
 	// hash join, can be more efficient
 	var missingNeedles []needle_map.NeedleValue
 	var counter int
-	subtrahend.AscendingVisit(func(value needle_map.NeedleValue) error {
+	minuend.AscendingVisit(func(value needle_map.NeedleValue) error {
 		counter++
-		if _, found := minuend.Get(value.Key); !found {
+		if _, found := subtrahend.Get(value.Key); !found {
 			missingNeedles = append(missingNeedles, value)
 		}
 		return nil

weed/shell/command_volume_configure_replication.go

@@ -59,7 +59,7 @@ func (c *commandVolumeConfigureReplication) Do(args []string, commandEnv *Comman
 	}
 
 	// collect topology information
-	topologyInfo, _, err := collectTopologyInfo(commandEnv)
+	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
 	if err != nil {
 		return err
 	}

weed/shell/command_volume_delete_empty.go

@@ -45,7 +45,7 @@ func (c *commandVolumeDeleteEmpty) Do(args []string, commandEnv *CommandEnv, wri
 	}
 
 	// collect topology information
-	topologyInfo, _, err := collectTopologyInfo(commandEnv)
+	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
 	if err != nil {
 		return err
 	}

weed/shell/command_volume_fix_replication.go

@@ -75,7 +75,7 @@ func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv,
 		fixedVolumeReplicas := map[string]int{}
 
 		// collect topology information
-		topologyInfo, _, err := collectTopologyInfo(commandEnv)
+		topologyInfo, _, err := collectTopologyInfo(commandEnv, 15*time.Second)
 		if err != nil {
 			return err
 		}

weed/shell/command_volume_fsck.go

@@ -455,7 +455,7 @@ func (c *commandVolumeFsck) collectVolumeIds(commandEnv *CommandEnv, verbose boo
 
 	volumeIdToServer = make(map[uint32]VInfo)
 	// collect topology information
-	topologyInfo, _, err := collectTopologyInfo(commandEnv)
+	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
 	if err != nil {
 		return
 	}

weed/shell/command_volume_list.go

@@ -39,7 +39,7 @@ func (c *commandVolumeList) Do(args []string, commandEnv *CommandEnv, writer io.
 	}
 
 	// collect topology information
-	topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv)
+	topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 0)
 	if err != nil {
 		return err
 	}

weed/shell/command_volume_list_test.go

@@ -1,6 +1,7 @@
 package shell
 
 import (
+	_ "embed"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
 	"github.com/golang/protobuf/proto"
 	"github.com/stretchr/testify/assert"
@@ -64,11 +65,14 @@ func parseOutput(output string) *master_pb.TopologyInfo {
 		case "Disk":
 			if disk == nil {
 				diskType := parts[1][:strings.Index(parts[1], "(")]
+				volumeCountStr := parts[1][strings.Index(parts[1], ":")+1 : strings.Index(parts[1], "/")]
 				maxVolumeCountStr := parts[1][strings.Index(parts[1], "/")+1:]
 				maxVolumeCount, _ := strconv.Atoi(maxVolumeCountStr)
+				volumeCount, _ := strconv.Atoi(volumeCountStr)
 				disk = &master_pb.DiskInfo{
 					Type:           diskType,
 					MaxVolumeCount: int64(maxVolumeCount),
+					VolumeCount:    int64(volumeCount),
 				}
 				dn.DiskInfos[types.ToDiskType(diskType).String()] = disk
 			} else {
@@ -85,809 +89,5 @@ func parseOutput(output string) *master_pb.TopologyInfo {
 	return topo
 }
 
-const topoData = `
-Topology volumeSizeLimit:1024 MB hdd(volume:760/7280 active:760 free:6520 remote:0)
-  DataCenter dc1 hdd(volume:0/0 active:0 free:0 remote:0)
-    Rack DefaultRack hdd(volume:0/0 active:0 free:0 remote:0)
-    Rack DefaultRack total size:0 file_count:0 
-  DataCenter dc1 total size:0 file_count:0 
-  DataCenter dc2 hdd(volume:86/430 active:86 free:344 remote:0)
-    Rack rack1 hdd(volume:50/240 active:50 free:190 remote:0)
-      DataNode 192.168.1.4:8080 hdd(volume:50/240 active:50 free:190 remote:0)
-        Disk hdd(volume:50/240 active:50 free:190 remote:0)
-          volume id:15  size:1115965064  collection:"collection0"  file_count:83  replica_placement:100  version:3  modified_at_second:1609923671 
-          volume id:21  size:1097631536  collection:"collection0"  file_count:82  delete_count:7  deleted_byte_count:68975485  replica_placement:100  version:3  modified_at_second:1609929578 
-          volume id:22  size:1086828272  collection:"collection0"  file_count:75  replica_placement:100  version:3  modified_at_second:1609930001 
-          volume id:23  size:1076380216  collection:"collection0"  file_count:68  replica_placement:100  version:3  modified_at_second:1609930434 
-          volume id:24  size:1074139776  collection:"collection0"  file_count:90  replica_placement:100  version:3  modified_at_second:1609930909 
-          volume id:25  size:690757512  collection:"collection0"  file_count:38  replica_placement:100  version:3  modified_at_second:1611144216 
-          volume id:27  size:298886792  file_count:1608  replica_placement:100  version:3  modified_at_second:1615632482 
-          volume id:28  size:308919192  file_count:1591  delete_count:1  deleted_byte_count:125280  replica_placement:100  version:3  modified_at_second:1615631762 
-          volume id:29  size:281582680  file_count:1537  replica_placement:100  version:3  modified_at_second:1615629422 
-          volume id:30  size:289466144  file_count:1566  delete_count:1  deleted_byte_count:124972  replica_placement:100  version:3  modified_at_second:1615632422 
-          volume id:31  size:273363256  file_count:1498  replica_placement:100  version:3  modified_at_second:1615631642 
-          volume id:33  size:1130226400  collection:"collection1"  file_count:7322  delete_count:172  deleted_byte_count:45199399  replica_placement:100  version:3  modified_at_second:1615618789 
-          volume id:38  size:1075545744  collection:"collection1"  file_count:13324  delete_count:100  deleted_byte_count:25223906  replica_placement:100  version:3  modified_at_second:1615569830 
-          volume id:51  size:1076796120  collection:"collection1"  file_count:10550  delete_count:39  deleted_byte_count:12723654  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615547786 
-          volume id:52  size:1083529728  collection:"collection1"  file_count:10128  delete_count:32  deleted_byte_count:10608391  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615599195 
-          volume id:54  size:1045022344  collection:"collection1"  file_count:9408  delete_count:30  deleted_byte_count:15132106  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630812 
-          volume id:63  size:956941112  collection:"collection1"  file_count:8271  delete_count:32  deleted_byte_count:15876189  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632036 
-          volume id:69  size:869213648  collection:"collection1"  file_count:7293  delete_count:102  deleted_byte_count:30643207  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630534 
-          volume id:74  size:957046128  collection:"collection1"  file_count:6982  delete_count:258  deleted_byte_count:73054259  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615631460 
-          volume id:80  size:827912928  collection:"collection1"  file_count:6914  delete_count:17  deleted_byte_count:5689635  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615631157 
-          volume id:84  size:873121856  collection:"collection1"  file_count:8200  delete_count:13  deleted_byte_count:3131676  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631161 
-          volume id:85  size:1023869320  collection:"collection1"  file_count:7788  delete_count:234  deleted_byte_count:78037967  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631723 
-          volume id:97  size:1053112992  collection:"collection1"  file_count:6789  delete_count:50  deleted_byte_count:38894001  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631193 
-          volume id:98  size:1077836440  collection:"collection1"  file_count:7605  delete_count:202  deleted_byte_count:73180379  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615523691 
-          volume id:105  size:1073996824  collection:"collection1"  file_count:6872  delete_count:20  deleted_byte_count:14482293  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615499757 
-          volume id:106  size:1075458664  collection:"collection1"  file_count:7182  delete_count:307  deleted_byte_count:69349053  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615598137 
-          volume id:112  size:1076392512  collection:"collection1"  file_count:8291  delete_count:156  deleted_byte_count:74120183  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615569823 
-          volume id:116  size:1074489504  collection:"collection1"  file_count:9981  delete_count:174  deleted_byte_count:53998777  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615611565 
-          volume id:119  size:1075940104  collection:"collection1"  file_count:9003  delete_count:12  deleted_byte_count:9128155  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615573878 
-          volume id:128  size:1074874632  collection:"collection1"  file_count:9821  delete_count:148  deleted_byte_count:43633334  replica_placement:100  version:3  modified_at_second:1615602670 
-          volume id:133  size:1075952760  collection:"collection1"  file_count:9538  delete_count:74  deleted_byte_count:19558008  replica_placement:100  version:3  modified_at_second:1615584779 
-          volume id:136  size:1074433552  collection:"collection1"  file_count:9593  delete_count:72  deleted_byte_count:26912512  replica_placement:100  version:3  modified_at_second:1615376036 
-          volume id:138  size:1074465744  collection:"collection1"  file_count:10120  delete_count:55  deleted_byte_count:15875438  replica_placement:100  version:3  modified_at_second:1615572231 
-          volume id:140  size:1076203744  collection:"collection1"  file_count:11219  delete_count:57  deleted_byte_count:19864498  replica_placement:100  version:3  modified_at_second:1615571947 
-          volume id:144  size:1074549720  collection:"collection1"  file_count:8780  delete_count:50  deleted_byte_count:52475146  replica_placement:100  version:3  modified_at_second:1615573451 
-          volume id:161  size:1077397192  collection:"collection1"  file_count:9988  delete_count:28  deleted_byte_count:12509164  replica_placement:100  version:3  modified_at_second:1615631452 
-          volume id:173  size:1074154704  collection:"collection1"  file_count:30884  delete_count:34  deleted_byte_count:2578509  replica_placement:100  version:3  modified_at_second:1615591904 
-          volume id:174  size:1073824232  collection:"collection1"  file_count:30689  delete_count:36  deleted_byte_count:2160116  replica_placement:100  version:3  modified_at_second:1615598914 
-          volume id:197  size:1075423240  collection:"collection1"  file_count:16473  delete_count:15  deleted_byte_count:12552442  replica_placement:100  version:3  modified_at_second:1615485254 
-          volume id:219  size:1092298904  collection:"collection1"  file_count:3193  delete_count:17  deleted_byte_count:2047576  replica_placement:100  version:3  modified_at_second:1615579316 
-          volume id:263  size:1077167352  collection:"collection2"  file_count:20227  delete_count:4  deleted_byte_count:97887  replica_placement:100  version:3  modified_at_second:1614871567 
-          volume id:272  size:1076146040  collection:"collection2"  file_count:21034  delete_count:2  deleted_byte_count:216564  replica_placement:100  version:3  modified_at_second:1614884139 
-          volume id:291  size:1076256760  collection:"collection2"  file_count:28301  delete_count:5  deleted_byte_count:116027  replica_placement:100  version:3  modified_at_second:1614904924 
-          volume id:299  size:1075147824  collection:"collection2"  file_count:22927  delete_count:4  deleted_byte_count:345569  replica_placement:100  version:3  modified_at_second:1614918454 
-          volume id:301  size:1074655600  collection:"collection2"  file_count:22543  delete_count:6  deleted_byte_count:136968  replica_placement:100  version:3  modified_at_second:1614918378 
-          volume id:302  size:1077559792  collection:"collection2"  file_count:23124  delete_count:7  deleted_byte_count:293111  replica_placement:100  version:3  modified_at_second:1614925500 
-          volume id:339  size:1078402392  collection:"collection2"  file_count:22309  replica_placement:100  version:3  modified_at_second:1614969996 
-          volume id:345  size:1074560760  collection:"collection2"  file_count:22117  delete_count:2  deleted_byte_count:373286  replica_placement:100  version:3  modified_at_second:1614977458 
-          volume id:355  size:1075239792  collection:"collection2"  file_count:22244  delete_count:1  deleted_byte_count:23282  replica_placement:100  version:3  modified_at_second:1614992157 
-          volume id:373  size:1080928000  collection:"collection2"  file_count:22617  delete_count:4  deleted_byte_count:91849  replica_placement:100  version:3  modified_at_second:1615016877 
-        Disk hdd total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253 
-      DataNode 192.168.1.4:8080 total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253 
-    Rack rack1 total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253 
-    Rack rack2 hdd(volume:36/190 active:36 free:154 remote:0)
-      DataNode 192.168.1.2:8080 hdd(volume:36/190 active:36 free:154 remote:0)
-        Disk hdd(volume:36/190 active:36 free:154 remote:0)
-          volume id:2  size:289228560  file_count:1640  delete_count:4  deleted_byte_count:480564  replica_placement:100  version:3  compact_revision:6  modified_at_second:1615630622 
-          volume id:3  size:308743136  file_count:1638  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632242 
-          volume id:4  size:285986968  file_count:1641  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632302 
-          volume id:6  size:302411024  file_count:1604  delete_count:2  deleted_byte_count:274587  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631402 
-          volume id:7  size:1924728  collection:"collection4"  file_count:15  replica_placement:100  version:3  modified_at_second:1609331040 
-          volume id:9  size:77337416  collection:"collection3"  file_count:58  replica_placement:100  version:3  ttl:772  modified_at_second:1615513762 
-          volume id:10  size:1212784656  collection:"collection0"  file_count:58  replica_placement:100  version:3  modified_at_second:1609814550 
-          volume id:12  size:1110923848  collection:"collection0"  file_count:45  replica_placement:100  version:3  modified_at_second:1609819732 
-          volume id:13  size:1184910656  collection:"collection0"  file_count:47  replica_placement:100  version:3  modified_at_second:1609827837 
-          volume id:14  size:1107475720  collection:"collection0"  file_count:80  delete_count:3  deleted_byte_count:6870  replica_placement:100  version:3  modified_at_second:1612956980 
-          volume id:16  size:1113666104  collection:"collection0"  file_count:73  delete_count:5  deleted_byte_count:6318  replica_placement:100  version:3  modified_at_second:1612957007 
-          volume id:17  size:1095115800  collection:"collection0"  file_count:83  delete_count:3  deleted_byte_count:7099  replica_placement:100  version:3  modified_at_second:1612957000 
-          volume id:21  size:1097631664  collection:"collection0"  file_count:82  delete_count:11  deleted_byte_count:68985100  replica_placement:100  version:3  modified_at_second:1612957007 
-          volume id:56  size:1001897616  collection:"collection1"  file_count:8762  delete_count:37  deleted_byte_count:65375405  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615632014 
-          volume id:81  size:880693104  collection:"collection1"  file_count:7481  delete_count:236  deleted_byte_count:80386421  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615631396 
-          volume id:104  size:1076383624  collection:"collection1"  file_count:7663  delete_count:184  deleted_byte_count:100728071  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615602658 
-          volume id:107  size:1073811840  collection:"collection1"  file_count:7436  delete_count:168  deleted_byte_count:57747484  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615293569 
-          volume id:113  size:1076709184  collection:"collection1"  file_count:9355  delete_count:177  deleted_byte_count:59796765  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615569822 
-          volume id:139  size:1074163936  collection:"collection1"  file_count:9315  delete_count:42  deleted_byte_count:10630966  replica_placement:100  version:3  modified_at_second:1615571946 
-          volume id:151  size:1098659752  collection:"collection1"  file_count:10808  delete_count:24  deleted_byte_count:7088102  replica_placement:100  version:3  modified_at_second:1615586389 
-          volume id:155  size:1075140688  collection:"collection1"  file_count:10882  delete_count:32  deleted_byte_count:9076141  replica_placement:100  version:3  modified_at_second:1615606614 
-          volume id:167  size:1073958176  collection:"collection1"  file_count:25229  delete_count:48  deleted_byte_count:25871565  replica_placement:100  version:3  modified_at_second:1615602669 
-          volume id:177  size:1074120216  collection:"collection1"  file_count:22293  delete_count:16  deleted_byte_count:3803952  replica_placement:100  version:3  modified_at_second:1615516892 
-          volume id:179  size:1074313920  collection:"collection1"  file_count:21829  delete_count:24  deleted_byte_count:45552859  replica_placement:100  version:3  modified_at_second:1615580308 
-          volume id:182  size:1076131280  collection:"collection1"  file_count:31987  delete_count:21  deleted_byte_count:1452346  replica_placement:100  version:3  modified_at_second:1615568922 
-          volume id:215  size:1068268216  collection:"collection1"  file_count:2813  delete_count:10  deleted_byte_count:5676795  replica_placement:100  version:3  modified_at_second:1615586386 
-          volume id:217  size:1075381872  collection:"collection1"  file_count:3331  delete_count:14  deleted_byte_count:2009141  replica_placement:100  version:3  modified_at_second:1615401638 
-          volume id:283  size:1080178944  collection:"collection2"  file_count:19462  delete_count:7  deleted_byte_count:660407  replica_placement:100  version:3  modified_at_second:1614896626 
-          volume id:303  size:1075944504  collection:"collection2"  file_count:22541  delete_count:2  deleted_byte_count:13617  replica_placement:100  version:3  modified_at_second:1614925431 
-          volume id:309  size:1075178624  collection:"collection2"  file_count:22692  delete_count:3  deleted_byte_count:171124  replica_placement:100  version:3  modified_at_second:1614931409 
-          volume id:323  size:1074608200  collection:"collection2"  file_count:21605  delete_count:4  deleted_byte_count:172090  replica_placement:100  version:3  modified_at_second:1614950526 
-          volume id:344  size:1075035448  collection:"collection2"  file_count:21765  delete_count:1  deleted_byte_count:24623  replica_placement:100  version:3  modified_at_second:1614977465 
-          volume id:347  size:1075145496  collection:"collection2"  file_count:22178  delete_count:1  deleted_byte_count:79392  replica_placement:100  version:3  modified_at_second:1614984727 
-          volume id:357  size:1074276208  collection:"collection2"  file_count:23137  delete_count:4  deleted_byte_count:188487  replica_placement:100  version:3  modified_at_second:1614998792 
-          volume id:380  size:1010760456  collection:"collection2"  file_count:14921  delete_count:6  deleted_byte_count:65678  replica_placement:100  version:3  modified_at_second:1615632322 
-          volume id:381  size:939292792  collection:"collection2"  file_count:14619  delete_count:2  deleted_byte_count:5119  replica_placement:100  version:3  modified_at_second:1615632324 
-        Disk hdd total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088 
-      DataNode 192.168.1.2:8080 total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088 
-    Rack rack2 total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088 
-  DataCenter dc2 total size:82098209920 file_count:907048 deleted_file:3671 deleted_bytes:1475897341 
-  DataCenter dc3 hdd(volume:108/1850 active:108 free:1742 remote:0)
-    Rack rack3 hdd(volume:108/1850 active:108 free:1742 remote:0)
-      DataNode 192.168.1.6:8080 hdd(volume:108/1850 active:108 free:1742 remote:0)
-        Disk hdd(volume:108/1850 active:108 free:1742 remote:0)
-          volume id:1  size:284685936  file_count:1557  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632062 
-          volume id:32  size:281390512  file_count:1496  delete_count:6  deleted_byte_count:546403  replica_placement:100  version:3  modified_at_second:1615632362 
-          volume id:47  size:444599784  collection:"collection1"  file_count:709  delete_count:19  deleted_byte_count:11913451  replica_placement:100  version:3  modified_at_second:1615632397 
-          volume id:49  size:1078775288  collection:"collection1"  file_count:9636  delete_count:22  deleted_byte_count:5625976  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630446 
-          volume id:68  size:898630584  collection:"collection1"  file_count:6934  delete_count:95  deleted_byte_count:27460707  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632284 
-          volume id:88  size:1073767976  collection:"collection1"  file_count:14995  delete_count:206  deleted_byte_count:81222360  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615629897 
-          volume id:202  size:1077533160  collection:"collection1"  file_count:2847  delete_count:67  deleted_byte_count:65172985  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615588497 
-          volume id:203  size:1027316272  collection:"collection1"  file_count:3040  delete_count:11  deleted_byte_count:3993230  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615631728 
-          volume id:205  size:1078485304  collection:"collection1"  file_count:2869  delete_count:43  deleted_byte_count:18290259  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615579314 
-          volume id:206  size:1082045848  collection:"collection1"  file_count:2979  delete_count:225  deleted_byte_count:88220074  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615564274 
-          volume id:209  size:1074083592  collection:"collection1"  file_count:3238  delete_count:4  deleted_byte_count:1494244  replica_placement:100  version:3  modified_at_second:1615419954 
-          volume id:211  size:1080610712  collection:"collection1"  file_count:3247  delete_count:7  deleted_byte_count:1891456  replica_placement:100  version:3  modified_at_second:1615269124 
-          volume id:212  size:1078293360  collection:"collection1"  file_count:3106  delete_count:6  deleted_byte_count:2085755  replica_placement:100  version:3  modified_at_second:1615586387 
-          volume id:213  size:1093587976  collection:"collection1"  file_count:3681  delete_count:12  deleted_byte_count:3138791  replica_placement:100  version:3  modified_at_second:1615586387 
-          volume id:214  size:1074486992  collection:"collection1"  file_count:3217  delete_count:10  deleted_byte_count:6392871  replica_placement:100  version:3  modified_at_second:1615586383 
-          volume id:216  size:1080073496  collection:"collection1"  file_count:3316  delete_count:4  deleted_byte_count:179819  replica_placement:100  version:3  modified_at_second:1615586387 
-          volume id:222  size:1106623104  collection:"collection1"  file_count:3273  delete_count:11  deleted_byte_count:2114627  replica_placement:100  version:3  modified_at_second:1615586243 
-          volume id:223  size:1075233064  collection:"collection1"  file_count:2966  delete_count:9  deleted_byte_count:744001  replica_placement:100  version:3  modified_at_second:1615586244 
-          volume id:227  size:1106699896  collection:"collection1"  file_count:2827  delete_count:20  deleted_byte_count:5496790  replica_placement:100  version:3  modified_at_second:1615609989 
-          volume id:229  size:1109855312  collection:"collection1"  file_count:2857  delete_count:22  deleted_byte_count:2839883  replica_placement:100  version:3  modified_at_second:1615609988 
-          volume id:230  size:1080722984  collection:"collection1"  file_count:2898  delete_count:15  deleted_byte_count:3929261  replica_placement:100  version:3  modified_at_second:1615610537 
-          volume id:231  size:1112917696  collection:"collection1"  file_count:3151  delete_count:20  deleted_byte_count:2989828  replica_placement:100  version:3  modified_at_second:1615611350 
-          volume id:233  size:1080526464  collection:"collection1"  file_count:3136  delete_count:61  deleted_byte_count:17991717  replica_placement:100  version:3  modified_at_second:1615611352 
-          volume id:234  size:1073835280  collection:"collection1"  file_count:2965  delete_count:41  deleted_byte_count:4960354  replica_placement:100  version:3  modified_at_second:1615611351 
-          volume id:235  size:1075586104  collection:"collection1"  file_count:2767  delete_count:33  deleted_byte_count:3216540  replica_placement:100  version:3  modified_at_second:1615611354 
-          volume id:237  size:375722792  collection:"collection1"  file_count:736  delete_count:16  deleted_byte_count:4464870  replica_placement:100  version:3  modified_at_second:1615631727 
-          volume id:239  size:426569024  collection:"collection1"  file_count:693  delete_count:19  deleted_byte_count:13020783  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630838 
-          volume id:241  size:380217424  collection:"collection1"  file_count:633  delete_count:6  deleted_byte_count:1715768  replica_placement:100  version:3  modified_at_second:1615632006 
-          volume id:244  size:1080295352  collection:"collection2"  file_count:10812  delete_count:1  deleted_byte_count:795  replica_placement:100  version:3  modified_at_second:1614852171 
-          volume id:245  size:1074597056  collection:"collection2"  file_count:10371  delete_count:3  deleted_byte_count:209701  replica_placement:100  version:3  modified_at_second:1614852093 
-          volume id:246  size:1075998648  collection:"collection2"  file_count:10365  delete_count:1  deleted_byte_count:13112  replica_placement:100  version:3  modified_at_second:1614852105 
-          volume id:248  size:1084301184  collection:"collection2"  file_count:11217  delete_count:4  deleted_byte_count:746488  replica_placement:100  version:3  modified_at_second:1614856285 
-          volume id:249  size:1074819136  collection:"collection2"  file_count:10763  delete_count:2  deleted_byte_count:271699  replica_placement:100  version:3  modified_at_second:1614856230 
-          volume id:251  size:1075684488  collection:"collection2"  file_count:10847  replica_placement:100  version:3  modified_at_second:1614856270 
-          volume id:252  size:1075065208  collection:"collection2"  file_count:14622  delete_count:2  deleted_byte_count:5228  replica_placement:100  version:3  modified_at_second:1614861196 
-          volume id:253  size:1087328816  collection:"collection2"  file_count:14920  delete_count:3  deleted_byte_count:522994  replica_placement:100  version:3  modified_at_second:1614861255 
-          volume id:255  size:1079581640  collection:"collection2"  file_count:14877  delete_count:3  deleted_byte_count:101223  replica_placement:100  version:3  modified_at_second:1614861233 
-          volume id:256  size:1074283592  collection:"collection2"  file_count:14157  delete_count:1  deleted_byte_count:18156  replica_placement:100  version:3  modified_at_second:1614861100 
-          volume id:258  size:1075527216  collection:"collection2"  file_count:18421  delete_count:4  deleted_byte_count:267833  replica_placement:100  version:3  modified_at_second:1614866420 
-          volume id:259  size:1075507776  collection:"collection2"  file_count:18079  delete_count:2  deleted_byte_count:71992  replica_placement:100  version:3  modified_at_second:1614866381 
-          volume id:264  size:1081624192  collection:"collection2"  file_count:21151  replica_placement:100  version:3  modified_at_second:1614871629 
-          volume id:265  size:1076401104  collection:"collection2"  file_count:19932  delete_count:2  deleted_byte_count:160823  replica_placement:100  version:3  modified_at_second:1615629130 
-          volume id:266  size:1075617464  collection:"collection2"  file_count:20075  delete_count:1  deleted_byte_count:1039  replica_placement:100  version:3  modified_at_second:1614871526 
-          volume id:267  size:1075699544  collection:"collection2"  file_count:21039  delete_count:3  deleted_byte_count:59956  replica_placement:100  version:3  modified_at_second:1614877294 
-          volume id:268  size:1074490592  collection:"collection2"  file_count:21698  delete_count:1  deleted_byte_count:33968  replica_placement:100  version:3  modified_at_second:1614877434 
-          volume id:269  size:1077552872  collection:"collection2"  file_count:21875  delete_count:4  deleted_byte_count:347272  replica_placement:100  version:3  modified_at_second:1614877481 
-          volume id:270  size:1076876568  collection:"collection2"  file_count:22057  delete_count:1  deleted_byte_count:43916  replica_placement:100  version:3  modified_at_second:1614877469 
-          volume id:275  size:1078349024  collection:"collection2"  file_count:20808  delete_count:1  deleted_byte_count:1118  replica_placement:100  version:3  modified_at_second:1614884147 
-          volume id:277  size:1074956288  collection:"collection2"  file_count:19260  delete_count:2  deleted_byte_count:172356  replica_placement:100  version:3  modified_at_second:1614889988 
-          volume id:278  size:1078798640  collection:"collection2"  file_count:20597  delete_count:5  deleted_byte_count:400060  replica_placement:100  version:3  modified_at_second:1614890292 
-          volume id:279  size:1077325040  collection:"collection2"  file_count:19671  delete_count:6  deleted_byte_count:379116  replica_placement:100  version:3  modified_at_second:1614890229 
-          volume id:280  size:1077432216  collection:"collection2"  file_count:20286  delete_count:1  deleted_byte_count:879  replica_placement:100  version:3  modified_at_second:1614890262 
-          volume id:281  size:1077581096  collection:"collection2"  file_count:20206  delete_count:3  deleted_byte_count:143964  replica_placement:100  version:3  modified_at_second:1614890237 
-          volume id:284  size:1074533384  collection:"collection2"  file_count:22196  delete_count:4  deleted_byte_count:154683  replica_placement:100  version:3  modified_at_second:1614897231 
-          volume id:285  size:1082128688  collection:"collection2"  file_count:21804  delete_count:1  deleted_byte_count:1064  replica_placement:100  version:3  modified_at_second:1614897165 
-          volume id:289  size:1075284256  collection:"collection2"  file_count:29342  delete_count:5  deleted_byte_count:100454  replica_placement:100  version:3  modified_at_second:1614904977 
-          volume id:290  size:1074723792  collection:"collection2"  file_count:28340  delete_count:4  deleted_byte_count:199064  replica_placement:100  version:3  modified_at_second:1614904924 
-          volume id:291  size:1076256768  collection:"collection2"  file_count:28301  delete_count:5  deleted_byte_count:116027  replica_placement:100  version:3  modified_at_second:1614904924 
-          volume id:293  size:1075409792  collection:"collection2"  file_count:26063  delete_count:4  deleted_byte_count:183834  replica_placement:100  version:3  modified_at_second:1614912235 
-          volume id:294  size:1075444048  collection:"collection2"  file_count:26076  delete_count:4  deleted_byte_count:194914  replica_placement:100  version:3  modified_at_second:1614912220 
-          volume id:296  size:1077824032  collection:"collection2"  file_count:26741  delete_count:4  deleted_byte_count:199906  replica_placement:100  version:3  modified_at_second:1614912301 
-          volume id:297  size:1080229136  collection:"collection2"  file_count:23409  delete_count:5  deleted_byte_count:46268  replica_placement:100  version:3  modified_at_second:1614918481 
-          volume id:298  size:1075410136  collection:"collection2"  file_count:23222  delete_count:2  deleted_byte_count:46110  replica_placement:100  version:3  modified_at_second:1614918474 
-          volume id:299  size:1075147936  collection:"collection2"  file_count:22927  delete_count:4  deleted_byte_count:345569  replica_placement:100  version:3  modified_at_second:1614918455 
-          volume id:300  size:1076212392  collection:"collection2"  file_count:22892  delete_count:2  deleted_byte_count:61320  replica_placement:100  version:3  modified_at_second:1614918464 
-          volume id:301  size:1074655600  collection:"collection2"  file_count:22543  delete_count:6  deleted_byte_count:136968  replica_placement:100  version:3  modified_at_second:1614918378 
-          volume id:303  size:1075944480  collection:"collection2"  file_count:22541  delete_count:2  deleted_byte_count:13617  replica_placement:100  version:3  modified_at_second:1614925431 
-          volume id:306  size:1074764016  collection:"collection2"  file_count:22939  replica_placement:100  version:3  modified_at_second:1614925462 
-          volume id:307  size:1076568000  collection:"collection2"  file_count:23377  delete_count:2  deleted_byte_count:25453  replica_placement:100  version:3  modified_at_second:1614931448 
-          volume id:308  size:1074022392  collection:"collection2"  file_count:23086  delete_count:2  deleted_byte_count:2127  replica_placement:100  version:3  modified_at_second:1614931401 
-          volume id:309  size:1075178664  collection:"collection2"  file_count:22692  delete_count:3  deleted_byte_count:171124  replica_placement:100  version:3  modified_at_second:1614931409 
-          volume id:310  size:1074761528  collection:"collection2"  file_count:21441  delete_count:3  deleted_byte_count:13934  replica_placement:100  version:3  modified_at_second:1614931077 
-          volume id:314  size:1074670840  collection:"collection2"  file_count:20964  delete_count:4  deleted_byte_count:304291  replica_placement:100  version:3  modified_at_second:1614937441 
-          volume id:315  size:1084153544  collection:"collection2"  file_count:23638  delete_count:2  deleted_byte_count:53956  replica_placement:100  version:3  modified_at_second:1614937885 
-          volume id:317  size:1076215096  collection:"collection2"  file_count:23572  delete_count:2  deleted_byte_count:1441356  replica_placement:100  version:3  modified_at_second:1614943965 
-          volume id:318  size:1075965168  collection:"collection2"  file_count:22459  delete_count:2  deleted_byte_count:37778  replica_placement:100  version:3  modified_at_second:1614943862 
-          volume id:319  size:1073952880  collection:"collection2"  file_count:22286  delete_count:2  deleted_byte_count:43421  replica_placement:100  version:3  modified_at_second:1614943810 
-          volume id:320  size:1082437792  collection:"collection2"  file_count:21544  delete_count:3  deleted_byte_count:16712  replica_placement:100  version:3  modified_at_second:1614943599 
-          volume id:321  size:1081477904  collection:"collection2"  file_count:23531  delete_count:5  deleted_byte_count:262564  replica_placement:100  version:3  modified_at_second:1614943982 
-          volume id:324  size:1075606680  collection:"collection2"  file_count:20799  delete_count:1  deleted_byte_count:251210  replica_placement:100  version:3  modified_at_second:1614950310 
-          volume id:325  size:1080701144  collection:"collection2"  file_count:21735  replica_placement:100  version:3  modified_at_second:1614950525 
-          volume id:330  size:1080825832  collection:"collection2"  file_count:22464  delete_count:2  deleted_byte_count:15771  replica_placement:100  version:3  modified_at_second:1614956477 
-          volume id:332  size:1075569928  collection:"collection2"  file_count:22097  delete_count:3  deleted_byte_count:98273  replica_placement:100  version:3  modified_at_second:1614962869 
-          volume id:334  size:1075607880  collection:"collection2"  file_count:22546  delete_count:6  deleted_byte_count:101538  replica_placement:100  version:3  modified_at_second:1614962978 
-          volume id:336  size:1087853056  collection:"collection2"  file_count:22801  delete_count:2  deleted_byte_count:26394  replica_placement:100  version:3  modified_at_second:1614963005 
-          volume id:337  size:1075646784  collection:"collection2"  file_count:21934  delete_count:1  deleted_byte_count:3397  replica_placement:100  version:3  modified_at_second:1614969937 
-          volume id:338  size:1076118304  collection:"collection2"  file_count:21680  replica_placement:100  version:3  modified_at_second:1614969850 
-          volume id:340  size:1079462184  collection:"collection2"  file_count:22319  delete_count:4  deleted_byte_count:93620  replica_placement:100  version:3  modified_at_second:1614969978 
-          volume id:341  size:1074448400  collection:"collection2"  file_count:21590  delete_count:5  deleted_byte_count:160085  replica_placement:100  version:3  modified_at_second:1614969858 
-          volume id:342  size:1080186424  collection:"collection2"  file_count:22405  delete_count:4  deleted_byte_count:64819  replica_placement:100  version:3  modified_at_second:1614977521 
-          volume id:344  size:1075035416  collection:"collection2"  file_count:21765  delete_count:1  deleted_byte_count:24623  replica_placement:100  version:3  modified_at_second:1614977465 
-          volume id:345  size:1074560760  collection:"collection2"  file_count:22117  delete_count:2  deleted_byte_count:373286  replica_placement:100  version:3  modified_at_second:1614977457 
-          volume id:346  size:1076464112  collection:"collection2"  file_count:22320  delete_count:4  deleted_byte_count:798258  replica_placement:100  version:3  modified_at_second:1615631322 
-          volume id:348  size:1080623640  collection:"collection2"  file_count:21667  delete_count:1  deleted_byte_count:2443  replica_placement:100  version:3  modified_at_second:1614984606 
-          volume id:350  size:1074756688  collection:"collection2"  file_count:21990  delete_count:3  deleted_byte_count:233881  replica_placement:100  version:3  modified_at_second:1614984682 
-          volume id:351  size:1078795112  collection:"collection2"  file_count:23660  delete_count:3  deleted_byte_count:102141  replica_placement:100  version:3  modified_at_second:1614984816 
-          volume id:352  size:1077145936  collection:"collection2"  file_count:22066  delete_count:1  deleted_byte_count:1018  replica_placement:100  version:3  modified_at_second:1614992130 
-          volume id:353  size:1074897496  collection:"collection2"  file_count:21266  delete_count:2  deleted_byte_count:3105374  replica_placement:100  version:3  modified_at_second:1614991951 
-          volume id:355  size:1075239728  collection:"collection2"  file_count:22244  delete_count:1  deleted_byte_count:23282  replica_placement:100  version:3  modified_at_second:1614992157 
-          volume id:356  size:1083305048  collection:"collection2"  file_count:21552  delete_count:4  deleted_byte_count:14472  replica_placement:100  version:3  modified_at_second:1614992028 
-          volume id:358  size:1085152368  collection:"collection2"  file_count:23756  delete_count:3  deleted_byte_count:44531  replica_placement:100  version:3  modified_at_second:1614998824 
-          volume id:360  size:1075532456  collection:"collection2"  file_count:22574  delete_count:3  deleted_byte_count:1774776  replica_placement:100  version:3  modified_at_second:1614998770 
-          volume id:361  size:1075362744  collection:"collection2"  file_count:22272  delete_count:1  deleted_byte_count:3497  replica_placement:100  version:3  modified_at_second:1614998668 
-          volume id:375  size:1076140568  collection:"collection2"  file_count:21880  delete_count:2  deleted_byte_count:51103  replica_placement:100  version:3  modified_at_second:1615016787 
-          volume id:376  size:1074845944  collection:"collection2"  file_count:22908  delete_count:4  deleted_byte_count:432305  replica_placement:100  version:3  modified_at_second:1615016916 
-          volume id:377  size:957284144  collection:"collection2"  file_count:14923  delete_count:1  deleted_byte_count:1797  replica_placement:100  version:3  modified_at_second:1615632323 
-          volume id:378  size:959273936  collection:"collection2"  file_count:15027  delete_count:4  deleted_byte_count:231414  replica_placement:100  version:3  modified_at_second:1615632323 
-          volume id:381  size:939261032  collection:"collection2"  file_count:14615  delete_count:5  deleted_byte_count:1192272  replica_placement:100  version:3  modified_at_second:1615632324 
-        Disk hdd total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 
-      DataNode 192.168.1.6:8080 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 
-    Rack rack3 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 
-  DataCenter dc3 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 
-  DataCenter dc4 hdd(volume:267/2000 active:267 free:1733 remote:0)
-    Rack DefaultRack hdd(volume:267/2000 active:267 free:1733 remote:0)
-      DataNode 192.168.1.1:8080 hdd(volume:267/2000 active:267 free:1733 remote:0)
-        Disk hdd(volume:267/2000 active:267 free:1733 remote:0)
-          volume id:1  size:284693256  file_count:1558  delete_count:2  deleted_byte_count:4818  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632062 
-          volume id:2  size:289228560  file_count:1640  delete_count:4  deleted_byte_count:464508  replica_placement:100  version:3  compact_revision:6  modified_at_second:1615630622 
-          volume id:3  size:308741952  file_count:1637  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632242 
-          volume id:4  size:285986968  file_count:1640  delete_count:1  deleted_byte_count:145095  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632302 
-          volume id:5  size:293806008  file_count:1669  delete_count:2  deleted_byte_count:274334  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631342 
-          volume id:6  size:302411024  file_count:1604  delete_count:2  deleted_byte_count:274587  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631402 
-          volume id:7  size:1924728  collection:"collection4"  file_count:15  replica_placement:100  version:3  modified_at_second:1609331040 
-          volume id:9  size:77337416  collection:"collection3"  file_count:58  replica_placement:100  version:3  ttl:772  modified_at_second:1615513762 
-          volume id:10  size:1212784656  collection:"collection0"  file_count:58  replica_placement:100  version:3  modified_at_second:1609814543 
-          volume id:11  size:1109224552  collection:"collection0"  file_count:44  replica_placement:100  version:3  modified_at_second:1609815123 
-          volume id:12  size:1110923848  collection:"collection0"  file_count:45  replica_placement:100  version:3  modified_at_second:1609819726 
-          volume id:13  size:1184910656  collection:"collection0"  file_count:47  replica_placement:100  version:3  modified_at_second:1609827832 
-          volume id:14  size:1107475720  collection:"collection0"  file_count:80  delete_count:3  deleted_byte_count:6870  replica_placement:100  version:3  modified_at_second:1612956983 
-          volume id:15  size:1115965160  collection:"collection0"  file_count:83  delete_count:3  deleted_byte_count:4956  replica_placement:100  version:3  modified_at_second:1612957001 
-          volume id:16  size:1113666048  collection:"collection0"  file_count:73  delete_count:5  deleted_byte_count:6318  replica_placement:100  version:3  modified_at_second:1612957007 
-          volume id:17  size:1095115800  collection:"collection0"  file_count:83  delete_count:3  deleted_byte_count:7099  replica_placement:100  version:3  modified_at_second:1612957000 
-          volume id:18  size:1096678688  collection:"collection0"  file_count:88  delete_count:4  deleted_byte_count:8633  replica_placement:100  version:3  modified_at_second:1612957000 
-          volume id:19  size:1096923792  collection:"collection0"  file_count:100  delete_count:10  deleted_byte_count:75779917  replica_placement:100  version:3  compact_revision:4  modified_at_second:1612957011 
-          volume id:20  size:1074760432  collection:"collection0"  file_count:82  delete_count:5  deleted_byte_count:12156  replica_placement:100  version:3  compact_revision:2  modified_at_second:1612957011 
-          volume id:22  size:1086828368  collection:"collection0"  file_count:75  delete_count:3  deleted_byte_count:5551  replica_placement:100  version:3  modified_at_second:1612957007 
-          volume id:23  size:1076380280  collection:"collection0"  file_count:68  delete_count:2  deleted_byte_count:2910  replica_placement:100  version:3  modified_at_second:1612957011 
-          volume id:24  size:1074139808  collection:"collection0"  file_count:90  delete_count:1  deleted_byte_count:1977  replica_placement:100  version:3  modified_at_second:1612957011 
-          volume id:25  size:690757544  collection:"collection0"  file_count:38  delete_count:1  deleted_byte_count:1944  replica_placement:100  version:3  modified_at_second:1612956995 
-          volume id:26  size:532657632  collection:"collection0"  file_count:100  delete_count:4  deleted_byte_count:9081  replica_placement:100  version:3  modified_at_second:1614170023 
-          volume id:34  size:1077111136  collection:"collection1"  file_count:9781  delete_count:110  deleted_byte_count:20894827  replica_placement:100  version:3  modified_at_second:1615619366 
-          volume id:35  size:1075241656  collection:"collection1"  file_count:10523  delete_count:96  deleted_byte_count:46618989  replica_placement:100  version:3  modified_at_second:1615618790 
-          volume id:36  size:1075118360  collection:"collection1"  file_count:10342  delete_count:116  deleted_byte_count:25493106  replica_placement:100  version:3  modified_at_second:1615606148 
-          volume id:37  size:1075895584  collection:"collection1"  file_count:12013  delete_count:98  deleted_byte_count:50747932  replica_placement:100  version:3  modified_at_second:1615594777 
-          volume id:39  size:1076606536  collection:"collection1"  file_count:12612  delete_count:78  deleted_byte_count:17462730  replica_placement:100  version:3  modified_at_second:1615611959 
-          volume id:40  size:1075358552  collection:"collection1"  file_count:12597  delete_count:62  deleted_byte_count:11657901  replica_placement:100  version:3  modified_at_second:1615612994 
-          volume id:41  size:1076283528  collection:"collection1"  file_count:12088  delete_count:84  deleted_byte_count:19319268  replica_placement:100  version:3  modified_at_second:1615596736 
-          volume id:42  size:1093948352  collection:"collection1"  file_count:7889  delete_count:47  deleted_byte_count:5697275  replica_placement:100  version:3  modified_at_second:1615548908 
-          volume id:43  size:1116445864  collection:"collection1"  file_count:7358  delete_count:54  deleted_byte_count:9534379  replica_placement:100  version:3  modified_at_second:1615566170 
-          volume id:44  size:1077582560  collection:"collection1"  file_count:7295  delete_count:50  deleted_byte_count:12618414  replica_placement:100  version:3  modified_at_second:1615566170 
-          volume id:45  size:1075254640  collection:"collection1"  file_count:10772  delete_count:76  deleted_byte_count:22426345  replica_placement:100  version:3  modified_at_second:1615573499 
-          volume id:46  size:1075286056  collection:"collection1"  file_count:9947  delete_count:309  deleted_byte_count:105601163  replica_placement:100  version:3  modified_at_second:1615569826 
-          volume id:48  size:1076778720  collection:"collection1"  file_count:9850  delete_count:77  deleted_byte_count:16641907  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630690 
-          volume id:50  size:1076688224  collection:"collection1"  file_count:7921  delete_count:26  deleted_byte_count:5162032  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615610879 
-          volume id:52  size:1083529704  collection:"collection1"  file_count:10128  delete_count:32  deleted_byte_count:10608391  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615599195 
-          volume id:53  size:1063089216  collection:"collection1"  file_count:9832  delete_count:31  deleted_byte_count:9273066  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632156 
-          volume id:55  size:1012890016  collection:"collection1"  file_count:8651  delete_count:27  deleted_byte_count:9418841  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631452 
-          volume id:57  size:839849792  collection:"collection1"  file_count:7514  delete_count:24  deleted_byte_count:6228543  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631774 
-          volume id:58  size:908064200  collection:"collection1"  file_count:8128  delete_count:21  deleted_byte_count:6113731  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632342 
-          volume id:59  size:988302272  collection:"collection1"  file_count:8098  delete_count:20  deleted_byte_count:3947615  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632238 
-          volume id:60  size:1010702480  collection:"collection1"  file_count:8969  delete_count:79  deleted_byte_count:24782814  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615632439 
-          volume id:61  size:975604488  collection:"collection1"  file_count:8683  delete_count:20  deleted_byte_count:10276072  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615631176 
-          volume id:62  size:873845936  collection:"collection1"  file_count:7897  delete_count:23  deleted_byte_count:10920170  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631133 
-          volume id:64  size:965638488  collection:"collection1"  file_count:8218  delete_count:27  deleted_byte_count:6922489  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631031 
-          volume id:65  size:823283552  collection:"collection1"  file_count:7834  delete_count:29  deleted_byte_count:5950610  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632306 
-          volume id:66  size:821343440  collection:"collection1"  file_count:7383  delete_count:29  deleted_byte_count:12010343  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631968 
-          volume id:67  size:878713872  collection:"collection1"  file_count:7299  delete_count:117  deleted_byte_count:24857326  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632156 
-          volume id:68  size:898630584  collection:"collection1"  file_count:6934  delete_count:95  deleted_byte_count:27460707  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632284 
-          volume id:70  size:886695472  collection:"collection1"  file_count:7769  delete_count:164  deleted_byte_count:45162513  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632398 
-          volume id:71  size:907608392  collection:"collection1"  file_count:7658  delete_count:122  deleted_byte_count:27622941  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632307 
-          volume id:72  size:903990720  collection:"collection1"  file_count:6996  delete_count:240  deleted_byte_count:74147727  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630982 
-          volume id:73  size:929047664  collection:"collection1"  file_count:7038  delete_count:227  deleted_byte_count:65336664  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630707 
-          volume id:74  size:957046128  collection:"collection1"  file_count:6981  delete_count:259  deleted_byte_count:73080838  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615631460 
-          volume id:75  size:908044992  collection:"collection1"  file_count:6911  delete_count:268  deleted_byte_count:73934373  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632430 
-          volume id:76  size:985296744  collection:"collection1"  file_count:6566  delete_count:61  deleted_byte_count:44464430  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632284 
-          volume id:77  size:929398296  collection:"collection1"  file_count:7427  delete_count:238  deleted_byte_count:59581579  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632013 
-          volume id:78  size:1075671512  collection:"collection1"  file_count:7540  delete_count:258  deleted_byte_count:71726846  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615582829 
-          volume id:79  size:948225472  collection:"collection1"  file_count:6997  delete_count:227  deleted_byte_count:60625763  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631326 
-          volume id:82  size:1041661800  collection:"collection1"  file_count:7043  delete_count:207  deleted_byte_count:52275724  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632430 
-          volume id:83  size:936195856  collection:"collection1"  file_count:7593  delete_count:13  deleted_byte_count:4633917  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632029 
-          volume id:85  size:1023867520  collection:"collection1"  file_count:7787  delete_count:240  deleted_byte_count:82091742  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631723 
-          volume id:86  size:1009437488  collection:"collection1"  file_count:8474  delete_count:236  deleted_byte_count:64543674  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630812 
-          volume id:87  size:922276640  collection:"collection1"  file_count:12902  delete_count:13  deleted_byte_count:3412959  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632438 
-          volume id:89  size:1044401976  collection:"collection1"  file_count:14943  delete_count:243  deleted_byte_count:58543159  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632208 
-          volume id:90  size:891145784  collection:"collection1"  file_count:14608  delete_count:10  deleted_byte_count:2564369  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615629390 
-          volume id:91  size:936572832  collection:"collection1"  file_count:14686  delete_count:11  deleted_byte_count:4717727  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631851 
-          volume id:92  size:992440712  collection:"collection1"  file_count:7061  delete_count:195  deleted_byte_count:60649573  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630566 
-          volume id:93  size:1079603768  collection:"collection1"  file_count:7878  delete_count:270  deleted_byte_count:74150048  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615556015 
-          volume id:94  size:1030685824  collection:"collection1"  file_count:7660  delete_count:207  deleted_byte_count:70150733  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631616 
-          volume id:95  size:990879168  collection:"collection1"  file_count:6620  delete_count:206  deleted_byte_count:60363604  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615631866 
-          volume id:96  size:989296136  collection:"collection1"  file_count:7544  delete_count:229  deleted_byte_count:59931853  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630778 
-          volume id:97  size:1053112992  collection:"collection1"  file_count:6789  delete_count:50  deleted_byte_count:38894001  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631194 
-          volume id:99  size:1071718504  collection:"collection1"  file_count:7470  delete_count:8  deleted_byte_count:9624950  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631175 
-          volume id:100  size:1083617440  collection:"collection1"  file_count:7018  delete_count:187  deleted_byte_count:61304236  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615505917 
-          volume id:101  size:1077109520  collection:"collection1"  file_count:7706  delete_count:226  deleted_byte_count:77864841  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630994 
-          volume id:102  size:1074359920  collection:"collection1"  file_count:7338  delete_count:7  deleted_byte_count:6499151  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615626683 
-          volume id:103  size:1075863904  collection:"collection1"  file_count:7184  delete_count:186  deleted_byte_count:58872238  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615628417 
-          volume id:104  size:1076383768  collection:"collection1"  file_count:7663  delete_count:184  deleted_byte_count:100578087  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615602661 
-          volume id:105  size:1073996824  collection:"collection1"  file_count:6873  delete_count:19  deleted_byte_count:14271533  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615499756 
-          volume id:108  size:1074648024  collection:"collection1"  file_count:7472  delete_count:194  deleted_byte_count:70864699  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615593232 
-          volume id:109  size:1075254560  collection:"collection1"  file_count:7556  delete_count:263  deleted_byte_count:55155265  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615502487 
-          volume id:110  size:1076575744  collection:"collection1"  file_count:6996  delete_count:163  deleted_byte_count:52954032  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615590786 
-          volume id:111  size:1073826232  collection:"collection1"  file_count:7355  delete_count:155  deleted_byte_count:50083578  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615593233 
-          volume id:114  size:1074762784  collection:"collection1"  file_count:8802  delete_count:156  deleted_byte_count:38470055  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615591826 
-          volume id:115  size:1076192240  collection:"collection1"  file_count:7690  delete_count:154  deleted_byte_count:32267193  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615285295 
-          volume id:116  size:1074489504  collection:"collection1"  file_count:9981  delete_count:174  deleted_byte_count:53998777  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615611567 
-          volume id:117  size:1073917192  collection:"collection1"  file_count:9520  delete_count:114  deleted_byte_count:21835126  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615573714 
-          volume id:118  size:1074064400  collection:"collection1"  file_count:8738  delete_count:15  deleted_byte_count:3460697  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615516265 
-          volume id:119  size:1075940104  collection:"collection1"  file_count:9003  delete_count:12  deleted_byte_count:9128155  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615573880 
-          volume id:120  size:1076115928  collection:"collection1"  file_count:9639  delete_count:118  deleted_byte_count:33357871  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615482567 
-          volume id:121  size:1078803248  collection:"collection1"  file_count:10113  delete_count:441  deleted_byte_count:94128627  replica_placement:100  version:3  modified_at_second:1615506629 
-          volume id:122  size:1076235312  collection:"collection1"  file_count:9106  delete_count:252  deleted_byte_count:93041272  replica_placement:100  version:3  modified_at_second:1615585913 
-          volume id:123  size:1080491112  collection:"collection1"  file_count:10623  delete_count:302  deleted_byte_count:83956998  replica_placement:100  version:3  modified_at_second:1615585916 
-          volume id:124  size:1074519360  collection:"collection1"  file_count:9457  delete_count:286  deleted_byte_count:74752459  replica_placement:100  version:3  modified_at_second:1615585916 
-          volume id:125  size:1088687040  collection:"collection1"  file_count:9518  delete_count:281  deleted_byte_count:76037905  replica_placement:100  version:3  modified_at_second:1615585913 
-          volume id:126  size:1073867464  collection:"collection1"  file_count:9320  delete_count:278  deleted_byte_count:94547424  replica_placement:100  version:3  modified_at_second:1615585912 
-          volume id:127  size:1074907336  collection:"collection1"  file_count:9900  delete_count:133  deleted_byte_count:48570820  replica_placement:100  version:3  modified_at_second:1615612991 
-          volume id:129  size:1074704272  collection:"collection1"  file_count:10012  delete_count:150  deleted_byte_count:64491721  replica_placement:100  version:3  modified_at_second:1615627566 
-          volume id:130  size:1075000632  collection:"collection1"  file_count:10633  delete_count:161  deleted_byte_count:34768201  replica_placement:100  version:3  modified_at_second:1615582330 
-          volume id:131  size:1075279584  collection:"collection1"  file_count:10075  delete_count:135  deleted_byte_count:29795712  replica_placement:100  version:3  modified_at_second:1615523898 
-          volume id:132  size:1088539496  collection:"collection1"  file_count:11051  delete_count:71  deleted_byte_count:17178322  replica_placement:100  version:3  modified_at_second:1615619584 
-          volume id:133  size:1075952760  collection:"collection1"  file_count:9538  delete_count:74  deleted_byte_count:19558008  replica_placement:100  version:3  modified_at_second:1615584780 
-          volume id:134  size:1074367304  collection:"collection1"  file_count:10662  delete_count:69  deleted_byte_count:25530139  replica_placement:100  version:3  modified_at_second:1615555876 
-          volume id:135  size:1073906720  collection:"collection1"  file_count:10446  delete_count:71  deleted_byte_count:28599975  replica_placement:100  version:3  modified_at_second:1615569816 
-          volume id:137  size:1074309264  collection:"collection1"  file_count:9633  delete_count:50  deleted_byte_count:27487972  replica_placement:100  version:3  modified_at_second:1615572231 
-          volume id:139  size:1074163936  collection:"collection1"  file_count:9314  delete_count:43  deleted_byte_count:10631353  replica_placement:100  version:3  modified_at_second:1615571946 
-          volume id:141  size:1074619488  collection:"collection1"  file_count:9840  delete_count:45  deleted_byte_count:40890181  replica_placement:100  version:3  modified_at_second:1615630994 
-          volume id:142  size:1075732992  collection:"collection1"  file_count:9009  delete_count:48  deleted_byte_count:9912854  replica_placement:100  version:3  modified_at_second:1615598914 
-          volume id:143  size:1075011280  collection:"collection1"  file_count:9608  delete_count:51  deleted_byte_count:37282460  replica_placement:100  version:3  modified_at_second:1615488586 
-          volume id:145  size:1074394928  collection:"collection1"  file_count:9255  delete_count:34  deleted_byte_count:38011392  replica_placement:100  version:3  modified_at_second:1615591825 
-          volume id:146  size:1076337520  collection:"collection1"  file_count:10492  delete_count:50  deleted_byte_count:17071505  replica_placement:100  version:3  modified_at_second:1615632005 
-          volume id:147  size:1077130544  collection:"collection1"  file_count:10451  delete_count:27  deleted_byte_count:8290907  replica_placement:100  version:3  modified_at_second:1615604117 
-          volume id:148  size:1076066568  collection:"collection1"  file_count:9547  delete_count:33  deleted_byte_count:7034089  replica_placement:100  version:3  modified_at_second:1615586393 
-          volume id:149  size:1074989016  collection:"collection1"  file_count:8352  delete_count:35  deleted_byte_count:7179742  replica_placement:100  version:3  modified_at_second:1615494496 
-          volume id:150  size:1076290408  collection:"collection1"  file_count:9328  delete_count:33  deleted_byte_count:43417791  replica_placement:100  version:3  modified_at_second:1615611569 
-          volume id:151  size:1098659752  collection:"collection1"  file_count:10805  delete_count:27  deleted_byte_count:7209106  replica_placement:100  version:3  modified_at_second:1615586390 
-          volume id:152  size:1075941376  collection:"collection1"  file_count:9951  delete_count:36  deleted_byte_count:25348335  replica_placement:100  version:3  modified_at_second:1615606614 
-          volume id:153  size:1078539784  collection:"collection1"  file_count:10924  delete_count:34  deleted_byte_count:12603081  replica_placement:100  version:3  modified_at_second:1615606614 
-          volume id:154  size:1081244752  collection:"collection1"  file_count:11002  delete_count:31  deleted_byte_count:8467560  replica_placement:100  version:3  modified_at_second:1615478471 
-          volume id:156  size:1074975832  collection:"collection1"  file_count:9535  delete_count:40  deleted_byte_count:11426621  replica_placement:100  version:3  modified_at_second:1615628342 
-          volume id:157  size:1076758536  collection:"collection1"  file_count:10012  delete_count:19  deleted_byte_count:11688737  replica_placement:100  version:3  modified_at_second:1615597782 
-          volume id:158  size:1087251976  collection:"collection1"  file_count:9972  delete_count:20  deleted_byte_count:10328429  replica_placement:100  version:3  modified_at_second:1615588879 
-          volume id:159  size:1074132336  collection:"collection1"  file_count:9382  delete_count:27  deleted_byte_count:11474574  replica_placement:100  version:3  modified_at_second:1615593593 
-          volume id:160  size:1075680976  collection:"collection1"  file_count:9772  delete_count:22  deleted_byte_count:4981968  replica_placement:100  version:3  modified_at_second:1615597782 
-          volume id:161  size:1077397136  collection:"collection1"  file_count:9988  delete_count:28  deleted_byte_count:12509164  replica_placement:100  version:3  modified_at_second:1615631452 
-          volume id:162  size:1074286880  collection:"collection1"  file_count:11220  delete_count:17  deleted_byte_count:1815547  replica_placement:100  version:3  modified_at_second:1615478127 
-          volume id:163  size:1074457224  collection:"collection1"  file_count:12524  delete_count:27  deleted_byte_count:6359619  replica_placement:100  version:3  modified_at_second:1615579313 
-          volume id:164  size:1074261256  collection:"collection1"  file_count:11922  delete_count:25  deleted_byte_count:2923288  replica_placement:100  version:3  modified_at_second:1615620085 
-          volume id:165  size:1073891080  collection:"collection1"  file_count:9152  delete_count:12  deleted_byte_count:19164659  replica_placement:100  version:3  modified_at_second:1615471910 
-          volume id:166  size:1075637536  collection:"collection1"  file_count:14211  delete_count:24  deleted_byte_count:20415490  replica_placement:100  version:3  modified_at_second:1615491021 
-          volume id:167  size:1073958280  collection:"collection1"  file_count:25231  delete_count:48  deleted_byte_count:26022344  replica_placement:100  version:3  modified_at_second:1615620014 
-          volume id:168  size:1074718864  collection:"collection1"  file_count:25702  delete_count:40  deleted_byte_count:4024775  replica_placement:100  version:3  modified_at_second:1615585664 
-          volume id:169  size:1073863032  collection:"collection1"  file_count:25248  delete_count:43  deleted_byte_count:3013817  replica_placement:100  version:3  modified_at_second:1615569832 
-          volume id:170  size:1075747088  collection:"collection1"  file_count:24596  delete_count:41  deleted_byte_count:3494711  replica_placement:100  version:3  modified_at_second:1615579207 
-          volume id:171  size:1081881400  collection:"collection1"  file_count:24215  delete_count:36  deleted_byte_count:3191335  replica_placement:100  version:3  modified_at_second:1615596486 
-          volume id:172  size:1074787304  collection:"collection1"  file_count:31236  delete_count:50  deleted_byte_count:3316482  replica_placement:100  version:3  modified_at_second:1615612385 
-          volume id:174  size:1073824160  collection:"collection1"  file_count:30689  delete_count:36  deleted_byte_count:2160116  replica_placement:100  version:3  modified_at_second:1615598914 
-          volume id:175  size:1077742472  collection:"collection1"  file_count:32353  delete_count:33  deleted_byte_count:1861403  replica_placement:100  version:3  modified_at_second:1615559516 
-          volume id:176  size:1073854800  collection:"collection1"  file_count:30582  delete_count:34  deleted_byte_count:7701976  replica_placement:100  version:3  modified_at_second:1615626169 
-          volume id:178  size:1087560112  collection:"collection1"  file_count:23482  delete_count:22  deleted_byte_count:18810492  replica_placement:100  version:3  modified_at_second:1615541369 
-          volume id:179  size:1074313920  collection:"collection1"  file_count:21829  delete_count:24  deleted_byte_count:45574435  replica_placement:100  version:3  modified_at_second:1615580308 
-          volume id:180  size:1078438448  collection:"collection1"  file_count:23614  delete_count:12  deleted_byte_count:4496474  replica_placement:100  version:3  modified_at_second:1614773243 
-          volume id:181  size:1074571672  collection:"collection1"  file_count:22898  delete_count:19  deleted_byte_count:6628413  replica_placement:100  version:3  modified_at_second:1614745117 
-          volume id:183  size:1076361616  collection:"collection1"  file_count:31293  delete_count:16  deleted_byte_count:468841  replica_placement:100  version:3  modified_at_second:1615572985 
-          volume id:184  size:1074594216  collection:"collection1"  file_count:31368  delete_count:22  deleted_byte_count:857453  replica_placement:100  version:3  modified_at_second:1615586578 
-          volume id:185  size:1074099592  collection:"collection1"  file_count:30612  delete_count:17  deleted_byte_count:2610847  replica_placement:100  version:3  modified_at_second:1615506835 
-          volume id:186  size:1074220664  collection:"collection1"  file_count:31450  delete_count:15  deleted_byte_count:391855  replica_placement:100  version:3  modified_at_second:1615614934 
-          volume id:187  size:1074396112  collection:"collection1"  file_count:31853  delete_count:17  deleted_byte_count:454283  replica_placement:100  version:3  modified_at_second:1615590491 
-          volume id:188  size:1074732632  collection:"collection1"  file_count:31867  delete_count:19  deleted_byte_count:393743  replica_placement:100  version:3  modified_at_second:1615487645 
-          volume id:189  size:1074847824  collection:"collection1"  file_count:31450  delete_count:16  deleted_byte_count:1040552  replica_placement:100  version:3  modified_at_second:1615335661 
-          volume id:190  size:1074008968  collection:"collection1"  file_count:31987  delete_count:11  deleted_byte_count:685125  replica_placement:100  version:3  modified_at_second:1615447162 
-          volume id:191  size:1075492960  collection:"collection1"  file_count:31301  delete_count:19  deleted_byte_count:708401  replica_placement:100  version:3  modified_at_second:1615357457 
-          volume id:192  size:1075857384  collection:"collection1"  file_count:31490  delete_count:25  deleted_byte_count:720617  replica_placement:100  version:3  modified_at_second:1615621632 
-          volume id:193  size:1076616760  collection:"collection1"  file_count:31907  delete_count:16  deleted_byte_count:464900  replica_placement:100  version:3  modified_at_second:1615507877 
-          volume id:194  size:1073985792  collection:"collection1"  file_count:31434  delete_count:18  deleted_byte_count:391432  replica_placement:100  version:3  modified_at_second:1615559502 
-          volume id:195  size:1074158304  collection:"collection1"  file_count:31453  delete_count:15  deleted_byte_count:718266  replica_placement:100  version:3  modified_at_second:1615559331 
-          volume id:196  size:1074594640  collection:"collection1"  file_count:31665  delete_count:18  deleted_byte_count:3468922  replica_placement:100  version:3  modified_at_second:1615501690 
-          volume id:198  size:1075104624  collection:"collection1"  file_count:16577  delete_count:18  deleted_byte_count:6583181  replica_placement:100  version:3  modified_at_second:1615623371 
-          volume id:199  size:1078117688  collection:"collection1"  file_count:16497  delete_count:14  deleted_byte_count:1514286  replica_placement:100  version:3  modified_at_second:1615585987 
-          volume id:200  size:1075630464  collection:"collection1"  file_count:16380  delete_count:18  deleted_byte_count:1103109  replica_placement:100  version:3  modified_at_second:1615485252 
-          volume id:201  size:1091460440  collection:"collection1"  file_count:16684  delete_count:26  deleted_byte_count:5590335  replica_placement:100  version:3  modified_at_second:1615585987 
-          volume id:204  size:1079766904  collection:"collection1"  file_count:3233  delete_count:255  deleted_byte_count:104707641  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615565702 
-          volume id:207  size:1081939960  collection:"collection1"  file_count:3010  delete_count:4  deleted_byte_count:692350  replica_placement:100  version:3  modified_at_second:1615269061 
-          volume id:208  size:1077863624  collection:"collection1"  file_count:3147  delete_count:6  deleted_byte_count:858726  replica_placement:100  version:3  modified_at_second:1615495515 
-          volume id:209  size:1074083592  collection:"collection1"  file_count:3238  delete_count:4  deleted_byte_count:1494244  replica_placement:100  version:3  modified_at_second:1615419954 
-          volume id:210  size:1094311304  collection:"collection1"  file_count:3468  delete_count:4  deleted_byte_count:466433  replica_placement:100  version:3  modified_at_second:1615495515 
-          volume id:211  size:1080610712  collection:"collection1"  file_count:3247  delete_count:7  deleted_byte_count:1891456  replica_placement:100  version:3  modified_at_second:1615269124 
-          volume id:216  size:1080073496  collection:"collection1"  file_count:3316  delete_count:4  deleted_byte_count:179819  replica_placement:100  version:3  modified_at_second:1615586387 
-          volume id:218  size:1081263944  collection:"collection1"  file_count:3433  delete_count:14  deleted_byte_count:3454237  replica_placement:100  version:3  modified_at_second:1615603637 
-          volume id:220  size:1081928312  collection:"collection1"  file_count:3166  delete_count:13  deleted_byte_count:4127709  replica_placement:100  version:3  modified_at_second:1615579317 
-          volume id:221  size:1106545536  collection:"collection1"  file_count:3153  delete_count:11  deleted_byte_count:1496835  replica_placement:100  version:3  modified_at_second:1615269138 
-          volume id:224  size:1093691520  collection:"collection1"  file_count:3463  delete_count:10  deleted_byte_count:1128328  replica_placement:100  version:3  modified_at_second:1615601870 
-          volume id:225  size:1080698928  collection:"collection1"  file_count:3115  delete_count:7  deleted_byte_count:18170416  replica_placement:100  version:3  modified_at_second:1615434685 
-          volume id:226  size:1103504792  collection:"collection1"  file_count:2965  delete_count:10  deleted_byte_count:2639254  replica_placement:100  version:3  modified_at_second:1615601870 
-          volume id:227  size:1106699864  collection:"collection1"  file_count:2827  delete_count:19  deleted_byte_count:5393310  replica_placement:100  version:3  modified_at_second:1615609989 
-          volume id:228  size:1109784072  collection:"collection1"  file_count:2504  delete_count:24  deleted_byte_count:5458950  replica_placement:100  version:3  modified_at_second:1615610489 
-          volume id:229  size:1109855256  collection:"collection1"  file_count:2857  delete_count:22  deleted_byte_count:2839883  replica_placement:100  version:3  modified_at_second:1615609989 
-          volume id:231  size:1112917664  collection:"collection1"  file_count:3151  delete_count:19  deleted_byte_count:2852517  replica_placement:100  version:3  modified_at_second:1615611350 
-          volume id:232  size:1073901520  collection:"collection1"  file_count:3004  delete_count:54  deleted_byte_count:10273081  replica_placement:100  version:3  modified_at_second:1615611352 
-          volume id:233  size:1080526464  collection:"collection1"  file_count:3136  delete_count:61  deleted_byte_count:17991717  replica_placement:100  version:3  modified_at_second:1615611354 
-          volume id:236  size:1089476200  collection:"collection1"  file_count:3231  delete_count:53  deleted_byte_count:11625921  replica_placement:100  version:3  modified_at_second:1615611351 
-          volume id:238  size:354320000  collection:"collection1"  file_count:701  delete_count:17  deleted_byte_count:5940420  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615632030 
-          volume id:240  size:424791528  collection:"collection1"  file_count:734  delete_count:12  deleted_byte_count:7353071  replica_placement:100  version:3  modified_at_second:1615631669 
-          volume id:242  size:1075383304  collection:"collection2"  file_count:10470  replica_placement:100  version:3  modified_at_second:1614852115 
-          volume id:243  size:1088174560  collection:"collection2"  file_count:11109  delete_count:1  deleted_byte_count:938  replica_placement:100  version:3  modified_at_second:1614852202 
-          volume id:245  size:1074597056  collection:"collection2"  file_count:10371  delete_count:3  deleted_byte_count:209701  replica_placement:100  version:3  modified_at_second:1614852093 
-          volume id:247  size:1075859784  collection:"collection2"  file_count:10443  delete_count:2  deleted_byte_count:564486  replica_placement:100  version:3  modified_at_second:1614856152 
-          volume id:249  size:1074819168  collection:"collection2"  file_count:10763  delete_count:2  deleted_byte_count:271699  replica_placement:100  version:3  modified_at_second:1614856231 
-          volume id:250  size:1080572256  collection:"collection2"  file_count:10220  replica_placement:100  version:3  modified_at_second:1614856129 
-          volume id:251  size:1075684408  collection:"collection2"  file_count:10847  replica_placement:100  version:3  modified_at_second:1614856270 
-          volume id:254  size:1074830800  collection:"collection2"  file_count:14140  delete_count:2  deleted_byte_count:105892  replica_placement:100  version:3  modified_at_second:1614861115 
-          volume id:257  size:1082621664  collection:"collection2"  file_count:18172  delete_count:2  deleted_byte_count:25125  replica_placement:100  version:3  modified_at_second:1614866395 
-          volume id:260  size:1075105664  collection:"collection2"  file_count:17316  delete_count:4  deleted_byte_count:2015310  replica_placement:100  version:3  modified_at_second:1614866226 
-          volume id:261  size:1076628592  collection:"collection2"  file_count:18355  delete_count:1  deleted_byte_count:1155  replica_placement:100  version:3  modified_at_second:1614866420 
-          volume id:262  size:1078492464  collection:"collection2"  file_count:20390  delete_count:3  deleted_byte_count:287601  replica_placement:100  version:3  modified_at_second:1614871601 
-          volume id:263  size:1077167440  collection:"collection2"  file_count:20227  delete_count:4  deleted_byte_count:97887  replica_placement:100  version:3  modified_at_second:1614871567 
-          volume id:268  size:1074490592  collection:"collection2"  file_count:21698  delete_count:1  deleted_byte_count:33968  replica_placement:100  version:3  modified_at_second:1614877435 
-          volume id:269  size:1077552720  collection:"collection2"  file_count:21875  delete_count:4  deleted_byte_count:347272  replica_placement:100  version:3  modified_at_second:1614877481 
-          volume id:271  size:1076992648  collection:"collection2"  file_count:22640  delete_count:1  deleted_byte_count:30645  replica_placement:100  version:3  modified_at_second:1614877504 
-          volume id:273  size:1074873432  collection:"collection2"  file_count:20511  delete_count:3  deleted_byte_count:46076  replica_placement:100  version:3  modified_at_second:1614884046 
-          volume id:274  size:1075994128  collection:"collection2"  file_count:20997  replica_placement:100  version:3  modified_at_second:1614884113 
-          volume id:276  size:1076899888  collection:"collection2"  file_count:20190  delete_count:1  deleted_byte_count:8798  replica_placement:100  version:3  modified_at_second:1614884003 
-          volume id:277  size:1074956160  collection:"collection2"  file_count:19260  delete_count:2  deleted_byte_count:172356  replica_placement:100  version:3  modified_at_second:1614889988 
-          volume id:279  size:1077325096  collection:"collection2"  file_count:19671  delete_count:6  deleted_byte_count:379116  replica_placement:100  version:3  modified_at_second:1614890230 
-          volume id:282  size:1075232240  collection:"collection2"  file_count:22659  delete_count:4  deleted_byte_count:67915  replica_placement:100  version:3  modified_at_second:1614897304 
-          volume id:284  size:1074533384  collection:"collection2"  file_count:22196  delete_count:4  deleted_byte_count:154683  replica_placement:100  version:3  modified_at_second:1614897231 
-          volume id:285  size:1082128576  collection:"collection2"  file_count:21804  delete_count:1  deleted_byte_count:1064  replica_placement:100  version:3  modified_at_second:1614897165 
-          volume id:286  size:1077464824  collection:"collection2"  file_count:23905  delete_count:6  deleted_byte_count:630577  replica_placement:100  version:3  modified_at_second:1614897401 
-          volume id:287  size:1074590528  collection:"collection2"  file_count:28163  delete_count:5  deleted_byte_count:35727  replica_placement:100  version:3  modified_at_second:1614904874 
-          volume id:288  size:1075406800  collection:"collection2"  file_count:27243  delete_count:2  deleted_byte_count:51519  replica_placement:100  version:3  modified_at_second:1614904738 
-          volume id:292  size:1092010744  collection:"collection2"  file_count:26781  delete_count:5  deleted_byte_count:508910  replica_placement:100  version:3  modified_at_second:1614912327 
-          volume id:293  size:1075409776  collection:"collection2"  file_count:26063  delete_count:4  deleted_byte_count:183834  replica_placement:100  version:3  modified_at_second:1614912235 
-          volume id:294  size:1075443992  collection:"collection2"  file_count:26076  delete_count:4  deleted_byte_count:194914  replica_placement:100  version:3  modified_at_second:1614912220 
-          volume id:295  size:1074702376  collection:"collection2"  file_count:24488  delete_count:3  deleted_byte_count:48555  replica_placement:100  version:3  modified_at_second:1614911929 
-          volume id:300  size:1076212424  collection:"collection2"  file_count:22892  delete_count:2  deleted_byte_count:61320  replica_placement:100  version:3  modified_at_second:1614918464 
-          volume id:304  size:1081038888  collection:"collection2"  file_count:24505  delete_count:2  deleted_byte_count:124447  replica_placement:100  version:3  modified_at_second:1614925567 
-          volume id:305  size:1074185552  collection:"collection2"  file_count:22074  delete_count:5  deleted_byte_count:20221  replica_placement:100  version:3  modified_at_second:1614925312 
-          volume id:310  size:1074761520  collection:"collection2"  file_count:21441  delete_count:3  deleted_byte_count:13934  replica_placement:100  version:3  modified_at_second:1614931077 
-          volume id:311  size:1088248208  collection:"collection2"  file_count:23553  delete_count:6  deleted_byte_count:191716  replica_placement:100  version:3  modified_at_second:1614931460 
-          volume id:312  size:1075037808  collection:"collection2"  file_count:22524  replica_placement:100  version:3  modified_at_second:1614937832 
-          volume id:313  size:1074876016  collection:"collection2"  file_count:22404  delete_count:4  deleted_byte_count:51728  replica_placement:100  version:3  modified_at_second:1614937755 
-          volume id:314  size:1074670840  collection:"collection2"  file_count:20964  delete_count:4  deleted_byte_count:304291  replica_placement:100  version:3  modified_at_second:1614937441 
-          volume id:315  size:1084153456  collection:"collection2"  file_count:23638  delete_count:2  deleted_byte_count:53956  replica_placement:100  version:3  modified_at_second:1614937884 
-          volume id:316  size:1077720784  collection:"collection2"  file_count:22605  delete_count:1  deleted_byte_count:8503  replica_placement:100  version:3  modified_at_second:1614937838 
-          volume id:317  size:1076215040  collection:"collection2"  file_count:23572  delete_count:2  deleted_byte_count:1441356  replica_placement:100  version:3  modified_at_second:1614943965 
-          volume id:319  size:1073952744  collection:"collection2"  file_count:22286  delete_count:2  deleted_byte_count:43421  replica_placement:100  version:3  modified_at_second:1614943810 
-          volume id:320  size:1082437736  collection:"collection2"  file_count:21544  delete_count:3  deleted_byte_count:16712  replica_placement:100  version:3  modified_at_second:1614943591 
-          volume id:321  size:1081477960  collection:"collection2"  file_count:23531  delete_count:5  deleted_byte_count:262564  replica_placement:100  version:3  modified_at_second:1614943982 
-          volume id:322  size:1078471600  collection:"collection2"  file_count:21905  delete_count:3  deleted_byte_count:145002  replica_placement:100  version:3  modified_at_second:1614950574 
-          volume id:324  size:1075606712  collection:"collection2"  file_count:20799  delete_count:1  deleted_byte_count:251210  replica_placement:100  version:3  modified_at_second:1614950310 
-          volume id:326  size:1076059936  collection:"collection2"  file_count:22564  delete_count:2  deleted_byte_count:192886  replica_placement:100  version:3  modified_at_second:1614950619 
-          volume id:327  size:1076121224  collection:"collection2"  file_count:22007  delete_count:3  deleted_byte_count:60358  replica_placement:100  version:3  modified_at_second:1614956487 
-          volume id:328  size:1074767928  collection:"collection2"  file_count:21720  delete_count:3  deleted_byte_count:56429  replica_placement:100  version:3  modified_at_second:1614956362 
-          volume id:329  size:1076691776  collection:"collection2"  file_count:22411  delete_count:5  deleted_byte_count:214092  replica_placement:100  version:3  modified_at_second:1614956485 
-          volume id:331  size:1074957192  collection:"collection2"  file_count:21230  delete_count:4  deleted_byte_count:62145  replica_placement:100  version:3  modified_at_second:1614956259 
-          volume id:333  size:1074270192  collection:"collection2"  file_count:21271  delete_count:2  deleted_byte_count:168122  replica_placement:100  version:3  modified_at_second:1614962697 
-          volume id:335  size:1076235176  collection:"collection2"  file_count:22391  delete_count:3  deleted_byte_count:8838  replica_placement:100  version:3  modified_at_second:1614962970 
-          volume id:336  size:1087853032  collection:"collection2"  file_count:22801  delete_count:2  deleted_byte_count:26394  replica_placement:100  version:3  modified_at_second:1614963003 
-          volume id:338  size:1076118360  collection:"collection2"  file_count:21680  replica_placement:100  version:3  modified_at_second:1614969850 
-          volume id:342  size:1080186296  collection:"collection2"  file_count:22405  delete_count:4  deleted_byte_count:64819  replica_placement:100  version:3  modified_at_second:1614977518 
-          volume id:343  size:1075345184  collection:"collection2"  file_count:21095  delete_count:2  deleted_byte_count:20581  replica_placement:100  version:3  modified_at_second:1614977148 
-          volume id:349  size:1075957824  collection:"collection2"  file_count:22395  delete_count:2  deleted_byte_count:61565  replica_placement:100  version:3  modified_at_second:1614984748 
-          volume id:350  size:1074756688  collection:"collection2"  file_count:21990  delete_count:3  deleted_byte_count:233881  replica_placement:100  version:3  modified_at_second:1614984682 
-          volume id:354  size:1085213992  collection:"collection2"  file_count:23150  delete_count:4  deleted_byte_count:82391  replica_placement:100  version:3  modified_at_second:1614992207 
-          volume id:356  size:1083304992  collection:"collection2"  file_count:21552  delete_count:4  deleted_byte_count:14472  replica_placement:100  version:3  modified_at_second:1614992027 
-          volume id:358  size:1085152312  collection:"collection2"  file_count:23756  delete_count:3  deleted_byte_count:44531  replica_placement:100  version:3  modified_at_second:1614998824 
-          volume id:359  size:1074211240  collection:"collection2"  file_count:22437  delete_count:2  deleted_byte_count:187953  replica_placement:100  version:3  modified_at_second:1614998711 
-          volume id:362  size:1074074120  collection:"collection2"  file_count:20595  delete_count:1  deleted_byte_count:112145  replica_placement:100  version:3  modified_at_second:1615004407 
-          volume id:363  size:1078859496  collection:"collection2"  file_count:23177  delete_count:4  deleted_byte_count:9601  replica_placement:100  version:3  modified_at_second:1615004822 
-          volume id:364  size:1081280816  collection:"collection2"  file_count:22686  delete_count:1  deleted_byte_count:84375  replica_placement:100  version:3  modified_at_second:1615004813 
-          volume id:365  size:1075736632  collection:"collection2"  file_count:22193  delete_count:5  deleted_byte_count:259033  replica_placement:100  version:3  modified_at_second:1615004776 
-          volume id:366  size:1075267272  collection:"collection2"  file_count:21856  delete_count:5  deleted_byte_count:138363  replica_placement:100  version:3  modified_at_second:1615004703 
-          volume id:367  size:1076403648  collection:"collection2"  file_count:22995  delete_count:2  deleted_byte_count:36955  replica_placement:100  version:3  modified_at_second:1615010985 
-          volume id:368  size:1074822016  collection:"collection2"  file_count:22252  delete_count:4  deleted_byte_count:3291946  replica_placement:100  version:3  modified_at_second:1615010878 
-          volume id:369  size:1091472040  collection:"collection2"  file_count:23709  delete_count:4  deleted_byte_count:400876  replica_placement:100  version:3  modified_at_second:1615011019 
-          volume id:370  size:1076040480  collection:"collection2"  file_count:22092  delete_count:2  deleted_byte_count:115388  replica_placement:100  version:3  modified_at_second:1615010877 
-          volume id:371  size:1078806160  collection:"collection2"  file_count:22685  delete_count:2  deleted_byte_count:68905  replica_placement:100  version:3  modified_at_second:1615010994 
-          volume id:372  size:1076193312  collection:"collection2"  file_count:22774  delete_count:1  deleted_byte_count:3495  replica_placement:100  version:3  modified_at_second:1615016911 
-          volume id:374  size:1085011080  collection:"collection2"  file_count:23054  delete_count:2  deleted_byte_count:89034  replica_placement:100  version:3  modified_at_second:1615016917 
-          volume id:375  size:1076140688  collection:"collection2"  file_count:21880  delete_count:2  deleted_byte_count:51103  replica_placement:100  version:3  modified_at_second:1615016787 
-          volume id:378  size:959273824  collection:"collection2"  file_count:15031  replica_placement:100  version:3  modified_at_second:1615632323 
-          volume id:379  size:1014108592  collection:"collection2"  file_count:15360  delete_count:8  deleted_byte_count:2524591  replica_placement:100  version:3  modified_at_second:1615632323 
-          volume id:380  size:1010760464  collection:"collection2"  file_count:14920  delete_count:7  deleted_byte_count:134859  replica_placement:100  version:3  modified_at_second:1615632322 
-        Disk hdd total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 
-      DataNode 192.168.1.1:8080 total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 
-    Rack DefaultRack total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 
-  DataCenter dc4 total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 
-  DataCenter dc5 hdd(volume:299/3000 active:299 free:2701 remote:0)
-    Rack DefaultRack hdd(volume:299/3000 active:299 free:2701 remote:0)
-      DataNode 192.168.1.5:8080 hdd(volume:299/3000 active:299 free:2701 remote:0)
-        Disk hdd(volume:299/3000 active:299 free:2701 remote:0)
-          volume id:5  size:293806008  file_count:1669  delete_count:2  deleted_byte_count:274334  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631342 
-          volume id:11  size:1109224552  collection:"collection0"  file_count:44  replica_placement:100  version:3  modified_at_second:1615629606 
-          volume id:18  size:1096678688  collection:"collection0"  file_count:88  delete_count:4  deleted_byte_count:8633  replica_placement:100  version:3  modified_at_second:1615631673 
-          volume id:19  size:1096923792  collection:"collection0"  file_count:100  delete_count:10  deleted_byte_count:75779917  replica_placement:100  version:3  compact_revision:4  modified_at_second:1615630117 
-          volume id:20  size:1074760432  collection:"collection0"  file_count:82  delete_count:5  deleted_byte_count:12156  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615629340 
-          volume id:26  size:532657632  collection:"collection0"  file_count:100  delete_count:4  deleted_byte_count:9081  replica_placement:100  version:3  modified_at_second:1614170024 
-          volume id:27  size:298886792  file_count:1608  replica_placement:100  version:3  modified_at_second:1615632482 
-          volume id:28  size:308919192  file_count:1591  delete_count:1  deleted_byte_count:125280  replica_placement:100  version:3  modified_at_second:1615631762 
-          volume id:29  size:281582688  file_count:1537  replica_placement:100  version:3  modified_at_second:1615629422 
-          volume id:30  size:289466144  file_count:1566  delete_count:1  deleted_byte_count:124972  replica_placement:100  version:3  modified_at_second:1615632422 
-          volume id:31  size:273363256  file_count:1498  replica_placement:100  version:3  modified_at_second:1615631642 
-          volume id:32  size:281343360  file_count:1497  replica_placement:100  version:3  modified_at_second:1615632362 
-          volume id:33  size:1130226344  collection:"collection1"  file_count:7322  delete_count:172  deleted_byte_count:45199399  replica_placement:100  version:3  modified_at_second:1615618789 
-          volume id:34  size:1077111136  collection:"collection1"  file_count:9781  delete_count:110  deleted_byte_count:20894827  replica_placement:100  version:3  modified_at_second:1615619366 
-          volume id:35  size:1075241744  collection:"collection1"  file_count:10523  delete_count:97  deleted_byte_count:46586217  replica_placement:100  version:3  modified_at_second:1615618790 
-          volume id:36  size:1075118336  collection:"collection1"  file_count:10341  delete_count:118  deleted_byte_count:24753278  replica_placement:100  version:3  modified_at_second:1615606148 
-          volume id:37  size:1075895576  collection:"collection1"  file_count:12013  delete_count:98  deleted_byte_count:50747932  replica_placement:100  version:3  modified_at_second:1615594776 
-          volume id:38  size:1075545744  collection:"collection1"  file_count:13324  delete_count:100  deleted_byte_count:25223906  replica_placement:100  version:3  modified_at_second:1615569830 
-          volume id:39  size:1076606536  collection:"collection1"  file_count:12612  delete_count:78  deleted_byte_count:17462730  replica_placement:100  version:3  modified_at_second:1615611959 
-          volume id:40  size:1075358552  collection:"collection1"  file_count:12597  delete_count:62  deleted_byte_count:11657901  replica_placement:100  version:3  modified_at_second:1615612994 
-          volume id:41  size:1076283592  collection:"collection1"  file_count:12088  delete_count:84  deleted_byte_count:19311237  replica_placement:100  version:3  modified_at_second:1615596736 
-          volume id:42  size:1093948352  collection:"collection1"  file_count:7889  delete_count:47  deleted_byte_count:5697275  replica_placement:100  version:3  modified_at_second:1615548906 
-          volume id:43  size:1116445864  collection:"collection1"  file_count:7355  delete_count:57  deleted_byte_count:9727158  replica_placement:100  version:3  modified_at_second:1615566167 
-          volume id:44  size:1077582560  collection:"collection1"  file_count:7295  delete_count:50  deleted_byte_count:12618414  replica_placement:100  version:3  modified_at_second:1615566170 
-          volume id:45  size:1075254640  collection:"collection1"  file_count:10772  delete_count:76  deleted_byte_count:22426345  replica_placement:100  version:3  modified_at_second:1615573498 
-          volume id:46  size:1075286056  collection:"collection1"  file_count:9947  delete_count:309  deleted_byte_count:105601163  replica_placement:100  version:3  modified_at_second:1615569825 
-          volume id:47  size:444599784  collection:"collection1"  file_count:709  delete_count:19  deleted_byte_count:11913451  replica_placement:100  version:3  modified_at_second:1615632397 
-          volume id:48  size:1076778664  collection:"collection1"  file_count:9850  delete_count:77  deleted_byte_count:16641907  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630690 
-          volume id:49  size:1078775288  collection:"collection1"  file_count:9631  delete_count:27  deleted_byte_count:5985628  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615575823 
-          volume id:50  size:1076688288  collection:"collection1"  file_count:7921  delete_count:26  deleted_byte_count:5162032  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615610876 
-          volume id:51  size:1076796120  collection:"collection1"  file_count:10550  delete_count:39  deleted_byte_count:12723654  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615547786 
-          volume id:53  size:1063089216  collection:"collection1"  file_count:9832  delete_count:31  deleted_byte_count:9273066  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632156 
-          volume id:54  size:1045022288  collection:"collection1"  file_count:9409  delete_count:29  deleted_byte_count:15102818  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630813 
-          volume id:55  size:1012890016  collection:"collection1"  file_count:8651  delete_count:27  deleted_byte_count:9418841  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631453 
-          volume id:56  size:1002412240  collection:"collection1"  file_count:8762  delete_count:40  deleted_byte_count:65885831  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615632014 
-          volume id:57  size:839849792  collection:"collection1"  file_count:7514  delete_count:24  deleted_byte_count:6228543  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631775 
-          volume id:58  size:908064192  collection:"collection1"  file_count:8128  delete_count:21  deleted_byte_count:6113731  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632343 
-          volume id:59  size:988302272  collection:"collection1"  file_count:8098  delete_count:20  deleted_byte_count:3947615  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632238 
-          volume id:60  size:1010702480  collection:"collection1"  file_count:8969  delete_count:79  deleted_byte_count:24782814  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615632439 
-          volume id:61  size:975604544  collection:"collection1"  file_count:8683  delete_count:20  deleted_byte_count:10276072  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615631176 
-          volume id:62  size:873845904  collection:"collection1"  file_count:7897  delete_count:23  deleted_byte_count:10920170  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631133 
-          volume id:63  size:956941176  collection:"collection1"  file_count:8271  delete_count:32  deleted_byte_count:15876189  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632036 
-          volume id:64  size:965638424  collection:"collection1"  file_count:8218  delete_count:27  deleted_byte_count:6922489  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631032 
-          volume id:65  size:823283608  collection:"collection1"  file_count:7834  delete_count:29  deleted_byte_count:5950610  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632307 
-          volume id:66  size:821343440  collection:"collection1"  file_count:7383  delete_count:29  deleted_byte_count:12010343  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631968 
-          volume id:67  size:878713880  collection:"collection1"  file_count:7299  delete_count:117  deleted_byte_count:24857326  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632156 
-          volume id:69  size:863913896  collection:"collection1"  file_count:7291  delete_count:100  deleted_byte_count:25335024  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630534 
-          volume id:70  size:886695472  collection:"collection1"  file_count:7769  delete_count:164  deleted_byte_count:45162513  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632398 
-          volume id:71  size:907608392  collection:"collection1"  file_count:7658  delete_count:122  deleted_byte_count:27622941  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632307 
-          volume id:72  size:903990720  collection:"collection1"  file_count:6996  delete_count:240  deleted_byte_count:74147727  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630985 
-          volume id:73  size:929047544  collection:"collection1"  file_count:7038  delete_count:227  deleted_byte_count:65336664  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630707 
-          volume id:75  size:908045000  collection:"collection1"  file_count:6911  delete_count:268  deleted_byte_count:73934373  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632430 
-          volume id:76  size:985296744  collection:"collection1"  file_count:6566  delete_count:61  deleted_byte_count:44464430  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632284 
-          volume id:77  size:929398296  collection:"collection1"  file_count:7427  delete_count:238  deleted_byte_count:59581579  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632014 
-          volume id:78  size:1075671512  collection:"collection1"  file_count:7540  delete_count:258  deleted_byte_count:71726846  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615582829 
-          volume id:79  size:948225472  collection:"collection1"  file_count:6997  delete_count:227  deleted_byte_count:60625763  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631326 
-          volume id:80  size:827912928  collection:"collection1"  file_count:6916  delete_count:15  deleted_byte_count:5611604  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615631159 
-          volume id:81  size:880693168  collection:"collection1"  file_count:7481  delete_count:238  deleted_byte_count:80880878  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615631395 
-          volume id:82  size:1041660512  collection:"collection1"  file_count:7043  delete_count:207  deleted_byte_count:52275724  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632430 
-          volume id:83  size:936194288  collection:"collection1"  file_count:7593  delete_count:13  deleted_byte_count:4633917  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632029 
-          volume id:84  size:871262320  collection:"collection1"  file_count:8190  delete_count:14  deleted_byte_count:3150948  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631161 
-          volume id:86  size:1009434632  collection:"collection1"  file_count:8474  delete_count:236  deleted_byte_count:64543674  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630812 
-          volume id:87  size:922274624  collection:"collection1"  file_count:12902  delete_count:13  deleted_byte_count:3412959  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632438 
-          volume id:88  size:1073767976  collection:"collection1"  file_count:14994  delete_count:207  deleted_byte_count:82380696  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615541383 
-          volume id:89  size:1044421824  collection:"collection1"  file_count:14943  delete_count:243  deleted_byte_count:58543159  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632208 
-          volume id:90  size:891163760  collection:"collection1"  file_count:14608  delete_count:10  deleted_byte_count:2564369  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615629392 
-          volume id:91  size:936573952  collection:"collection1"  file_count:14686  delete_count:11  deleted_byte_count:4717727  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631851 
-          volume id:92  size:992439144  collection:"collection1"  file_count:7061  delete_count:195  deleted_byte_count:60649573  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630566 
-          volume id:93  size:1079602592  collection:"collection1"  file_count:7878  delete_count:270  deleted_byte_count:74150048  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615556013 
-          volume id:94  size:1030684704  collection:"collection1"  file_count:7660  delete_count:207  deleted_byte_count:70150733  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631616 
-          volume id:95  size:990877824  collection:"collection1"  file_count:6620  delete_count:206  deleted_byte_count:60363604  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615631867 
-          volume id:96  size:989294848  collection:"collection1"  file_count:7544  delete_count:229  deleted_byte_count:59931853  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630778 
-          volume id:98  size:1077836472  collection:"collection1"  file_count:7605  delete_count:202  deleted_byte_count:73180379  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615523691 
-          volume id:99  size:1071718496  collection:"collection1"  file_count:7470  delete_count:8  deleted_byte_count:9624950  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631175 
-          volume id:100  size:1083617472  collection:"collection1"  file_count:7018  delete_count:187  deleted_byte_count:61304236  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615505914 
-          volume id:101  size:1077109408  collection:"collection1"  file_count:7706  delete_count:226  deleted_byte_count:77864780  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630994 
-          volume id:102  size:1074359920  collection:"collection1"  file_count:7338  delete_count:7  deleted_byte_count:6499151  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615626682 
-          volume id:103  size:1075863904  collection:"collection1"  file_count:7184  delete_count:186  deleted_byte_count:58872238  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615628417 
-          volume id:106  size:1075458680  collection:"collection1"  file_count:7182  delete_count:307  deleted_byte_count:69349053  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615598137 
-          volume id:107  size:1073811776  collection:"collection1"  file_count:7436  delete_count:168  deleted_byte_count:57747428  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615293569 
-          volume id:108  size:1074648024  collection:"collection1"  file_count:7472  delete_count:194  deleted_byte_count:70864699  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615593231 
-          volume id:109  size:1075254560  collection:"collection1"  file_count:7556  delete_count:263  deleted_byte_count:55155265  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615502487 
-          volume id:110  size:1076575744  collection:"collection1"  file_count:6996  delete_count:163  deleted_byte_count:52954032  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615590786 
-          volume id:111  size:1073826176  collection:"collection1"  file_count:7355  delete_count:155  deleted_byte_count:50083578  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615593232 
-          volume id:112  size:1076392512  collection:"collection1"  file_count:8291  delete_count:156  deleted_byte_count:74120183  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615569823 
-          volume id:113  size:1076709184  collection:"collection1"  file_count:9355  delete_count:177  deleted_byte_count:59796765  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615569822 
-          volume id:114  size:1074762792  collection:"collection1"  file_count:8802  delete_count:156  deleted_byte_count:38470055  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615591826 
-          volume id:115  size:1076192296  collection:"collection1"  file_count:7690  delete_count:154  deleted_byte_count:32267193  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615285296 
-          volume id:117  size:1073917192  collection:"collection1"  file_count:9520  delete_count:114  deleted_byte_count:21835126  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615573712 
-          volume id:118  size:1074064344  collection:"collection1"  file_count:8738  delete_count:15  deleted_byte_count:3460697  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615516264 
-          volume id:120  size:1076115928  collection:"collection1"  file_count:9639  delete_count:118  deleted_byte_count:33357871  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615482567 
-          volume id:121  size:1078803320  collection:"collection1"  file_count:10113  delete_count:441  deleted_byte_count:94128627  replica_placement:100  version:3  modified_at_second:1615506626 
-          volume id:122  size:1076235312  collection:"collection1"  file_count:9106  delete_count:252  deleted_byte_count:93041272  replica_placement:100  version:3  modified_at_second:1615585912 
-          volume id:123  size:1080491112  collection:"collection1"  file_count:10623  delete_count:302  deleted_byte_count:83956998  replica_placement:100  version:3  modified_at_second:1615585916 
-          volume id:124  size:1074519360  collection:"collection1"  file_count:9457  delete_count:286  deleted_byte_count:74752459  replica_placement:100  version:3  modified_at_second:1615585913 
-          volume id:125  size:1088687040  collection:"collection1"  file_count:9518  delete_count:281  deleted_byte_count:76037905  replica_placement:100  version:3  modified_at_second:1615585913 
-          volume id:126  size:1073867408  collection:"collection1"  file_count:9320  delete_count:278  deleted_byte_count:94547424  replica_placement:100  version:3  modified_at_second:1615585911 
-          volume id:127  size:1074907336  collection:"collection1"  file_count:9900  delete_count:133  deleted_byte_count:48570820  replica_placement:100  version:3  modified_at_second:1615612990 
-          volume id:128  size:1074874632  collection:"collection1"  file_count:9821  delete_count:148  deleted_byte_count:43633334  replica_placement:100  version:3  modified_at_second:1615602670 
-          volume id:129  size:1074704328  collection:"collection1"  file_count:10012  delete_count:150  deleted_byte_count:64491721  replica_placement:100  version:3  modified_at_second:1615627566 
-          volume id:130  size:1075000632  collection:"collection1"  file_count:10633  delete_count:161  deleted_byte_count:34768201  replica_placement:100  version:3  modified_at_second:1615582327 
-          volume id:131  size:1075279584  collection:"collection1"  file_count:10075  delete_count:135  deleted_byte_count:29795712  replica_placement:100  version:3  modified_at_second:1615523898 
-          volume id:132  size:1088539552  collection:"collection1"  file_count:11051  delete_count:71  deleted_byte_count:17178322  replica_placement:100  version:3  modified_at_second:1615619581 
-          volume id:134  size:1074367304  collection:"collection1"  file_count:10662  delete_count:69  deleted_byte_count:25530139  replica_placement:100  version:3  modified_at_second:1615555873 
-          volume id:135  size:1073906776  collection:"collection1"  file_count:10446  delete_count:71  deleted_byte_count:28599975  replica_placement:100  version:3  modified_at_second:1615569816 
-          volume id:136  size:1074433552  collection:"collection1"  file_count:9593  delete_count:72  deleted_byte_count:26912512  replica_placement:100  version:3  modified_at_second:1615376036 
-          volume id:137  size:1074309264  collection:"collection1"  file_count:9633  delete_count:50  deleted_byte_count:27487972  replica_placement:100  version:3  modified_at_second:1615572231 
-          volume id:138  size:1074465744  collection:"collection1"  file_count:10120  delete_count:55  deleted_byte_count:15875438  replica_placement:100  version:3  modified_at_second:1615572231 
-          volume id:140  size:1076203744  collection:"collection1"  file_count:11219  delete_count:57  deleted_byte_count:19864498  replica_placement:100  version:3  modified_at_second:1615571947 
-          volume id:141  size:1074619488  collection:"collection1"  file_count:9840  delete_count:45  deleted_byte_count:40890181  replica_placement:100  version:3  modified_at_second:1615630994 
-          volume id:142  size:1075733064  collection:"collection1"  file_count:9009  delete_count:48  deleted_byte_count:9912854  replica_placement:100  version:3  modified_at_second:1615598913 
-          volume id:143  size:1075011280  collection:"collection1"  file_count:9608  delete_count:51  deleted_byte_count:37282460  replica_placement:100  version:3  modified_at_second:1615488584 
-          volume id:144  size:1074549720  collection:"collection1"  file_count:8780  delete_count:50  deleted_byte_count:52475146  replica_placement:100  version:3  modified_at_second:1615573451 
-          volume id:145  size:1074394928  collection:"collection1"  file_count:9255  delete_count:34  deleted_byte_count:38011392  replica_placement:100  version:3  modified_at_second:1615591825 
-          volume id:146  size:1076337576  collection:"collection1"  file_count:10492  delete_count:50  deleted_byte_count:17071505  replica_placement:100  version:3  modified_at_second:1615632005 
-          volume id:147  size:1077130576  collection:"collection1"  file_count:10451  delete_count:27  deleted_byte_count:8290907  replica_placement:100  version:3  modified_at_second:1615604115 
-          volume id:148  size:1076066568  collection:"collection1"  file_count:9547  delete_count:33  deleted_byte_count:7034089  replica_placement:100  version:3  modified_at_second:1615586390 
-          volume id:149  size:1074989016  collection:"collection1"  file_count:8352  delete_count:35  deleted_byte_count:7179742  replica_placement:100  version:3  modified_at_second:1615494494 
-          volume id:150  size:1076290328  collection:"collection1"  file_count:9328  delete_count:33  deleted_byte_count:43417791  replica_placement:100  version:3  modified_at_second:1615611567 
-          volume id:152  size:1075941400  collection:"collection1"  file_count:9951  delete_count:36  deleted_byte_count:25348335  replica_placement:100  version:3  modified_at_second:1615606614 
-          volume id:153  size:1078539784  collection:"collection1"  file_count:10924  delete_count:34  deleted_byte_count:12603081  replica_placement:100  version:3  modified_at_second:1615606614 
-          volume id:154  size:1081244696  collection:"collection1"  file_count:11002  delete_count:31  deleted_byte_count:8467560  replica_placement:100  version:3  modified_at_second:1615478469 
-          volume id:155  size:1075140688  collection:"collection1"  file_count:10882  delete_count:32  deleted_byte_count:10076804  replica_placement:100  version:3  modified_at_second:1615606614 
-          volume id:156  size:1074975832  collection:"collection1"  file_count:9535  delete_count:40  deleted_byte_count:11426621  replica_placement:100  version:3  modified_at_second:1615628341 
-          volume id:157  size:1076758536  collection:"collection1"  file_count:10012  delete_count:19  deleted_byte_count:11688737  replica_placement:100  version:3  modified_at_second:1615597782 
-          volume id:158  size:1087251976  collection:"collection1"  file_count:9972  delete_count:20  deleted_byte_count:10328429  replica_placement:100  version:3  modified_at_second:1615588879 
-          volume id:159  size:1074132368  collection:"collection1"  file_count:9382  delete_count:27  deleted_byte_count:11474574  replica_placement:100  version:3  modified_at_second:1615593593 
-          volume id:160  size:1075680952  collection:"collection1"  file_count:9772  delete_count:22  deleted_byte_count:4981968  replica_placement:100  version:3  modified_at_second:1615597780 
-          volume id:162  size:1074286880  collection:"collection1"  file_count:11220  delete_count:17  deleted_byte_count:1815547  replica_placement:100  version:3  modified_at_second:1615478126 
-          volume id:163  size:1074457192  collection:"collection1"  file_count:12524  delete_count:27  deleted_byte_count:6359619  replica_placement:100  version:3  modified_at_second:1615579313 
-          volume id:164  size:1074261248  collection:"collection1"  file_count:11922  delete_count:25  deleted_byte_count:2923288  replica_placement:100  version:3  modified_at_second:1615620084 
-          volume id:165  size:1073891016  collection:"collection1"  file_count:9152  delete_count:12  deleted_byte_count:19164659  replica_placement:100  version:3  modified_at_second:1615471907 
-          volume id:166  size:1075637536  collection:"collection1"  file_count:14211  delete_count:24  deleted_byte_count:20415490  replica_placement:100  version:3  modified_at_second:1615491019 
-          volume id:168  size:1074718808  collection:"collection1"  file_count:25702  delete_count:40  deleted_byte_count:4024775  replica_placement:100  version:3  modified_at_second:1615585664 
-          volume id:169  size:1073863128  collection:"collection1"  file_count:25248  delete_count:43  deleted_byte_count:3013817  replica_placement:100  version:3  modified_at_second:1615569832 
-          volume id:170  size:1075747096  collection:"collection1"  file_count:24596  delete_count:41  deleted_byte_count:3494711  replica_placement:100  version:3  modified_at_second:1615579204 
-          volume id:171  size:1081881312  collection:"collection1"  file_count:24215  delete_count:36  deleted_byte_count:3191335  replica_placement:100  version:3  modified_at_second:1615596485 
-          volume id:172  size:1074787312  collection:"collection1"  file_count:31236  delete_count:50  deleted_byte_count:3316482  replica_placement:100  version:3  modified_at_second:1615612385 
-          volume id:173  size:1074154648  collection:"collection1"  file_count:30884  delete_count:34  deleted_byte_count:2430948  replica_placement:100  version:3  modified_at_second:1615591904 
-          volume id:175  size:1077742504  collection:"collection1"  file_count:32353  delete_count:33  deleted_byte_count:1861403  replica_placement:100  version:3  modified_at_second:1615559515 
-          volume id:176  size:1073854800  collection:"collection1"  file_count:30582  delete_count:34  deleted_byte_count:7701976  replica_placement:100  version:3  modified_at_second:1615626169 
-          volume id:177  size:1074120120  collection:"collection1"  file_count:22293  delete_count:16  deleted_byte_count:3719562  replica_placement:100  version:3  modified_at_second:1615516891 
-          volume id:178  size:1087560112  collection:"collection1"  file_count:23482  delete_count:22  deleted_byte_count:18810492  replica_placement:100  version:3  modified_at_second:1615541369 
-          volume id:180  size:1078438536  collection:"collection1"  file_count:23614  delete_count:12  deleted_byte_count:4496474  replica_placement:100  version:3  modified_at_second:1614773242 
-          volume id:181  size:1074571768  collection:"collection1"  file_count:22898  delete_count:19  deleted_byte_count:6628413  replica_placement:100  version:3  modified_at_second:1614745116 
-          volume id:182  size:1076131280  collection:"collection1"  file_count:31987  delete_count:21  deleted_byte_count:1416142  replica_placement:100  version:3  modified_at_second:1615568922 
-          volume id:183  size:1076361448  collection:"collection1"  file_count:31293  delete_count:16  deleted_byte_count:468841  replica_placement:100  version:3  modified_at_second:1615572982 
-          volume id:184  size:1074594160  collection:"collection1"  file_count:31368  delete_count:22  deleted_byte_count:857453  replica_placement:100  version:3  modified_at_second:1615586578 
-          volume id:185  size:1074099624  collection:"collection1"  file_count:30612  delete_count:17  deleted_byte_count:2610847  replica_placement:100  version:3  modified_at_second:1615506832 
-          volume id:186  size:1074220864  collection:"collection1"  file_count:31450  delete_count:15  deleted_byte_count:391855  replica_placement:100  version:3  modified_at_second:1615614933 
-          volume id:187  size:1074395944  collection:"collection1"  file_count:31853  delete_count:17  deleted_byte_count:454283  replica_placement:100  version:3  modified_at_second:1615590490 
-          volume id:188  size:1074732792  collection:"collection1"  file_count:31867  delete_count:19  deleted_byte_count:393743  replica_placement:100  version:3  modified_at_second:1615487645 
-          volume id:189  size:1074847896  collection:"collection1"  file_count:31450  delete_count:16  deleted_byte_count:1040552  replica_placement:100  version:3  modified_at_second:1615335661 
-          volume id:190  size:1074008912  collection:"collection1"  file_count:31987  delete_count:11  deleted_byte_count:685125  replica_placement:100  version:3  modified_at_second:1615447161 
-          volume id:191  size:1075493024  collection:"collection1"  file_count:31301  delete_count:19  deleted_byte_count:708401  replica_placement:100  version:3  modified_at_second:1615357456 
-          volume id:192  size:1075857400  collection:"collection1"  file_count:31490  delete_count:25  deleted_byte_count:720617  replica_placement:100  version:3  modified_at_second:1615621632 
-          volume id:193  size:1076616768  collection:"collection1"  file_count:31907  delete_count:16  deleted_byte_count:464900  replica_placement:100  version:3  modified_at_second:1615507875 
-          volume id:194  size:1073985624  collection:"collection1"  file_count:31434  delete_count:18  deleted_byte_count:391432  replica_placement:100  version:3  modified_at_second:1615559499 
-          volume id:195  size:1074158312  collection:"collection1"  file_count:31453  delete_count:15  deleted_byte_count:718266  replica_placement:100  version:3  modified_at_second:1615559331 
-          volume id:196  size:1074594784  collection:"collection1"  file_count:31665  delete_count:18  deleted_byte_count:3468922  replica_placement:100  version:3  modified_at_second:1615501688 
-          volume id:197  size:1075423296  collection:"collection1"  file_count:16473  delete_count:15  deleted_byte_count:12552442  replica_placement:100  version:3  modified_at_second:1615485253 
-          volume id:198  size:1075104712  collection:"collection1"  file_count:16577  delete_count:18  deleted_byte_count:6583181  replica_placement:100  version:3  modified_at_second:1615623369 
-          volume id:199  size:1078117688  collection:"collection1"  file_count:16497  delete_count:14  deleted_byte_count:1514286  replica_placement:100  version:3  modified_at_second:1615585984 
-          volume id:200  size:1075630536  collection:"collection1"  file_count:16380  delete_count:18  deleted_byte_count:1103109  replica_placement:100  version:3  modified_at_second:1615485252 
-          volume id:201  size:1091460440  collection:"collection1"  file_count:16684  delete_count:26  deleted_byte_count:5590335  replica_placement:100  version:3  modified_at_second:1615585987 
-          volume id:202  size:1077533160  collection:"collection1"  file_count:2847  delete_count:67  deleted_byte_count:65172985  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615588497 
-          volume id:203  size:1027316272  collection:"collection1"  file_count:3040  delete_count:11  deleted_byte_count:3993230  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615631728 
-          volume id:204  size:1079766872  collection:"collection1"  file_count:3233  delete_count:255  deleted_byte_count:104707641  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615565701 
-          volume id:205  size:1078485304  collection:"collection1"  file_count:2869  delete_count:43  deleted_byte_count:18290259  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615579314 
-          volume id:206  size:1082045848  collection:"collection1"  file_count:2979  delete_count:225  deleted_byte_count:88220074  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630989 
-          volume id:207  size:1081939960  collection:"collection1"  file_count:3010  delete_count:4  deleted_byte_count:692350  replica_placement:100  version:3  modified_at_second:1615269061 
-          volume id:208  size:1077863624  collection:"collection1"  file_count:3147  delete_count:6  deleted_byte_count:858726  replica_placement:100  version:3  modified_at_second:1615495515 
-          volume id:210  size:1094311304  collection:"collection1"  file_count:3468  delete_count:4  deleted_byte_count:466433  replica_placement:100  version:3  modified_at_second:1615495515 
-          volume id:212  size:1078293448  collection:"collection1"  file_count:3106  delete_count:6  deleted_byte_count:2085755  replica_placement:100  version:3  modified_at_second:1615586387 
-          volume id:213  size:1093588072  collection:"collection1"  file_count:3681  delete_count:12  deleted_byte_count:3138791  replica_placement:100  version:3  modified_at_second:1615586387 
-          volume id:214  size:1074486992  collection:"collection1"  file_count:3217  delete_count:10  deleted_byte_count:6392871  replica_placement:100  version:3  modified_at_second:1615586383 
-          volume id:215  size:1074798704  collection:"collection1"  file_count:2819  delete_count:31  deleted_byte_count:10873569  replica_placement:100  version:3  modified_at_second:1615586386 
-          volume id:217  size:1075381872  collection:"collection1"  file_count:3331  delete_count:14  deleted_byte_count:2009141  replica_placement:100  version:3  modified_at_second:1615401638 
-          volume id:218  size:1081263944  collection:"collection1"  file_count:3433  delete_count:14  deleted_byte_count:3454237  replica_placement:100  version:3  modified_at_second:1615603637 
-          volume id:219  size:1092298816  collection:"collection1"  file_count:3193  delete_count:17  deleted_byte_count:2047576  replica_placement:100  version:3  modified_at_second:1615579316 
-          volume id:220  size:1081928312  collection:"collection1"  file_count:3166  delete_count:13  deleted_byte_count:4127709  replica_placement:100  version:3  modified_at_second:1615579317 
-          volume id:221  size:1106545456  collection:"collection1"  file_count:3153  delete_count:11  deleted_byte_count:1496835  replica_placement:100  version:3  modified_at_second:1615269138 
-          volume id:222  size:1106623104  collection:"collection1"  file_count:3273  delete_count:11  deleted_byte_count:2114627  replica_placement:100  version:3  modified_at_second:1615586243 
-          volume id:223  size:1075233064  collection:"collection1"  file_count:2966  delete_count:9  deleted_byte_count:744001  replica_placement:100  version:3  modified_at_second:1615586244 
-          volume id:224  size:1093691520  collection:"collection1"  file_count:3463  delete_count:10  deleted_byte_count:1128328  replica_placement:100  version:3  modified_at_second:1615601870 
-          volume id:225  size:1080698928  collection:"collection1"  file_count:3115  delete_count:7  deleted_byte_count:18170416  replica_placement:100  version:3  modified_at_second:1615434684 
-          volume id:226  size:1103504768  collection:"collection1"  file_count:2965  delete_count:10  deleted_byte_count:2639254  replica_placement:100  version:3  modified_at_second:1615601867 
-          volume id:228  size:1109784072  collection:"collection1"  file_count:2504  delete_count:24  deleted_byte_count:5458950  replica_placement:100  version:3  modified_at_second:1615610489 
-          volume id:230  size:1080722984  collection:"collection1"  file_count:2898  delete_count:15  deleted_byte_count:3929261  replica_placement:100  version:3  modified_at_second:1615610537 
-          volume id:232  size:1073901520  collection:"collection1"  file_count:3004  delete_count:54  deleted_byte_count:10273081  replica_placement:100  version:3  modified_at_second:1615611351 
-          volume id:234  size:1073835280  collection:"collection1"  file_count:2965  delete_count:41  deleted_byte_count:4960354  replica_placement:100  version:3  modified_at_second:1615611351 
-          volume id:235  size:1075586104  collection:"collection1"  file_count:2767  delete_count:33  deleted_byte_count:3216540  replica_placement:100  version:3  modified_at_second:1615611354 
-          volume id:236  size:1089476136  collection:"collection1"  file_count:3231  delete_count:53  deleted_byte_count:11625921  replica_placement:100  version:3  modified_at_second:1615611351 
-          volume id:237  size:375722792  collection:"collection1"  file_count:736  delete_count:16  deleted_byte_count:4464870  replica_placement:100  version:3  modified_at_second:1615631727 
-          volume id:238  size:354320000  collection:"collection1"  file_count:701  delete_count:17  deleted_byte_count:5940420  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615632030 
-          volume id:239  size:426569024  collection:"collection1"  file_count:693  delete_count:19  deleted_byte_count:13020783  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630841 
-          volume id:240  size:424791528  collection:"collection1"  file_count:733  delete_count:13  deleted_byte_count:7515220  replica_placement:100  version:3  modified_at_second:1615631670 
-          volume id:241  size:380217424  collection:"collection1"  file_count:633  delete_count:6  deleted_byte_count:1715768  replica_placement:100  version:3  modified_at_second:1615632006 
-          volume id:242  size:1075383392  collection:"collection2"  file_count:10470  replica_placement:100  version:3  modified_at_second:1614852116 
-          volume id:243  size:1088174704  collection:"collection2"  file_count:11109  delete_count:1  deleted_byte_count:938  replica_placement:100  version:3  modified_at_second:1614852203 
-          volume id:244  size:1080295352  collection:"collection2"  file_count:10812  delete_count:1  deleted_byte_count:795  replica_placement:100  version:3  modified_at_second:1615628825 
-          volume id:246  size:1075998672  collection:"collection2"  file_count:10365  delete_count:1  deleted_byte_count:13112  replica_placement:100  version:3  modified_at_second:1614852106 
-          volume id:247  size:1075859808  collection:"collection2"  file_count:10443  delete_count:2  deleted_byte_count:564486  replica_placement:100  version:3  modified_at_second:1614856152 
-          volume id:248  size:1084301208  collection:"collection2"  file_count:11217  delete_count:4  deleted_byte_count:746488  replica_placement:100  version:3  modified_at_second:1614856285 
-          volume id:250  size:1080572168  collection:"collection2"  file_count:10220  replica_placement:100  version:3  modified_at_second:1614856129 
-          volume id:252  size:1075065264  collection:"collection2"  file_count:14622  delete_count:2  deleted_byte_count:5228  replica_placement:100  version:3  modified_at_second:1614861200 
-          volume id:253  size:1087328880  collection:"collection2"  file_count:14920  delete_count:3  deleted_byte_count:522994  replica_placement:100  version:3  modified_at_second:1614861258 
-          volume id:254  size:1074830736  collection:"collection2"  file_count:14140  delete_count:2  deleted_byte_count:105892  replica_placement:100  version:3  modified_at_second:1614861115 
-          volume id:255  size:1079581640  collection:"collection2"  file_count:14877  delete_count:3  deleted_byte_count:101223  replica_placement:100  version:3  modified_at_second:1614861233 
-          volume id:256  size:1074283592  collection:"collection2"  file_count:14157  delete_count:1  deleted_byte_count:18156  replica_placement:100  version:3  modified_at_second:1614861100 
-          volume id:257  size:1082621720  collection:"collection2"  file_count:18172  delete_count:2  deleted_byte_count:25125  replica_placement:100  version:3  modified_at_second:1614866402 
-          volume id:258  size:1075527216  collection:"collection2"  file_count:18421  delete_count:4  deleted_byte_count:267833  replica_placement:100  version:3  modified_at_second:1614866420 
-          volume id:259  size:1075507848  collection:"collection2"  file_count:18079  delete_count:2  deleted_byte_count:71992  replica_placement:100  version:3  modified_at_second:1614866381 
-          volume id:260  size:1075105664  collection:"collection2"  file_count:17316  delete_count:4  deleted_byte_count:2015310  replica_placement:100  version:3  modified_at_second:1614866226 
-          volume id:261  size:1076628592  collection:"collection2"  file_count:18355  delete_count:1  deleted_byte_count:1155  replica_placement:100  version:3  modified_at_second:1614866420 
-          volume id:262  size:1078492584  collection:"collection2"  file_count:20390  delete_count:3  deleted_byte_count:287601  replica_placement:100  version:3  modified_at_second:1614871601 
-          volume id:264  size:1081624192  collection:"collection2"  file_count:21151  replica_placement:100  version:3  modified_at_second:1614871629 
-          volume id:265  size:1076401104  collection:"collection2"  file_count:19932  delete_count:2  deleted_byte_count:160823  replica_placement:100  version:3  modified_at_second:1614871543 
-          volume id:266  size:1075617552  collection:"collection2"  file_count:20075  delete_count:1  deleted_byte_count:1039  replica_placement:100  version:3  modified_at_second:1614871526 
-          volume id:267  size:1075699376  collection:"collection2"  file_count:21039  delete_count:3  deleted_byte_count:59956  replica_placement:100  version:3  modified_at_second:1614877294 
-          volume id:270  size:1076876424  collection:"collection2"  file_count:22057  delete_count:1  deleted_byte_count:43916  replica_placement:100  version:3  modified_at_second:1614877469 
-          volume id:271  size:1076992704  collection:"collection2"  file_count:22640  delete_count:1  deleted_byte_count:30645  replica_placement:100  version:3  modified_at_second:1614877504 
-          volume id:272  size:1076145912  collection:"collection2"  file_count:21034  delete_count:2  deleted_byte_count:216564  replica_placement:100  version:3  modified_at_second:1614884139 
-          volume id:273  size:1074873432  collection:"collection2"  file_count:20511  delete_count:3  deleted_byte_count:46076  replica_placement:100  version:3  modified_at_second:1614884046 
-          volume id:274  size:1075994184  collection:"collection2"  file_count:20997  replica_placement:100  version:3  modified_at_second:1614884113 
-          volume id:275  size:1078349024  collection:"collection2"  file_count:20808  delete_count:1  deleted_byte_count:1118  replica_placement:100  version:3  modified_at_second:1614884147 
-          volume id:276  size:1076899880  collection:"collection2"  file_count:20190  delete_count:1  deleted_byte_count:8798  replica_placement:100  version:3  modified_at_second:1614884003 
-          volume id:278  size:1078798632  collection:"collection2"  file_count:20597  delete_count:5  deleted_byte_count:400060  replica_placement:100  version:3  modified_at_second:1614890292 
-          volume id:280  size:1077432160  collection:"collection2"  file_count:20286  delete_count:1  deleted_byte_count:879  replica_placement:100  version:3  modified_at_second:1614890262 
-          volume id:281  size:1077581064  collection:"collection2"  file_count:20206  delete_count:3  deleted_byte_count:143964  replica_placement:100  version:3  modified_at_second:1614890237 
-          volume id:282  size:1075232184  collection:"collection2"  file_count:22659  delete_count:4  deleted_byte_count:67915  replica_placement:100  version:3  modified_at_second:1614897304 
-          volume id:283  size:1080178880  collection:"collection2"  file_count:19462  delete_count:7  deleted_byte_count:660407  replica_placement:100  version:3  modified_at_second:1614896623 
-          volume id:286  size:1077464816  collection:"collection2"  file_count:23905  delete_count:6  deleted_byte_count:630577  replica_placement:100  version:3  modified_at_second:1614897401 
-          volume id:287  size:1074590536  collection:"collection2"  file_count:28163  delete_count:5  deleted_byte_count:35727  replica_placement:100  version:3  modified_at_second:1614904875 
-          volume id:288  size:1075406920  collection:"collection2"  file_count:27243  delete_count:2  deleted_byte_count:51519  replica_placement:100  version:3  modified_at_second:1614904738 
-          volume id:289  size:1075284312  collection:"collection2"  file_count:29342  delete_count:5  deleted_byte_count:100454  replica_placement:100  version:3  modified_at_second:1614904977 
-          volume id:290  size:1074723800  collection:"collection2"  file_count:28340  delete_count:4  deleted_byte_count:199064  replica_placement:100  version:3  modified_at_second:1614904924 
-          volume id:292  size:1092010672  collection:"collection2"  file_count:26781  delete_count:5  deleted_byte_count:508910  replica_placement:100  version:3  modified_at_second:1614912325 
-          volume id:295  size:1074702320  collection:"collection2"  file_count:24488  delete_count:3  deleted_byte_count:48555  replica_placement:100  version:3  modified_at_second:1614911929 
-          volume id:296  size:1077824056  collection:"collection2"  file_count:26741  delete_count:4  deleted_byte_count:199906  replica_placement:100  version:3  modified_at_second:1614912301 
-          volume id:297  size:1080229176  collection:"collection2"  file_count:23409  delete_count:5  deleted_byte_count:46268  replica_placement:100  version:3  modified_at_second:1614918481 
-          volume id:298  size:1075410024  collection:"collection2"  file_count:23222  delete_count:2  deleted_byte_count:46110  replica_placement:100  version:3  modified_at_second:1614918474 
-          volume id:302  size:1077559640  collection:"collection2"  file_count:23124  delete_count:7  deleted_byte_count:293111  replica_placement:100  version:3  modified_at_second:1614925500 
-          volume id:304  size:1081038944  collection:"collection2"  file_count:24505  delete_count:2  deleted_byte_count:124447  replica_placement:100  version:3  modified_at_second:1614925569 
-          volume id:305  size:1074185376  collection:"collection2"  file_count:22074  delete_count:5  deleted_byte_count:20221  replica_placement:100  version:3  modified_at_second:1614925312 
-          volume id:306  size:1074763952  collection:"collection2"  file_count:22939  replica_placement:100  version:3  modified_at_second:1614925462 
-          volume id:307  size:1076567912  collection:"collection2"  file_count:23377  delete_count:2  deleted_byte_count:25453  replica_placement:100  version:3  modified_at_second:1614931448 
-          volume id:308  size:1074022336  collection:"collection2"  file_count:23086  delete_count:2  deleted_byte_count:2127  replica_placement:100  version:3  modified_at_second:1614931401 
-          volume id:311  size:1088248344  collection:"collection2"  file_count:23553  delete_count:6  deleted_byte_count:191716  replica_placement:100  version:3  modified_at_second:1614931463 
-          volume id:312  size:1075037528  collection:"collection2"  file_count:22524  replica_placement:100  version:3  modified_at_second:1614937831 
-          volume id:313  size:1074875960  collection:"collection2"  file_count:22404  delete_count:4  deleted_byte_count:51728  replica_placement:100  version:3  modified_at_second:1614937755 
-          volume id:316  size:1077720776  collection:"collection2"  file_count:22605  delete_count:1  deleted_byte_count:8503  replica_placement:100  version:3  modified_at_second:1614937838 
-          volume id:318  size:1075965168  collection:"collection2"  file_count:22459  delete_count:2  deleted_byte_count:37778  replica_placement:100  version:3  modified_at_second:1614943862 
-          volume id:322  size:1078471536  collection:"collection2"  file_count:21905  delete_count:3  deleted_byte_count:145002  replica_placement:100  version:3  modified_at_second:1614950572 
-          volume id:323  size:1074608056  collection:"collection2"  file_count:21605  delete_count:4  deleted_byte_count:172090  replica_placement:100  version:3  modified_at_second:1614950526 
-          volume id:325  size:1080701232  collection:"collection2"  file_count:21735  replica_placement:100  version:3  modified_at_second:1614950525 
-          volume id:326  size:1076059920  collection:"collection2"  file_count:22564  delete_count:2  deleted_byte_count:192886  replica_placement:100  version:3  modified_at_second:1614950619 
-          volume id:327  size:1076121304  collection:"collection2"  file_count:22007  delete_count:3  deleted_byte_count:60358  replica_placement:100  version:3  modified_at_second:1614956487 
-          volume id:328  size:1074767816  collection:"collection2"  file_count:21720  delete_count:3  deleted_byte_count:56429  replica_placement:100  version:3  modified_at_second:1614956362 
-          volume id:329  size:1076691960  collection:"collection2"  file_count:22411  delete_count:5  deleted_byte_count:214092  replica_placement:100  version:3  modified_at_second:1614956485 
-          volume id:330  size:1080825760  collection:"collection2"  file_count:22464  delete_count:2  deleted_byte_count:15771  replica_placement:100  version:3  modified_at_second:1614956476 
-          volume id:331  size:1074957256  collection:"collection2"  file_count:21230  delete_count:4  deleted_byte_count:62145  replica_placement:100  version:3  modified_at_second:1614956259 
-          volume id:332  size:1075569928  collection:"collection2"  file_count:22097  delete_count:3  deleted_byte_count:98273  replica_placement:100  version:3  modified_at_second:1614962869 
-          volume id:333  size:1074270160  collection:"collection2"  file_count:21271  delete_count:2  deleted_byte_count:168122  replica_placement:100  version:3  modified_at_second:1614962697 
-          volume id:334  size:1075607880  collection:"collection2"  file_count:22546  delete_count:6  deleted_byte_count:101538  replica_placement:100  version:3  modified_at_second:1614962978 
-          volume id:335  size:1076235136  collection:"collection2"  file_count:22391  delete_count:3  deleted_byte_count:8838  replica_placement:100  version:3  modified_at_second:1614962970 
-          volume id:337  size:1075646896  collection:"collection2"  file_count:21934  delete_count:1  deleted_byte_count:3397  replica_placement:100  version:3  modified_at_second:1614969937 
-          volume id:339  size:1078402392  collection:"collection2"  file_count:22309  replica_placement:100  version:3  modified_at_second:1614969995 
-          volume id:340  size:1079462152  collection:"collection2"  file_count:22319  delete_count:4  deleted_byte_count:93620  replica_placement:100  version:3  modified_at_second:1614969977 
-          volume id:341  size:1074448360  collection:"collection2"  file_count:21590  delete_count:5  deleted_byte_count:160085  replica_placement:100  version:3  modified_at_second:1614969858 
-          volume id:343  size:1075345072  collection:"collection2"  file_count:21095  delete_count:2  deleted_byte_count:20581  replica_placement:100  version:3  modified_at_second:1614977148 
-          volume id:346  size:1076464112  collection:"collection2"  file_count:22320  delete_count:4  deleted_byte_count:798258  replica_placement:100  version:3  modified_at_second:1614977511 
-          volume id:347  size:1075145248  collection:"collection2"  file_count:22178  delete_count:1  deleted_byte_count:79392  replica_placement:100  version:3  modified_at_second:1614984727 
-          volume id:348  size:1080623544  collection:"collection2"  file_count:21667  delete_count:1  deleted_byte_count:2443  replica_placement:100  version:3  modified_at_second:1614984604 
-          volume id:349  size:1075957672  collection:"collection2"  file_count:22395  delete_count:2  deleted_byte_count:61565  replica_placement:100  version:3  modified_at_second:1614984748 
-          volume id:351  size:1078795120  collection:"collection2"  file_count:23660  delete_count:3  deleted_byte_count:102141  replica_placement:100  version:3  modified_at_second:1614984816 
-          volume id:352  size:1077145936  collection:"collection2"  file_count:22066  delete_count:1  deleted_byte_count:1018  replica_placement:100  version:3  modified_at_second:1614992130 
-          volume id:353  size:1074897496  collection:"collection2"  file_count:21266  delete_count:2  deleted_byte_count:3105374  replica_placement:100  version:3  modified_at_second:1614991951 
-          volume id:354  size:1085214104  collection:"collection2"  file_count:23150  delete_count:4  deleted_byte_count:82391  replica_placement:100  version:3  modified_at_second:1614992208 
-          volume id:357  size:1074276152  collection:"collection2"  file_count:23137  delete_count:4  deleted_byte_count:188487  replica_placement:100  version:3  modified_at_second:1614998792 
-          volume id:359  size:1074211296  collection:"collection2"  file_count:22437  delete_count:2  deleted_byte_count:187953  replica_placement:100  version:3  modified_at_second:1614998711 
-          volume id:360  size:1075532512  collection:"collection2"  file_count:22574  delete_count:3  deleted_byte_count:1774776  replica_placement:100  version:3  modified_at_second:1614998770 
-          volume id:361  size:1075362744  collection:"collection2"  file_count:22272  delete_count:1  deleted_byte_count:3497  replica_placement:100  version:3  modified_at_second:1614998668 
-          volume id:362  size:1074074176  collection:"collection2"  file_count:20595  delete_count:1  deleted_byte_count:112145  replica_placement:100  version:3  modified_at_second:1615004407 
-          volume id:363  size:1078859640  collection:"collection2"  file_count:23177  delete_count:4  deleted_byte_count:9601  replica_placement:100  version:3  modified_at_second:1615004823 
-          volume id:364  size:1081280880  collection:"collection2"  file_count:22686  delete_count:1  deleted_byte_count:84375  replica_placement:100  version:3  modified_at_second:1615004813 
-          volume id:365  size:1075736632  collection:"collection2"  file_count:22193  delete_count:5  deleted_byte_count:259033  replica_placement:100  version:3  modified_at_second:1615004776 
-          volume id:366  size:1075267272  collection:"collection2"  file_count:21856  delete_count:5  deleted_byte_count:138363  replica_placement:100  version:3  modified_at_second:1615004703 
-          volume id:367  size:1076403648  collection:"collection2"  file_count:22995  delete_count:2  deleted_byte_count:36955  replica_placement:100  version:3  modified_at_second:1615010985 
-          volume id:368  size:1074821960  collection:"collection2"  file_count:22252  delete_count:4  deleted_byte_count:3291946  replica_placement:100  version:3  modified_at_second:1615010877 
-          volume id:369  size:1091472040  collection:"collection2"  file_count:23709  delete_count:4  deleted_byte_count:400876  replica_placement:100  version:3  modified_at_second:1615011021 
-          volume id:370  size:1076040544  collection:"collection2"  file_count:22092  delete_count:2  deleted_byte_count:115388  replica_placement:100  version:3  modified_at_second:1615010877 
-          volume id:371  size:1078806216  collection:"collection2"  file_count:22685  delete_count:2  deleted_byte_count:68905  replica_placement:100  version:3  modified_at_second:1615010995 
-          volume id:372  size:1076193344  collection:"collection2"  file_count:22774  delete_count:1  deleted_byte_count:3495  replica_placement:100  version:3  modified_at_second:1615016911 
-          volume id:373  size:1080928088  collection:"collection2"  file_count:22617  delete_count:4  deleted_byte_count:91849  replica_placement:100  version:3  modified_at_second:1615016878 
-          volume id:374  size:1085011176  collection:"collection2"  file_count:23054  delete_count:2  deleted_byte_count:89034  replica_placement:100  version:3  modified_at_second:1615016917 
-          volume id:376  size:1074845832  collection:"collection2"  file_count:22908  delete_count:4  deleted_byte_count:432305  replica_placement:100  version:3  modified_at_second:1615016916 
-          volume id:377  size:957434264  collection:"collection2"  file_count:14929  delete_count:1  deleted_byte_count:43099  replica_placement:100  version:3  modified_at_second:1615632323 
-          volume id:379  size:1014108528  collection:"collection2"  file_count:15362  delete_count:6  deleted_byte_count:2481613  replica_placement:100  version:3  modified_at_second:1615632323 
-        Disk hdd total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 
-      DataNode 192.168.1.5:8080 total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 
-    Rack DefaultRack total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 
-  DataCenter dc5 total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 
-total size:775256653592 file_count:10478712 deleted_file:33754 deleted_bytes:10839266043 
-`
+//go:embed sample.topo.txt
+var topoData string

weed/shell/command_volume_server_evacuate.go

@@ -78,7 +78,7 @@ func volumeServerEvacuate(commandEnv *CommandEnv, volumeServer string, skipNonMo
 
 	// list all the volumes
 	// collect topology information
-	topologyInfo, _, err := collectTopologyInfo(commandEnv)
+	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
 	if err != nil {
 		return err
 	}

weed/shell/command_volume_tier_download.go

@@ -57,7 +57,7 @@ func (c *commandVolumeTierDownload) Do(args []string, commandEnv *CommandEnv, wr
 	vid := needle.VolumeId(*volumeId)
 
 	// collect topology information
-	topologyInfo, _, err := collectTopologyInfo(commandEnv)
+	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
 	if err != nil {
 		return err
 	}