2.13

from Jethro in slack:
is it possible to make the assign request a bit smarter? Currently I’m in the state that a disk failed but all assign request are being send to this volume. It would be cool if the master sees this and stopped using this volume.
e=HTTP(http://x:8089/913,045a782b63176edf) not 200 but 500 Internal Server Error
Body={"size":740167,"error":"failed to write to local disk: write /mnt/v9/913.dat: input/output error","eTag":"ee4381e202212ff3aee647704c036689"}
e=HTTP(http://x:8089/913,045a782c90240077) not 200 but 500 Internal Server Error
Body={"size":792779,"error":"failed to write to local disk: write /mnt/v9/913.dat: input/output error","eTag":"c43463ccc11eb6eb2fc306f407a6a953"}
e=HTTP(http://x:8089/913,045a782e6b7901ea) not 200 but 500 Internal Server Error
Body={"size":3962392,"error":"failed to write to local disk: write /mnt/v9/913.dat: input/output error","eTag":"04c91198e9b276c81f11dbf189af5d28"}

README.md

@@ -121,15 +121,16 @@ On top of the object store, optional [Filer] can support directories and POSIX a
 
 ## Filer Features ##
 * [Filer server][Filer] provides "normal" directories and files via http.
-* [Super Large Files][SuperLargeFiles] stores large or super large files in tens of TB.
+* [File TTL][FilerTTL] automatically expires file metadata and actual file data.
 * [Mount filer][Mount] reads and writes files directly as a local directory via FUSE.
+* [Filer Store Replication][FilerStoreReplication] enables HA for filer meta data stores.
 * [Active-Active Replication][ActiveActiveAsyncReplication] enables asynchronous one-way or two-way cross cluster continuous replication.
 * [Amazon S3 compatible API][AmazonS3API] accesses files with S3 tooling.
 * [Hadoop Compatible File System][Hadoop] accesses files from Hadoop/Spark/Flink/etc or even runs HBase.
 * [Async Replication To Cloud][BackupToCloud] has extremely fast local access and backups to Amazon S3, Google Cloud Storage, Azure, BackBlaze.
 * [WebDAV] accesses as a mapped drive on Mac and Windows, or from mobile devices.
 * [AES256-GCM Encrypted Storage][FilerDataEncryption] safely stores the encrypted data.
-* [File TTL][FilerTTL] automatically purges file metadata and actual file data.
+* [Super Large Files][SuperLargeFiles] stores large or super large files in tens of TB.
 * [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs-csi-driver.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)
 
 [Filer]: https://github.com/chrislusf/seaweedfs/wiki/Directories-and-Files
@@ -146,6 +147,7 @@ On top of the object store, optional [Filer] can support directories and POSIX a
 [VolumeServerTTL]: https://github.com/chrislusf/seaweedfs/wiki/Store-file-with-a-Time-To-Live
 [SeaweedFsCsiDriver]: https://github.com/seaweedfs/seaweedfs-csi-driver
 [ActiveActiveAsyncReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Active-Active-cross-cluster-continuous-synchronization
+[FilerStoreReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Store-Replication
 
 [Back to TOC](#table-of-contents)
 

go.mod

@@ -40,7 +40,7 @@ require (
 	github.com/json-iterator/go v1.1.10
 	github.com/karlseguin/ccache v2.0.3+incompatible
 	github.com/karlseguin/expect v1.0.1 // indirect
-	github.com/klauspost/compress v1.10.9
+	github.com/klauspost/compress v1.10.9 // indirect
 	github.com/klauspost/cpuid v1.2.1 // indirect
 	github.com/klauspost/crc32 v1.2.0
 	github.com/klauspost/reedsolomon v1.9.2

k8s/seaweedfs/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-version: 2.11
+version: 2.13

k8s/seaweedfs/values.yaml

@@ -4,7 +4,7 @@ global:
   registry: ""
   repository: ""
   imageName: chrislusf/seaweedfs
-  imageTag: "2.11"
+  imageTag: "2.13"
   imagePullPolicy: IfNotPresent
   imagePullSecrets: imagepullsecret
   restartPolicy: Always

other/java/client/pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.5.4</version>
+    <version>1.5.6</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -68,6 +68,11 @@
             <version>4.13.1</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>javax.annotation</groupId>
+            <artifactId>javax.annotation-api</artifactId>
+            <version>1.3.2</version>
+        </dependency>
     </dependencies>
 
     <distributionManagement>

other/java/client/pom.xml.deploy

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.5.4</version>
+    <version>1.5.6</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/pom_debug.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.5.4</version>
+    <version>1.5.6</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/src/main/proto/filer.proto

@@ -364,6 +364,7 @@ message FilerConf {
         }
         DiskType disk_type = 5;
         bool fsync = 6;
+        uint32 volume_growth_count = 7;
     }
     repeated PathConf locations = 2;
 }

other/java/examples/pom.xml

@@ -11,13 +11,13 @@
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-client</artifactId>
-            <version>1.5.4</version>
+            <version>1.5.6</version>
             <scope>compile</scope>
         </dependency>
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-hadoop2-client</artifactId>
-            <version>1.5.4</version>
+            <version>1.5.6</version>
             <scope>compile</scope>
         </dependency>
         <dependency>

other/java/hdfs2/dependency-reduced-pom.xml

@@ -301,7 +301,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.5.4</seaweedfs.client.version>
+    <seaweedfs.client.version>1.5.6</seaweedfs.client.version>
     <hadoop.version>2.9.2</hadoop.version>
   </properties>
 </project>

other/java/hdfs2/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.5.4</seaweedfs.client.version>
+        <seaweedfs.client.version>1.5.6</seaweedfs.client.version>
         <hadoop.version>2.9.2</hadoop.version>
     </properties>
 

other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@@ -25,6 +25,7 @@ public class SeaweedFileSystem extends FileSystem {
     public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
     public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
     public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size";
+    public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication";
     public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024;
 
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
@@ -67,8 +68,8 @@ public class SeaweedFileSystem extends FileSystem {
 
     @Override
     public void close() throws IOException {
-        this.seaweedFileSystemStore.close();
         super.close();
+        this.seaweedFileSystemStore.close();
     }
 
     @Override
@@ -97,7 +98,7 @@ public class SeaweedFileSystem extends FileSystem {
         path = qualify(path);
 
         try {
-            String replicaPlacement = String.format("%03d", replication - 1);
+            String replicaPlacement = this.getConf().get(FS_SEAWEED_REPLICATION, String.format("%03d", replication - 1));
             int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
             OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, seaweedBufferSize, replicaPlacement);
             return new FSDataOutputStream(outputStream, statistics);

other/java/hdfs3/dependency-reduced-pom.xml

@@ -309,7 +309,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.5.4</seaweedfs.client.version>
+    <seaweedfs.client.version>1.5.6</seaweedfs.client.version>
     <hadoop.version>3.1.1</hadoop.version>
   </properties>
 </project>

other/java/hdfs3/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.5.4</seaweedfs.client.version>
+        <seaweedfs.client.version>1.5.6</seaweedfs.client.version>
         <hadoop.version>3.1.1</hadoop.version>
     </properties>
 

other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@@ -25,6 +25,7 @@ public class SeaweedFileSystem extends FileSystem {
     public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
     public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
     public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size";
+    public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication";
     public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024;
 
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
@@ -67,8 +68,8 @@ public class SeaweedFileSystem extends FileSystem {
 
     @Override
     public void close() throws IOException {
-        this.seaweedFileSystemStore.close();
         super.close();
+        this.seaweedFileSystemStore.close();
     }
 
     @Override
@@ -97,7 +98,7 @@ public class SeaweedFileSystem extends FileSystem {
         path = qualify(path);
 
         try {
-            String replicaPlacement = String.format("%03d", replication - 1);
+            String replicaPlacement = this.getConf().get(FS_SEAWEED_REPLICATION, String.format("%03d", replication - 1));
             int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
             OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, seaweedBufferSize, replicaPlacement);
             return new FSDataOutputStream(outputStream, statistics);

other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java

@@ -287,4 +287,4 @@ public class SeaweedFileSystemStore {
 
     }
 
-}
+}

other/metrics/grafana_seaweedfs.json

@@ -386,7 +386,7 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(SeaweedFS_filer_request_total[1m]) * 5",
+              "expr": "rate(SeaweedFS_filer_request_total[1m])",
               "format": "time_series",
               "intervalFactor": 2,
               "legendFormat": "{{type}}",
@@ -761,7 +761,7 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(SeaweedFS_s3_request_total[1m]) * 5",
+              "expr": "rate(SeaweedFS_s3_request_total[1m])",
               "format": "time_series",
               "intervalFactor": 2,
               "legendFormat": "{{type}}",
@@ -1853,4 +1853,4 @@
   "timezone": "browser",
   "title": "SeaweedFS",
   "version": 2
-}
+}

other/metrics/grafana_seaweedfs_k8s.json

@@ -97,7 +97,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(rate(SeaweedFS_s3_request_total{namespace=\"$namespace\",service=~\"$service-api\",type=~\"$method\"}[1m]) * 5) by (code)",
+          "expr": "sum(rate(SeaweedFS_s3_request_total{namespace=\"$namespace\",service=~\"$service-api\",type=~\"$method\"}[1m])) by (code)",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 2,
@@ -210,7 +210,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(rate(SeaweedFS_s3_request_total{namespace=\"$namespace\",service=~\"$service-api\"}[1m]) * 5) by (type)",
+          "expr": "sum(rate(SeaweedFS_s3_request_total{namespace=\"$namespace\",service=~\"$service-api\"}[1m])) by (type)",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 2,
@@ -1014,7 +1014,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "rate(SeaweedFS_filer_request_total{namespace=\"$namespace\",service=~\"$service-api\"}[1m]) * 5",
+          "expr": "rate(SeaweedFS_filer_request_total{namespace=\"$namespace\",service=~\"$service-api\"}[1m])",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 2,
@@ -2359,4 +2359,4 @@
   "timezone": "browser",
   "title": "SeaweedFS",
   "version": 2
-}
+}

unmaintained/s3/benchmark/hsbench.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+hsbench -a accesstoken -s secret -z 4K -d 10 -t 10 -b 10 -u http://localhost:8333 -m "cxipgdx" -bp "hsbench-"

weed/command/backup.go

@@ -112,7 +112,7 @@ func runBackup(cmd *Command, args []string) bool {
 			return true
 		}
 	}
-	v, err := storage.NewVolume(util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0)
+	v, err := storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0)
 	if err != nil {
 		fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
 		return true
@@ -137,7 +137,7 @@ func runBackup(cmd *Command, args []string) bool {
 		// remove the old data
 		v.Destroy()
 		// recreate an empty volume
-		v, err = storage.NewVolume(util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0)
+		v, err = storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0)
 		if err != nil {
 			fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
 			return true

weed/command/compact.go

@@ -41,8 +41,7 @@ func runCompact(cmd *Command, args []string) bool {
 	preallocate := *compactVolumePreallocate * (1 << 20)
 
 	vid := needle.VolumeId(*compactVolumeId)
-	v, err := storage.NewVolume(util.ResolvePath(*compactVolumePath), *compactVolumeCollection, vid,
-		storage.NeedleMapInMemory, nil, nil, preallocate, 0)
+	v, err := storage.NewVolume(util.ResolvePath(*compactVolumePath), util.ResolvePath(*compactVolumePath), *compactVolumeCollection, vid, storage.NeedleMapInMemory, nil, nil, preallocate, 0)
 	if err != nil {
 		glog.Fatalf("Load Volume [ERROR] %s\n", err)
 	}

weed/command/server.go

@@ -105,6 +105,7 @@ func init() {
 	serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
 	serverOptions.v.preStopSeconds = cmdServer.Flag.Int("volume.preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
 	serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
+	serverOptions.v.idxFolder = cmdServer.Flag.String("volume.dir.idx", "", "directory to store .idx files")
 
 	s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
 	s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")

weed/command/volume.go

@@ -39,6 +39,7 @@ type VolumeServerOptions struct {
 	publicPort            *int
 	folders               []string
 	folderMaxLimits       []int
+	idxFolder             *string
 	ip                    *string
 	publicUrl             *string
 	bindIp                *string
@@ -83,6 +84,7 @@ func init() {
 	v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
 	v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
 	v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
+	v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files")
 }
 
 var cmdVolume = &Command{
@@ -211,6 +213,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 	volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
 		*v.ip, *v.port, *v.publicUrl,
 		v.folders, v.folderMaxLimits, v.minFreeSpacePercents,
+		*v.idxFolder,
 		volumeNeedleMapKind,
 		strings.Split(masters, ","), 5, *v.dataCenter, *v.rack,
 		v.whiteList,

weed/filer/filer.go

@@ -269,7 +269,7 @@ func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, e
 	entry, err = f.Store.FindEntry(ctx, p)
 	if entry != nil && entry.TtlSec > 0 {
 		if entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
-			f.Store.DeleteEntry(ctx, p.Child(entry.Name()))
+			f.Store.DeleteOneEntry(ctx, entry)
 			return nil, filer_pb.ErrNotFound
 		}
 	}
@@ -303,7 +303,7 @@ func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, sta
 		lastFileName = entry.Name()
 		if entry.TtlSec > 0 {
 			if entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
-				f.Store.DeleteEntry(ctx, p.Child(entry.Name()))
+				f.Store.DeleteOneEntry(ctx, entry)
 				expiredCount++
 				continue
 			}
@@ -317,11 +317,3 @@ func (f *Filer) Shutdown() {
 	f.LocalMetaLogBuffer.Shutdown()
 	f.Store.Shutdown()
 }
-
-func (f *Filer) maybeDeleteHardLinks(hardLinkIds []HardLinkId) {
-	for _, hardLinkId := range hardLinkIds {
-		if err := f.Store.DeleteHardLink(context.Background(), hardLinkId); err != nil {
-			glog.Errorf("delete hard link id %d : %v", hardLinkId, err)
-		}
-	}
-}

weed/filer/filer_conf.go

@@ -122,6 +122,9 @@ func mergePathConf(a, b *filer_pb.FilerConf_PathConf) {
 		a.DiskType = b.DiskType
 	}
 	a.Fsync = b.Fsync || a.Fsync
+	if b.VolumeGrowthCount > 0 {
+		a.VolumeGrowthCount = b.VolumeGrowthCount
+	}
 }
 
 func (fc *FilerConf) ToProto() *filer_pb.FilerConf {

weed/filer/filer_conf_test.go

@@ -22,7 +22,7 @@ func TestFilerConf(t *testing.T) {
 		},
 		{
 			LocationPrefix: "/buckets/",
-			Replication: "001",
+			Replication:    "001",
 		},
 	}}
 	fc.doLoadConf(conf)

weed/filer/filer_delete_entry.go

@@ -3,6 +3,7 @@ package filer
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
@@ -22,7 +23,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 		return findErr
 	}
 
-	isCollection := f.isBucket(entry)
+	isDeleteCollection := f.isBucket(entry)
 
 	var chunks []*filer_pb.FileChunk
 	var hardLinkIds []HardLinkId
@@ -31,7 +32,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 		// delete the folder children, not including the folder itself
 		var dirChunks []*filer_pb.FileChunk
 		var dirHardLinkIds []HardLinkId
-		dirChunks, dirHardLinkIds, err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks && !isCollection, isFromOtherCluster, signatures)
+		dirChunks, dirHardLinkIds, err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks && !isDeleteCollection, isFromOtherCluster, signatures)
 		if err != nil {
 			glog.V(0).Infof("delete directory %s: %v", p, err)
 			return fmt.Errorf("delete directory %s: %v", p, err)
@@ -46,8 +47,8 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 		return fmt.Errorf("delete file %s: %v", p, err)
 	}
 
-	if shouldDeleteChunks && !isCollection {
-		go f.DeleteChunks(chunks)
+	if shouldDeleteChunks && !isDeleteCollection {
+		f.DirectDeleteChunks(chunks)
 	}
 	// A case not handled:
 	// what if the chunk is in a different collection?
@@ -55,10 +56,15 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 		f.maybeDeleteHardLinks(hardLinkIds)
 	}
 
-	if isCollection {
+	if isDeleteCollection {
 		collectionName := entry.Name()
 		f.doDeleteCollection(collectionName)
 		f.deleteBucket(collectionName)
+	} else {
+		parent, _ := p.DirAndName()
+		if err := f.removeEmptyParentFolder(ctx, util.FullPath(parent)); err != nil {
+			glog.Errorf("clean up empty folders for %s : %v", p, err)
+		}
 	}
 
 	return nil
@@ -122,7 +128,7 @@ func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shou
 
 	glog.V(3).Infof("deleting entry %v, delete chunks: %v", entry.FullPath, shouldDeleteChunks)
 
-	if storeDeletionErr := f.Store.DeleteEntry(ctx, entry.FullPath); storeDeletionErr != nil {
+	if storeDeletionErr := f.Store.DeleteOneEntry(ctx, entry); storeDeletionErr != nil {
 		return fmt.Errorf("filer store delete: %v", storeDeletionErr)
 	}
 	if !entry.IsDirectory() {
@@ -145,3 +151,33 @@ func (f *Filer) doDeleteCollection(collectionName string) (err error) {
 	})
 
 }
+
+func (f *Filer) maybeDeleteHardLinks(hardLinkIds []HardLinkId) {
+	for _, hardLinkId := range hardLinkIds {
+		if err := f.Store.DeleteHardLink(context.Background(), hardLinkId); err != nil {
+			glog.Errorf("delete hard link id %d : %v", hardLinkId, err)
+		}
+	}
+}
+
+func (f *Filer) removeEmptyParentFolder(ctx context.Context, dir util.FullPath) error {
+	if !strings.HasPrefix(string(dir), f.DirBucketsPath) {
+		return nil
+	}
+	parent, _ := dir.DirAndName()
+	if parent == f.DirBucketsPath {
+		// should not delete bucket itself
+		return nil
+	}
+	entries, err := f.ListDirectoryEntries(ctx, dir, "", false, 1, "")
+	if err != nil {
+		return err
+	}
+	if len(entries) > 0 {
+		return nil
+	}
+	if err := f.Store.DeleteEntry(ctx, dir); err != nil {
+		return err
+	}
+	return f.removeEmptyParentFolder(ctx, util.FullPath(parent))
+}

weed/filer/filer_deletion.go

@@ -68,6 +68,50 @@ func (f *Filer) loopProcessingDeletion() {
 	}
 }
 
+func (f *Filer) doDeleteFileIds(fileIds []string) {
+
+	lookupFunc := LookupByMasterClientFn(f.MasterClient)
+	DeletionBatchSize := 100000 // roughly 20 bytes cost per file id.
+
+	for len(fileIds) > 0 {
+		var toDeleteFileIds []string
+		if len(fileIds) > DeletionBatchSize {
+			toDeleteFileIds = fileIds[:DeletionBatchSize]
+			fileIds = fileIds[DeletionBatchSize:]
+		} else {
+			toDeleteFileIds = fileIds
+			fileIds = fileIds[:0]
+		}
+		deletionCount := len(toDeleteFileIds)
+		_, err := operation.DeleteFilesWithLookupVolumeId(f.GrpcDialOption, toDeleteFileIds, lookupFunc)
+		if err != nil {
+			if !strings.Contains(err.Error(), "already deleted") {
+				glog.V(0).Infof("deleting fileIds len=%d error: %v", deletionCount, err)
+			}
+		}
+	}
+}
+
+func (f *Filer) DirectDeleteChunks(chunks []*filer_pb.FileChunk) {
+	var fildIdsToDelete []string
+	for _, chunk := range chunks {
+		if !chunk.IsChunkManifest {
+			fildIdsToDelete = append(fildIdsToDelete, chunk.GetFileIdString())
+			continue
+		}
+		dataChunks, manifestResolveErr := ResolveOneChunkManifest(f.MasterClient.LookupFileId, chunk)
+		if manifestResolveErr != nil {
+			glog.V(0).Infof("failed to resolve manifest %s: %v", chunk.FileId, manifestResolveErr)
+		}
+		for _, dChunk := range dataChunks {
+			fildIdsToDelete = append(fildIdsToDelete, dChunk.GetFileIdString())
+		}
+		fildIdsToDelete = append(fildIdsToDelete, chunk.GetFileIdString())
+	}
+
+	f.doDeleteFileIds(fildIdsToDelete)
+}
+
 func (f *Filer) DeleteChunks(chunks []*filer_pb.FileChunk) {
 	for _, chunk := range chunks {
 		if !chunk.IsChunkManifest {
@@ -107,3 +151,4 @@ func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) {
 	}
 	f.DeleteChunks(toDelete)
 }
+

weed/filer/filer_notify_append.go

@@ -53,7 +53,7 @@ func (f *Filer) assignAndUpload(targetFile string, data []byte) (*operation.Assi
 		Count:               1,
 		Collection:          util.Nvl(f.metaLogCollection, rule.Collection),
 		Replication:         util.Nvl(f.metaLogReplication, rule.Replication),
-		WritableVolumeCount: 1,
+		WritableVolumeCount: rule.VolumeGrowthCount,
 	}
 
 	assignResult, err := operation.Assign(f.GetMaster(), f.GrpcDialOption, assignRequest)

weed/filer/filerstore.go

@@ -45,6 +45,7 @@ type FilerStore interface {
 type VirtualFilerStore interface {
 	FilerStore
 	DeleteHardLink(ctx context.Context, hardLinkId HardLinkId) error
+	DeleteOneEntry(ctx context.Context, entry *Entry) error
 }
 
 type FilerStoreWrapper struct {
@@ -145,6 +146,23 @@ func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp util.FullPath)
 	return fsw.ActualStore.DeleteEntry(ctx, fp)
 }
 
+func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry *Entry) (err error) {
+	stats.FilerStoreCounter.WithLabelValues(fsw.ActualStore.GetName(), "delete").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "delete").Observe(time.Since(start).Seconds())
+	}()
+
+	if len(existingEntry.HardLinkId) != 0 {
+		// remove hard link
+		if err = fsw.DeleteHardLink(ctx, existingEntry.HardLinkId); err != nil {
+			return err
+		}
+	}
+
+	return fsw.ActualStore.DeleteEntry(ctx, existingEntry.FullPath)
+}
+
 func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) {
 	stats.FilerStoreCounter.WithLabelValues(fsw.ActualStore.GetName(), "deleteFolderChildren").Inc()
 	start := time.Now()

weed/filer/leveldb/leveldb_store.go

@@ -162,14 +162,14 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we
 	return nil
 }
 
-func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) {
-	return nil, filer.ErrUnsupportedListDirectoryPrefixed
-}
-
 func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool,
 	limit int) (entries []*filer.Entry, err error) {
+	return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "")
+}
 
-	directoryPrefix := genDirectoryKeyPrefix(fullpath, "")
+func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) {
+
+	directoryPrefix := genDirectoryKeyPrefix(fullpath, prefix)
 
 	iter := store.db.NewIterator(&leveldb_util.Range{Start: genDirectoryKeyPrefix(fullpath, startFileName)}, nil)
 	for iter.Next() {

weed/filer/leveldb2/leveldb2_store.go

@@ -171,14 +171,14 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w
 	return nil
 }
 
-func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) {
-	return nil, filer.ErrUnsupportedListDirectoryPrefixed
-}
-
 func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool,
 	limit int) (entries []*filer.Entry, err error) {
+	return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "")
+}
 
-	directoryPrefix, partitionId := genDirectoryKeyPrefix(fullpath, "", store.dbCount)
+func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) {
+
+	directoryPrefix, partitionId := genDirectoryKeyPrefix(fullpath, prefix, store.dbCount)
 	lastFileStart, _ := genDirectoryKeyPrefix(fullpath, startFileName, store.dbCount)
 
 	iter := store.dbs[partitionId].NewIterator(&leveldb_util.Range{Start: lastFileStart}, nil)

weed/filesys/wfs_filer_client.go

@@ -12,7 +12,7 @@ var _ = filer_pb.FilerClient(&WFS{})
 
 func (wfs *WFS) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	err := util.Retry("filer grpc", func() error {
+	err := util.Retry("filer grpc "+wfs.option.FilerGrpcAddress, func() error {
 		return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
 			client := filer_pb.NewSeaweedFilerClient(grpcConnection)
 			return fn(client)

weed/operation/assign_file_id.go

@@ -50,14 +50,14 @@ func Assign(server string, grpcDialOption grpc.DialOption, primaryRequest *Volum
 		lastError = WithMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
 
 			req := &master_pb.AssignRequest{
-				Count:               primaryRequest.Count,
-				Replication:         primaryRequest.Replication,
-				Collection:          primaryRequest.Collection,
-				Ttl:                 primaryRequest.Ttl,
-				DataCenter:          primaryRequest.DataCenter,
-				Rack:                primaryRequest.Rack,
-				DataNode:            primaryRequest.DataNode,
-				WritableVolumeCount: primaryRequest.WritableVolumeCount,
+				Count:               request.Count,
+				Replication:         request.Replication,
+				Collection:          request.Collection,
+				Ttl:                 request.Ttl,
+				DataCenter:          request.DataCenter,
+				Rack:                request.Rack,
+				DataNode:            request.DataNode,
+				WritableVolumeCount: request.WritableVolumeCount,
 			}
 			resp, grpcErr := masterClient.Assign(context.Background(), req)
 			if grpcErr != nil {
@@ -104,12 +104,13 @@ func LookupJwt(master string, fileId string) security.EncodedJwt {
 }
 
 type StorageOption struct {
-	Replication string
-	Collection  string
-	DataCenter  string
-	Rack        string
-	TtlSeconds  int32
-	Fsync       bool
+	Replication       string
+	Collection        string
+	DataCenter        string
+	Rack              string
+	TtlSeconds        int32
+	Fsync             bool
+	VolumeGrowthCount uint32
 }
 
 func (so *StorageOption) TtlString() string {
@@ -118,21 +119,23 @@ func (so *StorageOption) TtlString() string {
 
 func (so *StorageOption) ToAssignRequests(count int) (ar *VolumeAssignRequest, altRequest *VolumeAssignRequest) {
 	ar = &VolumeAssignRequest{
-		Count:       uint64(count),
-		Replication: so.Replication,
-		Collection:  so.Collection,
-		Ttl:         so.TtlString(),
-		DataCenter:  so.DataCenter,
-		Rack:        so.Rack,
+		Count:               uint64(count),
+		Replication:         so.Replication,
+		Collection:          so.Collection,
+		Ttl:                 so.TtlString(),
+		DataCenter:          so.DataCenter,
+		Rack:                so.Rack,
+		WritableVolumeCount: so.VolumeGrowthCount,
 	}
 	if so.DataCenter != "" || so.Rack != "" {
 		altRequest = &VolumeAssignRequest{
-			Count:       uint64(count),
-			Replication: so.Replication,
-			Collection:  so.Collection,
-			Ttl:         so.TtlString(),
-			DataCenter:  "",
-			Rack:        "",
+			Count:               uint64(count),
+			Replication:         so.Replication,
+			Collection:          so.Collection,
+			Ttl:                 so.TtlString(),
+			DataCenter:          "",
+			Rack:                "",
+			WritableVolumeCount: so.VolumeGrowthCount,
 		}
 	}
 	return

weed/operation/needle_parse_test.go

@@ -75,29 +75,31 @@ func TestCreateNeedleFromRequest(t *testing.T) {
 		Upload("http://localhost:8080/389,0f084d17353afda0", "t.txt", false, bytes.NewReader(gzippedData), true, "text/plain", nil, "")
 	}
 
-	{
-		mc.needleHandling = func(n *needle.Needle, originalSize int, err error) {
-			assert.Equal(t, nil, err, "upload: %v", err)
-			assert.Equal(t, "text/plain", string(n.Mime), "mime detection failed: %v", string(n.Mime))
-			assert.Equal(t, true, n.IsCompressed(), "this should be compressed")
-			assert.Equal(t, true, util.IsZstdContent(n.Data), "this should be zstd")
-			fmt.Printf("needle: %v, dataSize:%d originalSize:%d\n", n, len(n.Data), originalSize)
+	/*
+		{
+			mc.needleHandling = func(n *needle.Needle, originalSize int, err error) {
+				assert.Equal(t, nil, err, "upload: %v", err)
+				assert.Equal(t, "text/plain", string(n.Mime), "mime detection failed: %v", string(n.Mime))
+				assert.Equal(t, true, n.IsCompressed(), "this should be compressed")
+				assert.Equal(t, true, util.IsZstdContent(n.Data), "this should be zstd")
+				fmt.Printf("needle: %v, dataSize:%d originalSize:%d\n", n, len(n.Data), originalSize)
+			}
+			zstdData, _ := util.ZstdData([]byte(textContent))
+			Upload("http://localhost:8080/389,0f084d17353afda0", "t.txt", false, bytes.NewReader(zstdData), true, "text/plain", nil, "")
 		}
-		zstdData, _ := util.ZstdData([]byte(textContent))
-		Upload("http://localhost:8080/389,0f084d17353afda0", "t.txt", false, bytes.NewReader(zstdData), true, "text/plain", nil, "")
-	}
 
-	{
-		mc.needleHandling = func(n *needle.Needle, originalSize int, err error) {
-			assert.Equal(t, nil, err, "upload: %v", err)
-			assert.Equal(t, "application/zstd", string(n.Mime), "mime detection failed: %v", string(n.Mime))
-			assert.Equal(t, false, n.IsCompressed(), "this should not be compressed")
-			assert.Equal(t, true, util.IsZstdContent(n.Data), "this should still be zstd")
-			fmt.Printf("needle: %v, dataSize:%d originalSize:%d\n", n, len(n.Data), originalSize)
+		{
+			mc.needleHandling = func(n *needle.Needle, originalSize int, err error) {
+				assert.Equal(t, nil, err, "upload: %v", err)
+				assert.Equal(t, "application/zstd", string(n.Mime), "mime detection failed: %v", string(n.Mime))
+				assert.Equal(t, false, n.IsCompressed(), "this should not be compressed")
+				assert.Equal(t, true, util.IsZstdContent(n.Data), "this should still be zstd")
+				fmt.Printf("needle: %v, dataSize:%d originalSize:%d\n", n, len(n.Data), originalSize)
+			}
+			zstdData, _ := util.ZstdData([]byte(textContent))
+			Upload("http://localhost:8080/389,0f084d17353afda0", "t.txt", false, bytes.NewReader(zstdData), false, "application/zstd", nil, "")
 		}
-		zstdData, _ := util.ZstdData([]byte(textContent))
-		Upload("http://localhost:8080/389,0f084d17353afda0", "t.txt", false, bytes.NewReader(zstdData), false, "application/zstd", nil, "")
-	}
+	*/
 
 }
 

weed/pb/filer.proto

@@ -364,6 +364,7 @@ message FilerConf {
         }
         DiskType disk_type = 5;
         bool fsync = 6;
+        uint32 volume_growth_count = 7;
     }
     repeated PathConf locations = 2;
 }

weed/pb/filer_pb/filer.pb.go

@@ -3169,12 +3169,13 @@ type FilerConf_PathConf struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	LocationPrefix string                      `protobuf:"bytes,1,opt,name=location_prefix,json=locationPrefix,proto3" json:"location_prefix,omitempty"`
-	Collection     string                      `protobuf:"bytes,2,opt,name=collection,proto3" json:"collection,omitempty"`
-	Replication    string                      `protobuf:"bytes,3,opt,name=replication,proto3" json:"replication,omitempty"`
-	Ttl            string                      `protobuf:"bytes,4,opt,name=ttl,proto3" json:"ttl,omitempty"`
-	DiskType       FilerConf_PathConf_DiskType `protobuf:"varint,5,opt,name=disk_type,json=diskType,proto3,enum=filer_pb.FilerConf_PathConf_DiskType" json:"disk_type,omitempty"`
-	Fsync          bool                        `protobuf:"varint,6,opt,name=fsync,proto3" json:"fsync,omitempty"`
+	LocationPrefix    string                      `protobuf:"bytes,1,opt,name=location_prefix,json=locationPrefix,proto3" json:"location_prefix,omitempty"`
+	Collection        string                      `protobuf:"bytes,2,opt,name=collection,proto3" json:"collection,omitempty"`
+	Replication       string                      `protobuf:"bytes,3,opt,name=replication,proto3" json:"replication,omitempty"`
+	Ttl               string                      `protobuf:"bytes,4,opt,name=ttl,proto3" json:"ttl,omitempty"`
+	DiskType          FilerConf_PathConf_DiskType `protobuf:"varint,5,opt,name=disk_type,json=diskType,proto3,enum=filer_pb.FilerConf_PathConf_DiskType" json:"disk_type,omitempty"`
+	Fsync             bool                        `protobuf:"varint,6,opt,name=fsync,proto3" json:"fsync,omitempty"`
+	VolumeGrowthCount uint32                      `protobuf:"varint,7,opt,name=volume_growth_count,json=volumeGrowthCount,proto3" json:"volume_growth_count,omitempty"`
 }
 
 func (x *FilerConf_PathConf) Reset() {
@@ -3251,6 +3252,13 @@ func (x *FilerConf_PathConf) GetFsync() bool {
 	return false
 }
 
+func (x *FilerConf_PathConf) GetVolumeGrowthCount() uint32 {
+	if x != nil {
+		return x.VolumeGrowthCount
+	}
+	return 0
+}
+
 var File_filer_proto protoreflect.FileDescriptor
 
 var file_filer_proto_rawDesc = []byte{
@@ -3625,13 +3633,13 @@ var file_filer_proto_rawDesc = []byte{
 	0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c,
 	0x75, 0x65, 0x22, 0x25, 0x0a, 0x0d, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f,
 	0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xed, 0x02, 0x0a, 0x09, 0x46, 0x69,
+	0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x9d, 0x03, 0x0a, 0x09, 0x46, 0x69,
 	0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69,
 	0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f,
 	0x6e, 0x12, 0x3a, 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02,
 	0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
 	0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f,
-	0x6e, 0x66, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x89, 0x02,
+	0x6e, 0x66, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0xb9, 0x02,
 	0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x27, 0x0a, 0x0f, 0x6c, 0x6f,
 	0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x01, 0x20,
 	0x01, 0x28, 0x09, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x65,
@@ -3646,7 +3654,10 @@ var file_filer_proto_rawDesc = []byte{
 	0x50, 0x61, 0x74, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70,
 	0x65, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x66,
 	0x73, 0x79, 0x6e, 0x63, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x73, 0x79, 0x6e,
-	0x63, 0x22, 0x26, 0x0a, 0x08, 0x44, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a,
+	0x63, 0x12, 0x2e, 0x0a, 0x13, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x67, 0x72, 0x6f, 0x77,
+	0x74, 0x68, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x11,
+	0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x47, 0x72, 0x6f, 0x77, 0x74, 0x68, 0x43, 0x6f, 0x75, 0x6e,
+	0x74, 0x22, 0x26, 0x0a, 0x08, 0x44, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a,
 	0x04, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x48, 0x44, 0x44, 0x10, 0x01,
 	0x12, 0x07, 0x0a, 0x03, 0x53, 0x53, 0x44, 0x10, 0x02, 0x32, 0xdc, 0x0c, 0x0a, 0x0c, 0x53, 0x65,
 	0x61, 0x77, 0x65, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x12, 0x67, 0x0a, 0x14, 0x4c, 0x6f,

weed/pb/master.proto

@@ -25,6 +25,8 @@ service Seaweed {
     }
     rpc LookupEcVolume (LookupEcVolumeRequest) returns (LookupEcVolumeResponse) {
     }
+    rpc VacuumVolume (VacuumVolumeRequest) returns (VacuumVolumeResponse) {
+    }
     rpc GetMasterConfiguration (GetMasterConfigurationRequest) returns (GetMasterConfigurationResponse) {
     }
     rpc ListMasterClients (ListMasterClientsRequest) returns (ListMasterClientsResponse) {
@@ -264,6 +266,12 @@ message LookupEcVolumeResponse {
     repeated EcShardIdLocation shard_id_locations = 2;
 }
 
+message VacuumVolumeRequest {
+    float garbage_threshold = 1;
+}
+message VacuumVolumeResponse {
+}
+
 message GetMasterConfigurationRequest {
 }
 message GetMasterConfigurationResponse {

weed/pb/master_pb/master.pb.go

@@ -2189,6 +2189,91 @@ func (x *LookupEcVolumeResponse) GetShardIdLocations() []*LookupEcVolumeResponse
 	return nil
 }
 
+type VacuumVolumeRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	GarbageThreshold float32 `protobuf:"fixed32,1,opt,name=garbage_threshold,json=garbageThreshold,proto3" json:"garbage_threshold,omitempty"`
+}
+
+func (x *VacuumVolumeRequest) Reset() {
+	*x = VacuumVolumeRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_master_proto_msgTypes[30]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *VacuumVolumeRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*VacuumVolumeRequest) ProtoMessage() {}
+
+func (x *VacuumVolumeRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_master_proto_msgTypes[30]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use VacuumVolumeRequest.ProtoReflect.Descriptor instead.
+func (*VacuumVolumeRequest) Descriptor() ([]byte, []int) {
+	return file_master_proto_rawDescGZIP(), []int{30}
+}
+
+func (x *VacuumVolumeRequest) GetGarbageThreshold() float32 {
+	if x != nil {
+		return x.GarbageThreshold
+	}
+	return 0
+}
+
+type VacuumVolumeResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *VacuumVolumeResponse) Reset() {
+	*x = VacuumVolumeResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_master_proto_msgTypes[31]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *VacuumVolumeResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*VacuumVolumeResponse) ProtoMessage() {}
+
+func (x *VacuumVolumeResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_master_proto_msgTypes[31]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use VacuumVolumeResponse.ProtoReflect.Descriptor instead.
+func (*VacuumVolumeResponse) Descriptor() ([]byte, []int) {
+	return file_master_proto_rawDescGZIP(), []int{31}
+}
+
 type GetMasterConfigurationRequest struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -2198,7 +2283,7 @@ type GetMasterConfigurationRequest struct {
 func (x *GetMasterConfigurationRequest) Reset() {
 	*x = GetMasterConfigurationRequest{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_master_proto_msgTypes[30]
+		mi := &file_master_proto_msgTypes[32]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -2211,7 +2296,7 @@ func (x *GetMasterConfigurationRequest) String() string {
 func (*GetMasterConfigurationRequest) ProtoMessage() {}
 
 func (x *GetMasterConfigurationRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_master_proto_msgTypes[30]
+	mi := &file_master_proto_msgTypes[32]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -2224,7 +2309,7 @@ func (x *GetMasterConfigurationRequest) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use GetMasterConfigurationRequest.ProtoReflect.Descriptor instead.
 func (*GetMasterConfigurationRequest) Descriptor() ([]byte, []int) {
-	return file_master_proto_rawDescGZIP(), []int{30}
+	return file_master_proto_rawDescGZIP(), []int{32}
 }
 
 type GetMasterConfigurationResponse struct {
@@ -2242,7 +2327,7 @@ type GetMasterConfigurationResponse struct {
 func (x *GetMasterConfigurationResponse) Reset() {
 	*x = GetMasterConfigurationResponse{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_master_proto_msgTypes[31]
+		mi := &file_master_proto_msgTypes[33]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -2255,7 +2340,7 @@ func (x *GetMasterConfigurationResponse) String() string {
 func (*GetMasterConfigurationResponse) ProtoMessage() {}
 
 func (x *GetMasterConfigurationResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_master_proto_msgTypes[31]
+	mi := &file_master_proto_msgTypes[33]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -2268,7 +2353,7 @@ func (x *GetMasterConfigurationResponse) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use GetMasterConfigurationResponse.ProtoReflect.Descriptor instead.
 func (*GetMasterConfigurationResponse) Descriptor() ([]byte, []int) {
-	return file_master_proto_rawDescGZIP(), []int{31}
+	return file_master_proto_rawDescGZIP(), []int{33}
 }
 
 func (x *GetMasterConfigurationResponse) GetMetricsAddress() string {
@@ -2317,7 +2402,7 @@ type ListMasterClientsRequest struct {
 func (x *ListMasterClientsRequest) Reset() {
 	*x = ListMasterClientsRequest{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_master_proto_msgTypes[32]
+		mi := &file_master_proto_msgTypes[34]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -2330,7 +2415,7 @@ func (x *ListMasterClientsRequest) String() string {
 func (*ListMasterClientsRequest) ProtoMessage() {}
 
 func (x *ListMasterClientsRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_master_proto_msgTypes[32]
+	mi := &file_master_proto_msgTypes[34]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -2343,7 +2428,7 @@ func (x *ListMasterClientsRequest) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use ListMasterClientsRequest.ProtoReflect.Descriptor instead.
 func (*ListMasterClientsRequest) Descriptor() ([]byte, []int) {
-	return file_master_proto_rawDescGZIP(), []int{32}
+	return file_master_proto_rawDescGZIP(), []int{34}
 }
 
 func (x *ListMasterClientsRequest) GetClientType() string {
@@ -2364,7 +2449,7 @@ type ListMasterClientsResponse struct {
 func (x *ListMasterClientsResponse) Reset() {
 	*x = ListMasterClientsResponse{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_master_proto_msgTypes[33]
+		mi := &file_master_proto_msgTypes[35]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -2377,7 +2462,7 @@ func (x *ListMasterClientsResponse) String() string {
 func (*ListMasterClientsResponse) ProtoMessage() {}
 
 func (x *ListMasterClientsResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_master_proto_msgTypes[33]
+	mi := &file_master_proto_msgTypes[35]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -2390,7 +2475,7 @@ func (x *ListMasterClientsResponse) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use ListMasterClientsResponse.ProtoReflect.Descriptor instead.
 func (*ListMasterClientsResponse) Descriptor() ([]byte, []int) {
-	return file_master_proto_rawDescGZIP(), []int{33}
+	return file_master_proto_rawDescGZIP(), []int{35}
 }
 
 func (x *ListMasterClientsResponse) GetGrpcAddresses() []string {
@@ -2413,7 +2498,7 @@ type LeaseAdminTokenRequest struct {
 func (x *LeaseAdminTokenRequest) Reset() {
 	*x = LeaseAdminTokenRequest{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_master_proto_msgTypes[34]
+		mi := &file_master_proto_msgTypes[36]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -2426,7 +2511,7 @@ func (x *LeaseAdminTokenRequest) String() string {
 func (*LeaseAdminTokenRequest) ProtoMessage() {}
 
 func (x *LeaseAdminTokenRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_master_proto_msgTypes[34]
+	mi := &file_master_proto_msgTypes[36]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -2439,7 +2524,7 @@ func (x *LeaseAdminTokenRequest) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use LeaseAdminTokenRequest.ProtoReflect.Descriptor instead.
 func (*LeaseAdminTokenRequest) Descriptor() ([]byte, []int) {
-	return file_master_proto_rawDescGZIP(), []int{34}
+	return file_master_proto_rawDescGZIP(), []int{36}
 }
 
 func (x *LeaseAdminTokenRequest) GetPreviousToken() int64 {
@@ -2475,7 +2560,7 @@ type LeaseAdminTokenResponse struct {
 func (x *LeaseAdminTokenResponse) Reset() {
 	*x = LeaseAdminTokenResponse{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_master_proto_msgTypes[35]
+		mi := &file_master_proto_msgTypes[37]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -2488,7 +2573,7 @@ func (x *LeaseAdminTokenResponse) String() string {
 func (*LeaseAdminTokenResponse) ProtoMessage() {}
 
 func (x *LeaseAdminTokenResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_master_proto_msgTypes[35]
+	mi := &file_master_proto_msgTypes[37]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -2501,7 +2586,7 @@ func (x *LeaseAdminTokenResponse) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use LeaseAdminTokenResponse.ProtoReflect.Descriptor instead.
 func (*LeaseAdminTokenResponse) Descriptor() ([]byte, []int) {
-	return file_master_proto_rawDescGZIP(), []int{35}
+	return file_master_proto_rawDescGZIP(), []int{37}
 }
 
 func (x *LeaseAdminTokenResponse) GetToken() int64 {
@@ -2531,7 +2616,7 @@ type ReleaseAdminTokenRequest struct {
 func (x *ReleaseAdminTokenRequest) Reset() {
 	*x = ReleaseAdminTokenRequest{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_master_proto_msgTypes[36]
+		mi := &file_master_proto_msgTypes[38]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -2544,7 +2629,7 @@ func (x *ReleaseAdminTokenRequest) String() string {
 func (*ReleaseAdminTokenRequest) ProtoMessage() {}
 
 func (x *ReleaseAdminTokenRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_master_proto_msgTypes[36]
+	mi := &file_master_proto_msgTypes[38]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -2557,7 +2642,7 @@ func (x *ReleaseAdminTokenRequest) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use ReleaseAdminTokenRequest.ProtoReflect.Descriptor instead.
 func (*ReleaseAdminTokenRequest) Descriptor() ([]byte, []int) {
-	return file_master_proto_rawDescGZIP(), []int{36}
+	return file_master_proto_rawDescGZIP(), []int{38}
 }
 
 func (x *ReleaseAdminTokenRequest) GetPreviousToken() int64 {
@@ -2590,7 +2675,7 @@ type ReleaseAdminTokenResponse struct {
 func (x *ReleaseAdminTokenResponse) Reset() {
 	*x = ReleaseAdminTokenResponse{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_master_proto_msgTypes[37]
+		mi := &file_master_proto_msgTypes[39]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -2603,7 +2688,7 @@ func (x *ReleaseAdminTokenResponse) String() string {
 func (*ReleaseAdminTokenResponse) ProtoMessage() {}
 
 func (x *ReleaseAdminTokenResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_master_proto_msgTypes[37]
+	mi := &file_master_proto_msgTypes[39]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -2616,7 +2701,7 @@ func (x *ReleaseAdminTokenResponse) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use ReleaseAdminTokenResponse.ProtoReflect.Descriptor instead.
 func (*ReleaseAdminTokenResponse) Descriptor() ([]byte, []int) {
-	return file_master_proto_rawDescGZIP(), []int{37}
+	return file_master_proto_rawDescGZIP(), []int{39}
 }
 
 type SuperBlockExtra_ErasureCoding struct {
@@ -2632,7 +2717,7 @@ type SuperBlockExtra_ErasureCoding struct {
 func (x *SuperBlockExtra_ErasureCoding) Reset() {
 	*x = SuperBlockExtra_ErasureCoding{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_master_proto_msgTypes[39]
+		mi := &file_master_proto_msgTypes[41]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -2645,7 +2730,7 @@ func (x *SuperBlockExtra_ErasureCoding) String() string {
 func (*SuperBlockExtra_ErasureCoding) ProtoMessage() {}
 
 func (x *SuperBlockExtra_ErasureCoding) ProtoReflect() protoreflect.Message {
-	mi := &file_master_proto_msgTypes[39]
+	mi := &file_master_proto_msgTypes[41]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -2695,7 +2780,7 @@ type LookupVolumeResponse_VolumeIdLocation struct {
 func (x *LookupVolumeResponse_VolumeIdLocation) Reset() {
 	*x = LookupVolumeResponse_VolumeIdLocation{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_master_proto_msgTypes[40]
+		mi := &file_master_proto_msgTypes[42]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -2708,7 +2793,7 @@ func (x *LookupVolumeResponse_VolumeIdLocation) String() string {
 func (*LookupVolumeResponse_VolumeIdLocation) ProtoMessage() {}
 
 func (x *LookupVolumeResponse_VolumeIdLocation) ProtoReflect() protoreflect.Message {
-	mi := &file_master_proto_msgTypes[40]
+	mi := &file_master_proto_msgTypes[42]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -2757,7 +2842,7 @@ type LookupEcVolumeResponse_EcShardIdLocation struct {
 func (x *LookupEcVolumeResponse_EcShardIdLocation) Reset() {
 	*x = LookupEcVolumeResponse_EcShardIdLocation{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_master_proto_msgTypes[41]
+		mi := &file_master_proto_msgTypes[43]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -2770,7 +2855,7 @@ func (x *LookupEcVolumeResponse_EcShardIdLocation) String() string {
 func (*LookupEcVolumeResponse_EcShardIdLocation) ProtoMessage() {}
 
 func (x *LookupEcVolumeResponse_EcShardIdLocation) ProtoReflect() protoreflect.Message {
-	mi := &file_master_proto_msgTypes[41]
+	mi := &file_master_proto_msgTypes[43]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -3165,134 +3250,145 @@ var file_master_proto_rawDesc = []byte{
 	0x64, 0x12, 0x31, 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02,
 	0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
 	0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74,
-	0x69, 0x6f, 0x6e, 0x73, 0x22, 0x1f, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65,
-	0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x92, 0x02, 0x0a, 0x1e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73,
-	0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e,
-	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x6d, 0x65, 0x74, 0x72,
-	0x69, 0x63, 0x73, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x0e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73,
-	0x73, 0x12, 0x38, 0x0a, 0x18, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x69, 0x6e, 0x74,
-	0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x02, 0x20,
-	0x01, 0x28, 0x0d, 0x52, 0x16, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x49, 0x6e, 0x74, 0x65,
-	0x72, 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x44, 0x0a, 0x10, 0x73,
-	0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x18,
-	0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
-	0x62, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64,
-	0x52, 0x0f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64,
-	0x73, 0x12, 0x2f, 0x0a, 0x13, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x72, 0x65, 0x70,
-	0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12,
-	0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69,
-	0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x06, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x22, 0x3b, 0x0a, 0x18, 0x4c, 0x69,
-	0x73, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x52,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74,
-	0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6c, 0x69,
-	0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0x42, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x4d,
-	0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70,
-	0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x61, 0x64, 0x64,
-	0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x67, 0x72,
-	0x70, 0x63, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x22, 0x8a, 0x01, 0x0a, 0x16,
-	0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x70, 0x72, 0x65, 0x76, 0x69, 0x6f,
-	0x75, 0x73, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d,
-	0x70, 0x72, 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2c, 0x0a,
-	0x12, 0x70, 0x72, 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x5f, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x74,
-	0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x70, 0x72, 0x65, 0x76, 0x69,
-	0x6f, 0x75, 0x73, 0x4c, 0x6f, 0x63, 0x6b, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6c,
-	0x6f, 0x63, 0x6b, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08,
-	0x6c, 0x6f, 0x63, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x4d, 0x0a, 0x17, 0x4c, 0x65, 0x61, 0x73,
-	0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f,
-	0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01,
-	0x28, 0x03, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1c, 0x0a, 0x0a, 0x6c, 0x6f, 0x63,
-	0x6b, 0x5f, 0x74, 0x73, 0x5f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x6c,
-	0x6f, 0x63, 0x6b, 0x54, 0x73, 0x4e, 0x73, 0x22, 0x8c, 0x01, 0x0a, 0x18, 0x52, 0x65, 0x6c, 0x65,
-	0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x70, 0x72, 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73,
-	0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x70, 0x72,
-	0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2c, 0x0a, 0x12, 0x70,
-	0x72, 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x5f, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x74, 0x69, 0x6d,
-	0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x70, 0x72, 0x65, 0x76, 0x69, 0x6f, 0x75,
-	0x73, 0x4c, 0x6f, 0x63, 0x6b, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6c, 0x6f, 0x63,
-	0x6b, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f,
-	0x63, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x1b, 0x0a, 0x19, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73,
-	0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f,
-	0x6e, 0x73, 0x65, 0x32, 0xf7, 0x08, 0x0a, 0x07, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x12,
-	0x49, 0x0a, 0x0d, 0x53, 0x65, 0x6e, 0x64, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74,
-	0x12, 0x14, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x48, 0x65, 0x61,
-	0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f,
-	0x70, 0x62, 0x2e, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x52, 0x65, 0x73, 0x70,
-	0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x51, 0x0a, 0x0d, 0x4b, 0x65,
-	0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x1f, 0x2e, 0x6d, 0x61,
-	0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e,
-	0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6d,
-	0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c,
-	0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x51, 0x0a,
-	0x0c, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1e, 0x2e,
-	0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70,
-	0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e,
-	0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70,
-	0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00,
-	0x12, 0x3f, 0x0a, 0x06, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x12, 0x18, 0x2e, 0x6d, 0x61, 0x73,
-	0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
-	0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
-	0x00, 0x12, 0x4b, 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12,
-	0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74,
-	0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e,
-	0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73,
-	0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x57,
-	0x0a, 0x0e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74,
-	0x12, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c,
-	0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65,
-	0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43,
-	0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x10, 0x43, 0x6f, 0x6c, 0x6c, 0x65,
-	0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x22, 0x2e, 0x6d, 0x61,
+	0x69, 0x6f, 0x6e, 0x73, 0x22, 0x42, 0x0a, 0x13, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f,
+	0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x11, 0x67,
+	0x61, 0x72, 0x62, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x02, 0x52, 0x10, 0x67, 0x61, 0x72, 0x62, 0x61, 0x67, 0x65, 0x54,
+	0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x22, 0x16, 0x0a, 0x14, 0x56, 0x61, 0x63, 0x75,
+	0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x22, 0x1f, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e,
+	0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x22, 0x92, 0x02, 0x0a, 0x1e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43,
+	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f,
+	0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6d,
+	0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x38, 0x0a,
+	0x18, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61,
+	0x6c, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52,
+	0x16, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c,
+	0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x44, 0x0a, 0x10, 0x73, 0x74, 0x6f, 0x72, 0x61,
+	0x67, 0x65, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28,
+	0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74,
+	0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x0f, 0x73, 0x74,
+	0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x12, 0x2f, 0x0a,
+	0x13, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x64, 0x65, 0x66, 0x61,
+	0x75, 0x6c, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x16,
+	0x0a, 0x06, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06,
+	0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x22, 0x3b, 0x0a, 0x18, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61,
+	0x73, 0x74, 0x65, 0x72, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x79, 0x70,
+	0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54,
+	0x79, 0x70, 0x65, 0x22, 0x42, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65,
+	0x72, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x12, 0x25, 0x0a, 0x0e, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73,
+	0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x67, 0x72, 0x70, 0x63, 0x41, 0x64,
+	0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x22, 0x8a, 0x01, 0x0a, 0x16, 0x4c, 0x65, 0x61, 0x73,
+	0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x70, 0x72, 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x5f, 0x74,
+	0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x70, 0x72, 0x65, 0x76,
+	0x69, 0x6f, 0x75, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2c, 0x0a, 0x12, 0x70, 0x72, 0x65,
+	0x76, 0x69, 0x6f, 0x75, 0x73, 0x5f, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x70, 0x72, 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x4c,
+	0x6f, 0x63, 0x6b, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x6b, 0x5f,
+	0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x6b,
+	0x4e, 0x61, 0x6d, 0x65, 0x22, 0x4d, 0x0a, 0x17, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d,
+	0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
+	0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05,
+	0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1c, 0x0a, 0x0a, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x74, 0x73,
+	0x5f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x6b, 0x54,
+	0x73, 0x4e, 0x73, 0x22, 0x8c, 0x01, 0x0a, 0x18, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41,
+	0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x12, 0x25, 0x0a, 0x0e, 0x70, 0x72, 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x5f, 0x74, 0x6f, 0x6b,
+	0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x70, 0x72, 0x65, 0x76, 0x69, 0x6f,
+	0x75, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2c, 0x0a, 0x12, 0x70, 0x72, 0x65, 0x76, 0x69,
+	0x6f, 0x75, 0x73, 0x5f, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x03, 0x52, 0x10, 0x70, 0x72, 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x4c, 0x6f, 0x63,
+	0x6b, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6e, 0x61,
+	0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x6b, 0x4e, 0x61,
+	0x6d, 0x65, 0x22, 0x1b, 0x0a, 0x19, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d,
+	0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32,
+	0xca, 0x09, 0x0a, 0x07, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x12, 0x49, 0x0a, 0x0d, 0x53,
+	0x65, 0x6e, 0x64, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x12, 0x14, 0x2e, 0x6d,
+	0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65,
+	0x61, 0x74, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x48,
+	0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x51, 0x0a, 0x0d, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f,
+	0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72,
+	0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65,
+	0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65,
+	0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74,
+	0x69, 0x6f, 0x6e, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x51, 0x0a, 0x0c, 0x4c, 0x6f, 0x6f,
+	0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74,
+	0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75,
+	0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74,
+	0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75,
+	0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x06,
+	0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x12, 0x18, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f,
+	0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x1a, 0x19, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73,
+	0x69, 0x67, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a,
+	0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x1c, 0x2e, 0x6d, 0x61,
+	0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69,
+	0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x73, 0x74,
+	0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e, 0x43, 0x6f,
+	0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x20, 0x2e, 0x6d,
+	0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74,
+	0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21,
+	0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65,
+	0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x10, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f,
+	0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72,
+	0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65,
+	0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61,
 	0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69,
-	0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
-	0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c,
-	0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70,
-	0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65,
-	0x4c, 0x69, 0x73, 0x74, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
-	0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65,
-	0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56,
-	0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
-	0x65, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56,
-	0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
-	0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72,
-	0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75,
-	0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x16,
-	0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75,
-	0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f,
-	0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66,
-	0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x1a, 0x29, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74,
-	0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74,
-	0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a,
-	0x11, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6c, 0x69, 0x65, 0x6e,
-	0x74, 0x73, 0x12, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c,
-	0x69, 0x73, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72,
-	0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6c,
-	0x69, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12,
-	0x5a, 0x0a, 0x0f, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b,
-	0x65, 0x6e, 0x12, 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c,
-	0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
-	0x62, 0x2e, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65,
-	0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x11, 0x52,
-	0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e,
-	0x12, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6c,
-	0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
-	0x62, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f,
-	0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x32, 0x5a,
-	0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69,
-	0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f,
-	0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
-	0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74,
+	0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c,
+	0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d,
+	0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d,
+	0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12,
+	0x57, 0x0a, 0x0e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d,
+	0x65, 0x12, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f,
+	0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75,
+	0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
+	0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x51, 0x0a, 0x0c, 0x56, 0x61, 0x63, 0x75,
+	0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65,
+	0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d,
+	0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65,
+	0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d,
+	0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x16, 0x47,
+	0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72,
+	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
+	0x62, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69,
+	0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
+	0x29, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4d,
+	0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69,
+	0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x11,
+	0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74,
+	0x73, 0x12, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69,
+	0x73, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f,
+	0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6c, 0x69,
+	0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5a,
+	0x0a, 0x0f, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65,
+	0x6e, 0x12, 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x65,
+	0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
+	0x2e, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x11, 0x52, 0x65,
+	0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12,
+	0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6c, 0x65,
+	0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
+	0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b,
+	0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x32, 0x5a, 0x30,
+	0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73,
+	0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77,
+	0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
+	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -3307,7 +3403,7 @@ func file_master_proto_rawDescGZIP() []byte {
 	return file_master_proto_rawDescData
 }
 
-var file_master_proto_msgTypes = make([]protoimpl.MessageInfo, 42)
+var file_master_proto_msgTypes = make([]protoimpl.MessageInfo, 44)
 var file_master_proto_goTypes = []interface{}{
 	(*Heartbeat)(nil),                                // 0: master_pb.Heartbeat
 	(*HeartbeatResponse)(nil),                        // 1: master_pb.HeartbeatResponse
@@ -3339,18 +3435,20 @@ var file_master_proto_goTypes = []interface{}{
 	(*VolumeListResponse)(nil),                       // 27: master_pb.VolumeListResponse
 	(*LookupEcVolumeRequest)(nil),                    // 28: master_pb.LookupEcVolumeRequest
 	(*LookupEcVolumeResponse)(nil),                   // 29: master_pb.LookupEcVolumeResponse
-	(*GetMasterConfigurationRequest)(nil),            // 30: master_pb.GetMasterConfigurationRequest
-	(*GetMasterConfigurationResponse)(nil),           // 31: master_pb.GetMasterConfigurationResponse
-	(*ListMasterClientsRequest)(nil),                 // 32: master_pb.ListMasterClientsRequest
-	(*ListMasterClientsResponse)(nil),                // 33: master_pb.ListMasterClientsResponse
-	(*LeaseAdminTokenRequest)(nil),                   // 34: master_pb.LeaseAdminTokenRequest
-	(*LeaseAdminTokenResponse)(nil),                  // 35: master_pb.LeaseAdminTokenResponse
-	(*ReleaseAdminTokenRequest)(nil),                 // 36: master_pb.ReleaseAdminTokenRequest
-	(*ReleaseAdminTokenResponse)(nil),                // 37: master_pb.ReleaseAdminTokenResponse
-	nil,                                              // 38: master_pb.StorageBackend.PropertiesEntry
-	(*SuperBlockExtra_ErasureCoding)(nil),            // 39: master_pb.SuperBlockExtra.ErasureCoding
-	(*LookupVolumeResponse_VolumeIdLocation)(nil),    // 40: master_pb.LookupVolumeResponse.VolumeIdLocation
-	(*LookupEcVolumeResponse_EcShardIdLocation)(nil), // 41: master_pb.LookupEcVolumeResponse.EcShardIdLocation
+	(*VacuumVolumeRequest)(nil),                      // 30: master_pb.VacuumVolumeRequest
+	(*VacuumVolumeResponse)(nil),                     // 31: master_pb.VacuumVolumeResponse
+	(*GetMasterConfigurationRequest)(nil),            // 32: master_pb.GetMasterConfigurationRequest
+	(*GetMasterConfigurationResponse)(nil),           // 33: master_pb.GetMasterConfigurationResponse
+	(*ListMasterClientsRequest)(nil),                 // 34: master_pb.ListMasterClientsRequest
+	(*ListMasterClientsResponse)(nil),                // 35: master_pb.ListMasterClientsResponse
+	(*LeaseAdminTokenRequest)(nil),                   // 36: master_pb.LeaseAdminTokenRequest
+	(*LeaseAdminTokenResponse)(nil),                  // 37: master_pb.LeaseAdminTokenResponse
+	(*ReleaseAdminTokenRequest)(nil),                 // 38: master_pb.ReleaseAdminTokenRequest
+	(*ReleaseAdminTokenResponse)(nil),                // 39: master_pb.ReleaseAdminTokenResponse
+	nil,                                              // 40: master_pb.StorageBackend.PropertiesEntry
+	(*SuperBlockExtra_ErasureCoding)(nil),            // 41: master_pb.SuperBlockExtra.ErasureCoding
+	(*LookupVolumeResponse_VolumeIdLocation)(nil),    // 42: master_pb.LookupVolumeResponse.VolumeIdLocation
+	(*LookupEcVolumeResponse_EcShardIdLocation)(nil), // 43: master_pb.LookupEcVolumeResponse.EcShardIdLocation
 }
 var file_master_proto_depIdxs = []int32{
 	2,  // 0: master_pb.Heartbeat.volumes:type_name -> master_pb.VolumeInformationMessage
@@ -3360,9 +3458,9 @@ var file_master_proto_depIdxs = []int32{
 	4,  // 4: master_pb.Heartbeat.new_ec_shards:type_name -> master_pb.VolumeEcShardInformationMessage
 	4,  // 5: master_pb.Heartbeat.deleted_ec_shards:type_name -> master_pb.VolumeEcShardInformationMessage
 	5,  // 6: master_pb.HeartbeatResponse.storage_backends:type_name -> master_pb.StorageBackend
-	38, // 7: master_pb.StorageBackend.properties:type_name -> master_pb.StorageBackend.PropertiesEntry
-	39, // 8: master_pb.SuperBlockExtra.erasure_coding:type_name -> master_pb.SuperBlockExtra.ErasureCoding
-	40, // 9: master_pb.LookupVolumeResponse.volume_id_locations:type_name -> master_pb.LookupVolumeResponse.VolumeIdLocation
+	40, // 7: master_pb.StorageBackend.properties:type_name -> master_pb.StorageBackend.PropertiesEntry
+	41, // 8: master_pb.SuperBlockExtra.erasure_coding:type_name -> master_pb.SuperBlockExtra.ErasureCoding
+	42, // 9: master_pb.LookupVolumeResponse.volume_id_locations:type_name -> master_pb.LookupVolumeResponse.VolumeIdLocation
 	17, // 10: master_pb.CollectionListResponse.collections:type_name -> master_pb.Collection
 	2,  // 11: master_pb.DataNodeInfo.volume_infos:type_name -> master_pb.VolumeInformationMessage
 	4,  // 12: master_pb.DataNodeInfo.ec_shard_infos:type_name -> master_pb.VolumeEcShardInformationMessage
@@ -3370,7 +3468,7 @@ var file_master_proto_depIdxs = []int32{
 	23, // 14: master_pb.DataCenterInfo.rack_infos:type_name -> master_pb.RackInfo
 	24, // 15: master_pb.TopologyInfo.data_center_infos:type_name -> master_pb.DataCenterInfo
 	25, // 16: master_pb.VolumeListResponse.topology_info:type_name -> master_pb.TopologyInfo
-	41, // 17: master_pb.LookupEcVolumeResponse.shard_id_locations:type_name -> master_pb.LookupEcVolumeResponse.EcShardIdLocation
+	43, // 17: master_pb.LookupEcVolumeResponse.shard_id_locations:type_name -> master_pb.LookupEcVolumeResponse.EcShardIdLocation
 	5,  // 18: master_pb.GetMasterConfigurationResponse.storage_backends:type_name -> master_pb.StorageBackend
 	12, // 19: master_pb.LookupVolumeResponse.VolumeIdLocation.locations:type_name -> master_pb.Location
 	12, // 20: master_pb.LookupEcVolumeResponse.EcShardIdLocation.locations:type_name -> master_pb.Location
@@ -3383,25 +3481,27 @@ var file_master_proto_depIdxs = []int32{
 	20, // 27: master_pb.Seaweed.CollectionDelete:input_type -> master_pb.CollectionDeleteRequest
 	26, // 28: master_pb.Seaweed.VolumeList:input_type -> master_pb.VolumeListRequest
 	28, // 29: master_pb.Seaweed.LookupEcVolume:input_type -> master_pb.LookupEcVolumeRequest
-	30, // 30: master_pb.Seaweed.GetMasterConfiguration:input_type -> master_pb.GetMasterConfigurationRequest
-	32, // 31: master_pb.Seaweed.ListMasterClients:input_type -> master_pb.ListMasterClientsRequest
-	34, // 32: master_pb.Seaweed.LeaseAdminToken:input_type -> master_pb.LeaseAdminTokenRequest
-	36, // 33: master_pb.Seaweed.ReleaseAdminToken:input_type -> master_pb.ReleaseAdminTokenRequest
-	1,  // 34: master_pb.Seaweed.SendHeartbeat:output_type -> master_pb.HeartbeatResponse
-	9,  // 35: master_pb.Seaweed.KeepConnected:output_type -> master_pb.VolumeLocation
-	11, // 36: master_pb.Seaweed.LookupVolume:output_type -> master_pb.LookupVolumeResponse
-	14, // 37: master_pb.Seaweed.Assign:output_type -> master_pb.AssignResponse
-	16, // 38: master_pb.Seaweed.Statistics:output_type -> master_pb.StatisticsResponse
-	19, // 39: master_pb.Seaweed.CollectionList:output_type -> master_pb.CollectionListResponse
-	21, // 40: master_pb.Seaweed.CollectionDelete:output_type -> master_pb.CollectionDeleteResponse
-	27, // 41: master_pb.Seaweed.VolumeList:output_type -> master_pb.VolumeListResponse
-	29, // 42: master_pb.Seaweed.LookupEcVolume:output_type -> master_pb.LookupEcVolumeResponse
-	31, // 43: master_pb.Seaweed.GetMasterConfiguration:output_type -> master_pb.GetMasterConfigurationResponse
-	33, // 44: master_pb.Seaweed.ListMasterClients:output_type -> master_pb.ListMasterClientsResponse
-	35, // 45: master_pb.Seaweed.LeaseAdminToken:output_type -> master_pb.LeaseAdminTokenResponse
-	37, // 46: master_pb.Seaweed.ReleaseAdminToken:output_type -> master_pb.ReleaseAdminTokenResponse
-	34, // [34:47] is the sub-list for method output_type
-	21, // [21:34] is the sub-list for method input_type
+	30, // 30: master_pb.Seaweed.VacuumVolume:input_type -> master_pb.VacuumVolumeRequest
+	32, // 31: master_pb.Seaweed.GetMasterConfiguration:input_type -> master_pb.GetMasterConfigurationRequest
+	34, // 32: master_pb.Seaweed.ListMasterClients:input_type -> master_pb.ListMasterClientsRequest
+	36, // 33: master_pb.Seaweed.LeaseAdminToken:input_type -> master_pb.LeaseAdminTokenRequest
+	38, // 34: master_pb.Seaweed.ReleaseAdminToken:input_type -> master_pb.ReleaseAdminTokenRequest
+	1,  // 35: master_pb.Seaweed.SendHeartbeat:output_type -> master_pb.HeartbeatResponse
+	9,  // 36: master_pb.Seaweed.KeepConnected:output_type -> master_pb.VolumeLocation
+	11, // 37: master_pb.Seaweed.LookupVolume:output_type -> master_pb.LookupVolumeResponse
+	14, // 38: master_pb.Seaweed.Assign:output_type -> master_pb.AssignResponse
+	16, // 39: master_pb.Seaweed.Statistics:output_type -> master_pb.StatisticsResponse
+	19, // 40: master_pb.Seaweed.CollectionList:output_type -> master_pb.CollectionListResponse
+	21, // 41: master_pb.Seaweed.CollectionDelete:output_type -> master_pb.CollectionDeleteResponse
+	27, // 42: master_pb.Seaweed.VolumeList:output_type -> master_pb.VolumeListResponse
+	29, // 43: master_pb.Seaweed.LookupEcVolume:output_type -> master_pb.LookupEcVolumeResponse
+	31, // 44: master_pb.Seaweed.VacuumVolume:output_type -> master_pb.VacuumVolumeResponse
+	33, // 45: master_pb.Seaweed.GetMasterConfiguration:output_type -> master_pb.GetMasterConfigurationResponse
+	35, // 46: master_pb.Seaweed.ListMasterClients:output_type -> master_pb.ListMasterClientsResponse
+	37, // 47: master_pb.Seaweed.LeaseAdminToken:output_type -> master_pb.LeaseAdminTokenResponse
+	39, // 48: master_pb.Seaweed.ReleaseAdminToken:output_type -> master_pb.ReleaseAdminTokenResponse
+	35, // [35:49] is the sub-list for method output_type
+	21, // [21:35] is the sub-list for method input_type
 	21, // [21:21] is the sub-list for extension type_name
 	21, // [21:21] is the sub-list for extension extendee
 	0,  // [0:21] is the sub-list for field type_name
@@ -3774,7 +3874,7 @@ func file_master_proto_init() {
 			}
 		}
 		file_master_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*GetMasterConfigurationRequest); i {
+			switch v := v.(*VacuumVolumeRequest); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -3786,7 +3886,7 @@ func file_master_proto_init() {
 			}
 		}
 		file_master_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*GetMasterConfigurationResponse); i {
+			switch v := v.(*VacuumVolumeResponse); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -3798,7 +3898,7 @@ func file_master_proto_init() {
 			}
 		}
 		file_master_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ListMasterClientsRequest); i {
+			switch v := v.(*GetMasterConfigurationRequest); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -3810,7 +3910,7 @@ func file_master_proto_init() {
 			}
 		}
 		file_master_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ListMasterClientsResponse); i {
+			switch v := v.(*GetMasterConfigurationResponse); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -3822,7 +3922,7 @@ func file_master_proto_init() {
 			}
 		}
 		file_master_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*LeaseAdminTokenRequest); i {
+			switch v := v.(*ListMasterClientsRequest); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -3834,7 +3934,7 @@ func file_master_proto_init() {
 			}
 		}
 		file_master_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*LeaseAdminTokenResponse); i {
+			switch v := v.(*ListMasterClientsResponse); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -3846,7 +3946,7 @@ func file_master_proto_init() {
 			}
 		}
 		file_master_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ReleaseAdminTokenRequest); i {
+			switch v := v.(*LeaseAdminTokenRequest); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -3858,7 +3958,19 @@ func file_master_proto_init() {
 			}
 		}
 		file_master_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ReleaseAdminTokenResponse); i {
+			switch v := v.(*LeaseAdminTokenResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_master_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ReleaseAdminTokenRequest); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -3870,19 +3982,7 @@ func file_master_proto_init() {
 			}
 		}
 		file_master_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*SuperBlockExtra_ErasureCoding); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_master_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*LookupVolumeResponse_VolumeIdLocation); i {
+			switch v := v.(*ReleaseAdminTokenResponse); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -3894,6 +3994,30 @@ func file_master_proto_init() {
 			}
 		}
 		file_master_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SuperBlockExtra_ErasureCoding); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_master_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*LookupVolumeResponse_VolumeIdLocation); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_master_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*LookupEcVolumeResponse_EcShardIdLocation); i {
 			case 0:
 				return &v.state
@@ -3912,7 +4036,7 @@ func file_master_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_master_proto_rawDesc,
 			NumEnums:      0,
-			NumMessages:   42,
+			NumMessages:   44,
 			NumExtensions: 0,
 			NumServices:   1,
 		},
@@ -3947,6 +4071,7 @@ type SeaweedClient interface {
 	CollectionDelete(ctx context.Context, in *CollectionDeleteRequest, opts ...grpc.CallOption) (*CollectionDeleteResponse, error)
 	VolumeList(ctx context.Context, in *VolumeListRequest, opts ...grpc.CallOption) (*VolumeListResponse, error)
 	LookupEcVolume(ctx context.Context, in *LookupEcVolumeRequest, opts ...grpc.CallOption) (*LookupEcVolumeResponse, error)
+	VacuumVolume(ctx context.Context, in *VacuumVolumeRequest, opts ...grpc.CallOption) (*VacuumVolumeResponse, error)
 	GetMasterConfiguration(ctx context.Context, in *GetMasterConfigurationRequest, opts ...grpc.CallOption) (*GetMasterConfigurationResponse, error)
 	ListMasterClients(ctx context.Context, in *ListMasterClientsRequest, opts ...grpc.CallOption) (*ListMasterClientsResponse, error)
 	LeaseAdminToken(ctx context.Context, in *LeaseAdminTokenRequest, opts ...grpc.CallOption) (*LeaseAdminTokenResponse, error)
@@ -4086,6 +4211,15 @@ func (c *seaweedClient) LookupEcVolume(ctx context.Context, in *LookupEcVolumeRe
 	return out, nil
 }
 
+func (c *seaweedClient) VacuumVolume(ctx context.Context, in *VacuumVolumeRequest, opts ...grpc.CallOption) (*VacuumVolumeResponse, error) {
+	out := new(VacuumVolumeResponse)
+	err := c.cc.Invoke(ctx, "/master_pb.Seaweed/VacuumVolume", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *seaweedClient) GetMasterConfiguration(ctx context.Context, in *GetMasterConfigurationRequest, opts ...grpc.CallOption) (*GetMasterConfigurationResponse, error) {
 	out := new(GetMasterConfigurationResponse)
 	err := c.cc.Invoke(ctx, "/master_pb.Seaweed/GetMasterConfiguration", in, out, opts...)
@@ -4133,6 +4267,7 @@ type SeaweedServer interface {
 	CollectionDelete(context.Context, *CollectionDeleteRequest) (*CollectionDeleteResponse, error)
 	VolumeList(context.Context, *VolumeListRequest) (*VolumeListResponse, error)
 	LookupEcVolume(context.Context, *LookupEcVolumeRequest) (*LookupEcVolumeResponse, error)
+	VacuumVolume(context.Context, *VacuumVolumeRequest) (*VacuumVolumeResponse, error)
 	GetMasterConfiguration(context.Context, *GetMasterConfigurationRequest) (*GetMasterConfigurationResponse, error)
 	ListMasterClients(context.Context, *ListMasterClientsRequest) (*ListMasterClientsResponse, error)
 	LeaseAdminToken(context.Context, *LeaseAdminTokenRequest) (*LeaseAdminTokenResponse, error)
@@ -4170,6 +4305,9 @@ func (*UnimplementedSeaweedServer) VolumeList(context.Context, *VolumeListReques
 func (*UnimplementedSeaweedServer) LookupEcVolume(context.Context, *LookupEcVolumeRequest) (*LookupEcVolumeResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method LookupEcVolume not implemented")
 }
+func (*UnimplementedSeaweedServer) VacuumVolume(context.Context, *VacuumVolumeRequest) (*VacuumVolumeResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method VacuumVolume not implemented")
+}
 func (*UnimplementedSeaweedServer) GetMasterConfiguration(context.Context, *GetMasterConfigurationRequest) (*GetMasterConfigurationResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method GetMasterConfiguration not implemented")
 }
@@ -4365,6 +4503,24 @@ func _Seaweed_LookupEcVolume_Handler(srv interface{}, ctx context.Context, dec f
 	return interceptor(ctx, in, info, handler)
 }
 
+func _Seaweed_VacuumVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(VacuumVolumeRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedServer).VacuumVolume(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/master_pb.Seaweed/VacuumVolume",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedServer).VacuumVolume(ctx, req.(*VacuumVolumeRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _Seaweed_GetMasterConfiguration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(GetMasterConfigurationRequest)
 	if err := dec(in); err != nil {
@@ -4469,6 +4625,10 @@ var _Seaweed_serviceDesc = grpc.ServiceDesc{
 			MethodName: "LookupEcVolume",
 			Handler:    _Seaweed_LookupEcVolume_Handler,
 		},
+		{
+			MethodName: "VacuumVolume",
+			Handler:    _Seaweed_VacuumVolume_Handler,
+		},
 		{
 			MethodName: "GetMasterConfiguration",
 			Handler:    _Seaweed_GetMasterConfiguration_Handler,

weed/s3api/s3api_object_handlers.go

@@ -113,12 +113,6 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
 
 	bucket, object := getBucketAndObject(r)
 
-	response, _ := s3a.listFilerEntries(bucket, object, 1, "", "/")
-	if len(response.Contents) != 0 && strings.HasSuffix(object, "/") {
-		w.WriteHeader(http.StatusNoContent)
-		return
-	}
-
 	destUrl := fmt.Sprintf("http://%s%s/%s%s?recursive=true",
 		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)
 
@@ -266,11 +260,6 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
 
 	resp, postErr := client.Do(proxyReq)
 
-	if (resp.ContentLength == -1 || resp.StatusCode == 404) && !strings.HasSuffix(destUrl, "/") {
-		writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL)
-		return
-	}
-
 	if postErr != nil {
 		glog.Errorf("post to filer: %v", postErr)
 		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
@@ -278,6 +267,11 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
 	}
 	defer util.CloseResponse(resp)
 
+	if (resp.ContentLength == -1 || resp.StatusCode == 404) && !strings.HasSuffix(destUrl, "/") {
+		writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL)
+		return
+	}
+
 	responseFn(resp, w)
 
 }

weed/server/filer_server.go

@@ -126,6 +126,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 		defaultMux.HandleFunc("/", fs.filerHandler)
 	}
 	if defaultMux != readonlyMux {
+		handleStaticResources(readonlyMux)
 		readonlyMux.HandleFunc("/", fs.readonlyFilerHandler)
 	}
 

weed/server/filer_server_handlers_write.go

@@ -135,12 +135,13 @@ func (fs *FilerServer) detectStorageOption(requestURI, qCollection, qReplication
 	}
 
 	return &operation.StorageOption{
-		Replication: util.Nvl(replication, rule.Replication),
-		Collection:  util.Nvl(collection, rule.Collection),
-		DataCenter:  util.Nvl(dataCenter, fs.option.DataCenter),
-		Rack:        util.Nvl(rack, fs.option.Rack),
-		TtlSeconds:  ttlSeconds,
-		Fsync:       fsync || rule.Fsync,
+		Replication:       util.Nvl(replication, rule.Replication),
+		Collection:        util.Nvl(collection, rule.Collection),
+		DataCenter:        util.Nvl(dataCenter, fs.option.DataCenter),
+		Rack:              util.Nvl(rack, fs.option.Rack),
+		TtlSeconds:        ttlSeconds,
+		Fsync:             fsync || rule.Fsync,
+		VolumeGrowthCount: rule.VolumeGrowthCount,
 	}
 }
 

weed/server/master_grpc_server_volume.go

@@ -177,3 +177,16 @@ func (ms *MasterServer) LookupEcVolume(ctx context.Context, req *master_pb.Looku
 
 	return resp, nil
 }
+
+func (ms *MasterServer) VacuumVolume(ctx context.Context, req *master_pb.VacuumVolumeRequest) (*master_pb.VacuumVolumeResponse, error) {
+
+	if !ms.Topo.IsLeader() {
+		return nil, raft.NotLeaderError
+	}
+
+	resp := &master_pb.VacuumVolumeResponse{}
+
+	ms.Topo.Vacuum(ms.grpcDialOption, float64(req.GarbageThreshold), ms.preallocateSize)
+
+	return resp, nil
+}

weed/server/volume_grpc_copy.go

@@ -48,7 +48,7 @@ func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.Vo
 	//   send .dat file
 	//   confirm size and timestamp
 	var volFileInfoResp *volume_server_pb.ReadVolumeFileStatusResponse
-	var volumeFileName, idxFileName, datFileName string
+	var dataBaseFileName, indexBaseFileName, idxFileName, datFileName string
 	err := operation.WithVolumeServerClient(req.SourceDataNode, vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
 		var err error
 		volFileInfoResp, err = client.ReadVolumeFileStatus(context.Background(),
@@ -59,24 +59,25 @@ func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.Vo
 			return fmt.Errorf("read volume file status failed, %v", err)
 		}
 
-		volumeFileName = storage.VolumeFileName(location.Directory, volFileInfoResp.Collection, int(req.VolumeId))
+		dataBaseFileName = storage.VolumeFileName(location.Directory, volFileInfoResp.Collection, int(req.VolumeId))
+		indexBaseFileName = storage.VolumeFileName(location.IdxDirectory, volFileInfoResp.Collection, int(req.VolumeId))
 
-		ioutil.WriteFile(volumeFileName+".note", []byte(fmt.Sprintf("copying from %s", req.SourceDataNode)), 0755)
+		ioutil.WriteFile(dataBaseFileName+".note", []byte(fmt.Sprintf("copying from %s", req.SourceDataNode)), 0755)
 
 		// println("source:", volFileInfoResp.String())
-		if err := vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.DatFileSize, volumeFileName, ".dat", false, true); err != nil {
+		if err := vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.DatFileSize, dataBaseFileName, ".dat", false, true); err != nil {
 			return err
 		}
 
-		if err := vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.IdxFileSize, volumeFileName, ".idx", false, false); err != nil {
+		if err := vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.IdxFileSize, indexBaseFileName, ".idx", false, false); err != nil {
 			return err
 		}
 
-		if err := vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.DatFileSize, volumeFileName, ".vif", false, true); err != nil {
+		if err := vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.DatFileSize, dataBaseFileName, ".vif", false, true); err != nil {
 			return err
 		}
 
-		os.Remove(volumeFileName + ".note")
+		os.Remove(dataBaseFileName + ".note")
 
 		return nil
 	})
@@ -84,18 +85,18 @@ func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.Vo
 	if err != nil {
 		return nil, err
 	}
-	if volumeFileName == "" {
+	if dataBaseFileName == "" {
 		return nil, fmt.Errorf("not found volume %d file", req.VolumeId)
 	}
 
-	idxFileName = volumeFileName + ".idx"
-	datFileName = volumeFileName + ".dat"
+	idxFileName = indexBaseFileName + ".idx"
+	datFileName = dataBaseFileName + ".dat"
 
 	defer func() {
-		if err != nil && volumeFileName != "" {
+		if err != nil && dataBaseFileName != "" {
 			os.Remove(idxFileName)
 			os.Remove(datFileName)
-			os.Remove(volumeFileName + ".vif")
+			os.Remove(dataBaseFileName + ".vif")
 		}
 	}()
 
@@ -223,7 +224,7 @@ func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream v
 		if uint32(v.CompactionRevision) != req.CompactionRevision && req.CompactionRevision != math.MaxUint32 {
 			return fmt.Errorf("volume %d is compacted", req.VolumeId)
 		}
-		fileName = v.FileName() + req.Ext
+		fileName = v.FileName(req.Ext)
 	} else {
 		baseFileName := erasure_coding.EcShardBaseFileName(req.Collection, int(req.VolumeId)) + req.Ext
 		for _, location := range vs.store.Locations {
@@ -231,6 +232,10 @@ func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream v
 			if util.FileExists(tName) {
 				fileName = tName
 			}
+			tName = util.Join(location.IdxDirectory, baseFileName)
+			if util.FileExists(tName) {
+				fileName = tName
+			}
 		}
 		if fileName == "" {
 			if req.IgnoreSourceFileNotFound {

weed/server/volume_grpc_erasure_coding.go

@@ -8,7 +8,6 @@ import (
 	"math"
 	"os"
 	"path"
-	"path/filepath"
 	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -27,7 +26,7 @@ import (
 Steps to apply erasure coding to .dat .idx files
 0. ensure the volume is readonly
 1. client call VolumeEcShardsGenerate to generate the .ecx and .ec00 ~ .ec13 files
-2. client ask master for possible servers to hold the ec files, at least 4 servers
+2. client ask master for possible servers to hold the ec files
 3. client call VolumeEcShardsCopy on above target servers to copy ec files from the source server
 4. target servers report the new ec files to the master
 5.   master stores vid -> [14]*DataNode
@@ -44,7 +43,7 @@ func (vs *VolumeServer) VolumeEcShardsGenerate(ctx context.Context, req *volume_
 	if v == nil {
 		return nil, fmt.Errorf("volume %d not found", req.VolumeId)
 	}
-	baseFileName := v.FileName()
+	baseFileName := v.DataFileName()
 
 	if v.Collection != req.Collection {
 		return nil, fmt.Errorf("existing collection:%v unexpected input: %v", v.Collection, req.Collection)
@@ -56,8 +55,8 @@ func (vs *VolumeServer) VolumeEcShardsGenerate(ctx context.Context, req *volume_
 	}
 
 	// write .ecx file
-	if err := erasure_coding.WriteSortedFileFromIdx(baseFileName, ".ecx"); err != nil {
-		return nil, fmt.Errorf("WriteSortedFileFromIdx %s: %v", baseFileName, err)
+	if err := erasure_coding.WriteSortedFileFromIdx(v.IndexFileName(), ".ecx"); err != nil {
+		return nil, fmt.Errorf("WriteSortedFileFromIdx %s: %v", v.IndexFileName(), err)
 	}
 
 	// write .vif files
@@ -78,17 +77,18 @@ func (vs *VolumeServer) VolumeEcShardsRebuild(ctx context.Context, req *volume_s
 	var rebuiltShardIds []uint32
 
 	for _, location := range vs.store.Locations {
-		if util.FileExists(path.Join(location.Directory, baseFileName+".ecx")) {
+		if util.FileExists(path.Join(location.IdxDirectory, baseFileName+".ecx")) {
 			// write .ec00 ~ .ec13 files
-			baseFileName = path.Join(location.Directory, baseFileName)
-			if generatedShardIds, err := erasure_coding.RebuildEcFiles(baseFileName); err != nil {
-				return nil, fmt.Errorf("RebuildEcFiles %s: %v", baseFileName, err)
+			dataBaseFileName := path.Join(location.Directory, baseFileName)
+			if generatedShardIds, err := erasure_coding.RebuildEcFiles(dataBaseFileName); err != nil {
+				return nil, fmt.Errorf("RebuildEcFiles %s: %v", dataBaseFileName, err)
 			} else {
 				rebuiltShardIds = generatedShardIds
 			}
 
-			if err := erasure_coding.RebuildEcxFile(baseFileName); err != nil {
-				return nil, fmt.Errorf("RebuildEcxFile %s: %v", baseFileName, err)
+			indexBaseFileName := path.Join(location.IdxDirectory, baseFileName)
+			if err := erasure_coding.RebuildEcxFile(indexBaseFileName); err != nil {
+				return nil, fmt.Errorf("RebuildEcxFile %s: %v", dataBaseFileName, err)
 			}
 
 			break
@@ -110,13 +110,14 @@ func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_serv
 		return nil, fmt.Errorf("no space left")
 	}
 
-	baseFileName := storage.VolumeFileName(location.Directory, req.Collection, int(req.VolumeId))
+	dataBaseFileName := storage.VolumeFileName(location.Directory, req.Collection, int(req.VolumeId))
+	indexBaseFileName := storage.VolumeFileName(location.IdxDirectory, req.Collection, int(req.VolumeId))
 
 	err := operation.WithVolumeServerClient(req.SourceDataNode, vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
 
 		// copy ec data slices
 		for _, shardId := range req.ShardIds {
-			if err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, baseFileName, erasure_coding.ToExt(int(shardId)), false, false); err != nil {
+			if err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, dataBaseFileName, erasure_coding.ToExt(int(shardId)), false, false); err != nil {
 				return err
 			}
 		}
@@ -124,7 +125,7 @@ func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_serv
 		if req.CopyEcxFile {
 
 			// copy ecx file
-			if err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, baseFileName, ".ecx", false, false); err != nil {
+			if err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, indexBaseFileName, ".ecx", false, false); err != nil {
 				return err
 			}
 			return nil
@@ -132,14 +133,14 @@ func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_serv
 
 		if req.CopyEcjFile {
 			// copy ecj file
-			if err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, baseFileName, ".ecj", true, true); err != nil {
+			if err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, indexBaseFileName, ".ecj", true, true); err != nil {
 				return err
 			}
 		}
 
 		if req.CopyVifFile {
 			// copy vif file
-			if err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, baseFileName, ".vif", false, true); err != nil {
+			if err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, dataBaseFileName, ".vif", false, true); err != nil {
 				return err
 			}
 		}
@@ -157,17 +158,19 @@ func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_serv
 // the shard should not be mounted before calling this.
 func (vs *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_server_pb.VolumeEcShardsDeleteRequest) (*volume_server_pb.VolumeEcShardsDeleteResponse, error) {
 
-	baseFilename := erasure_coding.EcShardBaseFileName(req.Collection, int(req.VolumeId))
+	bName := erasure_coding.EcShardBaseFileName(req.Collection, int(req.VolumeId))
 
 	glog.V(0).Infof("ec volume %d shard delete %v", req.VolumeId, req.ShardIds)
 
 	found := false
+	var indexBaseFilename, dataBaseFilename string
 	for _, location := range vs.store.Locations {
-		if util.FileExists(path.Join(location.Directory, baseFilename+".ecx")) {
+		if util.FileExists(path.Join(location.IdxDirectory, bName+".ecx")) {
 			found = true
-			baseFilename = path.Join(location.Directory, baseFilename)
+			indexBaseFilename = path.Join(location.IdxDirectory, bName)
+			dataBaseFilename = path.Join(location.Directory, bName)
 			for _, shardId := range req.ShardIds {
-				os.Remove(baseFilename + erasure_coding.ToExt(int(shardId)))
+				os.Remove(dataBaseFilename + erasure_coding.ToExt(int(shardId)))
 			}
 			break
 		}
@@ -182,12 +185,18 @@ func (vs *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_se
 	hasIdxFile := false
 	existingShardCount := 0
 
-	bName := filepath.Base(baseFilename)
 	for _, location := range vs.store.Locations {
 		fileInfos, err := ioutil.ReadDir(location.Directory)
 		if err != nil {
 			continue
 		}
+		if location.IdxDirectory != location.Directory {
+			idxFileInfos, err := ioutil.ReadDir(location.IdxDirectory)
+			if err != nil {
+				continue
+			}
+			fileInfos = append(fileInfos, idxFileInfos...)
+		}
 		for _, fileInfo := range fileInfos {
 			if fileInfo.Name() == bName+".ecx" || fileInfo.Name() == bName+".ecj" {
 				hasEcxFile = true
@@ -204,14 +213,14 @@ func (vs *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_se
 	}
 
 	if hasEcxFile && existingShardCount == 0 {
-		if err := os.Remove(baseFilename + ".ecx"); err != nil {
+		if err := os.Remove(indexBaseFilename + ".ecx"); err != nil {
 			return nil, err
 		}
-		os.Remove(baseFilename + ".ecj")
+		os.Remove(indexBaseFilename + ".ecj")
 	}
 	if !hasIdxFile {
 		// .vif is used for ec volumes and normal volumes
-		os.Remove(baseFilename + ".vif")
+		os.Remove(dataBaseFilename + ".vif")
 	}
 
 	return &volume_server_pb.VolumeEcShardsDeleteResponse{}, nil
@@ -365,26 +374,26 @@ func (vs *VolumeServer) VolumeEcShardsToVolume(ctx context.Context, req *volume_
 	if !found {
 		return nil, fmt.Errorf("ec volume %d not found", req.VolumeId)
 	}
-	baseFileName := v.FileName()
 
 	if v.Collection != req.Collection {
 		return nil, fmt.Errorf("existing collection:%v unexpected input: %v", v.Collection, req.Collection)
 	}
 
+	dataBaseFileName, indexBaseFileName := v.DataBaseFileName(), v.IndexBaseFileName()
 	// calculate .dat file size
-	datFileSize, err := erasure_coding.FindDatFileSize(baseFileName)
+	datFileSize, err := erasure_coding.FindDatFileSize(dataBaseFileName, indexBaseFileName)
 	if err != nil {
-		return nil, fmt.Errorf("FindDatFileSize %s: %v", baseFileName, err)
+		return nil, fmt.Errorf("FindDatFileSize %s: %v", dataBaseFileName, err)
 	}
 
 	// write .dat file from .ec00 ~ .ec09 files
-	if err := erasure_coding.WriteDatFile(baseFileName, datFileSize); err != nil {
-		return nil, fmt.Errorf("WriteEcFiles %s: %v", baseFileName, err)
+	if err := erasure_coding.WriteDatFile(dataBaseFileName, datFileSize); err != nil {
+		return nil, fmt.Errorf("WriteEcFiles %s: %v", dataBaseFileName, err)
 	}
 
 	// write .idx file from .ecx and .ecj files
-	if err := erasure_coding.WriteIdxFileFromEcIndex(baseFileName); err != nil {
-		return nil, fmt.Errorf("WriteIdxFileFromEcIndex %s: %v", baseFileName, err)
+	if err := erasure_coding.WriteIdxFileFromEcIndex(indexBaseFileName); err != nil {
+		return nil, fmt.Errorf("WriteIdxFileFromEcIndex %s: %v", v.IndexBaseFileName(), err)
 	}
 
 	return &volume_server_pb.VolumeEcShardsToVolumeResponse{}, nil

weed/server/volume_grpc_tier_download.go

@@ -58,9 +58,9 @@ func (vs *VolumeServer) VolumeTierMoveDatFromRemote(req *volume_server_pb.Volume
 		})
 	}
 	// copy the data file
-	_, err := backendStorage.DownloadFile(v.FileName()+".dat", storageKey, fn)
+	_, err := backendStorage.DownloadFile(v.FileName(".dat"), storageKey, fn)
 	if err != nil {
-		return fmt.Errorf("backend %s copy file %s: %v", storageName, v.FileName()+".dat", err)
+		return fmt.Errorf("backend %s copy file %s: %v", storageName, v.FileName(".dat"), err)
 	}
 
 	if req.KeepRemoteDatFile {

weed/server/volume_grpc_tier_upload.go

@@ -93,7 +93,7 @@ func (vs *VolumeServer) VolumeTierMoveDatToRemote(req *volume_server_pb.VolumeTi
 	}
 
 	if !req.KeepLocalDatFile {
-		os.Remove(v.FileName() + ".dat")
+		os.Remove(v.FileName(".dat"))
 	}
 
 	return nil

weed/server/volume_server.go

@@ -38,6 +38,7 @@ type VolumeServer struct {
 func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 	port int, publicUrl string,
 	folders []string, maxCounts []int, minFreeSpacePercents []float32,
+	idxFolder string,
 	needleMapKind storage.NeedleMapType,
 	masterNodes []string, pulseSeconds int,
 	dataCenter string, rack string,
@@ -75,7 +76,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 
 	vs.checkWithMaster()
 
-	vs.store = storage.NewStore(vs.grpcDialOption, port, ip, publicUrl, folders, maxCounts, minFreeSpacePercents, vs.needleMapKind)
+	vs.store = storage.NewStore(vs.grpcDialOption, port, ip, publicUrl, folders, maxCounts, minFreeSpacePercents, idxFolder, vs.needleMapKind)
 	vs.guard = security.NewGuard(whiteList, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec)
 
 	handleStaticResources(adminMux)

weed/server/volume_server_handlers_read.go

@@ -159,8 +159,8 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 			if n.Data, err = util.DecompressData(n.Data); err != nil {
 				glog.V(0).Infoln("ungzip error:", err, r.URL.Path)
 			}
-		} else if strings.Contains(r.Header.Get("Accept-Encoding"), "zstd") && util.IsZstdContent(n.Data) {
-			w.Header().Set("Content-Encoding", "zstd")
+			// } else if strings.Contains(r.Header.Get("Accept-Encoding"), "zstd") && util.IsZstdContent(n.Data) {
+			//	w.Header().Set("Content-Encoding", "zstd")
 		} else if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") && util.IsGzippedContent(n.Data) {
 			w.Header().Set("Content-Encoding", "gzip")
 		} else {

weed/shell/command_collection_delete.go

@@ -22,7 +22,7 @@ func (c *commandCollectionDelete) Name() string {
 func (c *commandCollectionDelete) Help() string {
 	return `delete specified collection
 
-	collection.delete -collectin <collection_name> -force
+	collection.delete -collection <collection_name> -force
 
 `
 }

weed/shell/command_ec_decode.go

@@ -85,7 +85,7 @@ func doEcDecode(commandEnv *CommandEnv, topoInfo *master_pb.TopologyInfo, collec
 	}
 
 	// generate a normal volume
-	err = generateNormalVolume(commandEnv.option.GrpcDialOption, needle.VolumeId(vid), collection, targetNodeLocation)
+	err = generateNormalVolume(commandEnv.option.GrpcDialOption, vid, collection, targetNodeLocation)
 	if err != nil {
 		return fmt.Errorf("generate normal volume %d on %s: %v", vid, targetNodeLocation, err)
 	}

weed/shell/command_ec_encode.go

@@ -99,13 +99,13 @@ func doEcEncode(commandEnv *CommandEnv, collection string, vid needle.VolumeId)
 	// fmt.Printf("found ec %d shards on %v\n", vid, locations)
 
 	// mark the volume as readonly
-	err = markVolumeReadonly(commandEnv.option.GrpcDialOption, needle.VolumeId(vid), locations)
+	err = markVolumeReadonly(commandEnv.option.GrpcDialOption, vid, locations)
 	if err != nil {
 		return fmt.Errorf("mark volume %d as readonly on %s: %v", vid, locations[0].Url, err)
 	}
 
 	// generate ec shards
-	err = generateEcShards(commandEnv.option.GrpcDialOption, needle.VolumeId(vid), collection, locations[0].Url)
+	err = generateEcShards(commandEnv.option.GrpcDialOption, vid, collection, locations[0].Url)
 	if err != nil {
 		return fmt.Errorf("generate ec shards for volume %d on %s: %v", vid, locations[0].Url, err)
 	}

weed/shell/command_fs_configure.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -35,6 +36,9 @@ func (c *commandFsConfigure) Help() string {
 	fs.configure -locationPrfix=/my/folder -collection=abc
 	fs.configure -locationPrfix=/my/folder -collection=abc -ttl=7d
 
+	# example: configure adding only 1 physical volume for each bucket collection
+	fs.configure -locationPrfix=/buckets/ -volumeGrowthCount=1
+
 	# apply the changes
 	fs.configure -locationPrfix=/my/folder -collection=abc -apply
 
@@ -52,6 +56,7 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io
 	replication := fsConfigureCommand.String("replication", "", "assign writes with this replication")
 	ttl := fsConfigureCommand.String("ttl", "", "assign writes with this ttl")
 	fsync := fsConfigureCommand.Bool("fsync", false, "fsync for the writes")
+	volumeGrowthCount := fsConfigureCommand.Int("volumeGrowthCount", 0, "the number of physical volumes to add if no writable volumes")
 	isDelete := fsConfigureCommand.Bool("delete", false, "delete the configuration by locationPrefix")
 	apply := fsConfigureCommand.Bool("apply", false, "update and apply filer configuration")
 	if err = fsConfigureCommand.Parse(args); err != nil {
@@ -83,15 +88,31 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io
 
 	if *locationPrefix != "" {
 		locConf := &filer_pb.FilerConf_PathConf{
-			LocationPrefix: *locationPrefix,
-			Collection:     *collection,
-			Replication:    *replication,
-			Ttl:            *ttl,
-			Fsync:          *fsync,
+			LocationPrefix:    *locationPrefix,
+			Collection:        *collection,
+			Replication:       *replication,
+			Ttl:               *ttl,
+			Fsync:             *fsync,
+			VolumeGrowthCount: uint32(*volumeGrowthCount),
 		}
+
+		// check collection
 		if *collection != "" && strings.HasPrefix(*locationPrefix, "/buckets/") {
 			return fmt.Errorf("one s3 bucket goes to one collection and not customizable.")
 		}
+
+		// check replication
+		if *replication != "" {
+			rp, err := super_block.NewReplicaPlacementFromString(*replication)
+			if err != nil {
+				return fmt.Errorf("parse replication %s: %v", *replication, err)
+			}
+			if *volumeGrowthCount%rp.GetCopyCount() != 0 {
+				return fmt.Errorf("volumeGrowthCount %d should be devided by replication copy count %d", *volumeGrowthCount, rp.GetCopyCount())
+			}
+		}
+
+		// save it
 		if *isDelete {
 			fc.DeleteLocationConf(*locationPrefix)
 		} else {

weed/shell/command_fs_meta_cat.go

@@ -72,8 +72,9 @@ func (c *commandFsMetaCat) Do(args []string, commandEnv *CommandEnv, writer io.W
 
 		bytes, _ := proto.Marshal(respLookupEntry.Entry)
 		gzippedBytes, _ := util.GzipData(bytes)
-		zstdBytes, _ := util.ZstdData(bytes)
-		fmt.Fprintf(writer, "chunks %d meta size: %d gzip:%d zstd:%d\n", len(respLookupEntry.Entry.Chunks), len(bytes), len(gzippedBytes), len(zstdBytes))
+		// zstdBytes, _ := util.ZstdData(bytes)
+		// fmt.Fprintf(writer, "chunks %d meta size: %d gzip:%d zstd:%d\n", len(respLookupEntry.Entry.Chunks), len(bytes), len(gzippedBytes), len(zstdBytes))
+		fmt.Fprintf(writer, "chunks %d meta size: %d gzip:%d\n", len(respLookupEntry.Entry.Chunks), len(bytes), len(gzippedBytes))
 
 		return nil
 

weed/shell/command_volume_balance.go

@@ -306,16 +306,16 @@ func isGoodMove(placement *super_block.ReplicaPlacement, existingReplicas []*Vol
 	dcs[targetNode.dc] = true
 	racks[fmt.Sprintf("%s %s", targetNode.dc, targetNode.rack)]++
 
-	if len(dcs) > placement.DiffDataCenterCount+1 {
+	if len(dcs) != placement.DiffDataCenterCount+1 {
 		return false
 	}
 
-	if len(racks) > placement.DiffRackCount+placement.DiffDataCenterCount+1 {
+	if len(racks) != placement.DiffRackCount+placement.DiffDataCenterCount+1 {
 		return false
 	}
 
 	for _, sameRackCount := range racks {
-		if sameRackCount > placement.SameRackCount+1 {
+		if sameRackCount != placement.SameRackCount+1 {
 			return false
 		}
 	}

weed/shell/command_volume_balance_test.go

@@ -20,6 +20,22 @@ func TestIsGoodMove(t *testing.T) {
 
 	var tests = []testMoveCase{
 
+		{
+			name:        "test 100 move to wrong data centers",
+			replication: "100",
+			replicas: []*VolumeReplica{
+				{
+					location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
+				},
+				{
+					location: &location{"dc2", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
+				},
+			},
+			sourceLocation: location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
+			targetLocation: location{"dc2", "r3", &master_pb.DataNodeInfo{Id: "dn3"}},
+			expected:       false,
+		},
+
 		{
 			name:        "test 100 move to spread into proper data centers",
 			replication: "100",

weed/shell/command_volume_fsck.go

@@ -157,12 +157,12 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, volumeId
 			IgnoreSourceFileNotFound: false,
 		})
 		if err != nil {
-			return fmt.Errorf("failed to start copying volume %d.idx: %v", volumeId, err)
+			return fmt.Errorf("failed to start copying volume %d%s: %v", volumeId, ext, err)
 		}
 
 		err = writeToFile(copyFileClient, getVolumeFileIdFile(tempFolder, volumeId))
 		if err != nil {
-			return fmt.Errorf("failed to copy %d.idx from %s: %v", volumeId, vinfo.server, err)
+			return fmt.Errorf("failed to copy %d%s from %s: %v", volumeId, ext, vinfo.server, err)
 		}
 
 		return nil

weed/shell/command_volume_vacuum.go

@@ -0,0 +1,53 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"io"
+
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandVacuum{})
+}
+
+type commandVacuum struct {
+}
+
+func (c *commandVacuum) Name() string {
+	return "volume.vacuum"
+}
+
+func (c *commandVacuum) Help() string {
+	return `compact volumes if deleted entries are more than the limit
+
+	volume.vacuum [-garbageThreshold=0.3]
+
+`
+}
+
+func (c *commandVacuum) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	if err = commandEnv.confirmIsLocked(); err != nil {
+		return
+	}
+
+	volumeVacuumCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	garbageThreshold := volumeVacuumCommand.Float64("garbageThreshold", 0.3, "vacuum when garbage is more than this limit")
+	if err = volumeVacuumCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	err = commandEnv.MasterClient.WithClient(func(client master_pb.SeaweedClient) error {
+		_, err = client.VacuumVolume(context.Background(), &master_pb.VacuumVolumeRequest{
+			GarbageThreshold: float32(*garbageThreshold),
+		})
+		return err
+	})
+	if err != nil {
+		return
+	}
+
+	return nil
+}

weed/storage/disk_location.go

@@ -18,6 +18,7 @@ import (
 
 type DiskLocation struct {
 	Directory              string
+	IdxDirectory           string
 	MaxVolumeCount         int
 	OriginalMaxVolumeCount int
 	MinFreeSpacePercent    float32
@@ -31,23 +32,34 @@ type DiskLocation struct {
 	isDiskSpaceLow bool
 }
 
-func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpacePercent float32) *DiskLocation {
-	location := &DiskLocation{Directory: dir, MaxVolumeCount: maxVolumeCount, OriginalMaxVolumeCount: maxVolumeCount, MinFreeSpacePercent: minFreeSpacePercent}
+func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpacePercent float32, idxDir string) *DiskLocation {
+	dir = util.ResolvePath(dir)
+	if idxDir == "" {
+		idxDir = dir
+	} else {
+		idxDir = util.ResolvePath(idxDir)
+	}
+	location := &DiskLocation{
+		Directory:              dir,
+		IdxDirectory:           idxDir,
+		MaxVolumeCount:         maxVolumeCount,
+		OriginalMaxVolumeCount: maxVolumeCount,
+		MinFreeSpacePercent:    minFreeSpacePercent,
+	}
 	location.volumes = make(map[needle.VolumeId]*Volume)
 	location.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume)
 	go location.CheckDiskSpace()
 	return location
 }
 
-func (l *DiskLocation) volumeIdFromPath(dir os.FileInfo) (needle.VolumeId, string, error) {
-	name := dir.Name()
-	if !dir.IsDir() && strings.HasSuffix(name, ".idx") {
-		base := name[:len(name)-len(".idx")]
+func volumeIdFromFileName(filename string) (needle.VolumeId, string, error) {
+	if strings.HasSuffix(filename, ".idx") || strings.HasSuffix(filename, ".vif") {
+		base := filename[:len(filename)-4]
 		collection, volumeId, err := parseCollectionVolumeId(base)
 		return volumeId, collection, err
 	}
 
-	return 0, "", fmt.Errorf("Path is not a volume: %s", name)
+	return 0, "", fmt.Errorf("file is not a volume: %s", filename)
 }
 
 func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeId, err error) {
@@ -60,54 +72,63 @@ func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeI
 }
 
 func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapType) bool {
-	name := fileInfo.Name()
-	if !fileInfo.IsDir() && strings.HasSuffix(name, ".idx") {
-		name := name[:len(name)-len(".idx")]
-		noteFile := l.Directory + "/" + name + ".note"
-		if util.FileExists(noteFile) {
-			note, _ := ioutil.ReadFile(noteFile)
-			glog.Warningf("volume %s was not completed: %s", name, string(note))
-			removeVolumeFiles(l.Directory + "/" + name)
-			return false
-		}
-		vid, collection, err := l.volumeIdFromPath(fileInfo)
-		if err != nil {
-			glog.Warningf("get volume id failed, %s, err : %s", name, err)
-			return false
-		}
+	basename := fileInfo.Name()
+	if fileInfo.IsDir() {
+		return false
+	}
+	if !strings.HasSuffix(basename, ".idx") && !strings.HasSuffix(basename, ".vif") {
+		return false
+	}
+	volumeName := basename[:len(basename)-4]
 
-		// void loading one volume more than once
-		l.volumesLock.RLock()
-		_, found := l.volumes[vid]
-		l.volumesLock.RUnlock()
-		if found {
-			glog.V(1).Infof("loaded volume, %v", vid)
-			return true
-		}
+	// check for incomplete volume
+	noteFile := l.Directory + "/" + volumeName + ".note"
+	if util.FileExists(noteFile) {
+		note, _ := ioutil.ReadFile(noteFile)
+		glog.Warningf("volume %s was not completed: %s", volumeName, string(note))
+		removeVolumeFiles(l.Directory + "/" + volumeName)
+		removeVolumeFiles(l.IdxDirectory + "/" + volumeName)
+		return false
+	}
 
-		v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil, 0, 0)
-		if e != nil {
-			glog.V(0).Infof("new volume %s error %s", name, e)
-			return false
-		}
+	// parse out collection, volume id
+	vid, collection, err := volumeIdFromFileName(basename)
+	if err != nil {
+		glog.Warningf("get volume id failed, %s, err : %s", volumeName, err)
+		return false
+	}
 
-		l.SetVolume(vid, v)
-
-		size, _, _ := v.FileStat()
-		glog.V(0).Infof("data file %s, replicaPlacement=%s v=%d size=%d ttl=%s",
-			l.Directory+"/"+name+".dat", v.ReplicaPlacement, v.Version(), size, v.Ttl.String())
+	// avoid loading one volume more than once
+	l.volumesLock.RLock()
+	_, found := l.volumes[vid]
+	l.volumesLock.RUnlock()
+	if found {
+		glog.V(1).Infof("loaded volume, %v", vid)
 		return true
 	}
-	return false
+
+	// load the volume
+	v, e := NewVolume(l.Directory, l.IdxDirectory, collection, vid, needleMapKind, nil, nil, 0, 0)
+	if e != nil {
+		glog.V(0).Infof("new volume %s error %s", volumeName, e)
+		return false
+	}
+
+	l.SetVolume(vid, v)
+
+	size, _, _ := v.FileStat()
+	glog.V(0).Infof("data file %s, replicaPlacement=%s v=%d size=%d ttl=%s",
+		l.Directory+"/"+volumeName+".dat", v.ReplicaPlacement, v.Version(), size, v.Ttl.String())
+	return true
 }
 
 func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, concurrency int) {
 
 	task_queue := make(chan os.FileInfo, 10*concurrency)
 	go func() {
-		if dirs, err := ioutil.ReadDir(l.Directory); err == nil {
-			for _, dir := range dirs {
-				task_queue <- dir
+		if fileInfos, err := ioutil.ReadDir(l.Directory); err == nil {
+			for _, fi := range fileInfos {
+				task_queue <- fi
 			}
 		}
 		close(task_queue)
@@ -118,8 +139,8 @@ func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, con
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			for dir := range task_queue {
-				_ = l.loadExistingVolume(dir, needleMapKind)
+			for fi := range task_queue {
+				_ = l.loadExistingVolume(fi, needleMapKind)
 			}
 		}()
 	}
@@ -285,7 +306,7 @@ func (l *DiskLocation) Close() {
 func (l *DiskLocation) LocateVolume(vid needle.VolumeId) (os.FileInfo, bool) {
 	if fileInfos, err := ioutil.ReadDir(l.Directory); err == nil {
 		for _, fileInfo := range fileInfos {
-			volId, _, err := l.volumeIdFromPath(fileInfo)
+			volId, _, err := volumeIdFromFileName(fileInfo.Name())
 			if vid == volId && err == nil {
 				return fileInfo, true
 			}

weed/storage/disk_location_ec.go

@@ -68,7 +68,7 @@ func (l *DiskLocation) LoadEcShard(collection string, vid needle.VolumeId, shard
 	defer l.ecVolumesLock.Unlock()
 	ecVolume, found := l.ecVolumes[vid]
 	if !found {
-		ecVolume, err = erasure_coding.NewEcVolume(l.Directory, collection, vid)
+		ecVolume, err = erasure_coding.NewEcVolume(l.Directory, l.IdxDirectory, collection, vid)
 		if err != nil {
 			return fmt.Errorf("failed to create ec volume %d: %v", vid, err)
 		}
@@ -122,6 +122,13 @@ func (l *DiskLocation) loadAllEcShards() (err error) {
 	if err != nil {
 		return fmt.Errorf("load all ec shards in dir %s: %v", l.Directory, err)
 	}
+	if l.IdxDirectory != l.Directory {
+		indexFileInfos, err := ioutil.ReadDir(l.IdxDirectory)
+		if err != nil {
+			return fmt.Errorf("load all ec shards in dir %s: %v", l.IdxDirectory, err)
+		}
+		fileInfos = append(fileInfos, indexFileInfos...)
+	}
 
 	sort.Slice(fileInfos, func(i, j int) bool {
 		return fileInfos[i].Name() < fileInfos[j].Name()

weed/storage/erasure_coding/ec_decoder.go

@@ -45,14 +45,14 @@ func WriteIdxFileFromEcIndex(baseFileName string) (err error) {
 // FindDatFileSize calculate .dat file size from max offset entry
 // there may be extra deletions after that entry
 // but they are deletions anyway
-func FindDatFileSize(baseFileName string) (datSize int64, err error) {
+func FindDatFileSize(dataBaseFileName, indexBaseFileName string) (datSize int64, err error) {
 
-	version, err := readEcVolumeVersion(baseFileName)
+	version, err := readEcVolumeVersion(dataBaseFileName)
 	if err != nil {
-		return 0, fmt.Errorf("read ec volume %s version: %v", baseFileName, err)
+		return 0, fmt.Errorf("read ec volume %s version: %v", dataBaseFileName, err)
 	}
 
-	err = iterateEcxFile(baseFileName, func(key types.NeedleId, offset types.Offset, size types.Size) error {
+	err = iterateEcxFile(indexBaseFileName, func(key types.NeedleId, offset types.Offset, size types.Size) error {
 
 		if size.IsDeleted() {
 			return nil

weed/storage/erasure_coding/ec_volume.go

@@ -25,6 +25,7 @@ type EcVolume struct {
 	VolumeId                  needle.VolumeId
 	Collection                string
 	dir                       string
+	dirIdx                    string
 	ecxFile                   *os.File
 	ecxFileSize               int64
 	ecxCreatedAt              time.Time
@@ -37,33 +38,34 @@ type EcVolume struct {
 	ecjFileAccessLock         sync.Mutex
 }
 
-func NewEcVolume(dir string, collection string, vid needle.VolumeId) (ev *EcVolume, err error) {
-	ev = &EcVolume{dir: dir, Collection: collection, VolumeId: vid}
+func NewEcVolume(dir string, dirIdx string, collection string, vid needle.VolumeId) (ev *EcVolume, err error) {
+	ev = &EcVolume{dir: dir, dirIdx: dirIdx, Collection: collection, VolumeId: vid}
 
-	baseFileName := EcShardFileName(collection, dir, int(vid))
+	dataBaseFileName := EcShardFileName(collection, dir, int(vid))
+	indexBaseFileName := EcShardFileName(collection, dirIdx, int(vid))
 
 	// open ecx file
-	if ev.ecxFile, err = os.OpenFile(baseFileName+".ecx", os.O_RDWR, 0644); err != nil {
-		return nil, fmt.Errorf("cannot open ec volume index %s.ecx: %v", baseFileName, err)
+	if ev.ecxFile, err = os.OpenFile(indexBaseFileName+".ecx", os.O_RDWR, 0644); err != nil {
+		return nil, fmt.Errorf("cannot open ec volume index %s.ecx: %v", indexBaseFileName, err)
 	}
 	ecxFi, statErr := ev.ecxFile.Stat()
 	if statErr != nil {
-		return nil, fmt.Errorf("can not stat ec volume index %s.ecx: %v", baseFileName, statErr)
+		return nil, fmt.Errorf("can not stat ec volume index %s.ecx: %v", indexBaseFileName, statErr)
 	}
 	ev.ecxFileSize = ecxFi.Size()
 	ev.ecxCreatedAt = ecxFi.ModTime()
 
 	// open ecj file
-	if ev.ecjFile, err = os.OpenFile(baseFileName+".ecj", os.O_RDWR|os.O_CREATE, 0644); err != nil {
-		return nil, fmt.Errorf("cannot open ec volume journal %s.ecj: %v", baseFileName, err)
+	if ev.ecjFile, err = os.OpenFile(indexBaseFileName+".ecj", os.O_RDWR|os.O_CREATE, 0644); err != nil {
+		return nil, fmt.Errorf("cannot open ec volume journal %s.ecj: %v", indexBaseFileName, err)
 	}
 
 	// read volume info
 	ev.Version = needle.Version3
-	if volumeInfo, found, _ := pb.MaybeLoadVolumeInfo(baseFileName + ".vif"); found {
+	if volumeInfo, found, _ := pb.MaybeLoadVolumeInfo(dataBaseFileName + ".vif"); found {
 		ev.Version = needle.Version(volumeInfo.Version)
 	} else {
-		pb.SaveVolumeInfo(baseFileName+".vif", &volume_server_pb.VolumeInfo{Version: uint32(ev.Version)})
+		pb.SaveVolumeInfo(dataBaseFileName+".vif", &volume_server_pb.VolumeInfo{Version: uint32(ev.Version)})
 	}
 
 	ev.ShardLocations = make(map[ShardId][]string)
@@ -134,15 +136,26 @@ func (ev *EcVolume) Destroy() {
 	for _, s := range ev.Shards {
 		s.Destroy()
 	}
-	os.Remove(ev.FileName() + ".ecx")
-	os.Remove(ev.FileName() + ".ecj")
-	os.Remove(ev.FileName() + ".vif")
+	os.Remove(ev.FileName(".ecx"))
+	os.Remove(ev.FileName(".ecj"))
+	os.Remove(ev.FileName(".vif"))
 }
 
-func (ev *EcVolume) FileName() string {
+func (ev *EcVolume) FileName(ext string) string {
+	switch ext {
+	case ".ecx", ".ecj":
+		return ev.IndexBaseFileName() + ext
+	}
+	// .vif
+	return ev.DataBaseFileName() + ext
+}
 
+func (ev *EcVolume) DataBaseFileName() string {
 	return EcShardFileName(ev.Collection, ev.dir, int(ev.VolumeId))
+}
 
+func (ev *EcVolume) IndexBaseFileName() string {
+	return EcShardFileName(ev.Collection, ev.dirIdx, int(ev.VolumeId))
 }
 
 func (ev *EcVolume) ShardSize() uint64 {

weed/storage/needle/needle_parse_upload.go

@@ -18,12 +18,12 @@ import (
 )
 
 type ParsedUpload struct {
-	FileName         string
-	Data             []byte
-	MimeType         string
-	PairMap          map[string]string
-	IsGzipped        bool
-	IsZstd           bool
+	FileName  string
+	Data      []byte
+	MimeType  string
+	PairMap   map[string]string
+	IsGzipped bool
+	// IsZstd           bool
 	OriginalDataSize int
 	ModifiedTime     uint64
 	Ttl              *TTL
@@ -100,7 +100,7 @@ func ParseUpload(r *http.Request, sizeLimit int64) (pu *ParsedUpload, e error) {
 
 func parsePut(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
 	pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip"
-	pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
+	// pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
 	pu.MimeType = r.Header.Get("Content-Type")
 	pu.FileName = ""
 	pu.Data, e = ioutil.ReadAll(io.LimitReader(r.Body, sizeLimit+1))
@@ -194,7 +194,7 @@ func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error
 		}
 
 		pu.IsGzipped = part.Header.Get("Content-Encoding") == "gzip"
-		pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd"
+		// pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd"
 	}
 
 	return

weed/storage/needle_map.go

@@ -2,9 +2,11 @@ package storage
 
 import (
 	"fmt"
+	"io"
 	"os"
 	"sync"
 
+	"github.com/chrislusf/seaweedfs/weed/storage/idx"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
 	. "github.com/chrislusf/seaweedfs/weed/storage/types"
 )
@@ -31,6 +33,7 @@ type NeedleMapper interface {
 	MaxFileKey() NeedleId
 	IndexFileSize() uint64
 	Sync() error
+	ReadIndexEntry(n int64) (key NeedleId, offset Offset, size Size, err error)
 }
 
 type baseNeedleMapper struct {
@@ -64,3 +67,20 @@ func (nm *baseNeedleMapper) appendToIndexFile(key NeedleId, offset Offset, size
 func (nm *baseNeedleMapper) Sync() error {
 	return nm.indexFile.Sync()
 }
+
+func (nm *baseNeedleMapper) ReadIndexEntry(n int64) (key NeedleId, offset Offset, size Size, err error) {
+	bytes := make([]byte, NeedleMapEntrySize)
+	var readCount int
+	if readCount, err = nm.indexFile.ReadAt(bytes, n*NeedleMapEntrySize); err != nil {
+		if err == io.EOF {
+			if readCount == NeedleMapEntrySize {
+				err = nil
+			}
+		}
+		if err != nil {
+			return
+		}
+	}
+	key, offset, size = idx.IdxFileEntry(bytes)
+	return
+}

weed/storage/needle_map_sorted_file.go

@@ -16,18 +16,18 @@ type SortedFileNeedleMap struct {
 	dbFileSize   int64
 }
 
-func NewSortedFileNeedleMap(baseFileName string, indexFile *os.File) (m *SortedFileNeedleMap, err error) {
-	m = &SortedFileNeedleMap{baseFileName: baseFileName}
+func NewSortedFileNeedleMap(indexBaseFileName string, indexFile *os.File) (m *SortedFileNeedleMap, err error) {
+	m = &SortedFileNeedleMap{baseFileName: indexBaseFileName}
 	m.indexFile = indexFile
-	fileName := baseFileName + ".sdx"
+	fileName := indexBaseFileName + ".sdx"
 	if !isSortedFileFresh(fileName, indexFile) {
 		glog.V(0).Infof("Start to Generate %s from %s", fileName, indexFile.Name())
-		erasure_coding.WriteSortedFileFromIdx(baseFileName, ".sdx")
+		erasure_coding.WriteSortedFileFromIdx(indexBaseFileName, ".sdx")
 		glog.V(0).Infof("Finished Generating %s from %s", fileName, indexFile.Name())
 	}
 	glog.V(1).Infof("Opening %s...", fileName)
 
-	if m.dbFile, err = os.Open(baseFileName + ".sdx"); err != nil {
+	if m.dbFile, err = os.Open(indexBaseFileName + ".sdx"); err != nil {
 		return
 	}
 	dbStat, _ := m.dbFile.Stat()

weed/storage/store.go

@@ -16,7 +16,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
 	. "github.com/chrislusf/seaweedfs/weed/storage/types"
-	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 const (
@@ -53,11 +52,11 @@ func (s *Store) String() (str string) {
 	return
 }
 
-func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int, minFreeSpacePercents []float32, needleMapKind NeedleMapType) (s *Store) {
+func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int, minFreeSpacePercents []float32, idxFolder string, needleMapKind NeedleMapType) (s *Store) {
 	s = &Store{grpcDialOption: grpcDialOption, Port: port, Ip: ip, PublicUrl: publicUrl, NeedleMapType: needleMapKind}
 	s.Locations = make([]*DiskLocation, 0)
 	for i := 0; i < len(dirnames); i++ {
-		location := NewDiskLocation(util.ResolvePath(dirnames[i]), maxVolumeCounts[i], minFreeSpacePercents[i])
+		location := NewDiskLocation(dirnames[i], maxVolumeCounts[i], minFreeSpacePercents[i], idxFolder)
 		location.loadExistingVolumes(needleMapKind)
 		s.Locations = append(s.Locations, location)
 		stats.VolumeServerMaxVolumeCounter.Add(float64(maxVolumeCounts[i]))
@@ -122,7 +121,7 @@ func (s *Store) addVolume(vid needle.VolumeId, collection string, needleMapKind
 	if location := s.FindFreeLocation(); location != nil {
 		glog.V(0).Infof("In dir %s adds volume:%v collection:%s replicaPlacement:%v ttl:%v",
 			location.Directory, vid, collection, replicaPlacement, ttl)
-		if volume, err := NewVolume(location.Directory, collection, vid, needleMapKind, replicaPlacement, ttl, preallocate, memoryMapMaxSizeMb); err == nil {
+		if volume, err := NewVolume(location.Directory, location.IdxDirectory, collection, vid, needleMapKind, replicaPlacement, ttl, preallocate, memoryMapMaxSizeMb); err == nil {
 			location.SetVolume(vid, volume)
 			glog.V(0).Infof("add volume %d", vid)
 			s.NewVolumesChan <- master_pb.VolumeShortInformationMessage{
@@ -222,7 +221,12 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat {
 				if v.expiredLongEnough(MAX_TTL_VOLUME_REMOVAL_DELAY) {
 					deleteVids = append(deleteVids, v.Id)
 				} else {
-					glog.V(0).Infoln("volume", v.Id, "is expired.")
+					glog.V(0).Infof("volume %d is expired", v.Id)
+				}
+				if v.lastIoError != nil {
+					deleteVids = append(deleteVids, v.Id)
+				} else {
+					glog.Warningf("volume %d has IO error", v.Id)
 				}
 			}
 			collectionVolumeSize[v.Collection] += volumeMessage.Size

weed/storage/volume.go

@@ -21,6 +21,7 @@ import (
 type Volume struct {
 	Id                 needle.VolumeId
 	dir                string
+	dirIdx             string
 	Collection         string
 	DataBackend        backend.BackendStorageFile
 	nm                 NeedleMapper
@@ -45,11 +46,13 @@ type Volume struct {
 
 	volumeInfo *volume_server_pb.VolumeInfo
 	location   *DiskLocation
+
+	lastIoError     error
 }
 
-func NewVolume(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapType, replicaPlacement *super_block.ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapMaxSizeMb uint32) (v *Volume, e error) {
+func NewVolume(dirname string, dirIdx string, collection string, id needle.VolumeId, needleMapKind NeedleMapType, replicaPlacement *super_block.ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapMaxSizeMb uint32) (v *Volume, e error) {
 	// if replicaPlacement is nil, the superblock will be loaded from disk
-	v = &Volume{dir: dirname, Collection: collection, Id: id, MemoryMapMaxSizeMb: memoryMapMaxSizeMb,
+	v = &Volume{dir: dirname, dirIdx: dirIdx, Collection: collection, Id: id, MemoryMapMaxSizeMb: memoryMapMaxSizeMb,
 		asyncRequestsChan: make(chan *needle.AsyncRequest, 128)}
 	v.SuperBlock = super_block.SuperBlock{ReplicaPlacement: replicaPlacement, Ttl: ttl}
 	v.needleMapKind = needleMapKind
@@ -61,7 +64,7 @@ func NewVolume(dirname string, collection string, id needle.VolumeId, needleMapK
 func (v *Volume) String() string {
 	v.noWriteLock.RLock()
 	defer v.noWriteLock.RUnlock()
-	return fmt.Sprintf("Id:%v, dir:%s, Collection:%s, dataFile:%v, nm:%v, noWrite:%v canDelete:%v", v.Id, v.dir, v.Collection, v.DataBackend, v.nm, v.noWriteOrDelete || v.noWriteCanDelete, v.noWriteCanDelete)
+	return fmt.Sprintf("Id:%v dir:%s dirIdx:%s Collection:%s dataFile:%v nm:%v noWrite:%v canDelete:%v", v.Id, v.dir, v.dirIdx, v.Collection, v.DataBackend, v.nm, v.noWriteOrDelete || v.noWriteCanDelete, v.noWriteCanDelete)
 }
 
 func VolumeFileName(dir string, collection string, id int) (fileName string) {
@@ -74,10 +77,23 @@ func VolumeFileName(dir string, collection string, id int) (fileName string) {
 	return
 }
 
-func (v *Volume) FileName() (fileName string) {
+func (v *Volume) DataFileName() (fileName string) {
 	return VolumeFileName(v.dir, v.Collection, int(v.Id))
 }
 
+func (v *Volume) IndexFileName() (fileName string) {
+	return VolumeFileName(v.dirIdx, v.Collection, int(v.Id))
+}
+
+func (v *Volume) FileName(ext string) (fileName string) {
+	switch ext {
+	case ".idx", ".cpx", ".ldb":
+		return VolumeFileName(v.dirIdx, v.Collection, int(v.Id)) + ext
+	}
+	// .dat, .cpd, .vif
+	return VolumeFileName(v.dir, v.Collection, int(v.Id)) + ext
+}
+
 func (v *Volume) Version() needle.Version {
 	if v.volumeInfo.Version != 0 {
 		v.SuperBlock.Version = needle.Version(v.volumeInfo.Version)

weed/storage/volume_backup.go

@@ -124,9 +124,9 @@ func (v *Volume) findLastAppendAtNs() (uint64, error) {
 }
 
 func (v *Volume) locateLastAppendEntry() (Offset, error) {
-	indexFile, e := os.OpenFile(v.FileName()+".idx", os.O_RDONLY, 0644)
+	indexFile, e := os.OpenFile(v.FileName(".idx"), os.O_RDONLY, 0644)
 	if e != nil {
-		return Offset{}, fmt.Errorf("cannot read %s.idx: %v", v.FileName(), e)
+		return Offset{}, fmt.Errorf("cannot read %s: %v", v.FileName(".idx"), e)
 	}
 	defer indexFile.Close()
 
@@ -156,9 +156,9 @@ func (v *Volume) readAppendAtNs(offset Offset) (uint64, error) {
 
 	n, _, bodyLength, err := needle.ReadNeedleHeader(v.DataBackend, v.SuperBlock.Version, offset.ToAcutalOffset())
 	if err != nil {
-		return 0, fmt.Errorf("ReadNeedleHeader: %v", err)
+		return 0, fmt.Errorf("ReadNeedleHeader %s [%d,%d): %v", v.DataBackend.Name(), offset.ToAcutalOffset(), offset.ToAcutalOffset()+NeedleHeaderSize, err)
 	}
-	_, err = n.ReadNeedleBody(v.DataBackend, v.SuperBlock.Version, offset.ToAcutalOffset()+int64(NeedleHeaderSize), bodyLength)
+	_, err = n.ReadNeedleBody(v.DataBackend, v.SuperBlock.Version, offset.ToAcutalOffset()+NeedleHeaderSize, bodyLength)
 	if err != nil {
 		return 0, fmt.Errorf("ReadNeedleBody offset %d, bodyLength %d: %v", offset.ToAcutalOffset(), bodyLength, err)
 	}
@@ -168,25 +168,13 @@ func (v *Volume) readAppendAtNs(offset Offset) (uint64, error) {
 
 // on server side
 func (v *Volume) BinarySearchByAppendAtNs(sinceNs uint64) (offset Offset, isLast bool, err error) {
-	indexFile, openErr := os.OpenFile(v.FileName()+".idx", os.O_RDONLY, 0644)
-	if openErr != nil {
-		err = fmt.Errorf("cannot read %s.idx: %v", v.FileName(), openErr)
-		return
-	}
-	defer indexFile.Close()
 
-	fi, statErr := indexFile.Stat()
-	if statErr != nil {
-		err = fmt.Errorf("file %s stat error: %v", indexFile.Name(), statErr)
-		return
-	}
-	fileSize := fi.Size()
+	fileSize := int64(v.IndexFileSize())
 	if fileSize%NeedleMapEntrySize != 0 {
-		err = fmt.Errorf("unexpected file %s size: %d", indexFile.Name(), fileSize)
+		err = fmt.Errorf("unexpected file %s.idx size: %d", v.IndexFileName(), fileSize)
 		return
 	}
 
-	bytes := make([]byte, NeedleMapEntrySize)
 	entryCount := fileSize / NeedleMapEntrySize
 	l := int64(0)
 	h := entryCount
@@ -200,7 +188,7 @@ func (v *Volume) BinarySearchByAppendAtNs(sinceNs uint64) (offset Offset, isLast
 		}
 
 		// read the appendAtNs for entry m
-		offset, err = v.readAppendAtNsForIndexEntry(indexFile, bytes, m)
+		offset, err = v.readOffsetFromIndex(m)
 		if err != nil {
 			return
 		}
@@ -224,19 +212,21 @@ func (v *Volume) BinarySearchByAppendAtNs(sinceNs uint64) (offset Offset, isLast
 		return Offset{}, true, nil
 	}
 
-	offset, err = v.readAppendAtNsForIndexEntry(indexFile, bytes, l)
+	offset, err = v.readOffsetFromIndex(l)
 
 	return offset, false, err
 
 }
 
 // bytes is of size NeedleMapEntrySize
-func (v *Volume) readAppendAtNsForIndexEntry(indexFile *os.File, bytes []byte, m int64) (Offset, error) {
-	if _, readErr := indexFile.ReadAt(bytes, m*NeedleMapEntrySize); readErr != nil && readErr != io.EOF {
-		return Offset{}, readErr
+func (v *Volume) readOffsetFromIndex(m int64) (Offset, error) {
+	v.dataFileAccessLock.RLock()
+	defer v.dataFileAccessLock.RUnlock()
+	if v.nm == nil {
+		return Offset{}, io.EOF
 	}
-	_, offset, _ := idx.IdxFileEntry(bytes)
-	return offset, nil
+	_, offset, _, err := v.nm.ReadIndexEntry(m)
+	return offset, err
 }
 
 // generate the volume idx

weed/storage/volume_loading.go

@@ -23,7 +23,6 @@ func loadVolumeWithoutIndex(dirname string, collection string, id needle.VolumeI
 }
 
 func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind NeedleMapType, preallocate int64) (err error) {
-	fileName := v.FileName()
 	alreadyHasSuperBlock := false
 
 	hasVolumeInfoFile := v.maybeLoadVolumeInfo() && v.volumeInfo.Version != 0
@@ -34,17 +33,17 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind
 		glog.V(0).Infof("loading volume %d from remote %v", v.Id, v.volumeInfo.Files)
 		v.LoadRemoteFile()
 		alreadyHasSuperBlock = true
-	} else if exists, canRead, canWrite, modifiedTime, fileSize := util.CheckFile(fileName + ".dat"); exists {
+	} else if exists, canRead, canWrite, modifiedTime, fileSize := util.CheckFile(v.FileName(".dat")); exists {
 		// open dat file
 		if !canRead {
-			return fmt.Errorf("cannot read Volume Data file %s.dat", fileName)
+			return fmt.Errorf("cannot read Volume Data file %s", v.FileName(".dat"))
 		}
 		var dataFile *os.File
 		if canWrite {
-			dataFile, err = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
+			dataFile, err = os.OpenFile(v.FileName(".dat"), os.O_RDWR|os.O_CREATE, 0644)
 		} else {
-			glog.V(0).Infoln("opening " + fileName + ".dat in READONLY mode")
-			dataFile, err = os.Open(fileName + ".dat")
+			glog.V(0).Infof("opening %s in READONLY mode", v.FileName(".dat"))
+			dataFile, err = os.Open(v.FileName(".dat"))
 			v.noWriteOrDelete = true
 		}
 		v.lastModifiedTsSeconds = uint64(modifiedTime.Unix())
@@ -54,17 +53,17 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind
 		v.DataBackend = backend.NewDiskFile(dataFile)
 	} else {
 		if createDatIfMissing {
-			v.DataBackend, err = backend.CreateVolumeFile(fileName+".dat", preallocate, v.MemoryMapMaxSizeMb)
+			v.DataBackend, err = backend.CreateVolumeFile(v.FileName(".dat"), preallocate, v.MemoryMapMaxSizeMb)
 		} else {
-			return fmt.Errorf("Volume Data file %s.dat does not exist.", fileName)
+			return fmt.Errorf("volume data file %s does not exist", v.FileName(".dat"))
 		}
 	}
 
 	if err != nil {
 		if !os.IsPermission(err) {
-			return fmt.Errorf("cannot load Volume Data %s.dat: %v", fileName, err)
+			return fmt.Errorf("cannot load volume data %s: %v", v.FileName(".dat"), err)
 		} else {
-			return fmt.Errorf("load data file %s.dat: %v", fileName, err)
+			return fmt.Errorf("load data file %s: %v", v.FileName(".dat"), err)
 		}
 	}
 
@@ -72,21 +71,27 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind
 		err = v.readSuperBlock()
 	} else {
 		if !v.SuperBlock.Initialized() {
-			return fmt.Errorf("volume %s.dat not initialized", fileName)
+			return fmt.Errorf("volume %s not initialized", v.FileName(".dat"))
 		}
 		err = v.maybeWriteSuperBlock()
 	}
 	if err == nil && alsoLoadIndex {
+		// adjust for existing volumes with .idx together with .dat files
+		if v.dirIdx != v.dir {
+			if util.FileExists(v.DataFileName()+".idx") {
+				v.dirIdx = v.dir
+			}
+		}
 		var indexFile *os.File
 		if v.noWriteOrDelete {
-			glog.V(0).Infoln("open to read file", fileName+".idx")
-			if indexFile, err = os.OpenFile(fileName+".idx", os.O_RDONLY, 0644); err != nil {
-				return fmt.Errorf("cannot read Volume Index %s.idx: %v", fileName, err)
+			glog.V(0).Infoln("open to read file", v.FileName(".idx"))
+			if indexFile, err = os.OpenFile(v.FileName(".idx"), os.O_RDONLY, 0644); err != nil {
+				return fmt.Errorf("cannot read Volume Index %s: %v",  v.FileName(".idx"), err)
 			}
 		} else {
-			glog.V(1).Infoln("open to write file", fileName+".idx")
-			if indexFile, err = os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644); err != nil {
-				return fmt.Errorf("cannot write Volume Index %s.idx: %v", fileName, err)
+			glog.V(1).Infoln("open to write file", v.FileName(".idx"))
+			if indexFile, err = os.OpenFile(v.FileName(".idx"), os.O_RDWR|os.O_CREATE, 0644); err != nil {
+				return fmt.Errorf("cannot write Volume Index %s: %v",  v.FileName(".idx"), err)
 			}
 		}
 		if v.lastAppendAtNs, err = CheckAndFixVolumeDataIntegrity(v, indexFile); err != nil {
@@ -95,45 +100,45 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind
 		}
 
 		if v.noWriteOrDelete || v.noWriteCanDelete {
-			if v.nm, err = NewSortedFileNeedleMap(fileName, indexFile); err != nil {
-				glog.V(0).Infof("loading sorted db %s error: %v", fileName+".sdx", err)
+			if v.nm, err = NewSortedFileNeedleMap(v.IndexFileName(), indexFile); err != nil {
+				glog.V(0).Infof("loading sorted db %s error: %v",  v.FileName(".sdx"), err)
 			}
 		} else {
 			switch needleMapKind {
 			case NeedleMapInMemory:
-				glog.V(0).Infoln("loading index", fileName+".idx", "to memory")
+				glog.V(0).Infoln("loading index", v.FileName(".idx"), "to memory")
 				if v.nm, err = LoadCompactNeedleMap(indexFile); err != nil {
-					glog.V(0).Infof("loading index %s to memory error: %v", fileName+".idx", err)
+					glog.V(0).Infof("loading index %s to memory error: %v", v.FileName(".idx"), err)
 				}
 			case NeedleMapLevelDb:
-				glog.V(0).Infoln("loading leveldb", fileName+".ldb")
+				glog.V(0).Infoln("loading leveldb", v.FileName(".ldb"))
 				opts := &opt.Options{
 					BlockCacheCapacity:            2 * 1024 * 1024, // default value is 8MiB
 					WriteBuffer:                   1 * 1024 * 1024, // default value is 4MiB
 					CompactionTableSizeMultiplier: 10,              // default value is 1
 				}
-				if v.nm, err = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); err != nil {
-					glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", err)
+				if v.nm, err = NewLevelDbNeedleMap(v.FileName(".ldb"), indexFile, opts); err != nil {
+					glog.V(0).Infof("loading leveldb %s error: %v", v.FileName(".ldb"), err)
 				}
 			case NeedleMapLevelDbMedium:
-				glog.V(0).Infoln("loading leveldb medium", fileName+".ldb")
+				glog.V(0).Infoln("loading leveldb medium", v.FileName(".ldb"))
 				opts := &opt.Options{
 					BlockCacheCapacity:            4 * 1024 * 1024, // default value is 8MiB
 					WriteBuffer:                   2 * 1024 * 1024, // default value is 4MiB
 					CompactionTableSizeMultiplier: 10,              // default value is 1
 				}
-				if v.nm, err = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); err != nil {
-					glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", err)
+				if v.nm, err = NewLevelDbNeedleMap(v.FileName(".ldb"), indexFile, opts); err != nil {
+					glog.V(0).Infof("loading leveldb %s error: %v", v.FileName(".ldb"), err)
 				}
 			case NeedleMapLevelDbLarge:
-				glog.V(0).Infoln("loading leveldb large", fileName+".ldb")
+				glog.V(0).Infoln("loading leveldb large", v.FileName(".ldb"))
 				opts := &opt.Options{
 					BlockCacheCapacity:            8 * 1024 * 1024, // default value is 8MiB
 					WriteBuffer:                   4 * 1024 * 1024, // default value is 4MiB
 					CompactionTableSizeMultiplier: 10,              // default value is 1
 				}
-				if v.nm, err = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); err != nil {
-					glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", err)
+				if v.nm, err = NewLevelDbNeedleMap(v.FileName(".ldb"), indexFile, opts); err != nil {
+					glog.V(0).Infof("loading leveldb %s error: %v", v.FileName(".ldb"), err)
 				}
 			}
 		}

weed/storage/volume_read_write.go

@@ -19,6 +19,18 @@ var ErrorNotFound = errors.New("not found")
 var ErrorDeleted = errors.New("already deleted")
 var ErrorSizeMismatch = errors.New("size mismatch")
 
+func (v *Volume) checkReadWriteError(err error) {
+	if err == nil {
+		if v.lastIoError != nil {
+			v.lastIoError = nil
+		}
+		return
+	}
+	if err.Error() == "input/output error" {
+		v.lastIoError = err
+	}
+}
+
 // isFileUnchanged checks whether this needle to write is same as last one.
 // It requires serialized access in the same volume.
 func (v *Volume) isFileUnchanged(n *needle.Needle) bool {
@@ -56,18 +68,24 @@ func (v *Volume) Destroy() (err error) {
 		}
 	}
 	v.Close()
-	removeVolumeFiles(v.FileName())
+	removeVolumeFiles(v.DataFileName())
+	removeVolumeFiles(v.IndexFileName())
 	return
 }
 
 func removeVolumeFiles(filename string) {
+	// basic
 	os.Remove(filename + ".dat")
 	os.Remove(filename + ".idx")
 	os.Remove(filename + ".vif")
+	// sorted index file
 	os.Remove(filename + ".sdx")
+	// compaction
 	os.Remove(filename + ".cpd")
 	os.Remove(filename + ".cpx")
+	// level db indx file
 	os.RemoveAll(filename + ".ldb")
+	// marker for damaged or incomplete volume
 	os.Remove(filename + ".note")
 }
 
@@ -109,7 +127,9 @@ func (v *Volume) syncWrite(n *needle.Needle) (offset uint64, size Size, isUnchan
 
 	// append to dat file
 	n.AppendAtNs = uint64(time.Now().UnixNano())
-	if offset, size, _, err = n.Append(v.DataBackend, v.Version()); err != nil {
+	offset, size, _, err = n.Append(v.DataBackend, v.Version())
+	v.checkReadWriteError(err)
+	if err != nil {
 		return
 	}
 
@@ -173,7 +193,9 @@ func (v *Volume) doWriteRequest(n *needle.Needle) (offset uint64, size Size, isU
 
 	// append to dat file
 	n.AppendAtNs = uint64(time.Now().UnixNano())
-	if offset, size, _, err = n.Append(v.DataBackend, v.Version()); err != nil {
+	offset, size, _, err = n.Append(v.DataBackend, v.Version())
+	v.checkReadWriteError(err)
+	if err != nil {
 		return
 	}
 	v.lastAppendAtNs = n.AppendAtNs
@@ -208,6 +230,7 @@ func (v *Volume) syncDelete(n *needle.Needle) (Size, error) {
 		n.Data = nil
 		n.AppendAtNs = uint64(time.Now().UnixNano())
 		offset, _, _, err := n.Append(v.DataBackend, v.Version())
+		v.checkReadWriteError(err)
 		if err != nil {
 			return size, err
 		}
@@ -246,6 +269,7 @@ func (v *Volume) doDeleteRequest(n *needle.Needle) (Size, error) {
 		n.Data = nil
 		n.AppendAtNs = uint64(time.Now().UnixNano())
 		offset, _, _, err := n.Append(v.DataBackend, v.Version())
+		v.checkReadWriteError(err)
 		if err != nil {
 			return size, err
 		}
@@ -283,6 +307,7 @@ func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption) (int, erro
 	if err == needle.ErrorSizeMismatch && OffsetSize == 4 {
 		err = n.ReadData(v.DataBackend, nv.Offset.ToAcutalOffset()+int64(MaxPossibleVolumeSize), readSize, v.Version())
 	}
+	v.checkReadWriteError(err)
 	if err != nil {
 		return 0, err
 	}

weed/storage/volume_tier.go

@@ -14,7 +14,7 @@ func (v *Volume) GetVolumeInfo() *volume_server_pb.VolumeInfo {
 
 func (v *Volume) maybeLoadVolumeInfo() (found bool) {
 
-	v.volumeInfo, v.hasRemoteFile, _ = pb.MaybeLoadVolumeInfo(v.FileName() + ".vif")
+	v.volumeInfo, v.hasRemoteFile, _ = pb.MaybeLoadVolumeInfo(v.FileName(".vif"))
 
 	if v.hasRemoteFile {
 		glog.V(0).Infof("volume %d is tiered to %s as %s and read only", v.Id,
@@ -43,7 +43,7 @@ func (v *Volume) LoadRemoteFile() error {
 
 func (v *Volume) SaveVolumeInfo() error {
 
-	tierFileName := v.FileName() + ".vif"
+	tierFileName := v.FileName(".vif")
 
 	return pb.SaveVolumeInfo(tierFileName, v.volumeInfo)
 

weed/storage/volume_vacuum.go

@@ -49,7 +49,6 @@ func (v *Volume) Compact(preallocate int64, compactionBytePerSecond int64) error
 		v.isCompacting = false
 	}()
 
-	filePath := v.FileName()
 	v.lastCompactIndexOffset = v.IndexFileSize()
 	v.lastCompactRevision = v.SuperBlock.CompactionRevision
 	glog.V(3).Infof("creating copies for volume %d ,last offset %d...", v.Id, v.lastCompactIndexOffset)
@@ -59,7 +58,7 @@ func (v *Volume) Compact(preallocate int64, compactionBytePerSecond int64) error
 	if err := v.nm.Sync(); err != nil {
 		glog.V(0).Infof("compact fail to sync volume idx %d", v.Id)
 	}
-	return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx", preallocate, compactionBytePerSecond)
+	return v.copyDataAndGenerateIndexFile(v.FileName(".cpd"), v.FileName(".cpx"), preallocate, compactionBytePerSecond)
 }
 
 // compact a volume based on deletions in .idx files
@@ -75,7 +74,6 @@ func (v *Volume) Compact2(preallocate int64, compactionBytePerSecond int64) erro
 		v.isCompacting = false
 	}()
 
-	filePath := v.FileName()
 	v.lastCompactIndexOffset = v.IndexFileSize()
 	v.lastCompactRevision = v.SuperBlock.CompactionRevision
 	glog.V(3).Infof("creating copies for volume %d ...", v.Id)
@@ -85,7 +83,7 @@ func (v *Volume) Compact2(preallocate int64, compactionBytePerSecond int64) erro
 	if err := v.nm.Sync(); err != nil {
 		glog.V(0).Infof("compact2 fail to sync volume idx %d: %v", v.Id, err)
 	}
-	return copyDataBasedOnIndexFile(filePath+".dat", filePath+".idx", filePath+".cpd", filePath+".cpx", v.SuperBlock, v.Version(), preallocate, compactionBytePerSecond)
+	return copyDataBasedOnIndexFile(v.FileName(".dat"), v.FileName(".idx"), v.FileName(".cpd"), v.FileName(".cpx"), v.SuperBlock, v.Version(), preallocate, compactionBytePerSecond)
 }
 
 func (v *Volume) CommitCompact() error {
@@ -113,40 +111,40 @@ func (v *Volume) CommitCompact() error {
 	stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "volume").Dec()
 
 	var e error
-	if e = v.makeupDiff(v.FileName()+".cpd", v.FileName()+".cpx", v.FileName()+".dat", v.FileName()+".idx"); e != nil {
+	if e = v.makeupDiff(v.FileName(".cpd"), v.FileName(".cpx"), v.FileName(".dat"), v.FileName(".idx")); e != nil {
 		glog.V(0).Infof("makeupDiff in CommitCompact volume %d failed %v", v.Id, e)
-		e = os.Remove(v.FileName() + ".cpd")
+		e = os.Remove(v.FileName(".cpd"))
 		if e != nil {
 			return e
 		}
-		e = os.Remove(v.FileName() + ".cpx")
+		e = os.Remove(v.FileName(".cpx"))
 		if e != nil {
 			return e
 		}
 	} else {
 		if runtime.GOOS == "windows" {
-			e = os.RemoveAll(v.FileName() + ".dat")
+			e = os.RemoveAll(v.FileName(".dat"))
 			if e != nil {
 				return e
 			}
-			e = os.RemoveAll(v.FileName() + ".idx")
+			e = os.RemoveAll(v.FileName(".idx"))
 			if e != nil {
 				return e
 			}
 		}
 		var e error
-		if e = os.Rename(v.FileName()+".cpd", v.FileName()+".dat"); e != nil {
-			return fmt.Errorf("rename %s: %v", v.FileName()+".cpd", e)
+		if e = os.Rename(v.FileName(".cpd"), v.FileName(".dat")); e != nil {
+			return fmt.Errorf("rename %s: %v", v.FileName(".cpd"), e)
 		}
-		if e = os.Rename(v.FileName()+".cpx", v.FileName()+".idx"); e != nil {
-			return fmt.Errorf("rename %s: %v", v.FileName()+".cpx", e)
+		if e = os.Rename(v.FileName(".cpx"), v.FileName(".idx")); e != nil {
+			return fmt.Errorf("rename %s: %v", v.FileName(".cpx"), e)
 		}
 	}
 
 	//glog.V(3).Infof("Pretending to be vacuuming...")
 	//time.Sleep(20 * time.Second)
 
-	os.RemoveAll(v.FileName() + ".ldb")
+	os.RemoveAll(v.FileName(".ldb"))
 
 	glog.V(3).Infof("Loading volume %d commit file...", v.Id)
 	if e = v.load(true, false, v.needleMapKind, 0); e != nil {
@@ -158,8 +156,8 @@ func (v *Volume) CommitCompact() error {
 func (v *Volume) cleanupCompact() error {
 	glog.V(0).Infof("Cleaning up volume %d vacuuming...", v.Id)
 
-	e1 := os.Remove(v.FileName() + ".cpd")
-	e2 := os.Remove(v.FileName() + ".cpx")
+	e1 := os.Remove(v.FileName(".cpd"))
+	e2 := os.Remove(v.FileName(".cpx"))
 	if e1 != nil {
 		return e1
 	}

weed/storage/volume_vacuum_test.go

@@ -69,7 +69,7 @@ func TestCompaction(t *testing.T) {
 	}
 	defer os.RemoveAll(dir) // clean up
 
-	v, err := NewVolume(dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, 0)
+	v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, 0)
 	if err != nil {
 		t.Fatalf("volume creation: %v", err)
 	}
@@ -96,7 +96,7 @@ func TestCompaction(t *testing.T) {
 
 	v.Close()
 
-	v, err = NewVolume(dir, "", 1, NeedleMapInMemory, nil, nil, 0, 0)
+	v, err = NewVolume(dir, dir, "", 1, NeedleMapInMemory, nil, nil, 0, 0)
 	if err != nil {
 		t.Fatalf("volume reloading: %v", err)
 	}

weed/topology/topology.go

@@ -177,7 +177,9 @@ func (t *Topology) DeleteCollection(collectionName string) {
 }
 
 func (t *Topology) RegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) {
-	t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl).RegisterVolume(&v, dn)
+	vl := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl)
+	vl.RegisterVolume(&v, dn)
+	vl.EnsureCorrectWritables(&v)
 }
 func (t *Topology) UnRegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) {
 	glog.Infof("removing volume info:%+v", v)

weed/topology/topology_vacuum.go

@@ -144,12 +144,12 @@ func batchVacuumVolumeCleanup(grpcDialOption grpc.DialOption, vl *VolumeLayout,
 	}
 }
 
-func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float64, preallocate int64) int {
+func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float64, preallocate int64) {
 
 	// if there is vacuum going on, return immediately
 	swapped := atomic.CompareAndSwapInt64(&t.vacuumLockCounter, 0, 1)
 	if !swapped {
-		return 0
+		return
 	}
 	defer atomic.StoreInt64(&t.vacuumLockCounter, 0)
 
@@ -165,7 +165,6 @@ func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float
 			}
 		}
 	}
-	return 0
 }
 
 func vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeLayout *VolumeLayout, c *Collection, garbageThreshold float64, preallocate int64) {

weed/topology/volume_layout.go

@@ -132,6 +132,8 @@ func NewVolumeLayout(rp *super_block.ReplicaPlacement, ttl *needle.TTL, volumeSi
 }
 
 func (vl *VolumeLayout) String() string {
+	vl.accessLock.RLock()
+	defer vl.accessLock.RUnlock()
 	return fmt.Sprintf("rp:%v, ttl:%v, vid2location:%v, writables:%v, volumeSizeLimit:%v", vl.rp, vl.ttl, vl.vid2location, vl.writables, vl.volumeSizeLimit)
 }
 
@@ -139,7 +141,6 @@ func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) {
 	vl.accessLock.Lock()
 	defer vl.accessLock.Unlock()
 
-	defer vl.ensureCorrectWritables(v.Id)
 	defer vl.rememberOversizedVolume(v, dn)
 
 	if _, ok := vl.vid2location[v.Id]; !ok {

weed/util/compression.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/klauspost/compress/zstd"
+	// "github.com/klauspost/compress/zstd"
 )
 
 var (
@@ -55,19 +55,15 @@ func GzipData(input []byte) ([]byte, error) {
 	return buf.Bytes(), nil
 }
 
-var zstdEncoder, _ = zstd.NewWriter(nil)
-
-func ZstdData(input []byte) ([]byte, error) {
-	return zstdEncoder.EncodeAll(input, nil), nil
-}
-
 func DecompressData(input []byte) ([]byte, error) {
 	if IsGzippedContent(input) {
 		return ungzipData(input)
 	}
-	if IsZstdContent(input) {
-		return unzstdData(input)
-	}
+	/*
+		if IsZstdContent(input) {
+			return unzstdData(input)
+		}
+	*/
 	return input, UnsupportedCompression
 }
 
@@ -82,12 +78,6 @@ func ungzipData(input []byte) ([]byte, error) {
 	return output, err
 }
 
-var decoder, _ = zstd.NewReader(nil)
-
-func unzstdData(input []byte) ([]byte, error) {
-	return decoder.DecodeAll(input, nil)
-}
-
 func IsGzippedContent(data []byte) bool {
 	if len(data) < 2 {
 		return false
@@ -95,12 +85,26 @@ func IsGzippedContent(data []byte) bool {
 	return data[0] == 31 && data[1] == 139
 }
 
+/*
+var zstdEncoder, _ = zstd.NewWriter(nil)
+
+func ZstdData(input []byte) ([]byte, error) {
+	return zstdEncoder.EncodeAll(input, nil), nil
+}
+
+var decoder, _ = zstd.NewReader(nil)
+
+func unzstdData(input []byte) ([]byte, error) {
+	return decoder.DecodeAll(input, nil)
+}
+
 func IsZstdContent(data []byte) bool {
 	if len(data) < 4 {
 		return false
 	}
 	return data[3] == 0xFD && data[2] == 0x2F && data[1] == 0xB5 && data[0] == 0x28
 }
+*/
 
 /*
 * Default not to compressed since compression can be done on client side.

weed/util/constants.go

@@ -5,7 +5,7 @@ import (
 )
 
 var (
-	VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 11)
+	VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 13)
 	COMMIT  = ""
 )
 

weed/util/retry.go

@@ -22,9 +22,11 @@ func Retry(name string, job func() error) (err error) {
 		}
 		if strings.Contains(err.Error(), "transport") {
 			hasErr = true
-			glog.V(0).Infof("retry %s", name)
+			glog.V(0).Infof("retry %s: err: %v", name, err)
 			time.Sleep(waitTime)
 			waitTime += waitTime / 2
+		} else {
+			break
 		}
 	}
 	return err