1.29

replication: add GoCDK PubSub support

.travis.yml

@@ -1,18 +1,21 @@
 sudo: false
 language: go
 go:
-- 1.10.x
-- 1.11.x
-- tip
+  - 1.10.x
+  - 1.11.x
+  - 1.12.x
+  # - tip
 
 before_install:
-- export PATH=/home/travis/gopath/bin:$PATH
+  - export PATH=/home/travis/gopath/bin:$PATH
 
 install:
-- go get ./weed/...
+  - export CGO_ENABLED="0"
+  - go env
+  - go get -u ./weed/...
 
 script:
-- go test ./weed/...
+  - go test ./weed/...
 
 before_deploy:
   - make release
@@ -22,23 +25,26 @@ deploy:
   api_key:
     secure: ERL986+ncQ8lwAJUYDrQ8s2/FxF/cyNIwJIFCqspnWxQgGNNyokET9HapmlPSxjpFRF0q6L2WCg9OY3mSVRq4oI6hg1igOQ12KlLyN71XSJ3c8w0Ay5ho48TQ9l3f3Iu97mntBCe9l0R9pnT8wj1VI8YJxloXwUMG2yeTjA9aBI=
   file:
-  - build/linux_arm.tar.gz
-  - build/linux_arm64.tar.gz
-  - build/linux_386.tar.gz
-  - build/linux_amd64.tar.gz
-  - build/darwin_amd64.tar.gz
-  - build/windows_386.zip
-  - build/windows_amd64.zip
-  - build/freebsd_arm.tar.gz
-  - build/freebsd_amd64.tar.gz
-  - build/freebsd_386.tar.gz
-  - build/netbsd_arm.tar.gz
-  - build/netbsd_amd64.tar.gz
-  - build/netbsd_386.tar.gz
-  - build/openbsd_arm.tar.gz
-  - build/openbsd_amd64.tar.gz
-  - build/openbsd_386.tar.gz
+    - build/linux_arm.tar.gz
+    - build/linux_arm64.tar.gz
+    - build/linux_386.tar.gz
+    - build/linux_amd64.tar.gz
+    - build/linux_amd64_large_disk.tar.gz
+    - build/darwin_amd64.tar.gz
+    - build/darwin_amd64_large_disk.tar.gz
+    - build/windows_386.zip
+    - build/windows_amd64.zip
+    - build/windows_amd64_large_disk.zip
+    - build/freebsd_arm.tar.gz
+    - build/freebsd_amd64.tar.gz
+    - build/freebsd_386.tar.gz
+    - build/netbsd_arm.tar.gz
+    - build/netbsd_amd64.tar.gz
+    - build/netbsd_386.tar.gz
+    - build/openbsd_arm.tar.gz
+    - build/openbsd_amd64.tar.gz
+    - build/openbsd_386.tar.gz
   on:
     tags: true
     repo: chrislusf/seaweedfs
-    go: tip
+    go: 1.12.x

Makefile

@@ -12,6 +12,9 @@ build = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -ldflags "-extldflags -stat
 tar = cd build && tar -cvzf $(1)_$(2).tar.gz $(appname)$(3) && rm $(appname)$(3)
 zip = cd build && zip $(1)_$(2).zip $(appname)$(3) && rm $(appname)$(3)
 
+build_large = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -tags 5BytesOffset -ldflags "-extldflags -static" -o build/$(appname)$(3) $(SOURCE_DIR)
+tar_large = cd build && tar -cvzf $(1)_$(2)_large_disk.tar.gz $(appname)$(3) && rm $(appname)$(3)
+zip_large = cd build && zip $(1)_$(2)_large_disk.zip $(appname)$(3) && rm $(appname)$(3)
 
 all: build
 
@@ -32,9 +35,21 @@ linux: deps
 	mkdir -p linux
 	GOOS=linux GOARCH=amd64 go build $(GO_FLAGS) -o linux/$(BINARY) $(SOURCE_DIR)
 
-release: deps windows_build darwin_build linux_build bsd_build
+release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_build 5_byte_darwin_build 5_byte_windows_build
 
 ##### LINUX BUILDS #####
+5_byte_linux_build:
+	$(call build_large,linux,amd64,)
+	$(call tar_large,linux,amd64)
+
+5_byte_darwin_build:
+	$(call build_large,darwin,amd64,)
+	$(call tar_large,darwin,amd64)
+
+5_byte_windows_build:
+	$(call build_large,windows,amd64,.exe)
+	$(call zip_large,windows,amd64,.exe)
+
 linux_build: build/linux_arm.tar.gz build/linux_arm64.tar.gz build/linux_386.tar.gz build/linux_amd64.tar.gz
 
 build/linux_386.tar.gz: $(sources)

docker/Dockerfile

@@ -22,6 +22,8 @@ EXPOSE 9333
 # s3 server http port
 EXPOSE 8333
 
+RUN mkdir -p /data/filerdb
+
 VOLUME /data
 
 COPY filer.toml /etc/seaweedfs/filer.toml

docker/Dockerfile.go_build

@@ -16,6 +16,8 @@ EXPOSE 9333
 # s3 server http port
 EXPOSE 8333
 
+RUN mkdir -p /data/filerdb
+
 VOLUME /data
 
 RUN mkdir -p /etc/seaweedfs

docker/entrypoint.sh

@@ -33,7 +33,6 @@ case "$1" in
   	if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then
 		ARGS="$ARGS -master=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT"
 	fi
-	mkdir -p /data/filerdb
   	exec /usr/bin/weed $@ $ARGS
 	;;
 

other/java/client/pom.xml

@@ -4,7 +4,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.0.8</version>
+    <version>1.0.9</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/src/main/java/seaweedfs/client/FilerClient.java

@@ -51,12 +51,26 @@ public class FilerClient {
         }
 
         return createEntry(
-            parent,
-            newDirectoryEntry(name, mode, uid, gid, userName, groupNames).build()
+                parent,
+                newDirectoryEntry(name, mode, uid, gid, userName, groupNames).build()
         );
 
     }
 
+    public boolean mv(String oldPath, String newPath) {
+
+        Path oldPathObject = Paths.get(oldPath);
+        String oldParent = oldPathObject.getParent().toString();
+        String oldName = oldPathObject.getFileName().toString();
+
+        Path newPathObject = Paths.get(newPath);
+        String newParent = newPathObject.getParent().toString();
+        String newName = newPathObject.getFileName().toString();
+
+        return atomicRenameEntry(oldParent, oldName, newParent, newName);
+
+    }
+
     public boolean rm(String path, boolean isRecursive) {
 
         Path pathObject = Paths.get(path);
@@ -64,10 +78,10 @@ public class FilerClient {
         String name = pathObject.getFileName().toString();
 
         return deleteEntry(
-            parent,
-            name,
-            true,
-            isRecursive);
+                parent,
+                name,
+                true,
+                isRecursive);
     }
 
     public boolean touch(String path, int mode) {
@@ -84,18 +98,18 @@ public class FilerClient {
         FilerProto.Entry entry = lookupEntry(parent, name);
         if (entry == null) {
             return createEntry(
-                parent,
-                newFileEntry(name, mode, uid, gid, userName, groupNames).build()
+                    parent,
+                    newFileEntry(name, mode, uid, gid, userName, groupNames).build()
             );
         }
         long now = System.currentTimeMillis() / 1000L;
         FilerProto.FuseAttributes.Builder attr = entry.getAttributes().toBuilder()
-            .setMtime(now)
-            .setUid(uid)
-            .setGid(gid)
-            .setUserName(userName)
-            .clearGroupName()
-            .addAllGroupName(Arrays.asList(groupNames));
+                .setMtime(now)
+                .setUid(uid)
+                .setGid(gid)
+                .setUserName(userName)
+                .clearGroupName()
+                .addAllGroupName(Arrays.asList(groupNames));
         return updateEntry(parent, entry.toBuilder().setAttributes(attr).build());
     }
 
@@ -105,17 +119,17 @@ public class FilerClient {
         long now = System.currentTimeMillis() / 1000L;
 
         return FilerProto.Entry.newBuilder()
-            .setName(name)
-            .setIsDirectory(true)
-            .setAttributes(FilerProto.FuseAttributes.newBuilder()
-                .setMtime(now)
-                .setCrtime(now)
-                .setUid(uid)
-                .setGid(gid)
-                .setFileMode(mode | 1 << 31)
-                .setUserName(userName)
-                .clearGroupName()
-                .addAllGroupName(Arrays.asList(groupNames)));
+                .setName(name)
+                .setIsDirectory(true)
+                .setAttributes(FilerProto.FuseAttributes.newBuilder()
+                        .setMtime(now)
+                        .setCrtime(now)
+                        .setUid(uid)
+                        .setGid(gid)
+                        .setFileMode(mode | 1 << 31)
+                        .setUserName(userName)
+                        .clearGroupName()
+                        .addAllGroupName(Arrays.asList(groupNames)));
     }
 
     public FilerProto.Entry.Builder newFileEntry(String name, int mode,
@@ -124,17 +138,17 @@ public class FilerClient {
         long now = System.currentTimeMillis() / 1000L;
 
         return FilerProto.Entry.newBuilder()
-            .setName(name)
-            .setIsDirectory(false)
-            .setAttributes(FilerProto.FuseAttributes.newBuilder()
-                .setMtime(now)
-                .setCrtime(now)
-                .setUid(uid)
-                .setGid(gid)
-                .setFileMode(mode)
-                .setUserName(userName)
-                .clearGroupName()
-                .addAllGroupName(Arrays.asList(groupNames)));
+                .setName(name)
+                .setIsDirectory(false)
+                .setAttributes(FilerProto.FuseAttributes.newBuilder()
+                        .setMtime(now)
+                        .setCrtime(now)
+                        .setUid(uid)
+                        .setGid(gid)
+                        .setFileMode(mode)
+                        .setUserName(userName)
+                        .clearGroupName()
+                        .addAllGroupName(Arrays.asList(groupNames)));
     }
 
     public List<FilerProto.Entry> listEntries(String path) {
@@ -160,20 +174,20 @@ public class FilerClient {
 
     public List<FilerProto.Entry> listEntries(String path, String entryPrefix, String lastEntryName, int limit) {
         return filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
-            .setDirectory(path)
-            .setPrefix(entryPrefix)
-            .setStartFromFileName(lastEntryName)
-            .setLimit(limit)
-            .build()).getEntriesList();
+                .setDirectory(path)
+                .setPrefix(entryPrefix)
+                .setStartFromFileName(lastEntryName)
+                .setLimit(limit)
+                .build()).getEntriesList();
     }
 
     public FilerProto.Entry lookupEntry(String directory, String entryName) {
         try {
             return filerGrpcClient.getBlockingStub().lookupDirectoryEntry(
-                FilerProto.LookupDirectoryEntryRequest.newBuilder()
-                    .setDirectory(directory)
-                    .setName(entryName)
-                    .build()).getEntry();
+                    FilerProto.LookupDirectoryEntryRequest.newBuilder()
+                            .setDirectory(directory)
+                            .setName(entryName)
+                            .build()).getEntry();
         } catch (Exception e) {
             LOG.warn("lookupEntry {}/{}: {}", directory, entryName, e);
             return null;
@@ -184,9 +198,9 @@ public class FilerClient {
     public boolean createEntry(String parent, FilerProto.Entry entry) {
         try {
             filerGrpcClient.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder()
-                .setDirectory(parent)
-                .setEntry(entry)
-                .build());
+                    .setDirectory(parent)
+                    .setEntry(entry)
+                    .build());
         } catch (Exception e) {
             LOG.warn("createEntry {}/{}: {}", parent, entry.getName(), e);
             return false;
@@ -197,9 +211,9 @@ public class FilerClient {
     public boolean updateEntry(String parent, FilerProto.Entry entry) {
         try {
             filerGrpcClient.getBlockingStub().updateEntry(FilerProto.UpdateEntryRequest.newBuilder()
-                .setDirectory(parent)
-                .setEntry(entry)
-                .build());
+                    .setDirectory(parent)
+                    .setEntry(entry)
+                    .build());
         } catch (Exception e) {
             LOG.warn("createEntry {}/{}: {}", parent, entry.getName(), e);
             return false;
@@ -210,11 +224,11 @@ public class FilerClient {
     public boolean deleteEntry(String parent, String entryName, boolean isDeleteFileChunk, boolean isRecursive) {
         try {
             filerGrpcClient.getBlockingStub().deleteEntry(FilerProto.DeleteEntryRequest.newBuilder()
-                .setDirectory(parent)
-                .setName(entryName)
-                .setIsDeleteData(isDeleteFileChunk)
-                .setIsRecursive(isRecursive)
-                .build());
+                    .setDirectory(parent)
+                    .setName(entryName)
+                    .setIsDeleteData(isDeleteFileChunk)
+                    .setIsRecursive(isRecursive)
+                    .build());
         } catch (Exception e) {
             LOG.warn("deleteEntry {}/{}: {}", parent, entryName, e);
             return false;
@@ -222,4 +236,19 @@ public class FilerClient {
         return true;
     }
 
+    public boolean atomicRenameEntry(String oldParent, String oldName, String newParent, String newName) {
+        try {
+            filerGrpcClient.getBlockingStub().atomicRenameEntry(FilerProto.AtomicRenameEntryRequest.newBuilder()
+                    .setOldDirectory(oldParent)
+                    .setOldName(oldName)
+                    .setNewDirectory(newParent)
+                    .setNewName(newName)
+                    .build());
+        } catch (Exception e) {
+            LOG.warn("atomicRenameEntry {}/{} => {}/{}: {}", oldParent, oldName, newParent, newName, e);
+            return false;
+        }
+        return true;
+    }
+
 }

other/java/client/src/main/proto/filer.proto

@@ -24,6 +24,9 @@ service SeaweedFiler {
     rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) {
     }
 
+    rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) {
+    }
+
     rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
     }
 
@@ -126,6 +129,16 @@ message DeleteEntryRequest {
 message DeleteEntryResponse {
 }
 
+message AtomicRenameEntryRequest {
+    string old_directory = 1;
+    string old_name = 2;
+    string new_directory = 3;
+    string new_name = 4;
+}
+
+message AtomicRenameEntryResponse {
+}
+
 message AssignVolumeRequest {
     int32 count = 1;
     string collection = 2;

other/java/hdfs/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.0.8</seaweedfs.client.version>
+        <seaweedfs.client.version>1.0.9</seaweedfs.client.version>
         <hadoop.version>3.1.1</hadoop.version>
     </properties>
 

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java

@@ -151,35 +151,7 @@ public class SeaweedFileSystemStore {
             LOG.warn("rename non-existing source: {}", source);
             return;
         }
-        LOG.warn("rename moveEntry source: {}", source);
-        moveEntry(source.getParent(), entry, destination);
-    }
-
-    private boolean moveEntry(Path oldParent, FilerProto.Entry entry, Path destination) {
-
-        LOG.debug("moveEntry: {}/{}  => {}", oldParent, entry.getName(), destination);
-
-        FilerProto.Entry.Builder newEntry = entry.toBuilder().setName(destination.getName());
-        boolean isDirectoryCreated = filerClient.createEntry(getParentDirectory(destination), newEntry.build());
-
-        if (!isDirectoryCreated) {
-            return false;
-        }
-
-        if (entry.getIsDirectory()) {
-            Path entryPath = new Path(oldParent, entry.getName());
-            List<FilerProto.Entry> entries = filerClient.listEntries(entryPath.toUri().getPath());
-            for (FilerProto.Entry ent : entries) {
-                boolean isSucess = moveEntry(entryPath, ent, new Path(destination, ent.getName()));
-                if (!isSucess) {
-                    return false;
-                }
-            }
-        }
-
-        return filerClient.deleteEntry(
-            oldParent.toUri().getPath(), entry.getName(), false, false);
-
+        filerClient.mv(source.toUri().getPath(), destination.toUri().getPath());
     }
 
     public OutputStream createFile(final Path path,

unmaintained/see_dat/see_dat.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"time"
 )
 
 var (
@@ -22,11 +23,12 @@ func (scanner *VolumeFileScanner4SeeDat) VisitSuperBlock(superBlock storage.Supe
 
 }
 func (scanner *VolumeFileScanner4SeeDat) ReadNeedleBody() bool {
-	return false
+	return true
 }
 
 func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *storage.Needle, offset int64) error {
-	glog.V(0).Infof("%d,%s%x offset %d size %d cookie %x", *volumeId, n.Id, n.Cookie, offset, n.Size, n.Cookie)
+	t := time.Unix(int64(n.AppendAtNs)/int64(time.Second), int64(n.AppendAtNs)%int64(time.Second))
+	glog.V(0).Infof("%d,%s%x offset %d size %d cookie %x appendedAt %v", *volumeId, n.Id, n.Cookie, offset, n.Size, n.Cookie, t)
 	return nil
 }
 

weed/command/backup.go

@@ -88,7 +88,32 @@ func runBackup(cmd *Command, args []string) bool {
 		return true
 	}
 
-	if err := v.Synchronize(volumeServer, grpcDialOption); err != nil {
+	if v.SuperBlock.CompactRevision < uint16(stats.CompactRevision) {
+		if err = v.Compact(0); err != nil {
+			fmt.Printf("Compact Volume before synchronizing %v\n", err)
+			return true
+		}
+		if err = v.CommitCompact(); err != nil {
+			fmt.Printf("Commit Compact before synchronizing %v\n", err)
+			return true
+		}
+		v.SuperBlock.CompactRevision = uint16(stats.CompactRevision)
+		v.DataFile().WriteAt(v.SuperBlock.Bytes(), 0)
+	}
+
+	if uint64(v.Size()) > stats.TailOffset {
+		// remove the old data
+		v.Destroy()
+		// recreate an empty volume
+		v, err = storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0)
+		if err != nil {
+			fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
+			return true
+		}
+	}
+	defer v.Close()
+
+	if err := v.Follow(volumeServer, grpcDialOption); err != nil {
 		fmt.Printf("Error synchronizing volume %d: %v\n", vid, err)
 		return true
 	}

weed/command/benchmark.go

@@ -36,16 +36,17 @@ type BenchmarkOptions struct {
 	read             *bool
 	sequentialRead   *bool
 	collection       *string
+	replication      *string
 	cpuprofile       *string
 	maxCpu           *int
 	grpcDialOption   grpc.DialOption
+	masterClient     *wdclient.MasterClient
 }
 
 var (
-	b            BenchmarkOptions
-	sharedBytes  []byte
-	masterClient *wdclient.MasterClient
-	isSecure     bool
+	b           BenchmarkOptions
+	sharedBytes []byte
+	isSecure    bool
 )
 
 func init() {
@@ -61,6 +62,7 @@ func init() {
 	b.read = cmdBenchmark.Flag.Bool("read", true, "enable read")
 	b.sequentialRead = cmdBenchmark.Flag.Bool("readSequentially", false, "randomly read by ids from \"-list\" specified file")
 	b.collection = cmdBenchmark.Flag.String("collection", "benchmark", "write data to this collection")
+	b.replication = cmdBenchmark.Flag.String("replication", "000", "replication type")
 	b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file")
 	b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
 	sharedBytes = make([]byte, 1024)
@@ -123,9 +125,9 @@ func runBenchmark(cmd *Command, args []string) bool {
 		defer pprof.StopCPUProfile()
 	}
 
-	masterClient = wdclient.NewMasterClient(context.Background(), b.grpcDialOption, "client", strings.Split(*b.masters, ","))
-	go masterClient.KeepConnectedToMaster()
-	masterClient.WaitUntilConnected()
+	b.masterClient = wdclient.NewMasterClient(context.Background(), b.grpcDialOption, "client", strings.Split(*b.masters, ","))
+	go b.masterClient.KeepConnectedToMaster()
+	b.masterClient.WaitUntilConnected()
 
 	if *b.write {
 		benchWrite()
@@ -206,7 +208,7 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 				}
 				var jwtAuthorization security.EncodedJwt
 				if isSecure {
-					jwtAuthorization = operation.LookupJwt(masterClient.GetMaster(), df.fp.Fid)
+					jwtAuthorization = operation.LookupJwt(b.masterClient.GetMaster(), df.fp.Fid)
 				}
 				if e := util.Delete(fmt.Sprintf("http://%s/%s", df.fp.Server, df.fp.Fid), string(jwtAuthorization)); e == nil {
 					s.completed++
@@ -228,15 +230,16 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 			MimeType: "image/bench", // prevent gzip benchmark content
 		}
 		ar := &operation.VolumeAssignRequest{
-			Count:      1,
-			Collection: *b.collection,
+			Count:       1,
+			Collection:  *b.collection,
+			Replication: *b.replication,
 		}
-		if assignResult, err := operation.Assign(masterClient.GetMaster(), b.grpcDialOption, ar); err == nil {
+		if assignResult, err := operation.Assign(b.masterClient.GetMaster(), b.grpcDialOption, ar); err == nil {
 			fp.Server, fp.Fid, fp.Collection = assignResult.Url, assignResult.Fid, *b.collection
 			if !isSecure && assignResult.Auth != "" {
 				isSecure = true
 			}
-			if _, err := fp.Upload(0, masterClient.GetMaster(), assignResult.Auth, b.grpcDialOption); err == nil {
+			if _, err := fp.Upload(0, b.masterClient.GetMaster(), assignResult.Auth, b.grpcDialOption); err == nil {
 				if random.Intn(100) < *b.deletePercentage {
 					s.total++
 					delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp}
@@ -276,7 +279,7 @@ func readFiles(fileIdLineChan chan string, s *stat) {
 			fmt.Printf("reading file %s\n", fid)
 		}
 		start := time.Now()
-		url, err := masterClient.LookupFileId(fid)
+		url, err := b.masterClient.LookupFileId(fid)
 		if err != nil {
 			s.failed++
 			println("!!!! ", fid, " location not found!!!!!")

weed/command/export.go

@@ -107,7 +107,7 @@ func (scanner *VolumeFileScanner4Export) VisitNeedle(n *storage.Needle, offset i
 	nv, ok := needleMap.Get(n.Id)
 	glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
 		n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped(), ok, nv)
-	if ok && nv.Size > 0 && int64(nv.Offset)*types.NeedlePaddingSize == offset {
+	if ok && nv.Size > 0 && nv.Size != types.TombstoneFileSize && nv.Offset.ToAcutalOffset() == offset {
 		if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
 			glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
 				n.LastModified, newerThanUnix)

weed/command/filer.go

@@ -33,6 +33,7 @@ type FilerOptions struct {
 	dirListingLimit         *int
 	dataCenter              *string
 	enableNotification      *bool
+	disableHttp             *bool
 
 	// default leveldb directory, used in "weed server" mode
 	defaultLevelDbDirectory *string
@@ -52,6 +53,7 @@ func init() {
 	f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit")
 	f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
 	f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to write to volumes in this data center")
+	f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed")
 }
 
 var cmdFiler = &Command{
@@ -99,7 +101,7 @@ func (fo *FilerOptions) startFiler() {
 	}
 
 	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
-		Masters:            strings.Split(*f.masters, ","),
+		Masters:            strings.Split(*fo.masters, ","),
 		Collection:         *fo.collection,
 		DefaultReplication: *fo.defaultReplicaPlacement,
 		RedirectOnRead:     *fo.redirectOnRead,
@@ -108,6 +110,7 @@ func (fo *FilerOptions) startFiler() {
 		DirListingLimit:    *fo.dirListingLimit,
 		DataCenter:         *fo.dataCenter,
 		DefaultLevelDbDir:  defaultLevelDbDirectory,
+		DisableHttp:        *fo.disableHttp,
 	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)

weed/command/filer_copy.go

@@ -1,29 +1,31 @@
 package command
 
 import (
+	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/wdclient"
 	"github.com/spf13/viper"
 	"google.golang.org/grpc"
+	"io"
 	"io/ioutil"
+	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
-	"strings"
-
-	"context"
-	"github.com/chrislusf/seaweedfs/weed/operation"
-	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/util"
-	"io"
-	"net/http"
 	"strconv"
+	"strings"
+	"sync"
 	"time"
 )
 
 var (
-	copy CopyOptions
+	copy      CopyOptions
+	waitGroup sync.WaitGroup
 )
 
 type CopyOptions struct {
@@ -35,6 +37,8 @@ type CopyOptions struct {
 	ttl            *string
 	maxMB          *int
 	grpcDialOption grpc.DialOption
+	masterClient   *wdclient.MasterClient
+	concurrency    *int
 }
 
 func init() {
@@ -45,8 +49,9 @@ func init() {
 	copy.replication = cmdCopy.Flag.String("replication", "", "replication type")
 	copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name")
 	copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
-	copy.maxMB = cmdCopy.Flag.Int("maxMB", 0, "split files larger than the limit")
+	copy.maxMB = cmdCopy.Flag.Int("maxMB", 32, "split files larger than the limit")
 	copy.filerGrpcPort = cmdCopy.Flag.Int("filer.port.grpc", 0, "filer grpc server listen port, default to filer port + 10000")
+	copy.concurrency = cmdCopy.Flag.Int("c", 8, "concurrent file copy goroutines")
 }
 
 var cmdCopy = &Command{
@@ -105,61 +110,139 @@ func runCopy(cmd *Command, args []string) bool {
 	filerGrpcAddress := fmt.Sprintf("%s:%d", filerUrl.Hostname(), filerGrpcPort)
 	copy.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
 
-	for _, fileOrDir := range fileOrDirs {
-		if !doEachCopy(fileOrDir, filerUrl.Host, filerGrpcAddress, copy.grpcDialOption, urlPath) {
-			return false
-		}
+	copy.masterClient = wdclient.NewMasterClient(context.Background(), copy.grpcDialOption, "client", strings.Split(*copy.master, ","))
+	go copy.masterClient.KeepConnectedToMaster()
+	copy.masterClient.WaitUntilConnected()
+
+	if *cmdCopy.IsDebug {
+		util.SetupProfiling("filer.copy.cpu.pprof", "filer.copy.mem.pprof")
 	}
+
+	fileCopyTaskChan := make(chan FileCopyTask, *copy.concurrency)
+
+	ctx := context.Background()
+
+	go func() {
+		defer close(fileCopyTaskChan)
+		for _, fileOrDir := range fileOrDirs {
+			if err := genFileCopyTask(fileOrDir, urlPath, fileCopyTaskChan); err != nil {
+				fmt.Fprintf(os.Stderr, "gen file list error: %v\n", err)
+				break
+			}
+		}
+	}()
+	for i := 0; i < *copy.concurrency; i++ {
+		waitGroup.Add(1)
+		go func() {
+			defer waitGroup.Done()
+			worker := FileCopyWorker{
+				options:          &copy,
+				filerHost:        filerUrl.Host,
+				filerGrpcAddress: filerGrpcAddress,
+			}
+			if err := worker.copyFiles(ctx, fileCopyTaskChan); err != nil {
+				fmt.Fprintf(os.Stderr, "copy file error: %v\n", err)
+				return
+			}
+		}()
+	}
+	waitGroup.Wait()
+
 	return true
 }
 
-func doEachCopy(fileOrDir string, filerAddress, filerGrpcAddress string, grpcDialOption grpc.DialOption, path string) bool {
-	f, err := os.Open(fileOrDir)
-	if err != nil {
-		fmt.Printf("Failed to open file %s: %v\n", fileOrDir, err)
-		return false
-	}
-	defer f.Close()
+func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan FileCopyTask) error {
 
-	fi, err := f.Stat()
+	fi, err := os.Stat(fileOrDir)
 	if err != nil {
-		fmt.Printf("Failed to get stat for file %s: %v\n", fileOrDir, err)
-		return false
+		fmt.Fprintf(os.Stderr, "Failed to get stat for file %s: %v\n", fileOrDir, err)
+		return nil
 	}
 
 	mode := fi.Mode()
 	if mode.IsDir() {
 		files, _ := ioutil.ReadDir(fileOrDir)
 		for _, subFileOrDir := range files {
-			if !doEachCopy(fileOrDir+"/"+subFileOrDir.Name(), filerAddress, filerGrpcAddress, grpcDialOption, path+fi.Name()+"/") {
-				return false
+			if err = genFileCopyTask(fileOrDir+"/"+subFileOrDir.Name(), destPath+fi.Name()+"/", fileCopyTaskChan); err != nil {
+				return err
 			}
 		}
-		return true
+		return nil
 	}
 
+	uid, gid := util.GetFileUidGid(fi)
+
+	fileCopyTaskChan <- FileCopyTask{
+		sourceLocation:     fileOrDir,
+		destinationUrlPath: destPath,
+		fileSize:           fi.Size(),
+		fileMode:           fi.Mode(),
+		uid:                uid,
+		gid:                gid,
+	}
+
+	return nil
+}
+
+type FileCopyWorker struct {
+	options          *CopyOptions
+	filerHost        string
+	filerGrpcAddress string
+}
+
+func (worker *FileCopyWorker) copyFiles(ctx context.Context, fileCopyTaskChan chan FileCopyTask) error {
+	for task := range fileCopyTaskChan {
+		if err := worker.doEachCopy(ctx, task); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type FileCopyTask struct {
+	sourceLocation     string
+	destinationUrlPath string
+	fileSize           int64
+	fileMode           os.FileMode
+	uid                uint32
+	gid                uint32
+}
+
+func (worker *FileCopyWorker) doEachCopy(ctx context.Context, task FileCopyTask) error {
+
+	f, err := os.Open(task.sourceLocation)
+	if err != nil {
+		fmt.Printf("Failed to open file %s: %v\n", task.sourceLocation, err)
+		if _, ok := err.(*os.PathError); ok {
+			fmt.Printf("skipping %s\n", task.sourceLocation)
+			return nil
+		}
+		return err
+	}
+	defer f.Close()
+
 	// this is a regular file
-	if *copy.include != "" {
-		if ok, _ := filepath.Match(*copy.include, filepath.Base(fileOrDir)); !ok {
-			return true
+	if *worker.options.include != "" {
+		if ok, _ := filepath.Match(*worker.options.include, filepath.Base(task.sourceLocation)); !ok {
+			return nil
 		}
 	}
 
 	// find the chunk count
-	chunkSize := int64(*copy.maxMB * 1024 * 1024)
+	chunkSize := int64(*worker.options.maxMB * 1024 * 1024)
 	chunkCount := 1
-	if chunkSize > 0 && fi.Size() > chunkSize {
-		chunkCount = int(fi.Size()/chunkSize) + 1
+	if chunkSize > 0 && task.fileSize > chunkSize {
+		chunkCount = int(task.fileSize/chunkSize) + 1
 	}
 
 	if chunkCount == 1 {
-		return uploadFileAsOne(filerAddress, filerGrpcAddress, grpcDialOption, path, f, fi)
+		return worker.uploadFileAsOne(ctx, task, f)
 	}
 
-	return uploadFileInChunks(filerAddress, filerGrpcAddress, grpcDialOption, path, f, fi, chunkCount, chunkSize)
+	return worker.uploadFileInChunks(ctx, task, f, chunkCount, chunkSize)
 }
 
-func uploadFileAsOne(filerAddress, filerGrpcAddress string, grpcDialOption grpc.DialOption, urlFolder string, f *os.File, fi os.FileInfo) bool {
+func (worker *FileCopyWorker) uploadFileAsOne(ctx context.Context, task FileCopyTask, f *os.File) error {
 
 	// upload the file content
 	fileName := filepath.Base(f.Name())
@@ -167,29 +250,27 @@ func uploadFileAsOne(filerAddress, filerGrpcAddress string, grpcDialOption grpc.
 
 	var chunks []*filer_pb.FileChunk
 
-	if fi.Size() > 0 {
+	if task.fileSize > 0 {
 
 		// assign a volume
-		assignResult, err := operation.Assign(*copy.master, grpcDialOption, &operation.VolumeAssignRequest{
+		assignResult, err := operation.Assign(worker.options.masterClient.GetMaster(), worker.options.grpcDialOption, &operation.VolumeAssignRequest{
 			Count:       1,
-			Replication: *copy.replication,
-			Collection:  *copy.collection,
-			Ttl:         *copy.ttl,
+			Replication: *worker.options.replication,
+			Collection:  *worker.options.collection,
+			Ttl:         *worker.options.ttl,
 		})
 		if err != nil {
-			fmt.Printf("Failed to assign from %s: %v\n", *copy.master, err)
+			fmt.Printf("Failed to assign from %s: %v\n", *worker.options.master, err)
 		}
 
 		targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
 
 		uploadResult, err := operation.Upload(targetUrl, fileName, f, false, mimeType, nil, assignResult.Auth)
 		if err != nil {
-			fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err)
-			return false
+			return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
 		}
 		if uploadResult.Error != "" {
-			fmt.Printf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
-			return false
+			return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
 		}
 		fmt.Printf("uploaded %s to %s\n", fileName, targetUrl)
 
@@ -201,43 +282,42 @@ func uploadFileAsOne(filerAddress, filerGrpcAddress string, grpcDialOption grpc.
 			ETag:   uploadResult.ETag,
 		})
 
-		fmt.Printf("copied %s => http://%s%s%s\n", fileName, filerAddress, urlFolder, fileName)
+		fmt.Printf("copied %s => http://%s%s%s\n", fileName, worker.filerHost, task.destinationUrlPath, fileName)
 	}
 
-	if err := withFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if err := withFilerClient(ctx, worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
-			Directory: urlFolder,
+			Directory: task.destinationUrlPath,
 			Entry: &filer_pb.Entry{
 				Name: fileName,
 				Attributes: &filer_pb.FuseAttributes{
 					Crtime:      time.Now().Unix(),
 					Mtime:       time.Now().Unix(),
-					Gid:         uint32(os.Getgid()),
-					Uid:         uint32(os.Getuid()),
-					FileSize:    uint64(fi.Size()),
-					FileMode:    uint32(fi.Mode()),
+					Gid:         task.gid,
+					Uid:         task.uid,
+					FileSize:    uint64(task.fileSize),
+					FileMode:    uint32(task.fileMode),
 					Mime:        mimeType,
-					Replication: *copy.replication,
-					Collection:  *copy.collection,
-					TtlSec:      int32(util.ParseInt(*copy.ttl, 0)),
+					Replication: *worker.options.replication,
+					Collection:  *worker.options.collection,
+					TtlSec:      int32(util.ParseInt(*worker.options.ttl, 0)),
 				},
 				Chunks: chunks,
 			},
 		}
 
-		if _, err := client.CreateEntry(context.Background(), request); err != nil {
+		if _, err := client.CreateEntry(ctx, request); err != nil {
 			return fmt.Errorf("update fh: %v", err)
 		}
 		return nil
 	}); err != nil {
-		fmt.Printf("upload data %v to http://%s%s%s: %v\n", fileName, filerAddress, urlFolder, fileName, err)
-		return false
+		return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerHost, task.destinationUrlPath, fileName, err)
 	}
 
-	return true
+	return nil
 }
 
-func uploadFileInChunks(filerAddress, filerGrpcAddress string, grpcDialOption grpc.DialOption, urlFolder string, f *os.File, fi os.FileInfo, chunkCount int, chunkSize int64) bool {
+func (worker *FileCopyWorker) uploadFileInChunks(ctx context.Context, task FileCopyTask, f *os.File, chunkCount int, chunkSize int64) error {
 
 	fileName := filepath.Base(f.Name())
 	mimeType := detectMimeType(f)
@@ -247,14 +327,14 @@ func uploadFileInChunks(filerAddress, filerGrpcAddress string, grpcDialOption gr
 	for i := int64(0); i < int64(chunkCount); i++ {
 
 		// assign a volume
-		assignResult, err := operation.Assign(*copy.master, grpcDialOption, &operation.VolumeAssignRequest{
+		assignResult, err := operation.Assign(worker.options.masterClient.GetMaster(), worker.options.grpcDialOption, &operation.VolumeAssignRequest{
 			Count:       1,
-			Replication: *copy.replication,
-			Collection:  *copy.collection,
-			Ttl:         *copy.ttl,
+			Replication: *worker.options.replication,
+			Collection:  *worker.options.collection,
+			Ttl:         *worker.options.ttl,
 		})
 		if err != nil {
-			fmt.Printf("Failed to assign from %s: %v\n", *copy.master, err)
+			fmt.Printf("Failed to assign from %s: %v\n", *worker.options.master, err)
 		}
 
 		targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
@@ -264,12 +344,10 @@ func uploadFileInChunks(filerAddress, filerGrpcAddress string, grpcDialOption gr
 			io.LimitReader(f, chunkSize),
 			false, "application/octet-stream", nil, assignResult.Auth)
 		if err != nil {
-			fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err)
-			return false
+			return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
 		}
 		if uploadResult.Error != "" {
-			fmt.Printf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
-			return false
+			return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
 		}
 		chunks = append(chunks, &filer_pb.FileChunk{
 			FileId: assignResult.Fid,
@@ -281,39 +359,38 @@ func uploadFileInChunks(filerAddress, filerGrpcAddress string, grpcDialOption gr
 		fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
 	}
 
-	if err := withFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if err := withFilerClient(ctx, worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
-			Directory: urlFolder,
+			Directory: task.destinationUrlPath,
 			Entry: &filer_pb.Entry{
 				Name: fileName,
 				Attributes: &filer_pb.FuseAttributes{
 					Crtime:      time.Now().Unix(),
 					Mtime:       time.Now().Unix(),
-					Gid:         uint32(os.Getgid()),
-					Uid:         uint32(os.Getuid()),
-					FileSize:    uint64(fi.Size()),
-					FileMode:    uint32(fi.Mode()),
+					Gid:         task.gid,
+					Uid:         task.uid,
+					FileSize:    uint64(task.fileSize),
+					FileMode:    uint32(task.fileMode),
 					Mime:        mimeType,
-					Replication: *copy.replication,
-					Collection:  *copy.collection,
-					TtlSec:      int32(util.ParseInt(*copy.ttl, 0)),
+					Replication: *worker.options.replication,
+					Collection:  *worker.options.collection,
+					TtlSec:      int32(util.ParseInt(*worker.options.ttl, 0)),
 				},
 				Chunks: chunks,
 			},
 		}
 
-		if _, err := client.CreateEntry(context.Background(), request); err != nil {
+		if _, err := client.CreateEntry(ctx, request); err != nil {
 			return fmt.Errorf("update fh: %v", err)
 		}
 		return nil
 	}); err != nil {
-		fmt.Printf("upload data %v to http://%s%s%s: %v\n", fileName, filerAddress, urlFolder, fileName, err)
-		return false
+		return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerHost, task.destinationUrlPath, fileName, err)
 	}
 
-	fmt.Printf("copied %s => http://%s%s%s\n", fileName, filerAddress, urlFolder, fileName)
+	fmt.Printf("copied %s => http://%s%s%s\n", fileName, worker.filerHost, task.destinationUrlPath, fileName)
 
-	return true
+	return nil
 }
 
 func detectMimeType(f *os.File) string {
@@ -332,15 +409,11 @@ func detectMimeType(f *os.File) string {
 	return mimeType
 }
 
-func withFilerClient(filerAddress string, grpcDialOption grpc.DialOption, fn func(filer_pb.SeaweedFilerClient) error) error {
+func withFilerClient(ctx context.Context, filerAddress string, grpcDialOption grpc.DialOption, fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	grpcConnection, err := util.GrpcDial(filerAddress, grpcDialOption)
-	if err != nil {
-		return fmt.Errorf("fail to dial %s: %v", filerAddress, err)
-	}
-	defer grpcConnection.Close()
+	return util.WithCachedGrpcClient(ctx, func(clientConn *grpc.ClientConn) error {
+		client := filer_pb.NewSeaweedFilerClient(clientConn)
+		return fn(client)
+	}, filerAddress, grpcDialOption)
 
-	client := filer_pb.NewSeaweedFilerClient(grpcConnection)
-
-	return fn(client)
 }

weed/command/filer_export.go

@@ -1,6 +1,7 @@
 package command
 
 import (
+	"context"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/notification"
@@ -96,6 +97,8 @@ func runFilerExport(cmd *Command, args []string) bool {
 		return false
 	}
 
+	ctx := context.Background()
+
 	stat := statistics{}
 
 	var fn func(level int, entry *filer2.Entry) error
@@ -125,23 +128,23 @@ func runFilerExport(cmd *Command, args []string) bool {
 			if *dryRun {
 				return nil
 			}
-			return targetStore.InsertEntry(entry)
+			return targetStore.InsertEntry(ctx, entry)
 		}
 	}
 
-	doTraverse(&stat, sourceStore, filer2.FullPath(*dir), 0, fn)
+	doTraverse(ctx, &stat, sourceStore, filer2.FullPath(*dir), 0, fn)
 
 	glog.Infof("processed %d directories, %d files", stat.directoryCount, stat.fileCount)
 
 	return true
 }
 
-func doTraverse(stat *statistics, filerStore filer2.FilerStore, parentPath filer2.FullPath, level int, fn func(level int, entry *filer2.Entry) error) {
+func doTraverse(ctx context.Context, stat *statistics, filerStore filer2.FilerStore, parentPath filer2.FullPath, level int, fn func(level int, entry *filer2.Entry) error) {
 
 	limit := *dirListLimit
 	lastEntryName := ""
 	for {
-		entries, err := filerStore.ListDirectoryEntries(parentPath, lastEntryName, false, limit)
+		entries, err := filerStore.ListDirectoryEntries(ctx, parentPath, lastEntryName, false, limit)
 		if err != nil {
 			break
 		}
@@ -151,10 +154,11 @@ func doTraverse(stat *statistics, filerStore filer2.FilerStore, parentPath filer
 			}
 			if entry.IsDirectory() {
 				stat.directoryCount++
-				doTraverse(stat, filerStore, entry.FullPath, level+1, fn)
+				doTraverse(ctx, stat, filerStore, entry.FullPath, level+1, fn)
 			} else {
 				stat.fileCount++
 			}
+			lastEntryName = entry.Name()
 		}
 		if len(entries) < limit {
 			break

weed/command/filer_replication.go

@@ -1,6 +1,7 @@
 package command
 
 import (
+	"context"
 	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -116,7 +117,7 @@ func runFilerReplicate(cmd *Command, args []string) bool {
 		} else {
 			glog.V(1).Infof("modify: %s", key)
 		}
-		if err = replicator.Replicate(key, m); err != nil {
+		if err = replicator.Replicate(context.Background(), key, m); err != nil {
 			glog.Errorf("replicate %s: %+v", key, err)
 		} else {
 			glog.V(1).Infof("replicated %s", key)

weed/command/fix.go

@@ -44,12 +44,12 @@ func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool {
 
 func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *storage.Needle, offset int64) error {
 	glog.V(2).Infof("key %d offset %d size %d disk_size %d gzip %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped())
-	if n.Size > 0 {
-		pe := scanner.nm.Put(n.Id, types.Offset(offset/types.NeedlePaddingSize), n.Size)
+	if n.Size > 0 && n.Size != types.TombstoneFileSize {
+		pe := scanner.nm.Put(n.Id, types.ToOffset(offset), n.Size)
 		glog.V(2).Infof("saved %d with error %v", n.Size, pe)
 	} else {
 		glog.V(2).Infof("skipping deleted file ...")
-		return scanner.nm.Delete(n.Id, types.Offset(offset/types.NeedlePaddingSize))
+		return scanner.nm.Delete(n.Id, types.ToOffset(offset))
 	}
 	return nil
 }

weed/command/master.go

@@ -47,6 +47,7 @@ var (
 	mMaxCpu               = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
 	garbageThreshold      = cmdMaster.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	masterWhiteListOption = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
+	disableHttp           = cmdMaster.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
 	masterCpuProfile      = cmdMaster.Flag.String("cpuprofile", "", "cpu profile output file")
 	masterMemProfile      = cmdMaster.Flag.String("memprofile", "", "memory profile output file")
 
@@ -69,7 +70,7 @@ func runMaster(cmd *Command, args []string) bool {
 	if *masterWhiteListOption != "" {
 		masterWhiteList = strings.Split(*masterWhiteListOption, ",")
 	}
-	if *volumeSizeLimitMB > 30*1000 {
+	if *volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
 		glog.Fatalf("volumeSizeLimitMB should be smaller than 30000")
 	}
 
@@ -78,6 +79,7 @@ func runMaster(cmd *Command, args []string) bool {
 		*volumeSizeLimitMB, *volumePreallocate,
 		*mpulse, *defaultReplicaPlacement, *garbageThreshold,
 		masterWhiteList,
+		*disableHttp,
 	)
 
 	listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport)
@@ -95,6 +97,7 @@ func runMaster(cmd *Command, args []string) bool {
 		raftServer := weed_server.NewRaftServer(security.LoadClientTLS(viper.Sub("grpc"), "master"),
 			peers, myMasterAddress, *metaFolder, ms.Topo, *mpulse)
 		ms.SetRaftServer(raftServer)
+		r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
 
 		// starting grpc server
 		grpcPort := *mport + 10000

weed/command/mount.go

@@ -8,7 +8,6 @@ import (
 
 type MountOptions struct {
 	filer              *string
-	filerGrpcPort      *int
 	filerMountRootPath *string
 	dir                *string
 	dirListingLimit    *int
@@ -17,6 +16,7 @@ type MountOptions struct {
 	ttlSec             *int
 	chunkSizeLimitMB   *int
 	dataCenter         *string
+	allowOthers        *bool
 }
 
 var (
@@ -28,7 +28,6 @@ var (
 func init() {
 	cmdMount.Run = runMount // break init cycle
 	mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "weed filer location")
-	mountOptions.filerGrpcPort = cmdMount.Flag.Int("filer.grpc.port", 0, "filer grpc server listen port, default to http port + 10000")
 	mountOptions.filerMountRootPath = cmdMount.Flag.String("filer.path", "/", "mount this remote path from filer server")
 	mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory")
 	mountOptions.dirListingLimit = cmdMount.Flag.Int("dirListLimit", 100000, "limit directory listing size")
@@ -37,6 +36,7 @@ func init() {
 	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
 	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 4, "local write buffer size, also chunk large files")
 	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
+	mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
 	mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
 	mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
 }
@@ -59,7 +59,7 @@ var cmdMount = &Command{
   `,
 }
 
-func parseFilerGrpcAddress(filer string, optionalGrpcPort int) (filerGrpcAddress string, err error) {
+func parseFilerGrpcAddress(filer string) (filerGrpcAddress string, err error) {
 	hostnameAndPort := strings.Split(filer, ":")
 	if len(hostnameAndPort) != 2 {
 		return "", fmt.Errorf("The filer should have hostname:port format: %v", hostnameAndPort)
@@ -71,9 +71,6 @@ func parseFilerGrpcAddress(filer string, optionalGrpcPort int) (filerGrpcAddress
 	}
 
 	filerGrpcPort := int(filerPort) + 10000
-	if optionalGrpcPort != 0 {
-		filerGrpcPort = optionalGrpcPort
-	}
 
 	return fmt.Sprintf("%s:%d", hostnameAndPort[0], filerGrpcPort), nil
 }

weed/command/mount_std.go

@@ -56,8 +56,7 @@ func runMount(cmd *Command, args []string) bool {
 
 	util.SetupProfiling(*mountCpuProfile, *mountMemProfile)
 
-	c, err := fuse.Mount(
-		*mountOptions.dir,
+	options := []fuse.MountOption{
 		fuse.VolumeName("SeaweedFS"),
 		fuse.FSName("SeaweedFS"),
 		fuse.Subtype("SeaweedFS"),
@@ -67,13 +66,17 @@ func runMount(cmd *Command, args []string) bool {
 		fuse.AutoXattr(),
 		fuse.ExclCreate(),
 		fuse.DaemonTimeout("3600"),
-		fuse.AllowOther(),
 		fuse.AllowSUID(),
 		fuse.DefaultPermissions(),
-		fuse.MaxReadahead(1024*128),
+		fuse.MaxReadahead(1024 * 128),
 		fuse.AsyncRead(),
 		fuse.WritebackCache(),
-	)
+	}
+	if *mountOptions.allowOthers {
+		options = append(options, fuse.AllowOther())
+	}
+
+	c, err := fuse.Mount(*mountOptions.dir, options...)
 	if err != nil {
 		glog.Fatal(err)
 		return false
@@ -84,7 +87,7 @@ func runMount(cmd *Command, args []string) bool {
 		c.Close()
 	})
 
-	filerGrpcAddress, err := parseFilerGrpcAddress(*mountOptions.filer, *mountOptions.filerGrpcPort)
+	filerGrpcAddress, err := parseFilerGrpcAddress(*mountOptions.filer)
 	if err != nil {
 		glog.Fatal(err)
 		return false

weed/command/s3.go

@@ -51,7 +51,7 @@ func runS3(cmd *Command, args []string) bool {
 
 	weed_server.LoadConfiguration("security", false)
 
-	filerGrpcAddress, err := parseFilerGrpcAddress(*s3options.filer, *s3options.filerGrpcPort)
+	filerGrpcAddress, err := parseFilerGrpcAddress(*s3options.filer)
 	if err != nil {
 		glog.Fatal(err)
 		return false

weed/command/scaffold.go

@@ -72,9 +72,9 @@ dir = "."					# directory to store level db files
 
 [mysql]
 # CREATE TABLE IF NOT EXISTS filemeta (
-#   dirhash     BIGINT        COMMENT 'first 64 bits of MD5 hash value of directory field',
-#   name        VARCHAR(1000) COMMENT 'directory or file name',
-#   directory   VARCHAR(4096) COMMENT 'full path to parent directory',
+#   dirhash     BIGINT         COMMENT 'first 64 bits of MD5 hash value of directory field',
+#   name        VARCHAR(65535) COMMENT 'directory or file name',
+#   directory   VARCHAR(65535) COMMENT 'full path to parent directory',
 #   meta        BLOB,
 #   PRIMARY KEY (dirhash, name)
 # ) DEFAULT CHARSET=utf8;
@@ -90,8 +90,8 @@ connection_max_open = 100
 [postgres]
 # CREATE TABLE IF NOT EXISTS filemeta (
 #   dirhash     BIGINT,
-#   name        VARCHAR(1000),
-#   directory   VARCHAR(4096),
+#   name        VARCHAR(65535),
+#   directory   VARCHAR(65535),
 #   meta        bytea,
 #   PRIMARY KEY (dirhash, name)
 # );
@@ -180,6 +180,17 @@ google_application_credentials = "/path/to/x.json" # path to json credential fil
 project_id = ""                       # an existing project id
 topic = "seaweedfs_filer_topic"       # a topic, auto created if does not exists
 
+[notification.gocdk_pub_sub]
+# The Go Cloud Development Kit (https://gocloud.dev).
+# PubSub API (https://godoc.org/gocloud.dev/pubsub).
+# Supports AWS SNS/SQS, Azure Service Bus, Google PubSub, NATS and RabbitMQ.
+enabled = false
+# This URL will Dial the RabbitMQ server at the URL in the environment
+# variable RABBIT_SERVER_URL and open the exchange "myexchange".
+# The exchange must have already been created by some other means, like
+# the RabbitMQ management plugin.
+topic_url = "rabbit://myexchange"
+sub_url = "rabbit://myqueue"
 `
 
 	REPLICATION_TOML_EXAMPLE = `

weed/command/server.go

@@ -60,6 +60,7 @@ var (
 	serverDataCenter              = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
 	serverRack                    = cmdServer.Flag.String("rack", "", "current volume server's rack name")
 	serverWhiteListOption         = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
+	serverDisableHttp             = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
 	serverPeers                   = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
 	serverGarbageThreshold        = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	masterPort                    = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
@@ -114,6 +115,7 @@ func runServer(cmd *Command, args []string) bool {
 	}
 
 	master := *serverIp + ":" + strconv.Itoa(*masterPort)
+	filerOptions.masters = &master
 	filerOptions.ip = serverIp
 	serverOptions.v.ip = serverIp
 	serverOptions.v.bindIp = serverBindIp
@@ -125,6 +127,7 @@ func runServer(cmd *Command, args []string) bool {
 	serverOptions.v.pulseSeconds = pulseSeconds
 
 	filerOptions.dataCenter = serverDataCenter
+	filerOptions.disableHttp = serverDisableHttp
 
 	if *filerOptions.defaultReplicaPlacement == "" {
 		*filerOptions.defaultReplicaPlacement = *masterDefaultReplicaPlacement
@@ -137,7 +140,7 @@ func runServer(cmd *Command, args []string) bool {
 
 	folders := strings.Split(*volumeDataFolders, ",")
 
-	if *masterVolumeSizeLimitMB > 30*1000 {
+	if *masterVolumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
 		glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
 	}
 
@@ -171,7 +174,7 @@ func runServer(cmd *Command, args []string) bool {
 		ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
 			*masterVolumeSizeLimitMB, *masterVolumePreallocate,
 			*pulseSeconds, *masterDefaultReplicaPlacement, *serverGarbageThreshold,
-			serverWhiteList,
+			serverWhiteList, *serverDisableHttp,
 		)
 
 		glog.V(0).Infof("Start Seaweed Master %s at %s:%d", util.VERSION, *serverIp, *masterPort)
@@ -182,10 +185,11 @@ func runServer(cmd *Command, args []string) bool {
 
 		go func() {
 			// start raftServer
-			myMasterAddress, peers := checkPeers(*masterIp, *mport, *masterPeers)
+			myMasterAddress, peers := checkPeers(*serverIp, *masterPort, *serverPeers)
 			raftServer := weed_server.NewRaftServer(security.LoadClientTLS(viper.Sub("grpc"), "master"),
-				peers, myMasterAddress, *metaFolder, ms.Topo, *mpulse)
+				peers, myMasterAddress, *masterMetaFolder, ms.Topo, *pulseSeconds)
 			ms.SetRaftServer(raftServer)
+			r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
 
 			// starting grpc server
 			grpcPort := *masterPort + 10000

weed/command/shell.go

@@ -1,21 +1,25 @@
 package command
 
 import (
-	"bufio"
-	"fmt"
-	"os"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/shell"
+	"github.com/spf13/viper"
+)
 
-	"github.com/chrislusf/seaweedfs/weed/glog"
+var (
+	shellOptions shell.ShellOptions
 )
 
 func init() {
 	cmdShell.Run = runShell // break init cycle
+	shellOptions.Masters = cmdShell.Flag.String("master", "localhost:9333", "comma-separated master servers")
 }
 
 var cmdShell = &Command{
 	UsageLine: "shell",
-	Short:     "run interactive commands, now just echo",
-	Long: `run interactive commands.
+	Short:     "run interactive administrative commands",
+	Long: `run interactive administrative commands.
 
   `,
 }
@@ -23,39 +27,16 @@ var cmdShell = &Command{
 var ()
 
 func runShell(command *Command, args []string) bool {
-	r := bufio.NewReader(os.Stdin)
-	o := bufio.NewWriter(os.Stdout)
-	e := bufio.NewWriter(os.Stderr)
-	prompt := func() {
-		var err error
-		if _, err = o.WriteString("> "); err != nil {
-			glog.V(0).Infoln("error writing to stdout:", err)
-		}
-		if err = o.Flush(); err != nil {
-			glog.V(0).Infoln("error flushing stdout:", err)
-		}
-	}
-	readLine := func() string {
-		ret, err := r.ReadString('\n')
-		if err != nil {
-			fmt.Fprint(e, err)
-			os.Exit(1)
-		}
-		return ret
-	}
-	execCmd := func(cmd string) int {
-		if cmd != "" {
-			if _, err := o.WriteString(cmd); err != nil {
-				glog.V(0).Infoln("error writing to stdout:", err)
-			}
-		}
-		return 0
-	}
 
-	cmd := ""
-	for {
-		prompt()
-		cmd = readLine()
-		execCmd(cmd)
-	}
+	weed_server.LoadConfiguration("security", false)
+	shellOptions.GrpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
+
+	shellOptions.FilerHost = "localhost"
+	shellOptions.FilerPort = 8888
+	shellOptions.Directory = "/"
+
+	shell.RunShell(shellOptions)
+
+	return true
+
 }

weed/command/upload.go

@@ -37,7 +37,7 @@ func init() {
 	upload.collection = cmdUpload.Flag.String("collection", "", "optional collection name")
 	upload.dataCenter = cmdUpload.Flag.String("dataCenter", "", "optional data center name")
 	upload.ttl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
-	upload.maxMB = cmdUpload.Flag.Int("maxMB", 0, "split files larger than the limit")
+	upload.maxMB = cmdUpload.Flag.Int("maxMB", 32, "split files larger than the limit")
 }
 
 var cmdUpload = &Command{

weed/filer2/abstract_sql/abstract_sql_store.go

@@ -1,6 +1,7 @@
 package abstract_sql
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 
@@ -18,7 +19,44 @@ type AbstractSqlStore struct {
 	SqlListInclusive string
 }
 
-func (store *AbstractSqlStore) InsertEntry(entry *filer2.Entry) (err error) {
+type TxOrDB interface {
+	ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error)
+	QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row
+	QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error)
+}
+
+func (store *AbstractSqlStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	tx, err := store.DB.BeginTx(ctx, &sql.TxOptions{
+		Isolation: sql.LevelReadCommitted,
+		ReadOnly:  false,
+	})
+	if err != nil {
+		return ctx, err
+	}
+
+	return context.WithValue(ctx, "tx", tx), nil
+}
+func (store *AbstractSqlStore) CommitTransaction(ctx context.Context) error {
+	if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
+		return tx.Commit()
+	}
+	return nil
+}
+func (store *AbstractSqlStore) RollbackTransaction(ctx context.Context) error {
+	if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
+		return tx.Rollback()
+	}
+	return nil
+}
+
+func (store *AbstractSqlStore) getTxOrDB(ctx context.Context) TxOrDB {
+	if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
+		return tx
+	}
+	return store.DB
+}
+
+func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
 
 	dir, name := entry.FullPath.DirAndName()
 	meta, err := entry.EncodeAttributesAndChunks()
@@ -26,7 +64,7 @@ func (store *AbstractSqlStore) InsertEntry(entry *filer2.Entry) (err error) {
 		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
 	}
 
-	res, err := store.DB.Exec(store.SqlInsert, hashToLong(dir), name, dir, meta)
+	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlInsert, hashToLong(dir), name, dir, meta)
 	if err != nil {
 		return fmt.Errorf("insert %s: %s", entry.FullPath, err)
 	}
@@ -38,7 +76,7 @@ func (store *AbstractSqlStore) InsertEntry(entry *filer2.Entry) (err error) {
 	return nil
 }
 
-func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) {
+func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
 
 	dir, name := entry.FullPath.DirAndName()
 	meta, err := entry.EncodeAttributesAndChunks()
@@ -46,7 +84,7 @@ func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) {
 		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
 	}
 
-	res, err := store.DB.Exec(store.SqlUpdate, meta, hashToLong(dir), name, dir)
+	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlUpdate, meta, hashToLong(dir), name, dir)
 	if err != nil {
 		return fmt.Errorf("update %s: %s", entry.FullPath, err)
 	}
@@ -58,10 +96,10 @@ func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) {
 	return nil
 }
 
-func (store *AbstractSqlStore) FindEntry(fullpath filer2.FullPath) (*filer2.Entry, error) {
+func (store *AbstractSqlStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (*filer2.Entry, error) {
 
 	dir, name := fullpath.DirAndName()
-	row := store.DB.QueryRow(store.SqlFind, hashToLong(dir), name, dir)
+	row := store.getTxOrDB(ctx).QueryRowContext(ctx, store.SqlFind, hashToLong(dir), name, dir)
 	var data []byte
 	if err := row.Scan(&data); err != nil {
 		return nil, filer2.ErrNotFound
@@ -77,11 +115,11 @@ func (store *AbstractSqlStore) FindEntry(fullpath filer2.FullPath) (*filer2.Entr
 	return entry, nil
 }
 
-func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) error {
+func (store *AbstractSqlStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) error {
 
 	dir, name := fullpath.DirAndName()
 
-	res, err := store.DB.Exec(store.SqlDelete, hashToLong(dir), name, dir)
+	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDelete, hashToLong(dir), name, dir)
 	if err != nil {
 		return fmt.Errorf("delete %s: %s", fullpath, err)
 	}
@@ -94,14 +132,14 @@ func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) error {
 	return nil
 }
 
-func (store *AbstractSqlStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) {
+func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) {
 
 	sqlText := store.SqlListExclusive
 	if inclusive {
 		sqlText = store.SqlListInclusive
 	}
 
-	rows, err := store.DB.Query(sqlText, hashToLong(string(fullpath)), startFileName, string(fullpath), limit)
+	rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, hashToLong(string(fullpath)), startFileName, string(fullpath), limit)
 	if err != nil {
 		return nil, fmt.Errorf("list %s : %v", fullpath, err)
 	}

weed/filer2/cassandra/cassandra_store.go

@@ -1,6 +1,7 @@
 package cassandra
 
 import (
+	"context"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -39,7 +40,17 @@ func (store *CassandraStore) initialize(keyspace string, hosts []string) (err er
 	return
 }
 
-func (store *CassandraStore) InsertEntry(entry *filer2.Entry) (err error) {
+func (store *CassandraStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *CassandraStore) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *CassandraStore) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *CassandraStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
 
 	dir, name := entry.FullPath.DirAndName()
 	meta, err := entry.EncodeAttributesAndChunks()
@@ -56,12 +67,12 @@ func (store *CassandraStore) InsertEntry(entry *filer2.Entry) (err error) {
 	return nil
 }
 
-func (store *CassandraStore) UpdateEntry(entry *filer2.Entry) (err error) {
+func (store *CassandraStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
 
-	return store.InsertEntry(entry)
+	return store.InsertEntry(ctx, entry)
 }
 
-func (store *CassandraStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
+func (store *CassandraStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
 
 	dir, name := fullpath.DirAndName()
 	var data []byte
@@ -88,7 +99,7 @@ func (store *CassandraStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.
 	return entry, nil
 }
 
-func (store *CassandraStore) DeleteEntry(fullpath filer2.FullPath) error {
+func (store *CassandraStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) error {
 
 	dir, name := fullpath.DirAndName()
 
@@ -101,7 +112,7 @@ func (store *CassandraStore) DeleteEntry(fullpath filer2.FullPath) error {
 	return nil
 }
 
-func (store *CassandraStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool,
+func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool,
 	limit int) (entries []*filer2.Entry, err error) {
 
 	cqlStr := "SELECT NAME, meta FROM filemeta WHERE directory=? AND name>? ORDER BY NAME ASC LIMIT ?"

weed/filer2/filer.go

@@ -57,7 +57,19 @@ func (fs *Filer) KeepConnectedToMaster() {
 	fs.MasterClient.KeepConnectedToMaster()
 }
 
-func (f *Filer) CreateEntry(entry *Entry) error {
+func (f *Filer) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return f.store.BeginTransaction(ctx)
+}
+
+func (f *Filer) CommitTransaction(ctx context.Context) error {
+	return f.store.CommitTransaction(ctx)
+}
+
+func (f *Filer) RollbackTransaction(ctx context.Context) error {
+	return f.store.RollbackTransaction(ctx)
+}
+
+func (f *Filer) CreateEntry(ctx context.Context, entry *Entry) error {
 
 	if string(entry.FullPath) == "/" {
 		return nil
@@ -70,7 +82,7 @@ func (f *Filer) CreateEntry(entry *Entry) error {
 	var lastDirectoryEntry *Entry
 
 	for i := 1; i < len(dirParts); i++ {
-		dirPath := "/" + filepath.Join(dirParts[:i]...)
+		dirPath := "/" + filepath.ToSlash(filepath.Join(dirParts[:i]...))
 		// fmt.Printf("%d directory: %+v\n", i, dirPath)
 
 		// first check local cache
@@ -79,7 +91,7 @@ func (f *Filer) CreateEntry(entry *Entry) error {
 		// not found, check the store directly
 		if dirEntry == nil {
 			glog.V(4).Infof("find uncached directory: %s", dirPath)
-			dirEntry, _ = f.FindEntry(FullPath(dirPath))
+			dirEntry, _ = f.FindEntry(ctx, FullPath(dirPath))
 		} else {
 			glog.V(4).Infof("found cached directory: %s", dirPath)
 		}
@@ -102,9 +114,9 @@ func (f *Filer) CreateEntry(entry *Entry) error {
 			}
 
 			glog.V(2).Infof("create directory: %s %v", dirPath, dirEntry.Mode)
-			mkdirErr := f.store.InsertEntry(dirEntry)
+			mkdirErr := f.store.InsertEntry(ctx, dirEntry)
 			if mkdirErr != nil {
-				if _, err := f.FindEntry(FullPath(dirPath)); err == ErrNotFound {
+				if _, err := f.FindEntry(ctx, FullPath(dirPath)); err == ErrNotFound {
 					return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr)
 				}
 			} else {
@@ -137,14 +149,14 @@ func (f *Filer) CreateEntry(entry *Entry) error {
 		}
 	*/
 
-	oldEntry, _ := f.FindEntry(entry.FullPath)
+	oldEntry, _ := f.FindEntry(ctx, entry.FullPath)
 
 	if oldEntry == nil {
-		if err := f.store.InsertEntry(entry); err != nil {
+		if err := f.store.InsertEntry(ctx, entry); err != nil {
 			return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
 		}
 	} else {
-		if err := f.UpdateEntry(oldEntry, entry); err != nil {
+		if err := f.UpdateEntry(ctx, oldEntry, entry); err != nil {
 			return fmt.Errorf("update entry %s: %v", entry.FullPath, err)
 		}
 	}
@@ -156,7 +168,7 @@ func (f *Filer) CreateEntry(entry *Entry) error {
 	return nil
 }
 
-func (f *Filer) UpdateEntry(oldEntry, entry *Entry) (err error) {
+func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err error) {
 	if oldEntry != nil {
 		if oldEntry.IsDirectory() && !entry.IsDirectory() {
 			return fmt.Errorf("existing %s is a directory", entry.FullPath)
@@ -165,10 +177,10 @@ func (f *Filer) UpdateEntry(oldEntry, entry *Entry) (err error) {
 			return fmt.Errorf("existing %s is a file", entry.FullPath)
 		}
 	}
-	return f.store.UpdateEntry(entry)
+	return f.store.UpdateEntry(ctx, entry)
 }
 
-func (f *Filer) FindEntry(p FullPath) (entry *Entry, err error) {
+func (f *Filer) FindEntry(ctx context.Context, p FullPath) (entry *Entry, err error) {
 
 	now := time.Now()
 
@@ -184,11 +196,11 @@ func (f *Filer) FindEntry(p FullPath) (entry *Entry, err error) {
 			},
 		}, nil
 	}
-	return f.store.FindEntry(p)
+	return f.store.FindEntry(ctx, p)
 }
 
-func (f *Filer) DeleteEntryMetaAndData(p FullPath, isRecursive bool, shouldDeleteChunks bool) (err error) {
-	entry, err := f.FindEntry(p)
+func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p FullPath, isRecursive bool, shouldDeleteChunks bool) (err error) {
+	entry, err := f.FindEntry(ctx, p)
 	if err != nil {
 		return err
 	}
@@ -201,33 +213,36 @@ func (f *Filer) DeleteEntryMetaAndData(p FullPath, isRecursive bool, shouldDelet
 		lastFileName := ""
 		includeLastFile := false
 		for limit > 0 {
-			entries, err := f.ListDirectoryEntries(p, lastFileName, includeLastFile, 1024)
+			entries, err := f.ListDirectoryEntries(ctx, p, lastFileName, includeLastFile, 1024)
 			if err != nil {
 				return fmt.Errorf("list folder %s: %v", p, err)
 			}
+
 			if len(entries) == 0 {
 				break
-			} else {
-				if isRecursive {
-					for _, sub := range entries {
-						lastFileName = sub.Name()
-						f.DeleteEntryMetaAndData(sub.FullPath, isRecursive, shouldDeleteChunks)
-						limit--
-						if limit <= 0 {
-							break
-						}
+			}
+
+			if isRecursive {
+				for _, sub := range entries {
+					lastFileName = sub.Name()
+					err = f.DeleteEntryMetaAndData(ctx, sub.FullPath, isRecursive, shouldDeleteChunks)
+					if err != nil {
+						return err
 					}
-				} else {
-					if len(entries) > 0 {
-						return fmt.Errorf("folder %s is not empty", p)
+					limit--
+					if limit <= 0 {
+						break
 					}
 				}
-				f.cacheDelDirectory(string(p))
-				if len(entries) < 1024 {
-					break
-				}
+			}
+
+			if len(entries) < 1024 {
+				break
 			}
 		}
+
+		f.cacheDelDirectory(string(p))
+
 	}
 
 	if shouldDeleteChunks {
@@ -241,17 +256,22 @@ func (f *Filer) DeleteEntryMetaAndData(p FullPath, isRecursive bool, shouldDelet
 
 	f.NotifyUpdateEvent(entry, nil, shouldDeleteChunks)
 
-	return f.store.DeleteEntry(p)
+	return f.store.DeleteEntry(ctx, p)
 }
 
-func (f *Filer) ListDirectoryEntries(p FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) {
+func (f *Filer) ListDirectoryEntries(ctx context.Context, p FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) {
 	if strings.HasSuffix(string(p), "/") && len(p) > 1 {
 		p = p[0 : len(p)-1]
 	}
-	return f.store.ListDirectoryEntries(p, startFileName, inclusive, limit)
+	return f.store.ListDirectoryEntries(ctx, p, startFileName, inclusive, limit)
 }
 
 func (f *Filer) cacheDelDirectory(dirpath string) {
+
+	if dirpath == "/" {
+		return
+	}
+
 	if f.directoryCache == nil {
 		return
 	}
@@ -260,6 +280,7 @@ func (f *Filer) cacheDelDirectory(dirpath string) {
 }
 
 func (f *Filer) cacheGetDirectory(dirpath string) *Entry {
+
 	if f.directoryCache == nil {
 		return nil
 	}

weed/filer2/filerstore.go

@@ -1,6 +1,7 @@
 package filer2
 
 import (
+	"context"
 	"errors"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
@@ -10,12 +11,16 @@ type FilerStore interface {
 	GetName() string
 	// Initialize initializes the file store
 	Initialize(configuration util.Configuration) error
-	InsertEntry(*Entry) error
-	UpdateEntry(*Entry) (err error)
+	InsertEntry(context.Context, *Entry) error
+	UpdateEntry(context.Context, *Entry) (err error)
 	// err == filer2.ErrNotFound if not found
-	FindEntry(FullPath) (entry *Entry, err error)
-	DeleteEntry(FullPath) (err error)
-	ListDirectoryEntries(dirPath FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error)
+	FindEntry(context.Context, FullPath) (entry *Entry, err error)
+	DeleteEntry(context.Context, FullPath) (err error)
+	ListDirectoryEntries(ctx context.Context, dirPath FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error)
+
+	BeginTransaction(ctx context.Context) (context.Context, error)
+	CommitTransaction(ctx context.Context) error
+	RollbackTransaction(ctx context.Context) error
 }
 
 var ErrNotFound = errors.New("filer: no entry is found in filer store")

weed/filer2/fullpath.go

@@ -8,10 +8,7 @@ import (
 type FullPath string
 
 func NewFullPath(dir, name string) FullPath {
-	if strings.HasSuffix(dir, "/") {
-		return FullPath(dir + name)
-	}
-	return FullPath(dir + "/" + name)
+	return FullPath(dir).Child(name)
 }
 
 func (fp FullPath) DirAndName() (string, string) {
@@ -29,3 +26,11 @@ func (fp FullPath) Name() string {
 	_, name := filepath.Split(string(fp))
 	return name
 }
+
+func (fp FullPath) Child(name string) FullPath {
+	dir := string(fp)
+	if strings.HasSuffix(dir, "/") {
+		return FullPath(dir + name)
+	}
+	return FullPath(dir + "/" + name)
+}

weed/filer2/leveldb/leveldb_store.go

@@ -2,6 +2,7 @@ package leveldb
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 
 	"github.com/chrislusf/seaweedfs/weed/filer2"
@@ -45,7 +46,17 @@ func (store *LevelDBStore) initialize(dir string) (err error) {
 	return
 }
 
-func (store *LevelDBStore) InsertEntry(entry *filer2.Entry) (err error) {
+func (store *LevelDBStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *LevelDBStore) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *LevelDBStore) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
 	key := genKey(entry.DirAndName())
 
 	value, err := entry.EncodeAttributesAndChunks()
@@ -64,12 +75,12 @@ func (store *LevelDBStore) InsertEntry(entry *filer2.Entry) (err error) {
 	return nil
 }
 
-func (store *LevelDBStore) UpdateEntry(entry *filer2.Entry) (err error) {
+func (store *LevelDBStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
 
-	return store.InsertEntry(entry)
+	return store.InsertEntry(ctx, entry)
 }
 
-func (store *LevelDBStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
+func (store *LevelDBStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
 	key := genKey(fullpath.DirAndName())
 
 	data, err := store.db.Get(key, nil)
@@ -94,7 +105,7 @@ func (store *LevelDBStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.En
 	return entry, nil
 }
 
-func (store *LevelDBStore) DeleteEntry(fullpath filer2.FullPath) (err error) {
+func (store *LevelDBStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) (err error) {
 	key := genKey(fullpath.DirAndName())
 
 	err = store.db.Delete(key, nil)
@@ -105,7 +116,7 @@ func (store *LevelDBStore) DeleteEntry(fullpath filer2.FullPath) (err error) {
 	return nil
 }
 
-func (store *LevelDBStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool,
+func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool,
 	limit int) (entries []*filer2.Entry, err error) {
 
 	directoryPrefix := genDirectoryKeyPrefix(fullpath, "")

weed/filer2/leveldb/leveldb_store_test.go

@@ -1,6 +1,7 @@
 package leveldb
 
 import (
+	"context"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"io/ioutil"
 	"os"
@@ -18,6 +19,8 @@ func TestCreateAndFind(t *testing.T) {
 
 	fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg")
 
+	ctx := context.Background()
+
 	entry1 := &filer2.Entry{
 		FullPath: fullpath,
 		Attr: filer2.Attr{
@@ -27,12 +30,12 @@ func TestCreateAndFind(t *testing.T) {
 		},
 	}
 
-	if err := filer.CreateEntry(entry1); err != nil {
+	if err := filer.CreateEntry(ctx, entry1); err != nil {
 		t.Errorf("create entry %v: %v", entry1.FullPath, err)
 		return
 	}
 
-	entry, err := filer.FindEntry(fullpath)
+	entry, err := filer.FindEntry(ctx, fullpath)
 
 	if err != nil {
 		t.Errorf("find entry: %v", err)
@@ -45,14 +48,14 @@ func TestCreateAndFind(t *testing.T) {
 	}
 
 	// checking one upper directory
-	entries, _ := filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one"), "", false, 100)
+	entries, _ := filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is/one"), "", false, 100)
 	if len(entries) != 1 {
 		t.Errorf("list entries count: %v", len(entries))
 		return
 	}
 
 	// checking one upper directory
-	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/"), "", false, 100)
+	entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
 	if len(entries) != 1 {
 		t.Errorf("list entries count: %v", len(entries))
 		return
@@ -69,8 +72,10 @@ func TestEmptyRoot(t *testing.T) {
 	filer.SetStore(store)
 	filer.DisableDirectoryCache()
 
+	ctx := context.Background()
+
 	// checking one upper directory
-	entries, err := filer.ListDirectoryEntries(filer2.FullPath("/"), "", false, 100)
+	entries, err := filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
 	if err != nil {
 		t.Errorf("list entries: %v", err)
 		return

weed/filer2/memdb/memdb_store.go

@@ -1,6 +1,7 @@
 package memdb
 
 import (
+	"context"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/util"
@@ -33,21 +34,31 @@ func (store *MemDbStore) Initialize(configuration util.Configuration) (err error
 	return nil
 }
 
-func (store *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) {
+func (store *MemDbStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *MemDbStore) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *MemDbStore) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *MemDbStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
 	// println("inserting", entry.FullPath)
 	store.tree.ReplaceOrInsert(entryItem{entry})
 	return nil
 }
 
-func (store *MemDbStore) UpdateEntry(entry *filer2.Entry) (err error) {
-	if _, err = store.FindEntry(entry.FullPath); err != nil {
+func (store *MemDbStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
+	if _, err = store.FindEntry(ctx, entry.FullPath); err != nil {
 		return fmt.Errorf("no such file %s : %v", entry.FullPath, err)
 	}
 	store.tree.ReplaceOrInsert(entryItem{entry})
 	return nil
 }
 
-func (store *MemDbStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
+func (store *MemDbStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
 	item := store.tree.Get(entryItem{&filer2.Entry{FullPath: fullpath}})
 	if item == nil {
 		return nil, filer2.ErrNotFound
@@ -56,12 +67,12 @@ func (store *MemDbStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entr
 	return entry, nil
 }
 
-func (store *MemDbStore) DeleteEntry(fullpath filer2.FullPath) (err error) {
+func (store *MemDbStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) (err error) {
 	store.tree.Delete(entryItem{&filer2.Entry{FullPath: fullpath}})
 	return nil
 }
 
-func (store *MemDbStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) {
+func (store *MemDbStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) {
 
 	startFrom := string(fullpath)
 	if startFileName != "" {

weed/filer2/memdb/memdb_store_test.go

@@ -1,6 +1,7 @@
 package memdb
 
 import (
+	"context"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"testing"
 )
@@ -12,6 +13,8 @@ func TestCreateAndFind(t *testing.T) {
 	filer.SetStore(store)
 	filer.DisableDirectoryCache()
 
+	ctx := context.Background()
+
 	fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg")
 
 	entry1 := &filer2.Entry{
@@ -23,12 +26,12 @@ func TestCreateAndFind(t *testing.T) {
 		},
 	}
 
-	if err := filer.CreateEntry(entry1); err != nil {
+	if err := filer.CreateEntry(ctx, entry1); err != nil {
 		t.Errorf("create entry %v: %v", entry1.FullPath, err)
 		return
 	}
 
-	entry, err := filer.FindEntry(fullpath)
+	entry, err := filer.FindEntry(ctx, fullpath)
 
 	if err != nil {
 		t.Errorf("find entry: %v", err)
@@ -49,6 +52,8 @@ func TestCreateFileAndList(t *testing.T) {
 	filer.SetStore(store)
 	filer.DisableDirectoryCache()
 
+	ctx := context.Background()
+
 	entry1 := &filer2.Entry{
 		FullPath: filer2.FullPath("/home/chris/this/is/one/file1.jpg"),
 		Attr: filer2.Attr{
@@ -67,11 +72,11 @@ func TestCreateFileAndList(t *testing.T) {
 		},
 	}
 
-	filer.CreateEntry(entry1)
-	filer.CreateEntry(entry2)
+	filer.CreateEntry(ctx, entry1)
+	filer.CreateEntry(ctx, entry2)
 
 	// checking the 2 files
-	entries, err := filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one/"), "", false, 100)
+	entries, err := filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is/one/"), "", false, 100)
 
 	if err != nil {
 		t.Errorf("list entries: %v", err)
@@ -94,21 +99,21 @@ func TestCreateFileAndList(t *testing.T) {
 	}
 
 	// checking the offset
-	entries, err = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one/"), "file1.jpg", false, 100)
+	entries, err = filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is/one/"), "file1.jpg", false, 100)
 	if len(entries) != 1 {
 		t.Errorf("list entries count: %v", len(entries))
 		return
 	}
 
 	// checking one upper directory
-	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100)
+	entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is"), "", false, 100)
 	if len(entries) != 1 {
 		t.Errorf("list entries count: %v", len(entries))
 		return
 	}
 
 	// checking root directory
-	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/"), "", false, 100)
+	entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
 	if len(entries) != 1 {
 		t.Errorf("list entries count: %v", len(entries))
 		return
@@ -124,18 +129,18 @@ func TestCreateFileAndList(t *testing.T) {
 			Gid:  5678,
 		},
 	}
-	filer.CreateEntry(entry3)
+	filer.CreateEntry(ctx, entry3)
 
 	// checking one upper directory
-	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100)
+	entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is"), "", false, 100)
 	if len(entries) != 2 {
 		t.Errorf("list entries count: %v", len(entries))
 		return
 	}
 
 	// delete file and count
-	filer.DeleteEntryMetaAndData(file3Path, false, false)
-	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100)
+	filer.DeleteEntryMetaAndData(ctx, file3Path, false, false)
+	entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is"), "", false, 100)
 	if len(entries) != 1 {
 		t.Errorf("list entries count: %v", len(entries))
 		return

weed/filer2/postgres/README.txt

@@ -9,8 +9,8 @@ $PGHOME/bin/psql --username=postgres --password seaweedfs
 
 CREATE TABLE IF NOT EXISTS filemeta (
   dirhash     BIGINT,
-  name        VARCHAR(1000),
-  directory   VARCHAR(4096),
+  name        VARCHAR(65535),
+  directory   VARCHAR(65535),
   meta        bytea,
   PRIMARY KEY (dirhash, name)
 );

weed/filer2/redis/universal_redis_store.go

@@ -1,6 +1,7 @@
 package redis
 
 import (
+	"context"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -18,7 +19,17 @@ type UniversalRedisStore struct {
 	Client redis.UniversalClient
 }
 
-func (store *UniversalRedisStore) InsertEntry(entry *filer2.Entry) (err error) {
+func (store *UniversalRedisStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *UniversalRedisStore) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *UniversalRedisStore) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
 
 	value, err := entry.EncodeAttributesAndChunks()
 	if err != nil {
@@ -42,12 +53,12 @@ func (store *UniversalRedisStore) InsertEntry(entry *filer2.Entry) (err error) {
 	return nil
 }
 
-func (store *UniversalRedisStore) UpdateEntry(entry *filer2.Entry) (err error) {
+func (store *UniversalRedisStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
 
-	return store.InsertEntry(entry)
+	return store.InsertEntry(ctx, entry)
 }
 
-func (store *UniversalRedisStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
+func (store *UniversalRedisStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
 
 	data, err := store.Client.Get(string(fullpath)).Result()
 	if err == redis.Nil {
@@ -69,7 +80,7 @@ func (store *UniversalRedisStore) FindEntry(fullpath filer2.FullPath) (entry *fi
 	return entry, nil
 }
 
-func (store *UniversalRedisStore) DeleteEntry(fullpath filer2.FullPath) (err error) {
+func (store *UniversalRedisStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) (err error) {
 
 	_, err = store.Client.Del(string(fullpath)).Result()
 
@@ -88,7 +99,7 @@ func (store *UniversalRedisStore) DeleteEntry(fullpath filer2.FullPath) (err err
 	return nil
 }
 
-func (store *UniversalRedisStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool,
+func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool,
 	limit int) (entries []*filer2.Entry, err error) {
 
 	members, err := store.Client.SMembers(genDirectoryListKey(string(fullpath))).Result()
@@ -126,7 +137,7 @@ func (store *UniversalRedisStore) ListDirectoryEntries(fullpath filer2.FullPath,
 	// fetch entry meta
 	for _, fileName := range members {
 		path := filer2.NewFullPath(string(fullpath), fileName)
-		entry, err := store.FindEntry(path)
+		entry, err := store.FindEntry(ctx, path)
 		if err != nil {
 			glog.V(0).Infof("list %s : %v", path, err)
 		} else {

weed/filesys/dir.go

@@ -29,7 +29,7 @@ var _ = fs.NodeRemover(&Dir{})
 var _ = fs.NodeRenamer(&Dir{})
 var _ = fs.NodeSetattrer(&Dir{})
 
-func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
+func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error {
 
 	// https://github.com/bazil/fuse/issues/196
 	attr.Valid = time.Second
@@ -56,7 +56,7 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 
 	parent, name := filepath.Split(dir.Path)
 
-	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err := dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.LookupDirectoryEntryRequest{
 			Directory: parent,
@@ -64,7 +64,7 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 		}
 
 		glog.V(1).Infof("read dir %s attr: %v", dir.Path, request)
-		resp, err := client.LookupDirectoryEntry(context, request)
+		resp, err := client.LookupDirectoryEntry(ctx, request)
 		if err != nil {
 			if err == filer2.ErrNotFound {
 				return nil
@@ -132,7 +132,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
 	glog.V(1).Infof("create: %v", request)
 
 	if request.Entry.IsDirectory {
-		if err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		if err := dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 			if _, err := client.CreateEntry(ctx, request); err != nil {
 				glog.V(0).Infof("create %s/%s: %v", dir.Path, req.Name, err)
 				return fuse.EIO
@@ -155,7 +155,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
 
 func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
 
-	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err := dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.CreateEntryRequest{
 			Directory: dir.Path,
@@ -199,7 +199,7 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.
 	}
 
 	if entry == nil {
-		err = dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		err = dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 			request := &filer_pb.LookupDirectoryEntryRequest{
 				Directory: dir.Path,
@@ -243,7 +243,7 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.
 
 func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
 
-	err = dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err = dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		paginationLimit := 1024
 		remaining := dir.wfs.option.DirListingLimit
@@ -306,7 +306,7 @@ func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
 func (dir *Dir) removeOneFile(ctx context.Context, req *fuse.RemoveRequest) error {
 
 	var entry *filer_pb.Entry
-	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err := dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.LookupDirectoryEntryRequest{
 			Directory: dir.Path,
@@ -329,9 +329,9 @@ func (dir *Dir) removeOneFile(ctx context.Context, req *fuse.RemoveRequest) erro
 		return err
 	}
 
-	dir.wfs.deleteFileChunks(entry.Chunks)
+	dir.wfs.deleteFileChunks(ctx, entry.Chunks)
 
-	return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	return dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    dir.Path,
@@ -355,7 +355,7 @@ func (dir *Dir) removeOneFile(ctx context.Context, req *fuse.RemoveRequest) erro
 
 func (dir *Dir) removeFolder(ctx context.Context, req *fuse.RemoveRequest) error {
 
-	return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	return dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    dir.Path,
@@ -401,7 +401,7 @@ func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fus
 	}
 
 	parentDir, name := filer2.FullPath(dir.Path).DirAndName()
-	return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	return dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.UpdateEntryRequest{
 			Directory: parentDir,

weed/filesys/dir_link.go

@@ -35,7 +35,7 @@ func (dir *Dir) Symlink(ctx context.Context, req *fuse.SymlinkRequest) (fs.Node,
 		},
 	}
 
-	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err := dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 		if _, err := client.CreateEntry(ctx, request); err != nil {
 			glog.V(0).Infof("symlink %s/%s: %v", dir.Path, req.NewName, err)
 			return fuse.EIO

weed/filesys/dir_rename.go

@@ -2,118 +2,32 @@ package filesys
 
 import (
 	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/fuse"
 	"github.com/seaweedfs/fuse/fs"
-	"math"
-	"path/filepath"
-
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 )
 
 func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirectory fs.Node) error {
 
 	newDir := newDirectory.(*Dir)
 
-	return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	return dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
-		// find existing entry
-		request := &filer_pb.LookupDirectoryEntryRequest{
-			Directory: dir.Path,
-			Name:      req.OldName,
+		request := &filer_pb.AtomicRenameEntryRequest{
+			OldDirectory: dir.Path,
+			OldName:      req.OldName,
+			NewDirectory: newDir.Path,
+			NewName:      req.NewName,
 		}
 
-		glog.V(4).Infof("find existing directory entry: %v", request)
-		resp, err := client.LookupDirectoryEntry(ctx, request)
+		_, err := client.AtomicRenameEntry(ctx, request)
 		if err != nil {
-			glog.V(3).Infof("renaming find %s/%s: %v", dir.Path, req.OldName, err)
-			return fuse.ENOENT
+			return fmt.Errorf("renaming %s/%s => %s/%s: %v", dir.Path, req.OldName, newDir.Path, req.NewName, err)
 		}
 
-		entry := resp.Entry
+		return nil
 
-		glog.V(4).Infof("found existing directory entry resp: %+v", resp)
-
-		return moveEntry(ctx, client, dir.Path, entry, newDir.Path, req.NewName)
 	})
 
 }
-
-func moveEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, oldParent string, entry *filer_pb.Entry, newParent, newName string) error {
-	if entry.IsDirectory {
-		currentDirPath := filepath.Join(oldParent, entry.Name)
-
-		lastFileName := ""
-		includeLastFile := false
-		limit := math.MaxInt32
-		for limit > 0 {
-			request := &filer_pb.ListEntriesRequest{
-				Directory:          currentDirPath,
-				StartFromFileName:  lastFileName,
-				InclusiveStartFrom: includeLastFile,
-				Limit:              1024,
-			}
-			glog.V(4).Infof("read directory: %v", request)
-			resp, err := client.ListEntries(ctx, request)
-			if err != nil {
-				glog.V(0).Infof("list %s: %v", oldParent, err)
-				return fuse.EIO
-			}
-			if len(resp.Entries) == 0 {
-				break
-			}
-
-			for _, item := range resp.Entries {
-				lastFileName = item.Name
-				err := moveEntry(ctx, client, currentDirPath, item, filepath.Join(newParent, newName), item.Name)
-				if err != nil {
-					return err
-				}
-				limit--
-			}
-			if len(resp.Entries) < 1024 {
-				break
-			}
-		}
-
-	}
-
-	// add to new directory
-	{
-		request := &filer_pb.CreateEntryRequest{
-			Directory: newParent,
-			Entry: &filer_pb.Entry{
-				Name:        newName,
-				IsDirectory: entry.IsDirectory,
-				Attributes:  entry.Attributes,
-				Chunks:      entry.Chunks,
-			},
-		}
-
-		glog.V(1).Infof("create new entry: %v", request)
-		if _, err := client.CreateEntry(ctx, request); err != nil {
-			glog.V(0).Infof("renaming create %s/%s: %v", newParent, newName, err)
-			return fuse.EIO
-		}
-	}
-
-	// delete old entry
-	{
-		request := &filer_pb.DeleteEntryRequest{
-			Directory:    oldParent,
-			Name:         entry.Name,
-			IsDeleteData: false,
-		}
-
-		glog.V(1).Infof("remove old entry: %v", request)
-		_, err := client.DeleteEntry(ctx, request)
-		if err != nil {
-			glog.V(0).Infof("renaming delete %s/%s: %v", oldParent, entry.Name, err)
-			return fuse.EIO
-		}
-
-	}
-
-	return nil
-
-}

weed/filesys/dirty_page.go

@@ -167,7 +167,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte
 	var fileId, host string
 	var auth security.EncodedJwt
 
-	if err := pages.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	if err := pages.f.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.AssignVolumeRequest{
 			Count:       1,

weed/filesys/file.go

@@ -109,7 +109,7 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
 		return nil
 	}
 
-	return file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	return file.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.UpdateEntryRequest{
 			Directory: file.dir.Path,
@@ -144,7 +144,7 @@ func (file *File) maybeLoadAttributes(ctx context.Context) error {
 			file.setEntry(entry)
 			// glog.V(1).Infof("file attr read cached %v attributes", file.Name)
 		} else {
-			err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+			err := file.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 				request := &filer_pb.LookupDirectoryEntryRequest{
 					Name:      file.Name,

weed/filesys/filehandle.go

@@ -8,10 +8,12 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/gabriel-vasile/mimetype"
 	"github.com/seaweedfs/fuse"
 	"github.com/seaweedfs/fuse/fs"
 	"google.golang.org/grpc"
-	"net/http"
+	"mime"
+	"path"
 	"strings"
 	"sync"
 	"time"
@@ -73,7 +75,7 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus
 
 	vid2Locations := make(map[string]*filer_pb.Locations)
 
-	err := fh.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err := fh.f.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		glog.V(4).Infof("read fh lookup volume id locations: %v", vids)
 		resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{
@@ -154,7 +156,13 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f
 	resp.Size = len(req.Data)
 
 	if req.Offset == 0 {
-		fh.contentType = http.DetectContentType(req.Data)
+		// detect mime type
+		var possibleExt string
+		fh.contentType, possibleExt = mimetype.Detect(req.Data)
+		if ext := path.Ext(fh.f.Name); ext != possibleExt {
+			fh.contentType = mime.TypeByExtension(ext)
+		}
+
 		fh.dirtyMetadata = true
 	}
 
@@ -197,7 +205,7 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 		return nil
 	}
 
-	return fh.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	return fh.f.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		if fh.f.entry.Attributes != nil {
 			fh.f.entry.Attributes.Mime = fh.contentType
@@ -221,7 +229,7 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 		chunks, garbages := filer2.CompactFileChunks(fh.f.entry.Chunks)
 		fh.f.entry.Chunks = chunks
 		// fh.f.entryViewCache = nil
-		fh.f.wfs.deleteFileChunks(garbages)
+		fh.f.wfs.deleteFileChunks(ctx, garbages)
 
 		if _, err := client.CreateEntry(ctx, request); err != nil {
 			return fmt.Errorf("update fh: %v", err)

weed/filesys/wfs.go

@@ -73,9 +73,9 @@ func (wfs *WFS) Root() (fs.Node, error) {
 	return &Dir{Path: wfs.option.FilerMountRootPath, wfs: wfs}, nil
 }
 
-func (wfs *WFS) withFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
+func (wfs *WFS) withFilerClient(ctx context.Context, fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
+	return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
 		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
 		return fn(client)
 	}, wfs.option.FilerGrpcAddress, wfs.option.GrpcDialOption)
@@ -133,7 +133,7 @@ func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.
 
 	if wfs.stats.lastChecked < time.Now().Unix()-20 {
 
-		err := wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		err := wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 			request := &filer_pb.StatisticsRequest{
 				Collection:  wfs.option.Collection,

weed/filesys/wfs_deletion.go

@@ -5,7 +5,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 )
 
-func (wfs *WFS) deleteFileChunks(chunks []*filer_pb.FileChunk) {
+func (wfs *WFS) deleteFileChunks(ctx context.Context, chunks []*filer_pb.FileChunk) {
 	if len(chunks) == 0 {
 		return
 	}
@@ -15,8 +15,8 @@ func (wfs *WFS) deleteFileChunks(chunks []*filer_pb.FileChunk) {
 		fileIds = append(fileIds, chunk.FileId)
 	}
 
-	wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-		deleteFileIds(context.Background(), wfs.option.GrpcDialOption, client, fileIds)
+	wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+		deleteFileIds(ctx, wfs.option.GrpcDialOption, client, fileIds)
 		return nil
 	})
 }

weed/glide.lock

@@ -1,352 +0,0 @@
-hash: 306ab43df769fe8072733ba28c3d2b6308288f248ee65806ac7d0bfa3349ab73
-updated: 2019-02-25T00:25:56.429024-08:00
-imports:
-- name: cloud.google.com/go
-  version: e2c125ceac8b663cfcf4610477d4d67827377cb7
-  subpackages:
-  - compute/metadata
-  - iam
-  - internal
-  - internal/optional
-  - internal/trace
-  - internal/version
-  - pubsub
-  - pubsub/apiv1
-  - pubsub/internal/distribution
-  - storage
-- name: github.com/aws/aws-sdk-go
-  version: 5604f1add1ce6b18465fd50f7fe8de7561cc8a62
-  subpackages:
-  - aws
-  - aws/awserr
-  - aws/awsutil
-  - aws/client
-  - aws/client/metadata
-  - aws/corehandlers
-  - aws/credentials
-  - aws/credentials/ec2rolecreds
-  - aws/credentials/endpointcreds
-  - aws/credentials/processcreds
-  - aws/credentials/stscreds
-  - aws/csm
-  - aws/defaults
-  - aws/ec2metadata
-  - aws/endpoints
-  - aws/request
-  - aws/session
-  - aws/signer/v4
-  - internal/ini
-  - internal/s3err
-  - internal/sdkio
-  - internal/sdkrand
-  - internal/sdkuri
-  - internal/shareddefaults
-  - private/protocol
-  - private/protocol/eventstream
-  - private/protocol/eventstream/eventstreamapi
-  - private/protocol/query
-  - private/protocol/query/queryutil
-  - private/protocol/rest
-  - private/protocol/restxml
-  - private/protocol/xml/xmlutil
-  - service/s3
-  - service/s3/s3iface
-  - service/sqs
-  - service/sts
-- name: github.com/Azure/azure-pipeline-go
-  version: 76b57228f36adfbb7e6990ba1347a7fbbf3043da
-  subpackages:
-  - pipeline
-- name: github.com/Azure/azure-storage-blob-go
-  version: 457680cc0804810f6d02958481e0ffdda51d5c60
-  subpackages:
-  - azblob
-- name: github.com/boltdb/bolt
-  version: fd01fc79c553a8e99d512a07e8e0c63d4a3ccfc5
-- name: github.com/chrislusf/raft
-  version: 10d6e2182d923e93ec0cc1aa1d556e5b1f8a39e7
-  subpackages:
-  - protobuf
-- name: github.com/DataDog/zstd
-  version: 1e382f59b41eebd6f592c5db4fd1958ec38a0eba
-- name: github.com/davecgh/go-spew
-  version: d8f796af33cc11cb798c1aaeb27a4ebc5099927d
-  subpackages:
-  - spew
-- name: github.com/dgrijalva/jwt-go
-  version: 3af4c746e1c248ee8491a3e0c6f7a9cd831e95f8
-- name: github.com/disintegration/imaging
-  version: 5362c131d56305ce787e79a5b94ffc956df00d62
-- name: github.com/dustin/go-humanize
-  version: 9f541cc9db5d55bce703bd99987c9d5cb8eea45e
-- name: github.com/eapache/go-resiliency
-  version: 842e16ec2c98ef0c59eebfe60d2d3500a793ba19
-  subpackages:
-  - breaker
-- name: github.com/eapache/go-xerial-snappy
-  version: 776d5712da21bc4762676d614db1d8a64f4238b0
-- name: github.com/eapache/queue
-  version: 093482f3f8ce946c05bcba64badd2c82369e084d
-- name: github.com/fsnotify/fsnotify
-  version: ccc981bf80385c528a65fbfdd49bf2d8da22aa23
-- name: github.com/go-redis/redis
-  version: bd542089bb6e776e6fced5038edac8a0f526aa53
-  subpackages:
-  - internal
-  - internal/consistenthash
-  - internal/hashtag
-  - internal/pool
-  - internal/proto
-  - internal/util
-- name: github.com/go-sql-driver/mysql
-  version: 972a708cf97995463843c08c8585b26997daf0e1
-- name: github.com/gocql/gocql
-  version: ec4793573d1447b6f92a1b359a0594566fad9d0e
-  subpackages:
-  - internal/lru
-  - internal/murmur
-  - internal/streams
-- name: github.com/golang/protobuf
-  version: c823c79ea1570fb5ff454033735a8e68575d1d0f
-  subpackages:
-  - proto
-  - protoc-gen-go/descriptor
-  - ptypes
-  - ptypes/any
-  - ptypes/duration
-  - ptypes/empty
-  - ptypes/timestamp
-- name: github.com/golang/snappy
-  version: 2a8bb927dd31d8daada140a5d09578521ce5c36a
-- name: github.com/google/btree
-  version: 4030bb1f1f0c35b30ca7009e9ebd06849dd45306
-- name: github.com/googleapis/gax-go
-  version: ddfab93c3faef4935403ac75a7c11f0e731dc181
-  subpackages:
-  - v2
-- name: github.com/gorilla/mux
-  version: 8559a4f775fc329165fe32bd4c2543de8ada8fce
-- name: github.com/hailocab/go-hostpool
-  version: e80d13ce29ede4452c43dea11e79b9bc8a15b478
-- name: github.com/hashicorp/golang-lru
-  version: 20f1fb78b0740ba8c3cb143a61e86ba5c8669768
-  subpackages:
-  - simplelru
-- name: github.com/hashicorp/hcl
-  version: 65a6292f0157eff210d03ed1bf6c59b190b8b906
-  subpackages:
-  - hcl/ast
-  - hcl/parser
-  - hcl/printer
-  - hcl/scanner
-  - hcl/strconv
-  - hcl/token
-  - json/parser
-  - json/scanner
-  - json/token
-- name: github.com/jmespath/go-jmespath
-  version: c2b33e8439af944379acbdd9c3a5fe0bc44bd8a5
-- name: github.com/karlseguin/ccache
-  version: ec06cd93a07565b373789b0078ba88fe697fddd9
-- name: github.com/klauspost/crc32
-  version: bab58d77464aa9cf4e84200c3276da0831fe0c03
-- name: github.com/kurin/blazer
-  version: f20ef4f2aa8ccc2e94a1981dc37f199e90fa4ba5
-  subpackages:
-  - b2
-  - base
-  - internal/b2assets
-  - internal/b2types
-  - internal/blog
-  - x/window
-- name: github.com/lib/pq
-  version: 9eb73efc1fcc404148b56765b0d3f61d9a5ef8ee
-  subpackages:
-  - oid
-- name: github.com/magiconair/properties
-  version: 7757cc9fdb852f7579b24170bcacda2c7471bb6a
-- name: github.com/mitchellh/mapstructure
-  version: 3536a929edddb9a5b34bd6861dc4a9647cb459fe
-- name: github.com/pelletier/go-toml
-  version: 27c6b39a135b7dc87a14afb068809132fb7a9a8f
-- name: github.com/pierrec/lz4
-  version: 062282ea0dcff40c9fb8525789eef9644b1fbd6e
-  subpackages:
-  - internal/xxh32
-- name: github.com/rakyll/statik
-  version: 79258177a57a85a8ab2eca7ce0936aad80307f4e
-  subpackages:
-  - fs
-- name: github.com/rcrowley/go-metrics
-  version: 3113b8401b8a98917cde58f8bbd42a1b1c03b1fd
-- name: github.com/rwcarlsen/goexif
-  version: b1fd11e07dc5bc0d2ca3b79d28cbdf3c6d186247
-  subpackages:
-  - exif
-  - tiff
-- name: github.com/satori/go.uuid
-  version: b2ce2384e17bbe0c6d34077efa39dbab3e09123b
-- name: github.com/seaweedfs/fuse
-  version: 1aae43e32cadcfa182fc60777f20fb02673e8f82
-  subpackages:
-  - fs
-  - fuseutil
-- name: github.com/Shopify/sarama
-  version: 4602b5a8c6e826f9e0737865818dd43b2339a092
-- name: github.com/spaolacci/murmur3
-  version: f09979ecbc725b9e6d41a297405f65e7e8804acc
-- name: github.com/spf13/afero
-  version: f4711e4db9e9a1d3887343acb72b2bbfc2f686f5
-  subpackages:
-  - mem
-- name: github.com/spf13/cast
-  version: 8c9545af88b134710ab1cd196795e7f2388358d7
-- name: github.com/spf13/jwalterweatherman
-  version: 94f6ae3ed3bceceafa716478c5fbf8d29ca601a1
-- name: github.com/spf13/pflag
-  version: 24fa6976df40757dce6aea913e7b81ade90530e1
-- name: github.com/spf13/viper
-  version: d104d259b3380cb653bb793756823c3c41b37b53
-- name: github.com/syndtr/goleveldb
-  version: 9d007e481048296f09f59bd19bb7ae584563cd95
-  subpackages:
-  - leveldb
-  - leveldb/cache
-  - leveldb/comparer
-  - leveldb/errors
-  - leveldb/filter
-  - leveldb/iterator
-  - leveldb/journal
-  - leveldb/memdb
-  - leveldb/opt
-  - leveldb/storage
-  - leveldb/table
-  - leveldb/util
-- name: github.com/willf/bitset
-  version: 20ad246f50b49590afcb1ed8ad143da7163869cb
-- name: github.com/willf/bloom
-  version: 54e3b963ee1652b06c4562cb9b6020ebc6e36e59
-- name: go.opencensus.io
-  version: beafb2a85a579a4918ba259877a1625e9213a263
-  subpackages:
-  - exemplar
-  - internal
-  - internal/tagencoding
-  - plugin/ocgrpc
-  - plugin/ochttp
-  - plugin/ochttp/propagation/b3
-  - stats
-  - stats/internal
-  - stats/view
-  - tag
-  - trace
-  - trace/internal
-  - trace/propagation
-  - trace/tracestate
-- name: golang.org/x/image
-  version: 31aff87c08e9a5e5d524279a564f96968336f886
-  subpackages:
-  - bmp
-  - tiff
-  - tiff/lzw
-- name: golang.org/x/net
-  version: 3a22650c66bd7f4fb6d1e8072ffd7b75c8a27898
-  subpackages:
-  - context
-  - context/ctxhttp
-  - http/httpguts
-  - http2
-  - http2/hpack
-  - idna
-  - internal/timeseries
-  - trace
-- name: golang.org/x/oauth2
-  version: 9b3c75971fc92dd27c6436a37c05c831498658f1
-  subpackages:
-  - google
-  - internal
-  - jws
-  - jwt
-- name: golang.org/x/sync
-  version: 37e7f081c4d4c64e13b10787722085407fe5d15f
-  subpackages:
-  - errgroup
-  - semaphore
-- name: golang.org/x/text
-  version: 6c92c7dc7f53607809182301b96e4cc1975143f1
-  subpackages:
-  - secure/bidirule
-  - transform
-  - unicode/bidi
-  - unicode/norm
-- name: golang.org/x/tools
-  version: 83362c3779f5f48611068d488a03ea7bbaddc81e
-  subpackages:
-  - godoc/util
-  - godoc/vfs
-- name: google.golang.org/api
-  version: 8a550ba84cafabe9b2262c41303f31e5a4626318
-  subpackages:
-  - gensupport
-  - googleapi
-  - googleapi/internal/uritemplates
-  - googleapi/transport
-  - internal
-  - iterator
-  - option
-  - storage/v1
-  - support/bundler
-  - transport
-  - transport/grpc
-  - transport/http
-  - transport/http/internal/propagation
-- name: google.golang.org/genproto
-  version: 082222b4a5c572e33e82ee9162d1352c7cf38682
-  subpackages:
-  - googleapis/api/annotations
-  - googleapis/iam/v1
-  - googleapis/pubsub/v1
-  - googleapis/rpc/code
-  - googleapis/rpc/status
-  - protobuf/field_mask
-- name: google.golang.org/grpc
-  version: 2773c7bbcf81cf358d3f0038b1469b2d44062acb
-  subpackages:
-  - balancer
-  - balancer/base
-  - balancer/roundrobin
-  - binarylog/grpc_binarylog_v1
-  - codes
-  - connectivity
-  - credentials
-  - credentials/internal
-  - credentials/oauth
-  - encoding
-  - encoding/proto
-  - grpclog
-  - internal
-  - internal/backoff
-  - internal/binarylog
-  - internal/channelz
-  - internal/envconfig
-  - internal/grpcrand
-  - internal/grpcsync
-  - internal/syscall
-  - internal/transport
-  - keepalive
-  - metadata
-  - naming
-  - peer
-  - reflection
-  - reflection/grpc_reflection_v1alpha
-  - resolver
-  - resolver/dns
-  - resolver/passthrough
-  - stats
-  - status
-  - tap
-- name: gopkg.in/inf.v0
-  version: d2d2541c53f18d2a059457998ce2876cc8e67cbf
-- name: gopkg.in/yaml.v2
-  version: 51d6538a90f86fe93ac480b35f37b2be17fef232
-testImports: []

weed/glide.yaml

@@ -38,6 +38,7 @@ import:
   subpackages:
   - b2
 - package: github.com/lib/pq
+- package: github.com/peterh/liner
 - package: github.com/rakyll/statik
   subpackages:
   - fs
@@ -54,6 +55,14 @@ import:
   - leveldb
   - leveldb/util
 - package: github.com/willf/bloom
+- package: gocloud.dev
+  subpackages:
+  - pubsub
+  - pubsub/awssnssqs
+  - pubsub/azuresb
+  - pubsub/gcppubsub
+  - pubsub/natspubsub
+  - pubsub/rabbitpubsub
 - package: golang.org/x/net
   subpackages:
   - context

weed/notification/gocdk_pub_sub/gocdk_pub_sub.go

@@ -0,0 +1,71 @@
+// Package gocdk_pub_sub supports the Go CDK (Cloud Development Kit) PubSub API,
+// which in turn supports many providers, including Amazon SNS/SQS, Azure Service Bus,
+// Google Cloud PubSub, and RabbitMQ.
+//
+// In the config, select a provider and topic using a URL. See
+// https://godoc.org/gocloud.dev/pubsub and its sub-packages for details.
+//
+// The Go CDK PubSub API does not support administrative operations like topic
+// creation. Create the topic using a UI, CLI or provider-specific API before running
+// weed.
+//
+// The Go CDK obtains credentials via environment variables and other
+// provider-specific default mechanisms. See the provider's documentation for
+// details.
+package gocdk_pub_sub
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/notification"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/golang/protobuf/proto"
+	"gocloud.dev/pubsub"
+	_ "gocloud.dev/pubsub/awssnssqs"
+	_ "gocloud.dev/pubsub/azuresb"
+	_ "gocloud.dev/pubsub/gcppubsub"
+	_ "gocloud.dev/pubsub/natspubsub"
+	_ "gocloud.dev/pubsub/rabbitpubsub"
+)
+
+func init() {
+	notification.MessageQueues = append(notification.MessageQueues, &GoCDKPubSub{})
+}
+
+type GoCDKPubSub struct {
+	topicURL string
+	topic    *pubsub.Topic
+}
+
+func (k *GoCDKPubSub) GetName() string {
+	return "gocdk_pub_sub"
+}
+
+func (k *GoCDKPubSub) Initialize(config util.Configuration) error {
+	k.topicURL = config.GetString("topic_url")
+	glog.V(0).Infof("notification.gocdk_pub_sub.topic_url: %v", k.topicURL)
+	topic, err := pubsub.OpenTopic(context.Background(), k.topicURL)
+	if err != nil {
+		glog.Fatalf("Failed to open topic: %v", err)
+	}
+	k.topic = topic
+	return nil
+}
+
+func (k *GoCDKPubSub) SendMessage(key string, message proto.Message) error {
+	bytes, err := proto.Marshal(message)
+	if err != nil {
+		return err
+	}
+	ctx := context.Background()
+	err = k.topic.Send(ctx, &pubsub.Message{
+		Body:     bytes,
+		Metadata: map[string]string{"key": key},
+	})
+	if err != nil {
+		return fmt.Errorf("send message via Go CDK pubsub %s: %v", k.topicURL, err)
+	}
+	return nil
+}

weed/operation/compress.go

@@ -16,40 +16,9 @@ import (
  */
 func IsGzippable(ext, mtype string, data []byte) bool {
 
-	// text
-	if strings.HasPrefix(mtype, "text/") {
-		return true
-	}
-
-	// images
-	switch ext {
-	case ".svg", ".bmp":
-		return true
-	}
-	if strings.HasPrefix(mtype, "image/") {
-		return false
-	}
-
-	// by file name extension
-	switch ext {
-	case ".zip", ".rar", ".gz", ".bz2", ".xz":
-		return false
-	case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
-		return true
-	case ".php", ".java", ".go", ".rb", ".c", ".cpp", ".h", ".hpp":
-		return true
-	case ".png", ".jpg", ".jpeg":
-		return false
-	}
-
-	// by mime type
-	if strings.HasPrefix(mtype, "application/") {
-		if strings.HasSuffix(mtype, "xml") {
-			return true
-		}
-		if strings.HasSuffix(mtype, "script") {
-			return true
-		}
+	shouldBeZipped, iAmSure := IsGzippableFileType(ext, mtype)
+	if iAmSure {
+		return shouldBeZipped
 	}
 
 	isMostlyText := util.IsText(data)
@@ -57,9 +26,53 @@ func IsGzippable(ext, mtype string, data []byte) bool {
 	return isMostlyText
 }
 
+/*
+* Default more not to gzip since gzip can be done on client side.
+ */
+func IsGzippableFileType(ext, mtype string) (shouldBeZipped, iAmSure bool) {
+
+	// text
+	if strings.HasPrefix(mtype, "text/") {
+		return true, true
+	}
+
+	// images
+	switch ext {
+	case ".svg", ".bmp":
+		return true, true
+	}
+	if strings.HasPrefix(mtype, "image/") {
+		return false, true
+	}
+
+	// by file name extension
+	switch ext {
+	case ".zip", ".rar", ".gz", ".bz2", ".xz":
+		return false, true
+	case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
+		return true, true
+	case ".php", ".java", ".go", ".rb", ".c", ".cpp", ".h", ".hpp":
+		return true, true
+	case ".png", ".jpg", ".jpeg":
+		return false, true
+	}
+
+	// by mime type
+	if strings.HasPrefix(mtype, "application/") {
+		if strings.HasSuffix(mtype, "xml") {
+			return true, true
+		}
+		if strings.HasSuffix(mtype, "script") {
+			return true, true
+		}
+	}
+
+	return false, false
+}
+
 func GzipData(input []byte) ([]byte, error) {
 	buf := new(bytes.Buffer)
-	w, _ := gzip.NewWriterLevel(buf, flate.BestCompression)
+	w, _ := gzip.NewWriterLevel(buf, flate.BestSpeed)
 	if _, err := w.Write(input); err != nil {
 		glog.V(2).Infoln("error compressing data:", err)
 		return nil, err

weed/operation/grpc_client.go

@@ -1,6 +1,7 @@
 package operation
 
 import (
+	"context"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
@@ -13,12 +14,14 @@ import (
 
 func WithVolumeServerClient(volumeServer string, grpcDialOption grpc.DialOption, fn func(volume_server_pb.VolumeServerClient) error) error {
 
+	ctx := context.Background()
+
 	grpcAddress, err := toVolumeServerGrpcAddress(volumeServer)
 	if err != nil {
 		return err
 	}
 
-	return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
+	return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
 		client := volume_server_pb.NewVolumeServerClient(grpcConnection)
 		return fn(client)
 	}, grpcAddress, grpcDialOption)
@@ -37,12 +40,14 @@ func toVolumeServerGrpcAddress(volumeServer string) (grpcAddress string, err err
 
 func withMasterServerClient(masterServer string, grpcDialOption grpc.DialOption, fn func(masterClient master_pb.SeaweedClient) error) error {
 
-	masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(masterServer, 0)
+	ctx := context.Background()
+
+	masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(masterServer)
 	if parseErr != nil {
 		return fmt.Errorf("failed to parse master grpc %v", masterServer)
 	}
 
-	return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
+	return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
 		client := master_pb.NewSeaweedClient(grpcConnection)
 		return fn(client)
 	}, masterGrpcAddress, grpcDialOption)

weed/operation/sync_volume.go

@@ -2,12 +2,8 @@ package operation
 
 import (
 	"context"
-	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
-	. "github.com/chrislusf/seaweedfs/weed/storage/types"
-	"github.com/chrislusf/seaweedfs/weed/util"
 	"google.golang.org/grpc"
-	"io"
 )
 
 func GetVolumeSyncStatus(server string, grpcDialOption grpc.DialOption, vid uint32) (resp *volume_server_pb.VolumeSyncStatusResponse, err error) {
@@ -15,47 +11,10 @@ func GetVolumeSyncStatus(server string, grpcDialOption grpc.DialOption, vid uint
 	WithVolumeServerClient(server, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
 
 		resp, err = client.VolumeSyncStatus(context.Background(), &volume_server_pb.VolumeSyncStatusRequest{
-			VolumdId: vid,
+			VolumeId: vid,
 		})
 		return nil
 	})
 
 	return
 }
-
-func GetVolumeIdxEntries(server string, grpcDialOption grpc.DialOption, vid uint32, eachEntryFn func(key NeedleId, offset Offset, size uint32)) error {
-
-	return WithVolumeServerClient(server, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
-		stream, err := client.VolumeSyncIndex(context.Background(), &volume_server_pb.VolumeSyncIndexRequest{
-			VolumdId: vid,
-		})
-		if err != nil {
-			return err
-		}
-
-		var indexFileContent []byte
-
-		for {
-			resp, err := stream.Recv()
-			if err == io.EOF {
-				break
-			}
-			if err != nil {
-				return fmt.Errorf("read index entries: %v", err)
-			}
-			indexFileContent = append(indexFileContent, resp.IndexFileContent...)
-		}
-
-		dataSize := len(indexFileContent)
-
-		for idx := 0; idx+NeedleEntrySize <= dataSize; idx += NeedleEntrySize {
-			line := indexFileContent[idx : idx+NeedleEntrySize]
-			key := BytesToNeedleId(line[:NeedleIdSize])
-			offset := BytesToOffset(line[NeedleIdSize : NeedleIdSize+OffsetSize])
-			size := util.BytesToUint32(line[NeedleIdSize+OffsetSize : NeedleIdSize+OffsetSize+SizeSize])
-			eachEntryFn(key, offset, size)
-		}
-
-		return nil
-	})
-}

weed/operation/upload_content.go

@@ -2,6 +2,8 @@ package operation
 
 import (
 	"bytes"
+	"compress/flate"
+	"compress/gzip"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -39,10 +41,24 @@ var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
 
 // Upload sends a POST request to a volume server to upload the content
 func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
+	contentIsGzipped := isGzipped
+	shouldGzipNow := false
+	if !isGzipped {
+		if shouldBeZipped, iAmSure := IsGzippableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeZipped {
+			shouldGzipNow = true
+			contentIsGzipped = true
+		}
+	}
 	return upload_content(uploadUrl, func(w io.Writer) (err error) {
-		_, err = io.Copy(w, reader)
+		if shouldGzipNow {
+			gzWriter, _ := gzip.NewWriterLevel(w, flate.BestSpeed)
+			_, err = io.Copy(gzWriter, reader)
+			gzWriter.Close()
+		} else {
+			_, err = io.Copy(w, reader)
+		}
 		return
-	}, filename, isGzipped, mtype, pairMap, jwt)
+	}, filename, contentIsGzipped, mtype, pairMap, jwt)
 }
 func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
 	body_buf := bytes.NewBufferString("")
@@ -92,12 +108,6 @@ func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error
 		return nil, post_err
 	}
 	defer resp.Body.Close()
-
-	if resp.StatusCode < http.StatusOK ||
-		resp.StatusCode > http.StatusIMUsed {
-		return nil, errors.New(http.StatusText(resp.StatusCode))
-	}
-
 	etag := getEtag(resp)
 	resp_body, ra_err := ioutil.ReadAll(resp.Body)
 	if ra_err != nil {

weed/pb/filer.proto

@@ -24,6 +24,9 @@ service SeaweedFiler {
     rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) {
     }
 
+    rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) {
+    }
+
     rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
     }
 
@@ -126,6 +129,16 @@ message DeleteEntryRequest {
 message DeleteEntryResponse {
 }
 
+message AtomicRenameEntryRequest {
+    string old_directory = 1;
+    string old_name = 2;
+    string new_directory = 3;
+    string new_name = 4;
+}
+
+message AtomicRenameEntryResponse {
+}
+
 message AssignVolumeRequest {
     int32 count = 1;
     string collection = 2;

weed/pb/filer_pb/filer.pb.go

@@ -23,6 +23,8 @@ It has these top-level messages:
 	UpdateEntryResponse
 	DeleteEntryRequest
 	DeleteEntryResponse
+	AtomicRenameEntryRequest
+	AtomicRenameEntryResponse
 	AssignVolumeRequest
 	AssignVolumeResponse
 	LookupVolumeRequest
@@ -521,6 +523,54 @@ func (m *DeleteEntryResponse) String() string            { return proto.CompactT
 func (*DeleteEntryResponse) ProtoMessage()               {}
 func (*DeleteEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
 
+type AtomicRenameEntryRequest struct {
+	OldDirectory string `protobuf:"bytes,1,opt,name=old_directory,json=oldDirectory" json:"old_directory,omitempty"`
+	OldName      string `protobuf:"bytes,2,opt,name=old_name,json=oldName" json:"old_name,omitempty"`
+	NewDirectory string `protobuf:"bytes,3,opt,name=new_directory,json=newDirectory" json:"new_directory,omitempty"`
+	NewName      string `protobuf:"bytes,4,opt,name=new_name,json=newName" json:"new_name,omitempty"`
+}
+
+func (m *AtomicRenameEntryRequest) Reset()                    { *m = AtomicRenameEntryRequest{} }
+func (m *AtomicRenameEntryRequest) String() string            { return proto.CompactTextString(m) }
+func (*AtomicRenameEntryRequest) ProtoMessage()               {}
+func (*AtomicRenameEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
+
+func (m *AtomicRenameEntryRequest) GetOldDirectory() string {
+	if m != nil {
+		return m.OldDirectory
+	}
+	return ""
+}
+
+func (m *AtomicRenameEntryRequest) GetOldName() string {
+	if m != nil {
+		return m.OldName
+	}
+	return ""
+}
+
+func (m *AtomicRenameEntryRequest) GetNewDirectory() string {
+	if m != nil {
+		return m.NewDirectory
+	}
+	return ""
+}
+
+func (m *AtomicRenameEntryRequest) GetNewName() string {
+	if m != nil {
+		return m.NewName
+	}
+	return ""
+}
+
+type AtomicRenameEntryResponse struct {
+}
+
+func (m *AtomicRenameEntryResponse) Reset()                    { *m = AtomicRenameEntryResponse{} }
+func (m *AtomicRenameEntryResponse) String() string            { return proto.CompactTextString(m) }
+func (*AtomicRenameEntryResponse) ProtoMessage()               {}
+func (*AtomicRenameEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
+
 type AssignVolumeRequest struct {
 	Count       int32  `protobuf:"varint,1,opt,name=count" json:"count,omitempty"`
 	Collection  string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
@@ -532,7 +582,7 @@ type AssignVolumeRequest struct {
 func (m *AssignVolumeRequest) Reset()                    { *m = AssignVolumeRequest{} }
 func (m *AssignVolumeRequest) String() string            { return proto.CompactTextString(m) }
 func (*AssignVolumeRequest) ProtoMessage()               {}
-func (*AssignVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
+func (*AssignVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
 
 func (m *AssignVolumeRequest) GetCount() int32 {
 	if m != nil {
@@ -580,7 +630,7 @@ type AssignVolumeResponse struct {
 func (m *AssignVolumeResponse) Reset()                    { *m = AssignVolumeResponse{} }
 func (m *AssignVolumeResponse) String() string            { return proto.CompactTextString(m) }
 func (*AssignVolumeResponse) ProtoMessage()               {}
-func (*AssignVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
+func (*AssignVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
 
 func (m *AssignVolumeResponse) GetFileId() string {
 	if m != nil {
@@ -624,7 +674,7 @@ type LookupVolumeRequest struct {
 func (m *LookupVolumeRequest) Reset()                    { *m = LookupVolumeRequest{} }
 func (m *LookupVolumeRequest) String() string            { return proto.CompactTextString(m) }
 func (*LookupVolumeRequest) ProtoMessage()               {}
-func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
+func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
 
 func (m *LookupVolumeRequest) GetVolumeIds() []string {
 	if m != nil {
@@ -640,7 +690,7 @@ type Locations struct {
 func (m *Locations) Reset()                    { *m = Locations{} }
 func (m *Locations) String() string            { return proto.CompactTextString(m) }
 func (*Locations) ProtoMessage()               {}
-func (*Locations) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
+func (*Locations) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
 
 func (m *Locations) GetLocations() []*Location {
 	if m != nil {
@@ -657,7 +707,7 @@ type Location struct {
 func (m *Location) Reset()                    { *m = Location{} }
 func (m *Location) String() string            { return proto.CompactTextString(m) }
 func (*Location) ProtoMessage()               {}
-func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
+func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
 
 func (m *Location) GetUrl() string {
 	if m != nil {
@@ -680,7 +730,7 @@ type LookupVolumeResponse struct {
 func (m *LookupVolumeResponse) Reset()                    { *m = LookupVolumeResponse{} }
 func (m *LookupVolumeResponse) String() string            { return proto.CompactTextString(m) }
 func (*LookupVolumeResponse) ProtoMessage()               {}
-func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
+func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
 
 func (m *LookupVolumeResponse) GetLocationsMap() map[string]*Locations {
 	if m != nil {
@@ -696,7 +746,7 @@ type DeleteCollectionRequest struct {
 func (m *DeleteCollectionRequest) Reset()                    { *m = DeleteCollectionRequest{} }
 func (m *DeleteCollectionRequest) String() string            { return proto.CompactTextString(m) }
 func (*DeleteCollectionRequest) ProtoMessage()               {}
-func (*DeleteCollectionRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
+func (*DeleteCollectionRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
 
 func (m *DeleteCollectionRequest) GetCollection() string {
 	if m != nil {
@@ -711,7 +761,7 @@ type DeleteCollectionResponse struct {
 func (m *DeleteCollectionResponse) Reset()                    { *m = DeleteCollectionResponse{} }
 func (m *DeleteCollectionResponse) String() string            { return proto.CompactTextString(m) }
 func (*DeleteCollectionResponse) ProtoMessage()               {}
-func (*DeleteCollectionResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
+func (*DeleteCollectionResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
 
 type StatisticsRequest struct {
 	Replication string `protobuf:"bytes,1,opt,name=replication" json:"replication,omitempty"`
@@ -722,7 +772,7 @@ type StatisticsRequest struct {
 func (m *StatisticsRequest) Reset()                    { *m = StatisticsRequest{} }
 func (m *StatisticsRequest) String() string            { return proto.CompactTextString(m) }
 func (*StatisticsRequest) ProtoMessage()               {}
-func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
+func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
 
 func (m *StatisticsRequest) GetReplication() string {
 	if m != nil {
@@ -757,7 +807,7 @@ type StatisticsResponse struct {
 func (m *StatisticsResponse) Reset()                    { *m = StatisticsResponse{} }
 func (m *StatisticsResponse) String() string            { return proto.CompactTextString(m) }
 func (*StatisticsResponse) ProtoMessage()               {}
-func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
+func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
 
 func (m *StatisticsResponse) GetReplication() string {
 	if m != nil {
@@ -816,6 +866,8 @@ func init() {
 	proto.RegisterType((*UpdateEntryResponse)(nil), "filer_pb.UpdateEntryResponse")
 	proto.RegisterType((*DeleteEntryRequest)(nil), "filer_pb.DeleteEntryRequest")
 	proto.RegisterType((*DeleteEntryResponse)(nil), "filer_pb.DeleteEntryResponse")
+	proto.RegisterType((*AtomicRenameEntryRequest)(nil), "filer_pb.AtomicRenameEntryRequest")
+	proto.RegisterType((*AtomicRenameEntryResponse)(nil), "filer_pb.AtomicRenameEntryResponse")
 	proto.RegisterType((*AssignVolumeRequest)(nil), "filer_pb.AssignVolumeRequest")
 	proto.RegisterType((*AssignVolumeResponse)(nil), "filer_pb.AssignVolumeResponse")
 	proto.RegisterType((*LookupVolumeRequest)(nil), "filer_pb.LookupVolumeRequest")
@@ -844,6 +896,7 @@ type SeaweedFilerClient interface {
 	CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error)
 	UpdateEntry(ctx context.Context, in *UpdateEntryRequest, opts ...grpc.CallOption) (*UpdateEntryResponse, error)
 	DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error)
+	AtomicRenameEntry(ctx context.Context, in *AtomicRenameEntryRequest, opts ...grpc.CallOption) (*AtomicRenameEntryResponse, error)
 	AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error)
 	LookupVolume(ctx context.Context, in *LookupVolumeRequest, opts ...grpc.CallOption) (*LookupVolumeResponse, error)
 	DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error)
@@ -903,6 +956,15 @@ func (c *seaweedFilerClient) DeleteEntry(ctx context.Context, in *DeleteEntryReq
 	return out, nil
 }
 
+func (c *seaweedFilerClient) AtomicRenameEntry(ctx context.Context, in *AtomicRenameEntryRequest, opts ...grpc.CallOption) (*AtomicRenameEntryResponse, error) {
+	out := new(AtomicRenameEntryResponse)
+	err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/AtomicRenameEntry", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *seaweedFilerClient) AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error) {
 	out := new(AssignVolumeResponse)
 	err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/AssignVolume", in, out, c.cc, opts...)
@@ -947,6 +1009,7 @@ type SeaweedFilerServer interface {
 	CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error)
 	UpdateEntry(context.Context, *UpdateEntryRequest) (*UpdateEntryResponse, error)
 	DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error)
+	AtomicRenameEntry(context.Context, *AtomicRenameEntryRequest) (*AtomicRenameEntryResponse, error)
 	AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error)
 	LookupVolume(context.Context, *LookupVolumeRequest) (*LookupVolumeResponse, error)
 	DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error)
@@ -1047,6 +1110,24 @@ func _SeaweedFiler_DeleteEntry_Handler(srv interface{}, ctx context.Context, dec
 	return interceptor(ctx, in, info, handler)
 }
 
+func _SeaweedFiler_AtomicRenameEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(AtomicRenameEntryRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedFilerServer).AtomicRenameEntry(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/filer_pb.SeaweedFiler/AtomicRenameEntry",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedFilerServer).AtomicRenameEntry(ctx, req.(*AtomicRenameEntryRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _SeaweedFiler_AssignVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(AssignVolumeRequest)
 	if err := dec(in); err != nil {
@@ -1143,6 +1224,10 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 			MethodName: "DeleteEntry",
 			Handler:    _SeaweedFiler_DeleteEntry_Handler,
 		},
+		{
+			MethodName: "AtomicRenameEntry",
+			Handler:    _SeaweedFiler_AtomicRenameEntry_Handler,
+		},
 		{
 			MethodName: "AssignVolume",
 			Handler:    _SeaweedFiler_AssignVolume_Handler,
@@ -1167,87 +1252,92 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 func init() { proto.RegisterFile("filer.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 1301 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x57, 0x4d, 0x8f, 0xd3, 0xc6,
-	0x1b, 0xc7, 0x79, 0x23, 0x7e, 0x92, 0xf0, 0xdf, 0x9d, 0xec, 0xbf, 0x58, 0x61, 0x97, 0x06, 0xb7,
-	0x54, 0x8b, 0x8a, 0x22, 0x44, 0x7b, 0x80, 0xa2, 0x4a, 0x85, 0x65, 0x91, 0x90, 0x16, 0xa8, 0xbc,
-	0x50, 0xa9, 0xea, 0xc1, 0xf2, 0xda, 0x93, 0x30, 0x5a, 0xc7, 0x4e, 0x3d, 0xe3, 0x05, 0xfa, 0x11,
-	0x7a, 0xe9, 0xa5, 0xd7, 0x1e, 0x7a, 0xea, 0xb7, 0xe8, 0xa5, 0xdf, 0xa7, 0xf7, 0xde, 0xaa, 0x79,
-	0x66, 0xec, 0x8c, 0x63, 0x2f, 0xb4, 0xaa, 0xb8, 0xcd, 0xfc, 0x9e, 0xf7, 0x67, 0x9e, 0x17, 0x1b,
-	0x06, 0x73, 0x16, 0xd3, 0x6c, 0xb6, 0xca, 0x52, 0x91, 0x92, 0x3e, 0x5e, 0xfc, 0xd5, 0x89, 0xfb,
-	0x0c, 0xae, 0x1c, 0xa5, 0xe9, 0x69, 0xbe, 0x7a, 0xc8, 0x32, 0x1a, 0x8a, 0x34, 0x7b, 0x73, 0x98,
-	0x88, 0xec, 0x8d, 0x47, 0xbf, 0xcf, 0x29, 0x17, 0x64, 0x17, 0xec, 0xa8, 0x20, 0x38, 0xd6, 0xd4,
-	0xda, 0xb7, 0xbd, 0x35, 0x40, 0x08, 0x74, 0x92, 0x60, 0x49, 0x9d, 0x16, 0x12, 0xf0, 0xec, 0x1e,
-	0xc2, 0x6e, 0xb3, 0x42, 0xbe, 0x4a, 0x13, 0x4e, 0xc9, 0x75, 0xe8, 0x52, 0x09, 0xa0, 0xb6, 0xc1,
-	0xed, 0xff, 0xcd, 0x0a, 0x57, 0x66, 0x8a, 0x4f, 0x51, 0xdd, 0xdf, 0x2d, 0x20, 0x47, 0x8c, 0x0b,
-	0x09, 0x32, 0xca, 0xff, 0x99, 0x3f, 0x1f, 0x40, 0x6f, 0x95, 0xd1, 0x39, 0x7b, 0xad, 0x3d, 0xd2,
-	0x37, 0x72, 0x13, 0xb6, 0xb9, 0x08, 0x32, 0xf1, 0x28, 0x4b, 0x97, 0x8f, 0x58, 0x4c, 0x9f, 0x4a,
-	0xa7, 0xdb, 0xc8, 0x52, 0x27, 0x90, 0x19, 0x10, 0x96, 0x84, 0x71, 0xce, 0xd9, 0x19, 0x3d, 0x2e,
-	0xa8, 0x4e, 0x67, 0x6a, 0xed, 0xf7, 0xbd, 0x06, 0x0a, 0xd9, 0x81, 0x6e, 0xcc, 0x96, 0x4c, 0x38,
-	0xdd, 0xa9, 0xb5, 0x3f, 0xf2, 0xd4, 0xc5, 0xfd, 0x0a, 0xc6, 0x15, 0xff, 0x75, 0xf8, 0x37, 0xe0,
-	0x22, 0x55, 0x90, 0x63, 0x4d, 0xdb, 0x4d, 0x09, 0x28, 0xe8, 0xee, 0x2f, 0x2d, 0xe8, 0x22, 0x54,
-	0xe6, 0xd9, 0x5a, 0xe7, 0x99, 0x5c, 0x83, 0x21, 0xe3, 0xfe, 0x3a, 0x19, 0x2d, 0xf4, 0x6f, 0xc0,
-	0x78, 0x99, 0x77, 0xf2, 0x29, 0xf4, 0xc2, 0x97, 0x79, 0x72, 0xca, 0x9d, 0x36, 0x9a, 0x1a, 0xaf,
-	0x4d, 0xc9, 0x60, 0x0f, 0x24, 0xcd, 0xd3, 0x2c, 0xe4, 0x0e, 0x40, 0x20, 0x44, 0xc6, 0x4e, 0x72,
-	0x41, 0x39, 0x46, 0x3b, 0xb8, 0xed, 0x18, 0x02, 0x39, 0xa7, 0xf7, 0x4b, 0xba, 0x67, 0xf0, 0x92,
-	0xbb, 0xd0, 0xa7, 0xaf, 0x05, 0x4d, 0x22, 0x1a, 0x39, 0x5d, 0x34, 0xb4, 0xb7, 0x11, 0xd3, 0xec,
-	0x50, 0xd3, 0x55, 0x84, 0x25, 0xfb, 0xe4, 0x1e, 0x8c, 0x2a, 0x24, 0xb2, 0x05, 0xed, 0x53, 0x5a,
-	0xbc, 0xac, 0x3c, 0xca, 0xec, 0x9e, 0x05, 0x71, 0xae, 0x8a, 0x6c, 0xe8, 0xa9, 0xcb, 0x17, 0xad,
-	0x3b, 0x96, 0xfb, 0xb3, 0x05, 0xdb, 0x87, 0x67, 0x34, 0x11, 0x4f, 0x53, 0xc1, 0xe6, 0x2c, 0x0c,
-	0x04, 0x4b, 0x13, 0x72, 0x13, 0xec, 0x34, 0x8e, 0xfc, 0xb7, 0xd6, 0x58, 0x3f, 0x8d, 0xb5, 0xbd,
-	0x9b, 0x60, 0x27, 0xf4, 0x95, 0xe6, 0x6e, 0x9d, 0xc3, 0x9d, 0xd0, 0x57, 0x8a, 0xfb, 0x23, 0x18,
-	0x45, 0x34, 0xa6, 0x82, 0xfa, 0x65, 0x5e, 0x65, 0xd2, 0x87, 0x0a, 0xc4, 0x7c, 0x72, 0xf7, 0x57,
-	0x0b, 0xec, 0x32, 0xbd, 0xe4, 0x32, 0x5c, 0x94, 0xea, 0x7c, 0x16, 0xe9, 0xa0, 0x7a, 0xf2, 0xfa,
-	0x38, 0x92, 0xb5, 0x9a, 0xce, 0xe7, 0x9c, 0x0a, 0x34, 0xdb, 0xf6, 0xf4, 0x4d, 0xbe, 0x35, 0x67,
-	0x3f, 0xa8, 0xf2, 0xec, 0x78, 0x78, 0x96, 0x39, 0x58, 0x0a, 0xb6, 0xa4, 0xf8, 0x2c, 0x6d, 0x4f,
-	0x5d, 0xc8, 0x18, 0xba, 0xd4, 0x17, 0xc1, 0x02, 0xeb, 0xce, 0xf6, 0x3a, 0xf4, 0x79, 0xb0, 0x20,
-	0x1f, 0xc3, 0x25, 0x9e, 0xe6, 0x59, 0x48, 0xfd, 0xc2, 0x6c, 0x0f, 0xa9, 0x43, 0x85, 0x3e, 0x42,
-	0xe3, 0xee, 0x9f, 0x2d, 0xb8, 0x54, 0x7d, 0x51, 0x72, 0x05, 0x6c, 0x94, 0x40, 0xe3, 0x16, 0x1a,
-	0xc7, 0x29, 0x71, 0x5c, 0x71, 0xa0, 0x65, 0x3a, 0x50, 0x88, 0x2c, 0xd3, 0x48, 0xf9, 0x3b, 0x52,
-	0x22, 0x4f, 0xd2, 0x88, 0xca, 0x97, 0xcc, 0x59, 0x84, 0x1e, 0x8f, 0x3c, 0x79, 0x94, 0xc8, 0x82,
-	0x45, 0xba, 0x4b, 0xe4, 0x51, 0xe6, 0x20, 0xcc, 0x50, 0x6f, 0x4f, 0xe5, 0x40, 0xdd, 0x64, 0x0e,
-	0x96, 0x12, 0xbd, 0xa8, 0x02, 0x93, 0x67, 0x32, 0x85, 0x41, 0x46, 0x57, 0xb1, 0x7e, 0x66, 0xa7,
-	0x8f, 0x24, 0x13, 0x22, 0x57, 0x01, 0xc2, 0x34, 0x8e, 0x69, 0x88, 0x0c, 0x36, 0x32, 0x18, 0x88,
-	0x7c, 0x0a, 0x21, 0x62, 0x9f, 0xd3, 0xd0, 0x81, 0xa9, 0xb5, 0xdf, 0xf5, 0x7a, 0x42, 0xc4, 0xc7,
-	0x34, 0x94, 0x71, 0xe4, 0x9c, 0x66, 0x3e, 0xf6, 0xd8, 0x00, 0xe5, 0xfa, 0x12, 0xc0, 0x69, 0xb0,
-	0x07, 0xb0, 0xc8, 0xd2, 0x7c, 0xa5, 0xa8, 0xc3, 0x69, 0x5b, 0x8e, 0x1c, 0x44, 0x90, 0x7c, 0x1d,
-	0x2e, 0xf1, 0x37, 0xcb, 0x98, 0x25, 0xa7, 0xbe, 0x08, 0xb2, 0x05, 0x15, 0xce, 0x08, 0x15, 0x8c,
-	0x34, 0xfa, 0x1c, 0x41, 0xf7, 0x5b, 0x20, 0x07, 0x19, 0x0d, 0x04, 0xfd, 0x17, 0xd3, 0xb5, 0x9c,
-	0x94, 0xad, 0xb7, 0x4e, 0xca, 0xff, 0xc3, 0xb8, 0xa2, 0x5a, 0x0d, 0x1a, 0x69, 0xf1, 0xc5, 0x2a,
-	0x7a, 0x5f, 0x16, 0x2b, 0xaa, 0xb5, 0xc5, 0x9f, 0x2c, 0x20, 0x0f, 0xb1, 0x13, 0xfe, 0xdb, 0x0a,
-	0x91, 0x35, 0x2c, 0x47, 0x9b, 0xea, 0xb4, 0x28, 0x10, 0x81, 0x1e, 0xbe, 0x43, 0xc6, 0x95, 0xfe,
-	0x87, 0x81, 0x08, 0xf4, 0x00, 0xcc, 0x68, 0x98, 0x67, 0x72, 0x1e, 0x63, 0x5d, 0xe1, 0x00, 0xf4,
-	0x0a, 0x48, 0x3a, 0x5a, 0x71, 0x48, 0x3b, 0xfa, 0x9b, 0x05, 0xe3, 0xfb, 0x9c, 0xb3, 0x45, 0xf2,
-	0x4d, 0x1a, 0xe7, 0x4b, 0x5a, 0x78, 0xba, 0x03, 0xdd, 0x30, 0xcd, 0x13, 0x81, 0x5e, 0x76, 0x3d,
-	0x75, 0xd9, 0x28, 0xab, 0x56, 0xad, 0xac, 0x36, 0x0a, 0xb3, 0x5d, 0x2f, 0x4c, 0xa3, 0xf0, 0x3a,
-	0x95, 0xc2, 0xfb, 0x10, 0x06, 0x32, 0x3c, 0x3f, 0xa4, 0x89, 0xa0, 0x99, 0xee, 0x63, 0x90, 0xd0,
-	0x01, 0x22, 0xee, 0x8f, 0x16, 0xec, 0x54, 0x3d, 0xd5, 0x6b, 0xe4, 0xdc, 0xb1, 0x22, 0xdb, 0x2e,
-	0x8b, 0xb5, 0x9b, 0xf2, 0x28, 0x0b, 0x78, 0x95, 0x9f, 0xc4, 0x2c, 0xf4, 0x25, 0x41, 0xb9, 0x67,
-	0x2b, 0xe4, 0x45, 0x16, 0xaf, 0x83, 0xee, 0x98, 0x41, 0x13, 0xe8, 0x04, 0xb9, 0x78, 0x59, 0x8c,
-	0x16, 0x79, 0x76, 0x3f, 0x87, 0xb1, 0xda, 0xec, 0xd5, 0xac, 0xed, 0x01, 0x9c, 0x21, 0xe0, 0xb3,
-	0x48, 0x2d, 0x35, 0xdb, 0xb3, 0x15, 0xf2, 0x38, 0xe2, 0xee, 0x97, 0x60, 0x1f, 0xa5, 0x2a, 0x11,
-	0x9c, 0xdc, 0x02, 0x3b, 0x2e, 0x2e, 0x7a, 0xff, 0x91, 0x75, 0x91, 0x15, 0x7c, 0xde, 0x9a, 0xc9,
-	0xbd, 0x07, 0xfd, 0x02, 0x2e, 0x62, 0xb3, 0xce, 0x8b, 0xad, 0xb5, 0x11, 0x9b, 0xfb, 0x87, 0x05,
-	0x3b, 0x55, 0x97, 0x75, 0xfa, 0x5e, 0xc0, 0xa8, 0x34, 0xe1, 0x2f, 0x83, 0x95, 0xf6, 0xe5, 0x96,
-	0xe9, 0x4b, 0x5d, 0xac, 0x74, 0x90, 0x3f, 0x09, 0x56, 0xaa, 0xa4, 0x86, 0xb1, 0x01, 0x4d, 0x9e,
-	0xc3, 0x76, 0x8d, 0xa5, 0x61, 0xa5, 0xdd, 0x30, 0x57, 0x5a, 0x65, 0x2d, 0x97, 0xd2, 0xe6, 0x9e,
-	0xbb, 0x0b, 0x97, 0x55, 0x15, 0x1f, 0x94, 0x45, 0x57, 0xe4, 0xbe, 0x5a, 0x9b, 0xd6, 0x66, 0x6d,
-	0xba, 0x13, 0x70, 0xea, 0xa2, 0xba, 0x0b, 0x16, 0xb0, 0x7d, 0x2c, 0x02, 0xc1, 0xb8, 0x60, 0x61,
-	0xf9, 0x7d, 0xb5, 0x51, 0xcc, 0xd6, 0xbb, 0xa6, 0x6c, 0xbd, 0x1d, 0xb6, 0xa0, 0x2d, 0x44, 0x51,
-	0x67, 0xf2, 0x28, 0x5f, 0x81, 0x98, 0x96, 0xf4, 0x1b, 0xbc, 0x07, 0x53, 0xb2, 0x1e, 0x44, 0x2a,
-	0x82, 0x58, 0x6d, 0xb1, 0x0e, 0x6e, 0x31, 0x1b, 0x11, 0x5c, 0x63, 0x6a, 0xd0, 0x47, 0x8a, 0xda,
-	0x55, 0x3b, 0x4e, 0x02, 0x48, 0xdc, 0x03, 0xc0, 0x96, 0x52, 0xdd, 0xd0, 0x53, 0xb2, 0x12, 0x39,
-	0x90, 0xc0, 0xed, 0xbf, 0xba, 0x30, 0x3c, 0xa6, 0xc1, 0x2b, 0x4a, 0x23, 0xb9, 0x44, 0x33, 0xb2,
-	0x28, 0x6a, 0xab, 0xfa, 0xa1, 0x4b, 0xae, 0x6f, 0x16, 0x51, 0xe3, 0x97, 0xf5, 0xe4, 0x93, 0x77,
-	0xb1, 0xe9, 0x67, 0xba, 0x40, 0x8e, 0x60, 0x60, 0x7c, 0x49, 0x92, 0x5d, 0x43, 0xb0, 0xf6, 0x81,
-	0x3c, 0xd9, 0x3b, 0x87, 0x6a, 0x6a, 0x33, 0xd6, 0x85, 0xa9, 0xad, 0xbe, 0xa0, 0x4c, 0x6d, 0x4d,
-	0x3b, 0x06, 0xb5, 0x19, 0xab, 0xc0, 0xd4, 0x56, 0x5f, 0x3e, 0xa6, 0xb6, 0xa6, 0xfd, 0x81, 0xda,
-	0x8c, 0x79, 0x6d, 0x6a, 0xab, 0xef, 0x15, 0x53, 0x5b, 0xd3, 0x90, 0xbf, 0x40, 0x9e, 0xc1, 0xd0,
-	0x9c, 0x9d, 0xc4, 0x10, 0x68, 0x98, 0xfe, 0x93, 0xab, 0xe7, 0x91, 0x4d, 0x85, 0xe6, 0x58, 0x30,
-	0x15, 0x36, 0x0c, 0x46, 0x53, 0x61, 0xd3, 0x34, 0x71, 0x2f, 0x90, 0xef, 0x60, 0x6b, 0xb3, 0x3d,
-	0xc9, 0xb5, 0xcd, 0xb0, 0x6a, 0x5d, 0x3f, 0x71, 0xdf, 0xc6, 0x52, 0x2a, 0x7f, 0x0c, 0xb0, 0xee,
-	0x3a, 0x72, 0x65, 0x2d, 0x53, 0xeb, 0xfa, 0xc9, 0x6e, 0x33, 0xb1, 0x50, 0xf5, 0xe0, 0x2a, 0x6c,
-	0x71, 0x55, 0xfa, 0x73, 0x3e, 0x0b, 0x63, 0x46, 0x13, 0xf1, 0x00, 0xb0, 0x0b, 0xbe, 0x96, 0xbf,
-	0x93, 0x27, 0x3d, 0xfc, 0xab, 0xfc, 0xec, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x8c, 0x6d, 0xf7,
-	0x42, 0x64, 0x0e, 0x00, 0x00,
+	// 1391 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x57, 0xdd, 0x6e, 0xdc, 0x44,
+	0x14, 0xae, 0xf7, 0x2f, 0xf1, 0xd9, 0xdd, 0x92, 0x4c, 0x02, 0x75, 0x37, 0x49, 0xd9, 0x3a, 0x14,
+	0xa5, 0xa2, 0x8a, 0xaa, 0xc2, 0x45, 0x4b, 0x85, 0x44, 0x9b, 0xa6, 0x52, 0xa5, 0xb4, 0x45, 0x4e,
+	0x8b, 0x84, 0x90, 0xb0, 0x1c, 0x7b, 0xb2, 0x1d, 0xc5, 0x6b, 0x2f, 0x9e, 0x71, 0xd2, 0xf2, 0x08,
+	0xdc, 0x70, 0xc3, 0x15, 0x12, 0x17, 0x5c, 0xf1, 0x16, 0xdc, 0xf0, 0x14, 0xbc, 0x04, 0xcf, 0x80,
+	0xce, 0xcc, 0xd8, 0x3b, 0x5e, 0x3b, 0x29, 0x08, 0xf5, 0x6e, 0xe6, 0xfc, 0x7e, 0xe7, 0xcc, 0xf9,
+	0xb1, 0xa1, 0x7f, 0xcc, 0x62, 0x9a, 0xed, 0xce, 0xb2, 0x54, 0xa4, 0x64, 0x59, 0x5e, 0xfc, 0xd9,
+	0x91, 0xfb, 0x1c, 0x36, 0x0e, 0xd2, 0xf4, 0x24, 0x9f, 0x3d, 0x62, 0x19, 0x0d, 0x45, 0x9a, 0xbd,
+	0xd9, 0x4f, 0x44, 0xf6, 0xc6, 0xa3, 0xdf, 0xe7, 0x94, 0x0b, 0xb2, 0x09, 0x76, 0x54, 0x30, 0x1c,
+	0x6b, 0x6c, 0xed, 0xd8, 0xde, 0x9c, 0x40, 0x08, 0x74, 0x92, 0x60, 0x4a, 0x9d, 0x96, 0x64, 0xc8,
+	0xb3, 0xbb, 0x0f, 0x9b, 0xcd, 0x06, 0xf9, 0x2c, 0x4d, 0x38, 0x25, 0x37, 0xa0, 0x4b, 0x91, 0x20,
+	0xad, 0xf5, 0xef, 0xbc, 0xb7, 0x5b, 0x40, 0xd9, 0x55, 0x72, 0x8a, 0xeb, 0xfe, 0x61, 0x01, 0x39,
+	0x60, 0x5c, 0x20, 0x91, 0x51, 0xfe, 0xef, 0xf0, 0x7c, 0x00, 0xbd, 0x59, 0x46, 0x8f, 0xd9, 0x6b,
+	0x8d, 0x48, 0xdf, 0xc8, 0x2d, 0x58, 0xe5, 0x22, 0xc8, 0xc4, 0xe3, 0x2c, 0x9d, 0x3e, 0x66, 0x31,
+	0x7d, 0x86, 0xa0, 0xdb, 0x52, 0xa4, 0xce, 0x20, 0xbb, 0x40, 0x58, 0x12, 0xc6, 0x39, 0x67, 0xa7,
+	0xf4, 0xb0, 0xe0, 0x3a, 0x9d, 0xb1, 0xb5, 0xb3, 0xec, 0x35, 0x70, 0xc8, 0x3a, 0x74, 0x63, 0x36,
+	0x65, 0xc2, 0xe9, 0x8e, 0xad, 0x9d, 0xa1, 0xa7, 0x2e, 0xee, 0x97, 0xb0, 0x56, 0xc1, 0xaf, 0xc3,
+	0xbf, 0x09, 0x4b, 0x54, 0x91, 0x1c, 0x6b, 0xdc, 0x6e, 0x4a, 0x40, 0xc1, 0x77, 0x7f, 0x6d, 0x41,
+	0x57, 0x92, 0xca, 0x3c, 0x5b, 0xf3, 0x3c, 0x93, 0xeb, 0x30, 0x60, 0xdc, 0x9f, 0x27, 0xa3, 0x25,
+	0xf1, 0xf5, 0x19, 0x2f, 0xf3, 0x4e, 0x3e, 0x81, 0x5e, 0xf8, 0x2a, 0x4f, 0x4e, 0xb8, 0xd3, 0x96,
+	0xae, 0xd6, 0xe6, 0xae, 0x30, 0xd8, 0x3d, 0xe4, 0x79, 0x5a, 0x84, 0xdc, 0x05, 0x08, 0x84, 0xc8,
+	0xd8, 0x51, 0x2e, 0x28, 0x97, 0xd1, 0xf6, 0xef, 0x38, 0x86, 0x42, 0xce, 0xe9, 0x83, 0x92, 0xef,
+	0x19, 0xb2, 0xe4, 0x1e, 0x2c, 0xd3, 0xd7, 0x82, 0x26, 0x11, 0x8d, 0x9c, 0xae, 0x74, 0xb4, 0xb5,
+	0x10, 0xd3, 0xee, 0xbe, 0xe6, 0xab, 0x08, 0x4b, 0xf1, 0xd1, 0x7d, 0x18, 0x56, 0x58, 0x64, 0x05,
+	0xda, 0x27, 0xb4, 0x78, 0x59, 0x3c, 0x62, 0x76, 0x4f, 0x83, 0x38, 0x57, 0x45, 0x36, 0xf0, 0xd4,
+	0xe5, 0xf3, 0xd6, 0x5d, 0xcb, 0xfd, 0xd9, 0x82, 0xd5, 0xfd, 0x53, 0x9a, 0x88, 0x67, 0xa9, 0x60,
+	0xc7, 0x2c, 0x0c, 0x04, 0x4b, 0x13, 0x72, 0x0b, 0xec, 0x34, 0x8e, 0xfc, 0x0b, 0x6b, 0x6c, 0x39,
+	0x8d, 0xb5, 0xbf, 0x5b, 0x60, 0x27, 0xf4, 0x4c, 0x4b, 0xb7, 0xce, 0x91, 0x4e, 0xe8, 0x99, 0x92,
+	0xde, 0x86, 0x61, 0x44, 0x63, 0x2a, 0xa8, 0x5f, 0xe6, 0x15, 0x93, 0x3e, 0x50, 0x44, 0x99, 0x4f,
+	0xee, 0xfe, 0x66, 0x81, 0x5d, 0xa6, 0x97, 0x5c, 0x81, 0x25, 0x34, 0xe7, 0xb3, 0x48, 0x07, 0xd5,
+	0xc3, 0xeb, 0x93, 0x08, 0x6b, 0x35, 0x3d, 0x3e, 0xe6, 0x54, 0x48, 0xb7, 0x6d, 0x4f, 0xdf, 0xf0,
+	0xad, 0x39, 0xfb, 0x41, 0x95, 0x67, 0xc7, 0x93, 0x67, 0xcc, 0xc1, 0x54, 0xb0, 0x29, 0x95, 0xcf,
+	0xd2, 0xf6, 0xd4, 0x85, 0xac, 0x41, 0x97, 0xfa, 0x22, 0x98, 0xc8, 0xba, 0xb3, 0xbd, 0x0e, 0x7d,
+	0x11, 0x4c, 0xc8, 0x47, 0x70, 0x99, 0xa7, 0x79, 0x16, 0x52, 0xbf, 0x70, 0xdb, 0x93, 0xdc, 0x81,
+	0xa2, 0x3e, 0x96, 0xce, 0xdd, 0xbf, 0x5b, 0x70, 0xb9, 0xfa, 0xa2, 0x64, 0x03, 0x6c, 0xa9, 0x21,
+	0x9d, 0x5b, 0xd2, 0xb9, 0x9c, 0x12, 0x87, 0x15, 0x00, 0x2d, 0x13, 0x40, 0xa1, 0x32, 0x4d, 0x23,
+	0x85, 0x77, 0xa8, 0x54, 0x9e, 0xa6, 0x11, 0xc5, 0x97, 0xcc, 0x59, 0x24, 0x11, 0x0f, 0x3d, 0x3c,
+	0x22, 0x65, 0xc2, 0x22, 0xdd, 0x25, 0x78, 0xc4, 0x1c, 0x84, 0x99, 0xb4, 0xdb, 0x53, 0x39, 0x50,
+	0x37, 0xcc, 0xc1, 0x14, 0xa9, 0x4b, 0x2a, 0x30, 0x3c, 0x93, 0x31, 0xf4, 0x33, 0x3a, 0x8b, 0xf5,
+	0x33, 0x3b, 0xcb, 0x92, 0x65, 0x92, 0xc8, 0x35, 0x80, 0x30, 0x8d, 0x63, 0x1a, 0x4a, 0x01, 0x5b,
+	0x0a, 0x18, 0x14, 0x7c, 0x0a, 0x21, 0x62, 0x9f, 0xd3, 0xd0, 0x81, 0xb1, 0xb5, 0xd3, 0xf5, 0x7a,
+	0x42, 0xc4, 0x87, 0x34, 0xc4, 0x38, 0x72, 0x4e, 0x33, 0x5f, 0xf6, 0x58, 0x5f, 0xea, 0x2d, 0x23,
+	0x41, 0x4e, 0x83, 0x2d, 0x80, 0x49, 0x96, 0xe6, 0x33, 0xc5, 0x1d, 0x8c, 0xdb, 0x38, 0x72, 0x24,
+	0x45, 0xb2, 0x6f, 0xc0, 0x65, 0xfe, 0x66, 0x1a, 0xb3, 0xe4, 0xc4, 0x17, 0x41, 0x36, 0xa1, 0xc2,
+	0x19, 0x4a, 0x03, 0x43, 0x4d, 0x7d, 0x21, 0x89, 0xee, 0x37, 0x40, 0xf6, 0x32, 0x1a, 0x08, 0xfa,
+	0x1f, 0xa6, 0x6b, 0x39, 0x29, 0x5b, 0x17, 0x4e, 0xca, 0xf7, 0x61, 0xad, 0x62, 0x5a, 0x0d, 0x1a,
+	0xf4, 0xf8, 0x72, 0x16, 0xbd, 0x2b, 0x8f, 0x15, 0xd3, 0xda, 0xe3, 0x4f, 0x16, 0x90, 0x47, 0xb2,
+	0x13, 0xfe, 0xdf, 0x0a, 0xc1, 0x1a, 0xc6, 0xd1, 0xa6, 0x3a, 0x2d, 0x0a, 0x44, 0xa0, 0x87, 0xef,
+	0x80, 0x71, 0x65, 0xff, 0x51, 0x20, 0x02, 0x3d, 0x00, 0x33, 0x1a, 0xe6, 0x19, 0xce, 0x63, 0x59,
+	0x57, 0x72, 0x00, 0x7a, 0x05, 0x09, 0x81, 0x56, 0x00, 0x69, 0xa0, 0xbf, 0x58, 0xe0, 0x3c, 0x10,
+	0xe9, 0x94, 0x85, 0x1e, 0x45, 0x87, 0x15, 0xb8, 0xdb, 0x30, 0xc4, 0xf9, 0xb1, 0x08, 0x79, 0x90,
+	0xc6, 0xd1, 0x7c, 0xb2, 0x5e, 0x05, 0x1c, 0x21, 0xbe, 0x81, 0x7c, 0x29, 0x8d, 0x23, 0x59, 0x10,
+	0xdb, 0x30, 0xc4, 0x89, 0x32, 0xd7, 0x57, 0x7b, 0x66, 0x90, 0xd0, 0xb3, 0x8a, 0x3e, 0x0a, 0x49,
+	0xfd, 0x8e, 0xd2, 0x4f, 0xe8, 0x19, 0xea, 0xbb, 0x1b, 0x70, 0xb5, 0x01, 0x9b, 0x46, 0xfe, 0xbb,
+	0x05, 0x6b, 0x0f, 0x38, 0x67, 0x93, 0xe4, 0xeb, 0x34, 0xce, 0xa7, 0xb4, 0x00, 0xbd, 0x0e, 0xdd,
+	0x30, 0xcd, 0x13, 0x21, 0xc1, 0x76, 0x3d, 0x75, 0x59, 0x68, 0x88, 0x56, 0xad, 0x21, 0x16, 0x5a,
+	0xaa, 0x5d, 0x6f, 0x29, 0xa3, 0x65, 0x3a, 0x95, 0x96, 0xf9, 0x10, 0xfa, 0xf8, 0x30, 0x7e, 0x48,
+	0x13, 0x41, 0x33, 0x3d, 0x81, 0x00, 0x49, 0x7b, 0x92, 0xe2, 0xfe, 0x68, 0xc1, 0x7a, 0x15, 0xa9,
+	0x5e, 0x80, 0xe7, 0x0e, 0x44, 0x1c, 0x18, 0x59, 0xac, 0x61, 0xe2, 0x11, 0x5b, 0x6f, 0x96, 0x1f,
+	0xc5, 0x2c, 0xf4, 0x91, 0xa1, 0xe0, 0xd9, 0x8a, 0xf2, 0x32, 0x8b, 0xe7, 0x41, 0x77, 0xcc, 0xa0,
+	0x09, 0x74, 0x82, 0x5c, 0xbc, 0x2a, 0x86, 0x22, 0x9e, 0xdd, 0xcf, 0x60, 0x4d, 0x7d, 0x93, 0x54,
+	0xb3, 0xb6, 0x05, 0x70, 0x2a, 0x09, 0x3e, 0x8b, 0xd4, 0x3a, 0xb6, 0x3d, 0x5b, 0x51, 0x9e, 0x44,
+	0xdc, 0xfd, 0x02, 0xec, 0x83, 0x54, 0x25, 0x82, 0x93, 0xdb, 0x60, 0xc7, 0xc5, 0x45, 0x6f, 0x6e,
+	0x32, 0x6f, 0x8f, 0x42, 0xce, 0x9b, 0x0b, 0xb9, 0xf7, 0x61, 0xb9, 0x20, 0x17, 0xb1, 0x59, 0xe7,
+	0xc5, 0xd6, 0x5a, 0x88, 0xcd, 0xfd, 0xd3, 0x82, 0xf5, 0x2a, 0x64, 0x9d, 0xbe, 0x97, 0x30, 0x2c,
+	0x5d, 0xf8, 0xd3, 0x60, 0xa6, 0xb1, 0xdc, 0x36, 0xb1, 0xd4, 0xd5, 0x4a, 0x80, 0xfc, 0x69, 0x30,
+	0x53, 0x25, 0x35, 0x88, 0x0d, 0xd2, 0xe8, 0x05, 0xac, 0xd6, 0x44, 0x1a, 0x96, 0xf1, 0x4d, 0x73,
+	0x19, 0x57, 0x3e, 0x28, 0x4a, 0x6d, 0x73, 0x43, 0xdf, 0x83, 0x2b, 0xaa, 0xff, 0xf6, 0xca, 0xa2,
+	0x2b, 0x72, 0x5f, 0xad, 0x4d, 0x6b, 0xb1, 0x36, 0xdd, 0x11, 0x38, 0x75, 0x55, 0xdd, 0x05, 0x13,
+	0x58, 0x3d, 0x14, 0x81, 0x60, 0x5c, 0xb0, 0xb0, 0xfc, 0x32, 0x5c, 0x28, 0x66, 0xeb, 0x6d, 0xfb,
+	0xa1, 0xde, 0x0e, 0x2b, 0xd0, 0x16, 0xa2, 0xa8, 0x33, 0x3c, 0xe2, 0x2b, 0x10, 0xd3, 0x93, 0x7e,
+	0x83, 0x77, 0xe0, 0x0a, 0xeb, 0x41, 0xa4, 0x22, 0x88, 0xd5, 0xfe, 0xed, 0xc8, 0xfd, 0x6b, 0x4b,
+	0x8a, 0x5c, 0xc0, 0x6a, 0x45, 0x45, 0x8a, 0xdb, 0x55, 0xdb, 0x19, 0x09, 0x92, 0xb9, 0x05, 0x20,
+	0x5b, 0x4a, 0x75, 0x43, 0x4f, 0xe9, 0x22, 0x65, 0x0f, 0x09, 0x77, 0xfe, 0xea, 0xc1, 0xe0, 0x90,
+	0x06, 0x67, 0x94, 0x46, 0xb8, 0xfe, 0x33, 0x32, 0x29, 0x6a, 0xab, 0xfa, 0x89, 0x4e, 0x6e, 0x2c,
+	0x16, 0x51, 0xe3, 0x3f, 0xc1, 0xe8, 0xe3, 0xb7, 0x89, 0xe9, 0x67, 0xba, 0x44, 0x0e, 0xa0, 0x6f,
+	0x7c, 0x03, 0x93, 0x4d, 0x43, 0xb1, 0xf6, 0x69, 0x3f, 0xda, 0x3a, 0x87, 0x6b, 0x5a, 0x33, 0x16,
+	0x9d, 0x69, 0xad, 0xbe, 0x5a, 0x4d, 0x6b, 0x4d, 0xdb, 0x51, 0x5a, 0x33, 0x96, 0x98, 0x69, 0xad,
+	0xbe, 0x36, 0x4d, 0x6b, 0x4d, 0x9b, 0x4f, 0x5a, 0x33, 0x36, 0x8d, 0x69, 0xad, 0xbe, 0x11, 0x4d,
+	0x6b, 0x4d, 0xeb, 0xe9, 0x12, 0xf9, 0x0e, 0x56, 0x6b, 0x3b, 0x80, 0xb8, 0x73, 0xad, 0xf3, 0x96,
+	0xd7, 0x68, 0xfb, 0x42, 0x99, 0xd2, 0xfe, 0x73, 0x18, 0x98, 0xb3, 0x99, 0x18, 0x80, 0x1a, 0xb6,
+	0xcb, 0xe8, 0xda, 0x79, 0x6c, 0xd3, 0xa0, 0x39, 0x76, 0x4c, 0x83, 0x0d, 0x83, 0xd7, 0x34, 0xd8,
+	0x34, 0xad, 0xdc, 0x4b, 0xe4, 0x5b, 0x58, 0x59, 0x6c, 0x7f, 0x72, 0x7d, 0x31, 0x6d, 0xb5, 0xa9,
+	0x32, 0x72, 0x2f, 0x12, 0x29, 0x8d, 0x3f, 0x01, 0x98, 0x77, 0x35, 0xd9, 0x98, 0xeb, 0xd4, 0xa6,
+	0xca, 0x68, 0xb3, 0x99, 0x59, 0x98, 0x7a, 0x78, 0x0d, 0x56, 0xb8, 0x6a, 0xad, 0x63, 0xbe, 0x1b,
+	0xc6, 0x8c, 0x26, 0xe2, 0x21, 0xc8, 0x2e, 0xfb, 0x0a, 0x7f, 0xb4, 0x8f, 0x7a, 0xf2, 0x7f, 0xfb,
+	0xd3, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0xf3, 0x11, 0xc9, 0xda, 0x7e, 0x0f, 0x00, 0x00,
 }

weed/pb/master.proto

@@ -15,6 +15,12 @@ service Seaweed {
     }
     rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
     }
+    rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) {
+    }
+    rpc CollectionDelete (CollectionDeleteRequest) returns (CollectionDeleteResponse) {
+    }
+    rpc VolumeList (VolumeListRequest) returns (VolumeListResponse) {
+    }
 }
 
 //////////////////////////////////////////////////
@@ -50,6 +56,7 @@ message VolumeInformationMessage {
     uint32 replica_placement = 8;
     uint32 version = 9;
     uint32 ttl = 10;
+    uint32 compact_revision = 11;
 }
 
 message Empty {
@@ -124,3 +131,67 @@ message StatisticsResponse {
     uint64 used_size = 5;
     uint64 file_count = 6;
 }
+
+//
+// collection related
+//
+
+message StorageType {
+    string replication = 1;
+    string ttl = 2;
+}
+message Collection {
+    string name = 1;
+}
+message CollectionListRequest {
+}
+message CollectionListResponse {
+    repeated Collection collections = 1;
+}
+
+message CollectionDeleteRequest {
+    string name = 1;
+}
+message CollectionDeleteResponse {
+}
+
+//
+// volume related
+//
+message DataNodeInfo {
+    string id = 1;
+    uint64 volume_count = 2;
+    uint64 max_volume_count = 3;
+    uint64 free_volume_count = 4;
+    uint64 active_volume_count = 5;
+    repeated VolumeInformationMessage volume_infos = 6;
+}
+message RackInfo {
+    string id = 1;
+    uint64 volume_count = 2;
+    uint64 max_volume_count = 3;
+    uint64 free_volume_count = 4;
+    uint64 active_volume_count = 5;
+    repeated DataNodeInfo data_node_infos = 6;
+}
+message DataCenterInfo {
+    string id = 1;
+    uint64 volume_count = 2;
+    uint64 max_volume_count = 3;
+    uint64 free_volume_count = 4;
+    uint64 active_volume_count = 5;
+    repeated RackInfo rack_infos = 6;
+}
+message TopologyInfo {
+    string id = 1;
+    uint64 volume_count = 2;
+    uint64 max_volume_count = 3;
+    uint64 free_volume_count = 4;
+    uint64 active_volume_count = 5;
+    repeated DataCenterInfo data_center_infos = 6;
+}
+message VolumeListRequest {
+}
+message VolumeListResponse {
+    TopologyInfo topology_info = 1;
+}

weed/pb/master_pb/master.pb.go

@@ -23,6 +23,18 @@ It has these top-level messages:
 	AssignResponse
 	StatisticsRequest
 	StatisticsResponse
+	StorageType
+	Collection
+	CollectionListRequest
+	CollectionListResponse
+	CollectionDeleteRequest
+	CollectionDeleteResponse
+	DataNodeInfo
+	RackInfo
+	DataCenterInfo
+	TopologyInfo
+	VolumeListRequest
+	VolumeListResponse
 */
 package master_pb
 
@@ -178,6 +190,7 @@ type VolumeInformationMessage struct {
 	ReplicaPlacement uint32 `protobuf:"varint,8,opt,name=replica_placement,json=replicaPlacement" json:"replica_placement,omitempty"`
 	Version          uint32 `protobuf:"varint,9,opt,name=version" json:"version,omitempty"`
 	Ttl              uint32 `protobuf:"varint,10,opt,name=ttl" json:"ttl,omitempty"`
+	CompactRevision  uint32 `protobuf:"varint,11,opt,name=compact_revision,json=compactRevision" json:"compact_revision,omitempty"`
 }
 
 func (m *VolumeInformationMessage) Reset()                    { *m = VolumeInformationMessage{} }
@@ -255,6 +268,13 @@ func (m *VolumeInformationMessage) GetTtl() uint32 {
 	return 0
 }
 
+func (m *VolumeInformationMessage) GetCompactRevision() uint32 {
+	if m != nil {
+		return m.CompactRevision
+	}
+	return 0
+}
+
 type Empty struct {
 }
 
@@ -675,6 +695,345 @@ func (m *StatisticsResponse) GetFileCount() uint64 {
 	return 0
 }
 
+type StorageType struct {
+	Replication string `protobuf:"bytes,1,opt,name=replication" json:"replication,omitempty"`
+	Ttl         string `protobuf:"bytes,2,opt,name=ttl" json:"ttl,omitempty"`
+}
+
+func (m *StorageType) Reset()                    { *m = StorageType{} }
+func (m *StorageType) String() string            { return proto.CompactTextString(m) }
+func (*StorageType) ProtoMessage()               {}
+func (*StorageType) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
+
+func (m *StorageType) GetReplication() string {
+	if m != nil {
+		return m.Replication
+	}
+	return ""
+}
+
+func (m *StorageType) GetTtl() string {
+	if m != nil {
+		return m.Ttl
+	}
+	return ""
+}
+
+type Collection struct {
+	Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
+}
+
+func (m *Collection) Reset()                    { *m = Collection{} }
+func (m *Collection) String() string            { return proto.CompactTextString(m) }
+func (*Collection) ProtoMessage()               {}
+func (*Collection) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
+
+func (m *Collection) GetName() string {
+	if m != nil {
+		return m.Name
+	}
+	return ""
+}
+
+type CollectionListRequest struct {
+}
+
+func (m *CollectionListRequest) Reset()                    { *m = CollectionListRequest{} }
+func (m *CollectionListRequest) String() string            { return proto.CompactTextString(m) }
+func (*CollectionListRequest) ProtoMessage()               {}
+func (*CollectionListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
+
+type CollectionListResponse struct {
+	Collections []*Collection `protobuf:"bytes,1,rep,name=collections" json:"collections,omitempty"`
+}
+
+func (m *CollectionListResponse) Reset()                    { *m = CollectionListResponse{} }
+func (m *CollectionListResponse) String() string            { return proto.CompactTextString(m) }
+func (*CollectionListResponse) ProtoMessage()               {}
+func (*CollectionListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
+
+func (m *CollectionListResponse) GetCollections() []*Collection {
+	if m != nil {
+		return m.Collections
+	}
+	return nil
+}
+
+type CollectionDeleteRequest struct {
+	Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
+}
+
+func (m *CollectionDeleteRequest) Reset()                    { *m = CollectionDeleteRequest{} }
+func (m *CollectionDeleteRequest) String() string            { return proto.CompactTextString(m) }
+func (*CollectionDeleteRequest) ProtoMessage()               {}
+func (*CollectionDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
+
+func (m *CollectionDeleteRequest) GetName() string {
+	if m != nil {
+		return m.Name
+	}
+	return ""
+}
+
+type CollectionDeleteResponse struct {
+}
+
+func (m *CollectionDeleteResponse) Reset()                    { *m = CollectionDeleteResponse{} }
+func (m *CollectionDeleteResponse) String() string            { return proto.CompactTextString(m) }
+func (*CollectionDeleteResponse) ProtoMessage()               {}
+func (*CollectionDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
+
+//
+// volume related
+//
+type DataNodeInfo struct {
+	Id                string                      `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
+	VolumeCount       uint64                      `protobuf:"varint,2,opt,name=volume_count,json=volumeCount" json:"volume_count,omitempty"`
+	MaxVolumeCount    uint64                      `protobuf:"varint,3,opt,name=max_volume_count,json=maxVolumeCount" json:"max_volume_count,omitempty"`
+	FreeVolumeCount   uint64                      `protobuf:"varint,4,opt,name=free_volume_count,json=freeVolumeCount" json:"free_volume_count,omitempty"`
+	ActiveVolumeCount uint64                      `protobuf:"varint,5,opt,name=active_volume_count,json=activeVolumeCount" json:"active_volume_count,omitempty"`
+	VolumeInfos       []*VolumeInformationMessage `protobuf:"bytes,6,rep,name=volume_infos,json=volumeInfos" json:"volume_infos,omitempty"`
+}
+
+func (m *DataNodeInfo) Reset()                    { *m = DataNodeInfo{} }
+func (m *DataNodeInfo) String() string            { return proto.CompactTextString(m) }
+func (*DataNodeInfo) ProtoMessage()               {}
+func (*DataNodeInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
+
+func (m *DataNodeInfo) GetId() string {
+	if m != nil {
+		return m.Id
+	}
+	return ""
+}
+
+func (m *DataNodeInfo) GetVolumeCount() uint64 {
+	if m != nil {
+		return m.VolumeCount
+	}
+	return 0
+}
+
+func (m *DataNodeInfo) GetMaxVolumeCount() uint64 {
+	if m != nil {
+		return m.MaxVolumeCount
+	}
+	return 0
+}
+
+func (m *DataNodeInfo) GetFreeVolumeCount() uint64 {
+	if m != nil {
+		return m.FreeVolumeCount
+	}
+	return 0
+}
+
+func (m *DataNodeInfo) GetActiveVolumeCount() uint64 {
+	if m != nil {
+		return m.ActiveVolumeCount
+	}
+	return 0
+}
+
+func (m *DataNodeInfo) GetVolumeInfos() []*VolumeInformationMessage {
+	if m != nil {
+		return m.VolumeInfos
+	}
+	return nil
+}
+
+type RackInfo struct {
+	Id                string          `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
+	VolumeCount       uint64          `protobuf:"varint,2,opt,name=volume_count,json=volumeCount" json:"volume_count,omitempty"`
+	MaxVolumeCount    uint64          `protobuf:"varint,3,opt,name=max_volume_count,json=maxVolumeCount" json:"max_volume_count,omitempty"`
+	FreeVolumeCount   uint64          `protobuf:"varint,4,opt,name=free_volume_count,json=freeVolumeCount" json:"free_volume_count,omitempty"`
+	ActiveVolumeCount uint64          `protobuf:"varint,5,opt,name=active_volume_count,json=activeVolumeCount" json:"active_volume_count,omitempty"`
+	DataNodeInfos     []*DataNodeInfo `protobuf:"bytes,6,rep,name=data_node_infos,json=dataNodeInfos" json:"data_node_infos,omitempty"`
+}
+
+func (m *RackInfo) Reset()                    { *m = RackInfo{} }
+func (m *RackInfo) String() string            { return proto.CompactTextString(m) }
+func (*RackInfo) ProtoMessage()               {}
+func (*RackInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
+
+func (m *RackInfo) GetId() string {
+	if m != nil {
+		return m.Id
+	}
+	return ""
+}
+
+func (m *RackInfo) GetVolumeCount() uint64 {
+	if m != nil {
+		return m.VolumeCount
+	}
+	return 0
+}
+
+func (m *RackInfo) GetMaxVolumeCount() uint64 {
+	if m != nil {
+		return m.MaxVolumeCount
+	}
+	return 0
+}
+
+func (m *RackInfo) GetFreeVolumeCount() uint64 {
+	if m != nil {
+		return m.FreeVolumeCount
+	}
+	return 0
+}
+
+func (m *RackInfo) GetActiveVolumeCount() uint64 {
+	if m != nil {
+		return m.ActiveVolumeCount
+	}
+	return 0
+}
+
+func (m *RackInfo) GetDataNodeInfos() []*DataNodeInfo {
+	if m != nil {
+		return m.DataNodeInfos
+	}
+	return nil
+}
+
+type DataCenterInfo struct {
+	Id                string      `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
+	VolumeCount       uint64      `protobuf:"varint,2,opt,name=volume_count,json=volumeCount" json:"volume_count,omitempty"`
+	MaxVolumeCount    uint64      `protobuf:"varint,3,opt,name=max_volume_count,json=maxVolumeCount" json:"max_volume_count,omitempty"`
+	FreeVolumeCount   uint64      `protobuf:"varint,4,opt,name=free_volume_count,json=freeVolumeCount" json:"free_volume_count,omitempty"`
+	ActiveVolumeCount uint64      `protobuf:"varint,5,opt,name=active_volume_count,json=activeVolumeCount" json:"active_volume_count,omitempty"`
+	RackInfos         []*RackInfo `protobuf:"bytes,6,rep,name=rack_infos,json=rackInfos" json:"rack_infos,omitempty"`
+}
+
+func (m *DataCenterInfo) Reset()                    { *m = DataCenterInfo{} }
+func (m *DataCenterInfo) String() string            { return proto.CompactTextString(m) }
+func (*DataCenterInfo) ProtoMessage()               {}
+func (*DataCenterInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
+
+func (m *DataCenterInfo) GetId() string {
+	if m != nil {
+		return m.Id
+	}
+	return ""
+}
+
+func (m *DataCenterInfo) GetVolumeCount() uint64 {
+	if m != nil {
+		return m.VolumeCount
+	}
+	return 0
+}
+
+func (m *DataCenterInfo) GetMaxVolumeCount() uint64 {
+	if m != nil {
+		return m.MaxVolumeCount
+	}
+	return 0
+}
+
+func (m *DataCenterInfo) GetFreeVolumeCount() uint64 {
+	if m != nil {
+		return m.FreeVolumeCount
+	}
+	return 0
+}
+
+func (m *DataCenterInfo) GetActiveVolumeCount() uint64 {
+	if m != nil {
+		return m.ActiveVolumeCount
+	}
+	return 0
+}
+
+func (m *DataCenterInfo) GetRackInfos() []*RackInfo {
+	if m != nil {
+		return m.RackInfos
+	}
+	return nil
+}
+
+type TopologyInfo struct {
+	Id                string            `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
+	VolumeCount       uint64            `protobuf:"varint,2,opt,name=volume_count,json=volumeCount" json:"volume_count,omitempty"`
+	MaxVolumeCount    uint64            `protobuf:"varint,3,opt,name=max_volume_count,json=maxVolumeCount" json:"max_volume_count,omitempty"`
+	FreeVolumeCount   uint64            `protobuf:"varint,4,opt,name=free_volume_count,json=freeVolumeCount" json:"free_volume_count,omitempty"`
+	ActiveVolumeCount uint64            `protobuf:"varint,5,opt,name=active_volume_count,json=activeVolumeCount" json:"active_volume_count,omitempty"`
+	DataCenterInfos   []*DataCenterInfo `protobuf:"bytes,6,rep,name=data_center_infos,json=dataCenterInfos" json:"data_center_infos,omitempty"`
+}
+
+func (m *TopologyInfo) Reset()                    { *m = TopologyInfo{} }
+func (m *TopologyInfo) String() string            { return proto.CompactTextString(m) }
+func (*TopologyInfo) ProtoMessage()               {}
+func (*TopologyInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
+
+func (m *TopologyInfo) GetId() string {
+	if m != nil {
+		return m.Id
+	}
+	return ""
+}
+
+func (m *TopologyInfo) GetVolumeCount() uint64 {
+	if m != nil {
+		return m.VolumeCount
+	}
+	return 0
+}
+
+func (m *TopologyInfo) GetMaxVolumeCount() uint64 {
+	if m != nil {
+		return m.MaxVolumeCount
+	}
+	return 0
+}
+
+func (m *TopologyInfo) GetFreeVolumeCount() uint64 {
+	if m != nil {
+		return m.FreeVolumeCount
+	}
+	return 0
+}
+
+func (m *TopologyInfo) GetActiveVolumeCount() uint64 {
+	if m != nil {
+		return m.ActiveVolumeCount
+	}
+	return 0
+}
+
+func (m *TopologyInfo) GetDataCenterInfos() []*DataCenterInfo {
+	if m != nil {
+		return m.DataCenterInfos
+	}
+	return nil
+}
+
+type VolumeListRequest struct {
+}
+
+func (m *VolumeListRequest) Reset()                    { *m = VolumeListRequest{} }
+func (m *VolumeListRequest) String() string            { return proto.CompactTextString(m) }
+func (*VolumeListRequest) ProtoMessage()               {}
+func (*VolumeListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
+
+type VolumeListResponse struct {
+	TopologyInfo *TopologyInfo `protobuf:"bytes,1,opt,name=topology_info,json=topologyInfo" json:"topology_info,omitempty"`
+}
+
+func (m *VolumeListResponse) Reset()                    { *m = VolumeListResponse{} }
+func (m *VolumeListResponse) String() string            { return proto.CompactTextString(m) }
+func (*VolumeListResponse) ProtoMessage()               {}
+func (*VolumeListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
+
+func (m *VolumeListResponse) GetTopologyInfo() *TopologyInfo {
+	if m != nil {
+		return m.TopologyInfo
+	}
+	return nil
+}
+
 func init() {
 	proto.RegisterType((*Heartbeat)(nil), "master_pb.Heartbeat")
 	proto.RegisterType((*HeartbeatResponse)(nil), "master_pb.HeartbeatResponse")
@@ -692,6 +1051,18 @@ func init() {
 	proto.RegisterType((*AssignResponse)(nil), "master_pb.AssignResponse")
 	proto.RegisterType((*StatisticsRequest)(nil), "master_pb.StatisticsRequest")
 	proto.RegisterType((*StatisticsResponse)(nil), "master_pb.StatisticsResponse")
+	proto.RegisterType((*StorageType)(nil), "master_pb.StorageType")
+	proto.RegisterType((*Collection)(nil), "master_pb.Collection")
+	proto.RegisterType((*CollectionListRequest)(nil), "master_pb.CollectionListRequest")
+	proto.RegisterType((*CollectionListResponse)(nil), "master_pb.CollectionListResponse")
+	proto.RegisterType((*CollectionDeleteRequest)(nil), "master_pb.CollectionDeleteRequest")
+	proto.RegisterType((*CollectionDeleteResponse)(nil), "master_pb.CollectionDeleteResponse")
+	proto.RegisterType((*DataNodeInfo)(nil), "master_pb.DataNodeInfo")
+	proto.RegisterType((*RackInfo)(nil), "master_pb.RackInfo")
+	proto.RegisterType((*DataCenterInfo)(nil), "master_pb.DataCenterInfo")
+	proto.RegisterType((*TopologyInfo)(nil), "master_pb.TopologyInfo")
+	proto.RegisterType((*VolumeListRequest)(nil), "master_pb.VolumeListRequest")
+	proto.RegisterType((*VolumeListResponse)(nil), "master_pb.VolumeListResponse")
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -710,6 +1081,9 @@ type SeaweedClient interface {
 	LookupVolume(ctx context.Context, in *LookupVolumeRequest, opts ...grpc.CallOption) (*LookupVolumeResponse, error)
 	Assign(ctx context.Context, in *AssignRequest, opts ...grpc.CallOption) (*AssignResponse, error)
 	Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error)
+	CollectionList(ctx context.Context, in *CollectionListRequest, opts ...grpc.CallOption) (*CollectionListResponse, error)
+	CollectionDelete(ctx context.Context, in *CollectionDeleteRequest, opts ...grpc.CallOption) (*CollectionDeleteResponse, error)
+	VolumeList(ctx context.Context, in *VolumeListRequest, opts ...grpc.CallOption) (*VolumeListResponse, error)
 }
 
 type seaweedClient struct {
@@ -809,6 +1183,33 @@ func (c *seaweedClient) Statistics(ctx context.Context, in *StatisticsRequest, o
 	return out, nil
 }
 
+func (c *seaweedClient) CollectionList(ctx context.Context, in *CollectionListRequest, opts ...grpc.CallOption) (*CollectionListResponse, error) {
+	out := new(CollectionListResponse)
+	err := grpc.Invoke(ctx, "/master_pb.Seaweed/CollectionList", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedClient) CollectionDelete(ctx context.Context, in *CollectionDeleteRequest, opts ...grpc.CallOption) (*CollectionDeleteResponse, error) {
+	out := new(CollectionDeleteResponse)
+	err := grpc.Invoke(ctx, "/master_pb.Seaweed/CollectionDelete", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedClient) VolumeList(ctx context.Context, in *VolumeListRequest, opts ...grpc.CallOption) (*VolumeListResponse, error) {
+	out := new(VolumeListResponse)
+	err := grpc.Invoke(ctx, "/master_pb.Seaweed/VolumeList", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // Server API for Seaweed service
 
 type SeaweedServer interface {
@@ -817,6 +1218,9 @@ type SeaweedServer interface {
 	LookupVolume(context.Context, *LookupVolumeRequest) (*LookupVolumeResponse, error)
 	Assign(context.Context, *AssignRequest) (*AssignResponse, error)
 	Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error)
+	CollectionList(context.Context, *CollectionListRequest) (*CollectionListResponse, error)
+	CollectionDelete(context.Context, *CollectionDeleteRequest) (*CollectionDeleteResponse, error)
+	VolumeList(context.Context, *VolumeListRequest) (*VolumeListResponse, error)
 }
 
 func RegisterSeaweedServer(s *grpc.Server, srv SeaweedServer) {
@@ -929,6 +1333,60 @@ func _Seaweed_Statistics_Handler(srv interface{}, ctx context.Context, dec func(
 	return interceptor(ctx, in, info, handler)
 }
 
+func _Seaweed_CollectionList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(CollectionListRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedServer).CollectionList(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/master_pb.Seaweed/CollectionList",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedServer).CollectionList(ctx, req.(*CollectionListRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Seaweed_CollectionDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(CollectionDeleteRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedServer).CollectionDelete(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/master_pb.Seaweed/CollectionDelete",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedServer).CollectionDelete(ctx, req.(*CollectionDeleteRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Seaweed_VolumeList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(VolumeListRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedServer).VolumeList(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/master_pb.Seaweed/VolumeList",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedServer).VolumeList(ctx, req.(*VolumeListRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 var _Seaweed_serviceDesc = grpc.ServiceDesc{
 	ServiceName: "master_pb.Seaweed",
 	HandlerType: (*SeaweedServer)(nil),
@@ -945,6 +1403,18 @@ var _Seaweed_serviceDesc = grpc.ServiceDesc{
 			MethodName: "Statistics",
 			Handler:    _Seaweed_Statistics_Handler,
 		},
+		{
+			MethodName: "CollectionList",
+			Handler:    _Seaweed_CollectionList_Handler,
+		},
+		{
+			MethodName: "CollectionDelete",
+			Handler:    _Seaweed_CollectionDelete_Handler,
+		},
+		{
+			MethodName: "VolumeList",
+			Handler:    _Seaweed_VolumeList_Handler,
+		},
 	},
 	Streams: []grpc.StreamDesc{
 		{
@@ -966,71 +1436,94 @@ var _Seaweed_serviceDesc = grpc.ServiceDesc{
 func init() { proto.RegisterFile("master.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 1056 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0xcd, 0x6e, 0xe4, 0x44,
-	0x10, 0x8e, 0x3d, 0xbf, 0xae, 0xc9, 0x64, 0x27, 0x9d, 0x08, 0x79, 0x67, 0xd9, 0xdd, 0xc1, 0x5c,
-	0x06, 0x81, 0xa2, 0x25, 0x1c, 0x11, 0x42, 0x6c, 0x14, 0x44, 0x94, 0xc0, 0x06, 0x87, 0xdd, 0x03,
-	0x17, 0xd3, 0xb1, 0x2b, 0xa1, 0x15, 0xff, 0xe1, 0x6e, 0x27, 0x33, 0x7b, 0xe1, 0xc8, 0x03, 0xf0,
-	0x3e, 0x5c, 0xe0, 0xc6, 0xa3, 0x70, 0xe3, 0x09, 0x50, 0xff, 0xd8, 0xf1, 0x38, 0x19, 0x22, 0x21,
-	0x71, 0xeb, 0xfe, 0xba, 0xba, 0xab, 0xfa, 0xfb, 0xaa, 0xaa, 0x1b, 0x36, 0x13, 0xca, 0x05, 0x16,
-	0x7b, 0x79, 0x91, 0x89, 0x8c, 0x38, 0x7a, 0x16, 0xe4, 0xe7, 0xde, 0x5f, 0x36, 0x38, 0x5f, 0x21,
-	0x2d, 0xc4, 0x39, 0x52, 0x41, 0xb6, 0xc0, 0x66, 0xb9, 0x6b, 0xcd, 0xac, 0xb9, 0xe3, 0xdb, 0x2c,
-	0x27, 0x04, 0xba, 0x79, 0x56, 0x08, 0xd7, 0x9e, 0x59, 0xf3, 0xb1, 0xaf, 0xc6, 0xe4, 0x29, 0x40,
-	0x5e, 0x9e, 0xc7, 0x2c, 0x0c, 0xca, 0x22, 0x76, 0x3b, 0xca, 0xd6, 0xd1, 0xc8, 0xeb, 0x22, 0x26,
-	0x73, 0x98, 0x24, 0x74, 0x11, 0x5c, 0x67, 0x71, 0x99, 0x60, 0x10, 0x66, 0x65, 0x2a, 0xdc, 0xae,
-	0xda, 0xbe, 0x95, 0xd0, 0xc5, 0x1b, 0x05, 0x1f, 0x48, 0x94, 0xcc, 0x64, 0x54, 0x8b, 0xe0, 0x82,
-	0xc5, 0x18, 0x5c, 0xe1, 0xd2, 0xed, 0xcd, 0xac, 0x79, 0xd7, 0x87, 0x84, 0x2e, 0xbe, 0x64, 0x31,
-	0x1e, 0xe3, 0x92, 0x3c, 0x87, 0x51, 0x44, 0x05, 0x0d, 0x42, 0x4c, 0x05, 0x16, 0x6e, 0x5f, 0xf9,
-	0x02, 0x09, 0x1d, 0x28, 0x44, 0xc6, 0x57, 0xd0, 0xf0, 0xca, 0x1d, 0xa8, 0x15, 0x35, 0x96, 0xf1,
-	0xd1, 0x28, 0x61, 0x69, 0xa0, 0x22, 0x1f, 0x2a, 0xd7, 0x8e, 0x42, 0x4e, 0x65, 0xf8, 0x9f, 0xc1,
-	0x40, 0xc7, 0xc6, 0x5d, 0x67, 0xd6, 0x99, 0x8f, 0xf6, 0xdf, 0xdf, 0xab, 0xd9, 0xd8, 0xd3, 0xe1,
-	0x1d, 0xa5, 0x17, 0x59, 0x91, 0x50, 0xc1, 0xb2, 0xf4, 0x6b, 0xe4, 0x9c, 0x5e, 0xa2, 0x5f, 0xed,
-	0x21, 0x8f, 0x61, 0x98, 0xe2, 0x4d, 0x70, 0xcd, 0x22, 0xee, 0xc2, 0xac, 0x33, 0x1f, 0xfb, 0x83,
-	0x14, 0x6f, 0xde, 0xb0, 0x88, 0x93, 0xf7, 0x60, 0x33, 0xc2, 0x18, 0x05, 0x46, 0x7a, 0x79, 0xa4,
-	0x96, 0x47, 0x06, 0x93, 0x26, 0xde, 0x6b, 0xd8, 0xae, 0xc9, 0xf6, 0x91, 0xe7, 0x59, 0xca, 0x91,
-	0xcc, 0xe1, 0x91, 0x3e, 0xfd, 0x8c, 0xbd, 0xc5, 0x13, 0x96, 0x30, 0xa1, 0x14, 0xe8, 0xfa, 0x6d,
-	0x98, 0xbc, 0x03, 0xfd, 0x18, 0x69, 0x84, 0x85, 0xa1, 0xdd, 0xcc, 0xbc, 0x3f, 0x6c, 0x70, 0xd7,
-	0x85, 0xae, 0x34, 0x8d, 0xd4, 0x89, 0x63, 0xdf, 0x66, 0x91, 0xe4, 0x8c, 0xb3, 0xb7, 0xa8, 0x34,
-	0xed, 0xfa, 0x6a, 0x4c, 0x9e, 0x01, 0x84, 0x59, 0x1c, 0x63, 0x28, 0x37, 0x9a, 0xc3, 0x1b, 0x88,
-	0xe4, 0x54, 0xc9, 0x74, 0x2b, 0x67, 0xd7, 0x77, 0x24, 0xa2, 0x95, 0xac, 0x6f, 0x6e, 0x0c, 0xb4,
-	0x92, 0xe6, 0xe6, 0xda, 0xe4, 0x23, 0x20, 0x15, 0x39, 0xe7, 0xcb, 0xda, 0xb0, 0xaf, 0x0c, 0x27,
-	0x66, 0xe5, 0xe5, 0xb2, 0xb2, 0x7e, 0x02, 0x4e, 0x81, 0x34, 0x0a, 0xb2, 0x34, 0x5e, 0x2a, 0x71,
-	0x87, 0xfe, 0x50, 0x02, 0xaf, 0xd2, 0x78, 0x49, 0x3e, 0x84, 0xed, 0x02, 0xf3, 0x98, 0x85, 0x34,
-	0xc8, 0x63, 0x1a, 0x62, 0x82, 0x69, 0xa5, 0xf3, 0xc4, 0x2c, 0x9c, 0x56, 0x38, 0x71, 0x61, 0x70,
-	0x8d, 0x05, 0x97, 0xd7, 0x72, 0x94, 0x49, 0x35, 0x25, 0x13, 0xe8, 0x08, 0x11, 0xbb, 0xa0, 0x50,
-	0x39, 0xf4, 0x06, 0xd0, 0x3b, 0x4c, 0x72, 0xb1, 0xf4, 0x7e, 0xb3, 0xe0, 0xd1, 0x59, 0x99, 0x63,
-	0xf1, 0x32, 0xce, 0xc2, 0xab, 0xc3, 0x85, 0x28, 0x28, 0x79, 0x05, 0x5b, 0x58, 0x50, 0x5e, 0x16,
-	0x32, 0xf6, 0x88, 0xa5, 0x97, 0x8a, 0xd2, 0xd1, 0xfe, 0xbc, 0x91, 0x3e, 0xad, 0x3d, 0x7b, 0x87,
-	0x7a, 0xc3, 0x81, 0xb2, 0xf7, 0xc7, 0xd8, 0x9c, 0x4e, 0xbf, 0x87, 0xf1, 0xca, 0xba, 0x14, 0x46,
-	0xa6, 0xb6, 0x91, 0x4a, 0x8d, 0xa5, 0xe2, 0x39, 0x2d, 0x98, 0x58, 0x9a, 0x12, 0x34, 0x33, 0x29,
-	0x88, 0xa9, 0x30, 0x99, 0x69, 0x1d, 0x95, 0x69, 0x8e, 0x46, 0x8e, 0x22, 0xee, 0x7d, 0x00, 0x3b,
-	0x07, 0x31, 0xc3, 0x54, 0x9c, 0x30, 0x2e, 0x30, 0xf5, 0xf1, 0xa7, 0x12, 0xb9, 0x90, 0x1e, 0x52,
-	0x9a, 0xa0, 0x29, 0x70, 0x35, 0xf6, 0x7e, 0x86, 0x2d, 0x9d, 0x3a, 0x27, 0x59, 0xa8, 0xf2, 0x46,
-	0x12, 0x23, 0x2b, 0x5b, 0x1b, 0xc9, 0x61, 0xab, 0xe4, 0xed, 0x76, 0xc9, 0x37, 0x6b, 0xa2, 0xf3,
-	0xef, 0x35, 0xd1, 0xbd, 0x5b, 0x13, 0xdf, 0xc1, 0xce, 0x49, 0x96, 0x5d, 0x95, 0xb9, 0x0e, 0xa3,
-	0x8a, 0x75, 0xf5, 0x86, 0xd6, 0xac, 0x23, 0x7d, 0xd6, 0x37, 0x6c, 0x65, 0xac, 0xdd, 0xce, 0x58,
-	0xef, 0x6f, 0x0b, 0x76, 0x57, 0x8f, 0x35, 0xd5, 0xf6, 0x03, 0xec, 0xd4, 0xe7, 0x06, 0xb1, 0xb9,
-	0xb3, 0x76, 0x30, 0xda, 0x7f, 0xd1, 0x10, 0xf3, 0xbe, 0xdd, 0x55, 0x83, 0x88, 0x2a, 0xb2, 0xfc,
-	0xed, 0xeb, 0x16, 0xc2, 0xa7, 0x0b, 0x98, 0xb4, 0xcd, 0x64, 0x42, 0xd7, 0x5e, 0x0d, 0xb3, 0xc3,
-	0x6a, 0x27, 0xf9, 0x18, 0x9c, 0xdb, 0x40, 0x6c, 0x15, 0xc8, 0xce, 0x4a, 0x20, 0xc6, 0xd7, 0xad,
-	0x15, 0xd9, 0x85, 0x1e, 0x16, 0x45, 0x56, 0x35, 0x02, 0x3d, 0xf1, 0x3e, 0x85, 0xe1, 0x7f, 0x56,
-	0xd1, 0xfb, 0xd3, 0x82, 0xf1, 0x17, 0x9c, 0xb3, 0xcb, 0x3a, 0x5d, 0x76, 0xa1, 0xa7, 0xcb, 0x54,
-	0xb7, 0x23, 0x3d, 0x21, 0x33, 0x18, 0x99, 0x2a, 0x6b, 0x50, 0xdf, 0x84, 0x1e, 0xec, 0x26, 0xa6,
-	0xf2, 0xba, 0x3a, 0x34, 0x21, 0xe2, 0x76, 0xa3, 0xef, 0xad, 0x6d, 0xf4, 0xfd, 0x46, 0xa3, 0x7f,
-	0x02, 0x8e, 0xda, 0x94, 0x66, 0x11, 0x9a, 0x17, 0x60, 0x28, 0x81, 0x6f, 0xb2, 0x08, 0xbd, 0x5f,
-	0x2d, 0xd8, 0xaa, 0x6e, 0x63, 0x94, 0x9f, 0x40, 0xe7, 0xa2, 0x66, 0x5f, 0x0e, 0x2b, 0x8e, 0xec,
-	0x75, 0x1c, 0xdd, 0x79, 0xdc, 0x6a, 0x46, 0xba, 0x4d, 0x46, 0x6a, 0x31, 0x7a, 0x0d, 0x31, 0x64,
-	0xc8, 0xb4, 0x14, 0x3f, 0x56, 0x21, 0xcb, 0xb1, 0x77, 0x09, 0xdb, 0x67, 0x82, 0x0a, 0xc6, 0x05,
-	0x0b, 0x79, 0x45, 0x73, 0x8b, 0x50, 0xeb, 0x21, 0x42, 0xed, 0x75, 0x84, 0x76, 0x6a, 0x42, 0xbd,
-	0xdf, 0x2d, 0x20, 0x4d, 0x4f, 0x86, 0x82, 0xff, 0xc1, 0x95, 0xa4, 0x4c, 0x64, 0x82, 0xc6, 0x81,
-	0x7a, 0x55, 0xcc, 0xdb, 0xa0, 0x10, 0xf9, 0x70, 0x49, 0x95, 0x4a, 0x8e, 0x91, 0x5e, 0xd5, 0x0f,
-	0xc3, 0x50, 0x02, 0x6a, 0x71, 0xf5, 0x5d, 0xe9, 0xb7, 0xde, 0x95, 0xfd, 0x5f, 0x3a, 0x30, 0x38,
-	0x43, 0x7a, 0x83, 0x18, 0x91, 0x23, 0x18, 0x9f, 0x61, 0x1a, 0xdd, 0xfe, 0x55, 0x76, 0x1b, 0x25,
-	0x52, 0xa3, 0xd3, 0x77, 0xef, 0x43, 0xab, 0xfb, 0x7b, 0x1b, 0x73, 0xeb, 0x85, 0x45, 0x4e, 0x61,
-	0x7c, 0x8c, 0x98, 0x1f, 0x64, 0x69, 0x8a, 0xa1, 0xc0, 0x88, 0x3c, 0x6b, 0x6c, 0xba, 0xa7, 0x6f,
-	0x4e, 0x1f, 0xdf, 0xf9, 0x22, 0x54, 0x65, 0x66, 0x4e, 0xfc, 0x16, 0x36, 0x9b, 0xed, 0x62, 0xe5,
-	0xc0, 0x7b, 0x9a, 0xdb, 0xf4, 0xf9, 0x03, 0x7d, 0xc6, 0xdb, 0x20, 0x9f, 0x43, 0x5f, 0xe7, 0x2f,
-	0x71, 0x1b, 0xc6, 0x2b, 0x05, 0xba, 0x12, 0xd7, 0x6a, 0xb2, 0x7b, 0x1b, 0xe4, 0x18, 0xe0, 0x36,
-	0x03, 0x48, 0x93, 0x97, 0x3b, 0x29, 0x38, 0x7d, 0xba, 0x66, 0xb5, 0x3a, 0xec, 0xbc, 0xaf, 0x3e,
-	0x8e, 0x9f, 0xfc, 0x13, 0x00, 0x00, 0xff, 0xff, 0x25, 0xfc, 0xb7, 0x54, 0x48, 0x0a, 0x00, 0x00,
+	// 1416 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xd4, 0x58, 0xcd, 0x6f, 0xdc, 0x44,
+	0x14, 0x8f, 0xbd, 0x1f, 0x59, 0xbf, 0xfd, 0xc8, 0xee, 0x24, 0x6d, 0xdd, 0x2d, 0x6d, 0xb7, 0xee,
+	0x65, 0xcb, 0x47, 0x54, 0xc2, 0x81, 0x03, 0xa0, 0xaa, 0x4d, 0x53, 0x51, 0x35, 0xd0, 0xd6, 0x69,
+	0x8b, 0x84, 0x84, 0xcc, 0xc4, 0x7e, 0x09, 0x56, 0xbc, 0xb6, 0xb1, 0x67, 0xb7, 0xd9, 0x5e, 0xb8,
+	0x70, 0x44, 0x70, 0xe0, 0xff, 0xe1, 0xc2, 0x91, 0x3f, 0x85, 0x03, 0x12, 0x77, 0x24, 0x34, 0xe3,
+	0xb1, 0x77, 0xec, 0xdd, 0xa4, 0x15, 0x12, 0x87, 0xde, 0x3c, 0xbf, 0xf7, 0x66, 0xe6, 0xcd, 0xef,
+	0x7d, 0xee, 0x42, 0x67, 0x42, 0x53, 0x86, 0xc9, 0x76, 0x9c, 0x44, 0x2c, 0x22, 0x46, 0xb6, 0x72,
+	0xe2, 0x43, 0xeb, 0x4f, 0x1d, 0x8c, 0xcf, 0x91, 0x26, 0xec, 0x10, 0x29, 0x23, 0x3d, 0xd0, 0xfd,
+	0xd8, 0xd4, 0x46, 0xda, 0xd8, 0xb0, 0x75, 0x3f, 0x26, 0x04, 0xea, 0x71, 0x94, 0x30, 0x53, 0x1f,
+	0x69, 0xe3, 0xae, 0x2d, 0xbe, 0xc9, 0x55, 0x80, 0x78, 0x7a, 0x18, 0xf8, 0xae, 0x33, 0x4d, 0x02,
+	0xb3, 0x26, 0x74, 0x8d, 0x0c, 0x79, 0x9e, 0x04, 0x64, 0x0c, 0xfd, 0x09, 0x3d, 0x75, 0x66, 0x51,
+	0x30, 0x9d, 0xa0, 0xe3, 0x46, 0xd3, 0x90, 0x99, 0x75, 0xb1, 0xbd, 0x37, 0xa1, 0xa7, 0x2f, 0x04,
+	0xbc, 0xcb, 0x51, 0x32, 0xe2, 0x56, 0x9d, 0x3a, 0x47, 0x7e, 0x80, 0xce, 0x09, 0xce, 0xcd, 0xc6,
+	0x48, 0x1b, 0xd7, 0x6d, 0x98, 0xd0, 0xd3, 0x07, 0x7e, 0x80, 0x8f, 0x70, 0x4e, 0xae, 0x43, 0xdb,
+	0xa3, 0x8c, 0x3a, 0x2e, 0x86, 0x0c, 0x13, 0xb3, 0x29, 0xee, 0x02, 0x0e, 0xed, 0x0a, 0x84, 0xdb,
+	0x97, 0x50, 0xf7, 0xc4, 0x5c, 0x17, 0x12, 0xf1, 0xcd, 0xed, 0xa3, 0xde, 0xc4, 0x0f, 0x1d, 0x61,
+	0x79, 0x4b, 0x5c, 0x6d, 0x08, 0xe4, 0x09, 0x37, 0xff, 0x33, 0x58, 0xcf, 0x6c, 0x4b, 0x4d, 0x63,
+	0x54, 0x1b, 0xb7, 0x77, 0x6e, 0x6e, 0x17, 0x6c, 0x6c, 0x67, 0xe6, 0x3d, 0x0c, 0x8f, 0xa2, 0x64,
+	0x42, 0x99, 0x1f, 0x85, 0x5f, 0x60, 0x9a, 0xd2, 0x63, 0xb4, 0xf3, 0x3d, 0xe4, 0x32, 0xb4, 0x42,
+	0x7c, 0xe9, 0xcc, 0x7c, 0x2f, 0x35, 0x61, 0x54, 0x1b, 0x77, 0xed, 0xf5, 0x10, 0x5f, 0xbe, 0xf0,
+	0xbd, 0x94, 0xdc, 0x80, 0x8e, 0x87, 0x01, 0x32, 0xf4, 0x32, 0x71, 0x5b, 0x88, 0xdb, 0x12, 0xe3,
+	0x2a, 0xd6, 0x73, 0x18, 0x14, 0x64, 0xdb, 0x98, 0xc6, 0x51, 0x98, 0x22, 0x19, 0xc3, 0x46, 0x76,
+	0xfa, 0x81, 0xff, 0x0a, 0xf7, 0xfd, 0x89, 0xcf, 0x84, 0x07, 0xea, 0x76, 0x15, 0x26, 0x17, 0xa1,
+	0x19, 0x20, 0xf5, 0x30, 0x91, 0xb4, 0xcb, 0x95, 0xf5, 0x97, 0x0e, 0xe6, 0x59, 0xa6, 0x0b, 0x9f,
+	0x7a, 0xe2, 0xc4, 0xae, 0xad, 0xfb, 0x1e, 0xe7, 0x2c, 0xf5, 0x5f, 0xa1, 0xf0, 0x69, 0xdd, 0x16,
+	0xdf, 0xe4, 0x1a, 0x80, 0x1b, 0x05, 0x01, 0xba, 0x7c, 0xa3, 0x3c, 0x5c, 0x41, 0x38, 0xa7, 0xc2,
+	0x4d, 0x0b, 0x77, 0xd6, 0x6d, 0x83, 0x23, 0x99, 0x27, 0x8b, 0x97, 0x4b, 0x85, 0xcc, 0x93, 0xf2,
+	0xe5, 0x99, 0xca, 0xfb, 0x40, 0x72, 0x72, 0x0e, 0xe7, 0x85, 0x62, 0x53, 0x28, 0xf6, 0xa5, 0xe4,
+	0xde, 0x3c, 0xd7, 0xbe, 0x02, 0x46, 0x82, 0xd4, 0x73, 0xa2, 0x30, 0x98, 0x0b, 0xe7, 0xb6, 0xec,
+	0x16, 0x07, 0x1e, 0x87, 0xc1, 0x9c, 0xbc, 0x07, 0x83, 0x04, 0xe3, 0xc0, 0x77, 0xa9, 0x13, 0x07,
+	0xd4, 0xc5, 0x09, 0x86, 0xb9, 0x9f, 0xfb, 0x52, 0xf0, 0x24, 0xc7, 0x89, 0x09, 0xeb, 0x33, 0x4c,
+	0x52, 0xfe, 0x2c, 0x43, 0xa8, 0xe4, 0x4b, 0xd2, 0x87, 0x1a, 0x63, 0x81, 0x09, 0x02, 0xe5, 0x9f,
+	0xe4, 0x16, 0xf4, 0xdd, 0x68, 0x12, 0x53, 0x97, 0x39, 0x09, 0xce, 0x7c, 0xb1, 0xa9, 0x2d, 0xc4,
+	0x1b, 0x12, 0xb7, 0x25, 0x6c, 0xad, 0x43, 0x63, 0x6f, 0x12, 0xb3, 0xb9, 0xf5, 0x9b, 0x06, 0x1b,
+	0x07, 0xd3, 0x18, 0x93, 0x7b, 0x41, 0xe4, 0x9e, 0xec, 0x9d, 0xb2, 0x84, 0x92, 0xc7, 0xd0, 0xc3,
+	0x84, 0xa6, 0xd3, 0x84, 0x3f, 0xd3, 0xf3, 0xc3, 0x63, 0xc1, 0x7e, 0x7b, 0x67, 0xac, 0x44, 0x5a,
+	0x65, 0xcf, 0xf6, 0x5e, 0xb6, 0x61, 0x57, 0xe8, 0xdb, 0x5d, 0x54, 0x97, 0xc3, 0xaf, 0xa1, 0x5b,
+	0x92, 0x73, 0x1f, 0xf2, 0x2c, 0x90, 0x5e, 0x15, 0xdf, 0x3c, 0x38, 0x62, 0x9a, 0xf8, 0x6c, 0x2e,
+	0xb3, 0x55, 0xae, 0xb8, 0xef, 0x64, 0x32, 0xf2, 0xa0, 0xac, 0x89, 0xa0, 0x34, 0x32, 0xe4, 0xa1,
+	0x97, 0x5a, 0xb7, 0x60, 0x73, 0x37, 0xf0, 0x31, 0x64, 0xfb, 0x7e, 0xca, 0x30, 0xb4, 0xf1, 0xfb,
+	0x29, 0xa6, 0x8c, 0xdf, 0x10, 0xd2, 0x09, 0xca, 0x5a, 0x20, 0xbe, 0xad, 0x1f, 0xa0, 0x97, 0x45,
+	0xd9, 0x7e, 0xe4, 0x8a, 0x10, 0xe3, 0x1c, 0xf2, 0x22, 0x90, 0x29, 0xf1, 0xcf, 0x4a, 0x75, 0xd0,
+	0xab, 0xd5, 0x41, 0x4d, 0x9f, 0xda, 0xf9, 0xe9, 0x53, 0x5f, 0x4e, 0x9f, 0x67, 0xb0, 0xb9, 0x1f,
+	0x45, 0x27, 0xd3, 0x38, 0x33, 0x23, 0xb7, 0xb5, 0xfc, 0x42, 0x6d, 0x54, 0xe3, 0x77, 0x16, 0x2f,
+	0xac, 0x04, 0xb7, 0x5e, 0x0d, 0x6e, 0xeb, 0x6f, 0x0d, 0xb6, 0xca, 0xc7, 0xca, 0xc4, 0xfc, 0x16,
+	0x36, 0x8b, 0x73, 0x9d, 0x40, 0xbe, 0x39, 0xbb, 0xa0, 0xbd, 0x73, 0x5b, 0x71, 0xe6, 0xaa, 0xdd,
+	0x79, 0x2d, 0xf1, 0x72, 0xb2, 0xec, 0xc1, 0xac, 0x82, 0xa4, 0xc3, 0x53, 0xe8, 0x57, 0xd5, 0x78,
+	0xec, 0x17, 0xb7, 0x4a, 0x66, 0x5b, 0xf9, 0x4e, 0xf2, 0x21, 0x18, 0x0b, 0x43, 0x74, 0x61, 0xc8,
+	0x66, 0xc9, 0x10, 0x79, 0xd7, 0x42, 0x8b, 0x6c, 0x41, 0x03, 0x93, 0x24, 0xca, 0x6b, 0x46, 0xb6,
+	0xb0, 0x3e, 0x81, 0xd6, 0x7f, 0xf6, 0xa2, 0xf5, 0x87, 0x06, 0xdd, 0xbb, 0x69, 0xea, 0x1f, 0x17,
+	0xe1, 0xb2, 0x05, 0x8d, 0x2c, 0xa3, 0xb3, 0xca, 0x95, 0x2d, 0xc8, 0x08, 0xda, 0x32, 0x21, 0x15,
+	0xea, 0x55, 0xe8, 0xb5, 0x85, 0x47, 0x26, 0x69, 0x3d, 0x33, 0x8d, 0x27, 0x69, 0xa5, 0x27, 0x34,
+	0xce, 0xec, 0x09, 0x4d, 0xa5, 0x27, 0x5c, 0x01, 0x43, 0x6c, 0x0a, 0x23, 0x0f, 0x65, 0xb3, 0x68,
+	0x71, 0xe0, 0xcb, 0xc8, 0x43, 0xeb, 0x57, 0x0d, 0x7a, 0xf9, 0x6b, 0xa4, 0xe7, 0xfb, 0x50, 0x3b,
+	0x2a, 0xd8, 0xe7, 0x9f, 0x39, 0x47, 0xfa, 0x59, 0x1c, 0x2d, 0xf5, 0xc1, 0x82, 0x91, 0xba, 0xca,
+	0x48, 0xe1, 0x8c, 0x86, 0xe2, 0x0c, 0x6e, 0x32, 0x9d, 0xb2, 0xef, 0x72, 0x93, 0xf9, 0xb7, 0x75,
+	0x0c, 0x83, 0x03, 0x46, 0x99, 0x9f, 0x32, 0xdf, 0x4d, 0x73, 0x9a, 0x2b, 0x84, 0x6a, 0xaf, 0x23,
+	0x54, 0x3f, 0x8b, 0xd0, 0x5a, 0x41, 0xa8, 0xf5, 0xbb, 0x06, 0x44, 0xbd, 0x49, 0x52, 0xf0, 0x3f,
+	0x5c, 0xc5, 0x29, 0x63, 0x11, 0xa3, 0x81, 0x23, 0x1a, 0x90, 0x6c, 0x23, 0x02, 0xe1, 0x3d, 0x8e,
+	0x7b, 0x69, 0x9a, 0xa2, 0x97, 0x49, 0xb3, 0x1e, 0xd2, 0xe2, 0x80, 0x10, 0x96, 0x5b, 0x50, 0xb3,
+	0xd2, 0x82, 0xac, 0xbb, 0xd0, 0x3e, 0x60, 0x51, 0x42, 0x8f, 0xf1, 0xd9, 0x3c, 0x7e, 0x13, 0xeb,
+	0xa5, 0x75, 0xfa, 0x82, 0x88, 0x11, 0xc0, 0xee, 0xc2, 0xfa, 0x55, 0x05, 0xf0, 0x12, 0x5c, 0x58,
+	0x68, 0xf0, 0x7a, 0x29, 0xfd, 0x62, 0x3d, 0x85, 0x8b, 0x55, 0x81, 0xa4, 0xf1, 0x63, 0x68, 0x2f,
+	0x28, 0xc9, 0x6b, 0xc7, 0x05, 0x25, 0x65, 0x17, 0xfb, 0x6c, 0x55, 0xd3, 0xfa, 0x00, 0x2e, 0x2d,
+	0x44, 0xf7, 0x45, 0x11, 0x3c, 0xaf, 0x36, 0x0f, 0xc1, 0x5c, 0x56, 0xcf, 0x6c, 0xb0, 0x7e, 0xd1,
+	0xa1, 0x73, 0x5f, 0x46, 0x3b, 0x1f, 0x10, 0x94, 0x91, 0xc0, 0x10, 0x23, 0xc1, 0x0d, 0xe8, 0x94,
+	0xe6, 0xb5, 0x6c, 0x34, 0x68, 0xcf, 0x94, 0x61, 0x6d, 0xd5, 0x58, 0x57, 0x13, 0x6a, 0xd5, 0xb1,
+	0xee, 0x5d, 0x18, 0x1c, 0x25, 0x88, 0xcb, 0x13, 0x60, 0xdd, 0xde, 0xe0, 0x02, 0x55, 0x77, 0x1b,
+	0x36, 0xa9, 0xcb, 0xfc, 0x59, 0x45, 0x3b, 0xf3, 0xfd, 0x20, 0x13, 0xa9, 0xfa, 0x0f, 0x0a, 0x43,
+	0xfd, 0xf0, 0x28, 0x4a, 0xcd, 0xe6, 0x9b, 0x4f, 0x70, 0xf2, 0x35, 0x5c, 0x92, 0x5a, 0x3f, 0xea,
+	0xd0, 0xb2, 0xa9, 0x7b, 0xf2, 0x76, 0xb3, 0x71, 0x07, 0x36, 0x8a, 0xaa, 0x56, 0x22, 0xe4, 0x92,
+	0x42, 0x88, 0xea, 0x78, 0xbb, 0xeb, 0x29, 0xab, 0xd4, 0xfa, 0x47, 0x83, 0xde, 0xfd, 0xa2, 0x72,
+	0xbe, 0xdd, 0x64, 0xec, 0x00, 0xf0, 0x52, 0x5f, 0xe2, 0x41, 0x6d, 0x8d, 0xb9, 0xbb, 0x6d, 0x23,
+	0x91, 0x5f, 0xa9, 0xf5, 0xb3, 0x0e, 0x9d, 0x67, 0x51, 0x1c, 0x05, 0xd1, 0xf1, 0xfc, 0xed, 0x7e,
+	0xfd, 0x1e, 0x0c, 0x94, 0xae, 0x58, 0x22, 0xe1, 0x72, 0x25, 0x18, 0x16, 0xce, 0xb6, 0x37, 0xbc,
+	0xd2, 0x3a, 0xb5, 0x36, 0x61, 0x20, 0x27, 0x3c, 0xa5, 0xb8, 0xd9, 0x40, 0x54, 0x50, 0x16, 0xb6,
+	0x4f, 0xa1, 0xcb, 0x24, 0x75, 0xe2, 0x3a, 0x39, 0xe3, 0xaa, 0xa1, 0xa7, 0x52, 0x6b, 0x77, 0x98,
+	0xb2, 0xda, 0xf9, 0xa9, 0x01, 0xeb, 0x07, 0x48, 0x5f, 0x22, 0x7a, 0xe4, 0x21, 0x74, 0x0f, 0x30,
+	0xf4, 0x16, 0xbf, 0x42, 0xb7, 0x94, 0x33, 0x0a, 0x74, 0xf8, 0xce, 0x2a, 0xb4, 0xa8, 0x71, 0x6b,
+	0x63, 0xed, 0xb6, 0x46, 0x9e, 0x40, 0xf7, 0x11, 0x62, 0xbc, 0x1b, 0x85, 0x21, 0xba, 0x0c, 0x3d,
+	0x72, 0x4d, 0xad, 0xb4, 0xcb, 0x63, 0xee, 0xf0, 0xf2, 0x52, 0xe9, 0xc8, 0xa7, 0x22, 0x79, 0xe2,
+	0x53, 0xe8, 0xa8, 0xd3, 0x5d, 0xe9, 0xc0, 0x15, 0xb3, 0xe8, 0xf0, 0xfa, 0x6b, 0xc6, 0x42, 0x6b,
+	0x8d, 0xdc, 0x81, 0x66, 0x36, 0x6e, 0x10, 0x53, 0x51, 0x2e, 0xcd, 0x53, 0x25, 0xbb, 0xca, 0xb3,
+	0x89, 0xb5, 0x46, 0x1e, 0x01, 0x2c, 0x1a, 0x36, 0x51, 0x79, 0x59, 0x9a, 0x18, 0x86, 0x57, 0xcf,
+	0x90, 0x16, 0x87, 0x7d, 0x05, 0xbd, 0x72, 0xeb, 0x22, 0xa3, 0x95, 0xdd, 0x49, 0x89, 0x88, 0xe1,
+	0x8d, 0x73, 0x34, 0x8a, 0x83, 0xbf, 0x81, 0x7e, 0xb5, 0x23, 0x11, 0x6b, 0xe5, 0xc6, 0x52, 0x77,
+	0x1b, 0xde, 0x3c, 0x57, 0x47, 0x25, 0x61, 0x11, 0x95, 0x25, 0x12, 0x96, 0x22, 0xb8, 0x44, 0xc2,
+	0x72, 0x28, 0x5b, 0x6b, 0x87, 0x4d, 0xf1, 0xbf, 0xc8, 0x47, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff,
+	0x88, 0xac, 0x76, 0x9c, 0x27, 0x11, 0x00, 0x00,
 }

weed/pb/volume_server.proto

@@ -24,9 +24,7 @@ service VolumeServer {
 
     rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
     }
-    rpc VolumeSyncIndex (VolumeSyncIndexRequest) returns (stream VolumeSyncIndexResponse) {
-    }
-    rpc VolumeSyncData (VolumeSyncDataRequest) returns (stream VolumeSyncDataResponse) {
+    rpc VolumeFollow (VolumeFollowRequest) returns (stream VolumeFollowResponse) {
     }
 
     rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
@@ -36,7 +34,12 @@ service VolumeServer {
     rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
     }
 
-    // rpc VolumeUiPage (VolumeUiPageRequest) returns (VolumeUiPageResponse) {}
+    rpc ReplicateVolume (ReplicateVolumeRequest) returns (ReplicateVolumeResponse) {
+    }
+    rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
+    }
+    rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
+    }
 
 }
 
@@ -60,27 +63,27 @@ message Empty {
 }
 
 message VacuumVolumeCheckRequest {
-    uint32 volumd_id = 1;
+    uint32 volume_id = 1;
 }
 message VacuumVolumeCheckResponse {
     double garbage_ratio = 1;
 }
 
 message VacuumVolumeCompactRequest {
-    uint32 volumd_id = 1;
+    uint32 volume_id = 1;
     int64 preallocate = 2;
 }
 message VacuumVolumeCompactResponse {
 }
 
 message VacuumVolumeCommitRequest {
-    uint32 volumd_id = 1;
+    uint32 volume_id = 1;
 }
 message VacuumVolumeCommitResponse {
 }
 
 message VacuumVolumeCleanupRequest {
-    uint32 volumd_id = 1;
+    uint32 volume_id = 1;
 }
 message VacuumVolumeCleanupResponse {
 }
@@ -92,7 +95,7 @@ message DeleteCollectionResponse {
 }
 
 message AssignVolumeRequest {
-    uint32 volumd_id = 1;
+    uint32 volume_id = 1;
     string collection = 2;
     int64 preallocate = 3;
     string replication = 4;
@@ -102,10 +105,10 @@ message AssignVolumeResponse {
 }
 
 message VolumeSyncStatusRequest {
-    uint32 volumd_id = 1;
+    uint32 volume_id = 1;
 }
 message VolumeSyncStatusResponse {
-    uint32 volumd_id = 1;
+    uint32 volume_id = 1;
     string collection = 2;
     string replication = 4;
     string ttl = 5;
@@ -114,45 +117,60 @@ message VolumeSyncStatusResponse {
     uint64 idx_file_size = 8;
 }
 
-message VolumeSyncIndexRequest {
-    uint32 volumd_id = 1;
+message VolumeFollowRequest {
+    uint32 volume_id = 1;
+    uint64 since = 2;
 }
-message VolumeSyncIndexResponse {
-    bytes index_file_content = 1;
-}
-
-message VolumeSyncDataRequest {
-    uint32 volumd_id = 1;
-    uint32 revision = 2;
-    uint32 offset = 3;
-    uint32 size = 4;
-    string needle_id = 5;
-}
-message VolumeSyncDataResponse {
+message VolumeFollowResponse {
     bytes file_content = 1;
 }
 
 message VolumeMountRequest {
-    uint32 volumd_id = 1;
+    uint32 volume_id = 1;
 }
 message VolumeMountResponse {
 }
 
 message VolumeUnmountRequest {
-    uint32 volumd_id = 1;
+    uint32 volume_id = 1;
 }
 message VolumeUnmountResponse {
 }
 
 message VolumeDeleteRequest {
-    uint32 volumd_id = 1;
+    uint32 volume_id = 1;
 }
 message VolumeDeleteResponse {
 }
 
-message VolumeUiPageRequest {
+message ReplicateVolumeRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    string replication = 3;
+    string ttl = 4;
+    string source_data_node = 5;
 }
-message VolumeUiPageResponse {
+message ReplicateVolumeResponse {
+}
+
+message CopyFileRequest {
+    uint32 volume_id = 1;
+    bool is_idx_file = 2;
+    bool is_dat_file = 3;
+}
+message CopyFileResponse {
+    bytes file_content = 1;
+}
+
+message ReadVolumeFileStatusRequest {
+    uint32 volume_id = 1;
+}
+message ReadVolumeFileStatusResponse {
+    uint32 volume_id = 1;
+    uint64 idx_file_timestamp = 2;
+    uint64 idx_file_size = 3;
+    uint64 dat_file_timestamp = 4;
+    uint64 dat_file_size = 5;
 }
 
 message DiskStatus {

weed/replication/replicator.go

@@ -1,6 +1,7 @@
 package replication
 
 import (
+	"context"
 	"path/filepath"
 	"strings"
 
@@ -29,33 +30,33 @@ func NewReplicator(sourceConfig util.Configuration, dataSink sink.ReplicationSin
 	}
 }
 
-func (r *Replicator) Replicate(key string, message *filer_pb.EventNotification) error {
+func (r *Replicator) Replicate(ctx context.Context, key string, message *filer_pb.EventNotification) error {
 	if !strings.HasPrefix(key, r.source.Dir) {
 		glog.V(4).Infof("skipping %v outside of %v", key, r.source.Dir)
 		return nil
 	}
-	newKey := filepath.Join(r.sink.GetSinkToDirectory(), key[len(r.source.Dir):])
+	newKey := filepath.ToSlash(filepath.Join(r.sink.GetSinkToDirectory(), key[len(r.source.Dir):]))
 	glog.V(3).Infof("replicate %s => %s", key, newKey)
 	key = newKey
 	if message.OldEntry != nil && message.NewEntry == nil {
 		glog.V(4).Infof("deleting %v", key)
-		return r.sink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks)
+		return r.sink.DeleteEntry(ctx, key, message.OldEntry.IsDirectory, message.DeleteChunks)
 	}
 	if message.OldEntry == nil && message.NewEntry != nil {
 		glog.V(4).Infof("creating %v", key)
-		return r.sink.CreateEntry(key, message.NewEntry)
+		return r.sink.CreateEntry(ctx, key, message.NewEntry)
 	}
 	if message.OldEntry == nil && message.NewEntry == nil {
 		glog.V(0).Infof("weird message %+v", message)
 		return nil
 	}
 
-	foundExisting, err := r.sink.UpdateEntry(key, message.OldEntry, message.NewEntry, message.DeleteChunks)
+	foundExisting, err := r.sink.UpdateEntry(ctx, key, message.OldEntry, message.NewEntry, message.DeleteChunks)
 	if foundExisting {
 		glog.V(4).Infof("updated %v", key)
 		return err
 	}
 
 	glog.V(4).Infof("creating missing %v", key)
-	return r.sink.CreateEntry(key, message.NewEntry)
+	return r.sink.CreateEntry(ctx, key, message.NewEntry)
 }

weed/replication/sink/azuresink/azure_sink.go

@@ -70,7 +70,7 @@ func (g *AzureSink) initialize(accountName, accountKey, container, dir string) e
 	return nil
 }
 
-func (g *AzureSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool) error {
+func (g *AzureSink) DeleteEntry(ctx context.Context, key string, isDirectory, deleteIncludeChunks bool) error {
 
 	key = cleanKey(key)
 
@@ -78,8 +78,6 @@ func (g *AzureSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks boo
 		key = key + "/"
 	}
 
-	ctx := context.Background()
-
 	if _, err := g.containerURL.NewBlobURL(key).Delete(ctx,
 		azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{}); err != nil {
 		return fmt.Errorf("azure delete %s/%s: %v", g.container, key, err)
@@ -89,7 +87,7 @@ func (g *AzureSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks boo
 
 }
 
-func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry) error {
+func (g *AzureSink) CreateEntry(ctx context.Context, key string, entry *filer_pb.Entry) error {
 
 	key = cleanKey(key)
 
@@ -100,8 +98,6 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry) error {
 	totalSize := filer2.TotalSize(entry.Chunks)
 	chunkViews := filer2.ViewFromChunks(entry.Chunks, 0, int(totalSize))
 
-	ctx := context.Background()
-
 	// Create a URL that references a to-be-created blob in your
 	// Azure Storage account's container.
 	appendBlobURL := g.containerURL.NewAppendBlobURL(key)
@@ -113,7 +109,7 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry) error {
 
 	for _, chunk := range chunkViews {
 
-		fileUrl, err := g.filerSource.LookupFileId(chunk.FileId)
+		fileUrl, err := g.filerSource.LookupFileId(ctx, chunk.FileId)
 		if err != nil {
 			return err
 		}
@@ -136,7 +132,7 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry) error {
 
 }
 
-func (g *AzureSink) UpdateEntry(key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
+func (g *AzureSink) UpdateEntry(ctx context.Context, key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
 	key = cleanKey(key)
 	// TODO improve efficiency
 	return false, nil

weed/replication/sink/b2sink/b2_sink.go

@@ -58,7 +58,7 @@ func (g *B2Sink) initialize(accountId, accountKey, bucket, dir string) error {
 	return nil
 }
 
-func (g *B2Sink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool) error {
+func (g *B2Sink) DeleteEntry(ctx context.Context, key string, isDirectory, deleteIncludeChunks bool) error {
 
 	key = cleanKey(key)
 
@@ -66,8 +66,6 @@ func (g *B2Sink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool)
 		key = key + "/"
 	}
 
-	ctx := context.Background()
-
 	bucket, err := g.client.Bucket(ctx, g.bucket)
 	if err != nil {
 		return err
@@ -79,7 +77,7 @@ func (g *B2Sink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool)
 
 }
 
-func (g *B2Sink) CreateEntry(key string, entry *filer_pb.Entry) error {
+func (g *B2Sink) CreateEntry(ctx context.Context, key string, entry *filer_pb.Entry) error {
 
 	key = cleanKey(key)
 
@@ -90,8 +88,6 @@ func (g *B2Sink) CreateEntry(key string, entry *filer_pb.Entry) error {
 	totalSize := filer2.TotalSize(entry.Chunks)
 	chunkViews := filer2.ViewFromChunks(entry.Chunks, 0, int(totalSize))
 
-	ctx := context.Background()
-
 	bucket, err := g.client.Bucket(ctx, g.bucket)
 	if err != nil {
 		return err
@@ -102,7 +98,7 @@ func (g *B2Sink) CreateEntry(key string, entry *filer_pb.Entry) error {
 
 	for _, chunk := range chunkViews {
 
-		fileUrl, err := g.filerSource.LookupFileId(chunk.FileId)
+		fileUrl, err := g.filerSource.LookupFileId(ctx, chunk.FileId)
 		if err != nil {
 			return err
 		}
@@ -128,7 +124,7 @@ func (g *B2Sink) CreateEntry(key string, entry *filer_pb.Entry) error {
 
 }
 
-func (g *B2Sink) UpdateEntry(key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
+func (g *B2Sink) UpdateEntry(ctx context.Context, key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
 
 	key = cleanKey(key)
 

weed/replication/sink/filersink/fetch_write.go

@@ -3,6 +3,7 @@ package filersink
 import (
 	"context"
 	"fmt"
+	"google.golang.org/grpc"
 	"strings"
 	"sync"
 
@@ -13,7 +14,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
-func (fs *FilerSink) replicateChunks(sourceChunks []*filer_pb.FileChunk) (replicatedChunks []*filer_pb.FileChunk, err error) {
+func (fs *FilerSink) replicateChunks(ctx context.Context, sourceChunks []*filer_pb.FileChunk) (replicatedChunks []*filer_pb.FileChunk, err error) {
 	if len(sourceChunks) == 0 {
 		return
 	}
@@ -22,7 +23,7 @@ func (fs *FilerSink) replicateChunks(sourceChunks []*filer_pb.FileChunk) (replic
 		wg.Add(1)
 		go func(chunk *filer_pb.FileChunk) {
 			defer wg.Done()
-			replicatedChunk, e := fs.replicateOneChunk(chunk)
+			replicatedChunk, e := fs.replicateOneChunk(ctx, chunk)
 			if e != nil {
 				err = e
 			}
@@ -34,9 +35,9 @@ func (fs *FilerSink) replicateChunks(sourceChunks []*filer_pb.FileChunk) (replic
 	return
 }
 
-func (fs *FilerSink) replicateOneChunk(sourceChunk *filer_pb.FileChunk) (*filer_pb.FileChunk, error) {
+func (fs *FilerSink) replicateOneChunk(ctx context.Context, sourceChunk *filer_pb.FileChunk) (*filer_pb.FileChunk, error) {
 
-	fileId, err := fs.fetchAndWrite(sourceChunk)
+	fileId, err := fs.fetchAndWrite(ctx, sourceChunk)
 	if err != nil {
 		return nil, fmt.Errorf("copy %s: %v", sourceChunk.FileId, err)
 	}
@@ -51,9 +52,9 @@ func (fs *FilerSink) replicateOneChunk(sourceChunk *filer_pb.FileChunk) (*filer_
 	}, nil
 }
 
-func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk) (fileId string, err error) {
+func (fs *FilerSink) fetchAndWrite(ctx context.Context, sourceChunk *filer_pb.FileChunk) (fileId string, err error) {
 
-	filename, header, readCloser, err := fs.filerSource.ReadPart(sourceChunk.FileId)
+	filename, header, readCloser, err := fs.filerSource.ReadPart(ctx, sourceChunk.FileId)
 	if err != nil {
 		return "", fmt.Errorf("read part %s: %v", sourceChunk.FileId, err)
 	}
@@ -62,7 +63,7 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk) (fileId stri
 	var host string
 	var auth security.EncodedJwt
 
-	if err := fs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	if err := fs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.AssignVolumeRequest{
 			Count:       1,
@@ -72,7 +73,7 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk) (fileId stri
 			DataCenter:  fs.dataCenter,
 		}
 
-		resp, err := client.AssignVolume(context.Background(), request)
+		resp, err := client.AssignVolume(ctx, request)
 		if err != nil {
 			glog.V(0).Infof("assign volume failure %v: %v", request, err)
 			return err
@@ -103,17 +104,13 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk) (fileId stri
 	return
 }
 
-func (fs *FilerSink) withFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
+func (fs *FilerSink) withFilerClient(ctx context.Context, fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	grpcConnection, err := util.GrpcDial(fs.grpcAddress, fs.grpcDialOption)
-	if err != nil {
-		return fmt.Errorf("fail to dial %s: %v", fs.grpcAddress, err)
-	}
-	defer grpcConnection.Close()
+	return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
+		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+		return fn(client)
+	}, fs.grpcAddress, fs.grpcDialOption)
 
-	client := filer_pb.NewSeaweedFilerClient(grpcConnection)
-
-	return fn(client)
 }
 
 func volumeId(fileId string) string {

weed/replication/sink/filersink/filer_sink.go

@@ -63,8 +63,8 @@ func (fs *FilerSink) initialize(grpcAddress string, dir string,
 	return nil
 }
 
-func (fs *FilerSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool) error {
-	return fs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+func (fs *FilerSink) DeleteEntry(ctx context.Context, key string, isDirectory, deleteIncludeChunks bool) error {
+	return fs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		dir, name := filer2.FullPath(key).DirAndName()
 
@@ -75,7 +75,7 @@ func (fs *FilerSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bo
 		}
 
 		glog.V(1).Infof("delete entry: %v", request)
-		_, err := client.DeleteEntry(context.Background(), request)
+		_, err := client.DeleteEntry(ctx, request)
 		if err != nil {
 			glog.V(0).Infof("delete entry %s: %v", key, err)
 			return fmt.Errorf("delete entry %s: %v", key, err)
@@ -85,12 +85,11 @@ func (fs *FilerSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bo
 	})
 }
 
-func (fs *FilerSink) CreateEntry(key string, entry *filer_pb.Entry) error {
+func (fs *FilerSink) CreateEntry(ctx context.Context, key string, entry *filer_pb.Entry) error {
 
-	return fs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	return fs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		dir, name := filer2.FullPath(key).DirAndName()
-		ctx := context.Background()
 
 		// look up existing entry
 		lookupRequest := &filer_pb.LookupDirectoryEntryRequest{
@@ -105,7 +104,7 @@ func (fs *FilerSink) CreateEntry(key string, entry *filer_pb.Entry) error {
 			}
 		}
 
-		replicatedChunks, err := fs.replicateChunks(entry.Chunks)
+		replicatedChunks, err := fs.replicateChunks(ctx, entry.Chunks)
 
 		if err != nil {
 			glog.V(0).Infof("replicate entry chunks %s: %v", key, err)
@@ -134,15 +133,13 @@ func (fs *FilerSink) CreateEntry(key string, entry *filer_pb.Entry) error {
 	})
 }
 
-func (fs *FilerSink) UpdateEntry(key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
-
-	ctx := context.Background()
+func (fs *FilerSink) UpdateEntry(ctx context.Context, key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
 
 	dir, name := filer2.FullPath(key).DirAndName()
 
 	// read existing entry
 	var existingEntry *filer_pb.Entry
-	err = fs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err = fs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.LookupDirectoryEntryRequest{
 			Directory: dir,
@@ -186,7 +183,7 @@ func (fs *FilerSink) UpdateEntry(key string, oldEntry, newEntry *filer_pb.Entry,
 		}
 
 		// replicate the chunks that are new in the source
-		replicatedChunks, err := fs.replicateChunks(newChunks)
+		replicatedChunks, err := fs.replicateChunks(ctx, newChunks)
 		if err != nil {
 			return true, fmt.Errorf("replicte %s chunks error: %v", key, err)
 		}
@@ -194,7 +191,7 @@ func (fs *FilerSink) UpdateEntry(key string, oldEntry, newEntry *filer_pb.Entry,
 	}
 
 	// save updated meta data
-	return true, fs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	return true, fs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.UpdateEntryRequest{
 			Directory: dir,

weed/replication/sink/gcssink/gcs_sink.go

@@ -69,13 +69,13 @@ func (g *GcsSink) initialize(google_application_credentials, bucketName, dir str
 	return nil
 }
 
-func (g *GcsSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool) error {
+func (g *GcsSink) DeleteEntry(ctx context.Context, key string, isDirectory, deleteIncludeChunks bool) error {
 
 	if isDirectory {
 		key = key + "/"
 	}
 
-	if err := g.client.Bucket(g.bucket).Object(key).Delete(context.Background()); err != nil {
+	if err := g.client.Bucket(g.bucket).Object(key).Delete(ctx); err != nil {
 		return fmt.Errorf("gcs delete %s%s: %v", g.bucket, key, err)
 	}
 
@@ -83,7 +83,7 @@ func (g *GcsSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool)
 
 }
 
-func (g *GcsSink) CreateEntry(key string, entry *filer_pb.Entry) error {
+func (g *GcsSink) CreateEntry(ctx context.Context, key string, entry *filer_pb.Entry) error {
 
 	if entry.IsDirectory {
 		return nil
@@ -92,13 +92,11 @@ func (g *GcsSink) CreateEntry(key string, entry *filer_pb.Entry) error {
 	totalSize := filer2.TotalSize(entry.Chunks)
 	chunkViews := filer2.ViewFromChunks(entry.Chunks, 0, int(totalSize))
 
-	ctx := context.Background()
-
 	wc := g.client.Bucket(g.bucket).Object(key).NewWriter(ctx)
 
 	for _, chunk := range chunkViews {
 
-		fileUrl, err := g.filerSource.LookupFileId(chunk.FileId)
+		fileUrl, err := g.filerSource.LookupFileId(ctx, chunk.FileId)
 		if err != nil {
 			return err
 		}
@@ -121,7 +119,7 @@ func (g *GcsSink) CreateEntry(key string, entry *filer_pb.Entry) error {
 
 }
 
-func (g *GcsSink) UpdateEntry(key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
+func (g *GcsSink) UpdateEntry(ctx context.Context, key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
 	// TODO improve efficiency
 	return false, nil
 }

weed/replication/sink/replication_sink.go

@@ -1,6 +1,7 @@
 package sink
 
 import (
+	"context"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/replication/source"
 	"github.com/chrislusf/seaweedfs/weed/util"
@@ -9,9 +10,9 @@ import (
 type ReplicationSink interface {
 	GetName() string
 	Initialize(configuration util.Configuration) error
-	DeleteEntry(key string, isDirectory, deleteIncludeChunks bool) error
-	CreateEntry(key string, entry *filer_pb.Entry) error
-	UpdateEntry(key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error)
+	DeleteEntry(ctx context.Context, key string, isDirectory, deleteIncludeChunks bool) error
+	CreateEntry(ctx context.Context, key string, entry *filer_pb.Entry) error
+	UpdateEntry(ctx context.Context, key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error)
 	GetSinkToDirectory() string
 	SetSourceFiler(s *source.FilerSource)
 }

weed/replication/sink/s3sink/s3_sink.go

@@ -1,6 +1,7 @@
 package S3Sink
 
 import (
+	"context"
 	"fmt"
 	"strings"
 	"sync"
@@ -76,7 +77,7 @@ func (s3sink *S3Sink) initialize(awsAccessKeyId, aswSecretAccessKey, region, buc
 	return nil
 }
 
-func (s3sink *S3Sink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool) error {
+func (s3sink *S3Sink) DeleteEntry(ctx context.Context, key string, isDirectory, deleteIncludeChunks bool) error {
 
 	key = cleanKey(key)
 
@@ -88,7 +89,7 @@ func (s3sink *S3Sink) DeleteEntry(key string, isDirectory, deleteIncludeChunks b
 
 }
 
-func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry) error {
+func (s3sink *S3Sink) CreateEntry(ctx context.Context, key string, entry *filer_pb.Entry) error {
 
 	key = cleanKey(key)
 
@@ -111,7 +112,7 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry) error {
 		wg.Add(1)
 		go func(chunk *filer2.ChunkView) {
 			defer wg.Done()
-			if part, uploadErr := s3sink.uploadPart(key, uploadId, partId, chunk); uploadErr != nil {
+			if part, uploadErr := s3sink.uploadPart(ctx, key, uploadId, partId, chunk); uploadErr != nil {
 				err = uploadErr
 			} else {
 				parts = append(parts, part)
@@ -125,11 +126,11 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry) error {
 		return err
 	}
 
-	return s3sink.completeMultipartUpload(key, uploadId, parts)
+	return s3sink.completeMultipartUpload(ctx, key, uploadId, parts)
 
 }
 
-func (s3sink *S3Sink) UpdateEntry(key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
+func (s3sink *S3Sink) UpdateEntry(ctx context.Context, key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
 	key = cleanKey(key)
 	// TODO improve efficiency
 	return false, nil

weed/replication/sink/s3sink/s3_write.go

@@ -2,6 +2,7 @@ package S3Sink
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 
@@ -81,7 +82,7 @@ func (s3sink *S3Sink) abortMultipartUpload(key, uploadId string) error {
 }
 
 // To complete multipart upload
-func (s3sink *S3Sink) completeMultipartUpload(key, uploadId string, parts []*s3.CompletedPart) error {
+func (s3sink *S3Sink) completeMultipartUpload(ctx context.Context, key, uploadId string, parts []*s3.CompletedPart) error {
 	input := &s3.CompleteMultipartUploadInput{
 		Bucket:   aws.String(s3sink.bucket),
 		Key:      aws.String(key),
@@ -102,10 +103,10 @@ func (s3sink *S3Sink) completeMultipartUpload(key, uploadId string, parts []*s3.
 }
 
 // To upload a part
-func (s3sink *S3Sink) uploadPart(key, uploadId string, partId int, chunk *filer2.ChunkView) (*s3.CompletedPart, error) {
+func (s3sink *S3Sink) uploadPart(ctx context.Context, key, uploadId string, partId int, chunk *filer2.ChunkView) (*s3.CompletedPart, error) {
 	var readSeeker io.ReadSeeker
 
-	readSeeker, err := s3sink.buildReadSeeker(chunk)
+	readSeeker, err := s3sink.buildReadSeeker(ctx, chunk)
 	if err != nil {
 		glog.Errorf("[%s] uploadPart %s %d read: %v", s3sink.bucket, key, partId, err)
 		return nil, fmt.Errorf("[%s] uploadPart %s %d read: %v", s3sink.bucket, key, partId, err)
@@ -155,8 +156,8 @@ func (s3sink *S3Sink) uploadPartCopy(key, uploadId string, partId int64, copySou
 	return err
 }
 
-func (s3sink *S3Sink) buildReadSeeker(chunk *filer2.ChunkView) (io.ReadSeeker, error) {
-	fileUrl, err := s3sink.filerSource.LookupFileId(chunk.FileId)
+func (s3sink *S3Sink) buildReadSeeker(ctx context.Context, chunk *filer2.ChunkView) (io.ReadSeeker, error) {
+	fileUrl, err := s3sink.filerSource.LookupFileId(ctx, chunk.FileId)
 	if err != nil {
 		return nil, err
 	}

weed/replication/source/filer_source.go

@@ -39,16 +39,16 @@ func (fs *FilerSource) initialize(grpcAddress string, dir string) (err error) {
 	return nil
 }
 
-func (fs *FilerSource) LookupFileId(part string) (fileUrl string, err error) {
+func (fs *FilerSource) LookupFileId(ctx context.Context, part string) (fileUrl string, err error) {
 
 	vid2Locations := make(map[string]*filer_pb.Locations)
 
 	vid := volumeId(part)
 
-	err = fs.withFilerClient(fs.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	err = fs.withFilerClient(ctx, fs.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 
 		glog.V(4).Infof("read lookup volume id locations: %v", vid)
-		resp, err := client.LookupVolume(context.Background(), &filer_pb.LookupVolumeRequest{
+		resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{
 			VolumeIds: []string{vid},
 		})
 		if err != nil {
@@ -77,9 +77,9 @@ func (fs *FilerSource) LookupFileId(part string) (fileUrl string, err error) {
 	return
 }
 
-func (fs *FilerSource) ReadPart(part string) (filename string, header http.Header, readCloser io.ReadCloser, err error) {
+func (fs *FilerSource) ReadPart(ctx context.Context, part string) (filename string, header http.Header, readCloser io.ReadCloser, err error) {
 
-	fileUrl, err := fs.LookupFileId(part)
+	fileUrl, err := fs.LookupFileId(ctx, part)
 	if err != nil {
 		return "", nil, nil, err
 	}
@@ -89,17 +89,13 @@ func (fs *FilerSource) ReadPart(part string) (filename string, header http.Heade
 	return filename, header, readCloser, err
 }
 
-func (fs *FilerSource) withFilerClient(grpcDialOption grpc.DialOption, fn func(filer_pb.SeaweedFilerClient) error) error {
+func (fs *FilerSource) withFilerClient(ctx context.Context, grpcDialOption grpc.DialOption, fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	grpcConnection, err := util.GrpcDial(fs.grpcAddress, grpcDialOption)
-	if err != nil {
-		return fmt.Errorf("fail to dial %s: %v", fs.grpcAddress, err)
-	}
-	defer grpcConnection.Close()
+	return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
+		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+		return fn(client)
+	}, fs.grpcAddress, fs.grpcDialOption)
 
-	client := filer_pb.NewSeaweedFilerClient(grpcConnection)
-
-	return fn(client)
 }
 
 func volumeId(fileId string) string {

weed/replication/sub/notification_gocdk_pub_sub.go

@@ -0,0 +1,50 @@
+package sub
+
+import (
+	"context"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/golang/protobuf/proto"
+	"gocloud.dev/pubsub"
+	_ "gocloud.dev/pubsub/awssnssqs"
+	_ "gocloud.dev/pubsub/azuresb"
+	_ "gocloud.dev/pubsub/gcppubsub"
+	_ "gocloud.dev/pubsub/natspubsub"
+	_ "gocloud.dev/pubsub/rabbitpubsub"
+)
+
+func init() {
+	NotificationInputs = append(NotificationInputs, &GoCDKPubSubInput{})
+}
+
+type GoCDKPubSubInput struct {
+	sub *pubsub.Subscription
+}
+
+func (k *GoCDKPubSubInput) GetName() string {
+	return "gocdk_pub_sub"
+}
+
+func (k *GoCDKPubSubInput) Initialize(config util.Configuration) error {
+	subURL := config.GetString("sub_url")
+	glog.V(0).Infof("notification.gocdk_pub_sub.sub_url: %v", subURL)
+	sub, err := pubsub.OpenSubscription(context.Background(), subURL)
+	if err != nil {
+		return err
+	}
+	k.sub = sub
+	return nil
+}
+
+func (k *GoCDKPubSubInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, err error) {
+	msg, err := k.sub.Receive(context.Background())
+	key = msg.Metadata["key"]
+	message = &filer_pb.EventNotification{}
+	err = proto.Unmarshal(msg.Body, message)
+	if err != nil {
+		return "", nil, err
+	}
+	return key, message, nil
+}

weed/s3api/custom_types.go

@@ -0,0 +1,3 @@
+package s3api
+
+const s3TimeFormat = "2006-01-02T15:04:05.999Z07:00"

weed/s3api/filer_multipart.go

@@ -1,6 +1,8 @@
 package s3api
 
 import (
+	"context"
+	"encoding/xml"
 	"fmt"
 	"path/filepath"
 	"strconv"
@@ -16,14 +18,15 @@ import (
 )
 
 type InitiateMultipartUploadResult struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult"`
 	s3.CreateMultipartUploadOutput
 }
 
-func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInput) (output *InitiateMultipartUploadResult, code ErrorCode) {
+func (s3a *S3ApiServer) createMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (output *InitiateMultipartUploadResult, code ErrorCode) {
 	uploadId, _ := uuid.NewV4()
 	uploadIdString := uploadId.String()
 
-	if err := s3a.mkdir(s3a.genUploadsFolder(*input.Bucket), uploadIdString, func(entry *filer_pb.Entry) {
+	if err := s3a.mkdir(ctx, s3a.genUploadsFolder(*input.Bucket), uploadIdString, func(entry *filer_pb.Entry) {
 		if entry.Extended == nil {
 			entry.Extended = make(map[string][]byte)
 		}
@@ -34,7 +37,7 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp
 	}
 
 	output = &InitiateMultipartUploadResult{
-		s3.CreateMultipartUploadOutput{
+		CreateMultipartUploadOutput: s3.CreateMultipartUploadOutput{
 			Bucket:   input.Bucket,
 			Key:      input.Key,
 			UploadId: aws.String(uploadIdString),
@@ -45,14 +48,15 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp
 }
 
 type CompleteMultipartUploadResult struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult"`
 	s3.CompleteMultipartUploadOutput
 }
 
-func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploadInput) (output *CompleteMultipartUploadResult, code ErrorCode) {
+func (s3a *S3ApiServer) completeMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (output *CompleteMultipartUploadResult, code ErrorCode) {
 
 	uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId
 
-	entries, err := s3a.list(uploadDirectory, "", "", false, 0)
+	entries, err := s3a.list(ctx, uploadDirectory, "", "", false, 0)
 	if err != nil {
 		glog.Errorf("completeMultipartUpload %s %s error: %v", *input.Bucket, *input.UploadId, err)
 		return nil, ErrNoSuchUpload
@@ -87,7 +91,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
 	}
 	dirName = fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, *input.Bucket, dirName)
 
-	err = s3a.mkFile(dirName, entryName, finalParts)
+	err = s3a.mkFile(ctx, dirName, entryName, finalParts)
 
 	if err != nil {
 		glog.Errorf("completeMultipartUpload %s/%s error: %v", dirName, entryName, err)
@@ -95,29 +99,29 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
 	}
 
 	output = &CompleteMultipartUploadResult{
-		s3.CompleteMultipartUploadOutput{
+		CompleteMultipartUploadOutput: s3.CompleteMultipartUploadOutput{
 			Bucket: input.Bucket,
 			ETag:   aws.String("\"" + filer2.ETag(finalParts) + "\""),
 			Key:    input.Key,
 		},
 	}
 
-	if err = s3a.rm(s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true, false, true); err != nil {
+	if err = s3a.rm(ctx, s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true, false, true); err != nil {
 		glog.V(1).Infof("completeMultipartUpload cleanup %s upload %s: %v", *input.Bucket, *input.UploadId, err)
 	}
 
 	return
 }
 
-func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput) (output *s3.AbortMultipartUploadOutput, code ErrorCode) {
+func (s3a *S3ApiServer) abortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) (output *s3.AbortMultipartUploadOutput, code ErrorCode) {
 
-	exists, err := s3a.exists(s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true)
+	exists, err := s3a.exists(ctx, s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true)
 	if err != nil {
 		glog.V(1).Infof("bucket %s abort upload %s: %v", *input.Bucket, *input.UploadId, err)
 		return nil, ErrNoSuchUpload
 	}
 	if exists {
-		err = s3a.rm(s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true, true, true)
+		err = s3a.rm(ctx, s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true, true, true)
 	}
 	if err != nil {
 		glog.V(1).Infof("bucket %s remove upload %s: %v", *input.Bucket, *input.UploadId, err)
@@ -128,13 +132,14 @@ func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput
 }
 
 type ListMultipartUploadsResult struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult"`
 	s3.ListMultipartUploadsOutput
 }
 
-func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput) (output *ListMultipartUploadsResult, code ErrorCode) {
+func (s3a *S3ApiServer) listMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (output *ListMultipartUploadsResult, code ErrorCode) {
 
 	output = &ListMultipartUploadsResult{
-		s3.ListMultipartUploadsOutput{
+		ListMultipartUploadsOutput: s3.ListMultipartUploadsOutput{
 			Bucket:       input.Bucket,
 			Delimiter:    input.Delimiter,
 			EncodingType: input.EncodingType,
@@ -144,7 +149,7 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput
 		},
 	}
 
-	entries, err := s3a.list(s3a.genUploadsFolder(*input.Bucket), *input.Prefix, *input.KeyMarker, true, int(*input.MaxUploads))
+	entries, err := s3a.list(ctx, s3a.genUploadsFolder(*input.Bucket), *input.Prefix, *input.KeyMarker, true, int(*input.MaxUploads))
 	if err != nil {
 		glog.Errorf("listMultipartUploads %s error: %v", *input.Bucket, err)
 		return
@@ -164,12 +169,13 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput
 }
 
 type ListPartsResult struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult"`
 	s3.ListPartsOutput
 }
 
-func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListPartsResult, code ErrorCode) {
+func (s3a *S3ApiServer) listObjectParts(ctx context.Context, input *s3.ListPartsInput) (output *ListPartsResult, code ErrorCode) {
 	output = &ListPartsResult{
-		s3.ListPartsOutput{
+		ListPartsOutput: s3.ListPartsOutput{
 			Bucket:           input.Bucket,
 			Key:              input.Key,
 			UploadId:         input.UploadId,
@@ -178,7 +184,7 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP
 		},
 	}
 
-	entries, err := s3a.list(s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId,
+	entries, err := s3a.list(ctx, s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId,
 		"", fmt.Sprintf("%04d.part", *input.PartNumberMarker), false, int(*input.MaxParts))
 	if err != nil {
 		glog.Errorf("listObjectParts %s %s error: %v", *input.Bucket, *input.UploadId, err)

weed/s3api/filer_multipart_test.go

@@ -0,0 +1,26 @@
+package s3api
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"testing"
+)
+
+func TestInitiateMultipartUploadResult(t *testing.T) {
+
+	expected := `<?xml version="1.0" encoding="UTF-8"?>
+<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Bucket>example-bucket</Bucket><Key>example-object</Key><UploadId>VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA</UploadId></InitiateMultipartUploadResult>`
+	response := &InitiateMultipartUploadResult{
+		CreateMultipartUploadOutput: s3.CreateMultipartUploadOutput{
+			Bucket:   aws.String("example-bucket"),
+			Key:      aws.String("example-object"),
+			UploadId: aws.String("VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA"),
+		},
+	}
+
+	encoded := string(encodeResponse(response))
+	if encoded != expected {
+		t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
+	}
+
+}

weed/s3api/filer_util.go

@@ -10,8 +10,8 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 )
 
-func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string, fn func(entry *filer_pb.Entry)) error {
-	return s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+func (s3a *S3ApiServer) mkdir(ctx context.Context, parentDirectoryPath string, dirName string, fn func(entry *filer_pb.Entry)) error {
+	return s3a.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		entry := &filer_pb.Entry{
 			Name:        dirName,
@@ -35,7 +35,7 @@ func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string, fn fun
 		}
 
 		glog.V(1).Infof("mkdir: %v", request)
-		if _, err := client.CreateEntry(context.Background(), request); err != nil {
+		if _, err := client.CreateEntry(ctx, request); err != nil {
 			return fmt.Errorf("mkdir %s/%s: %v", parentDirectoryPath, dirName, err)
 		}
 
@@ -43,8 +43,8 @@ func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string, fn fun
 	})
 }
 
-func (s3a *S3ApiServer) mkFile(parentDirectoryPath string, fileName string, chunks []*filer_pb.FileChunk) error {
-	return s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+func (s3a *S3ApiServer) mkFile(ctx context.Context, parentDirectoryPath string, fileName string, chunks []*filer_pb.FileChunk) error {
+	return s3a.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		entry := &filer_pb.Entry{
 			Name:        fileName,
@@ -65,7 +65,7 @@ func (s3a *S3ApiServer) mkFile(parentDirectoryPath string, fileName string, chun
 		}
 
 		glog.V(1).Infof("create file: %s/%s", parentDirectoryPath, fileName)
-		if _, err := client.CreateEntry(context.Background(), request); err != nil {
+		if _, err := client.CreateEntry(ctx, request); err != nil {
 			return fmt.Errorf("create file %s/%s: %v", parentDirectoryPath, fileName, err)
 		}
 
@@ -73,9 +73,9 @@ func (s3a *S3ApiServer) mkFile(parentDirectoryPath string, fileName string, chun
 	})
 }
 
-func (s3a *S3ApiServer) list(parentDirectoryPath, prefix, startFrom string, inclusive bool, limit int) (entries []*filer_pb.Entry, err error) {
+func (s3a *S3ApiServer) list(ctx context.Context, parentDirectoryPath, prefix, startFrom string, inclusive bool, limit int) (entries []*filer_pb.Entry, err error) {
 
-	err = s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err = s3a.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.ListEntriesRequest{
 			Directory:          parentDirectoryPath,
@@ -86,7 +86,7 @@ func (s3a *S3ApiServer) list(parentDirectoryPath, prefix, startFrom string, incl
 		}
 
 		glog.V(4).Infof("read directory: %v", request)
-		resp, err := client.ListEntries(context.Background(), request)
+		resp, err := client.ListEntries(ctx, request)
 		if err != nil {
 			return fmt.Errorf("list dir %v: %v", parentDirectoryPath, err)
 		}
@@ -100,11 +100,9 @@ func (s3a *S3ApiServer) list(parentDirectoryPath, prefix, startFrom string, incl
 
 }
 
-func (s3a *S3ApiServer) rm(parentDirectoryPath string, entryName string, isDirectory, isDeleteData, isRecursive bool) error {
+func (s3a *S3ApiServer) rm(ctx context.Context, parentDirectoryPath string, entryName string, isDirectory, isDeleteData, isRecursive bool) error {
 
-	return s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-
-		ctx := context.Background()
+	return s3a.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    parentDirectoryPath,
@@ -123,11 +121,9 @@ func (s3a *S3ApiServer) rm(parentDirectoryPath string, entryName string, isDirec
 
 }
 
-func (s3a *S3ApiServer) exists(parentDirectoryPath string, entryName string, isDirectory bool) (exists bool, err error) {
+func (s3a *S3ApiServer) exists(ctx context.Context, parentDirectoryPath string, entryName string, isDirectory bool) (exists bool, err error) {
 
-	err = s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-
-		ctx := context.Background()
+	err = s3a.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.LookupDirectoryEntryRequest{
 			Directory: parentDirectoryPath,

weed/s3api/s3api_bucket_handlers.go

@@ -2,6 +2,7 @@ package s3api
 
 import (
 	"context"
+	"encoding/xml"
 	"fmt"
 	"math"
 	"net/http"
@@ -21,15 +22,16 @@ var (
 )
 
 type ListAllMyBucketsResult struct {
-	Buckets []*s3.Bucket `xml:"Buckets>Bucket"`
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult"`
 	Owner   *s3.Owner
+	Buckets []*s3.Bucket `xml:"Buckets>Bucket"`
 }
 
 func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
 
 	var response ListAllMyBucketsResult
 
-	entries, err := s3a.list(s3a.option.BucketsPath, "", "", false, math.MaxInt32)
+	entries, err := s3a.list(context.Background(), s3a.option.BucketsPath, "", "", false, math.MaxInt32)
 
 	if err != nil {
 		writeErrorResponse(w, ErrInternalError, r.URL)
@@ -63,7 +65,7 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
 	bucket := vars["bucket"]
 
 	// create the folder for bucket, but lazily create actual collection
-	if err := s3a.mkdir(s3a.option.BucketsPath, bucket, nil); err != nil {
+	if err := s3a.mkdir(context.Background(), s3a.option.BucketsPath, bucket, nil); err != nil {
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
 	}
@@ -76,9 +78,8 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
-	err := s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-
-		ctx := context.Background()
+	ctx := context.Background()
+	err := s3a.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		// delete collection
 		deleteCollectionRequest := &filer_pb.DeleteCollectionRequest{
@@ -93,7 +94,7 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
 		return nil
 	})
 
-	err = s3a.rm(s3a.option.BucketsPath, bucket, true, false, true)
+	err = s3a.rm(ctx, s3a.option.BucketsPath, bucket, true, false, true)
 
 	if err != nil {
 		writeErrorResponse(w, ErrInternalError, r.URL)
@@ -108,7 +109,9 @@ func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
-	err := s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	ctx := context.Background()
+
+	err := s3a.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.LookupDirectoryEntryRequest{
 			Directory: s3a.option.BucketsPath,
@@ -116,7 +119,7 @@ func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request
 		}
 
 		glog.V(1).Infof("lookup bucket: %v", request)
-		if _, err := client.LookupDirectoryEntry(context.Background(), request); err != nil {
+		if _, err := client.LookupDirectoryEntry(ctx, request); err != nil {
 			return fmt.Errorf("lookup bucket %s/%s: %v", s3a.option.BucketsPath, bucket, err)
 		}
 

weed/s3api/s3api_bucket_handlers_test.go

@@ -0,0 +1,39 @@
+package s3api
+
+import (
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+func TestListBucketsHandler(t *testing.T) {
+
+	expected := `<?xml version="1.0" encoding="UTF-8"?>
+<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><DisplayName></DisplayName><ID></ID></Owner><Buckets><Bucket><CreationDate>2011-04-09T12:34:49Z</CreationDate><Name>test1</Name></Bucket><Bucket><CreationDate>2011-02-09T12:34:49Z</CreationDate><Name>test2</Name></Bucket></Buckets></ListAllMyBucketsResult>`
+	var response ListAllMyBucketsResult
+
+	var buckets []*s3.Bucket
+	buckets = append(buckets, &s3.Bucket{
+		Name:         aws.String("test1"),
+		CreationDate: aws.Time(time.Date(2011, 4, 9, 12, 34, 49, 0, time.UTC)),
+	})
+	buckets = append(buckets, &s3.Bucket{
+		Name:         aws.String("test2"),
+		CreationDate: aws.Time(time.Date(2011, 2, 9, 12, 34, 49, 0, time.UTC)),
+	})
+
+	response = ListAllMyBucketsResult{
+		Owner: &s3.Owner{
+			ID:          aws.String(""),
+			DisplayName: aws.String(""),
+		},
+		Buckets: buckets,
+	}
+
+	encoded := string(encodeResponse(response))
+	if encoded != expected {
+		t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
+	}
+}

weed/s3api/s3api_handlers.go

@@ -2,12 +2,14 @@ package s3api
 
 import (
 	"bytes"
+	"context"
 	"encoding/base64"
 	"encoding/xml"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"google.golang.org/grpc"
 	"net/http"
 	"net/url"
 	"time"
@@ -35,17 +37,13 @@ func encodeResponse(response interface{}) []byte {
 	return bytesBuffer.Bytes()
 }
 
-func (s3a *S3ApiServer) withFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
+func (s3a *S3ApiServer) withFilerClient(ctx context.Context, fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	grpcConnection, err := util.GrpcDial(s3a.option.FilerGrpcAddress, s3a.option.GrpcDialOption)
-	if err != nil {
-		return fmt.Errorf("fail to dial %s: %v", s3a.option.FilerGrpcAddress, err)
-	}
-	defer grpcConnection.Close()
+	return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
+		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+		return fn(client)
+	}, s3a.option.FilerGrpcAddress, s3a.option.GrpcDialOption)
 
-	client := filer_pb.NewSeaweedFilerClient(grpcConnection)
-
-	return fn(client)
 }
 
 // If none of the http routes match respond with MethodNotAllowed

weed/s3api/s3api_object_multipart_handlers.go

@@ -1,6 +1,7 @@
 package s3api
 
 import (
+	"context"
 	"fmt"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
@@ -25,7 +26,7 @@ func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http
 	bucket = vars["bucket"]
 	object = vars["object"]
 
-	response, errCode := s3a.createMultipartUpload(&s3.CreateMultipartUploadInput{
+	response, errCode := s3a.createMultipartUpload(context.Background(), &s3.CreateMultipartUploadInput{
 		Bucket: aws.String(bucket),
 		Key:    aws.String(object),
 	})
@@ -50,7 +51,7 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r
 	// Get upload id.
 	uploadID, _, _, _ := getObjectResources(r.URL.Query())
 
-	response, errCode := s3a.completeMultipartUpload(&s3.CompleteMultipartUploadInput{
+	response, errCode := s3a.completeMultipartUpload(context.Background(), &s3.CompleteMultipartUploadInput{
 		Bucket:   aws.String(bucket),
 		Key:      aws.String(object),
 		UploadId: aws.String(uploadID),
@@ -76,7 +77,7 @@ func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *ht
 	// Get upload id.
 	uploadID, _, _, _ := getObjectResources(r.URL.Query())
 
-	response, errCode := s3a.abortMultipartUpload(&s3.AbortMultipartUploadInput{
+	response, errCode := s3a.abortMultipartUpload(context.Background(), &s3.AbortMultipartUploadInput{
 		Bucket:   aws.String(bucket),
 		Key:      aws.String(object),
 		UploadId: aws.String(uploadID),
@@ -111,7 +112,7 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht
 		}
 	}
 
-	response, errCode := s3a.listMultipartUploads(&s3.ListMultipartUploadsInput{
+	response, errCode := s3a.listMultipartUploads(context.Background(), &s3.ListMultipartUploadsInput{
 		Bucket:         aws.String(bucket),
 		Delimiter:      aws.String(delimiter),
 		EncodingType:   aws.String(encodingType),
@@ -148,7 +149,7 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	response, errCode := s3a.listObjectParts(&s3.ListPartsInput{
+	response, errCode := s3a.listObjectParts(context.Background(), &s3.ListPartsInput{
 		Bucket:           aws.String(bucket),
 		Key:              aws.String(object),
 		MaxParts:         aws.Int64(int64(maxParts)),
@@ -174,8 +175,10 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ
 
 	rAuthType := getRequestAuthType(r)
 
+	ctx := context.Background()
+
 	uploadID := r.URL.Query().Get("uploadId")
-	exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true)
+	exists, err := s3a.exists(ctx, s3a.genUploadsFolder(bucket), uploadID, true)
 	if !exists {
 		writeErrorResponse(w, ErrNoSuchUpload, r.URL)
 		return
@@ -197,8 +200,8 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ
 		dataReader = newSignV4ChunkedReader(r)
 	}
 
-	uploadUrl := fmt.Sprintf("http://%s%s/%s/%04d.part",
-		s3a.option.Filer, s3a.genUploadsFolder(bucket), uploadID, partID-1)
+	uploadUrl := fmt.Sprintf("http://%s%s/%s/%04d.part?collection=%s",
+		s3a.option.Filer, s3a.genUploadsFolder(bucket), uploadID, partID-1, bucket)
 
 	etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader)
 

weed/s3api/s3api_objects_list_handlers.go

@@ -9,8 +9,6 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
@@ -46,7 +44,9 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
 		marker = startAfter
 	}
 
-	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker)
+	ctx := context.Background()
+
+	response, err := s3a.listFilerEntries(ctx, bucket, originalPrefix, maxKeys, marker)
 
 	if err != nil {
 		writeErrorResponse(w, ErrInternalError, r.URL)
@@ -64,6 +64,8 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
+	ctx := context.Background()
+
 	originalPrefix, marker, delimiter, maxKeys := getListObjectsV1Args(r.URL.Query())
 
 	if maxKeys < 0 {
@@ -75,7 +77,7 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker)
+	response, err := s3a.listFilerEntries(ctx, bucket, originalPrefix, maxKeys, marker)
 
 	if err != nil {
 		writeErrorResponse(w, ErrInternalError, r.URL)
@@ -85,13 +87,13 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
 	writeSuccessResponseXML(w, encodeResponse(response))
 }
 
-func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys int, marker string) (response *s3.ListObjectsOutput, err error) {
+func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, bucket, originalPrefix string, maxKeys int, marker string) (response ListBucketResult, err error) {
 
 	// convert full path prefix into directory name and prefix for entry name
 	dir, prefix := filepath.Split(originalPrefix)
 
 	// check filer
-	err = s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err = s3a.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.ListEntriesRequest{
 			Directory:          fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, bucket, dir),
@@ -101,13 +103,13 @@ func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys
 			InclusiveStartFrom: false,
 		}
 
-		resp, err := client.ListEntries(context.Background(), request)
+		resp, err := client.ListEntries(ctx, request)
 		if err != nil {
 			return fmt.Errorf("list buckets: %v", err)
 		}
 
-		var contents []*s3.Object
-		var commonPrefixes []*s3.CommonPrefix
+		var contents []ListEntry
+		var commonPrefixes []PrefixEntry
 		var counter int
 		var lastEntryName string
 		var isTruncated bool
@@ -119,37 +121,37 @@ func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys
 			}
 			lastEntryName = entry.Name
 			if entry.IsDirectory {
-				commonPrefixes = append(commonPrefixes, &s3.CommonPrefix{
-					Prefix: aws.String(fmt.Sprintf("%s%s/", dir, entry.Name)),
+				commonPrefixes = append(commonPrefixes, PrefixEntry{
+					Prefix: fmt.Sprintf("%s%s/", dir, entry.Name),
 				})
 			} else {
-				contents = append(contents, &s3.Object{
-					Key:          aws.String(fmt.Sprintf("%s%s", dir, entry.Name)),
-					LastModified: aws.Time(time.Unix(entry.Attributes.Mtime, 0)),
-					ETag:         aws.String("\"" + filer2.ETag(entry.Chunks) + "\""),
-					Size:         aws.Int64(int64(filer2.TotalSize(entry.Chunks))),
-					Owner: &s3.Owner{
-						ID:          aws.String("bcaf161ca5fb16fd081034f"),
-						DisplayName: aws.String("webfile"),
+				contents = append(contents, ListEntry{
+					Key:          fmt.Sprintf("%s%s", dir, entry.Name),
+					LastModified: time.Unix(entry.Attributes.Mtime, 0),
+					ETag:         "\"" + filer2.ETag(entry.Chunks) + "\"",
+					Size:         int64(filer2.TotalSize(entry.Chunks)),
+					Owner: CanonicalUser{
+						ID:          fmt.Sprintf("%x", entry.Attributes.Uid),
+						DisplayName: entry.Attributes.UserName,
 					},
-					StorageClass: aws.String("STANDARD"),
+					StorageClass: "STANDARD",
 				})
 			}
 		}
 
-		response = &s3.ListObjectsOutput{
-			Name:           aws.String(bucket),
-			Prefix:         aws.String(originalPrefix),
-			Marker:         aws.String(marker),
-			NextMarker:     aws.String(lastEntryName),
-			MaxKeys:        aws.Int64(int64(maxKeys)),
-			Delimiter:      aws.String("/"),
-			IsTruncated:    aws.Bool(isTruncated),
+		response = ListBucketResult{
+			Name:           bucket,
+			Prefix:         originalPrefix,
+			Marker:         marker,
+			NextMarker:     lastEntryName,
+			MaxKeys:        maxKeys,
+			Delimiter:      "/",
+			IsTruncated:    isTruncated,
 			Contents:       contents,
 			CommonPrefixes: commonPrefixes,
 		}
 
-		glog.V(4).Infof("read directory: %v, found: %v", request, counter)
+		glog.V(4).Infof("read directory: %v, found: %v, %+v", request, counter, response)
 
 		return nil
 	})

weed/s3api/s3api_objects_list_handlers_test.go

@@ -0,0 +1,38 @@
+package s3api
+
+import (
+	"testing"
+	"time"
+)
+
+func TestListObjectsHandler(t *testing.T) {
+
+	// https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html
+
+	expected := `<?xml version="1.0" encoding="UTF-8"?>
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>test_container</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>1.zip</Key><ETag>&#34;4397da7a7649e8085de9916c240e8166&#34;</ETag><Size>1234567</Size><Owner><ID>65a011niqo39cdf8ec533ec3d1ccaafsa932</ID></Owner><StorageClass>STANDARD</StorageClass><LastModified>2011-04-09T12:34:49Z</LastModified></Contents></ListBucketResult>`
+
+	response := ListBucketResult{
+		Name:        "test_container",
+		Prefix:      "",
+		Marker:      "",
+		NextMarker:  "",
+		MaxKeys:     1000,
+		IsTruncated: false,
+		Contents: []ListEntry{{
+			Key:          "1.zip",
+			LastModified: time.Date(2011, 4, 9, 12, 34, 49, 0, time.UTC),
+			ETag:         "\"4397da7a7649e8085de9916c240e8166\"",
+			Size:         1234567,
+			Owner: CanonicalUser{
+				ID: "65a011niqo39cdf8ec533ec3d1ccaafsa932",
+			},
+			StorageClass: "STANDARD",
+		}},
+	}
+
+	encoded := string(encodeResponse(response))
+	if encoded != expected {
+		t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
+	}
+}

weed/s3api/s3api_xsd_generated.go

@@ -25,8 +25,8 @@ type BucketLoggingStatus struct {
 }
 
 type CanonicalUser struct {
-	ID          string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ID"`
-	DisplayName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DisplayName,omitempty"`
+	ID          string `xml:"ID"`
+	DisplayName string `xml:"DisplayName,omitempty"`
 }
 
 type CopyObject struct {
@@ -506,15 +506,15 @@ func (t *ListAllMyBuckets) UnmarshalXML(d *xml.Decoder, start xml.StartElement)
 }
 
 type ListAllMyBucketsEntry struct {
-	Name         string    `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Name"`
-	CreationDate time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreationDate"`
+	Name         string    `xml:"Name"`
+	CreationDate time.Time `xml:"CreationDate"`
 }
 
 func (t *ListAllMyBucketsEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type T ListAllMyBucketsEntry
 	var layout struct {
 		*T
-		CreationDate *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreationDate"`
+		CreationDate *xsdDateTime `xml:"CreationDate"`
 	}
 	layout.T = (*T)(t)
 	layout.CreationDate = (*xsdDateTime)(&layout.T.CreationDate)
@@ -524,7 +524,7 @@ func (t *ListAllMyBucketsEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElem
 	type T ListAllMyBucketsEntry
 	var overlay struct {
 		*T
-		CreationDate *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreationDate"`
+		CreationDate *xsdDateTime `xml:"CreationDate"`
 	}
 	overlay.T = (*T)(t)
 	overlay.CreationDate = (*xsdDateTime)(&overlay.T.CreationDate)
@@ -532,7 +532,7 @@ func (t *ListAllMyBucketsEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElem
 }
 
 type ListAllMyBucketsList struct {
-	Bucket []ListAllMyBucketsEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket,omitempty"`
+	Bucket []ListAllMyBucketsEntry `xml:"Bucket,omitempty"`
 }
 
 type ListAllMyBucketsResponse struct {
@@ -577,32 +577,33 @@ type ListBucketResponse struct {
 }
 
 type ListBucketResult struct {
-	Metadata       []MetadataEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Metadata,omitempty"`
-	Name           string          `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Name"`
-	Prefix         string          `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Prefix"`
-	Marker         string          `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Marker"`
-	NextMarker     string          `xml:"http://s3.amazonaws.com/doc/2006-03-01/ NextMarker,omitempty"`
-	MaxKeys        int             `xml:"http://s3.amazonaws.com/doc/2006-03-01/ MaxKeys"`
-	Delimiter      string          `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Delimiter,omitempty"`
-	IsTruncated    bool            `xml:"http://s3.amazonaws.com/doc/2006-03-01/ IsTruncated"`
-	Contents       []ListEntry     `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Contents,omitempty"`
-	CommonPrefixes []PrefixEntry   `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CommonPrefixes,omitempty"`
+	XMLName        xml.Name        `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
+	Metadata       []MetadataEntry `xml:"Metadata,omitempty"`
+	Name           string          `xml:"Name"`
+	Prefix         string          `xml:"Prefix"`
+	Marker         string          `xml:"Marker"`
+	NextMarker     string          `xml:"NextMarker,omitempty"`
+	MaxKeys        int             `xml:"MaxKeys"`
+	Delimiter      string          `xml:"Delimiter,omitempty"`
+	IsTruncated    bool            `xml:"IsTruncated"`
+	Contents       []ListEntry     `xml:"Contents,omitempty"`
+	CommonPrefixes []PrefixEntry   `xml:"CommonPrefixes,omitempty"`
 }
 
 type ListEntry struct {
-	Key          string        `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Key"`
-	LastModified time.Time     `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"`
-	ETag         string        `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ETag"`
-	Size         int64         `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Size"`
-	Owner        CanonicalUser `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Owner,omitempty"`
-	StorageClass StorageClass  `xml:"http://s3.amazonaws.com/doc/2006-03-01/ StorageClass"`
+	Key          string        `xml:"Key"`
+	LastModified time.Time     `xml:"LastModified"`
+	ETag         string        `xml:"ETag"`
+	Size         int64         `xml:"Size"`
+	Owner        CanonicalUser `xml:"Owner,omitempty"`
+	StorageClass StorageClass  `xml:"StorageClass"`
 }
 
 func (t *ListEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type T ListEntry
 	var layout struct {
 		*T
-		LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"`
+		LastModified *xsdDateTime `xml:"LastModified"`
 	}
 	layout.T = (*T)(t)
 	layout.LastModified = (*xsdDateTime)(&layout.T.LastModified)
@@ -612,7 +613,7 @@ func (t *ListEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	type T ListEntry
 	var overlay struct {
 		*T
-		LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"`
+		LastModified *xsdDateTime `xml:"LastModified"`
 	}
 	overlay.T = (*T)(t)
 	overlay.LastModified = (*xsdDateTime)(&overlay.T.LastModified)
@@ -965,10 +966,10 @@ func (b xsdBase64Binary) MarshalText() ([]byte, error) {
 type xsdDateTime time.Time
 
 func (t *xsdDateTime) UnmarshalText(text []byte) error {
-	return _unmarshalTime(text, (*time.Time)(t), "2006-01-02T15:04:05.999999999")
+	return _unmarshalTime(text, (*time.Time)(t), s3TimeFormat)
 }
 func (t xsdDateTime) MarshalText() ([]byte, error) {
-	return []byte((time.Time)(t).Format("2006-01-02T15:04:05.999999999")), nil
+	return []byte((time.Time)(t).Format(s3TimeFormat)), nil
 }
 func (t xsdDateTime) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	if (time.Time)(t).IsZero() {

weed/server/common_test.go

@@ -0,0 +1,31 @@
+package weed_server
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestParseURL(t *testing.T) {
+	if vid, fid, _, _, _ := parseURLPath("/1,06dfa8a684"); true {
+		if vid != "1" {
+			t.Errorf("fail to parse vid: %s", vid)
+		}
+		if fid != "06dfa8a684" {
+			t.Errorf("fail to parse fid: %s", fid)
+		}
+	}
+	if vid, fid, _, _, _ := parseURLPath("/1,06dfa8a684_1"); true {
+		if vid != "1" {
+			t.Errorf("fail to parse vid: %s", vid)
+		}
+		if fid != "06dfa8a684_1" {
+			t.Errorf("fail to parse fid: %s", fid)
+		}
+		if sepIndex := strings.LastIndex(fid, "_"); sepIndex > 0 {
+			fid = fid[:sepIndex]
+		}
+		if fid != "06dfa8a684" {
+			t.Errorf("fail to parse fid: %s", fid)
+		}
+	}
+}

weed/server/filer_grpc_server.go

@@ -14,12 +14,11 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
-	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
 
-	entry, err := fs.filer.FindEntry(filer2.FullPath(filepath.Join(req.Directory, req.Name)))
+	entry, err := fs.filer.FindEntry(ctx, filer2.FullPath(filepath.ToSlash(filepath.Join(req.Directory, req.Name))))
 	if err != nil {
 		return nil, fmt.Errorf("%s not found under %s: %v", req.Name, req.Directory, err)
 	}
@@ -45,7 +44,7 @@ func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntrie
 	lastFileName := req.StartFromFileName
 	includeLastFile := req.InclusiveStartFrom
 	for limit > 0 {
-		entries, err := fs.filer.ListDirectoryEntries(filer2.FullPath(req.Directory), lastFileName, includeLastFile, 1024)
+		entries, err := fs.filer.ListDirectoryEntries(ctx, filer2.FullPath(req.Directory), lastFileName, includeLastFile, 1024)
 		if err != nil {
 			return nil, err
 		}
@@ -112,7 +111,7 @@ func (fs *FilerServer) LookupVolume(ctx context.Context, req *filer_pb.LookupVol
 
 func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntryRequest) (resp *filer_pb.CreateEntryResponse, err error) {
 
-	fullpath := filer2.FullPath(filepath.Join(req.Directory, req.Entry.Name))
+	fullpath := filer2.FullPath(filepath.ToSlash(filepath.Join(req.Directory, req.Entry.Name)))
 	chunks, garbages := filer2.CompactFileChunks(req.Entry.Chunks)
 
 	fs.filer.DeleteChunks(garbages)
@@ -121,7 +120,7 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr
 		return nil, fmt.Errorf("can not create entry with empty attributes")
 	}
 
-	err = fs.filer.CreateEntry(&filer2.Entry{
+	err = fs.filer.CreateEntry(ctx, &filer2.Entry{
 		FullPath: fullpath,
 		Attr:     filer2.PbToEntryAttribute(req.Entry.Attributes),
 		Chunks:   chunks,
@@ -135,8 +134,8 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr
 
 func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) {
 
-	fullpath := filepath.Join(req.Directory, req.Entry.Name)
-	entry, err := fs.filer.FindEntry(filer2.FullPath(fullpath))
+	fullpath := filepath.ToSlash(filepath.Join(req.Directory, req.Entry.Name))
+	entry, err := fs.filer.FindEntry(ctx, filer2.FullPath(fullpath))
 	if err != nil {
 		return &filer_pb.UpdateEntryResponse{}, fmt.Errorf("not found %s: %v", fullpath, err)
 	}
@@ -147,7 +146,7 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr
 	chunks, garbages := filer2.CompactFileChunks(req.Entry.Chunks)
 
 	newEntry := &filer2.Entry{
-		FullPath: filer2.FullPath(filepath.Join(req.Directory, req.Entry.Name)),
+		FullPath: filer2.FullPath(filepath.ToSlash(filepath.Join(req.Directory, req.Entry.Name))),
 		Attr:     entry.Attr,
 		Chunks:   chunks,
 	}
@@ -175,7 +174,7 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr
 		return &filer_pb.UpdateEntryResponse{}, err
 	}
 
-	if err = fs.filer.UpdateEntry(entry, newEntry); err == nil {
+	if err = fs.filer.UpdateEntry(ctx, entry, newEntry); err == nil {
 		fs.filer.DeleteChunks(unusedChunks)
 		fs.filer.DeleteChunks(garbages)
 	}
@@ -186,7 +185,7 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr
 }
 
 func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest) (resp *filer_pb.DeleteEntryResponse, err error) {
-	err = fs.filer.DeleteEntryMetaAndData(filer2.FullPath(filepath.Join(req.Directory, req.Name)), req.IsRecursive, req.IsDeleteData)
+	err = fs.filer.DeleteEntryMetaAndData(ctx, filer2.FullPath(filepath.ToSlash(filepath.Join(req.Directory, req.Name))), req.IsRecursive, req.IsDeleteData)
 	return &filer_pb.DeleteEntryResponse{}, err
 }
 
@@ -239,9 +238,12 @@ func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVol
 
 func (fs *FilerServer) DeleteCollection(ctx context.Context, req *filer_pb.DeleteCollectionRequest) (resp *filer_pb.DeleteCollectionResponse, err error) {
 
-	for _, master := range fs.option.Masters {
-		_, err = util.Get(fmt.Sprintf("http://%s/col/delete?collection=%s", master, req.Collection))
-	}
+	err = fs.filer.MasterClient.WithClient(ctx, func(client master_pb.SeaweedClient) error {
+		_, err := client.CollectionDelete(ctx, &master_pb.CollectionDeleteRequest{
+			Name: req.GetCollection(),
+		})
+		return err
+	})
 
 	return &filer_pb.DeleteCollectionResponse{}, err
 }

weed/server/filer_grpc_server_rename.go

@@ -0,0 +1,123 @@
+package weed_server
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"path/filepath"
+)
+
+func (fs *FilerServer) AtomicRenameEntry(ctx context.Context, req *filer_pb.AtomicRenameEntryRequest) (*filer_pb.AtomicRenameEntryResponse, error) {
+
+	ctx, err := fs.filer.BeginTransaction(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	oldParent := filer2.FullPath(filepath.ToSlash(req.OldDirectory))
+
+	oldEntry, err := fs.filer.FindEntry(ctx, oldParent.Child(req.OldName))
+	if err != nil {
+		fs.filer.RollbackTransaction(ctx)
+		return nil, fmt.Errorf("%s/%s not found: %v", req.OldDirectory, req.OldName, err)
+	}
+
+	var events MoveEvents
+	moveErr := fs.moveEntry(ctx, oldParent, oldEntry, filer2.FullPath(filepath.ToSlash(req.NewDirectory)), req.NewName, &events)
+	if moveErr != nil {
+		fs.filer.RollbackTransaction(ctx)
+		return nil, fmt.Errorf("%s/%s move error: %v", req.OldDirectory, req.OldName, err)
+	} else {
+		if commitError := fs.filer.CommitTransaction(ctx); commitError != nil {
+			fs.filer.RollbackTransaction(ctx)
+			return nil, fmt.Errorf("%s/%s move commit error: %v", req.OldDirectory, req.OldName, err)
+		}
+	}
+
+	for _, entry := range events.newEntries {
+		fs.filer.NotifyUpdateEvent(nil, entry, false)
+	}
+	for _, entry := range events.oldEntries {
+		fs.filer.NotifyUpdateEvent(entry, nil, false)
+	}
+
+	return &filer_pb.AtomicRenameEntryResponse{}, nil
+}
+
+func (fs *FilerServer) moveEntry(ctx context.Context, oldParent filer2.FullPath, entry *filer2.Entry, newParent filer2.FullPath, newName string, events *MoveEvents) error {
+	if entry.IsDirectory() {
+		if err := fs.moveFolderSubEntries(ctx, oldParent, entry, newParent, newName, events); err != nil {
+			return err
+		}
+	}
+	return fs.moveSelfEntry(ctx, oldParent, entry, newParent, newName, events)
+}
+
+func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent filer2.FullPath, entry *filer2.Entry, newParent filer2.FullPath, newName string, events *MoveEvents) error {
+
+	currentDirPath := oldParent.Child(entry.Name())
+	newDirPath := newParent.Child(newName)
+
+	glog.V(1).Infof("moving folder %s => %s", currentDirPath, newDirPath)
+
+	lastFileName := ""
+	includeLastFile := false
+	for {
+
+		entries, err := fs.filer.ListDirectoryEntries(ctx, currentDirPath, lastFileName, includeLastFile, 1024)
+		if err != nil {
+			return err
+		}
+
+		println("found", len(entries), "entries under", currentDirPath)
+
+		for _, item := range entries {
+			lastFileName = item.Name()
+			println("processing", lastFileName)
+			err := fs.moveEntry(ctx, currentDirPath, item, newDirPath, item.Name(), events)
+			if err != nil {
+				return err
+			}
+		}
+		if len(entries) < 1024 {
+			break
+		}
+	}
+	return nil
+}
+
+func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent filer2.FullPath, entry *filer2.Entry, newParent filer2.FullPath, newName string, events *MoveEvents) (error) {
+
+	oldPath, newPath := oldParent.Child(entry.Name()), newParent.Child(newName)
+
+	glog.V(1).Infof("moving entry %s => %s", oldPath, newPath)
+
+	// add to new directory
+	newEntry := &filer2.Entry{
+		FullPath: newPath,
+		Attr:     entry.Attr,
+		Chunks:   entry.Chunks,
+	}
+	createErr := fs.filer.CreateEntry(ctx, newEntry)
+	if createErr != nil {
+		return createErr
+	}
+
+	// delete old entry
+	deleteErr := fs.filer.DeleteEntryMetaAndData(ctx, oldPath, false, false)
+	if deleteErr != nil {
+		return deleteErr
+	}
+
+	events.oldEntries = append(events.oldEntries, entry)
+	events.newEntries = append(events.newEntries, newEntry)
+	return nil
+
+}
+
+type MoveEvents struct {
+	oldEntries []*filer2.Entry
+	newEntries []*filer2.Entry
+}

weed/server/filer_server.go

@@ -1,10 +1,11 @@
 package weed_server
 
 import (
-	"google.golang.org/grpc"
 	"net/http"
 	"os"
 
+	"google.golang.org/grpc"
+
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	_ "github.com/chrislusf/seaweedfs/weed/filer2/cassandra"
 	_ "github.com/chrislusf/seaweedfs/weed/filer2/leveldb"
@@ -15,6 +16,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/notification"
 	_ "github.com/chrislusf/seaweedfs/weed/notification/aws_sqs"
+	_ "github.com/chrislusf/seaweedfs/weed/notification/gocdk_pub_sub"
 	_ "github.com/chrislusf/seaweedfs/weed/notification/google_pub_sub"
 	_ "github.com/chrislusf/seaweedfs/weed/notification/kafka"
 	_ "github.com/chrislusf/seaweedfs/weed/notification/log"
@@ -32,6 +34,7 @@ type FilerOption struct {
 	DirListingLimit    int
 	DataCenter         string
 	DefaultLevelDbDir  string
+	DisableHttp        bool
 }
 
 type FilerServer struct {
@@ -72,7 +75,9 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 	notification.LoadConfiguration(v.Sub("notification"))
 
 	handleStaticResources(defaultMux)
-	defaultMux.HandleFunc("/", fs.filerHandler)
+	if !option.DisableHttp {
+		defaultMux.HandleFunc("/", fs.filerHandler)
+	}
 	if defaultMux != readonlyMux {
 		readonlyMux.HandleFunc("/", fs.readonlyFilerHandler)
 	}
@@ -92,10 +97,10 @@ func LoadConfiguration(configFileName string, required bool) (loaded bool) {
 	viper.AddConfigPath("$HOME/.seaweedfs") // call multiple times to add many search paths
 	viper.AddConfigPath("/etc/seaweedfs/")  // path to look for the config file in
 
-	glog.V(0).Infof("Reading %s.toml from %s", configFileName, viper.ConfigFileUsed())
+	glog.V(1).Infof("Reading %s.toml from %s", configFileName, viper.ConfigFileUsed())
 
 	if err := viper.MergeInConfig(); err != nil { // Handle errors reading the config file
-		glog.V(0).Infof("Reading %s: %v", viper.ConfigFileUsed(), err)
+		glog.V(1).Infof("Reading %s: %v", viper.ConfigFileUsed(), err)
 		if required {
 			glog.Fatalf("Failed to load %s.toml file from current directory, or $HOME/.seaweedfs/, or /etc/seaweedfs/"+
 				"\n\nPlease follow this example and add a filer.toml file to "+

weed/server/filer_server_handlers_read.go

@@ -1,6 +1,7 @@
 package weed_server
 
 import (
+	"context"
 	"io"
 	"mime"
 	"mime/multipart"
@@ -21,7 +22,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 		path = path[:len(path)-1]
 	}
 
-	entry, err := fs.filer.FindEntry(filer2.FullPath(path))
+	entry, err := fs.filer.FindEntry(context.Background(), filer2.FullPath(path))
 	if err != nil {
 		if path == "/" {
 			fs.listDirectoryHandler(w, r)
@@ -51,6 +52,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 	if r.Method == "HEAD" {
 		w.Header().Set("Content-Length", strconv.FormatInt(int64(filer2.TotalSize(entry.Chunks)), 10))
 		w.Header().Set("Last-Modified", entry.Attr.Mtime.Format(http.TimeFormat))
+		setEtag(w, filer2.ETag(entry.Chunks))
 		return
 	}
 
@@ -109,13 +111,16 @@ func (fs *FilerServer) handleSingleChunk(w http.ResponseWriter, r *http.Request,
 	for k, v := range resp.Header {
 		w.Header()[k] = v
 	}
+	if entry.Attr.Mime != "" {
+		w.Header().Set("Content-Type", entry.Attr.Mime)
+	}
 	w.WriteHeader(resp.StatusCode)
 	io.Copy(w, resp.Body)
 }
 
 func (fs *FilerServer) handleMultipleChunks(w http.ResponseWriter, r *http.Request, entry *filer2.Entry) {
 
-	mimeType := entry.Mime
+	mimeType := entry.Attr.Mime
 	if mimeType == "" {
 		if ext := path.Ext(entry.Name()); ext != "" {
 			mimeType = mime.TypeByExtension(ext)

weed/server/filer_server_handlers_read_dir.go

@@ -1,6 +1,7 @@
 package weed_server
 
 import (
+	"context"
 	"net/http"
 	"strconv"
 	"strings"
@@ -27,7 +28,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
 
 	lastFileName := r.FormValue("lastFileName")
 
-	entries, err := fs.filer.ListDirectoryEntries(filer2.FullPath(path), lastFileName, false, limit)
+	entries, err := fs.filer.ListDirectoryEntries(context.Background(), filer2.FullPath(path), lastFileName, false, limit)
 
 	if err != nil {
 		glog.V(0).Infof("listDirectory %s %s %d: %s", path, lastFileName, limit, err)

weed/server/filer_server_handlers_write.go

@@ -1,12 +1,15 @@
 package weed_server
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"io/ioutil"
+	"mime"
 	"net/http"
 	"net/url"
 	"os"
+	filenamePath "path"
 	"strconv"
 	"strings"
 	"time"
@@ -66,6 +69,8 @@ func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request,
 
 func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 
+	ctx := context.Background()
+
 	query := r.URL.Query()
 	replication := query.Get("replication")
 	if replication == "" {
@@ -80,7 +85,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 		dataCenter = fs.option.DataCenter
 	}
 
-	if autoChunked := fs.autoChunk(w, r, replication, collection, dataCenter); autoChunked {
+	if autoChunked := fs.autoChunk(ctx, w, r, replication, collection, dataCenter); autoChunked {
 		return
 	}
 
@@ -163,7 +168,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// update metadata in filer store
-	existingEntry, err := fs.filer.FindEntry(filer2.FullPath(path))
+	existingEntry, err := fs.filer.FindEntry(ctx, filer2.FullPath(path))
 	crTime := time.Now()
 	if err == nil && existingEntry != nil {
 		// glog.V(4).Infof("existing %s => %+v", path, existingEntry)
@@ -192,8 +197,11 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 			ETag:   etag,
 		}},
 	}
+	if ext := filenamePath.Ext(path); ext != "" {
+		entry.Attr.Mime = mime.TypeByExtension(ext)
+	}
 	// glog.V(4).Infof("saving %s => %+v", path, entry)
-	if db_err := fs.filer.CreateEntry(entry); db_err != nil {
+	if db_err := fs.filer.CreateEntry(ctx, entry); db_err != nil {
 		fs.filer.DeleteFileByFileId(fileId)
 		glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err)
 		writeJsonError(w, r, http.StatusInternalServerError, db_err)
@@ -218,7 +226,7 @@ func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
 
 	isRecursive := r.FormValue("recursive") == "true"
 
-	err := fs.filer.DeleteEntryMetaAndData(filer2.FullPath(r.URL.Path), isRecursive, true)
+	err := fs.filer.DeleteEntryMetaAndData(context.Background(), filer2.FullPath(r.URL.Path), isRecursive, true)
 	if err != nil {
 		glog.V(1).Infoln("deleting", r.URL.Path, ":", err.Error())
 		writeJsonError(w, r, http.StatusInternalServerError, err)

weed/server/filer_server_handlers_write_autochunk.go

@@ -2,6 +2,7 @@ package weed_server
 
 import (
 	"bytes"
+	"context"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -18,7 +19,8 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
-func (fs *FilerServer) autoChunk(w http.ResponseWriter, r *http.Request, replication string, collection string, dataCenter string) bool {
+func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request,
+	replication string, collection string, dataCenter string) bool {
 	if r.Method != "POST" {
 		glog.V(4).Infoln("AutoChunking not supported for method", r.Method)
 		return false
@@ -54,7 +56,7 @@ func (fs *FilerServer) autoChunk(w http.ResponseWriter, r *http.Request, replica
 		return false
 	}
 
-	reply, err := fs.doAutoChunk(w, r, contentLength, chunkSize, replication, collection, dataCenter)
+	reply, err := fs.doAutoChunk(ctx, w, r, contentLength, chunkSize, replication, collection, dataCenter)
 	if err != nil {
 		writeJsonError(w, r, http.StatusInternalServerError, err)
 	} else if reply != nil {
@@ -63,7 +65,8 @@ func (fs *FilerServer) autoChunk(w http.ResponseWriter, r *http.Request, replica
 	return true
 }
 
-func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, contentLength int64, chunkSize int32, replication string, collection string, dataCenter string) (filerResult *FilerPostResult, replyerr error) {
+func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request,
+	contentLength int64, chunkSize int32, replication string, collection string, dataCenter string) (filerResult *FilerPostResult, replyerr error) {
 
 	multipartReader, multipartReaderErr := r.MultipartReader()
 	if multipartReaderErr != nil {
@@ -166,7 +169,7 @@ func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, conte
 		},
 		Chunks: fileChunks,
 	}
-	if db_err := fs.filer.CreateEntry(entry); db_err != nil {
+	if db_err := fs.filer.CreateEntry(ctx, entry); db_err != nil {
 		replyerr = db_err
 		filerResult.Error = db_err.Error()
 		glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err)
@@ -176,8 +179,8 @@ func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, conte
 	return
 }
 
-func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request, chunkBuf []byte, fileName string, contentType string, fileId string, auth security.EncodedJwt) (err error) {
-	err = nil
+func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request,
+	chunkBuf []byte, fileName string, contentType string, fileId string, auth security.EncodedJwt) (err error) {
 
 	ioReader := ioutil.NopCloser(bytes.NewBuffer(chunkBuf))
 	uploadResult, uploadError := operation.Upload(urlLocation, fileName, ioReader, false, contentType, nil, auth)

weed/server/filer_ui/breadcrumb.go

@@ -16,7 +16,7 @@ func ToBreadcrumb(fullpath string) (crumbs []Breadcrumb) {
 	for i := 0; i < len(parts); i++ {
 		crumbs = append(crumbs, Breadcrumb{
 			Name: parts[i] + "/",
-			Link: "/" + filepath.Join(parts[0:i+1]...),
+			Link: "/" + filepath.ToSlash(filepath.Join(parts[0:i+1]...)),
 		})
 	}
 

weed/server/filer_ui/templates.go

@@ -162,7 +162,7 @@ function uploadFile(file, i) {
   var url = window.location.href
   var xhr = new XMLHttpRequest()
   var formData = new FormData()
-  xhr.open('POST', url, true)
+  xhr.open('POST', url, false)
 
   formData.append('file', file)
   xhr.send(formData)

weed/server/master_grpc_server.go

@@ -63,7 +63,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 			rack := dc.GetOrCreateRack(rackName)
 			dn = rack.GetOrCreateDataNode(heartbeat.Ip,
 				int(heartbeat.Port), heartbeat.PublicUrl,
-				int(heartbeat.MaxVolumeCount))
+				int64(heartbeat.MaxVolumeCount))
 			glog.V(0).Infof("added volume server %v:%d", heartbeat.GetIp(), heartbeat.GetPort())
 			if err := stream.Send(&master_pb.HeartbeatResponse{
 				VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024,
@@ -102,12 +102,13 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 
 		// tell the volume servers about the leader
 		newLeader, err := t.Leader()
-		if err == nil {
-			if err := stream.Send(&master_pb.HeartbeatResponse{
-				Leader: newLeader,
-			}); err != nil {
-				return err
-			}
+		if err != nil {
+			return err
+		}
+		if err := stream.Send(&master_pb.HeartbeatResponse{
+			Leader: newLeader,
+		}); err != nil {
+			return err
 		}
 	}
 }

weed/server/master_grpc_server_collection.go

@@ -0,0 +1,56 @@
+package weed_server
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/raft"
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func (ms *MasterServer) CollectionList(ctx context.Context, req *master_pb.CollectionListRequest) (*master_pb.CollectionListResponse, error) {
+
+	if !ms.Topo.IsLeader() {
+		return nil, raft.NotLeaderError
+	}
+
+	resp := &master_pb.CollectionListResponse{}
+	collections := ms.Topo.ListCollections()
+	for _, c := range collections {
+		resp.Collections = append(resp.Collections, &master_pb.Collection{
+			Name: c.Name,
+		})
+	}
+
+	return resp, nil
+}
+
+func (ms *MasterServer) CollectionDelete(ctx context.Context, req *master_pb.CollectionDeleteRequest) (*master_pb.CollectionDeleteResponse, error) {
+
+	if !ms.Topo.IsLeader() {
+		return nil, raft.NotLeaderError
+	}
+
+	resp := &master_pb.CollectionDeleteResponse{}
+
+	collection, ok := ms.Topo.FindCollection(req.GetName())
+	if !ok {
+		return resp, fmt.Errorf("collection not found: %v", req.GetName())
+	}
+
+	for _, server := range collection.ListVolumeServers() {
+		err := operation.WithVolumeServerClient(server.Url(), ms.grpcDialOpiton, func(client volume_server_pb.VolumeServerClient) error {
+			_, deleteErr := client.DeleteCollection(context.Background(), &volume_server_pb.DeleteCollectionRequest{
+				Collection: collection.Name,
+			})
+			return deleteErr
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+	ms.Topo.DeleteCollection(req.GetName())
+
+	return resp, nil
+}

weed/server/master_grpc_server_volume.go

@@ -126,3 +126,16 @@ func (ms *MasterServer) Statistics(ctx context.Context, req *master_pb.Statistic
 
 	return resp, nil
 }
+
+func (ms *MasterServer) VolumeList(ctx context.Context, req *master_pb.VolumeListRequest) (*master_pb.VolumeListResponse, error) {
+
+	if !ms.Topo.IsLeader() {
+		return nil, raft.NotLeaderError
+	}
+
+	resp := &master_pb.VolumeListResponse{
+		TopologyInfo: ms.Topo.ToTopologyInfo(),
+	}
+
+	return resp, nil
+}

weed/server/master_server.go

@@ -49,6 +49,7 @@ func NewMasterServer(r *mux.Router, port int, metaFolder string,
 	defaultReplicaPlacement string,
 	garbageThreshold float64,
 	whiteList []string,
+	disableHttp bool,
 ) *MasterServer {
 
 	v := viper.GetViper()
@@ -76,21 +77,23 @@ func NewMasterServer(r *mux.Router, port int, metaFolder string,
 
 	ms.guard = security.NewGuard(whiteList, signingKey)
 
-	handleStaticResources2(r)
-	r.HandleFunc("/", ms.uiStatusHandler)
-	r.HandleFunc("/ui/index.html", ms.uiStatusHandler)
-	r.HandleFunc("/dir/assign", ms.proxyToLeader(ms.guard.WhiteList(ms.dirAssignHandler)))
-	r.HandleFunc("/dir/lookup", ms.proxyToLeader(ms.guard.WhiteList(ms.dirLookupHandler)))
-	r.HandleFunc("/dir/status", ms.proxyToLeader(ms.guard.WhiteList(ms.dirStatusHandler)))
-	r.HandleFunc("/col/delete", ms.proxyToLeader(ms.guard.WhiteList(ms.collectionDeleteHandler)))
-	r.HandleFunc("/vol/grow", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeGrowHandler)))
-	r.HandleFunc("/vol/status", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeStatusHandler)))
-	r.HandleFunc("/vol/vacuum", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeVacuumHandler)))
-	r.HandleFunc("/submit", ms.guard.WhiteList(ms.submitFromMasterServerHandler))
-	r.HandleFunc("/stats/health", ms.guard.WhiteList(statsHealthHandler))
-	r.HandleFunc("/stats/counter", ms.guard.WhiteList(statsCounterHandler))
-	r.HandleFunc("/stats/memory", ms.guard.WhiteList(statsMemoryHandler))
-	r.HandleFunc("/{fileId}", ms.proxyToLeader(ms.redirectHandler))
+	if !disableHttp {
+		handleStaticResources2(r)
+		r.HandleFunc("/", ms.proxyToLeader(ms.uiStatusHandler))
+		r.HandleFunc("/ui/index.html", ms.uiStatusHandler)
+		r.HandleFunc("/dir/assign", ms.proxyToLeader(ms.guard.WhiteList(ms.dirAssignHandler)))
+		r.HandleFunc("/dir/lookup", ms.proxyToLeader(ms.guard.WhiteList(ms.dirLookupHandler)))
+		r.HandleFunc("/dir/status", ms.proxyToLeader(ms.guard.WhiteList(ms.dirStatusHandler)))
+		r.HandleFunc("/col/delete", ms.proxyToLeader(ms.guard.WhiteList(ms.collectionDeleteHandler)))
+		r.HandleFunc("/vol/grow", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeGrowHandler)))
+		r.HandleFunc("/vol/status", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeStatusHandler)))
+		r.HandleFunc("/vol/vacuum", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeVacuumHandler)))
+		r.HandleFunc("/submit", ms.guard.WhiteList(ms.submitFromMasterServerHandler))
+		r.HandleFunc("/stats/health", ms.guard.WhiteList(statsHealthHandler))
+		r.HandleFunc("/stats/counter", ms.guard.WhiteList(statsCounterHandler))
+		r.HandleFunc("/stats/memory", ms.guard.WhiteList(statsMemoryHandler))
+		r.HandleFunc("/{fileId}", ms.proxyToLeader(ms.redirectHandler))
+	}
 
 	ms.Topo.StartRefreshWritableVolumes(ms.grpcDialOpiton, garbageThreshold, ms.preallocate)
 

weed/server/master_server_handlers_admin.go

@@ -68,8 +68,8 @@ func (ms *MasterServer) volumeGrowHandler(w http.ResponseWriter, r *http.Request
 	}
 	if err == nil {
 		if count, err = strconv.Atoi(r.FormValue("count")); err == nil {
-			if ms.Topo.FreeSpace() < count*option.ReplicaPlacement.GetCopyCount() {
-				err = errors.New("Only " + strconv.Itoa(ms.Topo.FreeSpace()) + " volumes left! Not enough for " + strconv.Itoa(count*option.ReplicaPlacement.GetCopyCount()))
+			if ms.Topo.FreeSpace() < int64(count*option.ReplicaPlacement.GetCopyCount()) {
+				err = fmt.Errorf("only %d volumes left, not enough for %d", ms.Topo.FreeSpace(), count*option.ReplicaPlacement.GetCopyCount())
 			} else {
 				count, err = ms.vg.GrowByCountAndType(ms.grpcDialOpiton, count, option, ms.Topo)
 			}

weed/server/master_ui/templates.go

@@ -41,7 +41,7 @@ var StatusTpl = template.Must(template.New("status").Parse(`<!DOCTYPE html>
                 <td class="col-sm-2 field-label"><label>Other Masters:</label></td>
                 <td class="col-sm-10"><ul class="list-unstyled">
                 {{ range $k, $p := .Peers }}
-                  <li><a href="http://{{ $p.Name }}">{{ $p.Name }}</a></li>
+                  <li><a href="http://{{ $p.Name }}/ui/index.html">{{ $p.Name }}</a></li>
                 {{ end }}
                 </ul></td>
               </tr>

weed/server/raft_server.go

@@ -61,7 +61,7 @@ func NewRaftServer(grpcDialOption grpc.DialOption, peers []string, serverAddr st
 	s.raftServer.Start()
 
 	for _, peer := range s.peers {
-		s.raftServer.AddPeer(peer, util.ServerToGrpcAddress(peer, 19333))
+		s.raftServer.AddPeer(peer, util.ServerToGrpcAddress(peer))
 	}
 
 	s.GrpcServer = raft.NewGrpcServer(s.raftServer)
@@ -72,7 +72,7 @@ func NewRaftServer(grpcDialOption grpc.DialOption, peers []string, serverAddr st
 
 		_, err := s.raftServer.Do(&raft.DefaultJoinCommand{
 			Name:             s.raftServer.Name(),
-			ConnectionString: util.ServerToGrpcAddress(s.serverAddr, 19333),
+			ConnectionString: util.ServerToGrpcAddress(s.serverAddr),
 		})
 
 		if err != nil {

weed/server/raft_server_handlers.go

@@ -0,0 +1,22 @@
+package weed_server
+
+import (
+	"net/http"
+)
+
+type ClusterStatusResult struct {
+	IsLeader bool     `json:"IsLeader,omitempty"`
+	Leader   string   `json:"Leader,omitempty"`
+	Peers    []string `json:"Peers,omitempty"`
+}
+
+func (s *RaftServer) StatusHandler(w http.ResponseWriter, r *http.Request) {
+	ret := ClusterStatusResult{
+		IsLeader: s.topo.IsLeader(),
+		Peers:    s.Peers(),
+	}
+	if leader, e := s.topo.Leader(); e == nil {
+		ret.Leader = leader
+	}
+	writeJsonQuiet(w, r, http.StatusOK, ret)
+}

weed/server/volume_grpc_admin.go

@@ -29,7 +29,7 @@ func (vs *VolumeServer) AssignVolume(ctx context.Context, req *volume_server_pb.
 	resp := &volume_server_pb.AssignVolumeResponse{}
 
 	err := vs.store.AddVolume(
-		storage.VolumeId(req.VolumdId),
+		storage.VolumeId(req.VolumeId),
 		req.Collection,
 		vs.needleMapKind,
 		req.Replication,
@@ -51,7 +51,7 @@ func (vs *VolumeServer) VolumeMount(ctx context.Context, req *volume_server_pb.V
 
 	resp := &volume_server_pb.VolumeMountResponse{}
 
-	err := vs.store.MountVolume(storage.VolumeId(req.VolumdId))
+	err := vs.store.MountVolume(storage.VolumeId(req.VolumeId))
 
 	if err != nil {
 		glog.Errorf("volume mount %v: %v", req, err)
@@ -67,7 +67,7 @@ func (vs *VolumeServer) VolumeUnmount(ctx context.Context, req *volume_server_pb
 
 	resp := &volume_server_pb.VolumeUnmountResponse{}
 
-	err := vs.store.UnmountVolume(storage.VolumeId(req.VolumdId))
+	err := vs.store.UnmountVolume(storage.VolumeId(req.VolumeId))
 
 	if err != nil {
 		glog.Errorf("volume unmount %v: %v", req, err)
@@ -83,7 +83,7 @@ func (vs *VolumeServer) VolumeDelete(ctx context.Context, req *volume_server_pb.
 
 	resp := &volume_server_pb.VolumeDeleteResponse{}
 
-	err := vs.store.DeleteVolume(storage.VolumeId(req.VolumdId))
+	err := vs.store.DeleteVolume(storage.VolumeId(req.VolumeId))
 
 	if err != nil {
 		glog.Errorf("volume delete %v: %v", req, err)