fix compilation error for windows

right before changing to filer2

.travis.yml

@@ -1,8 +1,7 @@
 sudo: false
 language: go
 go:
-- 1.5
-- 1.6
+- 1.9
 - tip
 
 before_install:

README.md

@@ -3,11 +3,54 @@
 [![Build Status](https://travis-ci.org/chrislusf/seaweedfs.svg?branch=master)](https://travis-ci.org/chrislusf/seaweedfs)
 [![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed)
 [![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki)
-[download](https://github.com/chrislusf/seaweedfs/releases/latest)
 
 ![SeaweedFS Logo](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/seaweedfs.png)
 
-[![](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=EEECLJ8QGTTPC) 
+<h2 align="center">Supporting SeaweedFS</h2>
+
+SeaweedFS is Apache-licensed open source project, independent project with its ongoing development made 
+possible entirely thanks to the support by these awesome [backers](https://github.com/chrislusf/seaweedfs/blob/master/backers.md). 
+If you'd like to grow SeaweedFS even stronger, please consider to 
+<a href="https://www.patreon.com/seaweedfs">Sponsor SeaweedFS via Patreon</a>.
+
+Platinum ($2500/month), Gold ($500/month): put your company logo on the SeaweedFS github page
+Generous Backer($50/month), Backer($10/month): put your name on the SeaweedFS backer page.
+
+Your support will be really appreciated by me and other supporters!
+
+<h3 align="center"><a href="https://www.patreon.com/seaweedfs">Sponsors SeaweedFS via Patreon</a></h3>
+
+<h4 align="center">Platinum</h4>
+
+<p align="center">
+  <a href="" target="_blank">
+    Add your name or icon here
+  </a>
+</p>
+
+<h4 align="center">Gold</h4>
+
+<table>
+  <tbody>
+    <tr>
+      <td align="center" valign="middle">
+        <a href="" target="_blank">
+          Add your name or icon here
+        </a>
+      </td>
+    </tr>
+    <tr></tr>
+  </tbody>
+</table>
+
+---
+
+
+- [Download Binaries for different platforms](https://github.com/chrislusf/seaweedfs/releases/latest)
+- [SeaweedFS on Slack](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTc4MmVlYmFlNjBmZTgzZmJlYmI1MDE1YzkyNWYyZjkwZDFiM2RlMDdjNjVlNjdjYzc4NGFhZGIyYzEyMzJkYTA)
+- [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
+- [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
+
 
 ## Introduction
 
@@ -16,7 +59,7 @@ SeaweedFS is a simple and highly scalable distributed file system. There are two
 1. to store billions of files!
 2. to serve the files fast!
 
-Instead of supporting full POSIX file system semantics, SeaweedFS choose to implement only a key->file mapping. Similar to "NoSQL", you might call it "NoFS".
+Instead of supporting full POSIX file system semantics, SeaweedFS chooses to implement only a key->file mapping. Similar to "NoSQL", you might call it "NoFS".
 
 Instead of managing all file metadata in a central master, the central master only manages file volumes, and it lets these volume servers manage files and their metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers, allowing faster file access (just one disk read operation).
 
@@ -24,10 +67,6 @@ There is only a 40 bytes disk storage overhead for each file's metadata. It is s
 
 SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf). SeaweedFS is currently growing, with more features on the way.
 
-[Download latest compiled binaries for different platforms](https://github.com/chrislusf/seaweedfs/releases/latest)
-
-[SeaweedFS Discussion Group](https://groups.google.com/d/forum/seaweedfs)
-
 ## Additional Features
 * Can choose no replication or different replication level, rack and data center aware
 * Automatic master servers failover - no single point of failure (SPOF)
@@ -38,12 +77,12 @@ SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.
 * Optional [filer server][Filer] provides "normal" directories and files via http
 * Optionally fix the orientation for jpeg pictures
 * Support Etag, Accept-Range, Last-Modified, etc.
-* Support in-memory/leveldb/boltdb mode tuning for memory/performance balance.
+* Support in-memory/leveldb/boltdb/btree mode tuning for memory/performance balance.
 
 [Filer]: https://github.com/chrislusf/seaweedfs/wiki/Filer
 
 ## Example Usage
-By default, the master node runs on port 9333, and the volume nodes runs on port 8080.
+By default, the master node runs on port 9333, and the volume nodes run on port 8080.
 Here I will start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. I just use localhost as an example.
 
 SeaweedFS uses HTTP REST operations to write, read, delete. The responses are in JSON or JSONP format.
@@ -103,14 +142,14 @@ If space is really a concern, you can store the file id in your own format. You
 
 Here is an example of how to render the URL.
 
-First lookup the volume server's URLs by the file's volumeId:
+First look up the volume server's URLs by the file's volumeId:
 
 ```
 > curl http://localhost:9333/dir/lookup?volumeId=3
 {"locations":[{"publicUrl":"localhost:8080","url":"localhost:8080"}]}
 ```
 
-Since (usually) there are not too many volume servers, and volumes don't move often, you can cache the results most of the time. Depends on the replication type, one volume can have multiple replica locations. Just randomly pick one location to read.)
+Since (usually) there are not too many volume servers, and volumes don't move often, you can cache the results most of the time. Depending on the replication type, one volume can have multiple replica locations. Just randomly pick one location to read.
 
 Now you can take the public url, render the url or directly read from the volume server via url:
 
@@ -130,10 +169,12 @@ If you want a nicer URL, you can use one of these alternative URL formats:
  http://localhost:8080/3,01637037d6
 ```
 
-If you want get a scaled version of an image, you can add some params:
+If you want to get a scaled version of an image, you can add some params:
 
 ```
 http://localhost:8080/3/01637037d6.jpg?height=200&width=200
+http://localhost:8080/3/01637037d6.jpg?height=200&width=200&mode=fit
+http://localhost:8080/3/01637037d6.jpg?height=200&width=200&mode=fill
 ```
 
 ### Rack-Aware and Data Center-Aware Replication ###
@@ -238,7 +279,7 @@ Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more com
 
 SeaweedFS is meant to be fast and simple, both during usage and during setup. If you do not understand how it works when you reach here, we failed! Please raise an issue with any questions or update this file with clarifications.
 
-SeaweedFS has a centralized master to lookup free volumes, while Ceph uses hashing to locate its objects. Having a centralized master makes it easy to code and manage. HDFS/GFS has the single name node for years. SeaweedFS now support multiple master nodes.
+SeaweedFS has a centralized master to look up free volumes, while Ceph uses hashing to locate its objects. Having a centralized master makes it easy to code and manage. HDFS/GFS has the single name node for years. SeaweedFS now support multiple master nodes.
 
 Ceph hashing avoids SPOF, but makes it complicated when moving or adding servers.
 
@@ -305,7 +346,7 @@ step 4: after you modify your code locally, you could start a local build by cal
 
 ### Hard Drive Performance ###
 
-When testing read performance on SeaweedFS, it basically becomes performance test your hard drive's random read speed. Hard Drive usually get 100MB/s~200MB/s.
+When testing read performance on SeaweedFS, it basically becomes performance test for your hard drive's random read speed. Hard Drive usually get 100MB/s~200MB/s.
 
 ### Solid State Disk
 

backers.md

@@ -0,0 +1,12 @@
+<h1 align="center">Sponsors &amp; Backers</h1>
+
+- [Become a backer or sponsor on Patreon](https://www.patreon.com/seaweedfs).
+
+<h2 align="center">Generous Backers ($50+)</h2>
+
+- [Evercam Camera Management Software](https://evercam.io/)
+
+<h2 align="center">Backers</h2>
+
+- [ColorfulClouds Tech Co. Ltd.](https://caiyunai.com/)
+- [Haravan - Ecommerce Platform](https://www.haravan.com)

docker/Dockerfile

@@ -1,9 +1,10 @@
-FROM frolvlad/alpine-glibc:alpine-3.4
+FROM frolvlad/alpine-glibc:alpine-3.5
 
-RUN apk add --no-cache --virtual=build-dependencies --update curl wget ca-certificates && \
-    wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/latest | egrep -o '/chrislusf/seaweedfs/releases/download/.*/linux_amd64.tar.gz') && \
+# Tried to use curl only (curl -o /tmp/linux_amd64.tar.gz ...), however it turned out that the following tar command failed with "gzip: stdin: not in gzip format"
+RUN apk add --no-cache --virtual build-dependencies --update wget curl ca-certificates && \
+    wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/latest | egrep -o 'chrislusf/seaweedfs/releases/download/.*/linux_amd64.tar.gz') && \
     tar -C /usr/bin/ -xzvf /tmp/linux_amd64.tar.gz && \
-    apk del curl wget ca-certificates build-dependencies && \
+    apk del build-dependencies && \
     rm -rf /tmp/*
 
 EXPOSE 8080

weed/command/benchmark.go

@@ -201,9 +201,11 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 		}()
 	}
 
+	random := rand.New(rand.NewSource(time.Now().UnixNano()))
+
 	for id := range idChan {
 		start := time.Now()
-		fileSize := int64(*b.fileSize + rand.Intn(64))
+		fileSize := int64(*b.fileSize + random.Intn(64))
 		fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: fileSize}, FileSize: fileSize}
 		ar := &operation.VolumeAssignRequest{
 			Count:      1,
@@ -212,7 +214,7 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 		if assignResult, err := operation.Assign(*b.server, ar); err == nil {
 			fp.Server, fp.Fid, fp.Collection = assignResult.Url, assignResult.Fid, *b.collection
 			if _, err := fp.Upload(0, *b.server, secret); err == nil {
-				if rand.Intn(100) < *b.deletePercentage {
+				if random.Intn(100) < *b.deletePercentage {
 					s.total++
 					delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp}
 				} else {
@@ -239,6 +241,8 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 
 func readFiles(fileIdLineChan chan string, s *stat) {
 	defer wait.Done()
+	random := rand.New(rand.NewSource(time.Now().UnixNano()))
+
 	for fid := range fileIdLineChan {
 		if len(fid) == 0 {
 			continue
@@ -258,7 +262,7 @@ func readFiles(fileIdLineChan chan string, s *stat) {
 			println("!!!! volume id ", vid, " location not found!!!!!")
 			continue
 		}
-		server := ret.Locations[rand.Intn(len(ret.Locations))].Url
+		server := ret.Locations[random.Intn(len(ret.Locations))].Url
 		url := "http://" + server + "/" + fid
 		if bytesRead, err := util.Get(url); err == nil {
 			s.completed++
@@ -297,6 +301,8 @@ func readFileIds(fileName string, fileIdLineChan chan string) {
 	}
 	defer file.Close()
 
+	random := rand.New(rand.NewSource(time.Now().UnixNano()))
+
 	r := bufio.NewReader(file)
 	if *b.sequentialRead {
 		for {
@@ -317,7 +323,7 @@ func readFileIds(fileName string, fileIdLineChan chan string) {
 		}
 		if len(lines) > 0 {
 			for i := 0; i < readStats.total; i++ {
-				fileIdLineChan <- lines[rand.Intn(len(lines))]
+				fileIdLineChan <- lines[random.Intn(len(lines))]
 			}
 		}
 	}

weed/command/compact.go

@@ -33,14 +33,16 @@ func runCompact(cmd *Command, args []string) bool {
 		return false
 	}
 
+	preallocate := *compactVolumePreallocate * (1 << 20)
+
 	vid := storage.VolumeId(*compactVolumeId)
 	v, err := storage.NewVolume(*compactVolumePath, *compactVolumeCollection, vid,
-		storage.NeedleMapInMemory, nil, nil, *compactVolumePreallocate*(1<<20))
+		storage.NeedleMapInMemory, nil, nil, preallocate)
 	if err != nil {
 		glog.Fatalf("Load Volume [ERROR] %s\n", err)
 	}
 	if *compactMethod == 0 {
-		if err = v.Compact(); err != nil {
+		if err = v.Compact(preallocate); err != nil {
 			glog.Fatalf("Compact Volume [ERROR] %s\n", err)
 		}
 	} else {

weed/command/export.go

@@ -118,7 +118,7 @@ func runExport(cmd *Command, args []string) bool {
 	}
 	defer indexFile.Close()
 
-	needleMap, err := storage.LoadNeedleMap(indexFile)
+	needleMap, err := storage.LoadBtreeNeedleMap(indexFile)
 	if err != nil {
 		glog.Fatalf("cannot load needle map from %s: %s", indexFile.Name(), err)
 	}

weed/command/filer.go

@@ -9,6 +9,10 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/soheilhy/cmux"
+	"google.golang.org/grpc/reflection"
+	"google.golang.org/grpc"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 )
 
 var (
@@ -19,6 +23,7 @@ type FilerOptions struct {
 	master                  *string
 	ip                      *string
 	port                    *int
+	publicPort              *int
 	collection              *string
 	defaultReplicaPlacement *string
 	dir                     *string
@@ -40,15 +45,16 @@ func init() {
 	f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection")
 	f.ip = cmdFiler.Flag.String("ip", "", "filer server http listen ip address")
 	f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
+	f.publicPort = cmdFiler.Flag.Int("port.public", 0, "port opened to public")
 	f.dir = cmdFiler.Flag.String("dir", os.TempDir(), "directory to store meta data")
 	f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified")
 	f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
 	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
 	f.confFile = cmdFiler.Flag.String("confFile", "", "json encoded filer conf file")
-	f.maxMB = cmdFiler.Flag.Int("maxMB", 0, "split files larger than the limit")
+	f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit")
 	f.cassandra_server = cmdFiler.Flag.String("cassandra.server", "", "host[:port] of the cassandra server")
 	f.cassandra_keyspace = cmdFiler.Flag.String("cassandra.keyspace", "seaweed", "keyspace of the cassandra server")
-	f.redis_server = cmdFiler.Flag.String("redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379")
+	f.redis_server = cmdFiler.Flag.String("redis.server", "", "comma separated host:port[,host2:port2]* of the redis server, e.g., 127.0.0.1:6379")
 	f.redis_password = cmdFiler.Flag.String("redis.password", "", "password in clear text")
 	f.redis_database = cmdFiler.Flag.Int("redis.database", 0, "the database on the redis server")
 	f.secretKey = cmdFiler.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
@@ -83,29 +89,72 @@ func runFiler(cmd *Command, args []string) bool {
 		glog.Fatalf("Check Meta Folder (-dir) Writable %s : %s", *f.dir, err)
 	}
 
-	r := http.NewServeMux()
-	_, nfs_err := weed_server.NewFilerServer(r, *f.ip, *f.port, *f.master, *f.dir, *f.collection,
-		*f.defaultReplicaPlacement, *f.redirectOnRead, *f.disableDirListing,
-		*f.confFile,
-		*f.maxMB,
-		*f.secretKey,
-		*f.cassandra_server, *f.cassandra_keyspace,
-		*f.redis_server, *f.redis_password, *f.redis_database,
+	f.start()
+
+	return true
+}
+
+func (fo *FilerOptions) start() {
+
+	defaultMux := http.NewServeMux()
+	publicVolumeMux := defaultMux
+
+	if *fo.publicPort != 0 {
+		publicVolumeMux = http.NewServeMux()
+	}
+
+	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux,
+		*fo.ip, *fo.port, *fo.master, *fo.dir, *fo.collection,
+		*fo.defaultReplicaPlacement, *fo.redirectOnRead, *fo.disableDirListing,
+		*fo.confFile,
+		*fo.maxMB,
+		*fo.secretKey,
+		*fo.cassandra_server, *fo.cassandra_keyspace,
+		*fo.redis_server, *fo.redis_password, *fo.redis_database,
 	)
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)
 	}
-	glog.V(0).Infoln("Start Seaweed Filer", util.VERSION, "at port", strconv.Itoa(*f.port))
+
+	if *fo.publicPort != 0 {
+		publicListeningAddress := *fo.ip + ":" + strconv.Itoa(*fo.publicPort)
+		glog.V(0).Infoln("Start Seaweed filer server", util.VERSION, "public at", publicListeningAddress)
+		publicListener, e := util.NewListener(publicListeningAddress, 0)
+		if e != nil {
+			glog.Fatalf("Filer server public listener error on port %d:%v", *fo.publicPort, e)
+		}
+		go func() {
+			if e := http.Serve(publicListener, publicVolumeMux); e != nil {
+				glog.Fatalf("Volume server fail to serve public: %v", e)
+			}
+		}()
+	}
+
+	glog.V(0).Infoln("Start Seaweed Filer", util.VERSION, "at port", strconv.Itoa(*fo.port))
 	filerListener, e := util.NewListener(
-		":"+strconv.Itoa(*f.port),
+		":"+strconv.Itoa(*fo.port),
 		time.Duration(10)*time.Second,
 	)
 	if e != nil {
 		glog.Fatalf("Filer listener error: %v", e)
 	}
-	if e := http.Serve(filerListener, r); e != nil {
+
+	m := cmux.New(filerListener)
+	grpcL := m.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))
+	httpL := m.Match(cmux.Any())
+
+	// Create your protocol servers.
+	grpcS := grpc.NewServer()
+	filer_pb.RegisterSeaweedFilerServer(grpcS, fs)
+	reflection.Register(grpcS)
+
+	httpS := &http.Server{Handler: defaultMux}
+
+	go grpcS.Serve(grpcL)
+	go httpS.Serve(httpL)
+
+	if err := m.Serve(); err != nil {
 		glog.Fatalf("Filer Fail to serve: %v", e)
 	}
 
-	return true
 }

weed/command/fix.go

@@ -43,7 +43,7 @@ func runFix(cmd *Command, args []string) bool {
 	}
 	defer indexFile.Close()
 
-	nm := storage.NewNeedleMap(indexFile)
+	nm := storage.NewBtreeNeedleMap(indexFile)
 	defer nm.Close()
 
 	vid := storage.VolumeId(*fixVolumeId)

weed/command/master.go

@@ -4,13 +4,12 @@ import (
 	"net/http"
 	"os"
 	"runtime"
-	"runtime/pprof"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/gorilla/mux"
@@ -41,7 +40,6 @@ var (
 	volumeSizeLimitMB       = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
 	volumePreallocate       = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.")
 	mpulse                  = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
-	confFile                = cmdMaster.Flag.String("conf", "/etc/weedfs/weedfs.conf", "Deprecating! xml configuration file")
 	defaultReplicaPlacement = cmdMaster.Flag.String("defaultReplication", "000", "Default replication type if not specified.")
 	// mTimeout                = cmdMaster.Flag.Int("idleTimeout", 30, "connection idle seconds")
 	mMaxCpu               = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
@@ -49,6 +47,7 @@ var (
 	masterWhiteListOption = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
 	masterSecureKey       = cmdMaster.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
 	masterCpuProfile      = cmdMaster.Flag.String("cpuprofile", "", "cpu profile output file")
+	masterMemProfile      = cmdMaster.Flag.String("memprofile", "", "memory profile output file")
 
 	masterWhiteList []string
 )
@@ -58,28 +57,22 @@ func runMaster(cmd *Command, args []string) bool {
 		*mMaxCpu = runtime.NumCPU()
 	}
 	runtime.GOMAXPROCS(*mMaxCpu)
-	if *masterCpuProfile != "" {
-		f, err := os.Create(*masterCpuProfile)
-		if err != nil {
-			glog.Fatal(err)
-		}
-		pprof.StartCPUProfile(f)
-		defer pprof.StopCPUProfile()
-		OnInterrupt(func() {
-			pprof.StopCPUProfile()
-		})
-	}
+	util.SetupProfiling(*masterCpuProfile, *masterMemProfile)
+
 	if err := util.TestFolderWritable(*metaFolder); err != nil {
 		glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *metaFolder, err)
 	}
 	if *masterWhiteListOption != "" {
 		masterWhiteList = strings.Split(*masterWhiteListOption, ",")
 	}
+	if *volumeSizeLimitMB > 30*1000 {
+		glog.Fatalf("volumeSizeLimitMB should be smaller than 30000")
+	}
 
 	r := mux.NewRouter()
 	ms := weed_server.NewMasterServer(r, *mport, *metaFolder,
 		*volumeSizeLimitMB, *volumePreallocate,
-		*mpulse, *confFile, *defaultReplicaPlacement, *garbageThreshold,
+		*mpulse, *defaultReplicaPlacement, *garbageThreshold,
 		masterWhiteList, *masterSecureKey,
 	)
 
@@ -111,7 +104,7 @@ func runMaster(cmd *Command, args []string) bool {
 
 	// Create your protocol servers.
 	grpcS := grpc.NewServer()
-	pb.RegisterSeaweedServer(grpcS, ms)
+	master_pb.RegisterSeaweedServer(grpcS, ms)
 	reflection.Register(grpcS)
 
 	httpS := &http.Server{Handler: r}

weed/command/mount.go

@@ -26,7 +26,7 @@ var cmdMount = &Command{
   2) have a "weed filer" running
   These 2 requirements can be achieved with one command "weed server -filer=true"
 
-  This uses bazil.org/fuse, whichenables writing FUSE file systems on
+  This uses bazil.org/fuse, which enables writing FUSE file systems on
   Linux, and OS X.
 
   On OS X, it requires OSXFUSE (http://osxfuse.github.com/).

weed/command/mount_std.go

@@ -8,11 +8,9 @@ import (
 
 	"bazil.org/fuse"
 	"bazil.org/fuse/fs"
-	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"golang.org/x/net/context"
+	"github.com/chrislusf/seaweedfs/weed/filesys"
 )
 
 func runMount(cmd *Command, args []string) bool {
@@ -22,18 +20,34 @@ func runMount(cmd *Command, args []string) bool {
 		return false
 	}
 
-	c, err := fuse.Mount(*mountOptions.dir)
+	fuse.Unmount(*mountOptions.dir)
+
+	c, err := fuse.Mount(
+		*mountOptions.dir,
+		fuse.VolumeName("SeaweedFS"),
+		fuse.FSName("SeaweedFS"),
+		fuse.NoAppleDouble(),
+		fuse.NoAppleXattr(),
+		fuse.ExclCreate(),
+		fuse.DaemonTimeout("3600"),
+		fuse.AllowOther(),
+		fuse.AllowSUID(),
+		fuse.DefaultPermissions(),
+		// fuse.MaxReadahead(1024*128), // TODO: not tested yet, possibly improving read performance
+		fuse.AsyncRead(),
+		fuse.WritebackCache(),
+	)
 	if err != nil {
 		glog.Fatal(err)
 		return false
 	}
 
-	OnInterrupt(func() {
+	util.OnInterrupt(func() {
 		fuse.Unmount(*mountOptions.dir)
 		c.Close()
 	})
 
-	err = fs.Serve(c, WFS{})
+	err = fs.Serve(c, filesys.NewSeaweedFileSystem(*mountOptions.filer))
 	if err != nil {
 		fuse.Unmount(*mountOptions.dir)
 	}
@@ -46,61 +60,3 @@ func runMount(cmd *Command, args []string) bool {
 
 	return true
 }
-
-type File struct {
-	FileId filer.FileId
-	Name   string
-}
-
-func (File) Attr(context context.Context, attr *fuse.Attr) error {
-	return nil
-}
-func (File) ReadAll(ctx context.Context) ([]byte, error) {
-	return []byte("hello, world\n"), nil
-}
-
-type Dir struct {
-	Path string
-	Id   uint64
-}
-
-func (dir Dir) Attr(context context.Context, attr *fuse.Attr) error {
-	return nil
-}
-
-func (dir Dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
-	files_result, e := filer.ListFiles(*mountOptions.filer, dir.Path, name)
-	if e != nil {
-		return nil, fuse.ENOENT
-	}
-	if len(files_result.Files) > 0 {
-		return File{files_result.Files[0].Id, files_result.Files[0].Name}, nil
-	}
-	return nil, fmt.Errorf("File Not Found for %s", name)
-}
-
-type WFS struct{}
-
-func (WFS) Root() (fs.Node, error) {
-	return Dir{}, nil
-}
-
-func (dir *Dir) ReadDir(ctx context.Context) ([]fuse.Dirent, error) {
-	var ret []fuse.Dirent
-	if dirs, e := filer.ListDirectories(*mountOptions.filer, dir.Path); e == nil {
-		for _, d := range dirs.Directories {
-			dirId := uint64(d.Id)
-			ret = append(ret, fuse.Dirent{Inode: dirId, Name: d.Name, Type: fuse.DT_Dir})
-		}
-	}
-	if files, e := filer.ListFiles(*mountOptions.filer, dir.Path, ""); e == nil {
-		for _, f := range files.Files {
-			if fileId, e := storage.ParseFileId(string(f.Id)); e == nil {
-				fileInode := uint64(fileId.VolumeId)<<48 + fileId.Key
-				ret = append(ret, fuse.Dirent{Inode: fileInode, Name: f.Name, Type: fuse.DT_File})
-			}
-
-		}
-	}
-	return ret, nil
-}

weed/command/server.go

@@ -11,7 +11,7 @@ import (
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/util"
@@ -66,18 +66,16 @@ var (
 	masterMetaFolder              = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
 	masterVolumeSizeLimitMB       = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
 	masterVolumePreallocate       = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
-	masterConfFile                = cmdServer.Flag.String("master.conf", "/etc/weedfs/weedfs.conf", "xml configuration file")
 	masterDefaultReplicaPlacement = cmdServer.Flag.String("master.defaultReplicaPlacement", "000", "Default replication type if not specified.")
 	volumePort                    = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
 	volumePublicPort              = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
 	volumeDataFolders             = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
 	volumeMaxDataVolumeCounts     = cmdServer.Flag.String("volume.max", "7", "maximum numbers of volumes, count[,count]...")
 	volumePulse                   = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
-	volumeIndexType               = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|boltdb] mode for memory~performance balance.")
+	volumeIndexType               = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|boltdb|btree] mode for memory~performance balance.")
 	volumeFixJpgOrientation       = cmdServer.Flag.Bool("volume.images.fix.orientation", true, "Adjust jpg orientation when uploading.")
 	volumeReadRedirect            = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
 	volumeServerPublicUrl         = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
-	volumeEnableBytesCache        = cmdServer.Flag.Bool("volume.cache.enable", false, "direct cache instead of OS cache, cost more memory.")
 	isStartingFiler               = cmdServer.Flag.Bool("filer", false, "whether to start filer")
 
 	serverWhiteList []string
@@ -88,12 +86,13 @@ func init() {
 	filerOptions.master = cmdServer.Flag.String("filer.master", "", "default to current master server")
 	filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
 	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
+	filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
 	filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -dir is specified")
 	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
 	filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
 	filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
 	filerOptions.confFile = cmdServer.Flag.String("filer.confFile", "", "json encoded filer conf file")
-	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 0, "split files larger than the limit")
+	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit")
 	filerOptions.cassandra_server = cmdServer.Flag.String("filer.cassandra.server", "", "host[:port] of the cassandra server")
 	filerOptions.cassandra_keyspace = cmdServer.Flag.String("filer.cassandra.keyspace", "seaweed", "keyspace of the cassandra server")
 	filerOptions.redis_server = cmdServer.Flag.String("filer.redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379")
@@ -117,6 +116,7 @@ func runServer(cmd *Command, args []string) bool {
 	}
 
 	*filerOptions.master = *serverIp + ":" + strconv.Itoa(*masterPort)
+	filerOptions.ip = serverIp
 
 	if *filerOptions.defaultReplicaPlacement == "" {
 		*filerOptions.defaultReplicaPlacement = *masterDefaultReplicaPlacement
@@ -150,6 +150,10 @@ func runServer(cmd *Command, args []string) bool {
 		}
 	}
 
+	if *masterVolumeSizeLimitMB > 30*1000 {
+		glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
+	}
+
 	if *masterMetaFolder == "" {
 		*masterMetaFolder = folders[0]
 	}
@@ -173,30 +177,9 @@ func runServer(cmd *Command, args []string) bool {
 	if *isStartingFiler {
 		go func() {
 			time.Sleep(1 * time.Second)
-			r := http.NewServeMux()
-			_, nfs_err := weed_server.NewFilerServer(r, *serverBindIp, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection,
-				*filerOptions.defaultReplicaPlacement,
-				*filerOptions.redirectOnRead, *filerOptions.disableDirListing,
-				*filerOptions.confFile,
-				*filerOptions.maxMB,
-				*filerOptions.secretKey,
-				*filerOptions.cassandra_server, *filerOptions.cassandra_keyspace,
-				*filerOptions.redis_server, *filerOptions.redis_password, *filerOptions.redis_database,
-			)
-			if nfs_err != nil {
-				glog.Fatalf("Filer startup error: %v", nfs_err)
-			}
-			glog.V(0).Infoln("Start Seaweed Filer", util.VERSION, "at port", strconv.Itoa(*filerOptions.port))
-			filerListener, e := util.NewListener(
-				":"+strconv.Itoa(*filerOptions.port),
-				time.Duration(10)*time.Second,
-			)
-			if e != nil {
-				glog.Fatalf("Filer listener error: %v", e)
-			}
-			if e := http.Serve(filerListener, r); e != nil {
-				glog.Fatalf("Filer Fail to serve: %v", e)
-			}
+
+			filerOptions.start()
+
 		}()
 	}
 
@@ -210,7 +193,7 @@ func runServer(cmd *Command, args []string) bool {
 		r := mux.NewRouter()
 		ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
 			*masterVolumeSizeLimitMB, *masterVolumePreallocate,
-			*volumePulse, *masterConfFile, *masterDefaultReplicaPlacement, *serverGarbageThreshold,
+			*volumePulse, *masterDefaultReplicaPlacement, *serverGarbageThreshold,
 			serverWhiteList, *serverSecureKey,
 		)
 
@@ -243,7 +226,7 @@ func runServer(cmd *Command, args []string) bool {
 
 		// Create your protocol servers.
 		grpcS := grpc.NewServer()
-		pb.RegisterSeaweedServer(grpcS, ms)
+		master_pb.RegisterSeaweedServer(grpcS, ms)
 		reflection.Register(grpcS)
 
 		httpS := &http.Server{Handler: r}
@@ -277,6 +260,8 @@ func runServer(cmd *Command, args []string) bool {
 		volumeNeedleMapKind = storage.NeedleMapLevelDb
 	case "boltdb":
 		volumeNeedleMapKind = storage.NeedleMapBoltDb
+	case "btree":
+		volumeNeedleMapKind = storage.NeedleMapBtree
 	}
 	volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
 		*serverIp, *volumePort, *volumeServerPublicUrl,
@@ -284,7 +269,6 @@ func runServer(cmd *Command, args []string) bool {
 		volumeNeedleMapKind,
 		*serverIp+":"+strconv.Itoa(*masterPort), *volumePulse, *serverDataCenter, *serverRack,
 		serverWhiteList, *volumeFixJpgOrientation, *volumeReadRedirect,
-		*volumeEnableBytesCache,
 	)
 
 	glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*volumePort))
@@ -309,7 +293,7 @@ func runServer(cmd *Command, args []string) bool {
 		}()
 	}
 
-	OnInterrupt(func() {
+	util.OnInterrupt(func() {
 		volumeServer.Shutdown()
 		pprof.StopCPUProfile()
 	})

weed/command/upload.go

@@ -41,7 +41,7 @@ func init() {
 }
 
 var cmdUpload = &Command{
-	UsageLine: "upload -master=localhost:9333 file1 [file2 file3]\n         weed upload -server=localhost:9333 -dir=one_directory -include=*.pdf",
+	UsageLine: "upload -master=localhost:9333 file1 [file2 file3]\n         weed upload -master=localhost:9333 -dir=one_directory -include=*.pdf",
 	Short:     "upload one or a list of files",
 	Long: `upload one or a list of files, or batch upload one whole folder recursively.
 

weed/command/volume.go

@@ -36,7 +36,8 @@ type VolumeServerOptions struct {
 	indexType             *string
 	fixJpgOrientation     *bool
 	readRedirect          *bool
-	enableBytesCache      *bool
+	cpuProfile            *string
+	memProfile            *string
 }
 
 func init() {
@@ -52,10 +53,11 @@ func init() {
 	v.maxCpu = cmdVolume.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
 	v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
 	v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
-	v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|boltdb] mode for memory~performance balance.")
+	v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|boltdb|btree] mode for memory~performance balance.")
 	v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", true, "Adjust jpg orientation when uploading.")
 	v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.")
-	v.enableBytesCache = cmdVolume.Flag.Bool("cache.enable", false, "direct cache instead of OS cache, cost more memory.")
+	v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
+	v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
 }
 
 var cmdVolume = &Command{
@@ -77,6 +79,7 @@ func runVolume(cmd *Command, args []string) bool {
 		*v.maxCpu = runtime.NumCPU()
 	}
 	runtime.GOMAXPROCS(*v.maxCpu)
+	util.SetupProfiling(*v.cpuProfile, *v.memProfile)
 
 	//Set multiple folders and each folder's max volume count limit'
 	v.folders = strings.Split(*volumeFolders, ",")
@@ -126,6 +129,8 @@ func runVolume(cmd *Command, args []string) bool {
 		volumeNeedleMapKind = storage.NeedleMapLevelDb
 	case "boltdb":
 		volumeNeedleMapKind = storage.NeedleMapBoltDb
+	case "btree":
+		volumeNeedleMapKind = storage.NeedleMapBtree
 	}
 	volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
 		*v.ip, *v.port, *v.publicUrl,
@@ -134,7 +139,6 @@ func runVolume(cmd *Command, args []string) bool {
 		*v.master, *v.pulseSeconds, *v.dataCenter, *v.rack,
 		v.whiteList,
 		*v.fixJpgOrientation, *v.readRedirect,
-		*v.enableBytesCache,
 	)
 
 	listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port)
@@ -157,7 +161,7 @@ func runVolume(cmd *Command, args []string) bool {
 		}()
 	}
 
-	OnInterrupt(func() {
+	util.OnInterrupt(func() {
 		volumeServer.Shutdown()
 	})
 

weed/compress/delta_binary_pack32.go

@@ -1,8 +1,8 @@
 package compress
 
 import (
-	"github.com/reducedb/encoding/cursor"
-	"github.com/reducedb/encoding/delta/bp32"
+	"github.com/dataence/encoding/cursor"
+	"github.com/dataence/encoding/delta/bp32"
 )
 
 // Compress compresses in[]int32 to out[]int32

weed/filer/cassandra_store/cassandra_store.go

@@ -41,7 +41,7 @@ func NewCassandraStore(keyspace string, hosts string) (c *CassandraStore, err er
 		c.cluster = gocql.NewCluster(s...)
 	}
 	c.cluster.Keyspace = keyspace
-	c.cluster.Consistency = gocql.Quorum
+	c.cluster.Consistency = gocql.LocalQuorum
 	c.session, err = c.cluster.CreateSession()
 	if err != nil {
 		glog.V(0).Infof("Failed to open cassandra store, hosts %v, keyspace %s", hosts, keyspace)

weed/filer/client_operations.go

@@ -1,70 +0,0 @@
-package filer
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-
-	"github.com/chrislusf/seaweedfs/weed/util"
-
-	"net/url"
-)
-
-type ApiRequest struct {
-	Command   string //"listFiles", "listDirectories"
-	Directory string
-	FileName  string
-}
-
-type ListFilesResult struct {
-	Files []FileEntry
-	Error string `json:"error,omitempty"`
-}
-
-func ListFiles(server string, directory string, fileName string) (*ListFilesResult, error) {
-	var ret ListFilesResult
-	if err := call(server, ApiRequest{Command: "listFiles", Directory: directory, FileName: fileName}, &ret); err == nil {
-		if ret.Error != "" {
-			return nil, errors.New(ret.Error)
-		}
-		return &ret, nil
-	} else {
-		return nil, err
-	}
-}
-
-type ListDirectoriesResult struct {
-	Directories []DirectoryEntry
-	Error       string `json:"error,omitempty"`
-}
-
-func ListDirectories(server string, directory string) (*ListDirectoriesResult, error) {
-	var ret ListDirectoriesResult
-	if err := call(server, ApiRequest{Command: "listDirectories", Directory: directory}, &ret); err == nil {
-		if ret.Error != "" {
-			return nil, errors.New(ret.Error)
-		}
-		return &ret, nil
-	} else {
-		return nil, err
-	}
-}
-
-func call(server string, request ApiRequest, ret interface{}) error {
-	b, err := json.Marshal(request)
-	if err != nil {
-		fmt.Println("error:", err)
-		return nil
-	}
-	values := make(url.Values)
-	values.Add("request", string(b))
-	jsonBlob, err := util.Post("http://"+server+"/__api__", values)
-	if err != nil {
-		return err
-	}
-	err = json.Unmarshal(jsonBlob, ret)
-	if err != nil {
-		return err
-	}
-	return nil
-}

weed/filer/embedded_filer/directory.go

@@ -5,11 +5,11 @@ import (
 )
 
 type DirectoryManager interface {
-	FindDirectory(dirPath string) (filer.DirectoryId, error)
-	ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error)
-	MakeDirectory(currentDirPath string, dirName string) (filer.DirectoryId, error)
+	FindDirectory(dirPath string) (DirectoryId, error)
+	ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error)
+	MakeDirectory(currentDirPath string, dirName string) (DirectoryId, error)
 	MoveUnderDirectory(oldDirPath string, newParentDirPath string) error
 	DeleteDirectory(dirPath string) error
 	//functions used by FUSE
-	FindDirectoryById(filer.DirectoryId, error)
+	FindDirectoryById(DirectoryId, error)
 }

weed/filer/embedded_filer/directory_in_map.go

@@ -16,12 +16,14 @@ import (
 
 var writeLock sync.Mutex //serialize changes to dir.log
 
+type DirectoryId int32
+
 type DirectoryEntryInMap struct {
 	sync.Mutex
 	Name           string
 	Parent         *DirectoryEntryInMap
 	subDirectories map[string]*DirectoryEntryInMap
-	Id             filer.DirectoryId
+	Id             DirectoryId
 }
 
 func (de *DirectoryEntryInMap) getChild(dirName string) (*DirectoryEntryInMap, bool) {
@@ -45,18 +47,18 @@ func (de *DirectoryEntryInMap) hasChildren() bool {
 	defer de.Unlock()
 	return len(de.subDirectories) > 0
 }
-func (de *DirectoryEntryInMap) children() (dirNames []filer.DirectoryEntry) {
+func (de *DirectoryEntryInMap) children() (dirNames []filer.DirectoryName) {
 	de.Lock()
 	defer de.Unlock()
-	for k, v := range de.subDirectories {
-		dirNames = append(dirNames, filer.DirectoryEntry{Name: k, Id: v.Id})
+	for k, _ := range de.subDirectories {
+		dirNames = append(dirNames, filer.DirectoryName(k))
 	}
 	return dirNames
 }
 
 type DirectoryManagerInMap struct {
 	Root      *DirectoryEntryInMap
-	max       filer.DirectoryId
+	max       DirectoryId
 	logFile   *os.File
 	isLoading bool
 }
@@ -113,7 +115,7 @@ func (dm *DirectoryManagerInMap) processEachLine(line string) error {
 		if pe != nil {
 			return pe
 		}
-		if e := dm.loadDirectory(parts[1], filer.DirectoryId(v)); e != nil {
+		if e := dm.loadDirectory(parts[1], DirectoryId(v)); e != nil {
 			return e
 		}
 	case "mov":
@@ -167,12 +169,12 @@ func (dm *DirectoryManagerInMap) findDirectory(dirPath string) (*DirectoryEntryI
 		if sub, ok := dir.getChild(parts[i]); ok {
 			dir = sub
 		} else {
-			return dm.Root, fmt.Errorf("Directory %s Not Found", dirPath)
+			return dm.Root, filer.ErrNotFound
 		}
 	}
 	return dir, nil
 }
-func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (filer.DirectoryId, error) {
+func (dm *DirectoryManagerInMap) findDirectoryId(dirPath string) (DirectoryId, error) {
 	d, e := dm.findDirectory(dirPath)
 	if e == nil {
 		return d.Id, nil
@@ -180,7 +182,7 @@ func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (filer.DirectoryI
 	return dm.Root.Id, e
 }
 
-func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId filer.DirectoryId) error {
+func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId DirectoryId) error {
 	dirPath = CleanFilePath(dirPath)
 	if dirPath == "/" {
 		return nil
@@ -248,7 +250,7 @@ func (dm *DirectoryManagerInMap) makeDirectory(dirPath string) (dir *DirectoryEn
 	return dir, created
 }
 
-func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (filer.DirectoryId, error) {
+func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (DirectoryId, error) {
 	dir, _ := dm.makeDirectory(dirPath)
 	return dir.Id, nil
 }
@@ -275,7 +277,7 @@ func (dm *DirectoryManagerInMap) MoveUnderDirectory(oldDirPath string, newParent
 	return nil
 }
 
-func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []filer.DirectoryEntry, err error) {
+func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []filer.DirectoryName, err error) {
 	d, e := dm.findDirectory(dirPath)
 	if e != nil {
 		return dirNames, e

weed/filer/embedded_filer/directory_test.go

@@ -19,17 +19,17 @@ func TestDirectory(t *testing.T) {
 	dm.MakeDirectory("/a/b/e/f")
 	dm.MakeDirectory("/a/b/e/f/g")
 	dm.MoveUnderDirectory("/a/b/e/f/g", "/a/b", "t")
-	if _, err := dm.FindDirectory("/a/b/e/f/g"); err == nil {
+	if _, err := dm.findDirectoryId("/a/b/e/f/g"); err == nil {
 		t.Fatal("/a/b/e/f/g should not exist any more after moving")
 	}
-	if _, err := dm.FindDirectory("/a/b/t"); err != nil {
+	if _, err := dm.findDirectoryId("/a/b/t"); err != nil {
 		t.Fatal("/a/b/t should exist after moving")
 	}
-	if _, err := dm.FindDirectory("/a/b/g"); err == nil {
+	if _, err := dm.findDirectoryId("/a/b/g"); err == nil {
 		t.Fatal("/a/b/g should not exist after moving")
 	}
 	dm.MoveUnderDirectory("/a/b/e/f", "/a/b", "")
-	if _, err := dm.FindDirectory("/a/b/f"); err != nil {
+	if _, err := dm.findDirectoryId("/a/b/f"); err != nil {
 		t.Fatal("/a/b/g should not exist after moving")
 	}
 	dm.MakeDirectory("/a/b/g/h/i")
@@ -37,7 +37,7 @@ func TestDirectory(t *testing.T) {
 	dm.DeleteDirectory("/a/b/e")
 	dirNames, _ := dm.ListDirectories("/a/b/e")
 	for _, v := range dirNames {
-		println("sub1 dir:", v.Name, "id", v.Id)
+		println("sub1 dir:", v)
 	}
 	dm.logFile.Close()
 

weed/filer/embedded_filer/filer_embedded.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"path/filepath"
 	"strings"
+	"sync"
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/operation"
@@ -14,6 +15,7 @@ type FilerEmbedded struct {
 	master      string
 	directories *DirectoryManagerInMap
 	files       *FileListInLevelDb
+	mvMutex     sync.Mutex
 }
 
 func NewFilerEmbedded(master string, dir string) (filer *FilerEmbedded, err error) {
@@ -43,27 +45,37 @@ func (filer *FilerEmbedded) CreateFile(filePath string, fid string) (err error)
 }
 func (filer *FilerEmbedded) FindFile(filePath string) (fid string, err error) {
 	dir, file := filepath.Split(filePath)
-	dirId, e := filer.directories.FindDirectory(dir)
+	return filer.findFileEntry(dir, file)
+}
+func (filer *FilerEmbedded) findFileEntry(parentPath string, fileName string) (fid string, err error) {
+	dirId, e := filer.directories.findDirectoryId(parentPath)
 	if e != nil {
 		return "", e
 	}
-	return filer.files.FindFile(dirId, file)
+	return filer.files.FindFile(dirId, fileName)
 }
-func (filer *FilerEmbedded) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) {
-	return filer.directories.FindDirectory(dirPath)
+
+func (filer *FilerEmbedded) LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error) {
+	if _, err = filer.directories.findDirectory(filepath.Join(dirPath, name)); err == nil {
+		return true, "", nil
+	}
+	if fileId, err = filer.findFileEntry(dirPath, name); err == nil {
+		return true, fileId, nil
+	}
+	return false, "", err
 }
-func (filer *FilerEmbedded) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) {
+func (filer *FilerEmbedded) ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error) {
 	return filer.directories.ListDirectories(dirPath)
 }
 func (filer *FilerEmbedded) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
-	dirId, e := filer.directories.FindDirectory(dirPath)
+	dirId, e := filer.directories.findDirectoryId(dirPath)
 	if e != nil {
 		return nil, e
 	}
 	return filer.files.ListFiles(dirId, lastFileName, limit), nil
 }
 func (filer *FilerEmbedded) DeleteDirectory(dirPath string, recursive bool) (err error) {
-	dirId, e := filer.directories.FindDirectory(dirPath)
+	dirId, e := filer.directories.findDirectoryId(dirPath)
 	if e != nil {
 		return e
 	}
@@ -72,7 +84,7 @@ func (filer *FilerEmbedded) DeleteDirectory(dirPath string, recursive bool) (err
 			return fmt.Errorf("Fail to delete directory %s: %d sub directories found!", dirPath, len(sub_dirs))
 		}
 		for _, sub := range sub_dirs {
-			if delete_sub_err := filer.DeleteDirectory(filepath.Join(dirPath, sub.Name), recursive); delete_sub_err != nil {
+			if delete_sub_err := filer.DeleteDirectory(filepath.Join(dirPath, string(sub)), recursive); delete_sub_err != nil {
 				return delete_sub_err
 			}
 		}
@@ -106,7 +118,7 @@ func (filer *FilerEmbedded) DeleteDirectory(dirPath string, recursive bool) (err
 
 func (filer *FilerEmbedded) DeleteFile(filePath string) (fid string, err error) {
 	dir, file := filepath.Split(filePath)
-	dirId, e := filer.directories.FindDirectory(dir)
+	dirId, e := filer.directories.findDirectoryId(dir)
 	if e != nil {
 		return "", e
 	}
@@ -121,8 +133,11 @@ mv fromFile toDir
 mv fromFile toFile
 */
 func (filer *FilerEmbedded) Move(fromPath string, toPath string) error {
-	if _, dir_err := filer.FindDirectory(fromPath); dir_err == nil {
-		if _, err := filer.FindDirectory(toPath); err == nil {
+	filer.mvMutex.Lock()
+	defer filer.mvMutex.Unlock()
+
+	if _, dir_err := filer.directories.findDirectoryId(fromPath); dir_err == nil {
+		if _, err := filer.directories.findDirectoryId(toPath); err == nil {
 			// move folder under an existing folder
 			return filer.directories.MoveUnderDirectory(fromPath, toPath, "")
 		}
@@ -130,7 +145,7 @@ func (filer *FilerEmbedded) Move(fromPath string, toPath string) error {
 		return filer.directories.MoveUnderDirectory(fromPath, filepath.Dir(toPath), filepath.Base(toPath))
 	}
 	if fid, file_err := filer.DeleteFile(fromPath); file_err == nil {
-		if _, err := filer.FindDirectory(toPath); err == nil {
+		if _, err := filer.directories.findDirectoryId(toPath); err == nil {
 			// move file under an existing folder
 			return filer.CreateFile(filepath.Join(toPath, filepath.Base(fromPath)), fid)
 		}

weed/filer/embedded_filer/files_in_leveldb.go

@@ -28,7 +28,7 @@ func NewFileListInLevelDb(dir string) (fl *FileListInLevelDb, err error) {
 	return
 }
 
-func genKey(dirId filer.DirectoryId, fileName string) []byte {
+func genKey(dirId DirectoryId, fileName string) []byte {
 	ret := make([]byte, 0, 4+len(fileName))
 	for i := 3; i >= 0; i-- {
 		ret = append(ret, byte(dirId>>(uint(i)*8)))
@@ -37,11 +37,11 @@ func genKey(dirId filer.DirectoryId, fileName string) []byte {
 	return ret
 }
 
-func (fl *FileListInLevelDb) CreateFile(dirId filer.DirectoryId, fileName string, fid string) (err error) {
+func (fl *FileListInLevelDb) CreateFile(dirId DirectoryId, fileName string, fid string) (err error) {
 	glog.V(4).Infoln("directory", dirId, "fileName", fileName, "fid", fid)
 	return fl.db.Put(genKey(dirId, fileName), []byte(fid), nil)
 }
-func (fl *FileListInLevelDb) DeleteFile(dirId filer.DirectoryId, fileName string) (fid string, err error) {
+func (fl *FileListInLevelDb) DeleteFile(dirId DirectoryId, fileName string) (fid string, err error) {
 	if fid, err = fl.FindFile(dirId, fileName); err != nil {
 		if err == leveldb.ErrNotFound {
 			return "", nil
@@ -51,7 +51,7 @@ func (fl *FileListInLevelDb) DeleteFile(dirId filer.DirectoryId, fileName string
 	err = fl.db.Delete(genKey(dirId, fileName), nil)
 	return fid, err
 }
-func (fl *FileListInLevelDb) FindFile(dirId filer.DirectoryId, fileName string) (fid string, err error) {
+func (fl *FileListInLevelDb) FindFile(dirId DirectoryId, fileName string) (fid string, err error) {
 	data, e := fl.db.Get(genKey(dirId, fileName), nil)
 	if e == leveldb.ErrNotFound {
 		return "", filer.ErrNotFound
@@ -60,7 +60,7 @@ func (fl *FileListInLevelDb) FindFile(dirId filer.DirectoryId, fileName string)
 	}
 	return string(data), nil
 }
-func (fl *FileListInLevelDb) ListFiles(dirId filer.DirectoryId, lastFileName string, limit int) (files []filer.FileEntry) {
+func (fl *FileListInLevelDb) ListFiles(dirId DirectoryId, lastFileName string, limit int) (files []filer.FileEntry) {
 	glog.V(4).Infoln("directory", dirId, "lastFileName", lastFileName, "limit", limit)
 	dirKey := genKey(dirId, "")
 	iter := fl.db.NewIterator(&util.Range{Start: genKey(dirId, lastFileName)}, nil)

weed/filer/filer.go

@@ -11,12 +11,7 @@ type FileEntry struct {
 	Id   FileId `json:"fid,omitempty"`
 }
 
-type DirectoryId int32
-
-type DirectoryEntry struct {
-	Name string //dir name without path
-	Id   DirectoryId
-}
+type DirectoryName string
 
 type Filer interface {
 	CreateFile(fullFileName string, fid string) (err error)
@@ -24,11 +19,11 @@ type Filer interface {
 	DeleteFile(fullFileName string) (fid string, err error)
 
 	//Optional functions. embedded filer support these
-	FindDirectory(dirPath string) (dirId DirectoryId, err error)
-	ListDirectories(dirPath string) (dirs []DirectoryEntry, err error)
+	ListDirectories(dirPath string) (dirs []DirectoryName, err error)
 	ListFiles(dirPath string, lastFileName string, limit int) (files []FileEntry, err error)
 	DeleteDirectory(dirPath string, recursive bool) (err error)
 	Move(fromPath string, toPath string) (err error)
+	LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error)
 }
 
 var ErrNotFound = errors.New("filer: no entry is found in filer store")

weed/filer/flat_namespace/flat_namespace_filer.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
+	"path/filepath"
 )
 
 type FlatNamespaceFiler struct {
@@ -28,10 +29,13 @@ func (filer *FlatNamespaceFiler) CreateFile(fullFileName string, fid string) (er
 func (filer *FlatNamespaceFiler) FindFile(fullFileName string) (fid string, err error) {
 	return filer.store.Get(fullFileName)
 }
-func (filer *FlatNamespaceFiler) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) {
-	return 0, ErrNotImplemented
+func (filer *FlatNamespaceFiler) LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error) {
+	if fileId, err = filer.FindFile(filepath.Join(dirPath, name)); err == nil {
+		return true, fileId, nil
+	}
+	return false, "", err
 }
-func (filer *FlatNamespaceFiler) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) {
+func (filer *FlatNamespaceFiler) ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error) {
 	return nil, ErrNotImplemented
 }
 func (filer *FlatNamespaceFiler) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {

weed/filer/mysql_store/mysql_store.go

@@ -184,7 +184,7 @@ func (s *MySqlStore) Close() {
 var createTable = `
 CREATE TABLE IF NOT EXISTS %s (
   id bigint(20) NOT NULL AUTO_INCREMENT,
-  uriPath char(256) NOT NULL DEFAULT "" COMMENT 'http uriPath',
+  uriPath char(255) NOT NULL DEFAULT "" COMMENT 'http uriPath',
   fid char(36) NOT NULL DEFAULT "" COMMENT 'seaweedfs fid',
   createTime int(10) NOT NULL DEFAULT 0 COMMENT 'createdTime in unix timestamp',
   updateTime int(10) NOT NULL DEFAULT 0 COMMENT 'updatedTime in unix timestamp',

weed/filer/postgres_store/postgres_native.go

@@ -0,0 +1,456 @@
+package postgres_store
+
+import (
+	"database/sql"
+	"fmt"
+	"path/filepath"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+
+	_ "github.com/lib/pq"
+	_ "path/filepath"
+	"strings"
+)
+
+type DirectoryId int32
+
+func databaseExists(db *sql.DB, databaseName string) (bool, error) {
+	sqlStatement := "SELECT datname from pg_database WHERE datname='%s'"
+	row := db.QueryRow(fmt.Sprintf(sqlStatement, databaseName))
+
+	var dbName string
+	err := row.Scan(&dbName)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+func createDatabase(db *sql.DB, databaseName string) error {
+	sqlStatement := "CREATE DATABASE %s ENCODING='UTF8'"
+	_, err := db.Exec(fmt.Sprintf(sqlStatement, databaseName))
+	return err
+}
+
+func getDbConnection(conf PostgresConf) *sql.DB {
+	_init_db.Do(func() {
+
+		sqlUrl := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s connect_timeout=30", conf.HostName, conf.Port, conf.User, conf.Password, "postgres", conf.SslMode)
+		glog.V(3).Infoln("Opening postgres master database")
+
+		var dbErr error
+		_db_connection, dbErr := sql.Open("postgres", sqlUrl)
+		if dbErr != nil {
+			_db_connection.Close()
+			_db_connection = nil
+			panic(dbErr)
+		}
+
+		pingErr := _db_connection.Ping()
+		if pingErr != nil {
+			_db_connection.Close()
+			_db_connection = nil
+			panic(pingErr)
+		}
+
+		glog.V(3).Infoln("Checking to see if DB exists: ", conf.DataBase)
+		var existsErr error
+		dbExists, existsErr := databaseExists(_db_connection, conf.DataBase)
+		if existsErr != nil {
+			_db_connection.Close()
+			_db_connection = nil
+			panic(existsErr)
+		}
+
+		if !dbExists {
+			glog.V(3).Infoln("Database doesn't exist. Attempting to create one:  ", conf.DataBase)
+			createErr := createDatabase(_db_connection, conf.DataBase)
+			if createErr != nil {
+				_db_connection.Close()
+				_db_connection = nil
+				panic(createErr)
+			}
+		}
+
+		glog.V(3).Infoln("Closing master postgres database and opening configured database: ", conf.DataBase)
+		_db_connection.Close()
+		_db_connection = nil
+
+		sqlUrl = fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s connect_timeout=30", conf.HostName, conf.Port, conf.User, conf.Password, conf.DataBase, conf.SslMode)
+		_db_connection, dbErr = sql.Open("postgres", sqlUrl)
+		if dbErr != nil {
+			_db_connection.Close()
+			_db_connection = nil
+			panic(dbErr)
+		}
+
+		pingErr = _db_connection.Ping()
+		if pingErr != nil {
+			_db_connection.Close()
+			_db_connection = nil
+			panic(pingErr)
+		}
+
+		maxIdleConnections, maxOpenConnections := default_maxIdleConnections, default_maxOpenConnections
+		if conf.MaxIdleConnections != 0 {
+			maxIdleConnections = conf.MaxIdleConnections
+		}
+		if conf.MaxOpenConnections != 0 {
+			maxOpenConnections = conf.MaxOpenConnections
+		}
+
+		_db_connection.SetMaxIdleConns(maxIdleConnections)
+		_db_connection.SetMaxOpenConns(maxOpenConnections)
+	})
+	return _db_connection
+}
+
+var createDirectoryTable = `
+
+CREATE TABLE IF NOT EXISTS %s (
+  id BIGSERIAL NOT NULL,
+  directoryRoot VARCHAR(1024) NOT NULL DEFAULT '',
+  directoryName VARCHAR(1024) NOT NULL DEFAULT '',
+  CONSTRAINT unique_directory UNIQUE (directoryRoot, directoryName)
+);
+`
+
+var createFileTable = `
+
+CREATE TABLE IF NOT EXISTS %s (
+  id BIGSERIAL NOT NULL,
+  directoryPart VARCHAR(1024) NOT NULL DEFAULT '',
+  filePart VARCHAR(1024) NOT NULL DEFAULT '',
+  fid VARCHAR(36) NOT NULL DEFAULT '',
+  createTime BIGINT NOT NULL DEFAULT 0,
+  updateTime BIGINT NOT NULL DEFAULT 0,
+  remark VARCHAR(20) NOT NULL DEFAULT '',
+  status SMALLINT NOT NULL DEFAULT '1',
+  PRIMARY KEY (id),
+  CONSTRAINT %s_unique_file UNIQUE (directoryPart, filePart)
+);
+`
+
+func (s *PostgresStore) createDirectoriesTable() error {
+	glog.V(3).Infoln("Creating postgres table if it doesn't exist: ", directoriesTableName)
+
+	sqlCreate := fmt.Sprintf(createDirectoryTable, directoriesTableName)
+
+	stmt, err := s.db.Prepare(sqlCreate)
+	if err != nil {
+		return err
+	}
+	defer stmt.Close()
+
+	_, err = stmt.Exec()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *PostgresStore) createFilesTable() error {
+
+	glog.V(3).Infoln("Creating postgres table if it doesn't exist: ", filesTableName)
+
+	sqlCreate := fmt.Sprintf(createFileTable, filesTableName, filesTableName)
+
+	stmt, err := s.db.Prepare(sqlCreate)
+	if err != nil {
+		return err
+	}
+	defer stmt.Close()
+
+	_, err = stmt.Exec()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *PostgresStore) query(uriPath string) (string, error) {
+	directoryPart, filePart := filepath.Split(uriPath)
+	sqlStatement := fmt.Sprintf("SELECT fid FROM %s WHERE directoryPart=$1 AND filePart=$2", filesTableName)
+
+	row := s.db.QueryRow(sqlStatement, directoryPart, filePart)
+	var fid string
+	err := row.Scan(&fid)
+
+	glog.V(3).Infof("Postgres query -- looking up path '%s' and found id '%s' ", uriPath, fid)
+
+	if err != nil {
+		return "", err
+	}
+	return fid, nil
+}
+
+func (s *PostgresStore) update(uriPath string, fid string) error {
+	directoryPart, filePart := filepath.Split(uriPath)
+	sqlStatement := fmt.Sprintf("UPDATE %s SET fid=$1, updateTime=$2 WHERE directoryPart=$3 AND filePart=$4", filesTableName)
+
+	glog.V(3).Infof("Postgres query -- updating path '%s' with id '%s'", uriPath, fid)
+
+	res, err := s.db.Exec(sqlStatement, fid, time.Now().Unix(), directoryPart, filePart)
+	if err != nil {
+		return err
+	}
+
+	_, err = res.RowsAffected()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *PostgresStore) insert(uriPath string, fid string) error {
+	directoryPart, filePart := filepath.Split(uriPath)
+
+	existingId, _, _ := s.lookupDirectory(directoryPart)
+	if existingId == 0 {
+		s.recursiveInsertDirectory(directoryPart)
+	}
+
+	sqlStatement := fmt.Sprintf("INSERT INTO %s (directoryPart,filePart,fid,createTime) VALUES($1, $2, $3, $4)", filesTableName)
+	glog.V(3).Infof("Postgres query -- inserting path '%s' with id '%s'", uriPath, fid)
+
+	res, err := s.db.Exec(sqlStatement, directoryPart, filePart, fid, time.Now().Unix())
+
+	if err != nil {
+		return err
+	}
+
+	rows, err := res.RowsAffected()
+	if rows != 1 {
+		return fmt.Errorf("Postgres insert -- rows affected = %d. Expecting 1", rows)
+	}
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *PostgresStore) recursiveInsertDirectory(dirPath string) {
+	pathParts := strings.Split(dirPath, "/")
+
+	var workingPath string = "/"
+	for _, part := range pathParts {
+		if part == "" {
+			continue
+		}
+		workingPath += (part + "/")
+		existingId, _, _ := s.lookupDirectory(workingPath)
+		if existingId == 0 {
+			s.insertDirectory(workingPath)
+		}
+	}
+}
+
+func (s *PostgresStore) insertDirectory(dirPath string) {
+	pathParts := strings.Split(dirPath, "/")
+
+	directoryRoot := "/"
+	directoryName := ""
+	if len(pathParts) > 1 {
+		directoryRoot = strings.Join(pathParts[0:len(pathParts)-2], "/") + "/"
+		directoryName = strings.Join(pathParts[len(pathParts)-2:], "/")
+	} else if len(pathParts) == 1 {
+		directoryRoot = "/"
+		directoryName = pathParts[0] + "/"
+	}
+	sqlInsertDirectoryStatement := fmt.Sprintf("INSERT INTO %s (directoryroot, directoryname)  "+
+		"SELECT $1, $2 WHERE NOT EXISTS ( SELECT id FROM %s WHERE directoryroot=$3 AND directoryname=$4 )",
+		directoriesTableName, directoriesTableName)
+
+	glog.V(4).Infof("Postgres query -- Inserting directory (if it doesn't exist) - root = %s, name = %s",
+		directoryRoot, directoryName)
+
+	_, err := s.db.Exec(sqlInsertDirectoryStatement, directoryRoot, directoryName, directoryRoot, directoryName)
+	if err != nil {
+		glog.V(0).Infof("Postgres query -- Error inserting directory - root = %s, name = %s: %s",
+			directoryRoot, directoryName, err)
+	}
+}
+
+func (s *PostgresStore) delete(uriPath string) error {
+	directoryPart, filePart := filepath.Split(uriPath)
+	sqlStatement := fmt.Sprintf("DELETE FROM %s WHERE directoryPart=$1 AND filePart=$2", filesTableName)
+
+	glog.V(3).Infof("Postgres query -- deleting path '%s'", uriPath)
+
+	res, err := s.db.Exec(sqlStatement, directoryPart, filePart)
+	if err != nil {
+		return err
+	}
+
+	_, err = res.RowsAffected()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *PostgresStore) lookupDirectory(dirPath string) (DirectoryId, string, error) {
+	directoryRoot, directoryName := s.mySplitPath(dirPath)
+
+	sqlStatement := fmt.Sprintf("SELECT id, directoryroot, directoryname FROM %s WHERE directoryRoot=$1 AND directoryName=$2", directoriesTableName)
+
+	row := s.db.QueryRow(sqlStatement, directoryRoot, directoryName)
+	var id DirectoryId
+	var dirRoot string
+	var dirName string
+	err := row.Scan(&id, &dirRoot, &dirName)
+
+	glog.V(3).Infof("Postgres lookupDirectory -- looking up directory '%s' and found id '%d', root '%s', name '%s' ", dirPath, id, dirRoot, dirName)
+
+	if err != nil {
+		return 0, "", err
+	}
+	return id, filepath.Join(dirRoot, dirName), err
+}
+
+func (s *PostgresStore) findDirectories(dirPath string, limit int) (dirs []filer.DirectoryName, err error) {
+	sqlStatement := fmt.Sprintf("SELECT id, directoryroot, directoryname FROM %s WHERE directoryRoot=$1 AND directoryName != '' ORDER BY id LIMIT $2", directoriesTableName)
+	rows, err := s.db.Query(sqlStatement, dirPath, limit)
+
+	if err != nil {
+		glog.V(0).Infof("Postgres findDirectories error: %s", err)
+	}
+
+	if rows != nil {
+		defer rows.Close()
+		for rows.Next() {
+			var id DirectoryId
+			var directoryRoot string
+			var directoryName string
+
+			scanErr := rows.Scan(&id, &directoryRoot, &directoryName)
+			if scanErr != nil {
+				err = scanErr
+			}
+			dirs = append(dirs, filer.DirectoryName(directoryName))
+		}
+	}
+	return
+}
+
+func (s *PostgresStore) safeToDeleteDirectory(dirPath string, recursive bool) bool {
+	if recursive {
+		return true
+	}
+	sqlStatement := fmt.Sprintf("SELECT id FROM %s WHERE directoryRoot LIKE $1 LIMIT 1", directoriesTableName)
+	row := s.db.QueryRow(sqlStatement, dirPath+"%")
+
+	var id DirectoryId
+	err := row.Scan(&id)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return true
+		}
+	}
+	return false
+}
+
+func (s *PostgresStore) mySplitPath(dirPath string) (directoryRoot string, directoryName string) {
+	pathParts := strings.Split(dirPath, "/")
+	directoryRoot = "/"
+	directoryName = ""
+	if len(pathParts) > 1 {
+		directoryRoot = strings.Join(pathParts[0:len(pathParts)-2], "/") + "/"
+		directoryName = strings.Join(pathParts[len(pathParts)-2:], "/")
+	} else if len(pathParts) == 1 {
+		directoryRoot = "/"
+		directoryName = pathParts[0] + "/"
+	}
+	return directoryRoot, directoryName
+}
+
+func (s *PostgresStore) deleteDirectory(dirPath string, recursive bool) (err error) {
+	directoryRoot, directoryName := s.mySplitPath(dirPath)
+
+	// delete files
+	sqlStatement := fmt.Sprintf("DELETE FROM %s WHERE directorypart=$1", filesTableName)
+	_, err = s.db.Exec(sqlStatement, dirPath)
+	if err != nil {
+		return err
+	}
+
+	// delete specific directory if it is empty or recursive delete was requested
+	safeToDelete := s.safeToDeleteDirectory(dirPath, recursive)
+	if safeToDelete {
+		sqlStatement = fmt.Sprintf("DELETE FROM %s WHERE directoryRoot=$1 AND directoryName=$2", directoriesTableName)
+		_, err = s.db.Exec(sqlStatement, directoryRoot, directoryName)
+		if err != nil {
+			return err
+		}
+	}
+
+	if recursive {
+		// delete descendant files
+		sqlStatement = fmt.Sprintf("DELETE FROM %s WHERE directorypart LIKE $1", filesTableName)
+		_, err = s.db.Exec(sqlStatement, dirPath+"%")
+		if err != nil {
+			return err
+		}
+
+		// delete descendant directories
+		sqlStatement = fmt.Sprintf("DELETE FROM %s WHERE directoryRoot LIKE $1", directoriesTableName)
+		_, err = s.db.Exec(sqlStatement, dirPath+"%")
+		if err != nil {
+			return err
+		}
+	}
+
+	return err
+}
+
+func (s *PostgresStore) findFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
+	var rows *sql.Rows = nil
+
+	if lastFileName == "" {
+		sqlStatement :=
+			fmt.Sprintf("SELECT fid, directorypart, filepart FROM %s WHERE directorypart=$1 ORDER BY id LIMIT $2", filesTableName)
+		rows, err = s.db.Query(sqlStatement, dirPath, limit)
+	} else {
+		sqlStatement :=
+			fmt.Sprintf("SELECT fid, directorypart, filepart FROM %s WHERE directorypart=$1 "+
+				"AND id > (SELECT id FROM %s WHERE directoryPart=$2 AND filepart=$3)  ORDER BY id LIMIT $4",
+				filesTableName, filesTableName)
+		_, lastFileNameName := filepath.Split(lastFileName)
+		rows, err = s.db.Query(sqlStatement, dirPath, dirPath, lastFileNameName, limit)
+	}
+
+	if err != nil {
+		glog.V(0).Infof("Postgres find files error: %s", err)
+	}
+
+	if rows != nil {
+		defer rows.Close()
+
+		for rows.Next() {
+			var fid filer.FileId
+			var directoryPart string
+			var filePart string
+
+			scanErr := rows.Scan(&fid, &directoryPart, &filePart)
+			if scanErr != nil {
+				err = scanErr
+			}
+
+			files = append(files, filer.FileEntry{Name: filepath.Join(directoryPart, filePart), Id: fid})
+			if len(files) >= limit {
+				break
+			}
+		}
+	}
+
+	glog.V(3).Infof("Postgres findFiles -- looking up files under '%s' and found %d files. Limit=%d, lastFileName=%s",
+		dirPath, len(files), limit, lastFileName)
+
+	return files, err
+}

weed/filer/postgres_store/postgres_store.go

@@ -4,16 +4,13 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
-	"path/filepath"
 	"sync"
-	"time"
-
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 
 	_ "github.com/lib/pq"
 	_ "path/filepath"
-	"strings"
+	"path/filepath"
 )
 
 const (
@@ -46,170 +43,8 @@ type PostgresStore struct {
 	password string
 }
 
-func (s *PostgresStore) CreateFile(fullFileName string, fid string) (err error) {
-	glog.V(3).Infoln("Calling posgres_store CreateFile")
-	return s.Put(fullFileName, fid)
-}
+func (s *PostgresStore) CreateFile(fullFilePath string, fid string) (err error) {
 
-func (s *PostgresStore) FindFile(fullFileName string) (fid string, err error) {
-	glog.V(3).Infoln("Calling posgres_store FindFile")
-	return s.Get(fullFileName)
-}
-
-func (s *PostgresStore) DeleteFile(fullFileName string) (fid string, err error) {
-	glog.V(3).Infoln("Calling posgres_store DeleteFile")
-	return "", s.Delete(fullFileName)
-}
-
-func (s *PostgresStore) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) {
-	glog.V(3).Infoln("Calling posgres_store FindDirectory")
-	return s.FindDir(dirPath)
-}
-
-func (s *PostgresStore) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) {
-	glog.V(3).Infoln("Calling posgres_store ListDirectories")
-	return s.ListDirs(dirPath)
-}
-
-func (s *PostgresStore) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
-	glog.V(3).Infoln("Calling posgres_store ListFiles")
-	return s.FindFiles(dirPath, lastFileName, limit)
-}
-
-func (s *PostgresStore) DeleteDirectory(dirPath string, recursive bool) (err error) {
-	glog.V(3).Infoln("Calling posgres_store DeleteDirectory")
-	return s.DeleteDir(dirPath, recursive)
-}
-
-func (s *PostgresStore) Move(fromPath string, toPath string) (err error) {
-	glog.V(3).Infoln("Calling posgres_store Move")
-	return errors.New("Move is not yet implemented for the PostgreSQL store.")
-}
-
-func databaseExists(db *sql.DB, databaseName string) (bool, error) {
-	sqlStatement := "SELECT datname from pg_database WHERE datname='%s'"
-	row := db.QueryRow(fmt.Sprintf(sqlStatement, databaseName))
-
-	var dbName string
-	err := row.Scan(&dbName)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return false, nil
-		}
-		return false, err
-	}
-	return true, nil
-}
-
-func createDatabase(db *sql.DB, databaseName string) error {
-	sqlStatement := "CREATE DATABASE %s ENCODING='UTF8'"
-	_, err := db.Exec(fmt.Sprintf(sqlStatement, databaseName))
-	return err
-}
-
-func getDbConnection(conf PostgresConf) *sql.DB {
-	_init_db.Do(func() {
-
-		sqlUrl := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s connect_timeout=30", conf.HostName, conf.Port, conf.User, conf.Password, "postgres", conf.SslMode)
-		glog.V(3).Infoln("Opening postgres master database")
-
-		var dbErr error
-		_db_connection, dbErr := sql.Open("postgres", sqlUrl)
-		if dbErr != nil {
-			_db_connection.Close()
-			_db_connection = nil
-			panic(dbErr)
-		}
-
-		pingErr := _db_connection.Ping()
-		if pingErr != nil {
-			_db_connection.Close()
-			_db_connection = nil
-			panic(pingErr)
-		}
-
-		glog.V(3).Infoln("Checking to see if DB exists: ", conf.DataBase)
-		var existsErr error
-		dbExists, existsErr := databaseExists(_db_connection, conf.DataBase)
-		if existsErr != nil {
-			_db_connection.Close()
-			_db_connection = nil
-			panic(existsErr)
-		}
-
-		if !dbExists {
-			glog.V(3).Infoln("Database doesn't exist. Attempting to create one:  ", conf.DataBase)
-			createErr := createDatabase(_db_connection, conf.DataBase)
-			if createErr != nil {
-				_db_connection.Close()
-				_db_connection = nil
-				panic(createErr)
-			}
-		}
-
-		glog.V(3).Infoln("Closing master postgres database and opening configured database: ", conf.DataBase)
-		_db_connection.Close()
-		_db_connection = nil
-
-		sqlUrl = fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s connect_timeout=30", conf.HostName, conf.Port, conf.User, conf.Password, conf.DataBase, conf.SslMode)
-		_db_connection, dbErr = sql.Open("postgres", sqlUrl)
-		if dbErr != nil {
-			_db_connection.Close()
-			_db_connection = nil
-			panic(dbErr)
-		}
-
-		pingErr = _db_connection.Ping()
-		if pingErr != nil {
-			_db_connection.Close()
-			_db_connection = nil
-			panic(pingErr)
-		}
-
-		var maxIdleConnections, maxOpenConnections int
-
-		if conf.MaxIdleConnections != 0 {
-			maxIdleConnections = conf.MaxIdleConnections
-		} else {
-			maxIdleConnections = default_maxIdleConnections
-		}
-		if conf.MaxOpenConnections != 0 {
-			maxOpenConnections = conf.MaxOpenConnections
-		} else {
-			maxOpenConnections = default_maxOpenConnections
-		}
-
-		_db_connection.SetMaxIdleConns(maxIdleConnections)
-		_db_connection.SetMaxOpenConns(maxOpenConnections)
-	})
-	return _db_connection
-}
-
-//func NewPostgresStore(master string, confs []PostgresConf, isSharding bool, shardCount int) *PostgresStore {
-func NewPostgresStore(master string, conf PostgresConf) *PostgresStore {
-	pg := &PostgresStore{
-		db: getDbConnection(conf),
-	}
-
-	pg.createDirectoriesTable()
-
-	if err := pg.createFilesTable(); err != nil {
-		fmt.Printf("create table failed %v", err)
-	}
-
-	return pg
-}
-
-func (s *PostgresStore) Get(fullFilePath string) (fid string, err error) {
-	if err != nil {
-		return "", fmt.Errorf("PostgresStore Get operation can not parse file path %s: err is %v", fullFilePath, err)
-	}
-	fid, err = s.query(fullFilePath)
-
-	return fid, err
-}
-
-func (s *PostgresStore) Put(fullFilePath string, fid string) (err error) {
 	var old_fid string
 	if old_fid, err = s.query(fullFilePath); err != nil && err != sql.ErrNoRows {
 		return fmt.Errorf("PostgresStore Put operation failed when querying path %s: err is %v", fullFilePath, err)
@@ -227,43 +62,61 @@ func (s *PostgresStore) Put(fullFilePath string, fid string) (err error) {
 		}
 	}
 	return
+
 }
 
-func (s *PostgresStore) Delete(fullFilePath string) (err error) {
-	var fid string
+func (s *PostgresStore) FindFile(fullFilePath string) (fid string, err error) {
+
 	if err != nil {
-		return fmt.Errorf("PostgresStore Delete operation can not parse file path %s: err is %v", fullFilePath, err)
+		return "", fmt.Errorf("PostgresStore Get operation can not parse file path %s: err is %v", fullFilePath, err)
+	}
+	fid, err = s.query(fullFilePath)
+
+	return fid, err
+}
+
+func (s *PostgresStore) LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error) {
+	fullPath := filepath.Join(dirPath, name)
+	if fileId, err = s.FindFile(fullPath); err == nil {
+		return true, fileId, nil
+	}
+	if _, _, err := s.lookupDirectory(fullPath); err == nil {
+		return true, "", err
+	}
+	return false, "", err
+}
+
+func (s *PostgresStore) DeleteFile(fullFilePath string) (fid string, err error) {
+	if err != nil {
+		return "", fmt.Errorf("PostgresStore Delete operation can not parse file path %s: err is %v", fullFilePath, err)
 	}
 	if fid, err = s.query(fullFilePath); err != nil {
-		return fmt.Errorf("PostgresStore Delete operation failed when querying path %s: err is %v", fullFilePath, err)
+		return "", fmt.Errorf("PostgresStore Delete operation failed when querying path %s: err is %v", fullFilePath, err)
 	} else if fid == "" {
-		return nil
+		return "", nil
 	}
 	if err = s.delete(fullFilePath); err != nil {
-		return fmt.Errorf("PostgresStore Delete operation failed when deleting path %s: err is %v", fullFilePath, err)
+		return "", fmt.Errorf("PostgresStore Delete operation failed when deleting path %s: err is %v", fullFilePath, err)
 	} else {
-		return nil
+		return "", nil
 	}
 }
 
-func (s *PostgresStore) Close() {
-	s.db.Close()
-}
+func (s *PostgresStore) ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error) {
 
-func (s *PostgresStore) FindDir(dirPath string) (dirId filer.DirectoryId, err error) {
-	dirId, _, err = s.lookupDirectory(dirPath)
-	return dirId, err
-}
-
-func (s *PostgresStore) ListDirs(dirPath string) (dirs []filer.DirectoryEntry, err error) {
-	dirs, err = s.findDirectories(dirPath, 20)
+	dirs, err = s.findDirectories(dirPath, 1000)
 
 	glog.V(3).Infof("Postgres ListDirs = found %d directories under %s", len(dirs), dirPath)
 
 	return dirs, err
 }
 
-func (s *PostgresStore) DeleteDir(dirPath string, recursive bool) (err error) {
+func (s *PostgresStore) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
+	files, err = s.findFiles(dirPath, lastFileName, limit)
+	return files, err
+}
+
+func (s *PostgresStore) DeleteDirectory(dirPath string, recursive bool) (err error) {
 	err = s.deleteDirectory(dirPath, recursive)
 	if err != nil {
 		glog.V(0).Infof("Error in Postgres DeleteDir '%s' (recursive = '%t'): %s", err)
@@ -271,353 +124,26 @@ func (s *PostgresStore) DeleteDir(dirPath string, recursive bool) (err error) {
 	return err
 }
 
-func (s *PostgresStore) FindFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
-	files, err = s.findFiles(dirPath, lastFileName, limit)
-
-	return files, err
+func (s *PostgresStore) Move(fromPath string, toPath string) (err error) {
+	glog.V(3).Infoln("Calling posgres_store Move")
+	return errors.New("Move is not yet implemented for the PostgreSQL store.")
 }
 
-var createDirectoryTable = `
-
-CREATE TABLE IF NOT EXISTS %s (
-  id BIGSERIAL NOT NULL,
-  directoryRoot VARCHAR(1024) NOT NULL DEFAULT '',
-  directoryName VARCHAR(1024) NOT NULL DEFAULT '',
-  CONSTRAINT unique_directory UNIQUE (directoryRoot, directoryName)
-);
-`
-
-var createFileTable = `
-
-CREATE TABLE IF NOT EXISTS %s (
-  id BIGSERIAL NOT NULL,
-  directoryPart VARCHAR(1024) NOT NULL DEFAULT '',
-  filePart VARCHAR(1024) NOT NULL DEFAULT '',
-  fid VARCHAR(36) NOT NULL DEFAULT '',
-  createTime BIGINT NOT NULL DEFAULT 0,
-  updateTime BIGINT NOT NULL DEFAULT 0,
-  remark VARCHAR(20) NOT NULL DEFAULT '',
-  status SMALLINT NOT NULL DEFAULT '1',
-  PRIMARY KEY (id),
-  CONSTRAINT %s_unique_file UNIQUE (directoryPart, filePart)
-);
-`
-
-func (s *PostgresStore) createDirectoriesTable() error {
-	glog.V(3).Infoln("Creating postgres table if it doesn't exist: ", directoriesTableName)
-
-	sqlCreate := fmt.Sprintf(createDirectoryTable, directoriesTableName)
-
-	stmt, err := s.db.Prepare(sqlCreate)
-	if err != nil {
-		return err
+//func NewPostgresStore(master string, confs []PostgresConf, isSharding bool, shardCount int) *PostgresStore {
+func NewPostgresStore(master string, conf PostgresConf) *PostgresStore {
+	pg := &PostgresStore{
+		db: getDbConnection(conf),
 	}
-	defer stmt.Close()
 
-	_, err = stmt.Exec()
-	if err != nil {
-		return err
+	pg.createDirectoriesTable()
+
+	if err := pg.createFilesTable(); err != nil {
+		fmt.Printf("create table failed %v", err)
 	}
-	return nil
+
+	return pg
 }
 
-func (s *PostgresStore) createFilesTable() error {
-
-	glog.V(3).Infoln("Creating postgres table if it doesn't exist: ", filesTableName)
-
-	sqlCreate := fmt.Sprintf(createFileTable, filesTableName, filesTableName)
-
-	stmt, err := s.db.Prepare(sqlCreate)
-	if err != nil {
-		return err
-	}
-	defer stmt.Close()
-
-	_, err = stmt.Exec()
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (s *PostgresStore) query(uriPath string) (string, error) {
-	directoryPart, filePart := filepath.Split(uriPath)
-	sqlStatement := fmt.Sprintf("SELECT fid FROM %s WHERE directoryPart=$1 AND filePart=$2", filesTableName)
-
-	row := s.db.QueryRow(sqlStatement, directoryPart, filePart)
-	var fid string
-	err := row.Scan(&fid)
-
-	glog.V(3).Infof("Postgres query -- looking up path '%s' and found id '%s' ", uriPath, fid)
-
-	if err != nil {
-		return "", err
-	}
-	return fid, nil
-}
-
-func (s *PostgresStore) update(uriPath string, fid string) error {
-	directoryPart, filePart := filepath.Split(uriPath)
-	sqlStatement := fmt.Sprintf("UPDATE %s SET fid=$1, updateTime=$2 WHERE directoryPart=$3 AND filePart=$4", filesTableName)
-
-	glog.V(3).Infof("Postgres query -- updating path '%s' with id '%s'", uriPath, fid)
-
-	res, err := s.db.Exec(sqlStatement, fid, time.Now().Unix(), directoryPart, filePart)
-	if err != nil {
-		return err
-	}
-
-	_, err = res.RowsAffected()
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (s *PostgresStore) insert(uriPath string, fid string) error {
-	directoryPart, filePart := filepath.Split(uriPath)
-
-	existingId, _, _ := s.lookupDirectory(directoryPart)
-	if existingId == 0 {
-		s.recursiveInsertDirectory(directoryPart)
-	}
-
-	sqlStatement := fmt.Sprintf("INSERT INTO %s (directoryPart,filePart,fid,createTime) VALUES($1, $2, $3, $4)", filesTableName)
-	glog.V(3).Infof("Postgres query -- inserting path '%s' with id '%s'", uriPath, fid)
-
-	res, err := s.db.Exec(sqlStatement, directoryPart, filePart, fid, time.Now().Unix())
-
-	if err != nil {
-		return err
-	}
-
-	rows, err := res.RowsAffected()
-	if rows != 1 {
-		return fmt.Errorf("Postgres insert -- rows affected = %d. Expecting 1", rows)
-	}
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (s *PostgresStore) recursiveInsertDirectory(dirPath string) {
-	pathParts := strings.Split(dirPath, "/")
-
-	var workingPath string = "/"
-	for _, part := range pathParts {
-		if part == "" {
-			continue
-		}
-		workingPath += (part + "/")
-		existingId, _, _ := s.lookupDirectory(workingPath)
-		if existingId == 0 {
-			s.insertDirectory(workingPath)
-		}
-	}
-}
-
-func (s *PostgresStore) insertDirectory(dirPath string) {
-	pathParts := strings.Split(dirPath, "/")
-
-	directoryRoot := "/"
-	directoryName := ""
-	if len(pathParts) > 1 {
-		directoryRoot = strings.Join(pathParts[0:len(pathParts)-2], "/") + "/"
-		directoryName = strings.Join(pathParts[len(pathParts)-2:], "/")
-	} else if len(pathParts) == 1 {
-		directoryRoot = "/"
-		directoryName = pathParts[0] + "/"
-	}
-	sqlInsertDirectoryStatement := fmt.Sprintf("INSERT INTO %s (directoryroot, directoryname)  "+
-		"SELECT $1, $2 WHERE NOT EXISTS ( SELECT id FROM %s WHERE directoryroot=$3 AND directoryname=$4 )",
-		directoriesTableName, directoriesTableName)
-
-	glog.V(4).Infof("Postgres query -- Inserting directory (if it doesn't exist) - root = %s, name = %s",
-		directoryRoot, directoryName)
-
-	_, err := s.db.Exec(sqlInsertDirectoryStatement, directoryRoot, directoryName, directoryRoot, directoryName)
-	if err != nil {
-		glog.V(0).Infof("Postgres query -- Error inserting directory - root = %s, name = %s: %s",
-			directoryRoot, directoryName, err)
-	}
-}
-
-func (s *PostgresStore) delete(uriPath string) error {
-	directoryPart, filePart := filepath.Split(uriPath)
-	sqlStatement := fmt.Sprintf("DELETE FROM %s WHERE directoryPart=$1 AND filePart=$2", filesTableName)
-
-	glog.V(3).Infof("Postgres query -- deleting path '%s'", uriPath)
-
-	res, err := s.db.Exec(sqlStatement, directoryPart, filePart)
-	if err != nil {
-		return err
-	}
-
-	_, err = res.RowsAffected()
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (s *PostgresStore) lookupDirectory(dirPath string) (filer.DirectoryId, string, error) {
-	directoryRoot, directoryName := s.mySplitPath(dirPath)
-
-	sqlStatement := fmt.Sprintf("SELECT id, directoryroot, directoryname FROM %s WHERE directoryRoot=$1 AND directoryName=$2", directoriesTableName)
-
-	row := s.db.QueryRow(sqlStatement, directoryRoot, directoryName)
-	var id filer.DirectoryId
-	var dirRoot string
-	var dirName string
-	err := row.Scan(&id, &dirRoot, &dirName)
-
-	glog.V(3).Infof("Postgres lookupDirectory -- looking up directory '%s' and found id '%d', root '%s', name '%s' ", dirPath, id, dirRoot, dirName)
-
-	if err != nil {
-		return 0, "", err
-	}
-	return id, filepath.Join(dirRoot, dirName), err
-}
-
-func (s *PostgresStore) findDirectories(dirPath string, limit int) (dirs []filer.DirectoryEntry, err error) {
-	sqlStatement := fmt.Sprintf("SELECT id, directoryroot, directoryname FROM %s WHERE directoryRoot=$1 AND directoryName != '' ORDER BY id LIMIT $2", directoriesTableName)
-	rows, err := s.db.Query(sqlStatement, dirPath, limit)
-
-	if err != nil {
-		glog.V(0).Infof("Postgres findDirectories error: %s", err)
-	}
-
-	if rows != nil {
-		defer rows.Close()
-		for rows.Next() {
-			var id filer.DirectoryId
-			var directoryRoot string
-			var directoryName string
-
-			scanErr := rows.Scan(&id, &directoryRoot, &directoryName)
-			if scanErr != nil {
-				err = scanErr
-			}
-			dirs = append(dirs, filer.DirectoryEntry{Name: (directoryName), Id: id})
-		}
-	}
-	return
-}
-
-func (s *PostgresStore) safeToDeleteDirectory(dirPath string, recursive bool) bool {
-	if recursive {
-		return true
-	}
-	sqlStatement := fmt.Sprintf("SELECT id FROM %s WHERE directoryRoot LIKE $1 LIMIT 1", directoriesTableName)
-	row := s.db.QueryRow(sqlStatement, dirPath+"%")
-
-	var id filer.DirectoryId
-	err := row.Scan(&id)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return true
-		}
-	}
-	return false
-}
-
-func (s *PostgresStore) mySplitPath(dirPath string) (directoryRoot string, directoryName string) {
-	pathParts := strings.Split(dirPath, "/")
-	directoryRoot = "/"
-	directoryName = ""
-	if len(pathParts) > 1 {
-		directoryRoot = strings.Join(pathParts[0:len(pathParts)-2], "/") + "/"
-		directoryName = strings.Join(pathParts[len(pathParts)-2:], "/")
-	} else if len(pathParts) == 1 {
-		directoryRoot = "/"
-		directoryName = pathParts[0] + "/"
-	}
-	return directoryRoot, directoryName
-}
-
-func (s *PostgresStore) deleteDirectory(dirPath string, recursive bool) (err error) {
-	directoryRoot, directoryName := s.mySplitPath(dirPath)
-
-	// delete files
-	sqlStatement := fmt.Sprintf("DELETE FROM %s WHERE directorypart=$1", filesTableName)
-	_, err = s.db.Exec(sqlStatement, dirPath)
-	if err != nil {
-		return err
-	}
-
-	// delete specific directory if it is empty or recursive delete was requested
-	safeToDelete := s.safeToDeleteDirectory(dirPath, recursive)
-	if safeToDelete {
-		sqlStatement = fmt.Sprintf("DELETE FROM %s WHERE directoryRoot=$1 AND directoryName=$2", directoriesTableName)
-		_, err = s.db.Exec(sqlStatement, directoryRoot, directoryName)
-		if err != nil {
-			return err
-		}
-	}
-
-	if recursive {
-		// delete descendant files
-		sqlStatement = fmt.Sprintf("DELETE FROM %s WHERE directorypart LIKE $1", filesTableName)
-		_, err = s.db.Exec(sqlStatement, dirPath+"%")
-		if err != nil {
-			return err
-		}
-
-		// delete descendant directories
-		sqlStatement = fmt.Sprintf("DELETE FROM %s WHERE directoryRoot LIKE $1", directoriesTableName)
-		_, err = s.db.Exec(sqlStatement, dirPath+"%")
-		if err != nil {
-			return err
-		}
-	}
-
-	return err
-}
-
-func (s *PostgresStore) findFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
-	var rows *sql.Rows = nil
-
-	if lastFileName == "" {
-		sqlStatement :=
-			fmt.Sprintf("SELECT fid, directorypart, filepart FROM %s WHERE directorypart=$1 ORDER BY id LIMIT $2", filesTableName)
-		rows, err = s.db.Query(sqlStatement, dirPath, limit)
-	} else {
-		sqlStatement :=
-			fmt.Sprintf("SELECT fid, directorypart, filepart FROM %s WHERE directorypart=$1 "+
-				"AND id > (SELECT id FROM %s WHERE directoryPart=$2 AND filepart=$3)  ORDER BY id LIMIT $4",
-				filesTableName, filesTableName)
-		_, lastFileNameName := filepath.Split(lastFileName)
-		rows, err = s.db.Query(sqlStatement, dirPath, dirPath, lastFileNameName, limit)
-	}
-
-	if err != nil {
-		glog.V(0).Infof("Postgres find files error: %s", err)
-	}
-
-	if rows != nil {
-		defer rows.Close()
-
-		for rows.Next() {
-			var fid filer.FileId
-			var directoryPart string
-			var filePart string
-
-			scanErr := rows.Scan(&fid, &directoryPart, &filePart)
-			if scanErr != nil {
-				err = scanErr
-			}
-
-			files = append(files, filer.FileEntry{Name: filepath.Join(directoryPart, filePart), Id: fid})
-			if len(files) >= limit {
-				break
-			}
-		}
-	}
-
-	glog.V(3).Infof("Postgres findFiles -- looking up files under '%s' and found %d files. Limit=%d, lastFileName=%s",
-		dirPath, len(files), limit, lastFileName)
-
-	return files, err
+func (s *PostgresStore) Close() {
+	s.db.Close()
 }

weed/filer/redis_store/redis_store.go

@@ -3,7 +3,7 @@ package redis_store
 import (
 	"github.com/chrislusf/seaweedfs/weed/filer"
 
-	redis "gopkg.in/redis.v2"
+	"github.com/go-redis/redis"
 )
 
 type RedisStore struct {
@@ -11,10 +11,10 @@ type RedisStore struct {
 }
 
 func NewRedisStore(hostPort string, password string, database int) *RedisStore {
-	client := redis.NewTCPClient(&redis.Options{
+	client := redis.NewClient(&redis.Options{
 		Addr:     hostPort,
 		Password: password,
-		DB:       int64(database),
+		DB:       database,
 	})
 	return &RedisStore{Client: client}
 }
@@ -27,7 +27,7 @@ func (s *RedisStore) Get(fullFileName string) (fid string, err error) {
 	return fid, err
 }
 func (s *RedisStore) Put(fullFileName string, fid string) (err error) {
-	_, err = s.Client.Set(fullFileName, fid).Result()
+	_, err = s.Client.Set(fullFileName, fid, 0).Result()
 	if err == redis.Nil {
 		err = nil
 	}

weed/filer/vasto_store/design.txt

@@ -0,0 +1,45 @@
+There are two main components of a filer: directories and files.
+
+My previous approach was to use some sequance number to generate directoryId.
+However, this is not scalable. The id generation itself is a bottleneck.
+It needs careful locking and deduplication checking to get a directoryId.
+
+In a second design, each directory is deterministically mapped to UUID version 3,
+which uses MD5 to map a tuple of <uuid, name> to a version 3 UUID.
+However, this UUID3 approach is logically the same as storing the full path.
+
+Storing the full path is the simplest design.
+
+separator is a special byte, 0x00.
+
+When writing a file:
+  <file parent full path, separator, file name> => fildId, file properties
+For folders:
+  The filer breaks the directory path into folders.
+  for each folder:
+    if it is not in cache:
+	  check whether the folder is created in the KVS, if not:
+        set <folder parent full path, separator, folder name> => directory properties
+	if no permission for the folder:
+	  break
+	  
+
+The filer caches the most recently used folder permissions with a TTL.
+  So any folder permission change needs to wait TTL interval to take effect.
+
+
+
+When listing the directory:
+  prefix scan of using (the folder full path + separator) as the prefix
+
+The downside:
+	1. Rename a folder will need to recursively process all sub folders and files.
+	2. Move   a folder will need to recursively process all sub folders and files.
+So these operations are not allowed if the folder is not empty.
+
+Allowing:
+	1. Rename a file
+	2. Move a file to a different folder
+	3. Delete an empty folder
+
+

weed/filer2/embedded/embedded_store.go

@@ -0,0 +1,42 @@
+package embedded
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+)
+
+type EmbeddedStore struct {
+	db *leveldb.DB
+}
+
+func NewEmbeddedStore(dir string) (filer *EmbeddedStore, err error) {
+	filer = &EmbeddedStore{}
+	if filer.db, err = leveldb.OpenFile(dir, nil); err != nil {
+		return
+	}
+	return
+}
+
+func (filer *EmbeddedStore) InsertEntry(entry *filer2.Entry) (err error) {
+	return nil
+}
+
+func (filer *EmbeddedStore) AddDirectoryLink(directory *filer2.Entry, delta int32) (err error) {
+	return nil
+}
+
+func (filer *EmbeddedStore) AppendFileChunk(fullpath filer2.FullPath, fileChunk filer2.FileChunk) (err error) {
+	return nil
+}
+
+func (filer *EmbeddedStore) FindEntry(fullpath filer2.FullPath) (found bool, entry *filer2.Entry, err error) {
+	return false, nil, nil
+}
+
+func (filer *EmbeddedStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
+	return nil, nil
+}
+
+func (filer *EmbeddedStore) ListDirectoryEntries(fullpath filer2.FullPath) (entries []*filer2.Entry, err error) {
+	return nil, nil
+}

weed/filer2/filechunks.go

@@ -0,0 +1,23 @@
+package filer2
+
+type Chunks []FileChunk
+
+func (chunks Chunks) TotalSize() (size uint64) {
+	for _, c := range chunks {
+		t := uint64(c.Offset + int64(c.Size))
+		if size < t {
+			size = t
+		}
+	}
+	return
+}
+
+func (chunks Chunks) Len() int {
+	return len(chunks)
+}
+func (chunks Chunks) Swap(i, j int) {
+	chunks[i], chunks[j] = chunks[j], chunks[i]
+}
+func (chunks Chunks) Less(i, j int) bool {
+	return chunks[i].Offset < chunks[j].Offset
+}

weed/filer2/filer.go

@@ -0,0 +1,150 @@
+package filer2
+
+import (
+	"fmt"
+
+	"github.com/karlseguin/ccache"
+	"strings"
+	"path/filepath"
+	"time"
+	"os"
+)
+
+type Filer struct {
+	master         string
+	store          FilerStore
+	directoryCache *ccache.Cache
+}
+
+func NewFiler(master string) *Filer {
+	return &Filer{
+		master:         master,
+		directoryCache: ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
+	}
+}
+
+func (f *Filer) SetStore(store FilerStore) () {
+	f.store = store
+}
+
+func (f *Filer) DisableDirectoryCache() () {
+	f.directoryCache = nil
+}
+
+func (f *Filer) CreateEntry(entry *Entry) (error) {
+
+	dirParts := strings.Split(string(entry.FullPath), "/")
+
+	// fmt.Printf("directory parts: %+v\n", dirParts)
+
+	var lastDirectoryEntry *Entry
+
+	for i := 1; i < len(dirParts); i++ {
+		dirPath := "/" + filepath.Join(dirParts[:i]...)
+		// fmt.Printf("%d directory: %+v\n", i, dirPath)
+
+		dirFound := false
+
+		// first check local cache
+		dirEntry := f.cacheGetDirectory(dirPath)
+
+		// not found, check the store directly
+		if dirEntry == nil {
+			var dirFindErr error
+			dirFound, dirEntry, dirFindErr = f.FindEntry(FullPath(dirPath))
+			if dirFindErr != nil {
+				return fmt.Errorf("findDirectory %s: %v", dirPath, dirFindErr)
+			}
+		}
+
+		// no such existing directory
+		if !dirFound {
+
+			// create the directory
+			now := time.Now()
+
+			dirEntry = &Entry{
+				FullPath: FullPath(dirPath),
+				Attr: Attr{
+					Mtime:  now,
+					Crtime: now,
+					Mode:   os.ModeDir | 0660,
+					Uid:    entry.Uid,
+					Gid:    entry.Gid,
+				},
+			}
+
+			mkdirErr := f.store.InsertEntry(dirEntry)
+			if mkdirErr != nil {
+				return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr)
+			}
+		}
+
+		// cache the directory entry
+		f.cacheSetDirectory(dirPath, dirEntry, i)
+
+		// remember the direct parent directory entry
+		if i == len(dirParts)-1 {
+			lastDirectoryEntry = dirEntry
+		}
+
+	}
+
+	if lastDirectoryEntry == nil {
+		return fmt.Errorf("parent folder not found: %v", entry.FullPath)
+	}
+
+	if !hasWritePermission(lastDirectoryEntry, entry) {
+		return fmt.Errorf("no write permission in folder %v", lastDirectoryEntry.FullPath)
+	}
+
+	if err := f.store.InsertEntry(entry); err != nil {
+		return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
+	}
+
+	return nil
+}
+
+func (f *Filer) AppendFileChunk(p FullPath, c FileChunk) (err error) {
+	return f.store.AppendFileChunk(p, c)
+}
+
+func (f *Filer) FindEntry(p FullPath) (found bool, entry *Entry, err error) {
+	return f.store.FindEntry(p)
+}
+
+func (f *Filer) DeleteEntry(p FullPath) (fileEntry *Entry, err error) {
+	return f.store.DeleteEntry(p)
+}
+
+func (f *Filer) ListDirectoryEntries(p FullPath) ([]*Entry, error) {
+	if strings.HasSuffix(string(p), "/") {
+		p = p[0:len(p)-1]
+	}
+	return f.store.ListDirectoryEntries(p)
+}
+
+func (f *Filer) cacheGetDirectory(dirpath string) (*Entry) {
+	if f.directoryCache == nil {
+		return nil
+	}
+	item := f.directoryCache.Get(dirpath)
+	if item == nil {
+		return nil
+	}
+	return item.Value().(*Entry)
+}
+
+func (f *Filer) cacheSetDirectory(dirpath string, dirEntry *Entry, level int) {
+
+	if f.directoryCache == nil {
+		return
+	}
+
+	minutes := 60
+	if level < 10 {
+		minutes -= level * 6
+	}
+
+	f.directoryCache.Set(dirpath, dirEntry, time.Duration(minutes)*time.Minute)
+}

weed/filer2/filer_structure.go

@@ -0,0 +1,66 @@
+package filer2
+
+import (
+	"errors"
+	"os"
+	"time"
+	"path/filepath"
+)
+
+type FileId string //file id in SeaweedFS
+type FullPath string
+
+func (fp FullPath) DirAndName() (string, string) {
+	dir, name := filepath.Split(string(fp))
+	if dir == "/" {
+		return dir, name
+	}
+	if len(dir) < 1 {
+		return "/", ""
+	}
+	return dir[:len(dir)-1], name
+}
+
+type Attr struct {
+	Mtime  time.Time   // time of last modification
+	Crtime time.Time   // time of creation (OS X only)
+	Mode   os.FileMode // file mode
+	Uid    uint32      // owner uid
+	Gid    uint32      // group gid
+}
+
+type Entry struct {
+	FullPath
+
+	Attr
+
+	// the following is for files
+	Chunks []FileChunk `json:"chunks,omitempty"`
+}
+
+type FileChunk struct {
+	Fid    FileId `json:"fid,omitempty"`
+	Offset int64  `json:"offset,omitempty"`
+	Size   uint64 `json:"size,omitempty"` // size in bytes
+}
+
+type AbstractFiler interface {
+	CreateEntry(*Entry) (error)
+	AppendFileChunk(FullPath, FileChunk) (err error)
+	FindEntry(FullPath) (found bool, fileEntry *Entry, err error)
+	DeleteEntry(FullPath) (fileEntry *Entry, err error)
+
+	ListDirectoryEntries(dirPath FullPath) ([]*Entry, error)
+	UpdateEntry(*Entry) (error)
+}
+
+var ErrNotFound = errors.New("filer: no entry is found in filer store")
+
+type FilerStore interface {
+	InsertEntry(*Entry) (error)
+	AppendFileChunk(FullPath, FileChunk) (err error)
+	FindEntry(FullPath) (found bool, entry *Entry, err error)
+	DeleteEntry(FullPath) (fileEntry *Entry, err error)
+
+	ListDirectoryEntries(dirPath FullPath) ([]*Entry, error)
+}

weed/filer2/memdb/memdb_store.go

@@ -0,0 +1,91 @@
+package memdb
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/google/btree"
+	"strings"
+	"fmt"
+	"time"
+)
+
+type MemDbStore struct {
+	tree *btree.BTree
+}
+
+type Entry struct {
+	*filer2.Entry
+}
+
+func (a Entry) Less(b btree.Item) bool {
+	return strings.Compare(string(a.FullPath), string(b.(Entry).FullPath)) < 0
+}
+
+func NewMemDbStore() (filer *MemDbStore) {
+	filer = &MemDbStore{}
+	filer.tree = btree.New(8)
+	return
+}
+
+func (filer *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) {
+	// println("inserting", entry.FullPath)
+	filer.tree.ReplaceOrInsert(Entry{entry})
+	return nil
+}
+
+func (filer *MemDbStore) AppendFileChunk(fullpath filer2.FullPath, fileChunk filer2.FileChunk) (err error) {
+	found, entry, err := filer.FindEntry(fullpath)
+	if !found {
+		return fmt.Errorf("No such file: %s", fullpath)
+	}
+	entry.Chunks = append(entry.Chunks, fileChunk)
+	entry.Mtime = time.Now()
+	return nil
+}
+
+func (filer *MemDbStore) FindEntry(fullpath filer2.FullPath) (found bool, entry *filer2.Entry, err error) {
+	item := filer.tree.Get(Entry{&filer2.Entry{FullPath: fullpath}})
+	if item == nil {
+		return false, nil, nil
+	}
+	entry = item.(Entry).Entry
+	return true, entry, nil
+}
+
+func (filer *MemDbStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
+	item := filer.tree.Delete(Entry{&filer2.Entry{FullPath: fullpath}})
+	if item == nil {
+		return nil, nil
+	}
+	entry = item.(Entry).Entry
+	return entry, nil
+}
+
+func (filer *MemDbStore) ListDirectoryEntries(fullpath filer2.FullPath) (entries []*filer2.Entry, err error) {
+	filer.tree.AscendGreaterOrEqual(Entry{&filer2.Entry{FullPath: fullpath}},
+		func(item btree.Item) bool {
+			entry := item.(Entry).Entry
+			// println("checking", entry.FullPath)
+			if entry.FullPath == fullpath {
+				// skipping the current directory
+				// println("skipping the folder", entry.FullPath)
+				return true
+			}
+			dir, _ := entry.FullPath.DirAndName()
+			if !strings.HasPrefix(dir, string(fullpath)) {
+				// println("directory is:", dir, "fullpath:", fullpath)
+				// println("breaking from", entry.FullPath)
+				return false
+			}
+			if dir != string(fullpath) {
+				// this could be items in deeper directories
+				// println("skipping deeper folder", entry.FullPath)
+				return true
+			}
+			// now process the directory items
+			// println("adding entry", entry.FullPath)
+			entries = append(entries, entry)
+			return true
+		},
+	)
+	return entries, nil
+}

weed/filer2/memdb/memdb_store_test.go

@@ -0,0 +1,130 @@
+package memdb
+
+import (
+	"testing"
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+)
+
+func TestCreateAndFind(t *testing.T) {
+	filer := filer2.NewFiler("")
+	filer.SetStore(NewMemDbStore())
+	filer.DisableDirectoryCache()
+
+	fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg")
+
+	entry1 := &filer2.Entry{
+		FullPath: fullpath,
+		Attr: filer2.Attr{
+			Mode: 0440,
+			Uid:  1234,
+			Gid:  5678,
+		},
+	}
+
+	if err := filer.CreateEntry(entry1); err != nil {
+		t.Errorf("create entry %v: %v", entry1.FullPath, err)
+		return
+	}
+
+	found, entry, err := filer.FindEntry(fullpath)
+
+	if err != nil {
+		t.Errorf("find entry: %v", err)
+		return
+	}
+
+	if !found {
+		t.Errorf("Failed to find newly created file")
+		return
+	}
+
+	if entry.FullPath != entry1.FullPath {
+		t.Errorf("find wrong entry: %v", entry.FullPath)
+		return
+	}
+
+}
+
+func TestCreateFileAndList(t *testing.T) {
+	filer := filer2.NewFiler("")
+	filer.SetStore(NewMemDbStore())
+	filer.DisableDirectoryCache()
+
+	entry1 := &filer2.Entry{
+		FullPath: filer2.FullPath("/home/chris/this/is/one/file1.jpg"),
+		Attr: filer2.Attr{
+			Mode: 0440,
+			Uid:  1234,
+			Gid:  5678,
+		},
+	}
+
+	entry2 := &filer2.Entry{
+		FullPath: filer2.FullPath("/home/chris/this/is/one/file2.jpg"),
+		Attr: filer2.Attr{
+			Mode: 0440,
+			Uid:  1234,
+			Gid:  5678,
+		},
+	}
+
+	filer.CreateEntry(entry1)
+	filer.CreateEntry(entry2)
+
+	// checking the 2 files
+	entries, err := filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one/"))
+
+	if err != nil {
+		t.Errorf("list entries: %v", err)
+		return
+	}
+
+	if len(entries) != 2 {
+		t.Errorf("list entries count: %v", len(entries))
+		return
+	}
+
+	if entries[0].FullPath != entry1.FullPath {
+		t.Errorf("find wrong entry 1: %v", entries[0].FullPath)
+		return
+	}
+
+	if entries[1].FullPath != entry2.FullPath {
+		t.Errorf("find wrong entry 2: %v", entries[1].FullPath)
+		return
+	}
+
+	// checking one upper directory
+	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"))
+	if len(entries) != 1 {
+		t.Errorf("list entries count: %v", len(entries))
+		return
+	}
+
+	file3Path := filer2.FullPath("/home/chris/this/is/file3.jpg")
+	entry3 := &filer2.Entry{
+		FullPath: file3Path,
+		Attr: filer2.Attr{
+			Mode: 0440,
+			Uid:  1234,
+			Gid:  5678,
+		},
+	}
+	filer.CreateEntry(entry3)
+
+	// checking one upper directory
+	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"))
+	if len(entries) != 2 {
+		t.Errorf("list entries count: %v", len(entries))
+		return
+	}
+
+	// delete file and count
+	filer.DeleteEntry(file3Path)
+	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"))
+	if len(entries) != 1 {
+		t.Errorf("list entries count: %v", len(entries))
+		return
+	}
+
+}

weed/filer2/permission.go

@@ -0,0 +1,22 @@
+package filer2
+
+func hasWritePermission(dir *Entry, entry *Entry) bool {
+
+	if dir == nil {
+		return false
+	}
+
+	if dir.Uid == entry.Uid && dir.Mode&0200 > 0 {
+		return true
+	}
+
+	if dir.Gid == entry.Gid && dir.Mode&0020 > 0 {
+		return true
+	}
+
+	if dir.Mode&0002 > 0 {
+		return true
+	}
+
+	return false
+}

weed/filesys/dir.go

@@ -0,0 +1,140 @@
+package filesys
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"sync"
+
+	"bazil.org/fuse/fs"
+	"bazil.org/fuse"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+)
+
+type Dir struct {
+	Path        string
+	NodeMap     map[string]fs.Node
+	NodeMapLock sync.Mutex
+	wfs         *WFS
+}
+
+func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
+	attr.Mode = os.ModeDir | 0777
+	return nil
+}
+
+func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
+	dir.NodeMapLock.Lock()
+	defer dir.NodeMapLock.Unlock()
+
+	fmt.Printf("mkdir %+v\n", req)
+
+	node := &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs}
+	dir.NodeMap[req.Name] = node
+
+	return node, nil
+}
+
+func (dir *Dir) Lookup(ctx context.Context, name string) (node fs.Node, err error) {
+
+	dir.NodeMapLock.Lock()
+	defer dir.NodeMapLock.Unlock()
+
+	if dir.NodeMap == nil {
+		dir.NodeMap = make(map[string]fs.Node)
+	}
+
+	if node, ok := dir.NodeMap[name]; ok {
+		return node, nil
+	}
+
+	var entry *filer_pb.Entry
+	err = dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.LookupDirectoryEntryRequest{
+			Directory: dir.Path,
+			Name:      name,
+		}
+
+		glog.V(1).Infof("lookup directory entry: %v", request)
+		resp, err := client.LookupDirectoryEntry(ctx, request)
+		if err != nil {
+			return err
+		}
+
+		entry = resp.Entry
+
+		return nil
+	})
+
+	if entry != nil {
+		if entry.IsDirectory {
+			node = &Dir{Path: path.Join(dir.Path, name), wfs: dir.wfs}
+		} else {
+			node = &File{FileId: filer.FileId(entry.FileId), Name: name, wfs: dir.wfs}
+		}
+		dir.NodeMap[name] = node
+		return node, nil
+	}
+
+	return nil, err
+}
+
+func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
+
+	err = dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.ListEntriesRequest{
+			Directory: dir.Path,
+		}
+
+		glog.V(1).Infof("read directory: %v", request)
+		resp, err := client.ListEntries(ctx, request)
+		if err != nil {
+			return err
+		}
+
+		for _, entry := range resp.Entries {
+			if entry.IsDirectory {
+				dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_Dir}
+				ret = append(ret, dirent)
+			} else {
+				dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_File}
+				ret = append(ret, dirent)
+			}
+		}
+
+		return nil
+	})
+
+	return ret, err
+}
+
+func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
+
+	dir.NodeMapLock.Lock()
+	defer dir.NodeMapLock.Unlock()
+
+	return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.DeleteEntryRequest{
+			Directory:   dir.Path,
+			Name:        req.Name,
+			IsDirectory: req.Dir,
+		}
+
+		glog.V(1).Infof("remove directory entry: %v", request)
+		_, err := client.DeleteEntry(ctx, request)
+		if err != nil {
+			return err
+		}
+
+		delete(dir.NodeMap, req.Name)
+
+		return nil
+	})
+
+}

weed/filesys/file.go

@@ -0,0 +1,75 @@
+package filesys
+
+import (
+	"context"
+	"fmt"
+
+	"bazil.org/fuse"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"bazil.org/fuse/fs"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+)
+
+var _ = fs.Node(&File{})
+// var _ = fs.NodeOpener(&File{})
+// var _ = fs.NodeFsyncer(&File{})
+var _ = fs.Handle(&File{})
+var _ = fs.HandleReadAller(&File{})
+// var _ = fs.HandleReader(&File{})
+var _ = fs.HandleWriter(&File{})
+
+type File struct {
+	FileId filer.FileId
+	Name   string
+	wfs    *WFS
+}
+
+func (file *File) Attr(context context.Context, attr *fuse.Attr) error {
+	attr.Mode = 0444
+	return file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.GetFileAttributesRequest{
+			Name:      file.Name,
+			ParentDir: "", //TODO add parent folder
+			FileId:    string(file.FileId),
+		}
+
+		glog.V(1).Infof("read file size: %v", request)
+		resp, err := client.GetFileAttributes(context, request)
+		if err != nil {
+			return err
+		}
+
+		attr.Size = resp.Attributes.FileSize
+
+		return nil
+	})
+}
+
+func (file *File) ReadAll(ctx context.Context) (content []byte, err error) {
+
+	err = file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.GetFileContentRequest{
+			FileId: string(file.FileId),
+		}
+
+		glog.V(1).Infof("read file content: %v", request)
+		resp, err := client.GetFileContent(ctx, request)
+		if err != nil {
+			return err
+		}
+
+		content = resp.Content
+
+		return nil
+	})
+
+	return content, err
+}
+
+func (file *File) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error {
+	fmt.Printf("write file %+v\n", req)
+	return nil
+}

weed/filesys/wfs.go

@@ -0,0 +1,35 @@
+package filesys
+
+import (
+	"bazil.org/fuse/fs"
+	"fmt"
+	"google.golang.org/grpc"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+)
+
+type WFS struct {
+	filer string
+}
+
+func NewSeaweedFileSystem(filer string) *WFS {
+	return &WFS{
+		filer: filer,
+	}
+}
+
+func (wfs *WFS) Root() (fs.Node, error) {
+	return &Dir{Path: "/", wfs: wfs}, nil
+}
+
+func (wfs *WFS) withFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
+
+	grpcConnection, err := grpc.Dial(wfs.filer, grpc.WithInsecure())
+	if err != nil {
+		return fmt.Errorf("fail to dial %s: %v", wfs.filer, err)
+	}
+	defer grpcConnection.Close()
+
+	client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+
+	return fn(client)
+}

weed/glide.lock

@@ -1,65 +1,85 @@
-hash: fc2f4ccfa5d703b62b9c93047bae2295e6926ce1603c2164510c9010ff829945
-updated: 2016-06-24T14:19:43.337337605-07:00
+hash: bc35cb94b57f621923b87a4e71a0175425d1d2dad00fdd4d5ee35ab80cef1c60
+updated: 2018-04-16T00:58:48.978801-07:00
 imports:
 - name: bazil.org/fuse
-  version: 9e8e65293a037ce998ba0df102710c49313aa513
+  version: 371fbbdaa8987b715bdd21d6adc4c9b20155f748
   subpackages:
   - fs
   - fuseutil
 - name: github.com/boltdb/bolt
-  version: 3f7947a25d970e1e5f512276c14d5dcf731ccd5e
+  version: 2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8
 - name: github.com/chrislusf/raft
   version: 5f7ddd8f479583daf05879d3d3b174aa202c8fb7
   subpackages:
   - protobuf
+- name: github.com/dataence/encoding
+  version: b90e310a0325f9b765b4be7220df3642ad93ad8d
+  subpackages:
+  - bitpacking
+  - cursor
+  - delta/bp32
 - name: github.com/dgrijalva/jwt-go
-  version: f0777076321ab64f6efc15a82d9d23b98539b943
+  version: 06ea1031745cb8b3dab3f6a236daf2b0aa468b7e
 - name: github.com/disintegration/imaging
-  version: d8bbae1de109b518dabc98c6c1633eb358c148a4
+  version: fd34ef7671b12cdf1b024d98c6b327b2770d32c4
+- name: github.com/go-redis/redis
+  version: 877867d2845fbaf86798befe410b6ceb6f5c29a3
+  subpackages:
+  - internal
+  - internal/consistenthash
+  - internal/hashtag
+  - internal/pool
+  - internal/proto
+  - internal/singleflight
+  - internal/util
+- name: github.com/go-sql-driver/mysql
+  version: a0583e0143b1624142adab07e0e97fe106d99561
 - name: github.com/gocql/gocql
-  version: b7b8a0e04b0cb0ca0b379421c58ec6fab9939b85
+  version: 3540fc649cd7fc57cef5612b7bacac7a4fc443d6
   subpackages:
   - internal/lru
   - internal/murmur
   - internal/streams
 - name: github.com/gogo/protobuf
-  version: 2752d97bbd91927dd1c43296dbf8700e50e2708c
+  version: 1ef32a8b9fc3f8ec940126907cedb5998f6318e4
   subpackages:
   - proto
 - name: github.com/golang/protobuf
-  version: 0c1f6d65b5a189c2250d10e71a5506f06f9fa0a0
+  version: 925541529c1fa6821df4e44ce2723319eb2be768
   subpackages:
   - proto
+  - protoc-gen-go/descriptor
+  - ptypes
+  - ptypes/any
+  - ptypes/duration
+  - ptypes/timestamp
 - name: github.com/golang/snappy
-  version: d9eb7a3d35ec988b8585d4a0068e462c27d28380
+  version: 553a641470496b2327abcac10b36396bd98e45c9
+- name: github.com/google/btree
+  version: e89373fe6b4a7413d7acd6da1725b83ef713e6e4
 - name: github.com/gorilla/context
-  version: aed02d124ae4a0e94fea4541c8effd05bf0c8296
+  version: 08b5f424b9271eedf6f9f0ce86cb9396ed337a42
 - name: github.com/gorilla/mux
-  version: 9fa818a44c2bf1396a17f9d5a3c0f6dd39d2ff8e
+  version: 53c1911da2b537f792e7cafcb446b05ffe33b996
 - name: github.com/hailocab/go-hostpool
   version: e80d13ce29ede4452c43dea11e79b9bc8a15b478
-- name: github.com/hashicorp/golang-lru
-  version: a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4
-  subpackages:
-  - simplelru
 - name: github.com/klauspost/crc32
-  version: 19b0b332c9e4516a6370a0456e6182c3b5036720
-- name: github.com/reducedb/encoding
-  version: e31efcfdf1b19f4698709a6f522feaf101b1b1f4
+  version: cb6bfca970f6908083f26f39a79009d608efd5cd
+- name: github.com/lib/pq
+  version: d34b9ff171c21ad295489235aec8b6626023cd04
   subpackages:
-  - cursor
-  - delta/bp32
-  - bitpacking
+  - oid
 - name: github.com/rwcarlsen/goexif
-  version: 709fab3d192d7c62f86043caff1e7e3fb0f42bd8
+  version: fb35d3c3290d09f3524b684b5d42d0063c227158
   subpackages:
   - exif
   - tiff
+- name: github.com/soheilhy/cmux
+  version: e09e9389d85d8492d313d73d1469c029e710623f
 - name: github.com/syndtr/goleveldb
-  version: fa5b5c78794bc5c18f330361059f871ae8c2b9d6
+  version: 714f901b98fdb3aa954b4193d8cbd64a28d80cad
   subpackages:
   - leveldb
-  - leveldb/util
   - leveldb/cache
   - leveldb/comparer
   - leveldb/errors
@@ -70,24 +90,65 @@ imports:
   - leveldb/opt
   - leveldb/storage
   - leveldb/table
+  - leveldb/util
 - name: golang.org/x/image
-  version: 97680175a5267bb8b31f1923e7a66df98013b11a
+  version: f315e440302883054d0c2bd85486878cb4f8572c
   subpackages:
   - bmp
   - tiff
   - tiff/lzw
 - name: golang.org/x/net
-  version: bc3663df0ac92f928d419e31e0d2af22e683a5a2
+  version: 500e7a4f953ddaf55d316b4d3adc516aa0379622
   subpackages:
   - context
+  - http2
+  - http2/hpack
+  - idna
+  - internal/timeseries
+  - lex/httplex
+  - trace
 - name: golang.org/x/sys
-  version: 62bee037599929a6e9146f29d10dd5208c43507d
+  version: 2281fa97ef7b0c26324634d5a22f04babdac8713
   subpackages:
   - unix
-- name: gopkg.in/bufio.v1
-  version: 567b2bfa514e796916c4747494d6ff5132a1dfce
+- name: golang.org/x/text
+  version: 7922cc490dd5a7dbaa7fd5d6196b49db59ac042f
+  subpackages:
+  - secure/bidirule
+  - transform
+  - unicode/bidi
+  - unicode/norm
+- name: google.golang.org/genproto
+  version: 7fd901a49ba6a7f87732eb344f6e3c5b19d1b200
+  subpackages:
+  - googleapis/rpc/status
+- name: google.golang.org/grpc
+  version: d11072e7ca9811b1100b80ca0269ac831f06d024
+  subpackages:
+  - balancer
+  - balancer/base
+  - balancer/roundrobin
+  - codes
+  - connectivity
+  - credentials
+  - encoding
+  - encoding/proto
+  - grpclb/grpc_lb_v1/messages
+  - grpclog
+  - internal
+  - keepalive
+  - metadata
+  - naming
+  - peer
+  - reflection
+  - reflection/grpc_reflection_v1alpha
+  - resolver
+  - resolver/dns
+  - resolver/passthrough
+  - stats
+  - status
+  - tap
+  - transport
 - name: gopkg.in/inf.v0
-  version: 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4
-- name: gopkg.in/redis.v2
-  version: e6179049628164864e6e84e973cfb56335748dea
-devImports: []
+  version: d2d2541c53f18d2a059457998ce2876cc8e67cbf
+testImports: []

weed/glide.yaml

@@ -4,23 +4,36 @@ import:
   subpackages:
   - fs
 - package: github.com/boltdb/bolt
+  version: ^1.3.1
 - package: github.com/chrislusf/raft
-- package: github.com/dgrijalva/jwt-go
-- package: github.com/disintegration/imaging
-- package: github.com/gocql/gocql
-- package: github.com/golang/protobuf
-  subpackages:
-  - proto
-- package: github.com/gorilla/mux
-- package: github.com/hashicorp/golang-lru
-- package: github.com/klauspost/crc32
-- package: github.com/reducedb/encoding
+- package: github.com/dataence/encoding
   subpackages:
   - cursor
   - delta/bp32
+- package: github.com/dgrijalva/jwt-go
+  version: ^3.2.0
+- package: github.com/disintegration/imaging
+  version: ^1.4.1
+- package: github.com/go-redis/redis
+  version: ^6.10.2
+- package: github.com/go-sql-driver/mysql
+  version: ^1.3.0
+- package: github.com/gocql/gocql
+- package: github.com/golang/protobuf
+  version: ^1.0.0
+  subpackages:
+  - proto
+- package: github.com/google/btree
+- package: github.com/gorilla/mux
+  version: ^1.6.1
+- package: github.com/klauspost/crc32
+  version: ^1.1.0
+- package: github.com/lib/pq
 - package: github.com/rwcarlsen/goexif
   subpackages:
   - exif
+- package: github.com/soheilhy/cmux
+  version: ^0.1.4
 - package: github.com/syndtr/goleveldb
   subpackages:
   - leveldb
@@ -28,4 +41,8 @@ import:
 - package: golang.org/x/net
   subpackages:
   - context
-- package: gopkg.in/redis.v2
+- package: google.golang.org/grpc
+  version: ^1.11.3
+  subpackages:
+  - peer
+  - reflection

weed/images/preprocess.go

@@ -18,10 +18,10 @@ func MaybePreprocessImage(filename string, data []byte, width, height int) (resi
 	ext = strings.ToLower(ext)
 	switch ext {
 	case ".png", ".gif":
-		return Resized(ext, data, width, height)
+		return Resized(ext, data, width, height, "")
 	case ".jpg", ".jpeg":
 		data = FixJpgOrientation(data)
-		return Resized(ext, data, width, height)
+		return Resized(ext, data, width, height, "")
 	}
 	return data, 0, 0
 }

weed/images/resizing.go

@@ -11,7 +11,7 @@ import (
 	"github.com/disintegration/imaging"
 )
 
-func Resized(ext string, data []byte, width, height int) (resized []byte, w int, h int) {
+func Resized(ext string, data []byte, width, height int, mode string) (resized []byte, w int, h int) {
 	if width == 0 && height == 0 {
 		return data, 0, 0
 	}
@@ -20,11 +20,18 @@ func Resized(ext string, data []byte, width, height int) (resized []byte, w int,
 		bounds := srcImage.Bounds()
 		var dstImage *image.NRGBA
 		if bounds.Dx() > width && width != 0 || bounds.Dy() > height && height != 0 {
-			if width == height && bounds.Dx() != bounds.Dy() {
-				dstImage = imaging.Thumbnail(srcImage, width, height, imaging.Lanczos)
-				w, h = width, height
-			} else {
-				dstImage = imaging.Resize(srcImage, width, height, imaging.Lanczos)
+			switch mode {
+			case "fit":
+				dstImage = imaging.Fit(srcImage, width, height, imaging.Lanczos)
+			case "fill":
+				dstImage = imaging.Fill(srcImage, width, height, imaging.Center, imaging.Lanczos)
+			default:
+				if width == height && bounds.Dx() != bounds.Dy() {
+					dstImage = imaging.Thumbnail(srcImage, width, height, imaging.Lanczos)
+					w, h = width, height
+				} else {
+					dstImage = imaging.Resize(srcImage, width, height, imaging.Lanczos)
+				}
 			}
 		} else {
 			return data, bounds.Dx(), bounds.Dy()

weed/operation/filer/register.go

@@ -25,7 +25,7 @@ func RegisterFile(filer string, path string, fileId string, secret security.Secr
 	values.Add("fileId", fileId)
 	_, err := util.Post("http://"+filer+"/admin/register", values)
 	if err != nil {
-		return fmt.Errorf("Failed to register path:%s on filer:%s to file id:%s", path, filer, fileId)
+		return fmt.Errorf("Failed to register path %s on filer %s to file id %s : %v", path, filer, fileId, err)
 	}
 	return nil
 }

weed/operation/submit.go

@@ -104,7 +104,7 @@ func newFilePart(fullPathFilename string) (ret FilePart, err error) {
 	ret.FileSize = fi.Size()
 	ext := strings.ToLower(path.Ext(fullPathFilename))
 	ret.IsGzipped = ext == ".gz"
-	ret.FileName = fullPathFilename
+	ret.FileName = fi.Name()
 	if ext != "" {
 		ret.MimeType = mime.TypeByExtension(ext)
 	}

weed/pb/Makefile

@@ -3,4 +3,5 @@ all: gen
 .PHONY : gen
 
 gen:
-	protoc seaweed.proto --go_out=plugins=grpc:.
+	protoc seaweed.proto --go_out=plugins=grpc:./master_pb
+	protoc filer.proto --go_out=plugins=grpc:./filer_pb

weed/pb/filer.proto

@@ -0,0 +1,85 @@
+syntax = "proto3";
+
+package filer_pb;
+
+//////////////////////////////////////////////////
+
+service SeaweedFiler {
+
+    rpc LookupDirectoryEntry (LookupDirectoryEntryRequest) returns (LookupDirectoryEntryResponse) {
+    }
+
+    rpc ListEntries (ListEntriesRequest) returns (ListEntriesResponse) {
+    }
+
+    rpc GetFileAttributes (GetFileAttributesRequest) returns (GetFileAttributesResponse) {
+    }
+
+    rpc GetFileContent (GetFileContentRequest) returns (GetFileContentResponse) {
+    }
+
+    rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) {
+    }
+
+}
+
+//////////////////////////////////////////////////
+
+message LookupDirectoryEntryRequest {
+    string directory = 1;
+    string name = 2;
+}
+
+message LookupDirectoryEntryResponse {
+    Entry entry = 1;
+}
+
+message ListEntriesRequest {
+    string directory = 1;
+}
+
+message ListEntriesResponse {
+    repeated Entry entries = 1;
+}
+
+message Entry {
+    string name = 1;
+    bool is_directory = 2;
+    string file_id = 3;
+    FuseAttributes attributes = 4;
+}
+
+message FuseAttributes {
+    uint64 file_size = 1;
+    int64 mtime = 2;
+    uint32 file_mode = 3;
+    uint32 uid = 4;
+    uint32 gid = 5;
+}
+
+message GetFileAttributesRequest {
+    string name = 1;
+    string parent_dir = 2;
+    string file_id = 3;
+}
+
+message GetFileAttributesResponse {
+    FuseAttributes attributes = 1;
+}
+
+message GetFileContentRequest {
+    string file_id = 1;
+}
+
+message GetFileContentResponse {
+    bytes content = 1;
+}
+
+message DeleteEntryRequest {
+    string directory = 1;
+    string name = 2;
+    bool is_directory = 3;
+}
+
+message DeleteEntryResponse {
+}

weed/pb/filer_pb/filer.pb.go

@@ -0,0 +1,584 @@
+// Code generated by protoc-gen-go.
+// source: filer.proto
+// DO NOT EDIT!
+
+/*
+Package filer_pb is a generated protocol buffer package.
+
+It is generated from these files:
+	filer.proto
+
+It has these top-level messages:
+	LookupDirectoryEntryRequest
+	LookupDirectoryEntryResponse
+	ListEntriesRequest
+	ListEntriesResponse
+	Entry
+	FuseAttributes
+	GetFileAttributesRequest
+	GetFileAttributesResponse
+	GetFileContentRequest
+	GetFileContentResponse
+	DeleteEntryRequest
+	DeleteEntryResponse
+*/
+package filer_pb
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+import (
+	context "golang.org/x/net/context"
+	grpc "google.golang.org/grpc"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+
+type LookupDirectoryEntryRequest struct {
+	Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
+	Name      string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"`
+}
+
+func (m *LookupDirectoryEntryRequest) Reset()                    { *m = LookupDirectoryEntryRequest{} }
+func (m *LookupDirectoryEntryRequest) String() string            { return proto.CompactTextString(m) }
+func (*LookupDirectoryEntryRequest) ProtoMessage()               {}
+func (*LookupDirectoryEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
+
+func (m *LookupDirectoryEntryRequest) GetDirectory() string {
+	if m != nil {
+		return m.Directory
+	}
+	return ""
+}
+
+func (m *LookupDirectoryEntryRequest) GetName() string {
+	if m != nil {
+		return m.Name
+	}
+	return ""
+}
+
+type LookupDirectoryEntryResponse struct {
+	Entry *Entry `protobuf:"bytes,1,opt,name=entry" json:"entry,omitempty"`
+}
+
+func (m *LookupDirectoryEntryResponse) Reset()                    { *m = LookupDirectoryEntryResponse{} }
+func (m *LookupDirectoryEntryResponse) String() string            { return proto.CompactTextString(m) }
+func (*LookupDirectoryEntryResponse) ProtoMessage()               {}
+func (*LookupDirectoryEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
+
+func (m *LookupDirectoryEntryResponse) GetEntry() *Entry {
+	if m != nil {
+		return m.Entry
+	}
+	return nil
+}
+
+type ListEntriesRequest struct {
+	Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
+}
+
+func (m *ListEntriesRequest) Reset()                    { *m = ListEntriesRequest{} }
+func (m *ListEntriesRequest) String() string            { return proto.CompactTextString(m) }
+func (*ListEntriesRequest) ProtoMessage()               {}
+func (*ListEntriesRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
+
+func (m *ListEntriesRequest) GetDirectory() string {
+	if m != nil {
+		return m.Directory
+	}
+	return ""
+}
+
+type ListEntriesResponse struct {
+	Entries []*Entry `protobuf:"bytes,1,rep,name=entries" json:"entries,omitempty"`
+}
+
+func (m *ListEntriesResponse) Reset()                    { *m = ListEntriesResponse{} }
+func (m *ListEntriesResponse) String() string            { return proto.CompactTextString(m) }
+func (*ListEntriesResponse) ProtoMessage()               {}
+func (*ListEntriesResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
+
+func (m *ListEntriesResponse) GetEntries() []*Entry {
+	if m != nil {
+		return m.Entries
+	}
+	return nil
+}
+
+type Entry struct {
+	Name        string          `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
+	IsDirectory bool            `protobuf:"varint,2,opt,name=is_directory,json=isDirectory" json:"is_directory,omitempty"`
+	FileId      string          `protobuf:"bytes,3,opt,name=file_id,json=fileId" json:"file_id,omitempty"`
+	Attributes  *FuseAttributes `protobuf:"bytes,4,opt,name=attributes" json:"attributes,omitempty"`
+}
+
+func (m *Entry) Reset()                    { *m = Entry{} }
+func (m *Entry) String() string            { return proto.CompactTextString(m) }
+func (*Entry) ProtoMessage()               {}
+func (*Entry) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
+
+func (m *Entry) GetName() string {
+	if m != nil {
+		return m.Name
+	}
+	return ""
+}
+
+func (m *Entry) GetIsDirectory() bool {
+	if m != nil {
+		return m.IsDirectory
+	}
+	return false
+}
+
+func (m *Entry) GetFileId() string {
+	if m != nil {
+		return m.FileId
+	}
+	return ""
+}
+
+func (m *Entry) GetAttributes() *FuseAttributes {
+	if m != nil {
+		return m.Attributes
+	}
+	return nil
+}
+
+type FuseAttributes struct {
+	FileSize uint64 `protobuf:"varint,1,opt,name=file_size,json=fileSize" json:"file_size,omitempty"`
+	Mtime    int64  `protobuf:"varint,2,opt,name=mtime" json:"mtime,omitempty"`
+	FileMode uint32 `protobuf:"varint,3,opt,name=file_mode,json=fileMode" json:"file_mode,omitempty"`
+	Uid      uint32 `protobuf:"varint,4,opt,name=uid" json:"uid,omitempty"`
+	Gid      uint32 `protobuf:"varint,5,opt,name=gid" json:"gid,omitempty"`
+}
+
+func (m *FuseAttributes) Reset()                    { *m = FuseAttributes{} }
+func (m *FuseAttributes) String() string            { return proto.CompactTextString(m) }
+func (*FuseAttributes) ProtoMessage()               {}
+func (*FuseAttributes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
+
+func (m *FuseAttributes) GetFileSize() uint64 {
+	if m != nil {
+		return m.FileSize
+	}
+	return 0
+}
+
+func (m *FuseAttributes) GetMtime() int64 {
+	if m != nil {
+		return m.Mtime
+	}
+	return 0
+}
+
+func (m *FuseAttributes) GetFileMode() uint32 {
+	if m != nil {
+		return m.FileMode
+	}
+	return 0
+}
+
+func (m *FuseAttributes) GetUid() uint32 {
+	if m != nil {
+		return m.Uid
+	}
+	return 0
+}
+
+func (m *FuseAttributes) GetGid() uint32 {
+	if m != nil {
+		return m.Gid
+	}
+	return 0
+}
+
+type GetFileAttributesRequest struct {
+	Name      string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
+	ParentDir string `protobuf:"bytes,2,opt,name=parent_dir,json=parentDir" json:"parent_dir,omitempty"`
+	FileId    string `protobuf:"bytes,3,opt,name=file_id,json=fileId" json:"file_id,omitempty"`
+}
+
+func (m *GetFileAttributesRequest) Reset()                    { *m = GetFileAttributesRequest{} }
+func (m *GetFileAttributesRequest) String() string            { return proto.CompactTextString(m) }
+func (*GetFileAttributesRequest) ProtoMessage()               {}
+func (*GetFileAttributesRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
+
+func (m *GetFileAttributesRequest) GetName() string {
+	if m != nil {
+		return m.Name
+	}
+	return ""
+}
+
+func (m *GetFileAttributesRequest) GetParentDir() string {
+	if m != nil {
+		return m.ParentDir
+	}
+	return ""
+}
+
+func (m *GetFileAttributesRequest) GetFileId() string {
+	if m != nil {
+		return m.FileId
+	}
+	return ""
+}
+
+type GetFileAttributesResponse struct {
+	Attributes *FuseAttributes `protobuf:"bytes,1,opt,name=attributes" json:"attributes,omitempty"`
+}
+
+func (m *GetFileAttributesResponse) Reset()                    { *m = GetFileAttributesResponse{} }
+func (m *GetFileAttributesResponse) String() string            { return proto.CompactTextString(m) }
+func (*GetFileAttributesResponse) ProtoMessage()               {}
+func (*GetFileAttributesResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
+
+func (m *GetFileAttributesResponse) GetAttributes() *FuseAttributes {
+	if m != nil {
+		return m.Attributes
+	}
+	return nil
+}
+
+type GetFileContentRequest struct {
+	FileId string `protobuf:"bytes,1,opt,name=file_id,json=fileId" json:"file_id,omitempty"`
+}
+
+func (m *GetFileContentRequest) Reset()                    { *m = GetFileContentRequest{} }
+func (m *GetFileContentRequest) String() string            { return proto.CompactTextString(m) }
+func (*GetFileContentRequest) ProtoMessage()               {}
+func (*GetFileContentRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
+
+func (m *GetFileContentRequest) GetFileId() string {
+	if m != nil {
+		return m.FileId
+	}
+	return ""
+}
+
+type GetFileContentResponse struct {
+	Content []byte `protobuf:"bytes,1,opt,name=content,proto3" json:"content,omitempty"`
+}
+
+func (m *GetFileContentResponse) Reset()                    { *m = GetFileContentResponse{} }
+func (m *GetFileContentResponse) String() string            { return proto.CompactTextString(m) }
+func (*GetFileContentResponse) ProtoMessage()               {}
+func (*GetFileContentResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
+
+func (m *GetFileContentResponse) GetContent() []byte {
+	if m != nil {
+		return m.Content
+	}
+	return nil
+}
+
+type DeleteEntryRequest struct {
+	Directory   string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
+	Name        string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"`
+	IsDirectory bool   `protobuf:"varint,3,opt,name=is_directory,json=isDirectory" json:"is_directory,omitempty"`
+}
+
+func (m *DeleteEntryRequest) Reset()                    { *m = DeleteEntryRequest{} }
+func (m *DeleteEntryRequest) String() string            { return proto.CompactTextString(m) }
+func (*DeleteEntryRequest) ProtoMessage()               {}
+func (*DeleteEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
+
+func (m *DeleteEntryRequest) GetDirectory() string {
+	if m != nil {
+		return m.Directory
+	}
+	return ""
+}
+
+func (m *DeleteEntryRequest) GetName() string {
+	if m != nil {
+		return m.Name
+	}
+	return ""
+}
+
+func (m *DeleteEntryRequest) GetIsDirectory() bool {
+	if m != nil {
+		return m.IsDirectory
+	}
+	return false
+}
+
+type DeleteEntryResponse struct {
+}
+
+func (m *DeleteEntryResponse) Reset()                    { *m = DeleteEntryResponse{} }
+func (m *DeleteEntryResponse) String() string            { return proto.CompactTextString(m) }
+func (*DeleteEntryResponse) ProtoMessage()               {}
+func (*DeleteEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
+
+func init() {
+	proto.RegisterType((*LookupDirectoryEntryRequest)(nil), "filer_pb.LookupDirectoryEntryRequest")
+	proto.RegisterType((*LookupDirectoryEntryResponse)(nil), "filer_pb.LookupDirectoryEntryResponse")
+	proto.RegisterType((*ListEntriesRequest)(nil), "filer_pb.ListEntriesRequest")
+	proto.RegisterType((*ListEntriesResponse)(nil), "filer_pb.ListEntriesResponse")
+	proto.RegisterType((*Entry)(nil), "filer_pb.Entry")
+	proto.RegisterType((*FuseAttributes)(nil), "filer_pb.FuseAttributes")
+	proto.RegisterType((*GetFileAttributesRequest)(nil), "filer_pb.GetFileAttributesRequest")
+	proto.RegisterType((*GetFileAttributesResponse)(nil), "filer_pb.GetFileAttributesResponse")
+	proto.RegisterType((*GetFileContentRequest)(nil), "filer_pb.GetFileContentRequest")
+	proto.RegisterType((*GetFileContentResponse)(nil), "filer_pb.GetFileContentResponse")
+	proto.RegisterType((*DeleteEntryRequest)(nil), "filer_pb.DeleteEntryRequest")
+	proto.RegisterType((*DeleteEntryResponse)(nil), "filer_pb.DeleteEntryResponse")
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConn
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion4
+
+// Client API for SeaweedFiler service
+
+type SeaweedFilerClient interface {
+	LookupDirectoryEntry(ctx context.Context, in *LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*LookupDirectoryEntryResponse, error)
+	ListEntries(ctx context.Context, in *ListEntriesRequest, opts ...grpc.CallOption) (*ListEntriesResponse, error)
+	GetFileAttributes(ctx context.Context, in *GetFileAttributesRequest, opts ...grpc.CallOption) (*GetFileAttributesResponse, error)
+	GetFileContent(ctx context.Context, in *GetFileContentRequest, opts ...grpc.CallOption) (*GetFileContentResponse, error)
+	DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error)
+}
+
+type seaweedFilerClient struct {
+	cc *grpc.ClientConn
+}
+
+func NewSeaweedFilerClient(cc *grpc.ClientConn) SeaweedFilerClient {
+	return &seaweedFilerClient{cc}
+}
+
+func (c *seaweedFilerClient) LookupDirectoryEntry(ctx context.Context, in *LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*LookupDirectoryEntryResponse, error) {
+	out := new(LookupDirectoryEntryResponse)
+	err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/LookupDirectoryEntry", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedFilerClient) ListEntries(ctx context.Context, in *ListEntriesRequest, opts ...grpc.CallOption) (*ListEntriesResponse, error) {
+	out := new(ListEntriesResponse)
+	err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/ListEntries", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedFilerClient) GetFileAttributes(ctx context.Context, in *GetFileAttributesRequest, opts ...grpc.CallOption) (*GetFileAttributesResponse, error) {
+	out := new(GetFileAttributesResponse)
+	err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/GetFileAttributes", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedFilerClient) GetFileContent(ctx context.Context, in *GetFileContentRequest, opts ...grpc.CallOption) (*GetFileContentResponse, error) {
+	out := new(GetFileContentResponse)
+	err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/GetFileContent", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedFilerClient) DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error) {
+	out := new(DeleteEntryResponse)
+	err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/DeleteEntry", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// Server API for SeaweedFiler service
+
+type SeaweedFilerServer interface {
+	LookupDirectoryEntry(context.Context, *LookupDirectoryEntryRequest) (*LookupDirectoryEntryResponse, error)
+	ListEntries(context.Context, *ListEntriesRequest) (*ListEntriesResponse, error)
+	GetFileAttributes(context.Context, *GetFileAttributesRequest) (*GetFileAttributesResponse, error)
+	GetFileContent(context.Context, *GetFileContentRequest) (*GetFileContentResponse, error)
+	DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error)
+}
+
+func RegisterSeaweedFilerServer(s *grpc.Server, srv SeaweedFilerServer) {
+	s.RegisterService(&_SeaweedFiler_serviceDesc, srv)
+}
+
+func _SeaweedFiler_LookupDirectoryEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(LookupDirectoryEntryRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedFilerServer).LookupDirectoryEntry(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/filer_pb.SeaweedFiler/LookupDirectoryEntry",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedFilerServer).LookupDirectoryEntry(ctx, req.(*LookupDirectoryEntryRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedFiler_ListEntries_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ListEntriesRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedFilerServer).ListEntries(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/filer_pb.SeaweedFiler/ListEntries",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedFilerServer).ListEntries(ctx, req.(*ListEntriesRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedFiler_GetFileAttributes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetFileAttributesRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedFilerServer).GetFileAttributes(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/filer_pb.SeaweedFiler/GetFileAttributes",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedFilerServer).GetFileAttributes(ctx, req.(*GetFileAttributesRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedFiler_GetFileContent_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetFileContentRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedFilerServer).GetFileContent(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/filer_pb.SeaweedFiler/GetFileContent",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedFilerServer).GetFileContent(ctx, req.(*GetFileContentRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedFiler_DeleteEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(DeleteEntryRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedFilerServer).DeleteEntry(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/filer_pb.SeaweedFiler/DeleteEntry",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedFilerServer).DeleteEntry(ctx, req.(*DeleteEntryRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "filer_pb.SeaweedFiler",
+	HandlerType: (*SeaweedFilerServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "LookupDirectoryEntry",
+			Handler:    _SeaweedFiler_LookupDirectoryEntry_Handler,
+		},
+		{
+			MethodName: "ListEntries",
+			Handler:    _SeaweedFiler_ListEntries_Handler,
+		},
+		{
+			MethodName: "GetFileAttributes",
+			Handler:    _SeaweedFiler_GetFileAttributes_Handler,
+		},
+		{
+			MethodName: "GetFileContent",
+			Handler:    _SeaweedFiler_GetFileContent_Handler,
+		},
+		{
+			MethodName: "DeleteEntry",
+			Handler:    _SeaweedFiler_DeleteEntry_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "filer.proto",
+}
+
+func init() { proto.RegisterFile("filer.proto", fileDescriptor0) }
+
+var fileDescriptor0 = []byte{
+	// 532 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xa4, 0x54, 0x4d, 0x6f, 0xd3, 0x40,
+	0x10, 0xad, 0x71, 0xd2, 0x34, 0x93, 0xb4, 0xc0, 0xb4, 0x05, 0x93, 0xa6, 0x22, 0x2c, 0x2a, 0x82,
+	0x4b, 0x84, 0xc2, 0x85, 0x23, 0x88, 0xb4, 0x08, 0x29, 0x08, 0xc9, 0x55, 0xaf, 0x44, 0x49, 0x3d,
+	0x8d, 0x56, 0x24, 0x76, 0xf0, 0xae, 0x85, 0xda, 0x33, 0x7f, 0x80, 0xbf, 0xc5, 0xaf, 0x42, 0xbb,
+	0xeb, 0x8f, 0x35, 0x76, 0x0a, 0x88, 0x9b, 0xf7, 0xcd, 0xbe, 0x99, 0x37, 0x6f, 0x66, 0x0d, 0x9d,
+	0x2b, 0xbe, 0xa4, 0x78, 0xb8, 0x8e, 0x23, 0x19, 0xe1, 0x8e, 0x3e, 0x4c, 0xd7, 0x73, 0xf6, 0x09,
+	0x8e, 0x26, 0x51, 0xf4, 0x25, 0x59, 0x8f, 0x79, 0x4c, 0x97, 0x32, 0x8a, 0xaf, 0x4f, 0x43, 0x19,
+	0x5f, 0xfb, 0xf4, 0x35, 0x21, 0x21, 0xb1, 0x0f, 0xed, 0x20, 0x0b, 0x78, 0xce, 0xc0, 0x79, 0xde,
+	0xf6, 0x0b, 0x00, 0x11, 0x1a, 0xe1, 0x6c, 0x45, 0xde, 0x1d, 0x1d, 0xd0, 0xdf, 0xec, 0x14, 0xfa,
+	0xf5, 0x09, 0xc5, 0x3a, 0x0a, 0x05, 0xe1, 0x09, 0x34, 0x49, 0x01, 0x3a, 0x5b, 0x67, 0x74, 0x77,
+	0x98, 0x49, 0x19, 0x9a, 0x7b, 0x26, 0xca, 0x46, 0x80, 0x13, 0x2e, 0xa4, 0xc2, 0x38, 0x89, 0xbf,
+	0x92, 0xc3, 0xde, 0xc0, 0x7e, 0x89, 0x93, 0x56, 0x7c, 0x01, 0x2d, 0x32, 0x90, 0xe7, 0x0c, 0xdc,
+	0xba, 0x9a, 0x59, 0x9c, 0xfd, 0x70, 0xa0, 0xa9, 0xa1, 0xbc, 0x35, 0xa7, 0x68, 0x0d, 0x9f, 0x40,
+	0x97, 0x8b, 0x69, 0x21, 0x40, 0xb5, 0xbd, 0xe3, 0x77, 0xb8, 0xc8, 0x5b, 0xc5, 0x87, 0xd0, 0x52,
+	0xb9, 0xa7, 0x3c, 0xf0, 0x5c, 0xcd, 0xdc, 0x56, 0xc7, 0x0f, 0x01, 0xbe, 0x06, 0x98, 0x49, 0x19,
+	0xf3, 0x79, 0x22, 0x49, 0x78, 0x0d, 0xdd, 0xbb, 0x57, 0xe8, 0x38, 0x4b, 0x04, 0xbd, 0xcd, 0xe3,
+	0xbe, 0x75, 0x97, 0x7d, 0x77, 0x60, 0xaf, 0x1c, 0xc6, 0x23, 0x68, 0xeb, 0x2a, 0x82, 0xdf, 0x18,
+	0x85, 0x0d, 0x5f, 0x4f, 0xf4, 0x9c, 0xdf, 0x10, 0x1e, 0x40, 0x73, 0x25, 0x79, 0x3a, 0x15, 0xd7,
+	0x37, 0x87, 0x9c, 0xb2, 0x8a, 0x02, 0xd2, 0xd2, 0x76, 0x0d, 0xe5, 0x63, 0x14, 0x10, 0xde, 0x03,
+	0x37, 0xe1, 0x81, 0x56, 0xb5, 0xeb, 0xab, 0x4f, 0x85, 0x2c, 0x78, 0xe0, 0x35, 0x0d, 0xb2, 0xe0,
+	0x01, 0xbb, 0x02, 0xef, 0x3d, 0xc9, 0x33, 0xbe, 0xb4, 0x75, 0xa6, 0x63, 0xa9, 0x33, 0xeb, 0x18,
+	0x60, 0x3d, 0x8b, 0x29, 0x94, 0xca, 0xb0, 0x74, 0x43, 0xda, 0x06, 0x19, 0xf3, 0x78, 0xa3, 0x51,
+	0xec, 0x02, 0x1e, 0xd5, 0xd4, 0x49, 0x47, 0x59, 0x76, 0xd1, 0xf9, 0x07, 0x17, 0x5f, 0xc2, 0x61,
+	0x9a, 0xf6, 0x5d, 0x14, 0x4a, 0x0a, 0x65, 0xa6, 0xdd, 0x12, 0xe2, 0x94, 0x84, 0x8c, 0xe0, 0xc1,
+	0xef, 0x8c, 0x54, 0x85, 0x07, 0xad, 0x4b, 0x03, 0x69, 0x4a, 0xd7, 0xcf, 0x8e, 0x8c, 0x03, 0x8e,
+	0x69, 0x49, 0x92, 0xfe, 0xef, 0x11, 0x55, 0x36, 0xcd, 0xad, 0x6c, 0x1a, 0x3b, 0x84, 0xfd, 0x52,
+	0x29, 0xa3, 0x6d, 0xf4, 0xd3, 0x85, 0xee, 0x39, 0xcd, 0xbe, 0x11, 0x05, 0x4a, 0x7a, 0x8c, 0x0b,
+	0x38, 0xa8, 0x7b, 0x8f, 0x78, 0x52, 0xd8, 0x76, 0xcb, 0x0f, 0xa0, 0xf7, 0xec, 0x4f, 0xd7, 0x4c,
+	0x5d, 0xb6, 0x85, 0x13, 0xe8, 0x58, 0xaf, 0x0f, 0xfb, 0x16, 0xb1, 0xf2, 0x90, 0x7b, 0xc7, 0x1b,
+	0xa2, 0x79, 0xb6, 0xcf, 0x70, 0xbf, 0xb2, 0x06, 0xc8, 0x0a, 0xd6, 0xa6, 0x5d, 0xec, 0x3d, 0xbd,
+	0xf5, 0x4e, 0x9e, 0xff, 0x02, 0xf6, 0xca, 0xd3, 0xc5, 0xc7, 0x15, 0x62, 0x79, 0x53, 0x7a, 0x83,
+	0xcd, 0x17, 0x6c, 0x13, 0xac, 0xa9, 0xd8, 0x26, 0x54, 0xf7, 0xc2, 0x36, 0xa1, 0x66, 0x94, 0x6c,
+	0x6b, 0xbe, 0xad, 0xff, 0xd6, 0xaf, 0x7e, 0x05, 0x00, 0x00, 0xff, 0xff, 0x6a, 0x83, 0x8a, 0x32,
+	0xbc, 0x05, 0x00, 0x00,
+}

weed/pb/master_pb/seaweed.pb.go

@@ -3,7 +3,7 @@
 // DO NOT EDIT!
 
 /*
-Package pb is a generated protocol buffer package.
+Package master_pb is a generated protocol buffer package.
 
 It is generated from these files:
 	seaweed.proto
@@ -13,7 +13,7 @@ It has these top-level messages:
 	HeartbeatResponse
 	VolumeInformationMessage
 */
-package pb
+package master_pb
 
 import proto "github.com/golang/protobuf/proto"
 import fmt "fmt"
@@ -118,6 +118,7 @@ func (m *Heartbeat) GetVolumes() []*VolumeInformationMessage {
 type HeartbeatResponse struct {
 	VolumeSizeLimit uint64 `protobuf:"varint,1,opt,name=volumeSizeLimit" json:"volumeSizeLimit,omitempty"`
 	SecretKey       string `protobuf:"bytes,2,opt,name=secretKey" json:"secretKey,omitempty"`
+	Leader          string `protobuf:"bytes,3,opt,name=leader" json:"leader,omitempty"`
 }
 
 func (m *HeartbeatResponse) Reset()                    { *m = HeartbeatResponse{} }
@@ -139,6 +140,13 @@ func (m *HeartbeatResponse) GetSecretKey() string {
 	return ""
 }
 
+func (m *HeartbeatResponse) GetLeader() string {
+	if m != nil {
+		return m.Leader
+	}
+	return ""
+}
+
 type VolumeInformationMessage struct {
 	Id               uint32 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"`
 	Size             uint64 `protobuf:"varint,2,opt,name=size" json:"size,omitempty"`
@@ -228,9 +236,9 @@ func (m *VolumeInformationMessage) GetTtl() uint32 {
 }
 
 func init() {
-	proto.RegisterType((*Heartbeat)(nil), "pb.Heartbeat")
-	proto.RegisterType((*HeartbeatResponse)(nil), "pb.HeartbeatResponse")
-	proto.RegisterType((*VolumeInformationMessage)(nil), "pb.VolumeInformationMessage")
+	proto.RegisterType((*Heartbeat)(nil), "master_pb.Heartbeat")
+	proto.RegisterType((*HeartbeatResponse)(nil), "master_pb.HeartbeatResponse")
+	proto.RegisterType((*VolumeInformationMessage)(nil), "master_pb.VolumeInformationMessage")
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -256,7 +264,7 @@ func NewSeaweedClient(cc *grpc.ClientConn) SeaweedClient {
 }
 
 func (c *seaweedClient) SendHeartbeat(ctx context.Context, opts ...grpc.CallOption) (Seaweed_SendHeartbeatClient, error) {
-	stream, err := grpc.NewClientStream(ctx, &_Seaweed_serviceDesc.Streams[0], c.cc, "/pb.Seaweed/SendHeartbeat", opts...)
+	stream, err := grpc.NewClientStream(ctx, &_Seaweed_serviceDesc.Streams[0], c.cc, "/master_pb.Seaweed/SendHeartbeat", opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -323,7 +331,7 @@ func (x *seaweedSendHeartbeatServer) Recv() (*Heartbeat, error) {
 }
 
 var _Seaweed_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "pb.Seaweed",
+	ServiceName: "master_pb.Seaweed",
 	HandlerType: (*SeaweedServer)(nil),
 	Methods:     []grpc.MethodDesc{},
 	Streams: []grpc.StreamDesc{
@@ -340,36 +348,37 @@ var _Seaweed_serviceDesc = grpc.ServiceDesc{
 func init() { proto.RegisterFile("seaweed.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 489 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x74, 0x93, 0xdf, 0x8a, 0xd3, 0x40,
-	0x14, 0xc6, 0x4d, 0x1a, 0xdb, 0xe6, 0x74, 0xbb, 0x76, 0x07, 0x84, 0x41, 0x57, 0x8d, 0xbd, 0x0a,
-	0x28, 0x45, 0x56, 0xf0, 0xc6, 0x3b, 0x17, 0x16, 0x65, 0x15, 0x97, 0x29, 0x7a, 0xe3, 0x45, 0x98,
-	0x24, 0x67, 0x65, 0xd8, 0xc9, 0x1f, 0x26, 0xd3, 0xb5, 0xd9, 0x07, 0xf2, 0x49, 0x7c, 0x30, 0x99,
-	0x33, 0x4d, 0xab, 0x82, 0x77, 0xe7, 0xfc, 0xce, 0x97, 0xe4, 0x9b, 0xf3, 0x4d, 0x60, 0xde, 0xa1,
-	0xfc, 0x81, 0x58, 0xae, 0x5a, 0xd3, 0xd8, 0x86, 0x85, 0x6d, 0xbe, 0xfc, 0x19, 0x42, 0xfc, 0x1e,
-	0xa5, 0xb1, 0x39, 0x4a, 0xcb, 0x8e, 0x21, 0x54, 0x2d, 0x0f, 0x92, 0x20, 0x8d, 0x45, 0xa8, 0x5a,
-	0xc6, 0x20, 0x6a, 0x1b, 0x63, 0x79, 0x98, 0x04, 0xe9, 0x5c, 0x50, 0xcd, 0x9e, 0x00, 0xb4, 0x9b,
-	0x5c, 0xab, 0x22, 0xdb, 0x18, 0xcd, 0x47, 0xa4, 0x8d, 0x3d, 0xf9, 0x62, 0x34, 0x4b, 0x61, 0x51,
-	0xc9, 0x6d, 0x76, 0xdb, 0xe8, 0x4d, 0x85, 0x59, 0xd1, 0x6c, 0x6a, 0xcb, 0x23, 0x7a, 0xfc, 0xb8,
-	0x92, 0xdb, 0xaf, 0x84, 0xcf, 0x1d, 0x65, 0x09, 0x1c, 0x39, 0xe5, 0xb5, 0xd2, 0x98, 0xdd, 0x60,
-	0xcf, 0xef, 0x27, 0x41, 0x1a, 0x09, 0xa8, 0xe4, 0xf6, 0x42, 0x69, 0xbc, 0xc4, 0x9e, 0x3d, 0x83,
-	0x59, 0x29, 0xad, 0xcc, 0x0a, 0xac, 0x2d, 0x1a, 0x3e, 0xa6, 0x6f, 0x81, 0x43, 0xe7, 0x44, 0x9c,
-	0x3f, 0x23, 0x8b, 0x1b, 0x3e, 0xa1, 0x09, 0xd5, 0xce, 0x9f, 0x2c, 0x2b, 0x55, 0x67, 0xe4, 0x7c,
-	0x4a, 0x9f, 0x8e, 0x89, 0x5c, 0x39, 0xfb, 0x6f, 0x60, 0xe2, 0xbd, 0x75, 0x3c, 0x4e, 0x46, 0xe9,
-	0xec, 0xec, 0x74, 0xd5, 0xe6, 0x2b, 0xef, 0xeb, 0x43, 0x7d, 0xdd, 0x98, 0x4a, 0x5a, 0xd5, 0xd4,
-	0x9f, 0xb0, 0xeb, 0xe4, 0x77, 0x14, 0x83, 0x78, 0xf9, 0x0d, 0x4e, 0xf6, 0x7b, 0x12, 0xd8, 0xb5,
-	0x4d, 0xdd, 0x21, 0x4b, 0xe1, 0x81, 0x9f, 0xaf, 0xd5, 0x1d, 0x7e, 0x54, 0x95, 0xb2, 0xb4, 0xbc,
-	0x48, 0xfc, 0x8b, 0xd9, 0x29, 0xc4, 0x1d, 0x16, 0x06, 0xed, 0x25, 0xf6, 0xb4, 0xce, 0x58, 0x1c,
-	0xc0, 0xf2, 0x57, 0x08, 0xfc, 0x7f, 0x16, 0x28, 0x94, 0x92, 0xde, 0x3b, 0x17, 0xa1, 0x2a, 0xdd,
-	0xa1, 0x3b, 0x75, 0x87, 0xf4, 0x96, 0x48, 0x50, 0xcd, 0x9e, 0x02, 0x14, 0x8d, 0xd6, 0x58, 0xb8,
-	0x07, 0x77, 0xa1, 0xfc, 0x41, 0xdc, 0x52, 0x68, 0xcf, 0x87, 0x3c, 0x22, 0x11, 0x3b, 0xe2, 0xa3,
-	0x78, 0x0e, 0x47, 0x25, 0x6a, 0xb4, 0x83, 0xc0, 0x47, 0x31, 0xf3, 0xcc, 0x4b, 0x5e, 0x02, 0xf3,
-	0x6d, 0x99, 0xe5, 0xfd, 0x5e, 0x38, 0x26, 0xe1, 0x62, 0x37, 0x79, 0xd7, 0x0f, 0xea, 0xc7, 0x10,
-	0x1b, 0x94, 0x65, 0xd6, 0xd4, 0xba, 0xa7, 0x74, 0xa6, 0x62, 0xea, 0xc0, 0xe7, 0x5a, 0xf7, 0xec,
-	0x05, 0x9c, 0x18, 0x6c, 0xb5, 0x2a, 0x64, 0xd6, 0x6a, 0x59, 0x60, 0x85, 0xf5, 0x10, 0xd4, 0x62,
-	0x37, 0xb8, 0x1a, 0x38, 0xe3, 0x30, 0xb9, 0x45, 0xd3, 0xb9, 0x63, 0xc5, 0x24, 0x19, 0x5a, 0xb6,
-	0x80, 0x91, 0xb5, 0x9a, 0x03, 0x51, 0x57, 0x9e, 0x5d, 0xc0, 0x64, 0xed, 0x6f, 0x38, 0x7b, 0x0b,
-	0xf3, 0x35, 0xd6, 0xe5, 0xe1, 0x6a, 0xcf, 0x5d, 0xcc, 0xfb, 0xf6, 0xd1, 0xc3, 0xbf, 0xda, 0x21,
-	0xd0, 0xe5, 0xbd, 0x34, 0x78, 0x15, 0xe4, 0x63, 0xfa, 0x3f, 0x5e, 0xff, 0x0e, 0x00, 0x00, 0xff,
-	0xff, 0xd5, 0x08, 0xa6, 0xf2, 0x30, 0x03, 0x00, 0x00,
+	// 504 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x74, 0x93, 0xdf, 0x6e, 0xd3, 0x30,
+	0x14, 0xc6, 0x49, 0x16, 0xda, 0xfa, 0x74, 0x1d, 0x9d, 0x85, 0x90, 0x05, 0x03, 0x4a, 0xb9, 0x89,
+	0x04, 0xaa, 0xd0, 0xb8, 0xe6, 0x86, 0x49, 0x88, 0x69, 0x20, 0x26, 0x17, 0xb8, 0x8d, 0xdc, 0xe4,
+	0x0c, 0x59, 0x73, 0xfe, 0xc8, 0x76, 0x47, 0xb3, 0x77, 0xe2, 0x2d, 0x78, 0x30, 0xe4, 0x93, 0xa6,
+	0x9d, 0x10, 0xdc, 0x1d, 0xff, 0xce, 0xe7, 0xf8, 0xe4, 0xfb, 0x6c, 0x98, 0x38, 0x54, 0x3f, 0x11,
+	0x8b, 0x45, 0x63, 0x6b, 0x5f, 0x73, 0x56, 0x2a, 0xe7, 0xd1, 0x66, 0xcd, 0x6a, 0xfe, 0x2b, 0x06,
+	0xf6, 0x11, 0x95, 0xf5, 0x2b, 0x54, 0x9e, 0x1f, 0x41, 0xac, 0x1b, 0x11, 0xcd, 0xa2, 0x94, 0xc9,
+	0x58, 0x37, 0x9c, 0x43, 0xd2, 0xd4, 0xd6, 0x8b, 0x78, 0x16, 0xa5, 0x13, 0x49, 0x35, 0x7f, 0x0a,
+	0xd0, 0xac, 0x57, 0x46, 0xe7, 0xd9, 0xda, 0x1a, 0x71, 0x40, 0x5a, 0xd6, 0x91, 0x6f, 0xd6, 0xf0,
+	0x14, 0xa6, 0xa5, 0xda, 0x64, 0x37, 0xb5, 0x59, 0x97, 0x98, 0xe5, 0xf5, 0xba, 0xf2, 0x22, 0xa1,
+	0xed, 0x47, 0xa5, 0xda, 0x7c, 0x27, 0x7c, 0x16, 0x28, 0x9f, 0xc1, 0x61, 0x50, 0x5e, 0x69, 0x83,
+	0xd9, 0x35, 0xb6, 0xe2, 0xfe, 0x2c, 0x4a, 0x13, 0x09, 0xa5, 0xda, 0x7c, 0xd0, 0x06, 0x2f, 0xb0,
+	0xe5, 0xcf, 0x61, 0x5c, 0x28, 0xaf, 0xb2, 0x1c, 0x2b, 0x8f, 0x56, 0x0c, 0xe8, 0x2c, 0x08, 0xe8,
+	0x8c, 0x48, 0x98, 0xcf, 0xaa, 0xfc, 0x5a, 0x0c, 0xa9, 0x43, 0x75, 0x98, 0x4f, 0x15, 0xa5, 0xae,
+	0x32, 0x9a, 0x7c, 0x44, 0x47, 0x33, 0x22, 0x97, 0x61, 0xfc, 0x77, 0x30, 0xec, 0x66, 0x73, 0x82,
+	0xcd, 0x0e, 0xd2, 0xf1, 0xe9, 0xcb, 0xc5, 0xce, 0x8d, 0x45, 0x37, 0xde, 0x79, 0x75, 0x55, 0xdb,
+	0x52, 0x79, 0x5d, 0x57, 0x9f, 0xd1, 0x39, 0xf5, 0x03, 0x65, 0xbf, 0x67, 0xee, 0xe0, 0x78, 0x67,
+	0x97, 0x44, 0xd7, 0xd4, 0x95, 0x43, 0x9e, 0xc2, 0x83, 0xae, 0xbf, 0xd4, 0xb7, 0xf8, 0x49, 0x97,
+	0xda, 0x93, 0x87, 0x89, 0xfc, 0x1b, 0xf3, 0x13, 0x60, 0x0e, 0x73, 0x8b, 0xfe, 0x02, 0x5b, 0x72,
+	0x95, 0xc9, 0x3d, 0xe0, 0x8f, 0x60, 0x60, 0x50, 0x15, 0x68, 0xb7, 0xb6, 0x6e, 0x57, 0xf3, 0xdf,
+	0x31, 0x88, 0xff, 0x8d, 0x46, 0x99, 0x15, 0x74, 0xde, 0x44, 0xc6, 0xba, 0x08, 0x9e, 0x38, 0x7d,
+	0x8b, 0xf4, 0xf5, 0x44, 0x52, 0xcd, 0x9f, 0x01, 0xe4, 0xb5, 0x31, 0x98, 0x87, 0x8d, 0xdb, 0x8f,
+	0xdf, 0x21, 0xc1, 0x33, 0x8a, 0x61, 0x1f, 0x57, 0x22, 0x59, 0x20, 0x5d, 0x52, 0x2f, 0xe0, 0xb0,
+	0x40, 0x83, 0xbe, 0x17, 0x74, 0x49, 0x8d, 0x3b, 0xd6, 0x49, 0x5e, 0x03, 0xef, 0x96, 0x45, 0xb6,
+	0x6a, 0x77, 0xc2, 0x01, 0x09, 0xa7, 0xdb, 0xce, 0xfb, 0xb6, 0x57, 0x3f, 0x01, 0x66, 0x51, 0x15,
+	0x59, 0x5d, 0x99, 0x96, 0xc2, 0x1b, 0xc9, 0x51, 0x00, 0x5f, 0x2a, 0xd3, 0xf2, 0x57, 0x70, 0x6c,
+	0xb1, 0x31, 0x3a, 0x57, 0x59, 0x63, 0x54, 0x8e, 0x25, 0x56, 0x7d, 0x8e, 0xd3, 0x6d, 0xe3, 0xb2,
+	0xe7, 0x5c, 0xc0, 0xf0, 0x06, 0xad, 0x0b, 0xbf, 0xc5, 0x48, 0xd2, 0x2f, 0xf9, 0x14, 0x0e, 0xbc,
+	0x37, 0x02, 0x88, 0x86, 0xf2, 0xf4, 0x2b, 0x0c, 0x97, 0xdd, 0x3b, 0xe0, 0xe7, 0x30, 0x59, 0x62,
+	0x55, 0xec, 0x6f, 0xfe, 0xc3, 0x3b, 0xb7, 0x60, 0x47, 0x1f, 0x9f, 0xfc, 0x8b, 0xf6, 0xb1, 0xcf,
+	0xef, 0xa5, 0xd1, 0x9b, 0x68, 0x35, 0xa0, 0x37, 0xf5, 0xf6, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff,
+	0x01, 0x14, 0xbb, 0x3a, 0x64, 0x03, 0x00, 0x00,
 }

weed/pb/seaweed.proto

@@ -1,6 +1,6 @@
 syntax = "proto3";
 
-package pb;
+package master_pb;
 
 //////////////////////////////////////////////////
 
@@ -24,6 +24,7 @@ message Heartbeat {
 message HeartbeatResponse {
   uint64 volumeSizeLimit = 1;
   string secretKey = 2;
+  string leader = 3;
 }
 
 message VolumeInformationMessage {

weed/server/common.go

@@ -170,6 +170,11 @@ func parseURLPath(path string) (vid, fid, filename, ext string, isVolumeIdOnly b
 	return
 }
 
+func statsHealthHandler(w http.ResponseWriter, r *http.Request) {
+	m := make(map[string]interface{})
+	m["Version"] = util.VERSION
+	writeJsonQuiet(w, r, http.StatusOK, m)
+}
 func statsCounterHandler(w http.ResponseWriter, r *http.Request) {
 	m := make(map[string]interface{})
 	m["Version"] = util.VERSION

weed/server/filer_grpc_server.go

@@ -0,0 +1,109 @@
+package weed_server
+
+import (
+	"context"
+	"strconv"
+
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"fmt"
+)
+
+func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
+
+	found, fileId, err := fs.filer.LookupDirectoryEntry(req.Directory, req.Name)
+	if err != nil {
+		return nil, err
+	}
+	if !found {
+		return nil, fmt.Errorf("%s not found under %s", req.Name, req.Directory)
+	}
+
+	return &filer_pb.LookupDirectoryEntryResponse{
+		Entry: &filer_pb.Entry{
+			Name:        req.Name,
+			IsDirectory: fileId == "",
+			FileId:      fileId,
+		},
+	}, nil
+}
+
+func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntriesRequest) (*filer_pb.ListEntriesResponse, error) {
+
+	directoryNames, err := fs.filer.ListDirectories(req.Directory)
+	if err != nil {
+		return nil, err
+	}
+	files, err := fs.filer.ListFiles(req.Directory, "", 1000)
+	if err != nil {
+		return nil, err
+	}
+
+	resp := &filer_pb.ListEntriesResponse{}
+	for _, dir := range directoryNames {
+		resp.Entries = append(resp.Entries, &filer_pb.Entry{
+			Name:        string(dir),
+			IsDirectory: true,
+		})
+	}
+	for _, fileEntry := range files {
+		resp.Entries = append(resp.Entries, &filer_pb.Entry{
+			Name:        fileEntry.Name,
+			IsDirectory: false,
+			FileId:      string(fileEntry.Id),
+		})
+	}
+
+	return resp, nil
+}
+
+func (fs *FilerServer) GetFileAttributes(ctx context.Context, req *filer_pb.GetFileAttributesRequest) (*filer_pb.GetFileAttributesResponse, error) {
+
+	attributes := &filer_pb.FuseAttributes{}
+
+	server, err := operation.LookupFileId(fs.getMasterNode(), req.FileId)
+	if err != nil {
+		return nil, err
+	}
+	head, err := util.Head(server)
+	if err != nil {
+		return nil, err
+	}
+	attributes.FileSize, err = strconv.ParseUint(head.Get("Content-Length"), 10, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	return &filer_pb.GetFileAttributesResponse{
+		Attributes: attributes,
+	}, nil
+}
+
+func (fs *FilerServer) GetFileContent(ctx context.Context, req *filer_pb.GetFileContentRequest) (*filer_pb.GetFileContentResponse, error) {
+
+	server, err := operation.LookupFileId(fs.getMasterNode(), req.FileId)
+	if err != nil {
+		return nil, err
+	}
+	content, err := util.Get(server)
+	if err != nil {
+		return nil, err
+	}
+
+	return &filer_pb.GetFileContentResponse{
+		Content: content,
+	}, nil
+}
+
+func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest) (resp *filer_pb.DeleteEntryResponse, err error) {
+	if req.IsDirectory {
+		err = fs.filer.DeleteDirectory(req.Directory+req.Name, false)
+	} else {
+		fid, err := fs.filer.DeleteFile(req.Directory + req.Name)
+		if err == nil && fid != "" {
+			err = operation.DeleteFile(fs.getMasterNode(), fid, fs.jwt(fid))
+		}
+	}
+	return nil, err
+}

weed/server/filer_server.go

@@ -57,7 +57,7 @@ type FilerServer struct {
 	masterNodes        *storage.MasterNodes
 }
 
-func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir string, collection string,
+func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, ip string, port int, master string, dir string, collection string,
 	replication string, redirectOnRead bool, disableDirListing bool,
 	confFile string,
 	maxMB int,
@@ -105,11 +105,14 @@ func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir st
 			return
 		}
 
-		r.HandleFunc("/admin/mv", fs.moveHandler)
-		r.HandleFunc("/admin/register", fs.registerHandler)
+		defaultMux.HandleFunc("/admin/mv", fs.moveHandler)
 	}
 
-	r.HandleFunc("/", fs.filerHandler)
+	defaultMux.HandleFunc("/admin/register", fs.registerHandler)
+	defaultMux.HandleFunc("/", fs.filerHandler)
+	if defaultMux != readonlyMux {
+		readonlyMux.HandleFunc("/", fs.readonlyFilerHandler)
+	}
 
 	go func() {
 		connected := true
@@ -181,7 +184,7 @@ func (fs *FilerServer) detectHealthyMaster(masterNode string) (master string, e
 }
 
 func checkMaster(masterNode string) error {
-	statUrl := "http://" + masterNode + "/stats"
+	statUrl := "http://" + masterNode + "/stats/health"
 	glog.V(4).Infof("Connecting to %s ...", statUrl)
 	_, e := util.Get(statUrl)
 	return e

weed/server/filer_server_handlers.go

@@ -18,3 +18,12 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
 		fs.PostHandler(w, r)
 	}
 }
+
+func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Request) {
+	switch r.Method {
+	case "GET":
+		fs.GetOrHeadHandler(w, r, true)
+	case "HEAD":
+		fs.GetOrHeadHandler(w, r, false)
+	}
+}

weed/server/filer_server_handlers_write.go

@@ -532,7 +532,7 @@ func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *ht
 }
 
 // curl -X DELETE http://localhost:8888/path/to
-// curl -X DELETE http://localhost:8888/path/to?recursive=true
+// curl -X DELETE http://localhost:8888/path/to/?recursive=true
 func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
 	var err error
 	var fid string

weed/server/filer_ui/templates.go

@@ -28,23 +28,26 @@ var StatusTpl = template.Must(template.New("status").Parse(`<!DOCTYPE html>
 				{{$path := .Path }}
 				{{ range $dirs_index, $dir := .Directories }}
 				<li>
-					<a href=		{{ print $path  $dir.Name  "/"}} >
-						{{ $dir.Name }}
+					<img src="https://www.w3.org/TR/WWWicn/folder.gif" width="20" height="23">
+					<a href={{ print $path  $dir  "/"}} >
+						{{ $dir }}
 					</a>
 				</li>
 				{{ end }}
 
 				{{ range $file_index, $file := .Files }}
 				<li>
+					<a href={{ print $path  $file.Name}} >
 					{{ $file.Name }}
+					</a>
 				</li>
 				{{ end }}
 			</ul>
 		</div>
 
-{{if .ShouldDisplayLoadMore}}
+		{{if .ShouldDisplayLoadMore}}
 		<div class="row">
-		<a href=		{{ print .Path "?limit=" .Limit	"&lastFileName=" .LastFileName}} >
+		<a href={{ print .Path "?limit=" .Limit	"&lastFileName=" .LastFileName}} >
 		Load more
 		</a>
 		</div>

weed/server/master_grpc_server.go

@@ -5,13 +5,13 @@ import (
 	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/topology"
 	"google.golang.org/grpc/peer"
 )
 
-func (ms MasterServer) SendHeartbeat(stream pb.Seaweed_SendHeartbeatServer) error {
+func (ms MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServer) error {
 	var dn *topology.DataNode
 	t := ms.Topo
 	for {
@@ -34,7 +34,7 @@ func (ms MasterServer) SendHeartbeat(stream pb.Seaweed_SendHeartbeatServer) erro
 					int(heartbeat.Port), heartbeat.PublicUrl,
 					int(heartbeat.MaxVolumeCount))
 				glog.V(0).Infof("added volume server %v:%d", heartbeat.GetIp(), heartbeat.GetPort())
-				if err := stream.Send(&pb.HeartbeatResponse{
+				if err := stream.Send(&master_pb.HeartbeatResponse{
 					VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024,
 					SecretKey:       string(ms.guard.SecretKey),
 				}); err != nil {
@@ -59,11 +59,21 @@ func (ms MasterServer) SendHeartbeat(stream pb.Seaweed_SendHeartbeatServer) erro
 			}
 
 		} else {
-			glog.V(0).Infof("lost volume server %s:%d", dn.Ip, dn.Port)
 			if dn != nil {
+				glog.V(0).Infof("lost volume server %s:%d", dn.Ip, dn.Port)
 				t.UnRegisterDataNode(dn)
 			}
 			return err
 		}
+
+		// tell the volume servers about the leader
+		newLeader, err := t.Leader()
+		if err == nil {
+			if err := stream.Send(&master_pb.HeartbeatResponse{
+				Leader: newLeader,
+			}); err != nil {
+				return err
+			}
+		}
 	}
 }

weed/server/master_server.go

@@ -37,7 +37,6 @@ func NewMasterServer(r *mux.Router, port int, metaFolder string,
 	volumeSizeLimitMB uint,
 	preallocate bool,
 	pulseSeconds int,
-	confFile string,
 	defaultReplicaPlacement string,
 	garbageThreshold string,
 	whiteList []string,
@@ -58,11 +57,7 @@ func NewMasterServer(r *mux.Router, port int, metaFolder string,
 	}
 	ms.bounedLeaderChan = make(chan int, 16)
 	seq := sequence.NewMemorySequencer()
-	var e error
-	if ms.Topo, e = topology.NewTopology("topo", confFile, seq,
-		uint64(volumeSizeLimitMB)*1024*1024, pulseSeconds); e != nil {
-		glog.Fatalf("cannot create topology:%s", e)
-	}
+	ms.Topo = topology.NewTopology("topo", seq, uint64(volumeSizeLimitMB)*1024*1024, pulseSeconds)
 	ms.vg = topology.NewDefaultVolumeGrowth()
 	glog.V(0).Infoln("Volume Size Limit is", volumeSizeLimitMB, "MB")
 
@@ -80,11 +75,12 @@ func NewMasterServer(r *mux.Router, port int, metaFolder string,
 	r.HandleFunc("/vol/vacuum", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeVacuumHandler)))
 	r.HandleFunc("/submit", ms.guard.WhiteList(ms.submitFromMasterServerHandler))
 	r.HandleFunc("/delete", ms.guard.WhiteList(ms.deleteFromMasterServerHandler))
-	r.HandleFunc("/{fileId}", ms.proxyToLeader(ms.redirectHandler))
+	r.HandleFunc("/stats/health", ms.guard.WhiteList(statsHealthHandler))
 	r.HandleFunc("/stats/counter", ms.guard.WhiteList(statsCounterHandler))
 	r.HandleFunc("/stats/memory", ms.guard.WhiteList(statsMemoryHandler))
+	r.HandleFunc("/{fileId}", ms.proxyToLeader(ms.redirectHandler))
 
-	ms.Topo.StartRefreshWritableVolumes(garbageThreshold)
+	ms.Topo.StartRefreshWritableVolumes(garbageThreshold, ms.preallocate)
 
 	return ms
 }

weed/server/master_server_handlers.go

@@ -31,10 +31,10 @@ func (ms *MasterServer) lookupVolumeId(vids []string, collection string) (volume
 				}
 				volumeLocations[vid] = operation.LookupResult{VolumeId: vid, Locations: ret}
 			} else {
-				volumeLocations[vid] = operation.LookupResult{VolumeId: vid, Error: "volumeId not found."}
+				volumeLocations[vid] = operation.LookupResult{VolumeId: vid, Error: fmt.Sprintf("volumeId %s not found.", vid)}
 			}
 		} else {
-			volumeLocations[vid] = operation.LookupResult{VolumeId: vid, Error: "Unknown volumeId format."}
+			volumeLocations[vid] = operation.LookupResult{VolumeId: vid, Error: fmt.Sprintf("Unknown volumeId format: %s", vid)}
 		}
 	}
 	return

weed/server/master_server_handlers_admin.go

@@ -42,7 +42,7 @@ func (ms *MasterServer) volumeVacuumHandler(w http.ResponseWriter, r *http.Reque
 		gcThreshold = ms.garbageThreshold
 	}
 	glog.Infoln("garbageThreshold =", gcThreshold)
-	ms.Topo.Vacuum(gcThreshold)
+	ms.Topo.Vacuum(gcThreshold, ms.preallocate)
 	ms.dirStatusHandler(w, r)
 }
 

weed/server/master_server_handlers_ui.go

@@ -3,26 +3,25 @@ package weed_server
 import (
 	"net/http"
 
+	"github.com/chrislusf/raft"
+	ui "github.com/chrislusf/seaweedfs/weed/server/master_ui"
 	"github.com/chrislusf/seaweedfs/weed/stats"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	ui "github.com/chrislusf/seaweedfs/weed/server/master_ui"
 )
 
 func (ms *MasterServer) uiStatusHandler(w http.ResponseWriter, r *http.Request) {
 	infos := make(map[string]interface{})
 	infos["Version"] = util.VERSION
 	args := struct {
-		Version  string
-		Topology interface{}
-		Leader   string
-		Peers    interface{}
-		Stats    map[string]interface{}
-		Counters *stats.ServerStats
+		Version    string
+		Topology   interface{}
+		RaftServer raft.Server
+		Stats      map[string]interface{}
+		Counters   *stats.ServerStats
 	}{
 		util.VERSION,
 		ms.Topo.ToMap(),
-		ms.Topo.RaftServer.Leader(),
-		ms.Topo.RaftServer.Peers(),
+		ms.Topo.RaftServer,
 		infos,
 		serverStats,
 	}

weed/server/master_ui/templates.go

@@ -33,6 +33,7 @@ var StatusTpl = template.Must(template.New("status").Parse(`<!DOCTYPE html>
                 <th>Max</th>
                 <td>{{ .Topology.Max }}</td>
               </tr>
+              {{ with .RaftServer }}
               <tr>
                 <th>Leader</th>
                 <td><a href="http://{{ .Leader }}">{{ .Leader }}</a></td>
@@ -45,6 +46,7 @@ var StatusTpl = template.Must(template.New("status").Parse(`<!DOCTYPE html>
                 {{ end }}
                 </ul></td>
               </tr>
+              {{ end }}
             </tbody>
           </table>
         </div>

weed/server/raft_server.go

@@ -65,8 +65,8 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin
 		return nil
 	}
 	transporter.Install(s.raftServer, s)
-	s.raftServer.SetHeartbeatInterval(1 * time.Second)
-	s.raftServer.SetElectionTimeout(time.Duration(pulseSeconds) * 3450 * time.Millisecond)
+	s.raftServer.SetHeartbeatInterval(500 * time.Millisecond)
+	s.raftServer.SetElectionTimeout(time.Duration(pulseSeconds) * 500 * time.Millisecond)
 	s.raftServer.Start()
 
 	s.router.HandleFunc("/cluster/join", s.joinHandler).Methods("POST")
@@ -141,6 +141,10 @@ func isPeersChanged(dir string, self string, peers []string) (oldPeers []string,
 	}
 	oldPeers = append(oldPeers, self)
 
+	if len(peers) == 0 && len(oldPeers) <= 1 {
+		return oldPeers, false
+	}
+
 	sort.Strings(peers)
 	sort.Strings(oldPeers)
 
@@ -189,14 +193,12 @@ func postFollowingOneRedirect(target string, contentType string, b bytes.Buffer)
 		return err
 	}
 	defer resp.Body.Close()
-	reply, _ := ioutil.ReadAll(resp.Body)
 	statusCode := resp.StatusCode
+	data, _ := ioutil.ReadAll(resp.Body)
+	reply := string(data)
 
-	if statusCode == http.StatusMovedPermanently {
-		var urlStr string
-		if urlStr = resp.Header.Get("Location"); urlStr == "" {
-			return fmt.Errorf("%d response missing Location header", resp.StatusCode)
-		}
+	if strings.HasPrefix(reply, "\"http") {
+		urlStr := reply[1: len(reply)-1]
 
 		glog.V(0).Infoln("Post redirected to ", urlStr)
 		resp2, err2 := http.Post(urlStr, contentType, backupReader)
@@ -204,13 +206,13 @@ func postFollowingOneRedirect(target string, contentType string, b bytes.Buffer)
 			return err2
 		}
 		defer resp2.Body.Close()
-		reply, _ = ioutil.ReadAll(resp2.Body)
+		data, _ = ioutil.ReadAll(resp2.Body)
 		statusCode = resp2.StatusCode
 	}
 
-	glog.V(0).Infoln("Post returned status: ", statusCode, string(reply))
+	glog.V(0).Infoln("Post returned status: ", statusCode, string(data))
 	if statusCode != http.StatusOK {
-		return errors.New(string(reply))
+		return errors.New(string(data))
 	}
 
 	return nil

weed/server/raft_server_handlers.go

@@ -44,8 +44,10 @@ func (s *RaftServer) HandleFunc(pattern string, handler func(http.ResponseWriter
 func (s *RaftServer) redirectToLeader(w http.ResponseWriter, req *http.Request) {
 	if leader, e := s.topo.Leader(); e == nil {
 		//http.StatusMovedPermanently does not cause http POST following redirection
-		glog.V(0).Infoln("Redirecting to", http.StatusMovedPermanently, "http://"+leader+req.URL.Path)
-		http.Redirect(w, req, "http://"+leader+req.URL.Path, http.StatusMovedPermanently)
+		learderLocation := "http://" + leader + req.URL.Path
+		glog.V(0).Infoln("Redirecting to", learderLocation)
+		writeJsonQuiet(w, req, http.StatusOK, learderLocation)
+		// http.Redirect(w, req, "http://"+leader+req.URL.Path, http.StatusFound) // not working any more
 	} else {
 		glog.V(0).Infoln("Error: Leader Unknown")
 		http.Error(w, "Leader unknown", http.StatusInternalServerError)

weed/server/volume_grpc_client.go

@@ -5,7 +5,7 @@ import (
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/storage"
 	"golang.org/x/net/context"
@@ -42,7 +42,7 @@ func (vs *VolumeServer) doHeartbeat(sleepInterval time.Duration) error {
 	}
 	defer grpcConection.Close()
 
-	client := pb.NewSeaweedClient(grpcConection)
+	client := master_pb.NewSeaweedClient(grpcConection)
 	stream, err := client.SendHeartbeat(context.Background())
 	if err != nil {
 		glog.V(0).Infof("%v.SendHeartbeat(_) = _, %v", client, err)
@@ -63,8 +63,17 @@ func (vs *VolumeServer) doHeartbeat(sleepInterval time.Duration) error {
 				doneChan <- err
 				return
 			}
-			vs.store.VolumeSizeLimit = in.GetVolumeSizeLimit()
-			vs.guard.SecretKey = security.Secret(in.GetSecretKey())
+			if in.GetVolumeSizeLimit() != 0 {
+				vs.store.VolumeSizeLimit = in.GetVolumeSizeLimit()
+			}
+			if in.GetSecretKey() != "" {
+				vs.guard.SecretKey = security.Secret(in.GetSecretKey())
+			}
+			if in.GetLeader() != "" && masterNode != in.GetLeader() {
+				vs.masterNodes.SetPossibleLeader(in.GetLeader())
+				doneChan <- nil
+				return
+			}
 		}
 	}()
 
@@ -73,7 +82,7 @@ func (vs *VolumeServer) doHeartbeat(sleepInterval time.Duration) error {
 		return err
 	}
 
-	tickChan := time.NewTimer(sleepInterval).C
+	tickChan := time.Tick(sleepInterval)
 
 	for {
 		select {
@@ -83,6 +92,7 @@ func (vs *VolumeServer) doHeartbeat(sleepInterval time.Duration) error {
 				return err
 			}
 		case err := <-doneChan:
+			glog.V(0).Infof("Volume Server heart beat stops with %v", err)
 			return err
 		}
 	}

weed/server/volume_server.go

@@ -32,8 +32,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 	dataCenter string, rack string,
 	whiteList []string,
 	fixJpgOrientation bool,
-	readRedirect bool,
-	enableBytesCache bool) *VolumeServer {
+	readRedirect bool) *VolumeServer {
 	vs := &VolumeServer{
 		pulseSeconds:      pulseSeconds,
 		dataCenter:        dataCenter,
@@ -44,7 +43,6 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 	}
 	vs.SetMasterNode(masterNode)
 	vs.store = storage.NewStore(port, ip, publicUrl, folders, maxCounts, vs.needleMapKind)
-	storage.EnableBytesCache = enableBytesCache
 
 	vs.guard = security.NewGuard(whiteList, "")
 
@@ -54,10 +52,14 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 	adminMux.HandleFunc("/admin/vacuum/check", vs.guard.WhiteList(vs.vacuumVolumeCheckHandler))
 	adminMux.HandleFunc("/admin/vacuum/compact", vs.guard.WhiteList(vs.vacuumVolumeCompactHandler))
 	adminMux.HandleFunc("/admin/vacuum/commit", vs.guard.WhiteList(vs.vacuumVolumeCommitHandler))
+	adminMux.HandleFunc("/admin/vacuum/cleanup", vs.guard.WhiteList(vs.vacuumVolumeCleanupHandler))
 	adminMux.HandleFunc("/admin/delete_collection", vs.guard.WhiteList(vs.deleteCollectionHandler))
 	adminMux.HandleFunc("/admin/sync/status", vs.guard.WhiteList(vs.getVolumeSyncStatusHandler))
 	adminMux.HandleFunc("/admin/sync/index", vs.guard.WhiteList(vs.getVolumeIndexContentHandler))
 	adminMux.HandleFunc("/admin/sync/data", vs.guard.WhiteList(vs.getVolumeDataContentHandler))
+	adminMux.HandleFunc("/admin/volume/mount", vs.guard.WhiteList(vs.getVolumeMountHandler))
+	adminMux.HandleFunc("/admin/volume/unmount", vs.guard.WhiteList(vs.getVolumeUnmountHandler))
+	adminMux.HandleFunc("/admin/volume/delete", vs.guard.WhiteList(vs.getVolumeDeleteHandler))
 	adminMux.HandleFunc("/stats/counter", vs.guard.WhiteList(statsCounterHandler))
 	adminMux.HandleFunc("/stats/memory", vs.guard.WhiteList(statsMemoryHandler))
 	adminMux.HandleFunc("/stats/disk", vs.guard.WhiteList(vs.statsDiskHandler))

weed/server/volume_server_handlers_admin.go

@@ -1,6 +1,7 @@
 package weed_server
 
 import (
+	"fmt"
 	"net/http"
 	"path/filepath"
 	"strconv"
@@ -8,6 +9,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/stats"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/storage"
 )
 
 func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) {
@@ -65,3 +67,45 @@ func (vs *VolumeServer) statsDiskHandler(w http.ResponseWriter, r *http.Request)
 	m["DiskStatuses"] = ds
 	writeJsonQuiet(w, r, http.StatusOK, m)
 }
+
+func (vs *VolumeServer) getVolume(volumeParameterName string, r *http.Request) (*storage.Volume, error) {
+	vid, err := vs.getVolumeId(volumeParameterName, r)
+	if err != nil {
+		return nil, err
+	}
+	v := vs.store.GetVolume(vid)
+	if v == nil {
+		return nil, fmt.Errorf("Not Found Volume Id %d", vid)
+	}
+	return v, nil
+}
+
+func (vs *VolumeServer) getVolumeMountHandler(w http.ResponseWriter, r *http.Request) {
+	vid, err := vs.getVolumeId("volume", r)
+	if err != nil {
+		writeJsonError(w, r, http.StatusNotFound, err)
+		return
+	}
+	vs.store.MountVolume(vid)
+	writeJsonQuiet(w, r, http.StatusOK, "Volume mounted")
+}
+
+func (vs *VolumeServer) getVolumeUnmountHandler(w http.ResponseWriter, r *http.Request) {
+	vid, err := vs.getVolumeId("volume", r)
+	if err != nil {
+		writeJsonError(w, r, http.StatusNotFound, err)
+		return
+	}
+	vs.store.UnmountVolume(vid)
+	writeJsonQuiet(w, r, http.StatusOK, "Volume unmounted")
+}
+
+func (vs *VolumeServer) getVolumeDeleteHandler(w http.ResponseWriter, r *http.Request) {
+	vid, err := vs.getVolumeId("volume", r)
+	if err != nil {
+		writeJsonError(w, r, http.StatusNotFound, err)
+		return
+	}
+	vs.store.DeleteVolume(vid)
+	writeJsonQuiet(w, r, http.StatusOK, "Volume deleted")
+}

weed/server/volume_server_handlers_read.go

@@ -67,12 +67,11 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 	cookie := n.Cookie
 	count, e := vs.store.ReadVolumeNeedle(volumeId, n)
 	glog.V(4).Infoln("read bytes", count, "error", e)
-	if e != nil || count <= 0 {
+	if e != nil || count < 0 {
 		glog.V(0).Infoln("read error:", e, r.URL.Path)
 		w.WriteHeader(http.StatusNotFound)
 		return
 	}
-	defer n.ReleaseMemory()
 	if n.Cookie != cookie {
 		glog.V(0).Infoln("request", r.URL.Path, "with unmaching cookie seen:", cookie, "expected:", n.Cookie, "from", r.RemoteAddr, "agent", r.UserAgent())
 		w.WriteHeader(http.StatusNotFound)
@@ -144,7 +143,7 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 		if r.FormValue("height") != "" {
 			height, _ = strconv.Atoi(r.FormValue("height"))
 		}
-		n.Data, _, _ = images.Resized(ext, n.Data, width, height)
+		n.Data, _, _ = images.Resized(ext, n.Data, width, height, r.FormValue("mode"))
 	}
 
 	if e := writeResponseContent(filename, mtype, bytes.NewReader(n.Data), w, r); e != nil {
@@ -165,7 +164,7 @@ func (vs *VolumeServer) FaviconHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func (vs *VolumeServer) tryHandleChunkedFile(n *storage.Needle, fileName string, w http.ResponseWriter, r *http.Request) (processed bool) {
-	if !n.IsChunkedManifest() {
+	if !n.IsChunkedManifest() || r.URL.Query().Get("cm") == "false" {
 		return false
 	}
 

weed/server/volume_server_handlers_sync.go

@@ -50,8 +50,7 @@ func (vs *VolumeServer) getVolumeDataContentHandler(w http.ResponseWriter, r *ht
 	}
 	offset := uint32(util.ParseUint64(r.FormValue("offset"), 0))
 	size := uint32(util.ParseUint64(r.FormValue("size"), 0))
-	content, block, err := storage.ReadNeedleBlob(v.DataFile(), int64(offset)*storage.NeedlePaddingSize, size)
-	defer storage.ReleaseBytes(block.Bytes)
+	content, err := storage.ReadNeedleBlob(v.DataFile(), int64(offset)*storage.NeedlePaddingSize, size)
 	if err != nil {
 		writeJsonError(w, r, http.StatusInternalServerError, err)
 		return
@@ -68,20 +67,19 @@ func (vs *VolumeServer) getVolumeDataContentHandler(w http.ResponseWriter, r *ht
 	w.Write(content)
 }
 
-func (vs *VolumeServer) getVolume(volumeParameterName string, r *http.Request) (*storage.Volume, error) {
+func (vs *VolumeServer) getVolumeId(volumeParameterName string, r *http.Request) (storage.VolumeId, error) {
 	volumeIdString := r.FormValue(volumeParameterName)
+
 	if volumeIdString == "" {
 		err := fmt.Errorf("Empty Volume Id: Need to pass in %s=the_volume_id.", volumeParameterName)
-		return nil, err
+		return 0, err
 	}
+
 	vid, err := storage.NewVolumeId(volumeIdString)
 	if err != nil {
 		err = fmt.Errorf("Volume Id %s is not a valid unsigned integer", volumeIdString)
-		return nil, err
+		return 0, err
 	}
-	v := vs.store.GetVolume(vid)
-	if v == nil {
-		return nil, fmt.Errorf("Not Found Volume Id %s: %d", volumeIdString, vid)
-	}
-	return v, nil
+
+	return vid, err
 }

weed/server/volume_server_handlers_vacuum.go

@@ -2,6 +2,7 @@ package weed_server
 
 import (
 	"net/http"
+	"strconv"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 )
@@ -16,7 +17,15 @@ func (vs *VolumeServer) vacuumVolumeCheckHandler(w http.ResponseWriter, r *http.
 	glog.V(2).Infoln("checked compacting volume =", r.FormValue("volume"), "garbageThreshold =", r.FormValue("garbageThreshold"), "vacuum =", ret)
 }
 func (vs *VolumeServer) vacuumVolumeCompactHandler(w http.ResponseWriter, r *http.Request) {
-	err := vs.store.CompactVolume(r.FormValue("volume"))
+	var preallocate int64
+	var err error
+	if r.FormValue("preallocate") != "" {
+		preallocate, err = strconv.ParseInt(r.FormValue("preallocate"), 10, 64)
+		if err != nil {
+			glog.V(0).Infoln("Failed to parse int64 preallocate = %s: %v", r.FormValue("preallocate"), err)
+		}
+	}
+	err = vs.store.CompactVolume(r.FormValue("volume"), preallocate)
 	if err == nil {
 		writeJsonQuiet(w, r, http.StatusOK, map[string]string{"error": ""})
 	} else {
@@ -33,3 +42,12 @@ func (vs *VolumeServer) vacuumVolumeCommitHandler(w http.ResponseWriter, r *http
 	}
 	glog.V(2).Infoln("commit compact volume =", r.FormValue("volume"), ", error =", err)
 }
+func (vs *VolumeServer) vacuumVolumeCleanupHandler(w http.ResponseWriter, r *http.Request) {
+	err := vs.store.CommitCleanupVolume(r.FormValue("volume"))
+	if err == nil {
+		writeJsonQuiet(w, r, http.StatusOK, map[string]string{"error": ""})
+	} else {
+		writeJsonError(w, r, http.StatusInternalServerError, err)
+	}
+	glog.V(2).Infoln("cleanup compact volume =", r.FormValue("volume"), ", error =", err)
+}

weed/server/volume_server_handlers_write.go

@@ -64,7 +64,6 @@ func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
 		writeJsonQuiet(w, r, http.StatusNotFound, m)
 		return
 	}
-	defer n.ReleaseMemory()
 
 	if n.Cookie != cookie {
 		glog.V(0).Infoln("delete", r.URL.Path, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
@@ -133,7 +132,6 @@ func (vs *VolumeServer) batchDeleteHandler(w http.ResponseWriter, r *http.Reques
 				Status: http.StatusNotAcceptable,
 				Error:  "ChunkManifest: not allowed in batch delete mode.",
 			})
-			n.ReleaseMemory()
 			continue
 		}
 
@@ -144,7 +142,6 @@ func (vs *VolumeServer) batchDeleteHandler(w http.ResponseWriter, r *http.Reques
 				Error:  "File Random Cookie does not match.",
 			})
 			glog.V(0).Infoln("deleting", fid, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
-			n.ReleaseMemory()
 			return
 		}
 		if size, err := vs.store.Delete(volumeId, n); err != nil {
@@ -160,7 +157,6 @@ func (vs *VolumeServer) batchDeleteHandler(w http.ResponseWriter, r *http.Reques
 				Size:   int(size)},
 			)
 		}
-		n.ReleaseMemory()
 	}
 
 	writeJsonQuiet(w, r, http.StatusAccepted, ret)

weed/storage/compact_map.go

@@ -39,14 +39,13 @@ func NewCompactSection(start Key) *CompactSection {
 }
 
 //return old entry size
-func (cs *CompactSection) Set(key Key, offset uint32, size uint32) uint32 {
-	ret := uint32(0)
+func (cs *CompactSection) Set(key Key, offset, size uint32) (oldOffset, oldSize uint32) {
 	cs.Lock()
 	if key > cs.end {
 		cs.end = key
 	}
 	if i := cs.binarySearchValues(key); i >= 0 {
-		ret = cs.values[i].Size
+		oldOffset, oldSize = cs.values[i].Offset, cs.values[i].Size
 		//println("key", key, "old size", ret)
 		cs.values[i].Offset, cs.values[i].Size = offset, size
 	} else {
@@ -55,7 +54,7 @@ func (cs *CompactSection) Set(key Key, offset uint32, size uint32) uint32 {
 		if needOverflow {
 			//println("start", cs.start, "counter", cs.counter, "key", key)
 			if oldValue, found := cs.overflow[key]; found {
-				ret = oldValue.Size
+				oldOffset, oldSize = oldValue.Offset, oldValue.Size
 			}
 			cs.overflow[key] = NeedleValue{Key: key, Offset: offset, Size: size}
 		} else {
@@ -66,7 +65,7 @@ func (cs *CompactSection) Set(key Key, offset uint32, size uint32) uint32 {
 		}
 	}
 	cs.Unlock()
-	return ret
+	return
 }
 
 //return old entry size
@@ -129,7 +128,7 @@ func NewCompactMap() CompactMap {
 	return CompactMap{}
 }
 
-func (cm *CompactMap) Set(key Key, offset uint32, size uint32) uint32 {
+func (cm *CompactMap) Set(key Key, offset, size uint32) (oldOffset, oldSize uint32) {
 	x := cm.binarySearchCompactSection(key)
 	if x < 0 {
 		//println(x, "creating", len(cm.list), "section, starting", key)

weed/storage/disk_location.go

@@ -6,6 +6,8 @@ import (
 	"strings"
 	"sync"
 
+	"fmt"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 )
 
@@ -22,7 +24,7 @@ func NewDiskLocation(dir string, maxVolumeCount int) *DiskLocation {
 	return location
 }
 
-func (l *DiskLocation) loadExistingVolume(dir os.FileInfo, needleMapKind NeedleMapType, mutex *sync.RWMutex) {
+func (l *DiskLocation) volumeIdFromPath(dir os.FileInfo) (VolumeId, string, error) {
 	name := dir.Name()
 	if !dir.IsDir() && strings.HasSuffix(name, ".dat") {
 		collection := ""
@@ -31,7 +33,18 @@ func (l *DiskLocation) loadExistingVolume(dir os.FileInfo, needleMapKind NeedleM
 		if i > 0 {
 			collection, base = base[0:i], base[i+1:]
 		}
-		if vid, err := NewVolumeId(base); err == nil {
+		vol, err := NewVolumeId(base)
+		return vol, collection, err
+	}
+
+	return 0, "", fmt.Errorf("Path is not a volume: %s", name)
+}
+
+func (l *DiskLocation) loadExistingVolume(dir os.FileInfo, needleMapKind NeedleMapType, mutex *sync.RWMutex) {
+	name := dir.Name()
+	if !dir.IsDir() && strings.HasSuffix(name, ".dat") {
+		vid, collection, err := l.volumeIdFromPath(dir)
+		if err == nil {
 			mutex.RLock()
 			_, found := l.volumes[vid]
 			mutex.RUnlock()
@@ -42,10 +55,6 @@ func (l *DiskLocation) loadExistingVolume(dir os.FileInfo, needleMapKind NeedleM
 					mutex.Unlock()
 					glog.V(0).Infof("data file %s, replicaPlacement=%s v=%d size=%d ttl=%s",
 						l.Directory+"/"+name, v.ReplicaPlacement, v.Version(), v.Size(), v.Ttl.String())
-					if v.Size() != v.dataFileSize {
-						glog.V(0).Infof("data file %s, size=%d expected=%d",
-							l.Directory+"/"+name, v.Size(), v.dataFileSize)
-					}
 				} else {
 					glog.V(0).Infof("new volume %s error %s", name, e)
 				}
@@ -125,6 +134,45 @@ func (l *DiskLocation) deleteVolumeById(vid VolumeId) (e error) {
 	return
 }
 
+func (l *DiskLocation) LoadVolume(vid VolumeId, needleMapKind NeedleMapType) bool {
+	if dirs, err := ioutil.ReadDir(l.Directory); err == nil {
+		for _, dir := range dirs {
+			volId, _, err := l.volumeIdFromPath(dir)
+			if vid == volId && err == nil {
+				var mutex sync.RWMutex
+				l.loadExistingVolume(dir, needleMapKind, &mutex)
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+func (l *DiskLocation) DeleteVolume(vid VolumeId) error {
+	l.Lock()
+	defer l.Unlock()
+
+	_, ok := l.volumes[vid]
+	if !ok {
+		return fmt.Errorf("Volume not found, VolumeId: %d", vid)
+	}
+	return l.deleteVolumeById(vid)
+}
+
+func (l *DiskLocation) UnloadVolume(vid VolumeId) error {
+	l.Lock()
+	defer l.Unlock()
+
+	v, ok := l.volumes[vid]
+	if !ok {
+		return fmt.Errorf("Volume not loaded, VolumeId: %d", vid)
+	}
+	v.Close()
+	delete(l.volumes, vid)
+	return nil
+}
+
 func (l *DiskLocation) SetVolume(vid VolumeId, volume *Volume) {
 	l.Lock()
 	defer l.Unlock()

weed/storage/needle.go

@@ -49,8 +49,6 @@ type Needle struct {
 
 	Checksum CRC    `comment:"CRC32 to check integrity"`
 	Padding  []byte `comment:"Aligned to 8 bytes"`
-
-	rawBlock *Block // underlying supporing []byte, fetched and released into a pool
 }
 
 func (n *Needle) String() (str string) {

weed/storage/needle/btree_map.go

@@ -0,0 +1,52 @@
+package needle
+
+import (
+	"github.com/google/btree"
+)
+
+//This map assumes mostly inserting increasing keys
+type BtreeMap struct {
+	tree *btree.BTree
+}
+
+func NewBtreeMap() *BtreeMap {
+	return &BtreeMap{
+		tree: btree.New(32),
+	}
+}
+
+func (cm *BtreeMap) Set(key Key, offset, size uint32) (oldOffset, oldSize uint32) {
+	found := cm.tree.ReplaceOrInsert(NeedleValue{key, offset, size})
+	if found != nil {
+		old := found.(NeedleValue)
+		return old.Offset, old.Size
+	}
+	return
+}
+
+func (cm *BtreeMap) Delete(key Key) (oldSize uint32) {
+	found := cm.tree.Delete(NeedleValue{key, 0, 0})
+	if found != nil {
+		old := found.(NeedleValue)
+		return old.Size
+	}
+	return
+}
+func (cm *BtreeMap) Get(key Key) (*NeedleValue, bool) {
+	found := cm.tree.Get(NeedleValue{key, 0, 0})
+	if found != nil {
+		old := found.(NeedleValue)
+		return &old, true
+	}
+	return nil, false
+}
+
+// Visit visits all entries or stop if any error when visiting
+func (cm *BtreeMap) Visit(visit func(NeedleValue) error) (ret error) {
+	cm.tree.Ascend(func(item btree.Item) bool {
+		needle := item.(NeedleValue)
+		ret = visit(needle)
+		return ret == nil
+	})
+	return ret
+}

weed/storage/needle/compact_map.go

@@ -0,0 +1,194 @@
+package needle
+
+import (
+	"sync"
+)
+
+type CompactSection struct {
+	sync.RWMutex
+	values   []NeedleValue
+	overflow map[Key]NeedleValue
+	start    Key
+	end      Key
+	counter  int
+}
+
+func NewCompactSection(start Key) *CompactSection {
+	return &CompactSection{
+		values:   make([]NeedleValue, batch),
+		overflow: make(map[Key]NeedleValue),
+		start:    start,
+	}
+}
+
+//return old entry size
+func (cs *CompactSection) Set(key Key, offset, size uint32) (oldOffset, oldSize uint32) {
+	cs.Lock()
+	if key > cs.end {
+		cs.end = key
+	}
+	if i := cs.binarySearchValues(key); i >= 0 {
+		oldOffset, oldSize = cs.values[i].Offset, cs.values[i].Size
+		//println("key", key, "old size", ret)
+		cs.values[i].Offset, cs.values[i].Size = offset, size
+	} else {
+		needOverflow := cs.counter >= batch
+		needOverflow = needOverflow || cs.counter > 0 && cs.values[cs.counter-1].Key > key
+		if needOverflow {
+			//println("start", cs.start, "counter", cs.counter, "key", key)
+			if oldValue, found := cs.overflow[key]; found {
+				oldOffset, oldSize = oldValue.Offset, oldValue.Size
+			}
+			cs.overflow[key] = NeedleValue{Key: key, Offset: offset, Size: size}
+		} else {
+			p := &cs.values[cs.counter]
+			p.Key, p.Offset, p.Size = key, offset, size
+			//println("added index", cs.counter, "key", key, cs.values[cs.counter].Key)
+			cs.counter++
+		}
+	}
+	cs.Unlock()
+	return
+}
+
+//return old entry size
+func (cs *CompactSection) Delete(key Key) uint32 {
+	cs.Lock()
+	ret := uint32(0)
+	if i := cs.binarySearchValues(key); i >= 0 {
+		if cs.values[i].Size > 0 {
+			ret = cs.values[i].Size
+			cs.values[i].Size = 0
+		}
+	}
+	if v, found := cs.overflow[key]; found {
+		delete(cs.overflow, key)
+		ret = v.Size
+	}
+	cs.Unlock()
+	return ret
+}
+func (cs *CompactSection) Get(key Key) (*NeedleValue, bool) {
+	cs.RLock()
+	if v, ok := cs.overflow[key]; ok {
+		cs.RUnlock()
+		return &v, true
+	}
+	if i := cs.binarySearchValues(key); i >= 0 {
+		cs.RUnlock()
+		return &cs.values[i], true
+	}
+	cs.RUnlock()
+	return nil, false
+}
+func (cs *CompactSection) binarySearchValues(key Key) int {
+	l, h := 0, cs.counter-1
+	if h >= 0 && cs.values[h].Key < key {
+		return -2
+	}
+	//println("looking for key", key)
+	for l <= h {
+		m := (l + h) / 2
+		//println("mid", m, "key", cs.values[m].Key, cs.values[m].Offset, cs.values[m].Size)
+		if cs.values[m].Key < key {
+			l = m + 1
+		} else if key < cs.values[m].Key {
+			h = m - 1
+		} else {
+			//println("found", m)
+			return m
+		}
+	}
+	return -1
+}
+
+//This map assumes mostly inserting increasing keys
+//This map assumes mostly inserting increasing keys
+type CompactMap struct {
+	list []*CompactSection
+}
+
+func NewCompactMap() *CompactMap {
+	return &CompactMap{}
+}
+
+func (cm *CompactMap) Set(key Key, offset, size uint32) (oldOffset, oldSize uint32) {
+	x := cm.binarySearchCompactSection(key)
+	if x < 0 {
+		//println(x, "creating", len(cm.list), "section, starting", key)
+		cm.list = append(cm.list, NewCompactSection(key))
+		x = len(cm.list) - 1
+		//keep compact section sorted by start
+		for x > 0 {
+			if cm.list[x-1].start > cm.list[x].start {
+				cm.list[x-1], cm.list[x] = cm.list[x], cm.list[x-1]
+				x = x - 1
+			} else {
+				break
+			}
+		}
+	}
+	return cm.list[x].Set(key, offset, size)
+}
+func (cm *CompactMap) Delete(key Key) uint32 {
+	x := cm.binarySearchCompactSection(key)
+	if x < 0 {
+		return uint32(0)
+	}
+	return cm.list[x].Delete(key)
+}
+func (cm *CompactMap) Get(key Key) (*NeedleValue, bool) {
+	x := cm.binarySearchCompactSection(key)
+	if x < 0 {
+		return nil, false
+	}
+	return cm.list[x].Get(key)
+}
+func (cm *CompactMap) binarySearchCompactSection(key Key) int {
+	l, h := 0, len(cm.list)-1
+	if h < 0 {
+		return -5
+	}
+	if cm.list[h].start <= key {
+		if cm.list[h].counter < batch || key <= cm.list[h].end {
+			return h
+		}
+		return -4
+	}
+	for l <= h {
+		m := (l + h) / 2
+		if key < cm.list[m].start {
+			h = m - 1
+		} else { // cm.list[m].start <= key
+			if cm.list[m+1].start <= key {
+				l = m + 1
+			} else {
+				return m
+			}
+		}
+	}
+	return -3
+}
+
+// Visit visits all entries or stop if any error when visiting
+func (cm *CompactMap) Visit(visit func(NeedleValue) error) error {
+	for _, cs := range cm.list {
+		cs.RLock()
+		for _, v := range cs.overflow {
+			if err := visit(v); err != nil {
+				cs.RUnlock()
+				return err
+			}
+		}
+		for _, v := range cs.values {
+			if _, found := cs.overflow[v.Key]; !found {
+				if err := visit(v); err != nil {
+					cs.RUnlock()
+					return err
+				}
+			}
+		}
+		cs.RUnlock()
+	}
+	return nil
+}

weed/storage/needle/compact_map_perf_test.go

@@ -1,4 +1,4 @@
-package storage
+package needle
 
 import (
 	"log"
@@ -11,15 +11,15 @@ import (
 
 func TestMemoryUsage(t *testing.T) {
 
-	indexFile, ie := os.OpenFile("../../test/sample.idx", os.O_RDWR|os.O_RDONLY, 0644)
+	indexFile, ie := os.OpenFile("../../../test/sample.idx", os.O_RDWR|os.O_RDONLY, 0644)
 	if ie != nil {
 		log.Fatalln(ie)
 	}
-	LoadNewNeedleMap(indexFile)
+	loadNewNeedleMap(indexFile)
 
 }
 
-func LoadNewNeedleMap(file *os.File) CompactMap {
+func loadNewNeedleMap(file *os.File) {
 	m := NewCompactMap()
 	bytes := make([]byte, 16*1024)
 	count, e := file.Read(bytes)
@@ -41,5 +41,4 @@ func LoadNewNeedleMap(file *os.File) CompactMap {
 
 		count, e = file.Read(bytes)
 	}
-	return m
 }

weed/storage/needle/compact_map_test.go

@@ -1,4 +1,4 @@
-package storage
+package needle
 
 import (
 	"testing"

weed/storage/needle/needle_value.go

@@ -0,0 +1,28 @@
+package needle
+
+import (
+	"strconv"
+
+	"github.com/google/btree"
+)
+
+const (
+	batch = 100000
+)
+
+type NeedleValue struct {
+	Key    Key
+	Offset uint32 `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
+	Size   uint32 `comment:"Size of the data portion"`
+}
+
+func (this NeedleValue) Less(than btree.Item) bool {
+	that := than.(NeedleValue)
+	return this.Key < that.Key
+}
+
+type Key uint64
+
+func (k Key) String() string {
+	return strconv.FormatUint(uint64(k), 10)
+}

weed/storage/needle/needle_value_map.go

@@ -0,0 +1,8 @@
+package needle
+
+type NeedleValueMap interface {
+	Set(key Key, offset, size uint32) (oldOffset, oldSize uint32)
+	Delete(key Key) uint32
+	Get(key Key) (*NeedleValue, bool)
+	Visit(visit func(NeedleValue) error) error
+}

weed/storage/needle_byte_cache.go

@@ -1,80 +1,11 @@
 package storage
 
 import (
-	"fmt"
 	"os"
-	"sync/atomic"
-
-	"github.com/hashicorp/golang-lru"
-
-	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
-var (
-	EnableBytesCache = true
-	bytesCache       *lru.Cache
-	bytesPool        *util.BytesPool
-)
-
-/*
-There are one level of caching, and one level of pooling.
-
-In pooling, all []byte are fetched and returned to the pool bytesPool.
-
-In caching, the string~[]byte mapping is cached
-*/
-func init() {
-	bytesPool = util.NewBytesPool()
-	bytesCache, _ = lru.NewWithEvict(512, func(key interface{}, value interface{}) {
-		value.(*Block).decreaseReference()
-	})
-}
-
-type Block struct {
-	Bytes    []byte
-	refCount int32
-}
-
-func (block *Block) decreaseReference() {
-	if atomic.AddInt32(&block.refCount, -1) == 0 {
-		bytesPool.Put(block.Bytes)
-	}
-}
-func (block *Block) increaseReference() {
-	atomic.AddInt32(&block.refCount, 1)
-}
-
-// get bytes from the LRU cache of []byte first, then from the bytes pool
-// when []byte in LRU cache is evicted, it will be put back to the bytes pool
-func getBytesForFileBlock(r *os.File, offset int64, readSize int) (dataSlice []byte, block *Block, err error) {
-	// check cache, return if found
-	cacheKey := fmt.Sprintf("%d:%d:%d", r.Fd(), offset>>3, readSize)
-	if EnableBytesCache {
-		if obj, found := bytesCache.Get(cacheKey); found {
-			block = obj.(*Block)
-			block.increaseReference()
-			dataSlice = block.Bytes[0:readSize]
-			return dataSlice, block, nil
-		}
-	}
-
-	// get the []byte from pool
-	b := bytesPool.Get(readSize)
-	// refCount = 2, one by the bytesCache, one by the actual needle object
-	block = &Block{Bytes: b, refCount: 2}
-	dataSlice = block.Bytes[0:readSize]
+func getBytesForFileBlock(r *os.File, offset int64, readSize int) (dataSlice []byte, err error) {
+	dataSlice = make([]byte, readSize)
 	_, err = r.ReadAt(dataSlice, offset)
-	if EnableBytesCache {
-		bytesCache.Add(cacheKey, block)
-	}
-	return dataSlice, block, err
-}
-
-func (n *Needle) ReleaseMemory() {
-	if n.rawBlock != nil {
-		n.rawBlock.decreaseReference()
-	}
-}
-func ReleaseBytes(b []byte) {
-	bytesPool.Put(b)
+	return dataSlice, err
 }

weed/storage/needle_map.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"sync"
 
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -15,6 +16,7 @@ const (
 	NeedleMapInMemory NeedleMapType = iota
 	NeedleMapLevelDb
 	NeedleMapBoltDb
+	NeedleMapBtree
 )
 
 const (
@@ -23,7 +25,7 @@ const (
 
 type NeedleMapper interface {
 	Put(key uint64, offset uint32, size uint32) error
-	Get(key uint64) (element *NeedleValue, ok bool)
+	Get(key uint64) (element *needle.NeedleValue, ok bool)
 	Delete(key uint64, offset uint32) error
 	Close()
 	Destroy() error

weed/storage/needle_map_boltdb.go

@@ -7,6 +7,7 @@ import (
 	"github.com/boltdb/bolt"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -18,6 +19,7 @@ type BoltDbNeedleMap struct {
 
 var boltdbBucket = []byte("weed")
 
+// TODO avoid using btree to count deletions.
 func NewBoltDbNeedleMap(dbFileName string, indexFile *os.File) (m *BoltDbNeedleMap, err error) {
 	m = &BoltDbNeedleMap{dbFileName: dbFileName}
 	m.indexFile = indexFile
@@ -31,7 +33,7 @@ func NewBoltDbNeedleMap(dbFileName string, indexFile *os.File) (m *BoltDbNeedleM
 		return
 	}
 	glog.V(1).Infof("Loading %s...", indexFile.Name())
-	nm, indexLoadError := LoadNeedleMap(indexFile)
+	nm, indexLoadError := LoadBtreeNeedleMap(indexFile)
 	if indexLoadError != nil {
 		return nil, indexLoadError
 	}
@@ -72,7 +74,7 @@ func generateBoltDbFile(dbFileName string, indexFile *os.File) error {
 	})
 }
 
-func (m *BoltDbNeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
+func (m *BoltDbNeedleMap) Get(key uint64) (element *needle.NeedleValue, ok bool) {
 	bytes := make([]byte, 8)
 	var data []byte
 	util.Uint64toBytes(bytes, key)
@@ -91,7 +93,7 @@ func (m *BoltDbNeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
 	}
 	offset := util.BytesToUint32(data[0:4])
 	size := util.BytesToUint32(data[4:8])
-	return &NeedleValue{Key: Key(key), Offset: offset, Size: size}, true
+	return &needle.NeedleValue{Key: needle.Key(key), Offset: offset, Size: size}, true
 }
 
 func (m *BoltDbNeedleMap) Put(key uint64, offset uint32, size uint32) error {

weed/storage/needle_map_leveldb.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/syndtr/goleveldb/leveldb"
 )
@@ -16,6 +17,7 @@ type LevelDbNeedleMap struct {
 	baseNeedleMapper
 }
 
+// TODO avoid using btree to count deletions.
 func NewLevelDbNeedleMap(dbFileName string, indexFile *os.File) (m *LevelDbNeedleMap, err error) {
 	m = &LevelDbNeedleMap{dbFileName: dbFileName}
 	m.indexFile = indexFile
@@ -29,7 +31,7 @@ func NewLevelDbNeedleMap(dbFileName string, indexFile *os.File) (m *LevelDbNeedl
 		return
 	}
 	glog.V(1).Infof("Loading %s...", indexFile.Name())
-	nm, indexLoadError := LoadNeedleMap(indexFile)
+	nm, indexLoadError := LoadBtreeNeedleMap(indexFile)
 	if indexLoadError != nil {
 		return nil, indexLoadError
 	}
@@ -70,7 +72,7 @@ func generateLevelDbFile(dbFileName string, indexFile *os.File) error {
 	})
 }
 
-func (m *LevelDbNeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
+func (m *LevelDbNeedleMap) Get(key uint64) (element *needle.NeedleValue, ok bool) {
 	bytes := make([]byte, 8)
 	util.Uint64toBytes(bytes, key)
 	data, err := m.db.Get(bytes, nil)
@@ -79,7 +81,7 @@ func (m *LevelDbNeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
 	}
 	offset := util.BytesToUint32(data[0:4])
 	size := util.BytesToUint32(data[4:8])
-	return &NeedleValue{Key: Key(key), Offset: offset, Size: size}, true
+	return &needle.NeedleValue{Key: needle.Key(key), Offset: offset, Size: size}, true
 }
 
 func (m *LevelDbNeedleMap) Put(key uint64, offset uint32, size uint32) error {
@@ -130,5 +132,5 @@ func (m *LevelDbNeedleMap) Close() {
 func (m *LevelDbNeedleMap) Destroy() error {
 	m.Close()
 	os.Remove(m.indexFile.Name())
-	return os.Remove(m.dbFileName)
+	return os.RemoveAll(m.dbFileName)
 }

weed/storage/needle_map_memory.go

@@ -5,17 +5,26 @@ import (
 	"os"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 )
 
 type NeedleMap struct {
-	m CompactMap
+	m needle.NeedleValueMap
 
 	baseNeedleMapper
 }
 
-func NewNeedleMap(file *os.File) *NeedleMap {
+func NewCompactNeedleMap(file *os.File) *NeedleMap {
 	nm := &NeedleMap{
-		m: NewCompactMap(),
+		m: needle.NewCompactMap(),
+	}
+	nm.indexFile = file
+	return nm
+}
+
+func NewBtreeNeedleMap(file *os.File) *NeedleMap {
+	nm := &NeedleMap{
+		m: needle.NewBtreeMap(),
 	}
 	nm.indexFile = file
 	return nm
@@ -25,23 +34,32 @@ const (
 	RowsToRead = 1024
 )
 
-func LoadNeedleMap(file *os.File) (*NeedleMap, error) {
-	nm := NewNeedleMap(file)
+func LoadCompactNeedleMap(file *os.File) (*NeedleMap, error) {
+	nm := NewCompactNeedleMap(file)
+	return doLoading(file, nm)
+}
+
+func LoadBtreeNeedleMap(file *os.File) (*NeedleMap, error) {
+	nm := NewBtreeNeedleMap(file)
+	return doLoading(file, nm)
+}
+
+func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) {
 	e := WalkIndexFile(file, func(key uint64, offset, size uint32) error {
 		if key > nm.MaximumFileKey {
 			nm.MaximumFileKey = key
 		}
-		nm.FileCounter++
-		nm.FileByteCounter = nm.FileByteCounter + uint64(size)
 		if offset > 0 && size != TombstoneFileSize {
-			oldSize := nm.m.Set(Key(key), offset, size)
+			nm.FileCounter++
+			nm.FileByteCounter = nm.FileByteCounter + uint64(size)
+			oldOffset, oldSize := nm.m.Set(needle.Key(key), offset, size)
 			glog.V(3).Infoln("reading key", key, "offset", offset*NeedlePaddingSize, "size", size, "oldSize", oldSize)
-			if oldSize > 0 {
+			if oldOffset > 0 && oldSize != TombstoneFileSize {
 				nm.DeletionCounter++
 				nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
 			}
 		} else {
-			oldSize := nm.m.Delete(Key(key))
+			oldSize := nm.m.Delete(needle.Key(key))
 			glog.V(3).Infoln("removing key", key, "offset", offset*NeedlePaddingSize, "size", size, "oldSize", oldSize)
 			nm.DeletionCounter++
 			nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
@@ -84,16 +102,16 @@ func WalkIndexFile(r *os.File, fn func(key uint64, offset, size uint32) error) e
 }
 
 func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) error {
-	oldSize := nm.m.Set(Key(key), offset, size)
+	_, oldSize := nm.m.Set(needle.Key(key), offset, size)
 	nm.logPut(key, oldSize, size)
 	return nm.appendToIndexFile(key, offset, size)
 }
-func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
-	element, ok = nm.m.Get(Key(key))
+func (nm *NeedleMap) Get(key uint64) (element *needle.NeedleValue, ok bool) {
+	element, ok = nm.m.Get(needle.Key(key))
 	return
 }
 func (nm *NeedleMap) Delete(key uint64, offset uint32) error {
-	deletedBytes := nm.m.Delete(Key(key))
+	deletedBytes := nm.m.Delete(needle.Key(key))
 	nm.logDelete(deletedBytes)
 	return nm.appendToIndexFile(key, offset, TombstoneFileSize)
 }

weed/storage/needle_read_write.go

@@ -151,16 +151,15 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, actualSize i
 	return 0, 0, fmt.Errorf("Unsupported Version! (%d)", version)
 }
 
-func ReadNeedleBlob(r *os.File, offset int64, size uint32) (dataSlice []byte, block *Block, err error) {
+func ReadNeedleBlob(r *os.File, offset int64, size uint32) (dataSlice []byte, err error) {
 	return getBytesForFileBlock(r, offset, int(getActualSize(size)))
 }
 
 func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version) (err error) {
-	bytes, block, err := ReadNeedleBlob(r, offset, size)
+	bytes, err := ReadNeedleBlob(r, offset, size)
 	if err != nil {
 		return err
 	}
-	n.rawBlock = block
 	n.ParseNeedleHeader(bytes)
 	if n.Size != size {
 		return fmt.Errorf("File Entry Not Found. Needle %d Memory %d", n.Size, size)

weed/storage/store.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/operation"
-	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 )
 
 const (
@@ -16,8 +16,9 @@ const (
 )
 
 type MasterNodes struct {
-	nodes  []string
-	leader string
+	nodes          []string
+	leader         string
+	possibleLeader string
 }
 
 func (mn *MasterNodes) String() string {
@@ -34,6 +35,10 @@ func (mn *MasterNodes) Reset() {
 		glog.V(0).Infof("Resetting master nodes: %v", mn)
 	}
 }
+func (mn *MasterNodes) SetPossibleLeader(possibleLeader string) {
+	// TODO try to check this leader first
+	mn.possibleLeader = possibleLeader
+}
 func (mn *MasterNodes) FindMaster() (leader string, err error) {
 	if len(mn.nodes) == 0 {
 		return "", errors.New("No master node found!")
@@ -71,7 +76,8 @@ type Store struct {
 	rack            string //optional information, overwriting master setting if exists
 	connected       bool
 	VolumeSizeLimit uint64 //read from the master
-	Client          pb.Seaweed_SendHeartbeatClient
+	Client          master_pb.Seaweed_SendHeartbeatClient
+	NeedleMapType   NeedleMapType
 }
 
 func (s *Store) String() (str string) {
@@ -80,7 +86,7 @@ func (s *Store) String() (str string) {
 }
 
 func NewStore(port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int, needleMapKind NeedleMapType) (s *Store) {
-	s = &Store{Port: port, Ip: ip, PublicUrl: publicUrl}
+	s = &Store{Port: port, Ip: ip, PublicUrl: publicUrl, NeedleMapType: needleMapKind}
 	s.Locations = make([]*DiskLocation, 0)
 	for i := 0; i < len(dirnames); i++ {
 		location := NewDiskLocation(dirnames[i], maxVolumeCounts[i])
@@ -202,8 +208,8 @@ func (s *Store) SetRack(rack string) {
 	s.rack = rack
 }
 
-func (s *Store) CollectHeartbeat() *pb.Heartbeat {
-	var volumeMessages []*pb.VolumeInformationMessage
+func (s *Store) CollectHeartbeat() *master_pb.Heartbeat {
+	var volumeMessages []*master_pb.VolumeInformationMessage
 	maxVolumeCount := 0
 	var maxFileKey uint64
 	for _, location := range s.Locations {
@@ -214,7 +220,7 @@ func (s *Store) CollectHeartbeat() *pb.Heartbeat {
 				maxFileKey = v.nm.MaxFileKey()
 			}
 			if !v.expired(s.VolumeSizeLimit) {
-				volumeMessage := &pb.VolumeInformationMessage{
+				volumeMessage := &master_pb.VolumeInformationMessage{
 					Id:               uint32(k),
 					Size:             uint64(v.Size()),
 					Collection:       v.Collection,
@@ -239,7 +245,7 @@ func (s *Store) CollectHeartbeat() *pb.Heartbeat {
 		location.Unlock()
 	}
 
-	return &pb.Heartbeat{
+	return &master_pb.Heartbeat{
 		Ip:             s.Ip,
 		Port:           uint32(s.Port),
 		PublicUrl:      s.PublicUrl,
@@ -256,6 +262,7 @@ func (s *Store) Close() {
 		location.Close()
 	}
 }
+
 func (s *Store) Write(i VolumeId, n *Needle) (size uint32, err error) {
 	if v := s.findVolume(i); v != nil {
 		if v.readOnly {
@@ -268,31 +275,33 @@ func (s *Store) Write(i VolumeId, n *Needle) (size uint32, err error) {
 		} else {
 			err = fmt.Errorf("Volume Size Limit %d Exceeded! Current size is %d", s.VolumeSizeLimit, v.ContentSize())
 		}
-		if s.VolumeSizeLimit < v.ContentSize()+3*uint64(size) {
-			glog.V(0).Infoln("volume", i, "size", v.ContentSize(), "will exceed limit", s.VolumeSizeLimit)
-			if s.Client != nil {
-				if e := s.Client.Send(s.CollectHeartbeat()); e != nil {
-					glog.V(0).Infoln("error when reporting size:", e)
-				}
-			}
-		}
 		return
 	}
 	glog.V(0).Infoln("volume", i, "not found!")
 	err = fmt.Errorf("Volume %d not found!", i)
 	return
 }
+
+func (s *Store) updateMaster() {
+	if s.Client != nil {
+		if e := s.Client.Send(s.CollectHeartbeat()); e != nil {
+			glog.V(0).Infoln("error when reporting size:", e)
+		}
+	}
+}
+
 func (s *Store) Delete(i VolumeId, n *Needle) (uint32, error) {
 	if v := s.findVolume(i); v != nil && !v.readOnly {
 		return v.deleteNeedle(n)
 	}
 	return 0, nil
 }
+
 func (s *Store) ReadVolumeNeedle(i VolumeId, n *Needle) (int, error) {
 	if v := s.findVolume(i); v != nil {
 		return v.readNeedle(n)
 	}
-	return 0, fmt.Errorf("Volume %v not found!", i)
+	return 0, fmt.Errorf("Volume %d not found!", i)
 }
 func (s *Store) GetVolume(i VolumeId) *Volume {
 	return s.findVolume(i)
@@ -302,3 +311,36 @@ func (s *Store) HasVolume(i VolumeId) bool {
 	v := s.findVolume(i)
 	return v != nil
 }
+
+func (s *Store) MountVolume(i VolumeId) error {
+	for _, location := range s.Locations {
+		if found := location.LoadVolume(i, s.NeedleMapType); found == true {
+			s.updateMaster()
+			return nil
+		}
+	}
+
+	return fmt.Errorf("Volume %d not found on disk", i)
+}
+
+func (s *Store) UnmountVolume(i VolumeId) error {
+	for _, location := range s.Locations {
+		if err := location.UnloadVolume(i); err == nil {
+			s.updateMaster()
+			return nil
+		}
+	}
+
+	return fmt.Errorf("Volume %d not found on disk", i)
+}
+
+func (s *Store) DeleteVolume(i VolumeId) error {
+	for _, location := range s.Locations {
+		if error := location.deleteVolumeById(i); error == nil {
+			s.updateMaster()
+			return nil
+		}
+	}
+
+	return fmt.Errorf("Volume %d not found on disk", i)
+}

weed/storage/store_vacuum.go

@@ -22,13 +22,13 @@ func (s *Store) CheckCompactVolume(volumeIdString string, garbageThresholdString
 	}
 	return fmt.Errorf("volume id %d is not found during check compact", vid), false
 }
-func (s *Store) CompactVolume(volumeIdString string) error {
+func (s *Store) CompactVolume(volumeIdString string, preallocate int64) error {
 	vid, err := NewVolumeId(volumeIdString)
 	if err != nil {
 		return fmt.Errorf("Volume Id %s is not a valid unsigned integer", volumeIdString)
 	}
 	if v := s.findVolume(vid); v != nil {
-		return v.Compact()
+		return v.Compact(preallocate)
 	}
 	return fmt.Errorf("volume id %d is not found during compact", vid)
 }
@@ -42,3 +42,13 @@ func (s *Store) CommitCompactVolume(volumeIdString string) error {
 	}
 	return fmt.Errorf("volume id %d is not found during commit compact", vid)
 }
+func (s *Store) CommitCleanupVolume(volumeIdString string) error {
+	vid, err := NewVolumeId(volumeIdString)
+	if err != nil {
+		return fmt.Errorf("Volume Id %s is not a valid unsigned integer", volumeIdString)
+	}
+	if v := s.findVolume(vid); v != nil {
+		return v.cleanupCompact()
+	}
+	return fmt.Errorf("volume id %d is not found during cleaning up", vid)
+}

weed/storage/volume.go

@@ -15,7 +15,6 @@ type Volume struct {
 	dir           string
 	Collection    string
 	dataFile      *os.File
-	dataFileSize  int64
 	nm            NeedleMapper
 	needleMapKind NeedleMapType
 	readOnly      bool
@@ -62,7 +61,7 @@ func (v *Volume) Size() int64 {
 		return stat.Size()
 	}
 	glog.V(0).Infof("Failed to read file size %s %v", v.dataFile.Name(), e)
-	return -1
+	return 0 // -1 causes integer overflow and the volume to become unwritable.
 }
 
 // Close cleanly shuts down this volume

weed/storage/volume_checking.go

@@ -12,28 +12,28 @@ func getActualSize(size uint32) int64 {
 	return NeedleHeaderSize + int64(size) + NeedleChecksumSize + int64(padding)
 }
 
-func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) (int64, error) {
+func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) (error) {
 	var indexSize int64
 	var e error
 	if indexSize, e = verifyIndexFileIntegrity(indexFile); e != nil {
-		return 0, fmt.Errorf("verifyIndexFileIntegrity %s failed: %v", indexFile.Name(), e)
+		return fmt.Errorf("verifyIndexFileIntegrity %s failed: %v", indexFile.Name(), e)
 	}
 	if indexSize == 0 {
-		return int64(SuperBlockSize), nil
+		return nil
 	}
 	var lastIdxEntry []byte
 	if lastIdxEntry, e = readIndexEntryAtOffset(indexFile, indexSize-NeedleIndexSize); e != nil {
-		return 0, fmt.Errorf("readLastIndexEntry %s failed: %v", indexFile.Name(), e)
+		return fmt.Errorf("readLastIndexEntry %s failed: %v", indexFile.Name(), e)
 	}
 	key, offset, size := idxFileEntry(lastIdxEntry)
 	if offset == 0 || size == TombstoneFileSize {
-		return 0, nil
+		return nil
 	}
 	if e = verifyNeedleIntegrity(v.dataFile, v.Version(), int64(offset)*NeedlePaddingSize, key, size); e != nil {
-		return 0, fmt.Errorf("verifyNeedleIntegrity %s failed: %v", indexFile.Name(), e)
+		return fmt.Errorf("verifyNeedleIntegrity %s failed: %v", indexFile.Name(), e)
 	}
 
-	return int64(offset)*int64(NeedlePaddingSize) + getActualSize(size), nil
+	return nil
 }
 
 func verifyIndexFileIntegrity(indexFile *os.File) (indexSize int64, err error) {

weed/storage/volume_info.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"sort"
 
-	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 )
 
 type VolumeInfo struct {
@@ -20,7 +20,7 @@ type VolumeInfo struct {
 	ReadOnly         bool
 }
 
-func NewVolumeInfo(m *pb.VolumeInformationMessage) (vi VolumeInfo, err error) {
+func NewVolumeInfo(m *master_pb.VolumeInformationMessage) (vi VolumeInfo, err error) {
 	vi = VolumeInfo{
 		Id:               VolumeId(m.Id),
 		Size:             m.Size,

weed/storage/volume_loading.go

@@ -64,26 +64,31 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind
 				return fmt.Errorf("cannot write Volume Index %s.idx: %v", fileName, e)
 			}
 		}
-		if v.dataFileSize, e = CheckVolumeDataIntegrity(v, indexFile); e != nil {
+		if e = CheckVolumeDataIntegrity(v, indexFile); e != nil {
 			v.readOnly = true
 			glog.V(0).Infof("volumeDataIntegrityChecking failed %v", e)
 		}
 		switch needleMapKind {
 		case NeedleMapInMemory:
-			glog.V(0).Infoln("loading index file", fileName+".idx", "readonly", v.readOnly)
-			if v.nm, e = LoadNeedleMap(indexFile); e != nil {
-				glog.V(0).Infof("loading index %s error: %v", fileName+".idx", e)
+			glog.V(0).Infoln("loading index", fileName+".idx", "to memory readonly", v.readOnly)
+			if v.nm, e = LoadCompactNeedleMap(indexFile); e != nil {
+				glog.V(0).Infof("loading index %s to memory error: %v", fileName+".idx", e)
 			}
 		case NeedleMapLevelDb:
-			glog.V(0).Infoln("loading leveldb file", fileName+".ldb")
+			glog.V(0).Infoln("loading leveldb", fileName+".ldb")
 			if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile); e != nil {
 				glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e)
 			}
 		case NeedleMapBoltDb:
-			glog.V(0).Infoln("loading boltdb file", fileName+".bdb")
+			glog.V(0).Infoln("loading boltdb", fileName+".bdb")
 			if v.nm, e = NewBoltDbNeedleMap(fileName+".bdb", indexFile); e != nil {
 				glog.V(0).Infof("loading boltdb %s error: %v", fileName+".bdb", e)
 			}
+		case NeedleMapBtree:
+			glog.V(0).Infoln("loading index", fileName+".idx", "to btree readonly", v.readOnly)
+			if v.nm, e = LoadBtreeNeedleMap(indexFile); e != nil {
+				glog.V(0).Infof("loading index %s to btree error: %v", fileName+".idx", e)
+			}
 		}
 	}
 

weed/storage/volume_read_write.go

@@ -25,7 +25,6 @@ func (v *Volume) isFileUnchanged(n *Needle) bool {
 			glog.V(0).Infof("Failed to check updated file %v", err)
 			return false
 		}
-		defer oldNeedle.ReleaseMemory()
 		if oldNeedle.Checksum == n.Checksum && bytes.Equal(oldNeedle.Data, n.Data) {
 			n.DataSize = oldNeedle.DataSize
 			return true
@@ -60,8 +59,6 @@ func (v *Volume) AppendBlob(b []byte) (offset int64, err error) {
 	if offset, err = v.dataFile.Seek(0, 2); err != nil {
 		glog.V(0).Infof("failed to seek the end of file: %v", err)
 		return
-	} else if offset != int64(v.dataFileSize) {
-		glog.V(0).Infof("dataFileSize %d != actual data file size: %d", v.dataFileSize, offset)
 	}
 	//ensure file writing starting from aligned positions
 	if offset%NeedlePaddingSize != 0 {
@@ -69,12 +66,9 @@ func (v *Volume) AppendBlob(b []byte) (offset int64, err error) {
 		if offset, err = v.dataFile.Seek(offset, 0); err != nil {
 			glog.V(0).Infof("failed to align in datafile %s: %v", v.dataFile.Name(), err)
 			return
-		} else if offset != int64(v.dataFileSize) {
-			glog.V(0).Infof("dataFileSize %d != actual data file size: %d", v.dataFileSize, offset)
 		}
 	}
 	_, err = v.dataFile.Write(b)
-	v.dataFileSize += int64(len(b))
 	return
 }
 
@@ -91,12 +85,10 @@ func (v *Volume) writeNeedle(n *Needle) (size uint32, err error) {
 		glog.V(4).Infof("needle is unchanged!")
 		return
 	}
-	var offset, actualSize int64
+	var offset int64
 	if offset, err = v.dataFile.Seek(0, 2); err != nil {
 		glog.V(0).Infof("failed to seek the end of file: %v", err)
 		return
-	} else if offset != int64(v.dataFileSize) {
-		glog.V(0).Infof("dataFileSize %d != actual data file size: %d", v.dataFileSize, offset)
 	}
 
 	//ensure file writing starting from aligned positions
@@ -108,13 +100,12 @@ func (v *Volume) writeNeedle(n *Needle) (size uint32, err error) {
 		}
 	}
 
-	if size, actualSize, err = n.Append(v.dataFile, v.Version()); err != nil {
+	if size, _, err = n.Append(v.dataFile, v.Version()); err != nil {
 		if e := v.dataFile.Truncate(offset); e != nil {
 			err = fmt.Errorf("%s\ncannot truncate %s: %v", err, v.dataFile.Name(), e)
 		}
 		return
 	}
-	v.dataFileSize += actualSize
 
 	nv, ok := v.nm.Get(n.Id)
 	if !ok || int64(nv.Offset)*NeedlePaddingSize < offset {
@@ -139,18 +130,15 @@ func (v *Volume) deleteNeedle(n *Needle) (uint32, error) {
 	//fmt.Println("key", n.Id, "volume offset", nv.Offset, "data_size", n.Size, "cached size", nv.Size)
 	if ok && nv.Size != TombstoneFileSize {
 		size := nv.Size
-		// println("adding tombstone", n.Id, "at offset", v.dataFileSize)
-		if err := v.nm.Delete(n.Id, uint32(v.dataFileSize/NeedlePaddingSize)); err != nil {
+		offset, err := v.dataFile.Seek(0, 2)
+		if err != nil {
 			return size, err
 		}
-		if offset, err := v.dataFile.Seek(0, 2); err != nil {
+		if err := v.nm.Delete(n.Id, uint32(offset/NeedlePaddingSize)); err != nil {
 			return size, err
-		} else if offset != int64(v.dataFileSize) {
-			glog.V(0).Infof("dataFileSize %d != actual data file size: %d, deleteMarker: %d", v.dataFileSize, offset, getActualSize(0))
 		}
 		n.Data = nil
-		_, actualSize, err := n.Append(v.dataFile, v.Version())
-		v.dataFileSize += actualSize
+		_, _, err = n.Append(v.dataFile, v.Version())
 		return size, err
 	}
 	return 0, nil
@@ -183,7 +171,6 @@ func (v *Volume) readNeedle(n *Needle) (int, error) {
 	if uint64(time.Now().Unix()) < n.LastModified+uint64(ttlMinutes*60) {
 		return bytesRead, nil
 	}
-	n.ReleaseMemory()
 	return -1, errors.New("Not Found")
 }
 

weed/storage/volume_super_block.go

@@ -56,7 +56,6 @@ func (v *Volume) maybeWriteSuperBlock() error {
 				}
 			}
 		}
-		v.dataFileSize = SuperBlockSize
 	}
 	return e
 }

weed/storage/volume_sync.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -48,13 +49,13 @@ optimized more later).
 func (v *Volume) Synchronize(volumeServer string) (err error) {
 	var lastCompactRevision uint16 = 0
 	var compactRevision uint16 = 0
-	var masterMap CompactMap
+	var masterMap *needle.CompactMap
 	for i := 0; i < 3; i++ {
 		if masterMap, _, compactRevision, err = fetchVolumeFileEntries(volumeServer, v.Id); err != nil {
 			return fmt.Errorf("Failed to sync volume %d entries with %s: %v", v.Id, volumeServer, err)
 		}
 		if lastCompactRevision != compactRevision && lastCompactRevision != 0 {
-			if err = v.Compact(); err != nil {
+			if err = v.Compact(0); err != nil {
 				return fmt.Errorf("Compact Volume before synchronizing %v", err)
 			}
 			if err = v.commitCompact(); err != nil {
@@ -69,7 +70,7 @@ func (v *Volume) Synchronize(volumeServer string) (err error) {
 	return
 }
 
-type ByOffset []NeedleValue
+type ByOffset []needle.NeedleValue
 
 func (a ByOffset) Len() int           { return len(a) }
 func (a ByOffset) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
@@ -77,18 +78,18 @@ func (a ByOffset) Less(i, j int) bool { return a[i].Offset < a[j].Offset }
 
 // trySynchronizing sync with remote volume server incrementally by
 // make up the local and remote delta.
-func (v *Volume) trySynchronizing(volumeServer string, masterMap CompactMap, compactRevision uint16) error {
+func (v *Volume) trySynchronizing(volumeServer string, masterMap *needle.CompactMap, compactRevision uint16) error {
 	slaveIdxFile, err := os.Open(v.nm.IndexFileName())
 	if err != nil {
 		return fmt.Errorf("Open volume %d index file: %v", v.Id, err)
 	}
 	defer slaveIdxFile.Close()
-	slaveMap, err := LoadNeedleMap(slaveIdxFile)
+	slaveMap, err := LoadBtreeNeedleMap(slaveIdxFile)
 	if err != nil {
 		return fmt.Errorf("Load volume %d index file: %v", v.Id, err)
 	}
-	var delta []NeedleValue
-	if err := masterMap.Visit(func(needleValue NeedleValue) error {
+	var delta []needle.NeedleValue
+	if err := masterMap.Visit(func(needleValue needle.NeedleValue) error {
 		if needleValue.Key == 0 {
 			return nil
 		}
@@ -100,7 +101,7 @@ func (v *Volume) trySynchronizing(volumeServer string, masterMap CompactMap, com
 	}); err != nil {
 		return fmt.Errorf("Add master entry: %v", err)
 	}
-	if err := slaveMap.m.Visit(func(needleValue NeedleValue) error {
+	if err := slaveMap.m.Visit(func(needleValue needle.NeedleValue) error {
 		if needleValue.Key == 0 {
 			return nil
 		}
@@ -137,8 +138,8 @@ func (v *Volume) trySynchronizing(volumeServer string, masterMap CompactMap, com
 	return nil
 }
 
-func fetchVolumeFileEntries(volumeServer string, vid VolumeId) (m CompactMap, lastOffset uint64, compactRevision uint16, err error) {
-	m = NewCompactMap()
+func fetchVolumeFileEntries(volumeServer string, vid VolumeId) (m *needle.CompactMap, lastOffset uint64, compactRevision uint16, err error) {
+	m = needle.NewCompactMap()
 
 	syncStatus, err := operation.GetVolumeSyncStatus(volumeServer, vid.String())
 	if err != nil {
@@ -149,9 +150,9 @@ func fetchVolumeFileEntries(volumeServer string, vid VolumeId) (m CompactMap, la
 	err = operation.GetVolumeIdxEntries(volumeServer, vid.String(), func(key uint64, offset, size uint32) {
 		// println("remote key", key, "offset", offset*NeedlePaddingSize, "size", size)
 		if offset > 0 && size != TombstoneFileSize {
-			m.Set(Key(key), offset, size)
+			m.Set(needle.Key(key), offset, size)
 		} else {
-			m.Delete(Key(key))
+			m.Delete(needle.Key(key))
 		}
 		total++
 	})
@@ -178,7 +179,7 @@ func (v *Volume) IndexFileContent() ([]byte, error) {
 }
 
 // removeNeedle removes one needle by needle key
-func (v *Volume) removeNeedle(key Key) {
+func (v *Volume) removeNeedle(key needle.Key) {
 	n := new(Needle)
 	n.Id = uint64(key)
 	v.deleteNeedle(n)
@@ -188,7 +189,7 @@ func (v *Volume) removeNeedle(key Key) {
 // The compact revision is checked first in case the remote volume
 // is compacted and the offset is invalid any more.
 func (v *Volume) fetchNeedle(volumeDataContentHandlerUrl string,
-	needleValue NeedleValue, compactRevision uint16) error {
+	needleValue needle.NeedleValue, compactRevision uint16) error {
 	// add master file entry to local data file
 	values := make(url.Values)
 	values.Add("revision", strconv.Itoa(int(compactRevision)))

weed/storage/volume_vacuum.go

@@ -13,7 +13,7 @@ func (v *Volume) garbageLevel() float64 {
 	return float64(v.nm.DeletedSize()) / float64(v.ContentSize())
 }
 
-func (v *Volume) Compact() error {
+func (v *Volume) Compact(preallocate int64) error {
 	glog.V(3).Infof("Compacting ...")
 	//no need to lock for copy on write
 	//v.accessLock.Lock()
@@ -24,7 +24,7 @@ func (v *Volume) Compact() error {
 	v.lastCompactIndexOffset = v.nm.IndexFileSize()
 	v.lastCompactRevision = v.SuperBlock.CompactRevision
 	glog.V(3).Infof("creating copies for volume %d ,last offset %d...", v.Id, v.lastCompactIndexOffset)
-	return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx", v.dataFileSize)
+	return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx", preallocate)
 }
 
 func (v *Volume) Compact2() error {
@@ -72,6 +72,20 @@ func (v *Volume) commitCompact() error {
 	return nil
 }
 
+func (v *Volume) cleanupCompact() error {
+	glog.V(0).Infof("Cleaning up vacuuming...")
+
+	e1 := os.Remove(v.FileName() + ".cpd")
+	e2 := os.Remove(v.FileName() + ".cpx")
+	if e1 != nil {
+		return e1
+	}
+	if e2 != nil {
+		return e2
+	}
+	return nil
+}
+
 func fetchCompactRevisionFromDatFile(file *os.File) (compactRevision uint16, err error) {
 	if _, err = file.Seek(0, 0); err != nil {
 		return 0, fmt.Errorf("cannot seek to the beginning of %s: %v", file.Name(), err)
@@ -178,7 +192,7 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI
 			if incre_idx_entry.offset != 0 && incre_idx_entry.size != 0 {
 				//even the needle cache in memory is hit, the need_bytes is correct
 				var needle_bytes []byte
-				needle_bytes, _, err = ReadNeedleBlob(oldDatFile, int64(incre_idx_entry.offset)*NeedlePaddingSize, incre_idx_entry.size)
+				needle_bytes, err = ReadNeedleBlob(oldDatFile, int64(incre_idx_entry.offset)*NeedlePaddingSize, incre_idx_entry.size)
 				if err != nil {
 					return
 				}
@@ -221,7 +235,7 @@ func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string, prealloca
 	}
 	defer idx.Close()
 
-	nm := NewNeedleMap(idx)
+	nm := NewBtreeNeedleMap(idx)
 	new_offset := int64(SuperBlockSize)
 
 	now := uint64(time.Now().Unix())
@@ -272,7 +286,7 @@ func (v *Volume) copyDataBasedOnIndexFile(dstName, idxName string) (err error) {
 	}
 	defer oldIndexFile.Close()
 
-	nm := NewNeedleMap(idx)
+	nm := NewBtreeNeedleMap(idx)
 	now := uint64(time.Now().Unix())
 
 	v.SuperBlock.CompactRevision++
@@ -291,7 +305,6 @@ func (v *Volume) copyDataBasedOnIndexFile(dstName, idxName string) (err error) {
 
 		n := new(Needle)
 		n.ReadData(v.dataFile, int64(offset)*NeedlePaddingSize, size, v.Version())
-		defer n.ReleaseMemory()
 
 		if n.HasTtl() && now >= n.LastModified+uint64(v.Ttl.Minutes()*60) {
 			return nil

weed/topology/cluster_commands.go

@@ -25,7 +25,7 @@ func (c *MaxVolumeIdCommand) Apply(server raft.Server) (interface{}, error) {
 	before := topo.GetMaxVolumeId()
 	topo.UpAdjustMaxVolumeId(c.MaxVolumeId)
 
-	glog.V(4).Infoln("max volume id", before, "==>", topo.GetMaxVolumeId())
+	glog.V(0).Infoln("max volume id", before, "==>", topo.GetMaxVolumeId())
 
 	return nil, nil
 }

weed/topology/configuration.go

@@ -25,20 +25,6 @@ type Configuration struct {
 	ip2location map[string]loc
 }
 
-func NewConfiguration(b []byte) (*Configuration, error) {
-	c := &Configuration{}
-	err := xml.Unmarshal(b, c)
-	c.ip2location = make(map[string]loc)
-	for _, dc := range c.Topo.DataCenters {
-		for _, rack := range dc.Racks {
-			for _, ip := range rack.Ips {
-				c.ip2location[ip] = loc{dcName: dc.Name, rackName: rack.Name}
-			}
-		}
-	}
-	return c, err
-}
-
 func (c *Configuration) String() string {
 	if b, e := xml.MarshalIndent(c, "  ", "  "); e == nil {
 		return string(b)

weed/topology/configuration_test.go

@@ -1,42 +0,0 @@
-package topology
-
-import (
-	"fmt"
-	"testing"
-)
-
-func TestLoadConfiguration(t *testing.T) {
-
-	confContent := `
-
-<?xml version="1.0" encoding="UTF-8" ?>
-<Configuration>
-  <Topology>
-    <DataCenter name="dc1">
-      <Rack name="rack1">
-        <Ip>192.168.1.1</Ip>
-      </Rack>
-    </DataCenter>
-    <DataCenter name="dc2">
-      <Rack name="rack1">
-        <Ip>192.168.1.2</Ip>
-      </Rack>
-      <Rack name="rack2">
-        <Ip>192.168.1.3</Ip>
-        <Ip>192.168.1.4</Ip>
-      </Rack>
-    </DataCenter>
-  </Topology>
-</Configuration>
-`
-	c, err := NewConfiguration([]byte(confContent))
-
-	fmt.Printf("%s\n", c)
-	if err != nil {
-		t.Fatalf("unmarshal error:%v", err)
-	}
-
-	if len(c.Topo.DataCenters) <= 0 || c.Topo.DataCenters[0].Name != "dc1" {
-		t.Fatalf("unmarshal error:%s", c)
-	}
-}