2.30

fix https://github.com/chrislusf/seaweedfs/issues/1870

.github/ISSUE_TEMPLATE/bug_report.md

@@ -9,6 +9,7 @@ assignees: ''
 
 Sponsors SeaweedFS via Patreon https://www.patreon.com/seaweedfs
 Report issues here. Ask questions here https://stackoverflow.com/questions/tagged/seaweedfs
+Please ask questions in https://github.com/chrislusf/seaweedfs/discussions
 
 example of a good issue report:
 https://github.com/chrislusf/seaweedfs/issues/1005

.github/workflows/cleanup.yml

@@ -0,0 +1,22 @@
+name: Cleanup
+
+on:
+  push:
+    branches: [ master ]
+
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Delete old release assets
+        uses: mknejp/delete-release-assets@v1
+        with:
+          token: ${{ github.token }}
+          tag: dev
+          fail-if-no-assets: false
+          assets: |
+            weed-*

.github/workflows/go.yml

@@ -0,0 +1,37 @@
+name: Go
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: ^1.13
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        cd weed; go get -v -t -d ./...
+        if [ -f Gopkg.toml ]; then
+            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+            dep ensure
+        fi
+
+    - name: Build
+      run: cd weed; go build -v .
+
+    - name: Test
+      run: cd weed; go test -v ./...

.github/workflows/release.yml

@@ -0,0 +1,62 @@
+name: Release
+
+on:
+  push:
+    branches: [ master ]
+
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [linux, windows, darwin, freebsd ]
+        goarch: [amd64, arm]
+        exclude:
+          - goarch: arm
+            goos: darwin
+          - goarch: arm
+            goos: windows
+
+    steps:
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Wait for the deletion
+        uses: jakejarvis/wait-action@master
+        with:
+          time: '30s'
+
+      - name: Set BUILD_TIME env
+        run: echo BUILD_TIME=$(date -u +%Y-%m-%d-%H-%M) >> ${GITHUB_ENV}
+
+      - name: Go Release Binaries
+        uses: wangyoucao577/go-release-action@v1.14
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          release_tag: dev
+          overwrite: true
+          build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed-large-disk
+          asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
+
+      - name: Go Release Binaries
+        uses: wangyoucao577/go-release-action@v1.14
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          release_tag: dev
+          overwrite: true
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed-
+          asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"

.travis.yml

@@ -1,9 +1,8 @@
 sudo: false
 language: go
 go:
-  - 1.12.x
-  - 1.13.x
-  - 1.14.x
+  - 1.15.x
+  - 1.16.x
 
 before_install:
   - export PATH=/home/travis/gopath/bin:$PATH
@@ -45,4 +44,4 @@ deploy:
   on:
     tags: true
     repo: chrislusf/seaweedfs
-    go: 1.14.x
+    go: 1.16.x

Makefile

@@ -8,11 +8,14 @@ appname := weed
 
 sources := $(wildcard *.go)
 
-build = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -ldflags "-extldflags -static" -o build/$(appname)$(3) $(SOURCE_DIR)
+COMMIT ?= $(shell git rev-parse --short HEAD)
+LDFLAGS ?= -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${COMMIT}
+
+build = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -ldflags "-extldflags -static $(LDFLAGS)" -o build/$(appname)$(3) $(SOURCE_DIR)
 tar = cd build && tar -cvzf $(1)_$(2).tar.gz $(appname)$(3) && rm $(appname)$(3)
 zip = cd build && zip $(1)_$(2).zip $(appname)$(3) && rm $(appname)$(3)
 
-build_large = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -tags 5BytesOffset -ldflags "-extldflags -static" -o build/$(appname)$(3) $(SOURCE_DIR)
+build_large = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -tags 5BytesOffset -ldflags "-extldflags -static $(LDFLAGS)" -o build/$(appname)$(3) $(SOURCE_DIR)
 tar_large = cd build && tar -cvzf $(1)_$(2)_large_disk.tar.gz $(appname)$(3) && rm $(appname)$(3)
 zip_large = cd build && zip $(1)_$(2)_large_disk.zip $(appname)$(3) && rm $(appname)$(3)
 
@@ -31,13 +34,16 @@ deps:
 	rm -rf /home/travis/gopath/src/go.etcd.io/etcd/vendor/golang.org/x/net/trace
 
 build: deps
-	go build $(GO_FLAGS) -o $(BINARY) $(SOURCE_DIR)
+	go build $(GO_FLAGS) -ldflags "$(LDFLAGS)" -o $(BINARY) $(SOURCE_DIR)
+
+install: deps
+	go install $(GO_FLAGS) -ldflags "$(LDFLAGS)" $(SOURCE_DIR)
 
 linux: deps
 	mkdir -p linux
-	GOOS=linux GOARCH=amd64 go build $(GO_FLAGS) -o linux/$(BINARY) $(SOURCE_DIR)
+	GOOS=linux GOARCH=amd64 go build $(GO_FLAGS) -ldflags "$(LDFLAGS)" -o linux/$(BINARY) $(SOURCE_DIR)
 
-release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_build 5_byte_darwin_build 5_byte_windows_build
+release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_build 5_byte_arm64_build 5_byte_darwin_build 5_byte_windows_build
 
 ##### LINUX BUILDS #####
 5_byte_linux_build:
@@ -52,6 +58,14 @@ release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_buil
 	$(call build_large,windows,amd64,.exe)
 	$(call zip_large,windows,amd64,.exe)
 
+5_byte_arm_build: $(sources)
+	$(call build_large,linux,arm,)
+	$(call tar_large,linux,arm)
+
+5_byte_arm64_build: $(sources)
+	$(call build_large,linux,arm64,)
+	$(call tar_large,linux,arm64)
+
 linux_build: build/linux_arm.tar.gz build/linux_arm64.tar.gz build/linux_386.tar.gz build/linux_amd64.tar.gz
 
 build/linux_386.tar.gz: $(sources)

README.md

@@ -1,14 +1,17 @@
 # SeaweedFS
 
 
+[![Slack](https://img.shields.io/badge/slack-purple)](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
+[![Twitter](https://img.shields.io/twitter/follow/seaweedfs.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=seaweedfs)
 [![Build Status](https://travis-ci.org/chrislusf/seaweedfs.svg?branch=master)](https://travis-ci.org/chrislusf/seaweedfs)
 [![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed)
 [![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki)
-[![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs.svg?maxAge=604800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
+[![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
+
 
 ![SeaweedFS Logo](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/seaweedfs.png)
 
-<h2 align="center">Supporting SeaweedFS</h2>
+<h2 align="center"><a href="https://www.patreon.com/seaweedfs">Sponsor SeaweedFS via Patreon</a></h2>
 
 SeaweedFS is an independent Apache-licensed open source project with its ongoing development made
 possible entirely thanks to the support of these awesome [backers](https://github.com/chrislusf/seaweedfs/blob/master/backers.md).
@@ -17,8 +20,6 @@ If you'd like to grow SeaweedFS even stronger, please consider joining our
 
 Your support will be really appreciated by me and other supporters!
 
-<h3 align="center"><a href="https://www.patreon.com/seaweedfs">Sponsor SeaweedFS via Patreon</a></h3>
-
 <!--
 <h4 align="center">Platinum</h4>
 
@@ -27,24 +28,12 @@ Your support will be really appreciated by me and other supporters!
     Add your name or icon here
   </a>
 </p>
-
-<h4 align="center">Gold</h4>
-
-<table>
-  <tbody>
-    <tr>
-      <td align="center" valign="middle">
-        <a href="" target="_blank">
-          Add your name or icon here
-        </a>
-      </td>
-    </tr>
-    <tr></tr>
-  </tbody>
-</table>
-
 -->
 
+
+### Gold Sponsors
+![shuguang](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/shuguang.png)
+
 ---
 
 
@@ -58,11 +47,12 @@ Your support will be really appreciated by me and other supporters!
 Table of Contents
 =================
 
+* [Quick Start](#quick-start)
 * [Introduction](#introduction)
 * [Features](#features)
     * [Additional Features](#additional-features)
     * [Filer Features](#filer-features)
-* [Example Usage](#example-usage)
+* [Example: Using Seaweed Object Store](#example-Using-Seaweed-Object-Store)
 * [Architecture](#architecture)
 * [Compared to Other File Systems](#compared-to-other-file-systems)
     * [Compared to HDFS](#compared-to-hdfs)
@@ -75,6 +65,13 @@ Table of Contents
 * [Benchmark](#Benchmark)
 * [License](#license)
 
+
+## Quick Start ##
+* Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
+* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
+
+Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thoudsands of machines. That is it!
+
 ## Introduction ##
 
 SeaweedFS is a simple and highly scalable distributed file system. There are two objectives:
@@ -82,15 +79,34 @@ SeaweedFS is a simple and highly scalable distributed file system. There are two
 1. to store billions of files!
 2. to serve the files fast!
 
-SeaweedFS started as an Object Store to handle small files efficiently. Instead of managing all file metadata in a central master, the central master only manages file volumes, and it lets these volume servers manage files and their metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers, allowing faster file access (O(1), usually just one disk read operation).
+SeaweedFS started as an Object Store to handle small files efficiently. 
+Instead of managing all file metadata in a central master, 
+the central master only manages volumes on volume servers, 
+and these volume servers manage files and their metadata. 
+This relieves concurrency pressure from the central master and spreads file metadata into volume servers, 
+allowing faster file access (O(1), usually just one disk read operation).
 
-SeaweedFS can transparently integrate with the cloud. With hot data on local cluster, and warm data on the cloud with O(1) access time, SeaweedFS can achieve both fast local access time and elastic cloud storage capacity, without any client side changes.
+SeaweedFS can transparently integrate with the cloud. 
+With hot data on local cluster, and warm data on the cloud with O(1) access time, 
+SeaweedFS can achieve both fast local access time and elastic cloud storage capacity. 
+What's more, the cloud storage access API cost is minimized. 
+Faster and Cheaper than direct cloud storage!
+Signup for future managed SeaweedFS cluster offering at "seaweedfilesystem at gmail dot com".
 
-There is only 40 bytes of disk storage overhead for each file's metadata. It is so simple with O(1) disk reads that you are welcome to challenge the performance with your actual use cases.
+There is only 40 bytes of disk storage overhead for each file's metadata. 
+It is so simple with O(1) disk reads that you are welcome to challenge the performance with your actual use cases.
 
-SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf). Also, SeaweedFS implements erasure coding with ideas from [f4: Facebook’s Warm BLOB Storage System](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-muralidhar.pdf)
+SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf). 
+Also, SeaweedFS implements erasure coding with ideas from 
+[f4: Facebook’s Warm BLOB Storage System](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-muralidhar.pdf), and has a lot of similarities with [Facebook’s Tectonic Filesystem](https://www.usenix.org/system/files/fast21-pan.pdf)
 
-On top of the object store, optional [Filer] can support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql, Postgres, Mongodb, Redis, Etcd, Cassandra, LevelDB, MemSql, TiDB, TiKV, CockroachDB, etc.
+On top of the object store, optional [Filer] can support directories and POSIX attributes. 
+Filer is a separate linearly-scalable stateless server with customizable metadata stores, 
+e.g., MySql, Postgres, Redis, Cassandra, HBase, Mongodb, Elastic Search, LevelDB, RocksDB, MemSql, TiDB, Etcd, CockroachDB, etc.
+
+For any distributed key value stores, the large values can be offloaded to SeaweedFS. 
+With the fast access speed and linearly scalable capacity, 
+SeaweedFS can work as a distributed [Key-Large-Value store][KeyLargeValueStore].
 
 [Back to TOC](#table-of-contents)
 
@@ -101,41 +117,55 @@ On top of the object store, optional [Filer] can support directories and POSIX a
 * Automatic compaction to reclaim disk space after deletion or update.
 * [Automatic entry TTL expiration][VolumeServerTTL].
 * Any server with some disk spaces can add to the total storage space.
-* Adding/Removing servers does **not** cause any data re-balancing.
+* Adding/Removing servers does **not** cause any data re-balancing unless triggered by admin commands.
 * Optional picture resizing.
 * Support ETag, Accept-Range, Last-Modified, etc.
 * Support in-memory/leveldb/readonly mode tuning for memory/performance balance.
 * Support rebalancing the writable and readonly volumes.
+* [Customizable Multiple Storage Tiers][TieredStorage]: Customizable storage disk types to balance performance and cost.
 * [Transparent cloud integration][CloudTier]: unlimited capacity via tiered cloud storage for warm data.
 * [Erasure Coding for warm storage][ErasureCoding]  Rack-Aware 10.4 erasure coding reduces storage cost and increases availability.
 
 [Back to TOC](#table-of-contents)
 
 ## Filer Features ##
-* [filer server][Filer] provide "normal" directories and files via http.
-* [mount filer][Mount] to read and write files directly as a local directory via FUSE.
-* [Amazon S3 compatible API][AmazonS3API] to access files with S3 tooling.
-* [Hadoop Compatible File System][Hadoop] to access files from Hadoop/Spark/Flink/etc jobs.
-* [Async Backup To Cloud][BackupToCloud] has extremely fast local access and backups to Amazon S3, Google Cloud Storage, Azure, BackBlaze.
-* [WebDAV] access as a mapped drive on Mac and Windows, or from mobile devices.
+* [Filer server][Filer] provides "normal" directories and files via http.
+* [File TTL][FilerTTL] automatically expires file metadata and actual file data.
+* [Mount filer][Mount] reads and writes files directly as a local directory via FUSE.
+* [Filer Store Replication][FilerStoreReplication] enables HA for filer meta data stores.
+* [Active-Active Replication][ActiveActiveAsyncReplication] enables asynchronous one-way or two-way cross cluster continuous replication.
+* [Amazon S3 compatible API][AmazonS3API] accesses files with S3 tooling.
+* [Hadoop Compatible File System][Hadoop] accesses files from Hadoop/Spark/Flink/etc or even runs HBase.
+* [Async Replication To Cloud][BackupToCloud] has extremely fast local access and backups to Amazon S3, Google Cloud Storage, Azure, BackBlaze.
+* [WebDAV] accesses as a mapped drive on Mac and Windows, or from mobile devices.
 * [AES256-GCM Encrypted Storage][FilerDataEncryption] safely stores the encrypted data.
-* [File TTL][FilerTTL] automatically purge file metadata and actual file data.
+* [Super Large Files][SuperLargeFiles] stores large or super large files in tens of TB.
+
+## Kubernetes ##
+* [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs-csi-driver.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)
+* [SeaweedFS Operator](https://github.com/seaweedfs/seaweedfs-operator)
 
 [Filer]: https://github.com/chrislusf/seaweedfs/wiki/Directories-and-Files
-[Mount]: https://github.com/chrislusf/seaweedfs/wiki/Mount
+[SuperLargeFiles]: https://github.com/chrislusf/seaweedfs/wiki/Data-Structure-for-Large-Files
+[Mount]: https://github.com/chrislusf/seaweedfs/wiki/FUSE-Mount
 [AmazonS3API]: https://github.com/chrislusf/seaweedfs/wiki/Amazon-S3-API
-[BackupToCloud]: https://github.com/chrislusf/seaweedfs/wiki/Backup-to-Cloud
+[BackupToCloud]: https://github.com/chrislusf/seaweedfs/wiki/Async-Replication-to-Cloud
 [Hadoop]: https://github.com/chrislusf/seaweedfs/wiki/Hadoop-Compatible-File-System
 [WebDAV]: https://github.com/chrislusf/seaweedfs/wiki/WebDAV
 [ErasureCoding]: https://github.com/chrislusf/seaweedfs/wiki/Erasure-coding-for-warm-storage
+[TieredStorage]: https://github.com/chrislusf/seaweedfs/wiki/Tiered-Storage
 [CloudTier]: https://github.com/chrislusf/seaweedfs/wiki/Cloud-Tier
 [FilerDataEncryption]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Data-Encryption
 [FilerTTL]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Stores
 [VolumeServerTTL]: https://github.com/chrislusf/seaweedfs/wiki/Store-file-with-a-Time-To-Live
+[SeaweedFsCsiDriver]: https://github.com/seaweedfs/seaweedfs-csi-driver
+[ActiveActiveAsyncReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Active-Active-cross-cluster-continuous-synchronization
+[FilerStoreReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Store-Replication
+[KeyLargeValueStore]: https://github.com/chrislusf/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store
 
 [Back to TOC](#table-of-contents)
 
-## Example Usage ##
+## Example: Using Seaweed Object Store ##
 
 By default, the master node runs on port 9333, and the volume nodes run on port 8080.
 Let's start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. We'll use localhost as an example.
@@ -333,7 +363,7 @@ Usually hot data are fresh and warm data are old. SeaweedFS puts the newly creat
 
 With the O(1) access time, the network latency cost is kept at minimum. 
 
-If the hot~warm data is split as 20~80, with 20 servers, you can achieve storage capacity of 100 servers. That's a cost saving of 80%! Or you can repurpose the 80 servers to store new data also, and get 5X storage throughput.
+If the hot/warm data is split as 20/80, with 20 servers, you can achieve storage capacity of 100 servers. That's a cost saving of 80%! Or you can repurpose the 80 servers to store new data also, and get 5X storage throughput.
 
 [Back to TOC](#table-of-contents)
 
@@ -343,6 +373,8 @@ Most other distributed file systems seem more complicated than necessary.
 
 SeaweedFS is meant to be fast and simple, in both setup and operation. If you do not understand how it works when you reach here, we've failed! Please raise an issue with any questions or update this file with clarifications.
 
+SeaweedFS is constantly moving forward. Same with other systems. These comparisons can be outdated quickly. Please help to keep them updated.
+
 [Back to TOC](#table-of-contents)
 
 ### Compared to HDFS ###
@@ -361,16 +393,17 @@ The architectures are mostly the same. SeaweedFS aims to store and read files fa
 
 * SeaweedFS optimizes for small files, ensuring O(1) disk seek operation, and can also handle large files.
 * SeaweedFS statically assigns a volume id for a file. Locating file content becomes just a lookup of the volume id, which can be easily cached.
-* SeaweedFS Filer metadata store can be any well-known and proven data stores, e.g., Cassandra, Mongodb, Redis, Etcd, MySql, Postgres, MemSql, TiDB, CockroachDB, etc, and is easy to customized.
+* SeaweedFS Filer metadata store can be any well-known and proven data stores, e.g., Redis, Cassandra, HBase, Mongodb, Elastic Search, MySql, Postgres, MemSql, TiDB, CockroachDB, Etcd etc, and is easy to customized.
 * SeaweedFS Volume server also communicates directly with clients via HTTP, supporting range queries, direct uploads, etc.
 
-| System         | File Meta                       | File Content Read| POSIX  | REST API | Optimized for small files |
+| System         | File Metadata                   | File Content Read| POSIX  | REST API | Optimized for large number of small files |
 | -------------  | ------------------------------- | ---------------- | ------ | -------- | ------------------------- |
 | SeaweedFS      | lookup volume id, cacheable     | O(1) disk seek   |        | Yes      | Yes                       |
 | SeaweedFS Filer| Linearly Scalable, Customizable | O(1) disk seek   | FUSE   | Yes      | Yes                       |
 | GlusterFS      | hashing          |                  | FUSE, NFS          |          |                           |
 | Ceph           | hashing + rules  |                  | FUSE               | Yes      |                           |
 | MooseFS        | in memory        |                  | FUSE               |       | No                          |
+| MinIO          | separate meta file for each file  |                  |         | Yes   | No                          |
 
 [Back to TOC](#table-of-contents)
 
@@ -398,11 +431,11 @@ SeaweedFS has a centralized master group to look up free volumes, while Ceph use
 
 Same as SeaweedFS, Ceph is also based on the object store RADOS. Ceph is rather complicated with mixed reviews.
 
-Ceph uses CRUSH hashing to automatically manage the data placement. SeaweedFS places data by assigned volumes.
+Ceph uses CRUSH hashing to automatically manage the data placement, which is efficient to locate the data. But the data has to be placed according to the CRUSH algorithm. Any wrong configuration would cause data loss. SeaweedFS places data by assigning them to any writable volumes. If writes to one volume failed, just pick another volume to write. Adding more volumes are also as simple as it can be.
 
 SeaweedFS is optimized for small files. Small files are stored as one continuous block of content, with at most 8 unused bytes between files. Small file access is O(1) disk read.
 
-SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Mongodb, Redis, Etcd, Cassandra, MemSql, TiDB, CockroachCB, to manage file directories. These stores are proven, scalable, and easier to manage.
+SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Mongodb, Redis, Elastic Search, Cassandra, HBase, MemSql, TiDB, CockroachCB, Etcd, to manage file directories. These stores are proven, scalable, and easier to manage.
 
 | SeaweedFS         | comparable to Ceph | advantage |
 | -------------  | ------------- | ---------------- |
@@ -412,19 +445,31 @@ SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Mongodb, Red
 
 [Back to TOC](#table-of-contents)
 
+### Compared to MinIO ###
+
+MinIO follows AWS S3 closely and is ideal for testing for S3 API. It has good UI, policies, versionings, etc. SeaweedFS is trying to catch up here. It is also possible to put MinIO as a gateway in front of SeaweedFS later.
+
+MinIO metadata are in simple files. Each file write will incur extra writes to corresponding meta file.
+
+MinIO does not have optimization for lots of small files. The files are simply stored as is to local disks.
+Plus the extra meta file and shards for erasure coding, it only amplifies the LOSF problem.
+
+MinIO has multiple disk IO to read one file. SeaweedFS has O(1) disk reads, even for erasure coded files.
+
+MinIO has full-time erasure coding. SeaweedFS uses replication on hot data for faster speed and optionally applies erasure coding on warm data.
+
+MinIO does not have POSIX-like API support.
+
+MinIO has specific requirements on storage layout. It is not flexible to adjust capacity. In SeaweedFS, just start one volume server pointing to the master. That's all.
+
 ## Dev Plan ##
 
-More tools and documentation, on how to maintain and scale the system. For example, how to move volumes, automatically balancing data, how to grow volumes, how to check system status, etc.
-Other key features include: Erasure Encoding, JWT security.
+* More tools and documentation, on how to manage and scale the system.
+* Read and write stream data.
+* Support structured data.
 
 This is a super exciting project! And we need helpers and [support](https://www.patreon.com/seaweedfs)!
 
-BTW, We suggest run the code style check script `util/gostd` before you push your branch to remote, it will make SeaweedFS easy to review, maintain and develop:
-
-```
-$ ./util/gostd
-```
-
 [Back to TOC](#table-of-contents)
 
 ## Installation Guide ##
@@ -438,30 +483,18 @@ https://golang.org/doc/install
 make sure you set up your $GOPATH
 
 
-Step 2: also you may need to install Mercurial by following the instructions at:
-
-http://mercurial.selenic.com/downloads
-
+Step 2: checkout this repo:
+```bash
+git clone https://github.com/chrislusf/seaweedfs.git
+```
 Step 3: download, compile, and install the project by executing the following command
 
 ```bash
-go get github.com/chrislusf/seaweedfs/weed
+make install
 ```
 
 Once this is done, you will find the executable "weed" in your `$GOPATH/bin` directory
 
-Note:
-* If you got into this problem, try to `rm -Rf $GOPATH/src/go.etcd.io/etcd/vendor/golang.org/x/net/trace` and build again.
-```
-panic: /debug/requests is already registered. You may have two independent copies of golang.org/x/net/trace in your binary, trying to maintain separate state. This may involve a vendored copy of golang.org/x/net/trace.
-```
-
-Step 4: after you modify your code locally, you could start a local build by calling `go install` under
-
-```
-$GOPATH/src/github.com/chrislusf/seaweedfs/weed
-```
-
 [Back to TOC](#table-of-contents)
 
 ## Disk Related Topics ##

backers.md

@@ -1,3 +1,4 @@
+
 <h1 align="center">Sponsors &amp; Backers</h1>
 
 - [Become a backer or sponsor on Patreon](https://www.patreon.com/seaweedfs).
@@ -12,3 +13,4 @@
 
 - [ColorfulClouds Tech Co. Ltd.](https://caiyunai.com/)
 - [Haravan - Ecommerce Platform](https://www.haravan.com)
+- PeterCxy - Creator of Shelter App

docker/Dockerfile

@@ -1,15 +1,22 @@
-FROM frolvlad/alpine-glibc
+FROM alpine
 
-# Supercronic install settings
-ENV SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.1.8/supercronic-linux-amd64 \
-    SUPERCRONIC=supercronic-linux-amd64 \
-    SUPERCRONIC_SHA1SUM=be43e64c45acd6ec4fce5831e03759c89676a0ea
+ARG RELEASE=latest # 'latest' or 'dev'
 
-# Install SeaweedFS and Supercronic ( for cron job mode )
-# Tried to use curl only (curl -o /tmp/linux_amd64.tar.gz ...), however it turned out that the following tar command failed with "gzip: stdin: not in gzip format"
-RUN apk add --no-cache --virtual build-dependencies --update wget curl ca-certificates && \
-    wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/latest | egrep -o 'chrislusf/seaweedfs/releases/download/.*/linux_amd64.tar.gz') && \
-    tar -C /usr/bin/ -xzvf /tmp/linux_amd64.tar.gz && \
+RUN \
+    ARCH=$(if [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "64" ]; then echo "amd64"; \
+         elif [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "32" ]; then echo "386"; \
+         elif [ $(uname -m) == "aarch64" ]; then echo "arm64"; \
+         elif [ $(uname -m) == "armv7l" ]; then echo "arm"; \
+         elif [ $(uname -m) == "armv6l" ]; then echo "arm"; fi;) && \
+    echo "Building for $ARCH" 1>&2 && \
+    SUPERCRONIC_SHA1SUM=$(echo $ARCH | sed 's/386/e0126b0102b9f388ecd55714358e3ad60d0cebdb/g' | sed 's/amd64/5ddf8ea26b56d4a7ff6faecdd8966610d5cb9d85/g' | sed 's/arm64/e2714c43e7781bf1579c85aa61259245f56dbba1/g' | sed 's/arm/47481c3341bc3a1ae91a728e0cc63c8e6d3791ad/g') && \
+    SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.1.9/supercronic-linux-$ARCH && \
+    SUPERCRONIC=supercronic-linux-$ARCH && \
+    # Install SeaweedFS and Supercronic ( for cron job mode )
+    apk add --no-cache --virtual build-dependencies --update wget curl ca-certificates && \
+    apk add fuse && \
+    wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/${RELEASE} | egrep -o "chrislusf/seaweedfs/releases/download/.*/linux_$ARCH.tar.gz") && \
+    tar -C /usr/bin/ -xzvf /tmp/linux_$ARCH.tar.gz && \
     curl -fsSLO "$SUPERCRONIC_URL" && \
     echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
     chmod +x "$SUPERCRONIC" && \
@@ -32,6 +39,8 @@ EXPOSE 19333
 EXPOSE 9333
 # s3 server http port
 EXPOSE 8333
+# webdav server http port
+EXPOSE 7333
 
 RUN mkdir -p /data/filerldb2
 

docker/Dockerfile.go_build

@@ -1,8 +1,12 @@
 FROM frolvlad/alpine-glibc as builder
-RUN apk add git go g++
+RUN apk add git go g++ fuse
 RUN mkdir -p /go/src/github.com/chrislusf/
 RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
-RUN cd /go/src/github.com/chrislusf/seaweedfs/weed && go install
+ARG BRANCH=${BRANCH:-master}
+RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
+RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
+  && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
+  && go install -ldflags "${LDFLAGS}"
 
 FROM alpine AS final
 LABEL author="Chris Lu"
@@ -10,6 +14,7 @@ COPY --from=builder /root/go/bin/weed /usr/bin/
 RUN mkdir -p /etc/seaweedfs
 COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
 COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
+RUN apk add fuse # for weed mount
 
 # volume server gprc port
 EXPOSE 18080
@@ -25,6 +30,8 @@ EXPOSE 19333
 EXPOSE 9333
 # s3 server http port
 EXPOSE 8333
+# webdav server http port
+EXPOSE 7333
 
 RUN mkdir -p /data/filerldb2
 

docker/Dockerfile.go_build_large

@@ -0,0 +1,42 @@
+FROM frolvlad/alpine-glibc as builder
+RUN apk add git go g++ fuse
+RUN mkdir -p /go/src/github.com/chrislusf/
+RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
+ARG BRANCH=${BRANCH:-master}
+RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
+RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
+  && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
+  && go install -tags 5BytesOffset -ldflags "${LDFLAGS}"
+
+FROM alpine AS final
+LABEL author="Chris Lu"
+COPY --from=builder /root/go/bin/weed /usr/bin/
+RUN mkdir -p /etc/seaweedfs
+COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
+COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
+RUN apk add fuse # for weed mount
+
+# volume server gprc port
+EXPOSE 18080
+# volume server http port
+EXPOSE 8080
+# filer server gprc port
+EXPOSE 18888
+# filer server http port
+EXPOSE 8888
+# master server shared gprc port
+EXPOSE 19333
+# master server shared http port
+EXPOSE 9333
+# s3 server http port
+EXPOSE 8333
+# webdav server http port
+EXPOSE 7333
+
+RUN mkdir -p /data/filerldb2
+
+VOLUME /data
+
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/Dockerfile.local

@@ -4,6 +4,7 @@ COPY  ./weed /usr/bin/
 RUN mkdir -p /etc/seaweedfs
 COPY ./filer.toml /etc/seaweedfs/filer.toml
 COPY ./entrypoint.sh /entrypoint.sh
+RUN apk add fuse # for weed mount
 
 # volume server grpc port
 EXPOSE 18080
@@ -19,6 +20,8 @@ EXPOSE 19333
 EXPOSE 9333
 # s3 server http port
 EXPOSE 8333
+# webdav server http port
+EXPOSE 7333
 
 RUN mkdir -p /data/filerldb2
 

docker/Dockerfile.s3tests

@@ -0,0 +1,31 @@
+FROM ubuntu:20.04
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        git \
+        sudo \
+        debianutils \
+        python3-pip \
+        python3-virtualenv \
+        python3-dev \
+        libevent-dev \
+        libffi-dev \
+        libxml2-dev \
+        libxslt-dev \
+        zlib1g-dev && \
+    DEBIAN_FRONTEND=noninteractive apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    git clone https://github.com/ceph/s3-tests.git /opt/s3-tests
+
+WORKDIR /opt/s3-tests
+RUN ./bootstrap
+
+ENV \
+    NOSETESTS_EXCLUDE="" \
+    NOSETESTS_ATTR="" \
+    NOSETESTS_OPTIONS="" \
+    S3TEST_CONF="/s3test.conf"
+
+ENTRYPOINT ["/bin/bash", "-c"]
+CMD ["exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"]

docker/Makefile

@@ -9,11 +9,35 @@ build:
 	docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
 	rm ./weed
 
+s3tests_build:
+	docker build --no-cache -t chrislusf/ceph-s3-tests:local -f Dockerfile.s3tests .
+
 dev: build
-	docker-compose -f local-dev-compose.yml -p seaweedfs up
+	docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
+
+dev_mount: build
+	docker-compose -f compose/local-mount-compose.yml -p seaweedfs up
+
+k8s: build
+	docker-compose -f compose/local-k8s-compose.yml -p seaweedfs up
+
+dev_registry: build
+	docker-compose -f compose/local-registry-compose.yml -p seaweedfs up
+
+dev_replicate: build
+	docker-compose -f compose/local-replicate-compose.yml -p seaweedfs up
 
 cluster: build
-	docker-compose -f local-cluster-compose.yml -p seaweedfs up
+	docker-compose -f compose/local-cluster-compose.yml -p seaweedfs up
+
+2clusters: build
+	docker-compose -f compose/local-clusters-compose.yml -p seaweedfs up
+
+s3tests: build s3tests_build
+	docker-compose -f compose/local-s3tests-compose.yml -p seaweedfs up
+
+filer_etcd: build
+	docker stack deploy -c compose/swarm-etcd.yml fs
 
 clean:
 	rm ./weed

docker/README.md

@@ -27,3 +27,13 @@ docker-compose -f seaweedfs-dev-compose.yml -p seaweedfs up
 cd $GOPATH/src/github.com/chrislusf/seaweedfs/docker
 make
 ```
+
+## Build and push a multiarch build
+
+Make sure that `docker buildx` is supported (might be an experimental docker feature)
+```bash
+BUILDER=$(docker buildx create --driver docker-container --use)
+docker buildx build --pull --push --platform linux/386,linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 . -t chrislusf/seaweedfs
+docker buildx stop $BUILDER
+```
+

docker/compose/local-cluster-compose.yml

@@ -6,25 +6,45 @@ services:
     ports:
       - 9333:9333
       - 19333:19333
-    command: "master -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335"
+    command: "master -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m1"
   master1:
     image: chrislusf/seaweedfs:local
     ports:
       - 9334:9334
       - 19334:19334
-    command: "master -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335"
+    command: "master -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m2"
   master2:
     image: chrislusf/seaweedfs:local
     ports:
       - 9335:9335
       - 19335:19335
-    command: "master -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335"
-  volume:
+    command: "master -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m3"
+  volume1:
     image: chrislusf/seaweedfs:local
     ports:
       - 8080:8080
       - 18080:18080
-    command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume'
+    command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1 -disk=ssd1'
+    depends_on:
+      - master0
+      - master1
+      - master2
+  volume2:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8082:8082
+      - 18082:18082
+    command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1 -disk=ssd1'
+    depends_on:
+      - master0
+      - master1
+      - master2
+  volume3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8083:8083
+      - 18083:18083
+    command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
     depends_on:
       - master0
       - master1
@@ -39,7 +59,8 @@ services:
       - master0
       - master1
       - master2
-      - volume
+      - volume1
+      - volume2
   s3:
     image: chrislusf/seaweedfs:local
     ports:
@@ -49,5 +70,6 @@ services:
       - master0
       - master1
       - master2
-      - volume
+      - volume1
+      - volume2
       - filer

docker/compose/local-clusters-compose.yml

@@ -0,0 +1,28 @@
+version: '2'
+
+services:
+  server1:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+      - 8084:8080
+      - 18084:18080
+      - 8888:8888
+      - 18888:18888
+    command: "server -ip=server1 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
+    volumes:
+      - ./master-cloud.toml:/etc/seaweedfs/master.toml
+    depends_on:
+      - server2
+  server2:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9334:9333
+      - 19334:19333
+      - 8085:8080
+      - 18085:18080
+      - 8889:8888
+      - 18889:18888
+      - 8334:8333
+    command: "server -ip=server2 -filer -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"

docker/compose/local-dev-compose.yml

@@ -12,7 +12,7 @@ services:
     ports:
       - 8080:8080
       - 18080:18080
-    command: "volume -mserver=master:9333 -port=8080 -ip=volume"
+    command: "volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
     depends_on:
       - master
   filer:
@@ -33,3 +33,14 @@ services:
       - master
       - volume
       - filer
+  mount:
+    image: chrislusf/seaweedfs:local
+    privileged: true
+    cap_add:
+      - SYS_ADMIN
+    mem_limit: 4096m
+    command: '-v=4 mount -filer="filer:8888" -dirAutoCreate -dir=/mnt/seaweedfs -cacheCapacityMB=100 -concurrentWriters=128'
+    depends_on:
+      - master
+      - volume
+      - filer

docker/compose/local-k8s-compose.yml

@@ -0,0 +1,94 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=master"
+  volume:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8080:8080
+      - 18080:18080
+    command: "volume -mserver=master:9333 -port=8080 -ip=volume"
+    depends_on:
+      - master
+  mysql:
+    image: percona/percona-server:5.7
+    ports:
+      - 3306:3306
+    volumes:
+      - ./seaweedfs.sql:/docker-entrypoint-initdb.d/seaweedfs.sql
+    environment:
+      - MYSQL_ROOT_PASSWORD=secret
+      - MYSQL_DATABASE=seaweedfs
+      - MYSQL_PASSWORD=secret
+      - MYSQL_USER=seaweedfs
+  filer:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8888:8888
+      - 18888:18888
+    environment:
+      - WEED_MYSQL_HOSTNAME=mysql
+      - WEED_MYSQL_PORT=3306
+      - WEED_MYSQL_DATABASE=seaweedfs
+      - WEED_MYSQL_USERNAME=seaweedfs
+      - WEED_MYSQL_PASSWORD=secret
+      - WEED_MYSQL_ENABLED=true
+      - WEED_MYSQL_CONNECTION_MAX_IDLE=5
+      - WEED_MYSQL_CONNECTION_MAX_OPEN=75
+      # "refresh" connection every 10 minutes, eliminating mysql closing "old" connections
+      - WEED_MYSQL_CONNECTION_MAX_LIFETIME_SECONDS=600
+      # enable usage of memsql as filer backend
+      - WEED_MYSQL_INTERPOLATEPARAMS=true
+      - WEED_LEVELDB2_ENABLED=false
+    command: '-v 9 filer -master="master:9333"'
+    depends_on:
+      - master
+      - volume
+      - mysql
+  ingress:
+    image: jwilder/nginx-proxy:alpine
+    ports:
+      - "80:80"
+    volumes:
+      - /var/run/docker.sock:/tmp/docker.sock:ro
+      - ./nginx/proxy.conf:/etc/nginx/proxy.conf
+  s3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8333:8333
+    command: '-v 9 s3 -filer="filer:8888"'
+    depends_on:
+      - master
+      - volume
+      - filer
+    environment:
+      - VIRTUAL_HOST=ingress
+      - VIRTUAL_PORT=8333
+  registry:
+    image: registry:2
+    environment:
+      REGISTRY_HTTP_ADDR: "0.0.0.0:5001" # seaweedfs s3
+      REGISTRY_LOG_LEVEL: "debug"
+      REGISTRY_STORAGE: "s3"
+      REGISTRY_STORAGE_S3_REGION: "us-east-1"
+      REGISTRY_STORAGE_S3_REGIONENDPOINT: "http://ingress"
+      REGISTRY_STORAGE_S3_BUCKET: "registry"
+      REGISTRY_STORAGE_S3_ACCESSKEY: "some_access_key1"
+      REGISTRY_STORAGE_S3_SECRETKEY: "some_secret_key1"
+      REGISTRY_STORAGE_S3_V4AUTH: "true"
+      REGISTRY_STORAGE_S3_SECURE: "false"
+      REGISTRY_STORAGE_S3_SKIPVERIFY: "true"
+      REGISTRY_STORAGE_S3_ROOTDIRECTORY: "/"
+      REGISTRY_STORAGE_DELETE_ENABLED: "true"
+      REGISTRY_STORAGE_REDIRECT_DISABLE: "true"
+      REGISTRY_VALIDATION_DISABLED: "true"
+    ports:
+      - 5001:5001
+    depends_on:
+      - s3
+      - ingress

docker/compose/local-mount-compose.yml

@@ -0,0 +1,46 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=master"
+  volume:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 7455:8080
+      - 9325:9325
+    command: 'volume -mserver="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455'
+    depends_on:
+      - master
+  filer:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8888:8888
+      - 18888:18888
+      - 9326:9326
+    command: 'filer -master="master:9333"  -metricsPort=9326'
+    tty: true
+    stdin_open: true
+    depends_on:
+      - master
+      - volume
+  mount_1:
+    image: chrislusf/seaweedfs:local
+    privileged: true
+    entrypoint: '/bin/sh -c "mkdir -p t1 && mkdir -p cache/t1 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t1 -dir=./t1 -filer.path=/c1 -volumeServerAccess=filerProxy"'
+    depends_on:
+      - master
+      - volume
+      - filer
+  mount_2:
+    image: chrislusf/seaweedfs:local
+    privileged: true
+    entrypoint: '/bin/sh -c "mkdir -p t2 && mkdir -p cache/t2 && weed -v=4  mount -filer=filer:8888 -cacheDir=./cache/t2 -dir=./t2 -filer.path=/c1 -volumeServerAcess=publicUrl"'
+    depends_on:
+      - master
+      - volume
+      - filer
+      - mount_1

docker/compose/local-registry-compose.yml

@@ -0,0 +1,91 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=master -volumeSizeLimitMB=1024"
+  volume:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8080:8080
+      - 18080:18080
+    command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
+    depends_on:
+      - master
+  filer:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8888:8888
+      - 18888:18888
+    command: 'filer -master="master:9333"'
+    depends_on:
+      - master
+      - volume
+  s3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8333:8333
+    command: '-v 9 s3 -filer="filer:8888"'
+    depends_on:
+      - master
+      - volume
+      - filer
+  minio:
+    image: minio/minio
+    ports:
+      - 9000:9000
+    command: 'minio server /data'
+    environment:
+      MINIO_ACCESS_KEY: "some_access_key1"
+      MINIO_SECRET_KEY: "some_secret_key1"
+    depends_on:
+      - master
+  registry1:
+    image: registry:2
+    environment:
+      REGISTRY_HTTP_ADDR: "0.0.0.0:5001" # seaweedfs s3
+      REGISTRY_LOG_LEVEL: "debug"
+      REGISTRY_STORAGE: "s3"
+      REGISTRY_STORAGE_S3_REGION: "us-east-1"
+      REGISTRY_STORAGE_S3_REGIONENDPOINT: "http://s3:8333"
+      REGISTRY_STORAGE_S3_BUCKET: "registry"
+      REGISTRY_STORAGE_S3_ACCESSKEY: "some_access_key1"
+      REGISTRY_STORAGE_S3_SECRETKEY: "some_secret_key1"
+      REGISTRY_STORAGE_S3_V4AUTH: "true"
+      REGISTRY_STORAGE_S3_SECURE: "false"
+      REGISTRY_STORAGE_S3_SKIPVERIFY: "true"
+      REGISTRY_STORAGE_S3_ROOTDIRECTORY: "/"
+      REGISTRY_STORAGE_DELETE_ENABLED: "true"
+      REGISTRY_STORAGE_REDIRECT_DISABLE: "true"
+      REGISTRY_VALIDATION_DISABLED: "true"
+    ports:
+      - 5001:5001
+    depends_on:
+      - s3
+      - minio
+  registry2:
+    image: registry:2
+    environment:
+      REGISTRY_HTTP_ADDR: "0.0.0.0:5002" # minio
+      REGISTRY_LOG_LEVEL: "debug"
+      REGISTRY_STORAGE: "s3"
+      REGISTRY_STORAGE_S3_REGION: "us-east-1"
+      REGISTRY_STORAGE_S3_REGIONENDPOINT: "http://minio:9000"
+      REGISTRY_STORAGE_S3_BUCKET: "registry"
+      REGISTRY_STORAGE_S3_ACCESSKEY: "some_access_key1"
+      REGISTRY_STORAGE_S3_SECRETKEY: "some_secret_key1"
+      REGISTRY_STORAGE_S3_V4AUTH: "true"
+      REGISTRY_STORAGE_S3_SECURE: "false"
+      REGISTRY_STORAGE_S3_SKIPVERIFY: "true"
+      REGISTRY_STORAGE_S3_ROOTDIRECTORY: "/"
+      REGISTRY_STORAGE_DELETE_ENABLED: "true"
+      REGISTRY_STORAGE_REDIRECT_DISABLE: "true"
+      REGISTRY_VALIDATION_DISABLED: "true"
+    ports:
+      - 5002:5002
+    depends_on:
+      - s3
+      - minio

docker/compose/local-replicate-compose.yml

@@ -0,0 +1,61 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=master"
+  volume:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8080:8080
+      - 18080:18080
+    command: "volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
+    depends_on:
+      - master
+  filer:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8888:8888
+      - 18888:18888
+    command: '-v=9 filer -master="master:9333"'
+    restart: on-failure
+    volumes:
+      - ./notification.toml:/etc/seaweedfs/notification.toml
+    depends_on:
+      - master
+      - volume
+      - rabbitmq
+      - replicate
+    environment:
+      RABBIT_SERVER_URL: "amqp://guest:guest@rabbitmq:5672/"
+  replicate:
+    image: chrislusf/seaweedfs:local
+    command: '-v=9 filer.replicate'
+    restart: on-failure
+    volumes:
+      - ./notification.toml:/etc/seaweedfs/notification.toml
+      - ./replication.toml:/etc/seaweedfs/replication.toml
+    depends_on:
+      - rabbitmq
+    environment:
+      RABBIT_SERVER_URL: "amqp://guest:guest@rabbitmq:5672/"
+  s3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8333:8333
+    command: 's3 -filer="filer:8888"'
+    depends_on:
+      - master
+      - volume
+      - filer
+  rabbitmq:
+    image: rabbitmq:3.8.10-management-alpine
+    ports:
+      - 5672:5672
+      - 15671:15671
+      - 15672:15672
+    environment:
+      RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "-rabbit log_levels [{connection,error},{queue,debug}]"

docker/compose/local-s3tests-compose.yml

@@ -0,0 +1,45 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=master -volumeSizeLimitMB=16"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+  volume:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8080:8080
+      - 18080:18080
+    command: "volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
+    depends_on:
+      - master
+  s3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8888:8888
+      - 18888:18888
+      - 8000:8000
+    command: 'filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000'
+    volumes:
+      - ./s3.json:/etc/seaweedfs/s3.json
+    depends_on:
+      - master
+      - volume
+  s3tests:
+    image: chrislusf/ceph-s3-tests:local
+    volumes:
+      - ./s3tests.conf:/opt/s3-tests/s3tests.conf
+    environment:
+      S3TEST_CONF: "s3tests.conf"
+      NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
+      NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
+      NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_delimiter_not_skip_special|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_head_zero_bytes|object_write_cache_control|object_write_expires|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifmatch_failed|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_authenticated|object_raw_response_headers|object_raw_authenticated_bucket_acl|object_raw_authenticated_object_acl|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_not_expired|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_create_exists|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|bucket_create_special_key_names|object_copy_zero_size|object_copy_verify_contenttype|object_copy_to_itself|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
+    depends_on:
+      - master
+      - volume
+      - s3

docker/compose/master-cloud.toml

@@ -0,0 +1,30 @@
+
+# Put this file to one of the location, with descending priority
+#    ./master.toml
+#    $HOME/.seaweedfs/master.toml
+#    /etc/seaweedfs/master.toml
+# this file is read by master
+
+[master.maintenance]
+# periodically run these scripts are the same as running them from 'weed shell'
+scripts = """
+  lock
+  ec.encode -fullPercent=95 -quietFor=1h
+  ec.rebuild -force
+  ec.balance -force
+  volume.balance -force
+  volume.fix.replication
+  unlock
+"""
+sleep_minutes = 17          # sleep minutes between each script execution
+
+# configurations for tiered cloud storage
+# old volumes are transparently moved to cloud for cost efficiency
+[storage.backend]
+	[storage.backend.s3.default]
+	enabled = true
+	aws_access_key_id     = "any"     # if empty, loads from the shared credentials file (~/.aws/credentials).
+	aws_secret_access_key = "any"     # if empty, loads from the shared credentials file (~/.aws/credentials).
+	region = "us-east-2"
+	bucket = "volume_bucket"    # an existing bucket
+	endpoint = "http://server2:8333"

docker/compose/notification.toml

@@ -0,0 +1,17 @@
+[notification.log]
+# this is only for debugging perpose and does not work with "weed filer.replicate"
+enabled = false
+
+
+[notification.gocdk_pub_sub]
+# The Go Cloud Development Kit (https://gocloud.dev).
+# PubSub API (https://godoc.org/gocloud.dev/pubsub).
+# Supports AWS SNS/SQS, Azure Service Bus, Google PubSub, NATS and RabbitMQ.
+enabled = true
+# This URL will Dial the RabbitMQ server at the URL in the environment
+# variable RABBIT_SERVER_URL and open the exchange "myexchange".
+# The exchange must have already been created by some other means, like
+# the RabbitMQ management plugin. Сreate myexchange of type fanout and myqueue then
+# create binding myexchange => myqueue
+topic_url = "rabbit://swexchange"
+sub_url = "rabbit://swqueue"

docker/compose/replication.toml

@@ -0,0 +1,11 @@
+[source.filer]
+enabled = true
+grpcAddress = "filer:18888"
+# all files under this directory tree are replicated.
+# this is not a directory on your hard drive, but on your filer.
+# i.e., all files with this "prefix" are sent to notification message queue.
+directory = "/buckets"
+
+[sink.local_incremental]
+enabled = true
+directory = "/data"

docker/compose/s3.json

@@ -0,0 +1,105 @@
+{
+  "identities": [
+    {
+      "name": "anonymous",
+      "actions": [
+        "Read"
+      ]
+    },
+    {
+      "name": "some_admin_user",
+      "credentials": [
+        {
+          "accessKey": "some_access_key1",
+          "secretKey": "some_secret_key1"
+        }
+      ],
+      "actions": [
+        "Admin",
+        "Read",
+        "List",
+        "Tagging",
+        "Write"
+      ]
+    },
+    {
+      "name": "s3_tests",
+      "credentials": [
+        {
+          "accessKey": "ABCDEFGHIJKLMNOPQRST",
+          "secretKey": "abcdefghijklmnopqrstuvwxyzabcdefghijklmn"
+        },
+        {
+          "accessKey": "0555b35654ad1656d804",
+          "secretKey": "h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=="
+        }
+      ],
+      "actions": [
+        "Admin",
+        "Read",
+        "List",
+        "Tagging",
+        "Write"
+      ]
+    },
+    {
+      "name": "s3_tests_alt",
+      "credentials": [
+        {
+          "accessKey": "NOPQRSTUVWXYZABCDEFG",
+          "secretKey": "nopqrstuvwxyzabcdefghijklmnabcdefghijklm"
+        }
+      ],
+      "actions": [
+        "Admin",
+        "Read",
+        "List",
+        "Tagging",
+        "Write"
+      ]
+    },
+    {
+      "name": "s3_tests_tenant",
+      "credentials": [
+        {
+          "accessKey": "HIJKLMNOPQRSTUVWXYZA",
+          "secretKey": "opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab"
+        }
+      ],
+      "actions": [
+        "Admin",
+        "Read",
+        "List",
+        "Tagging",
+        "Write"
+      ]
+    },
+    {
+      "name": "some_read_only_user",
+      "credentials": [
+        {
+          "accessKey": "some_access_key2",
+          "secretKey": "some_secret_key2"
+        }
+      ],
+      "actions": [
+        "Read"
+      ]
+    },
+    {
+      "name": "some_normal_user",
+      "credentials": [
+        {
+          "accessKey": "some_access_key3",
+          "secretKey": "some_secret_key3"
+        }
+      ],
+      "actions": [
+        "Read",
+        "List",
+        "Tagging",
+        "Write"
+      ]
+    }
+  ]
+}

docker/compose/s3tests.conf

@@ -0,0 +1,70 @@
+[DEFAULT]
+## this section is just used for host, port and bucket_prefix
+
+# host set for rgw in vstart.sh
+host = s3
+
+# port set for rgw in vstart.sh
+port = 8000
+
+## say "False" to disable TLS
+is_secure = False
+
+[fixtures]
+## all the buckets created will start with this prefix;
+## {random} will be filled with random characters to pad
+## the prefix to 30 characters long, and avoid collisions
+bucket prefix = yournamehere-{random}-
+
+[s3 main]
+# main display_name set in vstart.sh
+display_name = M. Tester
+
+# main user_idname set in vstart.sh
+user_id = testid
+
+# main email set in vstart.sh
+email = tester@ceph.com
+
+# zonegroup api_name for bucket location
+api_name = default
+
+## main AWS access key
+access_key = 0555b35654ad1656d804
+
+## main AWS secret key
+secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==
+
+## replace with key id obtained when secret is created, or delete if KMS not tested
+#kms_keyid = 01234567-89ab-cdef-0123-456789abcdef
+
+[s3 alt]
+# alt display_name set in vstart.sh
+display_name = john.doe
+## alt email set in vstart.sh
+email = john.doe@example.com
+
+# alt user_id set in vstart.sh
+user_id = 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234
+
+# alt AWS access key set in vstart.sh
+access_key = NOPQRSTUVWXYZABCDEFG
+
+# alt AWS secret key set in vstart.sh
+secret_key = nopqrstuvwxyzabcdefghijklmnabcdefghijklm
+
+[s3 tenant]
+# tenant display_name set in vstart.sh
+display_name = testx$tenanteduser
+
+# tenant user_id set in vstart.sh
+user_id = 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+
+# tenant AWS secret key set in vstart.sh
+access_key = HIJKLMNOPQRSTUVWXYZA
+
+# tenant AWS secret key set in vstart.sh
+secret_key = opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab
+
+# tenant email set in vstart.sh
+email = tenanteduser@example.com

docker/compose/swarm-etcd.yml

@@ -0,0 +1,84 @@
+# 2021-01-30 16:25:30
+version: '3.8'
+
+services:
+
+  etcd:
+    image: gasparekatapy/etcd
+    networks:
+      - net
+    deploy:
+      mode: replicated
+      replicas: 3
+
+  master:
+    image: chrislusf/seaweedfs:local
+    environment:
+      WEED_MASTER_FILER_DEFAULT: "filer:8888"
+      WEED_MASTER_SEQUENCER_TYPE: "raft"
+    ports:
+      - "9333:9333"
+      - "19333:19333"
+    networks:
+      - net
+    command:
+      - 'master'
+      - '-resumeState=true'
+      - '-ip=master'
+      - '-port=9333'
+    deploy:
+      mode: replicated
+      replicas: 1
+
+  filer:
+    image: chrislusf/seaweedfs:local
+    environment:
+      WEED_LEVELDB2_ENABLED: "false"
+      WEED_ETCD_ENABLED: "true"
+      WEED_ETCD_SERVERS: "etcd:2379"
+    ports:
+      - target: 8888
+        published: 8888
+        protocol: tcp
+        mode: host
+      - target: 18888
+        published: 18888
+        protocol: tcp
+        mode: host
+    networks:
+      - net
+    command:
+      - 'filer'
+      - '-ip=filer'
+      - '-port=8888'
+      - '-port.readonly=28888'
+      - '-master=master:9333'
+      - '-disableDirListing=true'
+    deploy:
+      mode: replicated
+      replicas: 1
+
+  volume:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - target: 8080
+        published: 8080
+        protocol: tcp
+        mode: host
+      - target: 18080
+        published: 18080
+        protocol: tcp
+        mode: host
+    networks:
+      - net
+    command:
+      - 'volume'
+      - '-mserver=master:9333'
+      - '-port=8080'
+    deploy:
+      mode: global
+
+  ###########################################################################
+
+networks:
+  net:

docker/entrypoint.sh

@@ -1,30 +1,60 @@
 #!/bin/sh
 
+isArgPassed() {
+  arg="$1"
+  argWithEqualSign="$1="
+  shift
+  while [ $# -gt 0 ]; do
+    passedArg="$1"
+    shift
+    case $passedArg in
+    $arg)
+      return 0
+      ;;
+    $argWithEqualSign*)
+      return 0
+      ;;
+    esac
+  done
+  return 1
+}
+
 case "$1" in
 
   'master')
   	ARGS="-mdir=/data -volumePreallocate -volumeSizeLimitMB=1024"
-  	exec /usr/bin/weed $@ $ARGS
+  	shift
+  	exec /usr/bin/weed master $ARGS $@
 	;;
 
   'volume')
   	ARGS="-dir=/data -max=0"
-  	exec /usr/bin/weed $@ $ARGS
+  	if isArgPassed "-max" "$@"; then
+  	  ARGS="-dir=/data"
+  	fi
+  	shift
+  	exec /usr/bin/weed volume $ARGS $@
 	;;
 
   'server')
   	ARGS="-dir=/data -volume.max=0 -master.volumePreallocate -master.volumeSizeLimitMB=1024"
-  	exec /usr/bin/weed $@ $ARGS
+  	if isArgPassed "-volume.max" "$@"; then
+  	  ARGS="-dir=/data -master.volumePreallocate -master.volumeSizeLimitMB=1024"
+  	fi
+ 	shift
+  	exec /usr/bin/weed server $ARGS $@
   	;;
 
   'filer')
   	ARGS=""
-  	exec /usr/bin/weed $@ $ARGS
+  	shift
+  	exec /usr/bin/weed filer $ARGS $@
 	;;
 
   's3')
   	ARGS="-domainName=$S3_DOMAIN_NAME -key.file=$S3_KEY_FILE -cert.file=$S3_CERT_FILE"
-  	exec /usr/bin/weed $@ $ARGS
+  	shift
+  	exec /usr/bin/weed s3 $ARGS $@
 	;;
 
   'cronjob')

docker/nginx/proxy.conf

@@ -0,0 +1,30 @@
+# HTTP 1.1 support
+proxy_http_version 1.1;
+#proxy_buffering off;
+proxy_set_header Host $http_host;
+proxy_set_header Upgrade $http_upgrade;
+proxy_set_header Connection $proxy_connection;
+proxy_set_header X-Real-IP $remote_addr;
+proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto;
+proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl;
+proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port;
+
+# Mitigate httpoxy attack (see README for details)
+proxy_set_header Proxy "";
+
+# aws default max_concurrent_requests 10
+# aws default multipart_threshold     8MB
+proxy_buffering on;     # GET buffering or “X-Accel-Buffering” enables or disables buffering of a response;
+proxy_buffers 64 1m;  # buffers used for reading a response from the proxied server, for a single connection
+proxy_buffer_size 8k; # maximum size of the data that nginx can receive from the server at a time is set
+proxy_busy_buffers_size 2m;
+
+proxy_request_buffering on;  # PUT buffering
+client_body_buffer_size 64m; # buffer size for reading client request body
+client_max_body_size    64m;
+
+proxy_next_upstream error timeout non_idempotent http_500; # PUT request should be passed to the next server:
+proxy_connect_timeout 200ms;
+proxy_read_timeout    3s; #timeout is set only between two successive read operations
+proxy_send_timeout    3s; #timeout is set only between two successive write operations

docker/prometheus/prometheus.yml

@@ -0,0 +1,13 @@
+global:
+  scrape_interval: 30s
+  scrape_timeout: 10s
+
+scrape_configs:
+  - job_name: services
+    metrics_path: /metrics
+    static_configs:
+      - targets:
+          - 'prometheus:9090'
+          - 'volume:9325'
+          - 'filer:9326'
+          - 's3:9327'

docker/seaweedfs-compose.yml

@@ -12,7 +12,8 @@ services:
     ports:
       - 8080:8080
       - 18080:18080
-    command: 'volume -mserver="master:9333" -port=8080'
+      - 9325:9325
+    command: 'volume -mserver="master:9333" -port=8080  -metricsPort=9325'
     depends_on:
       - master
   filer:
@@ -20,7 +21,8 @@ services:
     ports:
       - 8888:8888
       - 18888:18888
-    command: 'filer -master="master:9333"'
+      - 9326:9326
+    command: 'filer -master="master:9333"  -metricsPort=9326'
     tty: true
     stdin_open: true
     depends_on:
@@ -40,8 +42,27 @@ services:
     image: chrislusf/seaweedfs # use a remote image
     ports:
       - 8333:8333
-    command: 's3 -filer="filer:8888"'
+      - 9327:9327
+    command: 's3 -filer="filer:8888" -metricsPort=9327'
     depends_on:
       - master
       - volume
       - filer
+  webdav:
+    image: chrislusf/seaweedfs # use a remote image
+    ports:
+      - 7333:7333
+    command: 'webdav -filer="filer:8888"'
+    depends_on:
+      - master
+      - volume
+      - filer
+  prometheus:
+    image: prom/prometheus:v2.21.0
+    ports:
+      - 9000:9090
+    volumes:
+      - ./prometheus:/etc/prometheus
+    command: --web.enable-lifecycle  --config.file=/etc/prometheus/prometheus.yml
+    depends_on:
+      - s3

docker/seaweedfs-dev-compose.yml

@@ -33,3 +33,12 @@ services:
       - master
       - volume
       - filer
+  webdav:
+    image: chrislusf/seaweedfs:dev # use a remote dev image
+    ports:
+      - 7333:7333
+    command: 'webdav -filer="filer:8888"'
+    depends_on:
+      - master
+      - volume
+      - filer

docker/seaweedfs.sql

@@ -0,0 +1,12 @@
+CREATE DATABASE IF NOT EXISTS seaweedfs;
+CREATE USER IF NOT EXISTS 'seaweedfs'@'%' IDENTIFIED BY 'secret';
+GRANT ALL PRIVILEGES ON seaweedfs_fast.* TO 'seaweedfs'@'%';
+FLUSH PRIVILEGES;
+USE seaweedfs;
+CREATE TABLE IF NOT EXISTS filemeta (
+    dirhash     BIGINT         COMMENT 'first 64 bits of MD5 hash value of directory field',
+    name        VARCHAR(1000)  COMMENT 'directory or file name',
+    directory   TEXT           COMMENT 'full path to parent directory',
+    meta        LONGBLOB,
+    PRIMARY KEY (dirhash, name)
+) DEFAULT CHARSET=utf8;

go.mod

@@ -3,20 +3,19 @@ module github.com/chrislusf/seaweedfs
 go 1.12
 
 require (
-	cloud.google.com/go v0.44.3
+	cloud.google.com/go v0.58.0 // indirect
+	cloud.google.com/go/pubsub v1.3.1
+	cloud.google.com/go/storage v1.9.0
+	github.com/Azure/azure-amqp-common-go/v2 v2.1.0 // indirect
 	github.com/Azure/azure-pipeline-go v0.2.2 // indirect
-	github.com/Azure/azure-storage-blob-go v0.8.0
-	github.com/DataDog/zstd v1.4.1 // indirect
+	github.com/Azure/azure-storage-blob-go v0.9.0
 	github.com/OneOfOne/xxhash v1.2.2
 	github.com/Shopify/sarama v1.23.1
-	github.com/aws/aws-sdk-go v1.23.13
+	github.com/aws/aws-sdk-go v1.34.30
 	github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
 	github.com/cespare/xxhash v1.1.0
-	github.com/chrislusf/raft v0.0.0-20190225081310-10d6e2182d92
-	github.com/coreos/bbolt v1.3.3 // indirect
-	github.com/coreos/etcd v3.3.15+incompatible // indirect
+	github.com/chrislusf/raft v1.0.4
 	github.com/coreos/go-semver v0.3.0 // indirect
-	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/disintegration/imaging v1.6.2
 	github.com/dustin/go-humanize v1.0.0
@@ -26,75 +25,81 @@ require (
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
+	github.com/fclairamb/ftpserverlib v0.8.0
 	github.com/frankban/quicktest v1.7.2 // indirect
-	github.com/go-redis/redis v6.15.7+incompatible
-	github.com/go-sql-driver/mysql v1.4.1
+	github.com/go-errors/errors v1.1.1 // indirect
+	github.com/go-redis/redis/v8 v8.4.4
+	github.com/go-sql-driver/mysql v1.5.0
 	github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6
 	github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 // indirect
-	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
-	github.com/golang/protobuf v1.3.2
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e
+	github.com/golang/protobuf v1.4.2
 	github.com/google/btree v1.0.0
 	github.com/google/uuid v1.1.1
-	github.com/gorilla/mux v1.7.3
+	github.com/gorilla/mux v1.7.4
 	github.com/gorilla/websocket v1.4.1 // indirect
-	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
-	github.com/hashicorp/golang-lru v0.5.3 // indirect
 	github.com/jcmturner/gofork v1.0.0 // indirect
-	github.com/karlseguin/ccache v2.0.3+incompatible
-	github.com/karlseguin/expect v1.0.1 // indirect
+	github.com/json-iterator/go v1.1.10
+	github.com/karlseguin/ccache v2.0.3+incompatible // indirect
+	github.com/karlseguin/ccache/v2 v2.0.7
+	github.com/klauspost/compress v1.10.9 // indirect
 	github.com/klauspost/cpuid v1.2.1 // indirect
 	github.com/klauspost/crc32 v1.2.0
 	github.com/klauspost/reedsolomon v1.9.2
 	github.com/kurin/blazer v0.5.3
 	github.com/lib/pq v1.2.0
+	github.com/lunixbochs/vtclean v1.0.0 // indirect
 	github.com/magiconair/properties v1.8.1 // indirect
-	github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect
+	github.com/mattn/go-colorable v0.1.2 // indirect
 	github.com/mattn/go-runewidth v0.0.4 // indirect
-	github.com/nats-io/nats-server/v2 v2.0.4 // indirect
-	github.com/onsi/ginkgo v1.10.1 // indirect
-	github.com/onsi/gomega v1.7.0 // indirect
+	github.com/olivere/elastic/v7 v7.0.19
 	github.com/peterh/liner v1.1.0
 	github.com/pierrec/lz4 v2.2.7+incompatible // indirect
-	github.com/prometheus/client_golang v1.1.0
-	github.com/prometheus/procfs v0.0.4 // indirect
+	github.com/prometheus/client_golang v1.3.0
 	github.com/rakyll/statik v0.1.7
 	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
-	github.com/seaweedfs/fuse v0.0.0-20190510212405-310228904eff
+	github.com/seaweedfs/fuse v1.1.1
 	github.com/seaweedfs/goexif v1.0.2
+	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/spf13/afero v1.2.2 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/viper v1.4.0
-	github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 // indirect
-	github.com/stretchr/testify v1.3.0
+	github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71
+	github.com/stretchr/testify v1.6.1
 	github.com/syndtr/goleveldb v1.0.0
+	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
 	github.com/tidwall/gjson v1.3.2
 	github.com/tidwall/match v1.0.1
+	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
+	github.com/valyala/bytebufferpool v1.0.0
+	github.com/valyala/fasthttp v1.20.0
+	github.com/viant/assertly v0.5.4 // indirect
+	github.com/viant/ptrie v0.3.0
+	github.com/viant/toolbox v0.33.2 // indirect
 	github.com/willf/bitset v1.1.10 // indirect
 	github.com/willf/bloom v2.0.3+incompatible
 	github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 // indirect
-	go.etcd.io/bbolt v1.3.3 // indirect
 	go.etcd.io/etcd v3.3.15+incompatible
 	go.mongodb.org/mongo-driver v1.3.2
-	go.uber.org/multierr v1.2.0 // indirect
-	gocloud.dev v0.16.0
-	gocloud.dev/pubsub/natspubsub v0.16.0
-	gocloud.dev/pubsub/rabbitpubsub v0.16.0
-	golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 // indirect
+	gocloud.dev v0.20.0
+	gocloud.dev/pubsub/natspubsub v0.20.0
+	gocloud.dev/pubsub/rabbitpubsub v0.20.0
 	golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
-	golang.org/x/net v0.0.0-20190909003024-a7b16738d86b
-	golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b
-	golang.org/x/tools v0.0.0-20190911022129-16c5e0f7d110
-	google.golang.org/api v0.9.0
-	google.golang.org/appengine v1.6.2 // indirect
-	google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514 // indirect
-	google.golang.org/grpc v1.26.0
+	golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb
+	golang.org/x/sync v0.0.0-20200930132711-30421366ff76 // indirect
+	golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd
+	golang.org/x/tools v0.0.0-20200608174601-1b747fd94509
+	google.golang.org/api v0.26.0
+	google.golang.org/grpc v1.29.1
+	google.golang.org/protobuf v1.24.0
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect
 	gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect
 	gopkg.in/karlseguin/expect.v1 v1.0.1 // indirect
-	sigs.k8s.io/yaml v1.1.0 // indirect
 )
 
-replace github.com/satori/go.uuid v1.2.0 => github.com/satori/go.uuid v0.0.0-20181028125025-b2ce2384e17b
+// replace github.com/seaweedfs/fuse => /Users/chris/go/src/github.com/seaweedfs/fuse
+// replace github.com/chrislusf/raft => /Users/chris/go/src/github.com/chrislusf/raft
+
+replace go.etcd.io/etcd => go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547

k8s/README.md

@@ -9,15 +9,40 @@ and backup/HA memsql can provide.
 with ENV.
 * cert config exists and can be enabled, but not been tested.
 
-### current instances config (AIO):
-1 instance for each type (master/filer/volume/s3)
+### prerequisites
+kubernetes node have labels which help to define which node(Host) will run which pod.
 
-instances need node labels:
-* sw-volume: true  (for volume instance, specific tag)
-* sw-backend: true (for all others, as they less resource demanding)
+s3/filer/master needs the label **sw-backend=true**
+
+volume need the label **sw-volume=true**
+
+to label a node to be able to run all pod types in k8s:
+```
+kubectl label node YOUR_NODE_NAME sw-volume=true,sw-backend=true
+```
+
+on production k8s deployment you will want each pod to have a different host,
+especially the volume server & the masters, currently all pods (master/volume/filer)
+have anti-affinity rule to disallow running multiple pod type on the same host.
+if you still want to run multiple pods of the same type (master/volume/filer) on the same host
+please set/update the corresponding affinity rule in values.yaml to an empty one:
+
+```affinity: ""```
+
+### PVC - storage class ###
+
+on the volume stateful set added support for K8S PVC, currently example
+with the simple local-path-provisioner from Rancher (comes included with k3d / k3s)
+https://github.com/rancher/local-path-provisioner
+
+you can use ANY storage class you like, just update the correct storage-class
+for your deployment.
+
+### current instances config (AIO):
+1 instance for each type (master/filer+s3/volume)
 
 you can update the replicas count for each node type in values.yaml,
-need to add more nodes with the corresponding label.
+need to add more nodes with the corresponding labels.
 
 most of the configuration are available through values.yaml
 

k8s/seaweedfs/Chart.yaml

@@ -1,4 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-version: 1.78
+appVersion: "2.30"
+version: 2.30

k8s/seaweedfs/templates/_helpers.tpl

@@ -52,12 +52,12 @@ Inject extra environment vars in the format key:value, if populated
 {{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}}
 {{- $repositoryName := .Values.image.repository | toString -}}
 {{- $name := .Values.global.imageName | toString -}}
-{{- $tag := .Values.global.imageTag | toString -}}
+{{- $tag := .Chart.AppVersion | toString -}}
 {{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}}
 {{- end -}}
 {{- end -}}
 
-{{/* Return the proper postgresqlSchema image */}}
+{{/* Return the proper dbSchema image */}}
 {{- define "filer.dbSchema.image" -}}
 {{- if .Values.filer.dbSchema.imageOverride -}}
 {{- $imageOverride := .Values.filer.dbSchema.imageOverride -}}
@@ -80,7 +80,7 @@ Inject extra environment vars in the format key:value, if populated
 {{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}}
 {{- $repositoryName := .Values.image.repository | toString -}}
 {{- $name := .Values.global.imageName | toString -}}
-{{- $tag := .Values.global.imageTag | toString -}}
+{{- $tag := .Chart.AppVersion | toString -}}
 {{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}}
 {{- end -}}
 {{- end -}}
@@ -94,7 +94,7 @@ Inject extra environment vars in the format key:value, if populated
 {{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}}
 {{- $repositoryName := .Values.image.repository | toString -}}
 {{- $name := .Values.global.imageName | toString -}}
-{{- $tag := .Values.global.imageTag | toString -}}
+{{- $tag := .Chart.AppVersion | toString -}}
 {{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}}
 {{- end -}}
 {{- end -}}
@@ -108,7 +108,44 @@ Inject extra environment vars in the format key:value, if populated
 {{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}}
 {{- $repositoryName := .Values.image.repository | toString -}}
 {{- $name := .Values.global.imageName | toString -}}
-{{- $tag := .Values.global.imageTag | toString -}}
+{{- $tag := .Chart.AppVersion | toString -}}
 {{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}}
 {{- end -}}
-{{- end -}}
+{{- end -}}
+
+{{/* Return the proper cronjob image */}}
+{{- define "cronjob.image" -}}
+{{- if .Values.cronjob.imageOverride -}}
+{{- $imageOverride := .Values.cronjob.imageOverride -}}
+{{- printf "%s" $imageOverride -}}
+{{- else -}}
+{{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}}
+{{- $repositoryName := .Values.image.repository | toString -}}
+{{- $name := .Values.global.imageName | toString -}}
+{{- $tag := .Chart.AppVersion | toString -}}
+{{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}}
+{{- end -}}
+{{- end -}}
+
+
+{{/* check if any PVC exists */}}
+{{- define "volume.pvc_exists" -}}
+{{- if or (or (eq .Values.volume.data.type "persistentVolumeClaim") (and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "persistentVolumeClaim") -}}
+{{- printf "true" -}}
+{{- else -}}
+{{- printf "false" -}}
+{{- end -}}
+{{- end -}}
+
+{{/* check if any HostPath exists */}}
+{{- define "volume.hostpath_exists" -}}
+{{- if or (or (eq .Values.volume.data.type "hostPath") (and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "hostPath") -}}
+{{- printf "true" -}}
+{{- else -}}
+{{- if or .Values.global.enableSecurity .Values.volume.extraVolumes -}}
+{{- printf "true" -}}
+{{- else -}}
+{{- printf "false" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}

k8s/seaweedfs/templates/cronjob.yaml

@@ -0,0 +1,58 @@
+{{- if .Values.cronjob }}
+{{- if .Values.cronjob.enabled }}
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: {{ include "seaweedfs.fullname" . }}-cronjob
+spec:
+  schedule: "{{ .Values.cronjob.schedule }}"
+  startingDeadlineSeconds: 200
+  concurrencyPolicy: Forbid
+  failedJobsHistoryLimit: 2
+  successfulJobsHistoryLimit: 2
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        spec:
+          {{- if .Values.cronjob.nodeSelector }}
+          nodeSelector:
+            {{ tpl .Values.cronjob.nodeSelector . | indent 12 | trim }}
+          {{- end }}
+          {{- if .Values.cronjob.tolerations }}
+          tolerations:
+            {{ tpl .Values.cronjob.tolerations . | nindent 12 | trim }}
+          {{- end }}
+          restartPolicy: OnFailure
+          containers:
+            - name: shell
+              image: {{ template "cronjob.image" . }}
+              imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }}
+              resources:
+              {{- toYaml .Values.cronjob.resources| nindent 16 }}
+              command:
+                - sh
+                - -c
+                - |
+                  set -ex
+                  echo -e "lock\n\
+                  volume.balance -force \
+                  {{ if .Values.volume.dataCenter }} -dataCenter {{ .Values.volume.dataCenter }}{{ end }}\
+                  {{ if .Values.cronjob.collection }} -collection {{ .Values.cronjob.collection }}{{ end }}\n\
+                  {{- if .Values.cronjob.enableFixReplication }}
+                  volume.fix.replication -collectionPattern={{ .Values.cronjob.collectionPattern }} \n\
+                  {{- end }}
+                  unlock\n" | \
+                  /usr/bin/weed shell \
+                  {{- if .Values.cronjob.master }}
+                  -master {{ .Values.cronjob.master }} \
+                  {{- else }}
+                  -master {{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}.svc:{{ .Values.master.port }} \
+                  {{- end }}
+                  {{- if .Values.cronjob.filer }}
+                  -filer {{ .Values.cronjob.filer }}
+                  {{- else }}
+                  -filer {{ template "seaweedfs.name" . }}-filer.{{ .Release.Namespace }}.svc:{{ .Values.filer.port }}
+                  {{- end }}
+{{- end }}
+{{- end }}

k8s/seaweedfs/templates/filer-service-client.yaml

@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "seaweedfs.name" . }}-filer-client
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "seaweedfs.name" . }}
+    component: filer
+{{- if .Values.filer.metricsPort }}
+    monitoring: "true"
+{{- end }}
+spec:
+  clusterIP: None
+  ports:
+  - name: "swfs-filer"
+    port: {{ .Values.filer.port }}
+    targetPort: {{ .Values.filer.port }}
+    protocol: TCP
+  - name: "swfs-filer-grpc"
+    port: {{ .Values.filer.grpcPort }}
+    targetPort: {{ .Values.filer.grpcPort }}
+    protocol: TCP
+{{- if .Values.filer.metricsPort }}
+  - name: "metrics"
+    port: {{ .Values.filer.metricsPort }}
+    targetPort: {{ .Values.filer.metricsPort }}
+    protocol: TCP
+{{- end }}
+  selector:
+    app: {{ template "seaweedfs.name" . }}
+    component: filer

k8s/seaweedfs/templates/filer-service.yaml

@@ -1,6 +1,8 @@
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
   name: {{ template "seaweedfs.name" . }}-filer
   namespace: {{ .Release.Namespace }}
   labels:
@@ -8,6 +10,7 @@ metadata:
     component: filer
 spec:
   clusterIP: None
+  publishNotReadyAddresses: true
   ports:
   - name: "swfs-filer"
     port: {{ .Values.filer.port }}
@@ -19,4 +22,4 @@ spec:
     protocol: TCP
   selector:
     app: {{ template "seaweedfs.name" . }}
-    component: filer
+    component: filer

k8s/seaweedfs/templates/filer-servicemonitor.yaml

@@ -0,0 +1,18 @@
+{{- if .Values.filer.metricsPort }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ template "seaweedfs.name" . }}-filer
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "seaweedfs.name" . }}
+    component: filer
+spec:
+  endpoints:
+    - interval: 30s
+      port: swfs-filer-metrics
+      scrapeTimeout: 5s
+  selector:
+    app: {{ template "seaweedfs.name" . }}
+    component: filer
+{{- end }}

k8s/seaweedfs/templates/filer-statefulset.yaml

@@ -87,6 +87,12 @@ spec:
               value: {{ $value | quote }}
             {{- end }}
             {{- end }}
+            {{- if .Values.global.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.global.extraEnvironmentVars }}
+            - name: {{ $key }}
+              value: {{ $value | quote }}
+            {{- end }}
+            {{- end }}
           command:
             - "/bin/sh"
             - "-ec"
@@ -99,6 +105,12 @@ spec:
               {{- end }}
               filer \
               -port={{ .Values.filer.port }} \
+              {{- if .Values.filer.metricsPort }}
+              -metricsPort {{ .Values.filer.metricsPort }} \
+              {{- end }}
+              {{- if .Values.filer.redirectOnRead }}
+              -redirectOnRead \
+              {{- end }}
               {{- if .Values.filer.disableHttp }}
               -disableHttp \
               {{- end }}
@@ -106,12 +118,51 @@ spec:
               -disableDirListing \
               {{- end }}
               -dirListLimit={{ .Values.filer.dirListLimit }} \
+              {{- if .Values.global.enableReplication }}
+              -defaultReplicaPlacement={{ .Values.global.replicationPlacment }} \
+              {{- else }}
+              -defaultReplicaPlacement={{ .Values.filer.defaultReplicaPlacement }} \
+              {{- end }}
+              {{- if .Values.filer.disableDirListing }}
+              -disableDirListing \
+              {{- end }}
+              {{- if .Values.filer.maxMB }}
+              -maxMB={{ .Values.filer.maxMB }} \
+              {{- end }}
+              {{- if .Values.filer.encryptVolumeData }}
+              -encryptVolumeData \
+              {{- end }}
               -ip=${POD_IP} \
+              {{- if .Values.filer.enable_peers }}
+              {{- if gt (.Values.filer.replicas | int) 1 }}
+              -peers=$(echo -n "{{ range $index := until (.Values.filer.replicas | int) }}${SEAWEEDFS_FULLNAME}-filer-{{ $index }}.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}{{ if lt $index (sub ($.Values.filer.replicas | int) 1) }},{{ end }}{{ end }}" | sed "s/$HOSTNAME.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}//" | sed 's/,$//; 's/^,//'; s/,,/,/;' )  \
+              {{- end }}
+              {{- end }}
+              {{- if .Values.filer.s3.enabled }}
+              -s3 \
+              -s3.port={{ .Values.filer.s3.port }} \
+              {{- if .Values.filer.s3.domainName }}
+              -s3.domainName={{ .Values.filer.s3.domainName }} \
+              {{- end }}
+              {{- if .Values.global.enableSecurity }}
+              -s3.cert.file=/usr/local/share/ca-certificates/client/tls.crt \
+              -s3.key.file=/usr/local/share/ca-certificates/client/tls.key \
+              {{- end }}
+              {{- if .Values.filer.s3.allowEmptyFolder }}
+              -s3.allowEmptyFolder={{ .Values.filer.s3.allowEmptyFolder }} \
+              {{- end }}
+              {{- if .Values.filer.s3.enableAuth }}
+              -s3.config=/etc/sw/seaweedfs_s3_config \
+              {{- end }}
+              {{- end }}
               -master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
           {{- if or (.Values.global.enableSecurity) (.Values.filer.extraVolumeMounts) }}    
           volumeMounts:
             - name: seaweedfs-filer-log-volume
               mountPath: "/logs/"
+            - mountPath: /etc/sw
+              name: config-users
+              readOnly: true
             {{- if .Values.global.enableSecurity }}
             - name: security-config
               readOnly: true
@@ -149,6 +200,7 @@ spec:
             periodSeconds: 15
             successThreshold: 1
             failureThreshold: 100
+            timeoutSeconds: 10
           livenessProbe:
             httpGet:
               path: /
@@ -158,6 +210,7 @@ spec:
             periodSeconds: 30
             successThreshold: 1
             failureThreshold: 5
+            timeoutSeconds: 10
           {{- if .Values.filer.resources }}
           resources:
             {{ tpl .Values.filer.resources . | nindent 12 | trim }}
@@ -167,6 +220,13 @@ spec:
           hostPath:
             path: /storage/logs/seaweedfs/filer
             type: DirectoryOrCreate
+        - name: db-schema-config-volume
+          configMap:
+            name: seaweedfs-db-init-config
+        - name: config-users
+          secret:
+            defaultMode: 420
+            secretName: seaweedfs-s3-secret
         {{- if .Values.global.enableSecurity }}
         - name: security-config
           configMap:

k8s/seaweedfs/templates/master-service.yaml

@@ -10,6 +10,7 @@ metadata:
     service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"    
 spec:
   clusterIP: None
+  publishNotReadyAddresses: true
   ports:
   - name: "swfs-master"
     port: {{ .Values.master.port }}
@@ -21,4 +22,4 @@ spec:
     protocol: TCP
   selector:
     app: {{ template "seaweedfs.name" . }}
-    component: master
+    component: master

k8s/seaweedfs/templates/master-statefulset.yaml

@@ -70,6 +70,18 @@ spec:
                   fieldPath: metadata.namespace
             - name: SEAWEEDFS_FULLNAME
               value: "{{ template "seaweedfs.name" . }}"
+            {{- if .Values.master.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.master.extraEnvironmentVars }}
+            - name: {{ $key }}
+              value: {{ $value | quote }}
+            {{- end }}
+            {{- end }}
+            {{- if .Values.global.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.global.extraEnvironmentVars }}
+            - name: {{ $key }}
+              value: {{ $value | quote }}
+            {{- end }}
+            {{- end }}
           command:
             - "/bin/sh"
             - "-ec"
@@ -84,6 +96,11 @@ spec:
               -port={{ .Values.master.port }} \
               -mdir=/data \
               -ip.bind={{ .Values.master.ipBind }} \
+              {{- if .Values.global.enableReplication }}
+              -defaultReplication={{ .Values.global.replicationPlacment }} \
+              {{- else }}
+              -defaultReplication={{ .Values.master.defaultReplication }} \
+              {{- end }}
               {{- if .Values.master.volumePreallocate }}
               -volumePreallocate \
               {{- end }}
@@ -94,6 +111,15 @@ spec:
               {{- if .Values.master.disableHttp }}
               -disableHttp \
               {{- end }}
+              {{- if .Values.master.pulseSeconds }}
+              -pulseSeconds={{ .Values.master.pulseSeconds }} \
+              {{- end }}
+              {{- if .Values.master.garbageThreshold }}
+              -garbageThreshold={{ .Values.master.garbageThreshold }} \
+              {{- end }}
+              {{- if .Values.master.metricsIntervalSec }}
+              -metrics.intervalSeconds={{ .Values.master.metricsIntervalSec }} \
+              {{- end }}
               -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master \
               -peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
           volumeMounts:
@@ -133,19 +159,21 @@ spec:
               path: /cluster/status
               port: {{ .Values.master.port }}
               scheme: HTTP
-            initialDelaySeconds: 5
-            periodSeconds: 15
+            initialDelaySeconds: 10
+            periodSeconds: 45
             successThreshold: 2
             failureThreshold: 100
+            timeoutSeconds: 10
           livenessProbe:
             httpGet:
               path: /cluster/status
               port: {{ .Values.master.port }}
               scheme: HTTP
             initialDelaySeconds: 20
-            periodSeconds: 10
+            periodSeconds: 30
             successThreshold: 1
-            failureThreshold: 6
+            failureThreshold: 4
+            timeoutSeconds: 10
           {{- if .Values.master.resources }}
           resources:
             {{ tpl .Values.master.resources . | nindent 12 | trim }}

k8s/seaweedfs/templates/s3-deployment.yaml

@@ -59,11 +59,17 @@ spec:
                   fieldPath: metadata.namespace
             - name: SEAWEEDFS_FULLNAME
               value: "{{ template "seaweedfs.name" . }}"
+            {{- if .Values.global.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.global.extraEnvironmentVars }}
+            - name: {{ $key }}
+              value: {{ $value | quote }}
+            {{- end }}
+            {{- end }}
           command:
             - "/bin/sh"
             - "-ec"
             - | 
-              exec /usr/bin/weed \
+              exec /usr/bin/weed -logdir=/logs \
               {{- if .Values.s3.loggingOverrideLevel }}
               -v={{ .Values.s3.loggingOverrideLevel }} \
               {{- else }}
@@ -71,6 +77,9 @@ spec:
               {{- end }}
               s3 \
               -port={{ .Values.s3.port }} \
+              {{- if .Values.s3.metricsPort }}
+              -metricsPort {{ .Values.s3.metricsPort }} \
+              {{- end }}
               {{- if .Values.global.enableSecurity }}
               -cert.file=/usr/local/share/ca-certificates/client/tls.crt \
               -key.file=/usr/local/share/ca-certificates/client/tls.key \
@@ -78,9 +87,19 @@ spec:
               {{- if .Values.s3.domainName }}
               -domainName={{ .Values.s3.domainName }} \
               {{- end }}
-              -filer={{ template "seaweedfs.name" . }}-filer:{{ .Values.filer.port }}
-          {{- if or (.Values.global.enableSecurity) (.Values.s3.extraVolumeMounts) }}
+              {{- if .Values.s3.allowEmptyFolder }}
+              -allowEmptyFolder={{ .Values.s3.allowEmptyFolder }} \
+              {{- end }}
+              {{- if .Values.s3.enableAuth }}
+              -config=/etc/sw/seaweedfs_s3_config \
+              {{- end }}
+              -filer={{ template "seaweedfs.name" . }}-filer-client:{{ .Values.filer.port }}
           volumeMounts:
+            - name: logs
+              mountPath: "/logs/"
+            - mountPath: /etc/sw
+              name: config-users
+              readOnly: true
             {{- if .Values.global.enableSecurity }}
             - name: security-config
               readOnly: true
@@ -103,7 +122,6 @@ spec:
               mountPath: /usr/local/share/ca-certificates/client/
             {{- end }}
             {{ tpl .Values.s3.extraVolumeMounts . | nindent 12 | trim }}
-          {{- end }}
           ports:
             - containerPort: {{ .Values.s3.port }}
               name: swfs-s3
@@ -116,6 +134,7 @@ spec:
             periodSeconds: 15
             successThreshold: 1
             failureThreshold: 100
+            timeoutSeconds: 10
           livenessProbe:
             httpGet:
               path: /
@@ -125,11 +144,22 @@ spec:
             periodSeconds: 60
             successThreshold: 1
             failureThreshold: 20
+            timeoutSeconds: 10
           {{- if .Values.s3.resources }}
           resources:
             {{ tpl .Values.s3.resources . | nindent 12 | trim }}
           {{- end }}
       volumes:
+        - name: config-users
+          secret:
+            defaultMode: 420
+            secretName: seaweedfs-s3-secret
+        {{- if eq .Values.s3.logs.type "hostPath" }}
+        - name: logs
+          hostPath:
+            path: /storage/logs/seaweedfs/s3
+            type: DirectoryOrCreate
+        {{- end }}
         {{- if .Values.global.enableSecurity }}
         - name: security-config
           configMap:

k8s/seaweedfs/templates/s3-service.yaml

@@ -9,9 +9,15 @@ metadata:
 spec:
   ports:
   - name: "swfs-s3"
-    port: {{ .Values.s3.port }}
-    targetPort: {{ .Values.s3.port }}
+    port: {{ if .Values.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }}
+    targetPort: {{ if .Values.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }}
     protocol: TCP
+{{- if and .Values.s3.enabled .Values.s3.metricsPort }}
+  - name: "metrics"
+    port: {{ .Values.s3.metricsPort }}
+    targetPort: {{ .Values.s3.metricsPort }}
+    protocol: TCP
+{{- end }}
   selector:
     app: {{ template "seaweedfs.name" . }}
-    component: s3
+    component: {{ if .Values.s3.enabled }}s3{{ else }}filer{{ end }}

k8s/seaweedfs/templates/s3-servicemonitor.yaml

@@ -0,0 +1,18 @@
+{{- if .Values.s3.metricsPort }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ template "seaweedfs.name" . }}-s3
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "seaweedfs.name" . }}
+    component: s3
+spec:
+  endpoints:
+    - interval: 30s
+      port: swfs-s3-metrics
+      scrapeTimeout: 5s
+  selector:
+    app: {{ template "seaweedfs.name" . }}
+    component: s3
+{{- end }}

k8s/seaweedfs/templates/seaweedfs-grafana-dashboard.yaml

@@ -0,0 +1,20 @@
+{{- if .Values.global.monitoring.enabled }}
+{{- $files := .Files.Glob "dashboards/*.json" }}
+{{- if $files }}
+apiVersion: v1
+kind: ConfigMapList
+items:
+{{- range $path, $fileContents := $files }}
+{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }}
+- apiVersion: v1
+  kind: ConfigMap
+  metadata:
+    name: {{ printf "%s" $dashboardName | lower | replace "_" "-" }}
+    namespace: {{ $.Release.Namespace }}
+    labels:
+     grafana_dashboard: "1"
+  data:
+    {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }}
+{{- end }}
+{{- end }}
+{{- end }}

k8s/seaweedfs/templates/seaweedfs-s3-secret.yaml

@@ -0,0 +1,21 @@
+{{- if not (or .Values.filer.s3.skipAuthSecretCreation .Values.s3.skipAuthSecretCreation) }}
+{{- $access_key_admin := randAlphaNum 16 -}}
+{{- $secret_key_admin := randAlphaNum 32 -}}
+{{- $access_key_read := randAlphaNum 16 -}}
+{{- $secret_key_read := randAlphaNum 32 -}}
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: seaweedfs-s3-secret
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/resource-policy": keep
+    "helm.sh/hook": "pre-install"
+stringData:
+  admin_access_key_id: {{ $access_key_admin }}
+  admin_secret_access_key: {{ $secret_key_admin }}
+  read_access_key_id: {{ $access_key_read }}
+  read_secret_access_key: {{ $secret_key_read }}
+  seaweedfs_s3_config: '{"identities":[{"name":"anvAdmin","credentials":[{"accessKey":"{{ $access_key_admin }}","secretKey":"{{ $secret_key_admin }}"}],"actions":["Admin","Read","Write"]},{"name":"anvReadOnly","credentials":[{"accessKey":"{{ $access_key_read }}","secretKey":"{{ $secret_key_read }}"}],"actions":["Read"]}]}'
+{{- end }}

k8s/seaweedfs/templates/volume-service.yaml

@@ -17,6 +17,12 @@ spec:
     port: {{ .Values.volume.grpcPort }}
     targetPort: {{ .Values.volume.grpcPort }}
     protocol: TCP
-  selector:
+{{- if .Values.volume.metricsPort }}
+  - name: "swfs-volume-metrics"
+    port: {{ .Values.volume.metricsPort }}
+    targetPort: {{ .Values.volume.metricsPort }}
+    protocol: TCP
+{{- end }}
+selector:
     app: {{ template "seaweedfs.name" . }}
     component: volume

k8s/seaweedfs/templates/volume-servicemonitor.yaml

@@ -0,0 +1,18 @@
+{{- if .Values.volume.metricsPort }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ template "seaweedfs.name" . }}-volume
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "seaweedfs.name" . }}
+    component: volume
+spec:
+  endpoints:
+    - interval: 30s
+      port: swfs-volume-metrics
+      scrapeTimeout: 5s
+  selector:
+    app: {{ template "seaweedfs.name" . }}
+    component: volume
+{{- end }}

k8s/seaweedfs/templates/volume-statefulset.yaml

@@ -12,6 +12,7 @@ metadata:
 spec:
   serviceName: {{ template "seaweedfs.name" . }}-volume
   replicas: {{ .Values.volume.replicas }}
+  podManagementPolicy: Parallel
   selector:
     matchLabels:
       app: {{ template "seaweedfs.name" . }}
@@ -33,7 +34,7 @@ spec:
       restartPolicy: {{ default .Values.global.restartPolicy .Values.volume.restartPolicy }}
       {{- if .Values.volume.tolerations }}
       tolerations:
-        {{ tpl .Values.volume.tolerations . | nindent 8 | trim }}
+      {{ tpl .Values.volume.tolerations . | nindent 8 | trim }}
       {{- end }}
       {{- if .Values.global.imagePullSecrets }}
       imagePullSecrets:
@@ -44,6 +45,19 @@ spec:
       priorityClassName: {{ .Values.volume.priorityClassName | quote }}
       {{- end }}
       enableServiceLinks: false
+      {{- if .Values.volume.dir_idx }}
+      initContainers:
+        - name: seaweedfs-vol-move-idx
+          image: {{ template "volume.image" . }}
+          imagePullPolicy: {{ .Values.global.pullPolicy | default "IfNotPresent" }}
+          command: [ '/bin/sh', '-c' ]
+          args: ['if ls {{ .Values.volume.dir }}/*.idx >/dev/null 2>&1; then mv {{ .Values.volume.dir }}/*.idx {{ .Values.volume.dir_idx }}/; fi;']
+          volumeMounts:
+          - name: idx
+            mountPath: {{ .Values.volume.dir_idx }}
+          - name: data
+            mountPath: {{ .Values.volume.dir }}
+      {{- end }}
       containers:
         - name: seaweedfs
           image: {{ template "volume.image" . }}
@@ -62,7 +76,13 @@ spec:
                 fieldRef:
                   fieldPath: status.hostIP
             - name: SEAWEEDFS_FULLNAME
-              value: "{{ template "seaweedfs.name" . }}"                  
+              value: "{{ template "seaweedfs.name" . }}"
+            {{- if .Values.global.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.global.extraEnvironmentVars }}
+            - name: {{ $key }}
+              value: {{ $value | quote }}
+            {{- end }}
+            {{- end }}
           command:
             - "/bin/sh"
             - "-ec"
@@ -75,7 +95,13 @@ spec:
                 {{- end }}
                 volume \
                 -port={{ .Values.volume.port }} \
+                {{- if .Values.volume.metricsPort }}
+                -metricsPort {{ .Values.volume.metricsPort }} \
+                {{- end }}
                 -dir={{ .Values.volume.dir }} \
+                {{- if .Values.volume.dir_idx }}
+                -dir.idx={{ .Values.volume.dir_idx }} \
+                {{- end }}
                 -max={{ .Values.volume.maxVolumes }} \
                 {{- if .Values.volume.rack }}
                 -rack={{ .Values.volume.rack }} \
@@ -91,13 +117,27 @@ spec:
                 {{- if .Values.volume.imagesFixOrientation }}
                 -images.fix.orientation \
                 {{- end }}
+                {{- if .Values.volume.pulseSeconds }}
+                -pulseSeconds={{ .Values.volume.pulseSeconds }} \
+                {{- end }}
+                {{- if .Values.volume.index }}
+                -index={{ .Values.volume.index }} \
+                {{- end }}
+                {{- if .Values.volume.fileSizeLimitMB }}
+                -fileSizeLimitMB={{ .Values.volume.fileSizeLimitMB }} \
+                {{- end }}
+                -minFreeSpacePercent={{ .Values.volume.minFreeSpacePercent }} \
                 -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume \
                 -compactionMBps={{ .Values.volume.compactionMBps }} \
                 -mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
           volumeMounts:
-            - name: seaweedfs-volume-storage
-              mountPath: "/data/"
-            - name: seaweedfs-volume-log-volume
+            - name: data
+              mountPath: "{{ .Values.volume.dir }}/"
+            {{- if .Values.volume.dir_idx }}
+            - name: idx
+              mountPath: "{{ .Values.volume.dir_idx }}/"
+            {{- end }}
+            - name: logs
               mountPath: "/logs/"
             {{- if .Values.global.enableSecurity }}
             - name: security-config
@@ -131,32 +171,46 @@ spec:
               path: /status
               port: {{ .Values.volume.port }}
               scheme: HTTP
-            initialDelaySeconds: 5
-            periodSeconds: 15
+            initialDelaySeconds: 15
+            periodSeconds: 90
             successThreshold: 1
             failureThreshold: 100
+            timeoutSeconds: 30
           livenessProbe:
             httpGet:
               path: /status
               port: {{ .Values.volume.port }}
               scheme: HTTP
             initialDelaySeconds: 20
-            periodSeconds: 30
+            periodSeconds: 90
             successThreshold: 1
-            failureThreshold: 10
+            failureThreshold: 4
+            timeoutSeconds: 30
           {{- if .Values.volume.resources }}
           resources:
             {{ tpl .Values.volume.resources . | nindent 12 | trim }}
           {{- end }}
+      {{- $hostpath_exists := include "volume.hostpath_exists" . -}}
+      {{- if $hostpath_exists }}
       volumes:
-        - name: seaweedfs-volume-log-volume
-          hostPath:
-            path: /storage/logs/seaweedfs/volume
-            type: DirectoryOrCreate
-        - name: seaweedfs-volume-storage
+        {{- if eq .Values.volume.data.type "hostPath" }}
+        - name: data
           hostPath:
             path: /storage/object_store/
             type: DirectoryOrCreate
+        {{- end }}
+        {{- if and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx }}
+        - name: idx
+          hostPath:
+            path: /ssd/seaweedfs-volume-idx/
+            type: DirectoryOrCreate
+        {{- end }}
+        {{- if eq .Values.volume.logs.type "hostPath" }}
+        - name: logs
+          hostPath:
+            path: /storage/logs/seaweedfs/volume
+            type: DirectoryOrCreate
+        {{- end }}
         {{- if .Values.global.enableSecurity }}
         - name: security-config
           configMap:
@@ -180,8 +234,43 @@ spec:
       {{- if .Values.volume.extraVolumes }}
         {{ tpl .Values.volume.extraVolumes . | indent 8 | trim }}
       {{- end }}
+      {{- end }}
       {{- if .Values.volume.nodeSelector }}
       nodeSelector:
         {{ tpl .Values.volume.nodeSelector . | indent 8 | trim }}
       {{- end }}
+  {{- $pvc_exists := include "volume.pvc_exists" . -}}
+  {{- if $pvc_exists }}
+  volumeClaimTemplates:
+    {{- if eq .Values.volume.data.type "persistentVolumeClaim"}}
+    - metadata:
+        name: data
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        storageClassName: {{ .Values.volume.data.storageClass }}
+        resources:
+          requests:
+            storage: {{ .Values.volume.data.size }}
+    {{- end }}
+    {{- if and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx }}
+    - metadata:
+        name: idx
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        storageClassName: {{ .Values.volume.idx.storageClass }}
+        resources:
+          requests:
+            storage: {{ .Values.volume.idx.size }}
+    {{- end }}
+    {{- if eq .Values.volume.logs.type "persistentVolumeClaim" }}
+    - metadata:
+        name: logs
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        storageClassName: {{ .Values.volume.logs.storageClass }}
+        resources:
+          requests:
+            storage: {{ .Values.volume.logs.size }}
+    {{- end }}
+      {{- end }}
 {{- end }}

k8s/seaweedfs/values.yaml

@@ -4,7 +4,7 @@ global:
   registry: ""
   repository: ""
   imageName: chrislusf/seaweedfs
-  imageTag: "1.78"
+  # imageTag: "2.30" - started using {.Chart.appVersion}
   imagePullPolicy: IfNotPresent
   imagePullSecrets: imagepullsecret
   restartPolicy: Always
@@ -14,6 +14,17 @@ global:
     enabled: false
     gatewayHost: null
     gatewayPort: null
+  # if enabled will use global.replicationPlacment and override master & filer defaultReplicaPlacement config
+  enableReplication: false
+  #  replication type is XYZ:
+  # X number of replica in other data centers
+  # Y number of replica in other racks in the same data center
+  # Z number of replica in other servers in the same rack
+  replicationPlacment: "001"
+  extraEnvironmentVars:
+    WEED_CLUSTER_DEFAULT: "sw"
+    WEED_CLUSTER_SW_MASTER: "seaweedfs-master:9333"
+    WEED_CLUSTER_SW_FILER: "seaweedfs-filer-client:8888"
 
 image:
   registry: ""
@@ -31,8 +42,20 @@ master:
   grpcPort: 19333
   ipBind: "0.0.0.0"
   volumePreallocate: false
+  #Master stops directing writes to oversized volumes
   volumeSizeLimitMB: 30000
   loggingOverrideLevel: null
+  #number of seconds between heartbeats, default 5
+  pulseSeconds: null
+  #threshold to vacuum and reclaim spaces, default 0.3 (30%)
+  garbageThreshold: null
+  #Prometheus push interval in seconds, default 15
+  metricsIntervalSec: 15
+  #  replication type is XYZ:
+  # X number of replica in other data centers
+  # Y number of replica in other racks in the same data center
+  # Z number of replica in other servers in the same rack
+  defaultReplication: "000"
 
   # Disable http request, only gRpc operations are allowed
   disableHttp: false
@@ -87,6 +110,11 @@ master:
   # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
   priorityClassName: ""
 
+  extraEnvironmentVars:
+    WEED_MASTER_VOLUME_GROWTH_COPY_1: 7
+    WEED_MASTER_VOLUME_GROWTH_COPY_2: 6
+    WEED_MASTER_VOLUME_GROWTH_COPY_3: 3
+    WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
 
 volume:
   enabled: true
@@ -97,15 +125,45 @@ volume:
   restartPolicy: null
   port: 8080
   grpcPort: 18080
+  metricsPort: 9327
   ipBind: "0.0.0.0"
   replicas: 1
   loggingOverrideLevel: null
+  # number of seconds between heartbeats, must be smaller than or equal to the master's setting
+  pulseSeconds: null
+  # Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance., default memory
+  index: null
+  # limit file size to avoid out of memory, default 256mb
+  fileSizeLimitMB: null
+  # minimum free disk space(in percents). If free disk space lower this value - all volumes marks as ReadOnly
+  minFreeSpacePercent: 7
+
+# can use ANY storage-class , example with local-path-provisner
+#  data:
+#    type: "persistentVolumeClaim"
+#    size: "24Ti"
+#    storageClass: "local-path-provisioner"
+  data:
+    type: "hostPath"
+    size: ""
+    storageClass: ""
+  idx:
+    type: "hostPath"
+    size: ""
+    storageClass: ""
+
+  logs:
+    type: "hostPath"
+    size: ""
+    storageClass: ""
 
   # limit background compaction or copying speed in mega bytes per second
-  compactionMBps: "40"
+  compactionMBps: "50"
 
   # Directories to store data files. dir[,dir]... (default "/tmp")
   dir: "/data"
+  # Directories to store index files. dir[,dir]... (default "/tmp")
+  dir_idx: null
 
   # Maximum numbers of volumes, count[,count]...
   # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
@@ -176,7 +234,24 @@ filer:
   replicas: 1
   port: 8888
   grpcPort: 18888
+  metricsPort: 9327
   loggingOverrideLevel: null
+  #  replication type is XYZ:
+  # X number of replica in other data centers
+  # Y number of replica in other racks in the same data center
+  # Z number of replica in other servers in the same rack
+  defaultReplicaPlacement: "000"
+  # turn off directory listing
+  disableDirListing: false
+  # split files larger than the limit, default 32
+  maxMB: null
+  # encrypt data on volume servers
+  encryptVolumeData: false
+  # enable peers sync metadata, for leveldb (localdb for filer but with sync across)
+  enable_peers: false
+
+  # Whether proxy or redirect to volume server during file GET request
+  redirectOnRead: false
 
   # Limit sub dir listing size (default 100000)
   dirListLimit: 100000
@@ -237,19 +312,16 @@ filer:
   # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
   priorityClassName: ""
 
-  dbSchema:
-    imageName: db-schema
-    imageTag: "development"
-    imageOverride: ""
-
   # extraEnvVars is a list of extra enviroment variables to set with the stateful set.
   extraEnvironmentVars:
     WEED_MYSQL_ENABLED: "true"
     WEED_MYSQL_HOSTNAME: "mysql-db-host"
     WEED_MYSQL_PORT: "3306"
     WEED_MYSQL_DATABASE: "sw_database"
-    WEED_MYSQL_CONNECTION_MAX_IDLE: "10"
-    WEED_MYSQL_CONNECTION_MAX_OPEN: "150"
+    WEED_MYSQL_CONNECTION_MAX_IDLE: "5"
+    WEED_MYSQL_CONNECTION_MAX_OPEN: "75"
+    # "refresh" connection every 10 minutes, eliminating mysql closing "old" connections
+    WEED_MYSQL_CONNECTION_MAX_LIFETIME_SECONDS: "600"
     # enable usage of memsql as filer backend
     WEED_MYSQL_INTERPOLATEPARAMS: "true"
     WEED_LEVELDB2_ENABLED: "false"
@@ -258,18 +330,33 @@ filer:
     WEED_FILER_OPTIONS_RECURSIVE_DELETE: "false"
     # directories under this folder will be automatically creating a separate bucket
     WEED_FILER_BUCKETS_FOLDER: "/buckets"
-    # directories under this folder will be store message queue data
-    WEED_FILER_QUEUES_FOLDER: "/queues"
+
+  s3:
+    enabled: true
+    port: 8333
+    #allow empty folders
+    allowEmptyFolder: false
+    # Suffix of the host name, {bucket}.{domainName}
+    domainName: ""
+    # enable user & permission to s3 (need to inject to all services)
+    enableAuth: false
+    skipAuthSecretCreation: false
 
 s3:
-  enabled: true
+  enabled: false
   repository: null
   imageName: null
   imageTag: null
   restartPolicy: null
   replicas: 1
   port: 8333
+  metricsPort: 9327
   loggingOverrideLevel: null
+  #allow empty folders
+  allowEmptyFolder: true
+  # enable user & permission to s3 (need to inject to all services)
+  enableAuth: false
+  skipAuthSecretCreation: false
 
   # Suffix of the host name, {bucket}.{domainName}
   domainName: ""
@@ -300,6 +387,28 @@ s3:
   # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
   priorityClassName: ""
 
+  logs:
+    type: "hostPath"
+    size: ""
+    storageClass: ""
+
+cronjob:
+  enabled: true
+  master: "seaweedfs-master:9333"
+  filer: "seaweedfs-filer-client:8888"
+  tolerations: ""
+  nodeSelector: |
+    sw-backend: "true"
+  replication:
+    enable: true
+    collectionPattern: ""
+  schedule: "*/7 * * * *"
+  resources: null
+  # balance all volumes among volume servers
+  # ALL|EACH_COLLECTION|<collection_name>
+  collection: ""
+
+
 certificates:
   commonName: "SeaweedFS CA"
   ipAddresses: []

other/java/client/pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.2.7</version>
+    <version>1.6.2</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -65,9 +65,14 @@
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
-            <version>4.12</version>
+            <version>4.13.1</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>javax.annotation</groupId>
+            <artifactId>javax.annotation-api</artifactId>
+            <version>1.3.2</version>
+        </dependency>
     </dependencies>
 
     <distributionManagement>

other/java/client/pom.xml.deploy

@@ -0,0 +1,170 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.github.chrislusf</groupId>
+    <artifactId>seaweedfs-client</artifactId>
+    <version>1.6.2</version>
+
+    <parent>
+        <groupId>org.sonatype.oss</groupId>
+        <artifactId>oss-parent</artifactId>
+        <version>9</version>
+    </parent>
+
+    <properties>
+        <protobuf.version>3.9.1</protobuf.version>
+        <!-- follow https://github.com/grpc/grpc-java -->
+        <grpc.version>1.23.0</grpc.version>
+        <guava.version>28.0-jre</guava.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.moandjiezana.toml</groupId>
+            <artifactId>toml4j</artifactId>
+            <version>0.7.2</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java</artifactId>
+            <version>${protobuf.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>${guava.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-netty-shaded</artifactId>
+            <version>${grpc.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-protobuf</artifactId>
+            <version>${grpc.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-stub</artifactId>
+            <version>${grpc.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.25</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpmime</artifactId>
+            <version>4.5.6</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.12</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <distributionManagement>
+        <snapshotRepository>
+            <id>ossrh</id>
+            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+        </snapshotRepository>
+    </distributionManagement>
+    <build>
+        <extensions>
+            <extension>
+                <groupId>kr.motd.maven</groupId>
+                <artifactId>os-maven-plugin</artifactId>
+                <version>1.6.2</version>
+            </extension>
+        </extensions>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>8</source>
+                    <target>8</target>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.xolstice.maven.plugins</groupId>
+                <artifactId>protobuf-maven-plugin</artifactId>
+                <version>0.6.1</version>
+                <configuration>
+                    <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
+                    </protocArtifact>
+                    <pluginId>grpc-java</pluginId>
+                    <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}
+                    </pluginArtifact>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>compile</goal>
+                            <goal>compile-custom</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-gpg-plugin</artifactId>
+                <version>1.5</version>
+                <executions>
+                    <execution>
+                        <id>sign-artifacts</id>
+                        <phase>verify</phase>
+                        <goals>
+                            <goal>sign</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.sonatype.plugins</groupId>
+                <artifactId>nexus-staging-maven-plugin</artifactId>
+                <version>1.6.7</version>
+                <extensions>true</extensions>
+                <configuration>
+                    <serverId>ossrh</serverId>
+                    <nexusUrl>https://oss.sonatype.org/</nexusUrl>
+                    <autoReleaseAfterClose>true</autoReleaseAfterClose>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <version>2.2.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar-no-fork</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <version>2.9.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-javadocs</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

other/java/client/pom_debug.xml

@@ -0,0 +1,144 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.github.chrislusf</groupId>
+    <artifactId>seaweedfs-client</artifactId>
+    <version>1.6.2</version>
+
+    <parent>
+        <groupId>org.sonatype.oss</groupId>
+        <artifactId>oss-parent</artifactId>
+        <version>9</version>
+    </parent>
+
+    <properties>
+        <protobuf.version>3.9.1</protobuf.version>
+        <!-- follow https://github.com/grpc/grpc-java -->
+        <grpc.version>1.23.0</grpc.version>
+        <guava.version>28.0-jre</guava.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.moandjiezana.toml</groupId>
+            <artifactId>toml4j</artifactId>
+            <version>0.7.2</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java</artifactId>
+            <version>${protobuf.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>${guava.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-netty-shaded</artifactId>
+            <version>${grpc.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-protobuf</artifactId>
+            <version>${grpc.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-stub</artifactId>
+            <version>${grpc.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.25</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpmime</artifactId>
+            <version>4.5.6</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.13.1</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>javax.annotation</groupId>
+            <artifactId>javax.annotation-api</artifactId>
+            <version>1.3.2</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <extensions>
+            <extension>
+                <groupId>kr.motd.maven</groupId>
+                <artifactId>os-maven-plugin</artifactId>
+                <version>1.6.2</version>
+            </extension>
+        </extensions>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>8</source>
+                    <target>8</target>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.xolstice.maven.plugins</groupId>
+                <artifactId>protobuf-maven-plugin</artifactId>
+                <version>0.6.1</version>
+                <configuration>
+                    <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
+                    </protocArtifact>
+                    <pluginId>grpc-java</pluginId>
+                    <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}
+                    </pluginArtifact>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>compile</goal>
+                            <goal>compile-custom</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <version>2.2.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar-no-fork</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <version>2.9.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-javadocs</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

other/java/client/src/main/java/seaweedfs/client/ByteBufferPool.java

@@ -0,0 +1,41 @@
+package seaweedfs.client;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ByteBufferPool {
+
+    private static final int MIN_BUFFER_SIZE = 8 * 1024 * 1024;
+    private static final Logger LOG = LoggerFactory.getLogger(ByteBufferPool.class);
+
+    private static final List<ByteBuffer> bufferList = new ArrayList<>();
+
+    public static synchronized ByteBuffer request(int bufferSize) {
+        if (bufferSize < MIN_BUFFER_SIZE) {
+            bufferSize = MIN_BUFFER_SIZE;
+        }
+        LOG.debug("requested new buffer {}", bufferSize);
+        if (bufferList.isEmpty()) {
+            return ByteBuffer.allocate(bufferSize);
+        }
+        ByteBuffer buffer = bufferList.remove(bufferList.size() - 1);
+        if (buffer.capacity() >= bufferSize) {
+            return buffer;
+        }
+
+        LOG.info("add new buffer from {} to {}", buffer.capacity(), bufferSize);
+        bufferList.add(0, buffer);
+        return ByteBuffer.allocate(bufferSize);
+
+    }
+
+    public static synchronized void release(ByteBuffer obj) {
+        obj.clear();
+        bufferList.add(0, obj);
+    }
+
+}

other/java/client/src/main/java/seaweedfs/client/ChunkCache.java

@@ -7,9 +7,12 @@ import java.util.concurrent.TimeUnit;
 
 public class ChunkCache {
 
-    private final Cache<String, byte[]> cache;
+    private Cache<String, byte[]> cache = null;
 
     public ChunkCache(int maxEntries) {
+        if (maxEntries == 0) {
+            return;
+        }
         this.cache = CacheBuilder.newBuilder()
                 .maximumSize(maxEntries)
                 .expireAfterAccess(1, TimeUnit.HOURS)
@@ -17,10 +20,16 @@ public class ChunkCache {
     }
 
     public byte[] getChunk(String fileId) {
+        if (this.cache == null) {
+            return null;
+        }
         return this.cache.getIfPresent(fileId);
     }
 
     public void setChunk(String fileId, byte[] data) {
+        if (this.cache == null) {
+            return;
+        }
         this.cache.put(fileId, data);
     }
 

other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java

@@ -0,0 +1,140 @@
+package seaweedfs.client;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class FileChunkManifest {
+
+    private static final Logger LOG = LoggerFactory.getLogger(FileChunkManifest.class);
+
+    private static final int mergeFactor = 1000;
+
+    public static boolean hasChunkManifest(List<FilerProto.FileChunk> chunks) {
+        for (FilerProto.FileChunk chunk : chunks) {
+            if (chunk.getIsChunkManifest()) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public static List<FilerProto.FileChunk> resolveChunkManifest(
+            final FilerClient filerClient, List<FilerProto.FileChunk> chunks) throws IOException {
+
+        List<FilerProto.FileChunk> dataChunks = new ArrayList<>();
+
+        for (FilerProto.FileChunk chunk : chunks) {
+            if (!chunk.getIsChunkManifest()) {
+                dataChunks.add(chunk);
+                continue;
+            }
+
+            // IsChunkManifest
+            LOG.debug("fetching chunk manifest:{}", chunk);
+            byte[] data = fetchChunk(filerClient, chunk);
+            FilerProto.FileChunkManifest m = FilerProto.FileChunkManifest.newBuilder().mergeFrom(data).build();
+            List<FilerProto.FileChunk> resolvedChunks = new ArrayList<>();
+            for (FilerProto.FileChunk t : m.getChunksList()) {
+                // avoid deprecated chunk.getFileId()
+                resolvedChunks.add(t.toBuilder().setFileId(FilerClient.toFileId(t.getFid())).build());
+            }
+            dataChunks.addAll(resolveChunkManifest(filerClient, resolvedChunks));
+        }
+
+        return dataChunks;
+    }
+
+    private static byte[] fetchChunk(final FilerClient filerClient, FilerProto.FileChunk chunk) throws IOException {
+
+        String vid = "" + chunk.getFid().getVolumeId();
+        FilerProto.Locations locations = filerClient.vidLocations.get(vid);
+        if (locations == null) {
+            FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder();
+            lookupRequest.addVolumeIds(vid);
+            FilerProto.LookupVolumeResponse lookupResponse = filerClient
+                    .getBlockingStub().lookupVolume(lookupRequest.build());
+            locations = lookupResponse.getLocationsMapMap().get(vid);
+            filerClient.vidLocations.put(vid, locations);
+            LOG.debug("fetchChunk vid:{} locations:{}", vid, locations);
+        }
+
+        SeaweedRead.ChunkView chunkView = new SeaweedRead.ChunkView(
+                FilerClient.toFileId(chunk.getFid()), // avoid deprecated chunk.getFileId()
+                0,
+                -1,
+                0,
+                true,
+                chunk.getCipherKey().toByteArray(),
+                chunk.getIsCompressed());
+
+        byte[] chunkData = SeaweedRead.chunkCache.getChunk(chunkView.fileId);
+        if (chunkData == null) {
+            LOG.debug("doFetchFullChunkData:{}", chunkView);
+            chunkData = SeaweedRead.doFetchFullChunkData(filerClient, chunkView, locations);
+        }
+        if (chunk.getIsChunkManifest()){
+            LOG.debug("chunk {} size {}", chunkView.fileId, chunkData.length);
+            SeaweedRead.chunkCache.setChunk(chunkView.fileId, chunkData);
+        }
+
+        return chunkData;
+
+    }
+
+    public static List<FilerProto.FileChunk> maybeManifestize(
+            final FilerClient filerClient, List<FilerProto.FileChunk> inputChunks, String parentDirectory) throws IOException {
+        // the return variable
+        List<FilerProto.FileChunk> chunks = new ArrayList<>();
+
+        List<FilerProto.FileChunk> dataChunks = new ArrayList<>();
+        for (FilerProto.FileChunk chunk : inputChunks) {
+            if (!chunk.getIsChunkManifest()) {
+                dataChunks.add(chunk);
+            } else {
+                chunks.add(chunk);
+            }
+        }
+
+        int remaining = dataChunks.size();
+        for (int i = 0; i + mergeFactor < dataChunks.size(); i += mergeFactor) {
+            FilerProto.FileChunk chunk = mergeIntoManifest(filerClient, dataChunks.subList(i, i + mergeFactor), parentDirectory);
+            chunks.add(chunk);
+            remaining -= mergeFactor;
+        }
+
+        // remaining
+        for (int i = dataChunks.size() - remaining; i < dataChunks.size(); i++) {
+            chunks.add(dataChunks.get(i));
+        }
+        return chunks;
+    }
+
+    private static FilerProto.FileChunk mergeIntoManifest(final FilerClient filerClient, List<FilerProto.FileChunk> dataChunks, String parentDirectory) throws IOException {
+        // create and serialize the manifest
+        dataChunks = FilerClient.beforeEntrySerialization(dataChunks);
+        FilerProto.FileChunkManifest.Builder m = FilerProto.FileChunkManifest.newBuilder().addAllChunks(dataChunks);
+        byte[] data = m.build().toByteArray();
+
+        long minOffset = Long.MAX_VALUE;
+        long maxOffset = -1;
+        for (FilerProto.FileChunk chunk : dataChunks) {
+            minOffset = Math.min(minOffset, chunk.getOffset());
+            maxOffset = Math.max(maxOffset, chunk.getSize() + chunk.getOffset());
+        }
+
+        FilerProto.FileChunk.Builder manifestChunk = SeaweedWrite.writeChunk(
+                filerClient.getReplication(),
+                filerClient,
+                minOffset,
+                data, 0, data.length, parentDirectory);
+        manifestChunk.setIsChunkManifest(true);
+        manifestChunk.setSize(maxOffset - minOffset);
+        return manifestChunk.build();
+
+    }
+
+}

other/java/client/src/main/java/seaweedfs/client/FilerClient.java

@@ -1,27 +1,82 @@
 package seaweedfs.client;
 
+import com.google.common.base.Strings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.nio.file.Path;
-import java.nio.file.Paths;
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 
-public class FilerClient {
+public class FilerClient extends FilerGrpcClient {
 
     private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class);
 
-    private final FilerGrpcClient filerGrpcClient;
-
     public FilerClient(String host, int grpcPort) {
-        filerGrpcClient = new FilerGrpcClient(host, grpcPort);
+        super(host, grpcPort);
     }
 
-    public FilerClient(FilerGrpcClient filerGrpcClient) {
-        this.filerGrpcClient = filerGrpcClient;
+    public static String toFileId(FilerProto.FileId fid) {
+        if (fid == null) {
+            return null;
+        }
+        return String.format("%d,%x%08x", fid.getVolumeId(), fid.getFileKey(), fid.getCookie());
+    }
+
+    public static FilerProto.FileId toFileIdObject(String fileIdStr) {
+        if (fileIdStr == null || fileIdStr.length() == 0) {
+            return null;
+        }
+        int commaIndex = fileIdStr.lastIndexOf(',');
+        String volumeIdStr = fileIdStr.substring(0, commaIndex);
+        String fileKeyStr = fileIdStr.substring(commaIndex + 1, fileIdStr.length() - 8);
+        String cookieStr = fileIdStr.substring(fileIdStr.length() - 8);
+
+        return FilerProto.FileId.newBuilder()
+                .setVolumeId(Integer.parseInt(volumeIdStr))
+                .setFileKey(Long.parseLong(fileKeyStr, 16))
+                .setCookie((int) Long.parseLong(cookieStr, 16))
+                .build();
+    }
+
+    public static List<FilerProto.FileChunk> beforeEntrySerialization(List<FilerProto.FileChunk> chunks) {
+        List<FilerProto.FileChunk> cleanedChunks = new ArrayList<>();
+        for (FilerProto.FileChunk chunk : chunks) {
+            FilerProto.FileChunk.Builder chunkBuilder = chunk.toBuilder();
+            chunkBuilder.clearFileId();
+            chunkBuilder.clearSourceFileId();
+            chunkBuilder.setFid(toFileIdObject(chunk.getFileId()));
+            FilerProto.FileId sourceFid = toFileIdObject(chunk.getSourceFileId());
+            if (sourceFid != null) {
+                chunkBuilder.setSourceFid(sourceFid);
+            }
+            cleanedChunks.add(chunkBuilder.build());
+        }
+        return cleanedChunks;
+    }
+
+    public static FilerProto.Entry afterEntryDeserialization(FilerProto.Entry entry) {
+        if (entry.getChunksList().size() <= 0) {
+            return entry;
+        }
+        String fileId = entry.getChunks(0).getFileId();
+        if (fileId != null && fileId.length() != 0) {
+            return entry;
+        }
+        FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
+        entryBuilder.clearChunks();
+        for (FilerProto.FileChunk chunk : entry.getChunksList()) {
+            FilerProto.FileChunk.Builder chunkBuilder = chunk.toBuilder();
+            chunkBuilder.setFileId(toFileId(chunk.getFid()));
+            String sourceFileId = toFileId(chunk.getSourceFid());
+            if (sourceFileId != null) {
+                chunkBuilder.setSourceFileId(sourceFileId);
+            }
+            entryBuilder.addChunks(chunkBuilder);
+        }
+        return entryBuilder.build();
     }
 
     public boolean mkdirs(String path, int mode) {
@@ -38,9 +93,9 @@ public class FilerClient {
         if ("/".equals(path)) {
             return true;
         }
-        Path pathObject = Paths.get(path);
-        String parent = pathObject.getParent().toString();
-        String name = pathObject.getFileName().toString();
+        File pathFile = new File(path);
+        String parent = pathFile.getParent();
+        String name = pathFile.getName();
 
         mkdirs(parent, mode, uid, gid, userName, groupNames);
 
@@ -59,13 +114,13 @@ public class FilerClient {
 
     public boolean mv(String oldPath, String newPath) {
 
-        Path oldPathObject = Paths.get(oldPath);
-        String oldParent = oldPathObject.getParent().toString();
-        String oldName = oldPathObject.getFileName().toString();
+        File oldPathFile = new File(oldPath);
+        String oldParent = oldPathFile.getParent();
+        String oldName = oldPathFile.getName();
 
-        Path newPathObject = Paths.get(newPath);
-        String newParent = newPathObject.getParent().toString();
-        String newName = newPathObject.getFileName().toString();
+        File newPathFile = new File(newPath);
+        String newParent = newPathFile.getParent();
+        String newName = newPathFile.getName();
 
         return atomicRenameEntry(oldParent, oldName, newParent, newName);
 
@@ -73,9 +128,9 @@ public class FilerClient {
 
     public boolean rm(String path, boolean isRecursive, boolean ignoreRecusiveError) {
 
-        Path pathObject = Paths.get(path);
-        String parent = pathObject.getParent().toString();
-        String name = pathObject.getFileName().toString();
+        File pathFile = new File(path);
+        String parent = pathFile.getParent();
+        String name = pathFile.getName();
 
         return deleteEntry(
                 parent,
@@ -92,9 +147,9 @@ public class FilerClient {
 
     public boolean touch(String path, int mode, int uid, int gid, String userName, String[] groupNames) {
 
-        Path pathObject = Paths.get(path);
-        String parent = pathObject.getParent().toString();
-        String name = pathObject.getFileName().toString();
+        File pathFile = new File(path);
+        String parent = pathFile.getParent();
+        String name = pathFile.getName();
 
         FilerProto.Entry entry = lookupEntry(parent, name);
         if (entry == null) {
@@ -156,7 +211,7 @@ public class FilerClient {
         List<FilerProto.Entry> results = new ArrayList<FilerProto.Entry>();
         String lastFileName = "";
         for (int limit = Integer.MAX_VALUE; limit > 0; ) {
-            List<FilerProto.Entry> t = listEntries(path, "", lastFileName, 1024);
+            List<FilerProto.Entry> t = listEntries(path, "", lastFileName, 1024, false);
             if (t == null) {
                 break;
             }
@@ -173,24 +228,25 @@ public class FilerClient {
         return results;
     }
 
-    public List<FilerProto.Entry> listEntries(String path, String entryPrefix, String lastEntryName, int limit) {
-        Iterator<FilerProto.ListEntriesResponse> iter = filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
+    public List<FilerProto.Entry> listEntries(String path, String entryPrefix, String lastEntryName, int limit, boolean includeLastEntry) {
+        Iterator<FilerProto.ListEntriesResponse> iter = this.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
                 .setDirectory(path)
                 .setPrefix(entryPrefix)
                 .setStartFromFileName(lastEntryName)
+                .setInclusiveStartFrom(includeLastEntry)
                 .setLimit(limit)
                 .build());
         List<FilerProto.Entry> entries = new ArrayList<>();
         while (iter.hasNext()) {
             FilerProto.ListEntriesResponse resp = iter.next();
-            entries.add(fixEntryAfterReading(resp.getEntry()));
+            entries.add(afterEntryDeserialization(resp.getEntry()));
         }
         return entries;
     }
 
     public FilerProto.Entry lookupEntry(String directory, String entryName) {
         try {
-            FilerProto.Entry entry = filerGrpcClient.getBlockingStub().lookupDirectoryEntry(
+            FilerProto.Entry entry = this.getBlockingStub().lookupDirectoryEntry(
                     FilerProto.LookupDirectoryEntryRequest.newBuilder()
                             .setDirectory(directory)
                             .setName(entryName)
@@ -198,7 +254,7 @@ public class FilerClient {
             if (entry == null) {
                 return null;
             }
-            return fixEntryAfterReading(entry);
+            return afterEntryDeserialization(entry);
         } catch (Exception e) {
             if (e.getMessage().indexOf("filer: no entry is found in filer store") > 0) {
                 return null;
@@ -208,28 +264,32 @@ public class FilerClient {
         }
     }
 
-
     public boolean createEntry(String parent, FilerProto.Entry entry) {
         try {
-            filerGrpcClient.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder()
-                    .setDirectory(parent)
-                    .setEntry(entry)
-                    .build());
+            FilerProto.CreateEntryResponse createEntryResponse =
+                    this.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder()
+                            .setDirectory(parent)
+                            .setEntry(entry)
+                            .build());
+            if (Strings.isNullOrEmpty(createEntryResponse.getError())) {
+                return true;
+            }
+            LOG.warn("createEntry {}/{} error: {}", parent, entry.getName(), createEntryResponse.getError());
+            return false;
         } catch (Exception e) {
             LOG.warn("createEntry {}/{}: {}", parent, entry.getName(), e);
             return false;
         }
-        return true;
     }
 
     public boolean updateEntry(String parent, FilerProto.Entry entry) {
         try {
-            filerGrpcClient.getBlockingStub().updateEntry(FilerProto.UpdateEntryRequest.newBuilder()
+            this.getBlockingStub().updateEntry(FilerProto.UpdateEntryRequest.newBuilder()
                     .setDirectory(parent)
                     .setEntry(entry)
                     .build());
         } catch (Exception e) {
-            LOG.warn("createEntry {}/{}: {}", parent, entry.getName(), e);
+            LOG.warn("updateEntry {}/{}: {}", parent, entry.getName(), e);
             return false;
         }
         return true;
@@ -237,7 +297,7 @@ public class FilerClient {
 
     public boolean deleteEntry(String parent, String entryName, boolean isDeleteFileChunk, boolean isRecursive, boolean ignoreRecusiveError) {
         try {
-            filerGrpcClient.getBlockingStub().deleteEntry(FilerProto.DeleteEntryRequest.newBuilder()
+            this.getBlockingStub().deleteEntry(FilerProto.DeleteEntryRequest.newBuilder()
                     .setDirectory(parent)
                     .setName(entryName)
                     .setIsDeleteData(isDeleteFileChunk)
@@ -253,7 +313,7 @@ public class FilerClient {
 
     public boolean atomicRenameEntry(String oldParent, String oldName, String newParent, String newName) {
         try {
-            filerGrpcClient.getBlockingStub().atomicRenameEntry(FilerProto.AtomicRenameEntryRequest.newBuilder()
+            this.getBlockingStub().atomicRenameEntry(FilerProto.AtomicRenameEntryRequest.newBuilder()
                     .setOldDirectory(oldParent)
                     .setOldName(oldName)
                     .setNewDirectory(newParent)
@@ -266,24 +326,13 @@ public class FilerClient {
         return true;
     }
 
-    private FilerProto.Entry fixEntryAfterReading(FilerProto.Entry entry) {
-        if (entry.getChunksList().size() <= 0) {
-            return entry;
-        }
-        String fileId = entry.getChunks(0).getFileId();
-        if (fileId != null && fileId.length() != 0) {
-            return entry;
-        }
-        FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
-        entryBuilder.clearChunks();
-        for (FilerProto.FileChunk chunk : entry.getChunksList()) {
-            FilerProto.FileChunk.Builder chunkBuilder = chunk.toBuilder();
-            FilerProto.FileId fid = chunk.getFid();
-            fileId = String.format("%d,%d%x", fid.getVolumeId(), fid.getFileKey(), fid.getCookie());
-            chunkBuilder.setFileId(fileId);
-            entryBuilder.addChunks(chunkBuilder);
-        }
-        return entryBuilder.build();
+    public Iterator<FilerProto.SubscribeMetadataResponse> watch(String prefix, String clientName, long sinceNs) {
+        return this.getBlockingStub().subscribeMetadata(FilerProto.SubscribeMetadataRequest.newBuilder()
+                .setPathPrefix(prefix)
+                .setClientName(clientName)
+                .setSinceNs(sinceNs)
+                .build()
+        );
     }
 
 }

other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java

@@ -9,6 +9,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import javax.net.ssl.SSLException;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 public class FilerGrpcClient {
@@ -24,6 +26,10 @@ public class FilerGrpcClient {
         }
     }
 
+    public final int VOLUME_SERVER_ACCESS_DIRECT = 0;
+    public final int VOLUME_SERVER_ACCESS_PUBLIC_URL = 1;
+    public final int VOLUME_SERVER_ACCESS_FILER_PROXY = 2;
+    public final Map<String, FilerProto.Locations> vidLocations = new HashMap<>();
     private final ManagedChannel channel;
     private final SeaweedFilerGrpc.SeaweedFilerBlockingStub blockingStub;
     private final SeaweedFilerGrpc.SeaweedFilerStub asyncStub;
@@ -31,6 +37,8 @@ public class FilerGrpcClient {
     private boolean cipher = false;
     private String collection = "";
     private String replication = "";
+    private int volumeServerAccess = VOLUME_SERVER_ACCESS_DIRECT;
+    private String filerAddress;
 
     public FilerGrpcClient(String host, int grpcPort) {
         this(host, grpcPort, sslContext);
@@ -39,11 +47,15 @@ public class FilerGrpcClient {
     public FilerGrpcClient(String host, int grpcPort, SslContext sslContext) {
 
         this(sslContext == null ?
-                ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext() :
+                ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext()
+                        .maxInboundMessageSize(1024 * 1024 * 1024) :
                 NettyChannelBuilder.forAddress(host, grpcPort)
+                        .maxInboundMessageSize(1024 * 1024 * 1024)
                         .negotiationType(NegotiationType.TLS)
                         .sslContext(sslContext));
 
+        filerAddress = String.format("%s:%d", host, grpcPort - 10000);
+
         FilerProto.GetFilerConfigurationResponse filerConfigurationResponse =
                 this.getBlockingStub().getFilerConfiguration(
                         FilerProto.GetFilerConfigurationRequest.newBuilder().build());
@@ -53,7 +65,7 @@ public class FilerGrpcClient {
 
     }
 
-    public FilerGrpcClient(ManagedChannelBuilder<?> channelBuilder) {
+    private FilerGrpcClient(ManagedChannelBuilder<?> channelBuilder) {
         channel = channelBuilder.build();
         blockingStub = SeaweedFilerGrpc.newBlockingStub(channel);
         asyncStub = SeaweedFilerGrpc.newStub(channel);
@@ -88,4 +100,39 @@ public class FilerGrpcClient {
         return futureStub;
     }
 
+    public void setAccessVolumeServerDirectly() {
+        this.volumeServerAccess = VOLUME_SERVER_ACCESS_DIRECT;
+    }
+
+    public boolean isAccessVolumeServerDirectly() {
+        return this.volumeServerAccess == VOLUME_SERVER_ACCESS_DIRECT;
+    }
+
+    public void setAccessVolumeServerByPublicUrl() {
+        this.volumeServerAccess = VOLUME_SERVER_ACCESS_PUBLIC_URL;
+    }
+
+    public boolean isAccessVolumeServerByPublicUrl() {
+        return this.volumeServerAccess == VOLUME_SERVER_ACCESS_PUBLIC_URL;
+    }
+
+    public void setAccessVolumeServerByFilerProxy() {
+        this.volumeServerAccess = VOLUME_SERVER_ACCESS_FILER_PROXY;
+    }
+
+    public boolean isAccessVolumeServerByFilerProxy() {
+        return this.volumeServerAccess == VOLUME_SERVER_ACCESS_FILER_PROXY;
+    }
+
+    public String getChunkUrl(String chunkId, String url, String publicUrl) {
+        switch (this.volumeServerAccess) {
+            case VOLUME_SERVER_ACCESS_PUBLIC_URL:
+                return String.format("http://%s/%s", publicUrl, chunkId);
+            case VOLUME_SERVER_ACCESS_FILER_PROXY:
+                return String.format("http://%s/?proxyChunkId=%s", this.filerAddress, chunkId);
+            default:
+                return String.format("http://%s/%s", url, chunkId);
+        }
+    }
+
 }

other/java/client/src/main/java/seaweedfs/client/Gzip.java

@@ -18,14 +18,18 @@ public class Gzip {
         return compressed;
     }
 
-    public static byte[] decompress(byte[] compressed) throws IOException {
-        ByteArrayInputStream bis = new ByteArrayInputStream(compressed);
-        GZIPInputStream gis = new GZIPInputStream(bis);
-        return readAll(gis);
+    public static byte[] decompress(byte[] compressed) {
+        try {
+            ByteArrayInputStream bis = new ByteArrayInputStream(compressed);
+            GZIPInputStream gis = new GZIPInputStream(bis);
+            return readAll(gis);
+        } catch (Exception e) {
+            return compressed;
+        }
     }
 
     private static byte[] readAll(InputStream input) throws IOException {
-        try( ByteArrayOutputStream output = new ByteArrayOutputStream()){
+        try (ByteArrayOutputStream output = new ByteArrayOutputStream()) {
             byte[] buffer = new byte[4096];
             int n;
             while (-1 != (n = input.read(buffer))) {

other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java

@@ -0,0 +1,208 @@
+package seaweedfs.client;
+
+// based on org.apache.hadoop.fs.azurebfs.services.AbfsInputStream
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+public class SeaweedInputStream extends InputStream {
+
+    private static final Logger LOG = LoggerFactory.getLogger(SeaweedInputStream.class);
+    private static final IOException EXCEPTION_STREAM_IS_CLOSED = new IOException("Stream is closed!");
+
+    private final FilerClient filerClient;
+    private final String path;
+    private final FilerProto.Entry entry;
+    private final List<SeaweedRead.VisibleInterval> visibleIntervalList;
+    private final long contentLength;
+
+    private long position = 0;  // cursor of the file
+
+    private boolean closed = false;
+
+    public SeaweedInputStream(
+            final FilerClient filerClient,
+            final String fullpath) throws IOException {
+        this.path = fullpath;
+        this.filerClient = filerClient;
+        this.entry = filerClient.lookupEntry(
+                SeaweedOutputStream.getParentDirectory(fullpath),
+                SeaweedOutputStream.getFileName(fullpath));
+        this.contentLength = SeaweedRead.fileSize(entry);
+
+        this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerClient, entry.getChunksList());
+
+        LOG.debug("new path:{} entry:{} visibleIntervalList:{}", path, entry, visibleIntervalList);
+
+    }
+
+    public SeaweedInputStream(
+            final FilerClient filerClient,
+            final String path,
+            final FilerProto.Entry entry) throws IOException {
+        this.filerClient = filerClient;
+        this.path = path;
+        this.entry = entry;
+        this.contentLength = SeaweedRead.fileSize(entry);
+
+        this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerClient, entry.getChunksList());
+
+        LOG.debug("new path:{} entry:{} visibleIntervalList:{}", path, entry, visibleIntervalList);
+
+    }
+
+    public String getPath() {
+        return path;
+    }
+
+    @Override
+    public int read() throws IOException {
+        byte[] b = new byte[1];
+        int numberOfBytesRead = read(b, 0, 1);
+        if (numberOfBytesRead < 0) {
+            return -1;
+        } else {
+            return (b[0] & 0xFF);
+        }
+    }
+
+    @Override
+    public int read(final byte[] b, final int off, final int len) throws IOException {
+
+        if (b == null) {
+            throw new IllegalArgumentException("null byte array passed in to read() method");
+        }
+        if (off >= b.length) {
+            throw new IllegalArgumentException("offset greater than length of array");
+        }
+        if (len < 0) {
+            throw new IllegalArgumentException("requested read length is less than zero");
+        }
+        if (len > (b.length - off)) {
+            throw new IllegalArgumentException("requested read length is more than will fit after requested offset in buffer");
+        }
+
+        ByteBuffer buf = ByteBuffer.wrap(b, off, len);
+        return read(buf);
+
+    }
+
+    // implement ByteBufferReadable
+    public synchronized int read(ByteBuffer buf) throws IOException {
+
+        if (position < 0) {
+            throw new IllegalArgumentException("attempting to read from negative offset");
+        }
+        if (position >= contentLength) {
+            return -1;  // Hadoop prefers -1 to EOFException
+        }
+
+        long bytesRead = 0;
+        int len = buf.remaining();
+        int start = (int) this.position;
+        if (start+len <= entry.getContent().size()) {
+            entry.getContent().substring(start, start+len).copyTo(buf);
+        } else {
+            bytesRead = SeaweedRead.read(this.filerClient, this.visibleIntervalList, this.position, buf, SeaweedRead.fileSize(entry));
+        }
+
+        if (bytesRead > Integer.MAX_VALUE) {
+            throw new IOException("Unexpected Content-Length");
+        }
+
+        if (bytesRead > 0) {
+            this.position += bytesRead;
+        }
+
+        return (int) bytesRead;
+    }
+
+    public synchronized void seek(long n) throws IOException {
+        if (closed) {
+            throw EXCEPTION_STREAM_IS_CLOSED;
+        }
+        if (n < 0) {
+            throw new EOFException("Cannot seek to a negative offset");
+        }
+        if (n > contentLength) {
+            throw new EOFException("Attempted to seek or read past the end of the file");
+        }
+        this.position = n;
+    }
+
+    @Override
+    public synchronized long skip(long n) throws IOException {
+        if (closed) {
+            throw EXCEPTION_STREAM_IS_CLOSED;
+        }
+        if (this.position == contentLength) {
+            if (n > 0) {
+                throw new EOFException("Attempted to seek or read past the end of the file");
+            }
+        }
+        long newPos = this.position + n;
+        if (newPos < 0) {
+            newPos = 0;
+            n = newPos - this.position;
+        }
+        if (newPos > contentLength) {
+            newPos = contentLength;
+            n = newPos - this.position;
+        }
+        seek(newPos);
+        return n;
+    }
+
+    /**
+     * Return the size of the remaining available bytes
+     * if the size is less than or equal to {@link Integer#MAX_VALUE},
+     * otherwise, return {@link Integer#MAX_VALUE}.
+     * <p>
+     * This is to match the behavior of DFSInputStream.available(),
+     * which some clients may rely on (HBase write-ahead log reading in
+     * particular).
+     */
+    @Override
+    public synchronized int available() throws IOException {
+        if (closed) {
+            throw EXCEPTION_STREAM_IS_CLOSED;
+        }
+        final long remaining = this.contentLength - this.position;
+        return remaining <= Integer.MAX_VALUE
+                ? (int) remaining : Integer.MAX_VALUE;
+    }
+
+    /**
+     * Returns the length of the file that this stream refers to. Note that the length returned is the length
+     * as of the time the Stream was opened. Specifically, if there have been subsequent appends to the file,
+     * they wont be reflected in the returned length.
+     *
+     * @return length of the file.
+     * @throws IOException if the stream is closed
+     */
+    public long length() throws IOException {
+        if (closed) {
+            throw EXCEPTION_STREAM_IS_CLOSED;
+        }
+        return contentLength;
+    }
+
+    public synchronized long getPos() throws IOException {
+        if (closed) {
+            throw EXCEPTION_STREAM_IS_CLOSED;
+        }
+        return position;
+    }
+
+    @Override
+    public synchronized void close() throws IOException {
+        closed = true;
+    }
+
+}

other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java

@@ -1,48 +1,50 @@
-package seaweed.hdfs;
+package seaweedfs.client;
 
 // adapted from org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream
 
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.hadoop.fs.Path;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import seaweedfs.client.FilerGrpcClient;
-import seaweedfs.client.FilerProto;
-import seaweedfs.client.SeaweedWrite;
 
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.io.OutputStream;
+import java.nio.Buffer;
+import java.nio.ByteBuffer;
 import java.util.concurrent.*;
 
-import static seaweed.hdfs.SeaweedFileSystemStore.getParentDirectory;
-
 public class SeaweedOutputStream extends OutputStream {
 
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedOutputStream.class);
-
-    private final FilerGrpcClient filerGrpcClient;
-    private final Path path;
+    protected final boolean supportFlush = true;
+    private final FilerClient filerClient;
+    private final String path;
     private final int bufferSize;
     private final int maxConcurrentRequestCount;
     private final ThreadPoolExecutor threadExecutor;
     private final ExecutorCompletionService<Void> completionService;
+    private final ConcurrentLinkedDeque<WriteOperation> writeOperations;
+    private final boolean shouldSaveMetadata = false;
     private FilerProto.Entry.Builder entry;
     private long position;
     private boolean closed;
-    private boolean supportFlush = true;
     private volatile IOException lastError;
     private long lastFlushOffset;
     private long lastTotalAppendOffset = 0;
-    private byte[] buffer;
-    private int bufferIndex;
-    private ConcurrentLinkedDeque<WriteOperation> writeOperations;
+    private ByteBuffer buffer;
+    private long outputIndex;
     private String replication = "000";
 
-    public SeaweedOutputStream(FilerGrpcClient filerGrpcClient, final Path path, FilerProto.Entry.Builder entry,
+    public SeaweedOutputStream(FilerClient filerClient, final String fullpath) {
+        this(filerClient, fullpath, "000");
+    }
+
+    public SeaweedOutputStream(FilerClient filerClient, final String fullpath, final String replication) {
+        this(filerClient, fullpath, null, 0, 8 * 1024 * 1024, "000");
+    }
+
+    public SeaweedOutputStream(FilerClient filerClient, final String path, FilerProto.Entry.Builder entry,
                                final long position, final int bufferSize, final String replication) {
-        this.filerGrpcClient = filerGrpcClient;
+        this.filerClient = filerClient;
         this.replication = replication;
         this.path = path;
         this.position = position;
@@ -50,30 +52,65 @@ public class SeaweedOutputStream extends OutputStream {
         this.lastError = null;
         this.lastFlushOffset = 0;
         this.bufferSize = bufferSize;
-        this.buffer = new byte[bufferSize];
-        this.bufferIndex = 0;
+        this.buffer = ByteBufferPool.request(bufferSize);
+        this.outputIndex = 0;
         this.writeOperations = new ConcurrentLinkedDeque<>();
 
-        this.maxConcurrentRequestCount = 4 * Runtime.getRuntime().availableProcessors();
+        this.maxConcurrentRequestCount = Runtime.getRuntime().availableProcessors();
 
         this.threadExecutor
-            = new ThreadPoolExecutor(maxConcurrentRequestCount,
-            maxConcurrentRequestCount,
-            10L,
-            TimeUnit.SECONDS,
-            new LinkedBlockingQueue<Runnable>());
+                = new ThreadPoolExecutor(maxConcurrentRequestCount,
+                maxConcurrentRequestCount,
+                120L,
+                TimeUnit.SECONDS,
+                new LinkedBlockingQueue<Runnable>());
         this.completionService = new ExecutorCompletionService<>(this.threadExecutor);
 
         this.entry = entry;
+        if (this.entry == null) {
+            long now = System.currentTimeMillis() / 1000L;
+
+            this.entry = FilerProto.Entry.newBuilder()
+                    .setName(getFileName(path))
+                    .setIsDirectory(false)
+                    .setAttributes(FilerProto.FuseAttributes.newBuilder()
+                            .setFileMode(0755)
+                            .setReplication(replication)
+                            .setCrtime(now)
+                            .setMtime(now)
+                            .clearGroupName()
+                    );
+        }
 
     }
 
+    public static String getParentDirectory(String path) {
+        int protoIndex = path.indexOf("://");
+        if (protoIndex >= 0) {
+            int pathStart = path.indexOf("/", protoIndex+3);
+            path = path.substring(pathStart);
+        }
+        if (path.equals("/")) {
+            return path;
+        }
+        int lastSlashIndex = path.lastIndexOf("/");
+        if (lastSlashIndex == 0) {
+            return "/";
+        }
+        return path.substring(0, lastSlashIndex);
+    }
+
+    public static String getFileName(String path) {
+        if (path.indexOf("/") < 0) {
+            return path;
+        }
+        int lastSlashIndex = path.lastIndexOf("/");
+        return path.substring(lastSlashIndex + 1);
+    }
+
     private synchronized void flushWrittenBytesToServiceInternal(final long offset) throws IOException {
-
-        LOG.debug("SeaweedWrite.writeMeta path: {} entry:{}", path, entry);
-
         try {
-            SeaweedWrite.writeMeta(filerGrpcClient, getParentDirectory(path), entry);
+            SeaweedWrite.writeMeta(filerClient, getParentDirectory(path), entry);
         } catch (Exception ex) {
             throw new IOException(ex);
         }
@@ -87,34 +124,40 @@ public class SeaweedOutputStream extends OutputStream {
 
     @Override
     public synchronized void write(final byte[] data, final int off, final int length)
-        throws IOException {
+            throws IOException {
         maybeThrowLastError();
 
-        Preconditions.checkArgument(data != null, "null data");
+        if (data == null) {
+            return;
+        }
 
         if (off < 0 || length < 0 || length > data.length - off) {
             throw new IndexOutOfBoundsException();
         }
 
+        // System.out.println(path + " write [" + (outputIndex + off) + "," + ((outputIndex + off) + length) + ")");
+
         int currentOffset = off;
-        int writableBytes = bufferSize - bufferIndex;
+        int writableBytes = bufferSize - buffer.position();
         int numberOfBytesToWrite = length;
 
         while (numberOfBytesToWrite > 0) {
-            if (writableBytes <= numberOfBytesToWrite) {
-                System.arraycopy(data, currentOffset, buffer, bufferIndex, writableBytes);
-                bufferIndex += writableBytes;
-                writeCurrentBufferToService();
-                currentOffset += writableBytes;
-                numberOfBytesToWrite = numberOfBytesToWrite - writableBytes;
-            } else {
-                System.arraycopy(data, currentOffset, buffer, bufferIndex, numberOfBytesToWrite);
-                bufferIndex += numberOfBytesToWrite;
-                numberOfBytesToWrite = 0;
+
+            if (numberOfBytesToWrite < writableBytes) {
+                buffer.put(data, currentOffset, numberOfBytesToWrite);
+                outputIndex += numberOfBytesToWrite;
+                break;
             }
 
-            writableBytes = bufferSize - bufferIndex;
+            // System.out.println(path + "     [" + (outputIndex + currentOffset) + "," + ((outputIndex + currentOffset) + writableBytes) + ") " + buffer.capacity());
+            buffer.put(data, currentOffset, writableBytes);
+            outputIndex += writableBytes;
+            currentOffset += writableBytes;
+            writeCurrentBufferToService();
+            numberOfBytesToWrite = numberOfBytesToWrite - writableBytes;
+            writableBytes = bufferSize - buffer.position();
         }
+
     }
 
     /**
@@ -149,47 +192,53 @@ public class SeaweedOutputStream extends OutputStream {
             flushInternal();
             threadExecutor.shutdown();
         } finally {
-            lastError = new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+            lastError = new IOException("Stream is closed!");
+            ByteBufferPool.release(buffer);
             buffer = null;
-            bufferIndex = 0;
+            outputIndex = 0;
             closed = true;
             writeOperations.clear();
             if (!threadExecutor.isShutdown()) {
                 threadExecutor.shutdownNow();
             }
         }
+
     }
 
     private synchronized void writeCurrentBufferToService() throws IOException {
-        if (bufferIndex == 0) {
+        if (buffer.position() == 0) {
             return;
         }
 
-        final byte[] bytes = buffer;
-        final int bytesLength = bufferIndex;
+        position += submitWriteBufferToService(buffer, position);
 
-        buffer = new byte[bufferSize];
-        bufferIndex = 0;
-        final long offset = position;
-        position += bytesLength;
+        buffer = ByteBufferPool.request(bufferSize);
 
-        if (threadExecutor.getQueue().size() >= maxConcurrentRequestCount * 2) {
+    }
+
+    private synchronized int submitWriteBufferToService(final ByteBuffer bufferToWrite, final long writePosition) throws IOException {
+
+        ((Buffer)bufferToWrite).flip();
+        int bytesLength = bufferToWrite.limit() - bufferToWrite.position();
+
+        if (threadExecutor.getQueue().size() >= maxConcurrentRequestCount) {
             waitForTaskToComplete();
         }
-
-        final Future<Void> job = completionService.submit(new Callable<Void>() {
-            @Override
-            public Void call() throws Exception {
-                // originally: client.append(path, offset, bytes, 0, bytesLength);
-                SeaweedWrite.writeData(entry, replication, filerGrpcClient, offset, bytes, 0, bytesLength);
-                return null;
-            }
+        final Future<Void> job = completionService.submit(() -> {
+            // System.out.println(path + " is going to save [" + (writePosition) + "," + ((writePosition) + bytesLength) + ")");
+            SeaweedWrite.writeData(entry, replication, filerClient, writePosition, bufferToWrite.array(), bufferToWrite.position(), bufferToWrite.limit(), path);
+            // System.out.println(path + " saved [" + (writePosition) + "," + ((writePosition) + bytesLength) + ")");
+            ByteBufferPool.release(bufferToWrite);
+            return null;
         });
 
-        writeOperations.add(new WriteOperation(job, offset, bytesLength));
+        writeOperations.add(new WriteOperation(job, writePosition, bytesLength));
 
         // Try to shrink the queue
         shrinkWriteOperationQueue();
+
+        return bytesLength;
+
     }
 
     private void waitForTaskToComplete() throws IOException {
@@ -231,13 +280,13 @@ public class SeaweedOutputStream extends OutputStream {
         }
     }
 
-    private synchronized void flushInternal() throws IOException {
+    protected synchronized void flushInternal() throws IOException {
         maybeThrowLastError();
         writeCurrentBufferToService();
         flushWrittenBytesToService();
     }
 
-    private synchronized void flushInternalAsync() throws IOException {
+    protected synchronized void flushInternalAsync() throws IOException {
         maybeThrowLastError();
         writeCurrentBufferToService();
         flushWrittenBytesToServiceAsync();
@@ -270,10 +319,6 @@ public class SeaweedOutputStream extends OutputStream {
         private final long length;
 
         WriteOperation(final Future<Void> task, final long startOffset, final long length) {
-            Preconditions.checkNotNull(task, "task");
-            Preconditions.checkArgument(startOffset >= 0, "startOffset");
-            Preconditions.checkArgument(length >= 0, "length");
-
             this.task = task;
             this.startOffset = startOffset;
             this.length = length;

other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java

@@ -1,102 +1,182 @@
 package seaweedfs.client;
 
+import org.apache.http.Header;
+import org.apache.http.HeaderElement;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHeaders;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
+import org.apache.http.client.entity.GzipDecompressingEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.util.EntityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import java.io.Closeable;
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.*;
 
 public class SeaweedRead {
 
-    // private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class);
+    private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class);
 
-    static ChunkCache chunkCache = new ChunkCache(1000);
+    static ChunkCache chunkCache = new ChunkCache(4);
+    static VolumeIdCache volumeIdCache = new VolumeIdCache(4 * 1024);
 
     // returns bytesRead
-    public static long read(FilerGrpcClient filerGrpcClient, List<VisibleInterval> visibleIntervals,
-                            final long position, final byte[] buffer, final int bufferOffset,
-                            final int bufferLength) throws IOException {
+    public static long read(FilerClient filerClient, List<VisibleInterval> visibleIntervals,
+                            final long position, final ByteBuffer buf, final long fileSize) throws IOException {
 
-        List<ChunkView> chunkViews = viewFromVisibles(visibleIntervals, position, bufferLength);
+        List<ChunkView> chunkViews = viewFromVisibles(visibleIntervals, position, buf.remaining());
+
+        Map<String, FilerProto.Locations> knownLocations = new HashMap<>();
 
         FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder();
         for (ChunkView chunkView : chunkViews) {
             String vid = parseVolumeId(chunkView.fileId);
-            lookupRequest.addVolumeIds(vid);
+            FilerProto.Locations locations = volumeIdCache.getLocations(vid);
+            if (locations == null) {
+                lookupRequest.addVolumeIds(vid);
+            } else {
+                knownLocations.put(vid, locations);
+            }
         }
 
-        FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient
-                .getBlockingStub().lookupVolume(lookupRequest.build());
-
-        Map<String, FilerProto.Locations> vid2Locations = lookupResponse.getLocationsMapMap();
+        if (lookupRequest.getVolumeIdsCount() > 0) {
+            FilerProto.LookupVolumeResponse lookupResponse = filerClient
+                    .getBlockingStub().lookupVolume(lookupRequest.build());
+            Map<String, FilerProto.Locations> vid2Locations = lookupResponse.getLocationsMapMap();
+            for (Map.Entry<String, FilerProto.Locations> entry : vid2Locations.entrySet()) {
+                volumeIdCache.setLocations(entry.getKey(), entry.getValue());
+                knownLocations.put(entry.getKey(), entry.getValue());
+            }
+        }
 
         //TODO parallel this
         long readCount = 0;
-        int startOffset = bufferOffset;
+        long startOffset = position;
         for (ChunkView chunkView : chunkViews) {
-            FilerProto.Locations locations = vid2Locations.get(parseVolumeId(chunkView.fileId));
-            if (locations.getLocationsCount() == 0) {
+
+            if (startOffset < chunkView.logicOffset) {
+                long gap = chunkView.logicOffset - startOffset;
+                LOG.debug("zero [{},{})", startOffset, startOffset + gap);
+                buf.position(buf.position()+ (int)gap);
+                readCount += gap;
+                startOffset += gap;
+            }
+
+            FilerProto.Locations locations = knownLocations.get(parseVolumeId(chunkView.fileId));
+            if (locations == null || locations.getLocationsCount() == 0) {
+                LOG.error("failed to locate {}", chunkView.fileId);
                 // log here!
                 return 0;
             }
 
-            int len = readChunkView(position, buffer, startOffset, chunkView, locations);
+            int len = readChunkView(filerClient, startOffset, buf, chunkView, locations);
+
+            LOG.debug("read [{},{}) {} size {}", startOffset, startOffset + len, chunkView.fileId, chunkView.size);
 
             readCount += len;
             startOffset += len;
 
         }
 
+        long limit = Math.min(buf.limit(), fileSize);
+
+        if (startOffset < limit) {
+            long gap = limit - startOffset;
+            LOG.debug("zero2 [{},{})", startOffset, startOffset + gap);
+            buf.position(buf.position()+ (int)gap);
+            readCount += gap;
+            startOffset += gap;
+        }
+
         return readCount;
     }
 
-    private static int readChunkView(long position, byte[] buffer, int startOffset, ChunkView chunkView, FilerProto.Locations locations) throws IOException {
+    private static int readChunkView(FilerClient filerClient, long startOffset, ByteBuffer buf, ChunkView chunkView, FilerProto.Locations locations) throws IOException {
 
         byte[] chunkData = chunkCache.getChunk(chunkView.fileId);
 
         if (chunkData == null) {
-            chunkData = doFetchFullChunkData(chunkView, locations);
+            chunkData = doFetchFullChunkData(filerClient, chunkView, locations);
+            chunkCache.setChunk(chunkView.fileId, chunkData);
         }
 
-        int len = (int) (chunkView.logicOffset - position + chunkView.size);
-        System.arraycopy(chunkData, (int) chunkView.offset, buffer, startOffset, len);
-
-        chunkCache.setChunk(chunkView.fileId, chunkData);
+        int len = (int) chunkView.size;
+        LOG.debug("readChunkView fid:{} chunkData.length:{} chunkView.offset:{} chunkView[{};{}) startOffset:{}",
+                chunkView.fileId, chunkData.length, chunkView.offset, chunkView.logicOffset, chunkView.logicOffset + chunkView.size, startOffset);
+        buf.put(chunkData, (int) (startOffset - chunkView.logicOffset + chunkView.offset), len);
 
         return len;
     }
 
-    private static byte[] doFetchFullChunkData(ChunkView chunkView, FilerProto.Locations locations) throws IOException {
-
-        HttpClient client = new DefaultHttpClient();
-        HttpGet request = new HttpGet(
-                String.format("http://%s/%s", locations.getLocations(0).getUrl(), chunkView.fileId));
-
-        request.setHeader(HttpHeaders.ACCEPT_ENCODING, "");
+    public static byte[] doFetchFullChunkData(FilerClient filerClient, ChunkView chunkView, FilerProto.Locations locations) throws IOException {
 
         byte[] data = null;
-
-        try {
-            HttpResponse response = client.execute(request);
-            HttpEntity entity = response.getEntity();
-
-            data = EntityUtils.toByteArray(entity);
-
-        } finally {
-            if (client instanceof Closeable) {
-                Closeable t = (Closeable) client;
-                t.close();
+        IOException lastException = null;
+        for (long waitTime = 1000L; waitTime < 10 * 1000; waitTime += waitTime / 2) {
+            for (FilerProto.Location location : locations.getLocationsList()) {
+                String url = filerClient.getChunkUrl(chunkView.fileId, location.getUrl(), location.getPublicUrl());
+                try {
+                    data = doFetchOneFullChunkData(chunkView, url);
+                    lastException = null;
+                    break;
+                } catch (IOException ioe) {
+                    LOG.debug("doFetchFullChunkData {} :{}", url, ioe);
+                    lastException = ioe;
+                }
+            }
+            if (data != null) {
+                break;
+            }
+            try {
+                Thread.sleep(waitTime);
+            } catch (InterruptedException e) {
             }
         }
 
-        if (chunkView.isGzipped) {
-            data = Gzip.decompress(data);
+        if (lastException != null) {
+            throw lastException;
+        }
+
+        LOG.debug("doFetchFullChunkData fid:{} chunkData.length:{}", chunkView.fileId, data.length);
+
+        return data;
+
+    }
+
+    private static byte[] doFetchOneFullChunkData(ChunkView chunkView, String url) throws IOException {
+
+        HttpGet request = new HttpGet(url);
+
+        request.setHeader(HttpHeaders.ACCEPT_ENCODING, "gzip");
+
+        byte[] data = null;
+
+        CloseableHttpResponse response = SeaweedUtil.getClosableHttpClient().execute(request);
+
+        try {
+            HttpEntity entity = response.getEntity();
+
+            Header contentEncodingHeader = entity.getContentEncoding();
+
+            if (contentEncodingHeader != null) {
+                HeaderElement[] encodings = contentEncodingHeader.getElements();
+                for (int i = 0; i < encodings.length; i++) {
+                    if (encodings[i].getName().equalsIgnoreCase("gzip")) {
+                        entity = new GzipDecompressingEntity(entity);
+                        break;
+                    }
+                }
+            }
+
+            data = EntityUtils.toByteArray(entity);
+
+            EntityUtils.consume(entity);
+
+        } finally {
+            response.close();
+            request.releaseConnection();
         }
 
         if (chunkView.cipherKey != null && chunkView.cipherKey.length != 0) {
@@ -107,6 +187,12 @@ public class SeaweedRead {
             }
         }
 
+        if (chunkView.isCompressed) {
+            data = Gzip.decompress(data);
+        }
+
+        LOG.debug("doFetchOneFullChunkData url:{} chunkData.length:{}", url, data.length);
+
         return data;
 
     }
@@ -116,29 +202,40 @@ public class SeaweedRead {
 
         long stop = offset + size;
         for (VisibleInterval chunk : visibleIntervals) {
-            if (chunk.start <= offset && offset < chunk.stop && offset < stop) {
+            long chunkStart = Math.max(offset, chunk.start);
+            long chunkStop = Math.min(stop, chunk.stop);
+            if (chunkStart < chunkStop) {
                 boolean isFullChunk = chunk.isFullChunk && chunk.start == offset && chunk.stop <= stop;
                 views.add(new ChunkView(
                         chunk.fileId,
-                        offset - chunk.start,
-                        Math.min(chunk.stop, stop) - offset,
-                        offset,
+                        chunkStart - chunk.start + chunk.chunkOffset,
+                        chunkStop - chunkStart,
+                        chunkStart,
                         isFullChunk,
                         chunk.cipherKey,
-                        chunk.isGzipped
+                        chunk.isCompressed
                 ));
-                offset = Math.min(chunk.stop, stop);
             }
         }
         return views;
     }
 
-    public static List<VisibleInterval> nonOverlappingVisibleIntervals(List<FilerProto.FileChunk> chunkList) {
+    public static List<VisibleInterval> nonOverlappingVisibleIntervals(
+            final FilerClient filerClient, List<FilerProto.FileChunk> chunkList) throws IOException {
+
+        chunkList = FileChunkManifest.resolveChunkManifest(filerClient, chunkList);
+
         FilerProto.FileChunk[] chunks = chunkList.toArray(new FilerProto.FileChunk[0]);
         Arrays.sort(chunks, new Comparator<FilerProto.FileChunk>() {
             @Override
             public int compare(FilerProto.FileChunk a, FilerProto.FileChunk b) {
-                return (int) (a.getMtime() - b.getMtime());
+                // if just a.getMtime() - b.getMtime(), it will overflow!
+                if (a.getMtime() < b.getMtime()) {
+                    return -1;
+                } else if (a.getMtime() > b.getMtime()) {
+                    return 1;
+                }
+                return 0;
             }
         });
 
@@ -159,9 +256,10 @@ public class SeaweedRead {
                 chunk.getOffset() + chunk.getSize(),
                 chunk.getFileId(),
                 chunk.getMtime(),
+                0,
                 true,
                 chunk.getCipherKey().toByteArray(),
-                chunk.getIsGzipped()
+                chunk.getIsCompressed()
         );
 
         // easy cases to speed up
@@ -181,9 +279,10 @@ public class SeaweedRead {
                         chunk.getOffset(),
                         v.fileId,
                         v.modifiedTime,
+                        v.chunkOffset,
                         false,
                         v.cipherKey,
-                        v.isGzipped
+                        v.isCompressed
                 ));
             }
             long chunkStop = chunk.getOffset() + chunk.getSize();
@@ -193,9 +292,10 @@ public class SeaweedRead {
                         v.stop,
                         v.fileId,
                         v.modifiedTime,
+                        v.chunkOffset + (chunkStop - v.start),
                         false,
                         v.cipherKey,
-                        v.isGzipped
+                        v.isCompressed
                 ));
             }
             if (chunkStop <= v.start || v.stop <= chunk.getOffset()) {
@@ -225,6 +325,10 @@ public class SeaweedRead {
         return fileId;
     }
 
+    public static long fileSize(FilerProto.Entry entry) {
+        return Math.max(totalSize(entry.getChunksList()), entry.getAttributes().getFileSize());
+    }
+
     public static long totalSize(List<FilerProto.FileChunk> chunksList) {
         long size = 0;
         for (FilerProto.FileChunk chunk : chunksList) {
@@ -241,18 +345,20 @@ public class SeaweedRead {
         public final long stop;
         public final long modifiedTime;
         public final String fileId;
+        public final long chunkOffset;
         public final boolean isFullChunk;
         public final byte[] cipherKey;
-        public final boolean isGzipped;
+        public final boolean isCompressed;
 
-        public VisibleInterval(long start, long stop, String fileId, long modifiedTime, boolean isFullChunk, byte[] cipherKey, boolean isGzipped) {
+        public VisibleInterval(long start, long stop, String fileId, long modifiedTime, long chunkOffset, boolean isFullChunk, byte[] cipherKey, boolean isCompressed) {
             this.start = start;
             this.stop = stop;
             this.modifiedTime = modifiedTime;
             this.fileId = fileId;
+            this.chunkOffset = chunkOffset;
             this.isFullChunk = isFullChunk;
             this.cipherKey = cipherKey;
-            this.isGzipped = isGzipped;
+            this.isCompressed = isCompressed;
         }
 
         @Override
@@ -264,7 +370,7 @@ public class SeaweedRead {
                     ", fileId='" + fileId + '\'' +
                     ", isFullChunk=" + isFullChunk +
                     ", cipherKey=" + Arrays.toString(cipherKey) +
-                    ", isGzipped=" + isGzipped +
+                    ", isCompressed=" + isCompressed +
                     '}';
         }
     }
@@ -276,16 +382,16 @@ public class SeaweedRead {
         public final long logicOffset;
         public final boolean isFullChunk;
         public final byte[] cipherKey;
-        public final boolean isGzipped;
+        public final boolean isCompressed;
 
-        public ChunkView(String fileId, long offset, long size, long logicOffset, boolean isFullChunk, byte[] cipherKey, boolean isGzipped) {
+        public ChunkView(String fileId, long offset, long size, long logicOffset, boolean isFullChunk, byte[] cipherKey, boolean isCompressed) {
             this.fileId = fileId;
             this.offset = offset;
             this.size = size;
             this.logicOffset = logicOffset;
             this.isFullChunk = isFullChunk;
             this.cipherKey = cipherKey;
-            this.isGzipped = isGzipped;
+            this.isCompressed = isCompressed;
         }
 
         @Override
@@ -297,7 +403,7 @@ public class SeaweedRead {
                     ", logicOffset=" + logicOffset +
                     ", isFullChunk=" + isFullChunk +
                     ", cipherKey=" + Arrays.toString(cipherKey) +
-                    ", isGzipped=" + isGzipped +
+                    ", isCompressed=" + isCompressed +
                     '}';
         }
     }

other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java

@@ -0,0 +1,30 @@
+package seaweedfs.client;
+
+import org.apache.http.impl.DefaultConnectionReuseStrategy;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+
+public class SeaweedUtil {
+
+    static PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
+    static CloseableHttpClient httpClient;
+
+    static {
+        // Increase max total connection to 200
+        cm.setMaxTotal(200);
+        // Increase default max connection per route to 20
+        cm.setDefaultMaxPerRoute(20);
+
+        httpClient = HttpClientBuilder.create()
+                .setConnectionManager(cm)
+                .setConnectionReuseStrategy(DefaultConnectionReuseStrategy.INSTANCE)
+                .setKeepAliveStrategy(DefaultConnectionKeepAliveStrategy.INSTANCE)
+                .build();
+    }
+
+    public static CloseableHttpClient getClosableHttpClient() {
+        return httpClient;
+    }
+}

other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java

@@ -1,72 +1,95 @@
 package seaweedfs.client;
 
 import com.google.protobuf.ByteString;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.entity.mime.HttpMultipartMode;
 import org.apache.http.entity.mime.MultipartEntityBuilder;
-import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.util.EntityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.ByteArrayInputStream;
-import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.security.SecureRandom;
+import java.util.List;
 
 public class SeaweedWrite {
 
-    private static SecureRandom random = new SecureRandom();
+    private static final Logger LOG = LoggerFactory.getLogger(SeaweedWrite.class);
+
+    private static final SecureRandom random = new SecureRandom();
 
     public static void writeData(FilerProto.Entry.Builder entry,
                                  final String replication,
-                                 final FilerGrpcClient filerGrpcClient,
+                                 final FilerClient filerClient,
                                  final long offset,
                                  final byte[] bytes,
-                                 final long bytesOffset, final long bytesLength) throws IOException {
-        FilerProto.AssignVolumeResponse response = filerGrpcClient.getBlockingStub().assignVolume(
+                                 final long bytesOffset, final long bytesLength,
+                                 final String path) throws IOException {
+        FilerProto.FileChunk.Builder chunkBuilder = writeChunk(
+                replication, filerClient, offset, bytes, bytesOffset, bytesLength, path);
+        synchronized (entry) {
+            entry.addChunks(chunkBuilder);
+        }
+    }
+
+    public static FilerProto.FileChunk.Builder writeChunk(final String replication,
+                                                          final FilerClient filerClient,
+                                                          final long offset,
+                                                          final byte[] bytes,
+                                                          final long bytesOffset,
+                                                          final long bytesLength,
+                                                          final String path) throws IOException {
+        FilerProto.AssignVolumeResponse response = filerClient.getBlockingStub().assignVolume(
                 FilerProto.AssignVolumeRequest.newBuilder()
-                        .setCollection(filerGrpcClient.getCollection())
-                        .setReplication(replication == null ? filerGrpcClient.getReplication() : replication)
+                        .setCollection(filerClient.getCollection())
+                        .setReplication(replication == null ? filerClient.getReplication() : replication)
                         .setDataCenter("")
                         .setTtlSec(0)
+                        .setPath(path)
                         .build());
         String fileId = response.getFileId();
-        String url = response.getUrl();
         String auth = response.getAuth();
-        String targetUrl = String.format("http://%s/%s", url, fileId);
+
+        String targetUrl = filerClient.getChunkUrl(fileId, response.getUrl(), response.getPublicUrl());
 
         ByteString cipherKeyString = com.google.protobuf.ByteString.EMPTY;
         byte[] cipherKey = null;
-        if (filerGrpcClient.isCipher()) {
+        if (filerClient.isCipher()) {
             cipherKey = genCipherKey();
             cipherKeyString = ByteString.copyFrom(cipherKey);
         }
 
         String etag = multipartUpload(targetUrl, auth, bytes, bytesOffset, bytesLength, cipherKey);
 
-        // cache fileId ~ bytes
-        SeaweedRead.chunkCache.setChunk(fileId, bytes);
+        LOG.debug("write file chunk {} size {}", targetUrl, bytesLength);
 
-        entry.addChunks(FilerProto.FileChunk.newBuilder()
+        return FilerProto.FileChunk.newBuilder()
                 .setFileId(fileId)
                 .setOffset(offset)
                 .setSize(bytesLength)
                 .setMtime(System.currentTimeMillis() / 10000L)
                 .setETag(etag)
-                .setCipherKey(cipherKeyString)
-        );
-
+                .setCipherKey(cipherKeyString);
     }
 
-    public static void writeMeta(final FilerGrpcClient filerGrpcClient,
-                                 final String parentDirectory, final FilerProto.Entry.Builder entry) {
-        filerGrpcClient.getBlockingStub().createEntry(
-                FilerProto.CreateEntryRequest.newBuilder()
-                        .setDirectory(parentDirectory)
-                        .setEntry(entry)
-                        .build()
-        );
+    public static void writeMeta(final FilerClient filerClient,
+                                 final String parentDirectory,
+                                 final FilerProto.Entry.Builder entry) throws IOException {
+
+        synchronized (entry) {
+            List<FilerProto.FileChunk> chunks = FileChunkManifest.maybeManifestize(filerClient, entry.getChunksList(), parentDirectory);
+            entry.clearChunks();
+            entry.addAllChunks(chunks);
+            filerClient.getBlockingStub().createEntry(
+                    FilerProto.CreateEntryRequest.newBuilder()
+                            .setDirectory(parentDirectory)
+                            .setEntry(entry)
+                            .build()
+            );
+        }
     }
 
     private static String multipartUpload(String targetUrl,
@@ -75,8 +98,6 @@ public class SeaweedWrite {
                                           final long bytesOffset, final long bytesLength,
                                           byte[] cipherKey) throws IOException {
 
-        HttpClient client = new DefaultHttpClient();
-
         InputStream inputStream = null;
         if (cipherKey == null || cipherKey.length == 0) {
             inputStream = new ByteArrayInputStream(bytes, (int) bytesOffset, (int) bytesLength);
@@ -99,8 +120,9 @@ public class SeaweedWrite {
                 .addBinaryBody("upload", inputStream)
                 .build());
 
+        CloseableHttpResponse response = SeaweedUtil.getClosableHttpClient().execute(post);
+
         try {
-            HttpResponse response = client.execute(post);
 
             String etag = response.getLastHeader("ETag").getValue();
 
@@ -108,12 +130,12 @@ public class SeaweedWrite {
                 etag = etag.substring(1, etag.length() - 1);
             }
 
+            EntityUtils.consume(response.getEntity());
+
             return etag;
         } finally {
-            if (client instanceof Closeable) {
-                Closeable t = (Closeable) client;
-                t.close();
-            }
+            response.close();
+            post.releaseConnection();
         }
 
     }

other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java

@@ -0,0 +1,36 @@
+package seaweedfs.client;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+
+import java.util.concurrent.TimeUnit;
+
+public class VolumeIdCache {
+
+    private Cache<String, FilerProto.Locations> cache = null;
+
+    public VolumeIdCache(int maxEntries) {
+        if (maxEntries == 0) {
+            return;
+        }
+        this.cache = CacheBuilder.newBuilder()
+                .maximumSize(maxEntries)
+                .expireAfterAccess(5, TimeUnit.MINUTES)
+                .build();
+    }
+
+    public FilerProto.Locations getLocations(String volumeId) {
+        if (this.cache == null) {
+            return null;
+        }
+        return this.cache.getIfPresent(volumeId);
+    }
+
+    public void setLocations(String volumeId, FilerProto.Locations locations) {
+        if (this.cache == null) {
+            return;
+        }
+        this.cache.put(volumeId, locations);
+    }
+
+}

other/java/client/src/main/proto/filer.proto

@@ -2,6 +2,7 @@ syntax = "proto3";
 
 package filer_pb;
 
+option go_package = "github.com/chrislusf/seaweedfs/weed/pb/filer_pb";
 option java_package = "seaweedfs.client";
 option java_outer_classname = "FilerProto";
 
@@ -36,6 +37,9 @@ service SeaweedFiler {
     rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) {
     }
 
+    rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) {
+    }
+
     rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
     }
 
@@ -48,12 +52,21 @@ service SeaweedFiler {
     rpc SubscribeMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) {
     }
 
+    rpc SubscribeLocalMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) {
+    }
+
     rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) {
     }
 
     rpc LocateBroker (LocateBrokerRequest) returns (LocateBrokerResponse) {
     }
 
+    rpc KvGet (KvGetRequest) returns (KvGetResponse) {
+    }
+
+    rpc KvPut (KvPutRequest) returns (KvPutResponse) {
+    }
+
 }
 
 //////////////////////////////////////////////////
@@ -85,6 +98,9 @@ message Entry {
     repeated FileChunk chunks = 3;
     FuseAttributes attributes = 4;
     map<string, bytes> extended = 5;
+    bytes hard_link_id = 7;
+    int32 hard_link_counter = 8; // only exists in hard link meta data
+    bytes content = 9; // if not empty, the file content
 }
 
 message FullEntry {
@@ -97,6 +113,8 @@ message EventNotification {
     Entry new_entry = 2;
     bool delete_chunks = 3;
     string new_parent_path = 4;
+    bool is_from_other_cluster = 5;
+    repeated int32 signatures = 6;
 }
 
 message FileChunk {
@@ -109,7 +127,12 @@ message FileChunk {
     FileId fid = 7;
     FileId source_fid = 8;
     bytes cipher_key = 9;
-    bool is_gzipped = 10;
+    bool is_compressed = 10;
+    bool is_chunk_manifest = 11; // content is a list of FileChunks
+}
+
+message FileChunkManifest {
+    repeated FileChunk chunks = 1;
 }
 
 message FileId {
@@ -133,12 +156,15 @@ message FuseAttributes {
     repeated string group_name = 12; // for hdfs
     string symlink_target = 13;
     bytes md5 = 14;
+    string disk_type = 15;
 }
 
 message CreateEntryRequest {
     string directory = 1;
     Entry entry = 2;
     bool o_excl = 3;
+    bool is_from_other_cluster = 4;
+    repeated int32 signatures = 5;
 }
 
 message CreateEntryResponse {
@@ -148,6 +174,8 @@ message CreateEntryResponse {
 message UpdateEntryRequest {
     string directory = 1;
     Entry entry = 2;
+    bool is_from_other_cluster = 3;
+    repeated int32 signatures = 4;
 }
 message UpdateEntryResponse {
 }
@@ -167,6 +195,8 @@ message DeleteEntryRequest {
     bool is_delete_data = 4;
     bool is_recursive = 5;
     bool ignore_recursive_error = 6;
+    bool is_from_other_cluster = 7;
+    repeated int32 signatures = 8;
 }
 
 message DeleteEntryResponse {
@@ -189,7 +219,9 @@ message AssignVolumeRequest {
     string replication = 3;
     int32 ttl_sec = 4;
     string data_center = 5;
-    string parent_path = 6;
+    string path = 6;
+    string rack = 7;
+    string disk_type = 8;
 }
 
 message AssignVolumeResponse {
@@ -219,6 +251,16 @@ message LookupVolumeResponse {
     map<string, Locations> locations_map = 1;
 }
 
+message Collection {
+    string name = 1;
+}
+message CollectionListRequest {
+    bool include_normal_volumes = 1;
+    bool include_ec_volumes = 2;
+}
+message CollectionListResponse {
+    repeated Collection collections = 1;
+}
 message DeleteCollectionRequest {
     string collection = 1;
 }
@@ -230,11 +272,9 @@ message StatisticsRequest {
     string replication = 1;
     string collection = 2;
     string ttl = 3;
+    string disk_type = 4;
 }
 message StatisticsResponse {
-    string replication = 1;
-    string collection = 2;
-    string ttl = 3;
     uint64 total_size = 4;
     uint64 used_size = 5;
     uint64 file_count = 6;
@@ -249,12 +289,16 @@ message GetFilerConfigurationResponse {
     uint32 max_mb = 4;
     string dir_buckets = 5;
     bool cipher = 7;
+    int32 signature = 8;
+    string metrics_address = 9;
+    int32 metrics_interval_sec = 10;
 }
 
 message SubscribeMetadataRequest {
     string client_name = 1;
     string path_prefix = 2;
     int64 since_ns = 3;
+    int32 signature = 4;
 }
 message SubscribeMetadataResponse {
     string directory = 1;
@@ -289,3 +333,34 @@ message LocateBrokerResponse {
     }
     repeated Resource resources = 2;
 }
+
+// Key-Value operations
+message KvGetRequest {
+    bytes key = 1;
+}
+message KvGetResponse {
+    bytes value = 1;
+    string error = 2;
+}
+message KvPutRequest {
+    bytes key = 1;
+    bytes value = 2;
+}
+message KvPutResponse {
+    string error = 1;
+}
+
+// path-based configurations
+message FilerConf {
+    int32 version = 1;
+    message PathConf {
+        string location_prefix = 1;
+        string collection = 2;
+        string replication = 3;
+        string ttl = 4;
+        string disk_type = 5;
+        bool fsync = 6;
+        uint32 volume_growth_count = 7;
+    }
+    repeated PathConf locations = 2;
+}

other/java/client/src/test/java/seaweedfs/client/SeaweedReadTest.java

@@ -3,13 +3,14 @@ package seaweedfs.client;
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
 public class SeaweedReadTest {
 
     @Test
-    public void testNonOverlappingVisibleIntervals() {
+    public void testNonOverlappingVisibleIntervals() throws IOException {
         List<FilerProto.FileChunk> chunks = new ArrayList<>();
         chunks.add(FilerProto.FileChunk.newBuilder()
             .setFileId("aaa")
@@ -24,7 +25,7 @@ public class SeaweedReadTest {
             .setMtime(2000)
             .build());
 
-        List<SeaweedRead.VisibleInterval> visibleIntervals = SeaweedRead.nonOverlappingVisibleIntervals(chunks);
+        List<SeaweedRead.VisibleInterval> visibleIntervals = SeaweedRead.nonOverlappingVisibleIntervals(null, chunks);
         for (SeaweedRead.VisibleInterval visibleInterval : visibleIntervals) {
             System.out.println("visible:" + visibleInterval);
         }

other/java/examples/pom.xml

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.example</groupId>
+    <artifactId>unzip</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <dependencies>
+        <dependency>
+            <groupId>com.github.chrislusf</groupId>
+            <artifactId>seaweedfs-client</artifactId>
+            <version>1.6.2</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.github.chrislusf</groupId>
+            <artifactId>seaweedfs-hadoop2-client</artifactId>
+            <version>1.6.2</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <version>2.9.2</version>
+            <scope>compile</scope>
+        </dependency>
+    </dependencies>
+
+
+</project>

other/java/examples/src/main/java/com/seaweedfs/examples/ExampleReadFile.java

@@ -0,0 +1,48 @@
+package com.seaweedfs.examples;
+
+import seaweedfs.client.FilerClient;
+import seaweedfs.client.SeaweedInputStream;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+public class ExampleReadFile {
+
+    public static void main(String[] args) throws IOException {
+
+        FilerClient filerClient = new FilerClient("localhost", 18888);
+
+        long startTime = System.currentTimeMillis();
+        parseZip("/Users/chris/tmp/test.zip");
+
+        long startTime2 = System.currentTimeMillis();
+
+        long localProcessTime = startTime2 - startTime;
+
+        SeaweedInputStream seaweedInputStream = new SeaweedInputStream(
+                filerClient, "/test.zip");
+        parseZip(seaweedInputStream);
+
+        long swProcessTime = System.currentTimeMillis() - startTime2;
+
+        System.out.println("Local time: " + localProcessTime);
+        System.out.println("SeaweedFS time: " + swProcessTime);
+
+    }
+
+    public static void parseZip(String filename) throws IOException {
+        FileInputStream fileInputStream = new FileInputStream(filename);
+        parseZip(fileInputStream);
+    }
+
+    public static void parseZip(InputStream is) throws IOException {
+        ZipInputStream zin = new ZipInputStream(is);
+        ZipEntry ze;
+        while ((ze = zin.getNextEntry()) != null) {
+            System.out.println(ze.getName());
+        }
+    }
+}

other/java/examples/src/main/java/com/seaweedfs/examples/ExampleWatchFileChanges.java

@@ -0,0 +1,46 @@
+package com.seaweedfs.examples;
+
+import seaweedfs.client.FilerClient;
+import seaweedfs.client.FilerProto;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.Iterator;
+
+public class ExampleWatchFileChanges {
+
+    public static void main(String[] args) throws IOException {
+
+        FilerClient filerClient = new FilerClient("localhost", 18888);
+
+        long sinceNs = (System.currentTimeMillis() - 3600 * 1000) * 1000000L;
+
+        Iterator<FilerProto.SubscribeMetadataResponse> watch = filerClient.watch(
+                "/buckets",
+                "exampleClientName",
+                sinceNs
+        );
+
+        System.out.println("Connected to filer, subscribing from " + new Date());
+
+        while (watch.hasNext()) {
+            FilerProto.SubscribeMetadataResponse event = watch.next();
+            FilerProto.EventNotification notification = event.getEventNotification();
+            if (!event.getDirectory().equals(notification.getNewParentPath())) {
+                // move an entry to a new directory, possibly with a new name
+                if (notification.hasOldEntry() && notification.hasNewEntry()) {
+                    System.out.println("moved " + event.getDirectory() + "/" + notification.getOldEntry().getName() + " to " + notification.getNewParentPath() + "/" + notification.getNewEntry().getName());
+                } else {
+                    System.out.println("this should not happen.");
+                }
+            } else if (notification.hasNewEntry() && !notification.hasOldEntry()) {
+                System.out.println("created entry " + event.getDirectory() + "/" + notification.getNewEntry().getName());
+            } else if (!notification.hasNewEntry() && notification.hasOldEntry()) {
+                System.out.println("deleted entry " + event.getDirectory() + "/" + notification.getOldEntry().getName());
+            } else if (notification.hasNewEntry() && notification.hasOldEntry()) {
+                System.out.println("updated entry " + event.getDirectory() + "/" + notification.getNewEntry().getName());
+            }
+        }
+
+    }
+}

other/java/examples/src/main/java/com/seaweedfs/examples/ExampleWriteFile.java

@@ -0,0 +1,47 @@
+package com.seaweedfs.examples;
+
+import seaweedfs.client.FilerClient;
+import seaweedfs.client.SeaweedInputStream;
+import seaweedfs.client.SeaweedOutputStream;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+public class ExampleWriteFile {
+
+    public static void main(String[] args) throws IOException {
+
+        FilerClient filerClient = new FilerClient("localhost", 18888);
+
+        SeaweedInputStream seaweedInputStream = new SeaweedInputStream(filerClient, "/test.zip");
+        unZipFiles(filerClient, seaweedInputStream);
+
+    }
+
+    public static void unZipFiles(FilerClient filerClient, InputStream is) throws IOException {
+        ZipInputStream zin = new ZipInputStream(is);
+        ZipEntry ze;
+        while ((ze = zin.getNextEntry()) != null) {
+
+            String filename = ze.getName();
+            if (filename.indexOf("/") >= 0) {
+                filename = filename.substring(filename.lastIndexOf("/") + 1);
+            }
+            if (filename.length()==0) {
+                continue;
+            }
+
+            SeaweedOutputStream seaweedOutputStream = new SeaweedOutputStream(filerClient, "/test/"+filename);
+            byte[] bytesIn = new byte[16 * 1024];
+            int read = 0;
+            while ((read = zin.read(bytesIn))!=-1) {
+                seaweedOutputStream.write(bytesIn,0,read);
+            }
+            seaweedOutputStream.close();
+
+            System.out.println(ze.getName());
+        }
+    }
+}

other/java/examples/src/main/java/com/seaweedfs/examples/HdfsCopyFile.java

@@ -0,0 +1,25 @@
+package com.seaweedfs.examples;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+
+import java.io.*;
+
+public class HdfsCopyFile {
+    public static void main(String[] args) throws IOException {
+        Configuration configuration = new Configuration();
+
+        configuration.set("fs.defaultFS", "seaweedfs://localhost:8888");
+        configuration.set("fs.seaweedfs.impl", "seaweed.hdfs.SeaweedFileSystem");
+
+        FileSystem fs = FileSystem.get(configuration);
+        String source = "/Users/chris/tmp/test.zip";
+        String destination = "/buckets/spark/test01.zip";
+        InputStream in = new BufferedInputStream(new FileInputStream(source));
+
+        OutputStream out = fs.create(new Path(destination));
+        IOUtils.copyBytes(in, out, 4096, true);
+    }
+}

other/java/hdfs2/dependency-reduced-pom.xml

@@ -15,8 +15,8 @@
       <plugin>
         <artifactId>maven-compiler-plugin</artifactId>
         <configuration>
-          <source>7</source>
-          <target>7</target>
+          <source>8</source>
+          <target>8</target>
         </configuration>
       </plugin>
       <plugin>
@@ -120,6 +120,180 @@
       </plugin>
     </plugins>
   </build>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client</artifactId>
+      <version>2.9.2</version>
+      <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <artifactId>hadoop-hdfs-client</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-mapreduce-client-app</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-yarn-api</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-mapreduce-client-core</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-annotations</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <version>2.9.2</version>
+      <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <artifactId>commons-cli</artifactId>
+          <groupId>commons-cli</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-math3</artifactId>
+          <groupId>org.apache.commons</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>xmlenc</artifactId>
+          <groupId>xmlenc</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-io</artifactId>
+          <groupId>commons-io</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-net</artifactId>
+          <groupId>commons-net</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-collections</artifactId>
+          <groupId>commons-collections</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>servlet-api</artifactId>
+          <groupId>javax.servlet</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jetty</artifactId>
+          <groupId>org.mortbay.jetty</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jetty-util</artifactId>
+          <groupId>org.mortbay.jetty</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jetty-sslengine</artifactId>
+          <groupId>org.mortbay.jetty</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jsp-api</artifactId>
+          <groupId>javax.servlet.jsp</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jersey-core</artifactId>
+          <groupId>com.sun.jersey</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jersey-json</artifactId>
+          <groupId>com.sun.jersey</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jersey-server</artifactId>
+          <groupId>com.sun.jersey</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>log4j</artifactId>
+          <groupId>log4j</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jets3t</artifactId>
+          <groupId>net.java.dev.jets3t</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-lang</artifactId>
+          <groupId>commons-lang</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-configuration</artifactId>
+          <groupId>commons-configuration</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-lang3</artifactId>
+          <groupId>org.apache.commons</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>slf4j-log4j12</artifactId>
+          <groupId>org.slf4j</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jackson-core-asl</artifactId>
+          <groupId>org.codehaus.jackson</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jackson-mapper-asl</artifactId>
+          <groupId>org.codehaus.jackson</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>avro</artifactId>
+          <groupId>org.apache.avro</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-auth</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jsch</artifactId>
+          <groupId>com.jcraft</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>curator-client</artifactId>
+          <groupId>org.apache.curator</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>curator-recipes</artifactId>
+          <groupId>org.apache.curator</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>htrace-core4</artifactId>
+          <groupId>org.apache.htrace</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>zookeeper</artifactId>
+          <groupId>org.apache.zookeeper</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-compress</artifactId>
+          <groupId>org.apache.commons</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>stax2-api</artifactId>
+          <groupId>org.codehaus.woodstox</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>woodstox-core</artifactId>
+          <groupId>com.fasterxml.woodstox</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-annotations</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+  </dependencies>
   <distributionManagement>
     <snapshotRepository>
       <id>ossrh</id>
@@ -127,7 +301,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.2.7</seaweedfs.client.version>
+    <seaweedfs.client.version>1.6.2</seaweedfs.client.version>
     <hadoop.version>2.9.2</hadoop.version>
   </properties>
 </project>

other/java/hdfs2/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.2.7</seaweedfs.client.version>
+        <seaweedfs.client.version>1.6.2</seaweedfs.client.version>
         <hadoop.version>2.9.2</hadoop.version>
     </properties>
 
@@ -31,8 +31,8 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
                 <configuration>
-                    <source>7</source>
-                    <target>7</target>
+                    <source>8</source>
+                    <target>8</target>
                 </configuration>
             </plugin>
             <plugin>
@@ -147,6 +147,7 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client</artifactId>
             <version>${hadoop.version}</version>
+            <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>com.github.chrislusf</groupId>
@@ -157,6 +158,7 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
             <version>${hadoop.version}</version>
+            <scope>provided</scope>
         </dependency>
     </dependencies>
 

other/java/hdfs2/src/main/java/seaweed/hdfs/BufferedByteBufferReadableInputStream.java

@@ -0,0 +1,25 @@
+package seaweed.hdfs;
+
+import org.apache.hadoop.fs.*;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class BufferedByteBufferReadableInputStream extends BufferedFSInputStream implements ByteBufferReadable {
+
+    public BufferedByteBufferReadableInputStream(FSInputStream in, int size) {
+        super(in, size);
+        if (!(in instanceof Seekable) || !(in instanceof PositionedReadable)) {
+            throw new IllegalArgumentException("In is not an instance of Seekable or PositionedReadable");
+        }
+    }
+
+    @Override
+    public int read(ByteBuffer buf) throws IOException {
+        if (this.in instanceof ByteBufferReadable) {
+            return ((ByteBufferReadable)this.in).read(buf);
+        } else {
+            throw new UnsupportedOperationException("Byte-buffer read unsupported by input stream");
+        }
+    }
+}

other/java/hdfs2/src/main/java/seaweed/hdfs/ReadBuffer.java

@@ -1,137 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package seaweed.hdfs;
-
-import java.util.concurrent.CountDownLatch;
-
-class ReadBuffer {
-
-  private SeaweedInputStream stream;
-  private long offset;                   // offset within the file for the buffer
-  private int length;                    // actual length, set after the buffer is filles
-  private int requestedLength;           // requested length of the read
-  private byte[] buffer;                 // the buffer itself
-  private int bufferindex = -1;          // index in the buffers array in Buffer manager
-  private ReadBufferStatus status;             // status of the buffer
-  private CountDownLatch latch = null;   // signaled when the buffer is done reading, so any client
-  // waiting on this buffer gets unblocked
-
-  // fields to help with eviction logic
-  private long timeStamp = 0;  // tick at which buffer became available to read
-  private boolean isFirstByteConsumed = false;
-  private boolean isLastByteConsumed = false;
-  private boolean isAnyByteConsumed = false;
-
-  public SeaweedInputStream getStream() {
-    return stream;
-  }
-
-  public void setStream(SeaweedInputStream stream) {
-    this.stream = stream;
-  }
-
-  public long getOffset() {
-    return offset;
-  }
-
-  public void setOffset(long offset) {
-    this.offset = offset;
-  }
-
-  public int getLength() {
-    return length;
-  }
-
-  public void setLength(int length) {
-    this.length = length;
-  }
-
-  public int getRequestedLength() {
-    return requestedLength;
-  }
-
-  public void setRequestedLength(int requestedLength) {
-    this.requestedLength = requestedLength;
-  }
-
-  public byte[] getBuffer() {
-    return buffer;
-  }
-
-  public void setBuffer(byte[] buffer) {
-    this.buffer = buffer;
-  }
-
-  public int getBufferindex() {
-    return bufferindex;
-  }
-
-  public void setBufferindex(int bufferindex) {
-    this.bufferindex = bufferindex;
-  }
-
-  public ReadBufferStatus getStatus() {
-    return status;
-  }
-
-  public void setStatus(ReadBufferStatus status) {
-    this.status = status;
-  }
-
-  public CountDownLatch getLatch() {
-    return latch;
-  }
-
-  public void setLatch(CountDownLatch latch) {
-    this.latch = latch;
-  }
-
-  public long getTimeStamp() {
-    return timeStamp;
-  }
-
-  public void setTimeStamp(long timeStamp) {
-    this.timeStamp = timeStamp;
-  }
-
-  public boolean isFirstByteConsumed() {
-    return isFirstByteConsumed;
-  }
-
-  public void setFirstByteConsumed(boolean isFirstByteConsumed) {
-    this.isFirstByteConsumed = isFirstByteConsumed;
-  }
-
-  public boolean isLastByteConsumed() {
-    return isLastByteConsumed;
-  }
-
-  public void setLastByteConsumed(boolean isLastByteConsumed) {
-    this.isLastByteConsumed = isLastByteConsumed;
-  }
-
-  public boolean isAnyByteConsumed() {
-    return isAnyByteConsumed;
-  }
-
-  public void setAnyByteConsumed(boolean isAnyByteConsumed) {
-    this.isAnyByteConsumed = isAnyByteConsumed;
-  }
-
-}

other/java/hdfs2/src/main/java/seaweed/hdfs/ReadBufferManager.java

@@ -1,394 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package seaweed.hdfs;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.Queue;
-import java.util.Stack;
-import java.util.concurrent.CountDownLatch;
-
-/**
- * The Read Buffer Manager for Rest AbfsClient.
- */
-final class ReadBufferManager {
-  private static final Logger LOGGER = LoggerFactory.getLogger(ReadBufferManager.class);
-
-  private static final int NUM_BUFFERS = 16;
-  private static final int BLOCK_SIZE = 4 * 1024 * 1024;
-  private static final int NUM_THREADS = 8;
-  private static final int THRESHOLD_AGE_MILLISECONDS = 3000; // have to see if 3 seconds is a good threshold
-
-  private Thread[] threads = new Thread[NUM_THREADS];
-  private byte[][] buffers;    // array of byte[] buffers, to hold the data that is read
-  private Stack<Integer> freeList = new Stack<>();   // indices in buffers[] array that are available
-
-  private Queue<ReadBuffer> readAheadQueue = new LinkedList<>(); // queue of requests that are not picked up by any worker thread yet
-  private LinkedList<ReadBuffer> inProgressList = new LinkedList<>(); // requests being processed by worker threads
-  private LinkedList<ReadBuffer> completedReadList = new LinkedList<>(); // buffers available for reading
-  private static final ReadBufferManager BUFFER_MANAGER; // singleton, initialized in static initialization block
-
-  static {
-    BUFFER_MANAGER = new ReadBufferManager();
-    BUFFER_MANAGER.init();
-  }
-
-  static ReadBufferManager getBufferManager() {
-    return BUFFER_MANAGER;
-  }
-
-  private void init() {
-    buffers = new byte[NUM_BUFFERS][];
-    for (int i = 0; i < NUM_BUFFERS; i++) {
-      buffers[i] = new byte[BLOCK_SIZE];  // same buffers are reused. The byte array never goes back to GC
-      freeList.add(i);
-    }
-    for (int i = 0; i < NUM_THREADS; i++) {
-      Thread t = new Thread(new ReadBufferWorker(i));
-      t.setDaemon(true);
-      threads[i] = t;
-      t.setName("SeaweedFS-prefetch-" + i);
-      t.start();
-    }
-    ReadBufferWorker.UNLEASH_WORKERS.countDown();
-  }
-
-  // hide instance constructor
-  private ReadBufferManager() {
-  }
-
-
-  /*
-   *
-   *  SeaweedInputStream-facing methods
-   *
-   */
-
-
-  /**
-   * {@link SeaweedInputStream} calls this method to queue read-aheads.
-   *
-   * @param stream          The {@link SeaweedInputStream} for which to do the read-ahead
-   * @param requestedOffset The offset in the file which shoukd be read
-   * @param requestedLength The length to read
-   */
-  void queueReadAhead(final SeaweedInputStream stream, final long requestedOffset, final int requestedLength) {
-    if (LOGGER.isTraceEnabled()) {
-      LOGGER.trace("Start Queueing readAhead for {} offset {} length {}",
-          stream.getPath(), requestedOffset, requestedLength);
-    }
-    ReadBuffer buffer;
-    synchronized (this) {
-      if (isAlreadyQueued(stream, requestedOffset)) {
-        return; // already queued, do not queue again
-      }
-      if (freeList.isEmpty() && !tryEvict()) {
-        return; // no buffers available, cannot queue anything
-      }
-
-      buffer = new ReadBuffer();
-      buffer.setStream(stream);
-      buffer.setOffset(requestedOffset);
-      buffer.setLength(0);
-      buffer.setRequestedLength(requestedLength);
-      buffer.setStatus(ReadBufferStatus.NOT_AVAILABLE);
-      buffer.setLatch(new CountDownLatch(1));
-
-      Integer bufferIndex = freeList.pop();  // will return a value, since we have checked size > 0 already
-
-      buffer.setBuffer(buffers[bufferIndex]);
-      buffer.setBufferindex(bufferIndex);
-      readAheadQueue.add(buffer);
-      notifyAll();
-    }
-    if (LOGGER.isTraceEnabled()) {
-      LOGGER.trace("Done q-ing readAhead for file {} offset {} buffer idx {}",
-          stream.getPath(), requestedOffset, buffer.getBufferindex());
-    }
-  }
-
-
-  /**
-   * {@link SeaweedInputStream} calls this method read any bytes already available in a buffer (thereby saving a
-   * remote read). This returns the bytes if the data already exists in buffer. If there is a buffer that is reading
-   * the requested offset, then this method blocks until that read completes. If the data is queued in a read-ahead
-   * but not picked up by a worker thread yet, then it cancels that read-ahead and reports cache miss. This is because
-   * depending on worker thread availability, the read-ahead may take a while - the calling thread can do it's own
-   * read to get the data faster (copmared to the read waiting in queue for an indeterminate amount of time).
-   *
-   * @param stream   the file to read bytes for
-   * @param position the offset in the file to do a read for
-   * @param length   the length to read
-   * @param buffer   the buffer to read data into. Note that the buffer will be written into from offset 0.
-   * @return the number of bytes read
-   */
-  int getBlock(final SeaweedInputStream stream, final long position, final int length, final byte[] buffer) {
-    // not synchronized, so have to be careful with locking
-    if (LOGGER.isTraceEnabled()) {
-      LOGGER.trace("getBlock for file {}  position {}  thread {}",
-          stream.getPath(), position, Thread.currentThread().getName());
-    }
-
-    waitForProcess(stream, position);
-
-    int bytesRead = 0;
-    synchronized (this) {
-      bytesRead = getBlockFromCompletedQueue(stream, position, length, buffer);
-    }
-    if (bytesRead > 0) {
-      if (LOGGER.isTraceEnabled()) {
-        LOGGER.trace("Done read from Cache for {} position {} length {}",
-            stream.getPath(), position, bytesRead);
-      }
-      return bytesRead;
-    }
-
-    // otherwise, just say we got nothing - calling thread can do its own read
-    return 0;
-  }
-
-  /*
-   *
-   *  Internal methods
-   *
-   */
-
-  private void waitForProcess(final SeaweedInputStream stream, final long position) {
-    ReadBuffer readBuf;
-    synchronized (this) {
-      clearFromReadAheadQueue(stream, position);
-      readBuf = getFromList(inProgressList, stream, position);
-    }
-    if (readBuf != null) {         // if in in-progress queue, then block for it
-      try {
-        if (LOGGER.isTraceEnabled()) {
-          LOGGER.trace("got a relevant read buffer for file {} offset {} buffer idx {}",
-              stream.getPath(), readBuf.getOffset(), readBuf.getBufferindex());
-        }
-        readBuf.getLatch().await();  // blocking wait on the caller stream's thread
-        // Note on correctness: readBuf gets out of inProgressList only in 1 place: after worker thread
-        // is done processing it (in doneReading). There, the latch is set after removing the buffer from
-        // inProgressList. So this latch is safe to be outside the synchronized block.
-        // Putting it in synchronized would result in a deadlock, since this thread would be holding the lock
-        // while waiting, so no one will be able to  change any state. If this becomes more complex in the future,
-        // then the latch cane be removed and replaced with wait/notify whenever inProgressList is touched.
-      } catch (InterruptedException ex) {
-        Thread.currentThread().interrupt();
-      }
-      if (LOGGER.isTraceEnabled()) {
-        LOGGER.trace("latch done for file {} buffer idx {} length {}",
-            stream.getPath(), readBuf.getBufferindex(), readBuf.getLength());
-      }
-    }
-  }
-
-  /**
-   * If any buffer in the completedlist can be reclaimed then reclaim it and return the buffer to free list.
-   * The objective is to find just one buffer - there is no advantage to evicting more than one.
-   *
-   * @return whether the eviction succeeeded - i.e., were we able to free up one buffer
-   */
-  private synchronized boolean tryEvict() {
-    ReadBuffer nodeToEvict = null;
-    if (completedReadList.size() <= 0) {
-      return false;  // there are no evict-able buffers
-    }
-
-    // first, try buffers where all bytes have been consumed (approximated as first and last bytes consumed)
-    for (ReadBuffer buf : completedReadList) {
-      if (buf.isFirstByteConsumed() && buf.isLastByteConsumed()) {
-        nodeToEvict = buf;
-        break;
-      }
-    }
-    if (nodeToEvict != null) {
-      return evict(nodeToEvict);
-    }
-
-    // next, try buffers where any bytes have been consumed (may be a bad idea? have to experiment and see)
-    for (ReadBuffer buf : completedReadList) {
-      if (buf.isAnyByteConsumed()) {
-        nodeToEvict = buf;
-        break;
-      }
-    }
-
-    if (nodeToEvict != null) {
-      return evict(nodeToEvict);
-    }
-
-    // next, try any old nodes that have not been consumed
-    long earliestBirthday = Long.MAX_VALUE;
-    for (ReadBuffer buf : completedReadList) {
-      if (buf.getTimeStamp() < earliestBirthday) {
-        nodeToEvict = buf;
-        earliestBirthday = buf.getTimeStamp();
-      }
-    }
-    if ((currentTimeMillis() - earliestBirthday > THRESHOLD_AGE_MILLISECONDS) && (nodeToEvict != null)) {
-      return evict(nodeToEvict);
-    }
-
-    // nothing can be evicted
-    return false;
-  }
-
-  private boolean evict(final ReadBuffer buf) {
-    freeList.push(buf.getBufferindex());
-    completedReadList.remove(buf);
-    if (LOGGER.isTraceEnabled()) {
-      LOGGER.trace("Evicting buffer idx {}; was used for file {} offset {} length {}",
-          buf.getBufferindex(), buf.getStream().getPath(), buf.getOffset(), buf.getLength());
-    }
-    return true;
-  }
-
-  private boolean isAlreadyQueued(final SeaweedInputStream stream, final long requestedOffset) {
-    // returns true if any part of the buffer is already queued
-    return (isInList(readAheadQueue, stream, requestedOffset)
-        || isInList(inProgressList, stream, requestedOffset)
-        || isInList(completedReadList, stream, requestedOffset));
-  }
-
-  private boolean isInList(final Collection<ReadBuffer> list, final SeaweedInputStream stream, final long requestedOffset) {
-    return (getFromList(list, stream, requestedOffset) != null);
-  }
-
-  private ReadBuffer getFromList(final Collection<ReadBuffer> list, final SeaweedInputStream stream, final long requestedOffset) {
-    for (ReadBuffer buffer : list) {
-      if (buffer.getStream() == stream) {
-        if (buffer.getStatus() == ReadBufferStatus.AVAILABLE
-            && requestedOffset >= buffer.getOffset()
-            && requestedOffset < buffer.getOffset() + buffer.getLength()) {
-          return buffer;
-        } else if (requestedOffset >= buffer.getOffset()
-            && requestedOffset < buffer.getOffset() + buffer.getRequestedLength()) {
-          return buffer;
-        }
-      }
-    }
-    return null;
-  }
-
-  private void clearFromReadAheadQueue(final SeaweedInputStream stream, final long requestedOffset) {
-    ReadBuffer buffer = getFromList(readAheadQueue, stream, requestedOffset);
-    if (buffer != null) {
-      readAheadQueue.remove(buffer);
-      notifyAll();   // lock is held in calling method
-      freeList.push(buffer.getBufferindex());
-    }
-  }
-
-  private int getBlockFromCompletedQueue(final SeaweedInputStream stream, final long position, final int length,
-                                         final byte[] buffer) {
-    ReadBuffer buf = getFromList(completedReadList, stream, position);
-    if (buf == null || position >= buf.getOffset() + buf.getLength()) {
-      return 0;
-    }
-    int cursor = (int) (position - buf.getOffset());
-    int availableLengthInBuffer = buf.getLength() - cursor;
-    int lengthToCopy = Math.min(length, availableLengthInBuffer);
-    System.arraycopy(buf.getBuffer(), cursor, buffer, 0, lengthToCopy);
-    if (cursor == 0) {
-      buf.setFirstByteConsumed(true);
-    }
-    if (cursor + lengthToCopy == buf.getLength()) {
-      buf.setLastByteConsumed(true);
-    }
-    buf.setAnyByteConsumed(true);
-    return lengthToCopy;
-  }
-
-  /*
-   *
-   *  ReadBufferWorker-thread-facing methods
-   *
-   */
-
-  /**
-   * ReadBufferWorker thread calls this to get the next buffer that it should work on.
-   *
-   * @return {@link ReadBuffer}
-   * @throws InterruptedException if thread is interrupted
-   */
-  ReadBuffer getNextBlockToRead() throws InterruptedException {
-    ReadBuffer buffer = null;
-    synchronized (this) {
-      //buffer = readAheadQueue.take();  // blocking method
-      while (readAheadQueue.size() == 0) {
-        wait();
-      }
-      buffer = readAheadQueue.remove();
-      notifyAll();
-      if (buffer == null) {
-        return null;            // should never happen
-      }
-      buffer.setStatus(ReadBufferStatus.READING_IN_PROGRESS);
-      inProgressList.add(buffer);
-    }
-    if (LOGGER.isTraceEnabled()) {
-      LOGGER.trace("ReadBufferWorker picked file {} for offset {}",
-          buffer.getStream().getPath(), buffer.getOffset());
-    }
-    return buffer;
-  }
-
-  /**
-   * ReadBufferWorker thread calls this method to post completion.
-   *
-   * @param buffer            the buffer whose read was completed
-   * @param result            the {@link ReadBufferStatus} after the read operation in the worker thread
-   * @param bytesActuallyRead the number of bytes that the worker thread was actually able to read
-   */
-  void doneReading(final ReadBuffer buffer, final ReadBufferStatus result, final int bytesActuallyRead) {
-    if (LOGGER.isTraceEnabled()) {
-      LOGGER.trace("ReadBufferWorker completed file {} for offset {} bytes {}",
-          buffer.getStream().getPath(),  buffer.getOffset(), bytesActuallyRead);
-    }
-    synchronized (this) {
-      inProgressList.remove(buffer);
-      if (result == ReadBufferStatus.AVAILABLE && bytesActuallyRead > 0) {
-        buffer.setStatus(ReadBufferStatus.AVAILABLE);
-        buffer.setTimeStamp(currentTimeMillis());
-        buffer.setLength(bytesActuallyRead);
-        completedReadList.add(buffer);
-      } else {
-        freeList.push(buffer.getBufferindex());
-        // buffer should go out of scope after the end of the calling method in ReadBufferWorker, and eligible for GC
-      }
-    }
-    //outside the synchronized, since anyone receiving a wake-up from the latch must see safe-published results
-    buffer.getLatch().countDown(); // wake up waiting threads (if any)
-  }
-
-  /**
-   * Similar to System.currentTimeMillis, except implemented with System.nanoTime().
-   * System.currentTimeMillis can go backwards when system clock is changed (e.g., with NTP time synchronization),
-   * making it unsuitable for measuring time intervals. nanotime is strictly monotonically increasing per CPU core.
-   * Note: it is not monotonic across Sockets, and even within a CPU, its only the
-   * more recent parts which share a clock across all cores.
-   *
-   * @return current time in milliseconds
-   */
-  private long currentTimeMillis() {
-    return System.nanoTime() / 1000 / 1000;
-  }
-}

other/java/hdfs2/src/main/java/seaweed/hdfs/ReadBufferWorker.java

@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package seaweed.hdfs;
-
-import java.util.concurrent.CountDownLatch;
-
-class ReadBufferWorker implements Runnable {
-
-  protected static final CountDownLatch UNLEASH_WORKERS = new CountDownLatch(1);
-  private int id;
-
-  ReadBufferWorker(final int id) {
-    this.id = id;
-  }
-
-  /**
-   * return the ID of ReadBufferWorker.
-   */
-  public int getId() {
-    return this.id;
-  }
-
-  /**
-   * Waits until a buffer becomes available in ReadAheadQueue.
-   * Once a buffer becomes available, reads the file specified in it and then posts results back to buffer manager.
-   * Rinse and repeat. Forever.
-   */
-  public void run() {
-    try {
-      UNLEASH_WORKERS.await();
-    } catch (InterruptedException ex) {
-      Thread.currentThread().interrupt();
-    }
-    ReadBufferManager bufferManager = ReadBufferManager.getBufferManager();
-    ReadBuffer buffer;
-    while (true) {
-      try {
-        buffer = bufferManager.getNextBlockToRead();   // blocks, until a buffer is available for this thread
-      } catch (InterruptedException ex) {
-        Thread.currentThread().interrupt();
-        return;
-      }
-      if (buffer != null) {
-        try {
-          // do the actual read, from the file.
-          int bytesRead = buffer.getStream().readRemote(buffer.getOffset(), buffer.getBuffer(), 0, buffer.getRequestedLength());
-          bufferManager.doneReading(buffer, ReadBufferStatus.AVAILABLE, bytesRead);  // post result back to ReadBufferManager
-        } catch (Exception ex) {
-          bufferManager.doneReading(buffer, ReadBufferStatus.READ_FAILED, 0);
-        }
-      }
-    }
-  }
-}

other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedAbstractFileSystem.java

@@ -18,12 +18,18 @@
 
 package seaweed.hdfs;
 
-/**
- * The ReadBufferStatus for Rest AbfsClient
- */
-public enum ReadBufferStatus {
-  NOT_AVAILABLE,  // buffers sitting in readaheadqueue have this stats
-  READING_IN_PROGRESS,  // reading is in progress on this buffer. Buffer should be in inProgressList
-  AVAILABLE,  // data is available in buffer. It should be in completedList
-  READ_FAILED  // read completed, but failed.
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegateToFileSystem;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+public class SeaweedAbstractFileSystem extends DelegateToFileSystem {
+
+    SeaweedAbstractFileSystem(final URI uri, final Configuration conf)
+            throws IOException, URISyntaxException {
+        super(uri, new SeaweedFileSystem(), conf, "seaweedfs", false);
+    }
+
 }

other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@@ -5,31 +5,31 @@ import org.apache.hadoop.fs.*;
 import org.apache.hadoop.fs.permission.AclEntry;
 import org.apache.hadoop.fs.permission.AclStatus;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Progressable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import seaweedfs.client.FilerProto;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URI;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
 
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
+public class SeaweedFileSystem extends FileSystem {
 
-public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
-
-    public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
     public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host";
     public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
+    public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
+    public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size";
+    public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication";
+    public static final String FS_SEAWEED_VOLUME_SERVER_ACCESS = "fs.seaweed.volume.server.access";
+    public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024;
 
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
-    private static int BUFFER_SIZE = 16 * 1024 * 1024;
 
     private URI uri;
     private Path workingDirectory = new Path("/");
@@ -60,15 +60,19 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
         port = (port == -1) ? FS_SEAWEED_DEFAULT_PORT : port;
         conf.setInt(FS_SEAWEED_FILER_PORT, port);
 
-        conf.setInt(IO_FILE_BUFFER_SIZE_KEY, BUFFER_SIZE);
-
         setConf(conf);
         this.uri = uri;
 
-        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port);
+        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, conf);
 
     }
 
+    @Override
+    public void close() throws IOException {
+        super.close();
+        this.seaweedFileSystemStore.close();
+    }
+
     @Override
     public FSDataInputStream open(Path path, int bufferSize) throws IOException {
 
@@ -77,8 +81,9 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
         path = qualify(path);
 
         try {
-            InputStream inputStream = seaweedFileSystemStore.openFileForRead(path, statistics, bufferSize);
-            return new FSDataInputStream(inputStream);
+            int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
+            FSInputStream inputStream = seaweedFileSystemStore.openFileForRead(path, statistics);
+            return new FSDataInputStream(new BufferedByteBufferReadableInputStream(inputStream, 4 * seaweedBufferSize));
         } catch (Exception ex) {
             LOG.warn("open path: {} bufferSize:{}", path, bufferSize, ex);
             return null;
@@ -94,8 +99,9 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
         path = qualify(path);
 
         try {
-            String replicaPlacement = String.format("%03d", replication - 1);
-            OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, bufferSize, replicaPlacement);
+            String replicaPlacement = this.getConf().get(FS_SEAWEED_REPLICATION, String.format("%03d", replication - 1));
+            int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
+            OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, seaweedBufferSize, replicaPlacement);
             return new FSDataOutputStream(outputStream, statistics);
         } catch (Exception ex) {
             LOG.warn("create path: {} bufferSize:{} blockSize:{}", path, bufferSize, blockSize, ex);
@@ -105,8 +111,9 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
 
     /**
      * {@inheritDoc}
+     *
      * @throws FileNotFoundException if the parent directory is not present -or
-     * is not a directory.
+     *                               is not a directory.
      */
     @Override
     public FSDataOutputStream createNonRecursive(Path path,
@@ -123,9 +130,10 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
                 throw new FileAlreadyExistsException("Not a directory: " + parent);
             }
         }
+        int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
         return create(path, permission,
                 flags.contains(CreateFlag.OVERWRITE), bufferSize,
-                replication, blockSize, progress);
+                replication, seaweedBufferSize, progress);
     }
 
     @Override
@@ -135,7 +143,8 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
 
         path = qualify(path);
         try {
-            OutputStream outputStream = seaweedFileSystemStore.createFile(path, false, null, bufferSize, "");
+            int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
+            OutputStream outputStream = seaweedFileSystemStore.createFile(path, false, null, seaweedBufferSize, "");
             return new FSDataOutputStream(outputStream, statistics);
         } catch (Exception ex) {
             LOG.warn("append path: {} bufferSize:{}", path, bufferSize, ex);
@@ -144,7 +153,7 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
     }
 
     @Override
-    public boolean rename(Path src, Path dst) {
+    public boolean rename(Path src, Path dst) throws IOException {
 
         LOG.debug("rename path: {} => {}", src, dst);
 
@@ -155,12 +164,13 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
         if (src.equals(dst)) {
             return true;
         }
-        FileStatus dstFileStatus = getFileStatus(dst);
+        FilerProto.Entry entry = seaweedFileSystemStore.lookupEntry(dst);
 
-        String sourceFileName = src.getName();
         Path adjustedDst = dst;
 
-        if (dstFileStatus != null) {
+        if (entry != null) {
+            FileStatus dstFileStatus = getFileStatus(dst);
+            String sourceFileName = src.getName();
             if (!dstFileStatus.isDirectory()) {
                 return false;
             }
@@ -175,18 +185,20 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
     }
 
     @Override
-    public boolean delete(Path path, boolean recursive) {
+    public boolean delete(Path path, boolean recursive) throws IOException {
 
         LOG.debug("delete path: {} recursive:{}", path, recursive);
 
         path = qualify(path);
 
-        FileStatus fileStatus = getFileStatus(path);
+        FilerProto.Entry entry = seaweedFileSystemStore.lookupEntry(path);
 
-        if (fileStatus == null) {
+        if (entry == null) {
             return true;
         }
 
+        FileStatus fileStatus = getFileStatus(path);
+
         return seaweedFileSystemStore.deleteEntries(path, fileStatus.isDirectory(), recursive);
 
     }
@@ -222,9 +234,9 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
 
         path = qualify(path);
 
-        FileStatus fileStatus = getFileStatus(path);
+        FilerProto.Entry entry = seaweedFileSystemStore.lookupEntry(path);
 
-        if (fileStatus == null) {
+        if (entry == null) {
 
             UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
             return seaweedFileSystemStore.createDirectory(path, currentUser,
@@ -233,6 +245,8 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
 
         }
 
+        FileStatus fileStatus = getFileStatus(path);
+
         if (fileStatus.isDirectory()) {
             return true;
         } else {
@@ -241,7 +255,7 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
     }
 
     @Override
-    public FileStatus getFileStatus(Path path) {
+    public FileStatus getFileStatus(Path path) throws IOException {
 
         LOG.debug("getFileStatus path: {}", path);
 
@@ -335,9 +349,7 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
 
     @Override
     public void createSymlink(final Path target, final Path link,
-                              final boolean createParent) throws AccessControlException,
-            FileAlreadyExistsException, FileNotFoundException,
-            ParentNotDirectoryException, UnsupportedFileSystemException,
+                              final boolean createParent) throws
             IOException {
         // Supporting filesystems should override this method
         throw new UnsupportedOperationException(

other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java

@@ -1,5 +1,7 @@
 package seaweed.hdfs;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSInputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -7,30 +9,43 @@ import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import seaweedfs.client.FilerClient;
-import seaweedfs.client.FilerGrpcClient;
-import seaweedfs.client.FilerProto;
-import seaweedfs.client.SeaweedRead;
+import seaweedfs.client.*;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
+import static seaweed.hdfs.SeaweedFileSystem.*;
+
 public class SeaweedFileSystemStore {
 
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystemStore.class);
 
-    private FilerGrpcClient filerGrpcClient;
     private FilerClient filerClient;
+    private Configuration conf;
 
-    public SeaweedFileSystemStore(String host, int port) {
+    public SeaweedFileSystemStore(String host, int port, Configuration conf) {
         int grpcPort = 10000 + port;
-        filerGrpcClient = new FilerGrpcClient(host, grpcPort);
-        filerClient = new FilerClient(filerGrpcClient);
+        filerClient = new FilerClient(host, grpcPort);
+        this.conf = conf;
+        String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct");
+        if (volumeServerAccessMode.equals("publicUrl")) {
+            filerClient.setAccessVolumeServerByPublicUrl();
+        } else if (volumeServerAccessMode.equals("filerProxy")) {
+            filerClient.setAccessVolumeServerByFilerProxy();
+        }
+
+    }
+
+    public void close() {
+        try {
+            this.filerClient.shutdown();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
     }
 
     public static String getParentDirectory(Path path) {
@@ -61,9 +76,19 @@ public class SeaweedFileSystemStore {
         );
     }
 
-    public FileStatus[] listEntries(final Path path) {
+    public FileStatus[] listEntries(final Path path) throws IOException {
         LOG.debug("listEntries path: {}", path);
 
+        FileStatus pathStatus = getFileStatus(path);
+
+        if (pathStatus == null) {
+            return new FileStatus[0];
+        }
+
+        if (!pathStatus.isDirectory()) {
+            return new FileStatus[]{pathStatus};
+        }
+
         List<FileStatus> fileStatuses = new ArrayList<FileStatus>();
 
         List<FilerProto.Entry> entries = filerClient.listEntries(path.toUri().getPath());
@@ -74,14 +99,16 @@ public class SeaweedFileSystemStore {
 
             fileStatuses.add(fileStatus);
         }
+        LOG.debug("listEntries path: {} size {}", fileStatuses, fileStatuses.size());
         return fileStatuses.toArray(new FileStatus[0]);
+
     }
 
-    public FileStatus getFileStatus(final Path path) {
+    public FileStatus getFileStatus(final Path path) throws IOException {
 
         FilerProto.Entry entry = lookupEntry(path);
         if (entry == null) {
-            return null;
+            throw new FileNotFoundException("File does not exist: " + path);
         }
         LOG.debug("doGetFileStatus path:{} entry:{}", path, entry);
 
@@ -111,10 +138,10 @@ public class SeaweedFileSystemStore {
 
     private FileStatus doGetFileStatus(Path path, FilerProto.Entry entry) {
         FilerProto.FuseAttributes attributes = entry.getAttributes();
-        long length = SeaweedRead.totalSize(entry.getChunksList());
+        long length = SeaweedRead.fileSize(entry);
         boolean isDir = entry.getIsDirectory();
         int block_replication = 1;
-        int blocksize = 512;
+        int blocksize = this.conf.getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
         long modification_time = attributes.getMtime() * 1000; // milliseconds
         long access_time = 0;
         FsPermission permission = FsPermission.createImmutable((short) attributes.getFileMode());
@@ -124,7 +151,7 @@ public class SeaweedFileSystemStore {
             modification_time, access_time, permission, owner, group, null, path);
     }
 
-    private FilerProto.Entry lookupEntry(Path path) {
+    public FilerProto.Entry lookupEntry(Path path) {
 
         return filerClient.lookupEntry(getParentDirectory(path), path.getName());
 
@@ -170,9 +197,10 @@ public class SeaweedFileSystemStore {
             if (existingEntry != null) {
                 entry = FilerProto.Entry.newBuilder();
                 entry.mergeFrom(existingEntry);
+                entry.clearContent();
                 entry.getAttributesBuilder().setMtime(now);
                 LOG.debug("createFile merged entry path:{} entry:{} from:{}", path, entry, existingEntry);
-                writePosition = SeaweedRead.totalSize(existingEntry.getChunksList());
+                writePosition = SeaweedRead.fileSize(existingEntry);
                 replication = existingEntry.getAttributes().getReplication();
             }
         }
@@ -189,30 +217,27 @@ public class SeaweedFileSystemStore {
                     .clearGroupName()
                     .addAllGroupName(Arrays.asList(userGroupInformation.getGroupNames()))
                 );
+            SeaweedWrite.writeMeta(filerClient, getParentDirectory(path), entry);
         }
 
-        return new SeaweedOutputStream(filerGrpcClient, path, entry, writePosition, bufferSize, replication);
+        return new SeaweedHadoopOutputStream(filerClient, path.toString(), entry, writePosition, bufferSize, replication);
 
     }
 
-    public InputStream openFileForRead(final Path path, FileSystem.Statistics statistics,
-                                       int bufferSize) throws IOException {
+    public FSInputStream openFileForRead(final Path path, FileSystem.Statistics statistics) throws IOException {
 
-        LOG.debug("openFileForRead path:{} bufferSize:{}", path, bufferSize);
+        LOG.debug("openFileForRead path:{}", path);
 
-        int readAheadQueueDepth = 2;
         FilerProto.Entry entry = lookupEntry(path);
 
         if (entry == null) {
             throw new FileNotFoundException("read non-exist file " + path);
         }
 
-        return new SeaweedInputStream(filerGrpcClient,
+        return new SeaweedHadoopInputStream(filerClient,
             statistics,
             path.toUri().getPath(),
-            entry,
-            bufferSize,
-            readAheadQueueDepth);
+            entry);
     }
 
     public void setOwner(Path path, String owner, String group) {

other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java

@@ -0,0 +1,150 @@
+package seaweed.hdfs;
+
+// based on org.apache.hadoop.fs.azurebfs.services.AbfsInputStream
+
+import org.apache.hadoop.fs.ByteBufferReadable;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileSystem.Statistics;
+import seaweedfs.client.FilerClient;
+import seaweedfs.client.FilerProto;
+import seaweedfs.client.SeaweedInputStream;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class SeaweedHadoopInputStream extends FSInputStream implements ByteBufferReadable {
+
+    private final SeaweedInputStream seaweedInputStream;
+    private final Statistics statistics;
+
+    public SeaweedHadoopInputStream(
+            final FilerClient filerClient,
+            final Statistics statistics,
+            final String path,
+            final FilerProto.Entry entry) throws IOException {
+        this.seaweedInputStream = new SeaweedInputStream(filerClient, path, entry);
+        this.statistics = statistics;
+    }
+
+    @Override
+    public int read() throws IOException {
+        return seaweedInputStream.read();
+    }
+
+    @Override
+    public int read(final byte[] b, final int off, final int len) throws IOException {
+        return seaweedInputStream.read(b, off, len);
+    }
+
+    // implement ByteBufferReadable
+    @Override
+    public synchronized int read(ByteBuffer buf) throws IOException {
+        int bytesRead = seaweedInputStream.read(buf);
+
+        if (bytesRead > 0) {
+            if (statistics != null) {
+                statistics.incrementBytesRead(bytesRead);
+            }
+        }
+
+        return bytesRead;
+    }
+
+    /**
+     * Seek to given position in stream.
+     *
+     * @param n position to seek to
+     * @throws IOException  if there is an error
+     * @throws EOFException if attempting to seek past end of file
+     */
+    @Override
+    public synchronized void seek(long n) throws IOException {
+        seaweedInputStream.seek(n);
+    }
+
+    @Override
+    public synchronized long skip(long n) throws IOException {
+        return seaweedInputStream.skip(n);
+    }
+
+    /**
+     * Return the size of the remaining available bytes
+     * if the size is less than or equal to {@link Integer#MAX_VALUE},
+     * otherwise, return {@link Integer#MAX_VALUE}.
+     * <p>
+     * This is to match the behavior of DFSInputStream.available(),
+     * which some clients may rely on (HBase write-ahead log reading in
+     * particular).
+     */
+    @Override
+    public synchronized int available() throws IOException {
+        return seaweedInputStream.available();
+    }
+
+    /**
+     * Returns the length of the file that this stream refers to. Note that the length returned is the length
+     * as of the time the Stream was opened. Specifically, if there have been subsequent appends to the file,
+     * they wont be reflected in the returned length.
+     *
+     * @return length of the file.
+     * @throws IOException if the stream is closed
+     */
+    public long length() throws IOException {
+        return seaweedInputStream.length();
+    }
+
+    /**
+     * Return the current offset from the start of the file
+     *
+     * @throws IOException throws {@link IOException} if there is an error
+     */
+    @Override
+    public synchronized long getPos() throws IOException {
+        return seaweedInputStream.getPos();
+    }
+
+    /**
+     * Seeks a different copy of the data.  Returns true if
+     * found a new source, false otherwise.
+     *
+     * @throws IOException throws {@link IOException} if there is an error
+     */
+    @Override
+    public boolean seekToNewSource(long l) throws IOException {
+        return false;
+    }
+
+    @Override
+    public synchronized void close() throws IOException {
+        seaweedInputStream.close();
+    }
+
+    /**
+     * Not supported by this stream. Throws {@link UnsupportedOperationException}
+     *
+     * @param readlimit ignored
+     */
+    @Override
+    public synchronized void mark(int readlimit) {
+        throw new UnsupportedOperationException("mark()/reset() not supported on this stream");
+    }
+
+    /**
+     * Not supported by this stream. Throws {@link UnsupportedOperationException}
+     */
+    @Override
+    public synchronized void reset() throws IOException {
+        throw new UnsupportedOperationException("mark()/reset() not supported on this stream");
+    }
+
+    /**
+     * gets whether mark and reset are supported by {@code ADLFileInputStream}. Always returns false.
+     *
+     * @return always {@code false}
+     */
+    @Override
+    public boolean markSupported() {
+        return false;
+    }
+}

other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java

@@ -0,0 +1,16 @@
+package seaweed.hdfs;
+
+// adapted from org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream
+
+import seaweedfs.client.FilerClient;
+import seaweedfs.client.FilerProto;
+import seaweedfs.client.SeaweedOutputStream;
+
+public class SeaweedHadoopOutputStream extends SeaweedOutputStream {
+
+    public SeaweedHadoopOutputStream(FilerClient filerClient, final String path, FilerProto.Entry.Builder entry,
+                                     final long position, final int bufferSize, final String replication) {
+        super(filerClient, path, entry, position, bufferSize, replication);
+    }
+
+}

other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedInputStream.java

@@ -1,371 +0,0 @@
-package seaweed.hdfs;
-
-// based on org.apache.hadoop.fs.azurebfs.services.AbfsInputStream
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.hadoop.fs.FSInputStream;
-import org.apache.hadoop.fs.FileSystem.Statistics;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import seaweedfs.client.FilerGrpcClient;
-import seaweedfs.client.FilerProto;
-import seaweedfs.client.SeaweedRead;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.util.List;
-
-public class SeaweedInputStream extends FSInputStream {
-
-    private static final Logger LOG = LoggerFactory.getLogger(SeaweedInputStream.class);
-
-    private final FilerGrpcClient filerGrpcClient;
-    private final Statistics statistics;
-    private final String path;
-    private final FilerProto.Entry entry;
-    private final List<SeaweedRead.VisibleInterval> visibleIntervalList;
-    private final long contentLength;
-    private final int bufferSize; // default buffer size
-    private final int readAheadQueueDepth;         // initialized in constructor
-    private final boolean readAheadEnabled; // whether enable readAhead;
-
-    private byte[] buffer = null;            // will be initialized on first use
-
-    private long fCursor = 0;  // cursor of buffer within file - offset of next byte to read from remote server
-    private long fCursorAfterLastRead = -1;
-    private int bCursor = 0;   // cursor of read within buffer - offset of next byte to be returned from buffer
-    private int limit = 0;     // offset of next byte to be read into buffer from service (i.e., upper marker+1
-    //                                                      of valid bytes in buffer)
-    private boolean closed = false;
-
-    public SeaweedInputStream(
-        final FilerGrpcClient filerGrpcClient,
-        final Statistics statistics,
-        final String path,
-        final FilerProto.Entry entry,
-        final int bufferSize,
-        final int readAheadQueueDepth) {
-        this.filerGrpcClient = filerGrpcClient;
-        this.statistics = statistics;
-        this.path = path;
-        this.entry = entry;
-        this.contentLength = SeaweedRead.totalSize(entry.getChunksList());
-        this.bufferSize = bufferSize;
-        this.readAheadQueueDepth = (readAheadQueueDepth >= 0) ? readAheadQueueDepth : Runtime.getRuntime().availableProcessors();
-        this.readAheadEnabled = true;
-
-        this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(entry.getChunksList());
-
-        LOG.debug("new path:{} entry:{} visibleIntervalList:{}", path, entry, visibleIntervalList);
-
-    }
-
-    public String getPath() {
-        return path;
-    }
-
-    @Override
-    public int read() throws IOException {
-        byte[] b = new byte[1];
-        int numberOfBytesRead = read(b, 0, 1);
-        if (numberOfBytesRead < 0) {
-            return -1;
-        } else {
-            return (b[0] & 0xFF);
-        }
-    }
-
-    @Override
-    public synchronized int read(final byte[] b, final int off, final int len) throws IOException {
-        int currentOff = off;
-        int currentLen = len;
-        int lastReadBytes;
-        int totalReadBytes = 0;
-        do {
-            lastReadBytes = readOneBlock(b, currentOff, currentLen);
-            if (lastReadBytes > 0) {
-                currentOff += lastReadBytes;
-                currentLen -= lastReadBytes;
-                totalReadBytes += lastReadBytes;
-            }
-            if (currentLen <= 0 || currentLen > b.length - currentOff) {
-                break;
-            }
-        } while (lastReadBytes > 0);
-        return totalReadBytes > 0 ? totalReadBytes : lastReadBytes;
-    }
-
-    private int readOneBlock(final byte[] b, final int off, final int len) throws IOException {
-        if (closed) {
-            throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
-        }
-
-        Preconditions.checkNotNull(b);
-
-        if (len == 0) {
-            return 0;
-        }
-
-        if (this.available() == 0) {
-            return -1;
-        }
-
-        if (off < 0 || len < 0 || len > b.length - off) {
-            throw new IndexOutOfBoundsException();
-        }
-
-        //If buffer is empty, then fill the buffer.
-        if (bCursor == limit) {
-            //If EOF, then return -1
-            if (fCursor >= contentLength) {
-                return -1;
-            }
-
-            long bytesRead = 0;
-            //reset buffer to initial state - i.e., throw away existing data
-            bCursor = 0;
-            limit = 0;
-            if (buffer == null) {
-                buffer = new byte[bufferSize];
-            }
-
-            // Enable readAhead when reading sequentially
-            if (-1 == fCursorAfterLastRead || fCursorAfterLastRead == fCursor || b.length >= bufferSize) {
-                bytesRead = readInternal(fCursor, buffer, 0, bufferSize, false);
-            } else {
-                bytesRead = readInternal(fCursor, buffer, 0, b.length, true);
-            }
-
-            if (bytesRead == -1) {
-                return -1;
-            }
-
-            limit += bytesRead;
-            fCursor += bytesRead;
-            fCursorAfterLastRead = fCursor;
-        }
-
-        //If there is anything in the buffer, then return lesser of (requested bytes) and (bytes in buffer)
-        //(bytes returned may be less than requested)
-        int bytesRemaining = limit - bCursor;
-        int bytesToRead = Math.min(len, bytesRemaining);
-        System.arraycopy(buffer, bCursor, b, off, bytesToRead);
-        bCursor += bytesToRead;
-        if (statistics != null) {
-            statistics.incrementBytesRead(bytesToRead);
-        }
-        return bytesToRead;
-    }
-
-
-    private int readInternal(final long position, final byte[] b, final int offset, final int length,
-                             final boolean bypassReadAhead) throws IOException {
-        if (readAheadEnabled && !bypassReadAhead) {
-            // try reading from read-ahead
-            if (offset != 0) {
-                throw new IllegalArgumentException("readahead buffers cannot have non-zero buffer offsets");
-            }
-            int receivedBytes;
-
-            // queue read-aheads
-            int numReadAheads = this.readAheadQueueDepth;
-            long nextSize;
-            long nextOffset = position;
-            while (numReadAheads > 0 && nextOffset < contentLength) {
-                nextSize = Math.min((long) bufferSize, contentLength - nextOffset);
-                ReadBufferManager.getBufferManager().queueReadAhead(this, nextOffset, (int) nextSize);
-                nextOffset = nextOffset + nextSize;
-                numReadAheads--;
-            }
-
-            // try reading from buffers first
-            receivedBytes = ReadBufferManager.getBufferManager().getBlock(this, position, length, b);
-            if (receivedBytes > 0) {
-                return receivedBytes;
-            }
-
-            // got nothing from read-ahead, do our own read now
-            receivedBytes = readRemote(position, b, offset, length);
-            return receivedBytes;
-        } else {
-            return readRemote(position, b, offset, length);
-        }
-    }
-
-    int readRemote(long position, byte[] b, int offset, int length) throws IOException {
-        if (position < 0) {
-            throw new IllegalArgumentException("attempting to read from negative offset");
-        }
-        if (position >= contentLength) {
-            return -1;  // Hadoop prefers -1 to EOFException
-        }
-        if (b == null) {
-            throw new IllegalArgumentException("null byte array passed in to read() method");
-        }
-        if (offset >= b.length) {
-            throw new IllegalArgumentException("offset greater than length of array");
-        }
-        if (length < 0) {
-            throw new IllegalArgumentException("requested read length is less than zero");
-        }
-        if (length > (b.length - offset)) {
-            throw new IllegalArgumentException("requested read length is more than will fit after requested offset in buffer");
-        }
-
-        long bytesRead = SeaweedRead.read(filerGrpcClient, visibleIntervalList, position, b, offset, length);
-        if (bytesRead > Integer.MAX_VALUE) {
-            throw new IOException("Unexpected Content-Length");
-        }
-        return (int) bytesRead;
-    }
-
-    /**
-     * Seek to given position in stream.
-     *
-     * @param n position to seek to
-     * @throws IOException  if there is an error
-     * @throws EOFException if attempting to seek past end of file
-     */
-    @Override
-    public synchronized void seek(long n) throws IOException {
-        if (closed) {
-            throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
-        }
-        if (n < 0) {
-            throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
-        }
-        if (n > contentLength) {
-            throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF);
-        }
-
-        if (n >= fCursor - limit && n <= fCursor) { // within buffer
-            bCursor = (int) (n - (fCursor - limit));
-            return;
-        }
-
-        // next read will read from here
-        fCursor = n;
-
-        //invalidate buffer
-        limit = 0;
-        bCursor = 0;
-    }
-
-    @Override
-    public synchronized long skip(long n) throws IOException {
-        if (closed) {
-            throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
-        }
-        long currentPos = getPos();
-        if (currentPos == contentLength) {
-            if (n > 0) {
-                throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF);
-            }
-        }
-        long newPos = currentPos + n;
-        if (newPos < 0) {
-            newPos = 0;
-            n = newPos - currentPos;
-        }
-        if (newPos > contentLength) {
-            newPos = contentLength;
-            n = newPos - currentPos;
-        }
-        seek(newPos);
-        return n;
-    }
-
-    /**
-     * Return the size of the remaining available bytes
-     * if the size is less than or equal to {@link Integer#MAX_VALUE},
-     * otherwise, return {@link Integer#MAX_VALUE}.
-     * <p>
-     * This is to match the behavior of DFSInputStream.available(),
-     * which some clients may rely on (HBase write-ahead log reading in
-     * particular).
-     */
-    @Override
-    public synchronized int available() throws IOException {
-        if (closed) {
-            throw new IOException(
-                FSExceptionMessages.STREAM_IS_CLOSED);
-        }
-        final long remaining = this.contentLength - this.getPos();
-        return remaining <= Integer.MAX_VALUE
-            ? (int) remaining : Integer.MAX_VALUE;
-    }
-
-    /**
-     * Returns the length of the file that this stream refers to. Note that the length returned is the length
-     * as of the time the Stream was opened. Specifically, if there have been subsequent appends to the file,
-     * they wont be reflected in the returned length.
-     *
-     * @return length of the file.
-     * @throws IOException if the stream is closed
-     */
-    public long length() throws IOException {
-        if (closed) {
-            throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
-        }
-        return contentLength;
-    }
-
-    /**
-     * Return the current offset from the start of the file
-     *
-     * @throws IOException throws {@link IOException} if there is an error
-     */
-    @Override
-    public synchronized long getPos() throws IOException {
-        if (closed) {
-            throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
-        }
-        return fCursor - limit + bCursor;
-    }
-
-    /**
-     * Seeks a different copy of the data.  Returns true if
-     * found a new source, false otherwise.
-     *
-     * @throws IOException throws {@link IOException} if there is an error
-     */
-    @Override
-    public boolean seekToNewSource(long l) throws IOException {
-        return false;
-    }
-
-    @Override
-    public synchronized void close() throws IOException {
-        closed = true;
-        buffer = null; // de-reference the buffer so it can be GC'ed sooner
-    }
-
-    /**
-     * Not supported by this stream. Throws {@link UnsupportedOperationException}
-     *
-     * @param readlimit ignored
-     */
-    @Override
-    public synchronized void mark(int readlimit) {
-        throw new UnsupportedOperationException("mark()/reset() not supported on this stream");
-    }
-
-    /**
-     * Not supported by this stream. Throws {@link UnsupportedOperationException}
-     */
-    @Override
-    public synchronized void reset() throws IOException {
-        throw new UnsupportedOperationException("mark()/reset() not supported on this stream");
-    }
-
-    /**
-     * gets whether mark and reset are supported by {@code ADLFileInputStream}. Always returns false.
-     *
-     * @return always {@code false}
-     */
-    @Override
-    public boolean markSupported() {
-        return false;
-    }
-}

other/java/hdfs3/dependency-reduced-pom.xml

@@ -15,8 +15,8 @@
       <plugin>
         <artifactId>maven-compiler-plugin</artifactId>
         <configuration>
-          <source>7</source>
-          <target>7</target>
+          <source>8</source>
+          <target>8</target>
         </configuration>
       </plugin>
       <plugin>
@@ -120,6 +120,188 @@
       </plugin>
     </plugins>
   </build>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client</artifactId>
+      <version>3.1.1</version>
+      <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <artifactId>hadoop-hdfs-client</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-yarn-api</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-yarn-client</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-mapreduce-client-core</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-annotations</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <version>3.1.1</version>
+      <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <artifactId>commons-cli</artifactId>
+          <groupId>commons-cli</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-math3</artifactId>
+          <groupId>org.apache.commons</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-io</artifactId>
+          <groupId>commons-io</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-net</artifactId>
+          <groupId>commons-net</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-collections</artifactId>
+          <groupId>commons-collections</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>javax.servlet-api</artifactId>
+          <groupId>javax.servlet</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jetty-server</artifactId>
+          <groupId>org.eclipse.jetty</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jetty-util</artifactId>
+          <groupId>org.eclipse.jetty</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jetty-servlet</artifactId>
+          <groupId>org.eclipse.jetty</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jetty-webapp</artifactId>
+          <groupId>org.eclipse.jetty</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jsp-api</artifactId>
+          <groupId>javax.servlet.jsp</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jersey-core</artifactId>
+          <groupId>com.sun.jersey</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jersey-servlet</artifactId>
+          <groupId>com.sun.jersey</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jersey-json</artifactId>
+          <groupId>com.sun.jersey</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jersey-server</artifactId>
+          <groupId>com.sun.jersey</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>log4j</artifactId>
+          <groupId>log4j</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-lang</artifactId>
+          <groupId>commons-lang</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-beanutils</artifactId>
+          <groupId>commons-beanutils</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-configuration2</artifactId>
+          <groupId>org.apache.commons</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-lang3</artifactId>
+          <groupId>org.apache.commons</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>slf4j-log4j12</artifactId>
+          <groupId>org.slf4j</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>avro</artifactId>
+          <groupId>org.apache.avro</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>re2j</artifactId>
+          <groupId>com.google.re2j</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-auth</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jsch</artifactId>
+          <groupId>com.jcraft</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>curator-client</artifactId>
+          <groupId>org.apache.curator</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>curator-recipes</artifactId>
+          <groupId>org.apache.curator</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>htrace-core4</artifactId>
+          <groupId>org.apache.htrace</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>zookeeper</artifactId>
+          <groupId>org.apache.zookeeper</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-compress</artifactId>
+          <groupId>org.apache.commons</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>kerb-simplekdc</artifactId>
+          <groupId>org.apache.kerby</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jackson-databind</artifactId>
+          <groupId>com.fasterxml.jackson.core</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>stax2-api</artifactId>
+          <groupId>org.codehaus.woodstox</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>woodstox-core</artifactId>
+          <groupId>com.fasterxml.woodstox</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-annotations</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+  </dependencies>
   <distributionManagement>
     <snapshotRepository>
       <id>ossrh</id>
@@ -127,7 +309,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.2.7</seaweedfs.client.version>
+    <seaweedfs.client.version>1.6.2</seaweedfs.client.version>
     <hadoop.version>3.1.1</hadoop.version>
   </properties>
 </project>

other/java/hdfs3/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.2.7</seaweedfs.client.version>
+        <seaweedfs.client.version>1.6.2</seaweedfs.client.version>
         <hadoop.version>3.1.1</hadoop.version>
     </properties>
 
@@ -31,8 +31,8 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
                 <configuration>
-                    <source>7</source>
-                    <target>7</target>
+                    <source>8</source>
+                    <target>8</target>
                 </configuration>
             </plugin>
             <plugin>
@@ -147,6 +147,7 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client</artifactId>
             <version>${hadoop.version}</version>
+            <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>com.github.chrislusf</groupId>
@@ -157,6 +158,7 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
             <version>${hadoop.version}</version>
+            <scope>provided</scope>
         </dependency>
     </dependencies>
 

other/java/hdfs3/src/main/java/seaweed/hdfs/BufferedByteBufferReadableInputStream.java

@@ -0,0 +1,25 @@
+package seaweed.hdfs;
+
+import org.apache.hadoop.fs.*;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class BufferedByteBufferReadableInputStream extends BufferedFSInputStream implements ByteBufferReadable {
+
+    public BufferedByteBufferReadableInputStream(FSInputStream in, int size) {
+        super(in, size);
+        if (!(in instanceof Seekable) || !(in instanceof PositionedReadable)) {
+            throw new IllegalArgumentException("In is not an instance of Seekable or PositionedReadable");
+        }
+    }
+
+    @Override
+    public int read(ByteBuffer buf) throws IOException {
+        if (this.in instanceof ByteBufferReadable) {
+            return ((ByteBufferReadable)this.in).read(buf);
+        } else {
+            throw new UnsupportedOperationException("Byte-buffer read unsupported by input stream");
+        }
+    }
+}

other/java/hdfs3/src/main/java/seaweed/hdfs/ReadBuffer.java

@@ -1,137 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package seaweed.hdfs;
-
-import java.util.concurrent.CountDownLatch;
-
-class ReadBuffer {
-
-  private SeaweedInputStream stream;
-  private long offset;                   // offset within the file for the buffer
-  private int length;                    // actual length, set after the buffer is filles
-  private int requestedLength;           // requested length of the read
-  private byte[] buffer;                 // the buffer itself
-  private int bufferindex = -1;          // index in the buffers array in Buffer manager
-  private ReadBufferStatus status;             // status of the buffer
-  private CountDownLatch latch = null;   // signaled when the buffer is done reading, so any client
-  // waiting on this buffer gets unblocked
-
-  // fields to help with eviction logic
-  private long timeStamp = 0;  // tick at which buffer became available to read
-  private boolean isFirstByteConsumed = false;
-  private boolean isLastByteConsumed = false;
-  private boolean isAnyByteConsumed = false;
-
-  public SeaweedInputStream getStream() {
-    return stream;
-  }
-
-  public void setStream(SeaweedInputStream stream) {
-    this.stream = stream;
-  }
-
-  public long getOffset() {
-    return offset;
-  }
-
-  public void setOffset(long offset) {
-    this.offset = offset;
-  }
-
-  public int getLength() {
-    return length;
-  }
-
-  public void setLength(int length) {
-    this.length = length;
-  }
-
-  public int getRequestedLength() {
-    return requestedLength;
-  }
-
-  public void setRequestedLength(int requestedLength) {
-    this.requestedLength = requestedLength;
-  }
-
-  public byte[] getBuffer() {
-    return buffer;
-  }
-
-  public void setBuffer(byte[] buffer) {
-    this.buffer = buffer;
-  }
-
-  public int getBufferindex() {
-    return bufferindex;
-  }
-
-  public void setBufferindex(int bufferindex) {
-    this.bufferindex = bufferindex;
-  }
-
-  public ReadBufferStatus getStatus() {
-    return status;
-  }
-
-  public void setStatus(ReadBufferStatus status) {
-    this.status = status;
-  }
-
-  public CountDownLatch getLatch() {
-    return latch;
-  }
-
-  public void setLatch(CountDownLatch latch) {
-    this.latch = latch;
-  }
-
-  public long getTimeStamp() {
-    return timeStamp;
-  }
-
-  public void setTimeStamp(long timeStamp) {
-    this.timeStamp = timeStamp;
-  }
-
-  public boolean isFirstByteConsumed() {
-    return isFirstByteConsumed;
-  }
-
-  public void setFirstByteConsumed(boolean isFirstByteConsumed) {
-    this.isFirstByteConsumed = isFirstByteConsumed;
-  }
-
-  public boolean isLastByteConsumed() {
-    return isLastByteConsumed;
-  }
-
-  public void setLastByteConsumed(boolean isLastByteConsumed) {
-    this.isLastByteConsumed = isLastByteConsumed;
-  }
-
-  public boolean isAnyByteConsumed() {
-    return isAnyByteConsumed;
-  }
-
-  public void setAnyByteConsumed(boolean isAnyByteConsumed) {
-    this.isAnyByteConsumed = isAnyByteConsumed;
-  }
-
-}

other/java/hdfs3/src/main/java/seaweed/hdfs/ReadBufferManager.java

@@ -1,394 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package seaweed.hdfs;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.Queue;
-import java.util.Stack;
-import java.util.concurrent.CountDownLatch;
-
-/**
- * The Read Buffer Manager for Rest AbfsClient.
- */
-final class ReadBufferManager {
-  private static final Logger LOGGER = LoggerFactory.getLogger(ReadBufferManager.class);
-
-  private static final int NUM_BUFFERS = 16;
-  private static final int BLOCK_SIZE = 4 * 1024 * 1024;
-  private static final int NUM_THREADS = 8;
-  private static final int THRESHOLD_AGE_MILLISECONDS = 3000; // have to see if 3 seconds is a good threshold
-
-  private Thread[] threads = new Thread[NUM_THREADS];
-  private byte[][] buffers;    // array of byte[] buffers, to hold the data that is read
-  private Stack<Integer> freeList = new Stack<>();   // indices in buffers[] array that are available
-
-  private Queue<ReadBuffer> readAheadQueue = new LinkedList<>(); // queue of requests that are not picked up by any worker thread yet
-  private LinkedList<ReadBuffer> inProgressList = new LinkedList<>(); // requests being processed by worker threads
-  private LinkedList<ReadBuffer> completedReadList = new LinkedList<>(); // buffers available for reading
-  private static final ReadBufferManager BUFFER_MANAGER; // singleton, initialized in static initialization block
-
-  static {
-    BUFFER_MANAGER = new ReadBufferManager();
-    BUFFER_MANAGER.init();
-  }
-
-  static ReadBufferManager getBufferManager() {
-    return BUFFER_MANAGER;
-  }
-
-  private void init() {
-    buffers = new byte[NUM_BUFFERS][];
-    for (int i = 0; i < NUM_BUFFERS; i++) {
-      buffers[i] = new byte[BLOCK_SIZE];  // same buffers are reused. The byte array never goes back to GC
-      freeList.add(i);
-    }
-    for (int i = 0; i < NUM_THREADS; i++) {
-      Thread t = new Thread(new ReadBufferWorker(i));
-      t.setDaemon(true);
-      threads[i] = t;
-      t.setName("SeaweedFS-prefetch-" + i);
-      t.start();
-    }
-    ReadBufferWorker.UNLEASH_WORKERS.countDown();
-  }
-
-  // hide instance constructor
-  private ReadBufferManager() {
-  }
-
-
-  /*
-   *
-   *  SeaweedInputStream-facing methods
-   *
-   */
-
-
-  /**
-   * {@link SeaweedInputStream} calls this method to queue read-aheads.
-   *
-   * @param stream          The {@link SeaweedInputStream} for which to do the read-ahead
-   * @param requestedOffset The offset in the file which shoukd be read
-   * @param requestedLength The length to read
-   */
-  void queueReadAhead(final SeaweedInputStream stream, final long requestedOffset, final int requestedLength) {
-    if (LOGGER.isTraceEnabled()) {
-      LOGGER.trace("Start Queueing readAhead for {} offset {} length {}",
-          stream.getPath(), requestedOffset, requestedLength);
-    }
-    ReadBuffer buffer;
-    synchronized (this) {
-      if (isAlreadyQueued(stream, requestedOffset)) {
-        return; // already queued, do not queue again
-      }
-      if (freeList.isEmpty() && !tryEvict()) {
-        return; // no buffers available, cannot queue anything
-      }
-
-      buffer = new ReadBuffer();
-      buffer.setStream(stream);
-      buffer.setOffset(requestedOffset);
-      buffer.setLength(0);
-      buffer.setRequestedLength(requestedLength);
-      buffer.setStatus(ReadBufferStatus.NOT_AVAILABLE);
-      buffer.setLatch(new CountDownLatch(1));
-
-      Integer bufferIndex = freeList.pop();  // will return a value, since we have checked size > 0 already
-
-      buffer.setBuffer(buffers[bufferIndex]);
-      buffer.setBufferindex(bufferIndex);
-      readAheadQueue.add(buffer);
-      notifyAll();
-    }
-    if (LOGGER.isTraceEnabled()) {
-      LOGGER.trace("Done q-ing readAhead for file {} offset {} buffer idx {}",
-          stream.getPath(), requestedOffset, buffer.getBufferindex());
-    }
-  }
-
-
-  /**
-   * {@link SeaweedInputStream} calls this method read any bytes already available in a buffer (thereby saving a
-   * remote read). This returns the bytes if the data already exists in buffer. If there is a buffer that is reading
-   * the requested offset, then this method blocks until that read completes. If the data is queued in a read-ahead
-   * but not picked up by a worker thread yet, then it cancels that read-ahead and reports cache miss. This is because
-   * depending on worker thread availability, the read-ahead may take a while - the calling thread can do it's own
-   * read to get the data faster (copmared to the read waiting in queue for an indeterminate amount of time).
-   *
-   * @param stream   the file to read bytes for
-   * @param position the offset in the file to do a read for
-   * @param length   the length to read
-   * @param buffer   the buffer to read data into. Note that the buffer will be written into from offset 0.
-   * @return the number of bytes read
-   */
-  int getBlock(final SeaweedInputStream stream, final long position, final int length, final byte[] buffer) {
-    // not synchronized, so have to be careful with locking
-    if (LOGGER.isTraceEnabled()) {
-      LOGGER.trace("getBlock for file {}  position {}  thread {}",
-          stream.getPath(), position, Thread.currentThread().getName());
-    }
-
-    waitForProcess(stream, position);
-
-    int bytesRead = 0;
-    synchronized (this) {
-      bytesRead = getBlockFromCompletedQueue(stream, position, length, buffer);
-    }
-    if (bytesRead > 0) {
-      if (LOGGER.isTraceEnabled()) {
-        LOGGER.trace("Done read from Cache for {} position {} length {}",
-            stream.getPath(), position, bytesRead);
-      }
-      return bytesRead;
-    }
-
-    // otherwise, just say we got nothing - calling thread can do its own read
-    return 0;
-  }
-
-  /*
-   *
-   *  Internal methods
-   *
-   */
-
-  private void waitForProcess(final SeaweedInputStream stream, final long position) {
-    ReadBuffer readBuf;
-    synchronized (this) {
-      clearFromReadAheadQueue(stream, position);
-      readBuf = getFromList(inProgressList, stream, position);
-    }
-    if (readBuf != null) {         // if in in-progress queue, then block for it
-      try {
-        if (LOGGER.isTraceEnabled()) {
-          LOGGER.trace("got a relevant read buffer for file {} offset {} buffer idx {}",
-              stream.getPath(), readBuf.getOffset(), readBuf.getBufferindex());
-        }
-        readBuf.getLatch().await();  // blocking wait on the caller stream's thread
-        // Note on correctness: readBuf gets out of inProgressList only in 1 place: after worker thread
-        // is done processing it (in doneReading). There, the latch is set after removing the buffer from
-        // inProgressList. So this latch is safe to be outside the synchronized block.
-        // Putting it in synchronized would result in a deadlock, since this thread would be holding the lock
-        // while waiting, so no one will be able to  change any state. If this becomes more complex in the future,
-        // then the latch cane be removed and replaced with wait/notify whenever inProgressList is touched.
-      } catch (InterruptedException ex) {
-        Thread.currentThread().interrupt();
-      }
-      if (LOGGER.isTraceEnabled()) {
-        LOGGER.trace("latch done for file {} buffer idx {} length {}",
-            stream.getPath(), readBuf.getBufferindex(), readBuf.getLength());
-      }
-    }
-  }
-
-  /**
-   * If any buffer in the completedlist can be reclaimed then reclaim it and return the buffer to free list.
-   * The objective is to find just one buffer - there is no advantage to evicting more than one.
-   *
-   * @return whether the eviction succeeeded - i.e., were we able to free up one buffer
-   */
-  private synchronized boolean tryEvict() {
-    ReadBuffer nodeToEvict = null;
-    if (completedReadList.size() <= 0) {
-      return false;  // there are no evict-able buffers
-    }
-
-    // first, try buffers where all bytes have been consumed (approximated as first and last bytes consumed)
-    for (ReadBuffer buf : completedReadList) {
-      if (buf.isFirstByteConsumed() && buf.isLastByteConsumed()) {
-        nodeToEvict = buf;
-        break;
-      }
-    }
-    if (nodeToEvict != null) {
-      return evict(nodeToEvict);
-    }
-
-    // next, try buffers where any bytes have been consumed (may be a bad idea? have to experiment and see)
-    for (ReadBuffer buf : completedReadList) {
-      if (buf.isAnyByteConsumed()) {
-        nodeToEvict = buf;
-        break;
-      }
-    }
-
-    if (nodeToEvict != null) {
-      return evict(nodeToEvict);
-    }
-
-    // next, try any old nodes that have not been consumed
-    long earliestBirthday = Long.MAX_VALUE;
-    for (ReadBuffer buf : completedReadList) {
-      if (buf.getTimeStamp() < earliestBirthday) {
-        nodeToEvict = buf;
-        earliestBirthday = buf.getTimeStamp();
-      }
-    }
-    if ((currentTimeMillis() - earliestBirthday > THRESHOLD_AGE_MILLISECONDS) && (nodeToEvict != null)) {
-      return evict(nodeToEvict);
-    }
-
-    // nothing can be evicted
-    return false;
-  }
-
-  private boolean evict(final ReadBuffer buf) {
-    freeList.push(buf.getBufferindex());
-    completedReadList.remove(buf);
-    if (LOGGER.isTraceEnabled()) {
-      LOGGER.trace("Evicting buffer idx {}; was used for file {} offset {} length {}",
-          buf.getBufferindex(), buf.getStream().getPath(), buf.getOffset(), buf.getLength());
-    }
-    return true;
-  }
-
-  private boolean isAlreadyQueued(final SeaweedInputStream stream, final long requestedOffset) {
-    // returns true if any part of the buffer is already queued
-    return (isInList(readAheadQueue, stream, requestedOffset)
-        || isInList(inProgressList, stream, requestedOffset)
-        || isInList(completedReadList, stream, requestedOffset));
-  }
-
-  private boolean isInList(final Collection<ReadBuffer> list, final SeaweedInputStream stream, final long requestedOffset) {
-    return (getFromList(list, stream, requestedOffset) != null);
-  }
-
-  private ReadBuffer getFromList(final Collection<ReadBuffer> list, final SeaweedInputStream stream, final long requestedOffset) {
-    for (ReadBuffer buffer : list) {
-      if (buffer.getStream() == stream) {
-        if (buffer.getStatus() == ReadBufferStatus.AVAILABLE
-            && requestedOffset >= buffer.getOffset()
-            && requestedOffset < buffer.getOffset() + buffer.getLength()) {
-          return buffer;
-        } else if (requestedOffset >= buffer.getOffset()
-            && requestedOffset < buffer.getOffset() + buffer.getRequestedLength()) {
-          return buffer;
-        }
-      }
-    }
-    return null;
-  }
-
-  private void clearFromReadAheadQueue(final SeaweedInputStream stream, final long requestedOffset) {
-    ReadBuffer buffer = getFromList(readAheadQueue, stream, requestedOffset);
-    if (buffer != null) {
-      readAheadQueue.remove(buffer);
-      notifyAll();   // lock is held in calling method
-      freeList.push(buffer.getBufferindex());
-    }
-  }
-
-  private int getBlockFromCompletedQueue(final SeaweedInputStream stream, final long position, final int length,
-                                         final byte[] buffer) {
-    ReadBuffer buf = getFromList(completedReadList, stream, position);
-    if (buf == null || position >= buf.getOffset() + buf.getLength()) {
-      return 0;
-    }
-    int cursor = (int) (position - buf.getOffset());
-    int availableLengthInBuffer = buf.getLength() - cursor;
-    int lengthToCopy = Math.min(length, availableLengthInBuffer);
-    System.arraycopy(buf.getBuffer(), cursor, buffer, 0, lengthToCopy);
-    if (cursor == 0) {
-      buf.setFirstByteConsumed(true);
-    }
-    if (cursor + lengthToCopy == buf.getLength()) {
-      buf.setLastByteConsumed(true);
-    }
-    buf.setAnyByteConsumed(true);
-    return lengthToCopy;
-  }
-
-  /*
-   *
-   *  ReadBufferWorker-thread-facing methods
-   *
-   */
-
-  /**
-   * ReadBufferWorker thread calls this to get the next buffer that it should work on.
-   *
-   * @return {@link ReadBuffer}
-   * @throws InterruptedException if thread is interrupted
-   */
-  ReadBuffer getNextBlockToRead() throws InterruptedException {
-    ReadBuffer buffer = null;
-    synchronized (this) {
-      //buffer = readAheadQueue.take();  // blocking method
-      while (readAheadQueue.size() == 0) {
-        wait();
-      }
-      buffer = readAheadQueue.remove();
-      notifyAll();
-      if (buffer == null) {
-        return null;            // should never happen
-      }
-      buffer.setStatus(ReadBufferStatus.READING_IN_PROGRESS);
-      inProgressList.add(buffer);
-    }
-    if (LOGGER.isTraceEnabled()) {
-      LOGGER.trace("ReadBufferWorker picked file {} for offset {}",
-          buffer.getStream().getPath(), buffer.getOffset());
-    }
-    return buffer;
-  }
-
-  /**
-   * ReadBufferWorker thread calls this method to post completion.
-   *
-   * @param buffer            the buffer whose read was completed
-   * @param result            the {@link ReadBufferStatus} after the read operation in the worker thread
-   * @param bytesActuallyRead the number of bytes that the worker thread was actually able to read
-   */
-  void doneReading(final ReadBuffer buffer, final ReadBufferStatus result, final int bytesActuallyRead) {
-    if (LOGGER.isTraceEnabled()) {
-      LOGGER.trace("ReadBufferWorker completed file {} for offset {} bytes {}",
-          buffer.getStream().getPath(),  buffer.getOffset(), bytesActuallyRead);
-    }
-    synchronized (this) {
-      inProgressList.remove(buffer);
-      if (result == ReadBufferStatus.AVAILABLE && bytesActuallyRead > 0) {
-        buffer.setStatus(ReadBufferStatus.AVAILABLE);
-        buffer.setTimeStamp(currentTimeMillis());
-        buffer.setLength(bytesActuallyRead);
-        completedReadList.add(buffer);
-      } else {
-        freeList.push(buffer.getBufferindex());
-        // buffer should go out of scope after the end of the calling method in ReadBufferWorker, and eligible for GC
-      }
-    }
-    //outside the synchronized, since anyone receiving a wake-up from the latch must see safe-published results
-    buffer.getLatch().countDown(); // wake up waiting threads (if any)
-  }
-
-  /**
-   * Similar to System.currentTimeMillis, except implemented with System.nanoTime().
-   * System.currentTimeMillis can go backwards when system clock is changed (e.g., with NTP time synchronization),
-   * making it unsuitable for measuring time intervals. nanotime is strictly monotonically increasing per CPU core.
-   * Note: it is not monotonic across Sockets, and even within a CPU, its only the
-   * more recent parts which share a clock across all cores.
-   *
-   * @return current time in milliseconds
-   */
-  private long currentTimeMillis() {
-    return System.nanoTime() / 1000 / 1000;
-  }
-}

other/java/hdfs3/src/main/java/seaweed/hdfs/ReadBufferWorker.java

@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package seaweed.hdfs;
-
-import java.util.concurrent.CountDownLatch;
-
-class ReadBufferWorker implements Runnable {
-
-  protected static final CountDownLatch UNLEASH_WORKERS = new CountDownLatch(1);
-  private int id;
-
-  ReadBufferWorker(final int id) {
-    this.id = id;
-  }
-
-  /**
-   * return the ID of ReadBufferWorker.
-   */
-  public int getId() {
-    return this.id;
-  }
-
-  /**
-   * Waits until a buffer becomes available in ReadAheadQueue.
-   * Once a buffer becomes available, reads the file specified in it and then posts results back to buffer manager.
-   * Rinse and repeat. Forever.
-   */
-  public void run() {
-    try {
-      UNLEASH_WORKERS.await();
-    } catch (InterruptedException ex) {
-      Thread.currentThread().interrupt();
-    }
-    ReadBufferManager bufferManager = ReadBufferManager.getBufferManager();
-    ReadBuffer buffer;
-    while (true) {
-      try {
-        buffer = bufferManager.getNextBlockToRead();   // blocks, until a buffer is available for this thread
-      } catch (InterruptedException ex) {
-        Thread.currentThread().interrupt();
-        return;
-      }
-      if (buffer != null) {
-        try {
-          // do the actual read, from the file.
-          int bytesRead = buffer.getStream().readRemote(buffer.getOffset(), buffer.getBuffer(), 0, buffer.getRequestedLength());
-          bufferManager.doneReading(buffer, ReadBufferStatus.AVAILABLE, bytesRead);  // post result back to ReadBufferManager
-        } catch (Exception ex) {
-          bufferManager.doneReading(buffer, ReadBufferStatus.READ_FAILED, 0);
-        }
-      }
-    }
-  }
-}

other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedAbstractFileSystem.java

@@ -18,12 +18,18 @@
 
 package seaweed.hdfs;
 
-/**
- * The ReadBufferStatus for Rest AbfsClient
- */
-public enum ReadBufferStatus {
-  NOT_AVAILABLE,  // buffers sitting in readaheadqueue have this stats
-  READING_IN_PROGRESS,  // reading is in progress on this buffer. Buffer should be in inProgressList
-  AVAILABLE,  // data is available in buffer. It should be in completedList
-  READ_FAILED  // read completed, but failed.
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegateToFileSystem;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+public class SeaweedAbstractFileSystem extends DelegateToFileSystem {
+
+    SeaweedAbstractFileSystem(final URI uri, final Configuration conf)
+            throws IOException, URISyntaxException {
+        super(uri, new SeaweedFileSystem(), conf, "seaweedfs", false);
+    }
+
 }