Merge branch 'master' into ftp

WIP
2026-10-02 12:42:13 +00:00 · 2021-01-21 00:47:27 -08:00 · 2020-12-26 23:46:20 -08:00
603 changed files with 11360 additions and 34966 deletions
@@ -9,7 +9,6 @@ assignees: ''

 Sponsors SeaweedFS via Patreon https://www.patreon.com/seaweedfs
 Report issues here. Ask questions here https://stackoverflow.com/questions/tagged/seaweedfs
-Please ask questions in https://github.com/chrislusf/seaweedfs/discussions

 example of a good issue report:
 https://github.com/chrislusf/seaweedfs/issues/1005
@@ -1,77 +0,0 @@
-name: "go: build dev binaries"
-
-on:
-  push:
-    branches: [ master ]
-
-jobs:
-
-  cleanup:
-    runs-on: ubuntu-latest
-
-    steps:
-
-      - name: Delete old release assets
-        uses: mknejp/delete-release-assets@v1
-        with:
-          token: ${{ github.token }}
-          tag: dev
-          fail-if-no-assets: false
-          assets: |
-            weed-*
-
-  build_dev:
-    needs: cleanup
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        goos: [linux, windows, darwin, freebsd]
-        goarch: [amd64, arm, arm64]
-        exclude:
-          - goarch: arm
-            goos: darwin
-          - goarch: 386
-            goos: darwin
-          - goarch: arm
-            goos: windows
-          - goarch: arm64
-            goos: windows
-
-    steps:
-
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Set BUILD_TIME env
-        run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
-
-      - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@v1.20
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          goos: ${{ matrix.goos }}
-          goarch: ${{ matrix.goarch }}
-          release_tag: dev
-          overwrite: true
-          pre_command: export CGO_ENABLED=0
-          build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
-          # Where to run `go build .`
-          project_path: weed
-          binary_name: weed-large-disk
-          asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
-
-      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.20
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          goos: ${{ matrix.goos }}
-          goarch: ${{ matrix.goarch }}
-          release_tag: dev
-          overwrite: true
-          pre_command: export CGO_ENABLED=0
-          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
-          # Where to run `go build .`
-          project_path: weed
-          binary_name: weed-normal-disk
-          asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
@@ -1,63 +0,0 @@
-# This is a basic workflow to help you get started with Actions
-
-name: "go: build versioned binaries"
-
-on:
-  push:
-    tags:
-      - '*'
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
-jobs:
-
-  build-release-binaries:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        goos: [linux, windows, darwin, freebsd]
-        goarch: [amd64, arm, arm64]
-        exclude:
-          - goarch: arm
-            goos: darwin
-          - goarch: 386
-            goos: darwin
-          - goarch: arm
-            goos: windows
-          - goarch: arm64
-            goos: windows
-
-    # Steps represent a sequence of tasks that will be executed as part of the job
-    steps:
-      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
-      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.20
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          goos: ${{ matrix.goos }}
-          goarch: ${{ matrix.goarch }}
-          overwrite: true
-          pre_command: export CGO_ENABLED=0
-          # build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
-          # Where to run `go build .`
-          project_path: weed
-          binary_name: weed
-          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
-      - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@v1.20
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          goos: ${{ matrix.goos }}
-          goarch: ${{ matrix.goarch }}
-          overwrite: true
-          pre_command: export CGO_ENABLED=0
-          build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
-          # Where to run `go build .`
-          project_path: weed
-          binary_name: weed
-          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk"
@@ -0,0 +1,22 @@
+name: Cleanup
+
+on:
+  push:
+    branches: [ master ]
+
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+
+    steps:
+
+    - name: Delete old release assets
+      uses: mknejp/delete-release-assets@v1
+      with:
+        token: ${{ github.token }}
+        tag: dev
+        fail-if-no-assets: false 
+        assets: | 
+          weed-*
@@ -1,63 +0,0 @@
-name: "docker: build dev containers"
-
-on:
-  push:
-    branches: [ master ]
-  workflow_dispatch: []
-
-jobs:
-
-  build-dev-containers:
-    runs-on: [ubuntu-latest]
-
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v2
-      -
-        name: Docker meta
-        id: docker_meta
-        uses: docker/metadata-action@v3
-        with:
-          images: |
-            chrislusf/seaweedfs
-            ghcr.io/chrislusf/seaweedfs
-          tags: |
-            type=raw,value=dev
-          labels: |
-            org.opencontainers.image.title=seaweedfs
-            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
-            org.opencontainers.image.vendor=Chris Lu
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          buildkitd-flags: "--debug"
-      -
-        name: Login to Docker Hub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      -
-        name: Login to GHCR
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ secrets.GHCR_USERNAME }}
-          password: ${{ secrets.GHCR_TOKEN }}
-      -
-        name: Build
-        uses: docker/build-push-action@v2
-        with:
-          context: ./docker
-          push: ${{ github.event_name != 'pull_request' }}
-          file: ./docker/Dockerfile.go_build
-          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
-          tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
@@ -1,63 +0,0 @@
-name: "docker: build latest container"
-
-on:
-  push:
-    branches: [ master ]
-  workflow_dispatch: []
-
-jobs:
-
-  build-latest-container:
-    runs-on: [ubuntu-latest]
-
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v2
-      -
-        name: Docker meta
-        id: docker_meta
-        uses: docker/metadata-action@v3
-        with:
-          images: |
-            chrislusf/seaweedfs
-            ghcr.io/chrislusf/seaweedfs
-          tags: |
-            type=raw,value=dev
-          labels: |
-            org.opencontainers.image.title=seaweedfs
-            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
-            org.opencontainers.image.vendor=Chris Lu
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          buildkitd-flags: "--debug"
-      -
-        name: Login to Docker Hub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      -
-        name: Login to GHCR
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ secrets.GHCR_USERNAME }}
-          password: ${{ secrets.GHCR_TOKEN }}
-      -
-        name: Build
-        uses: docker/build-push-action@v2
-        with:
-          context: ./docker
-          push: ${{ github.event_name != 'pull_request' }}
-          file: ./docker/Dockerfile.go_build
-          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
-          tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
@@ -1,121 +0,0 @@
-name: "docker: build release containers"
-
-on:
-  push:
-    tags:
-      - '*'
-  workflow_dispatch: []
-
-jobs:
-  build-default-release-container:
-    runs-on: [ubuntu-latest]
-
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v2
-      -
-        name: Docker meta
-        id: docker_meta
-        uses: docker/metadata-action@v3
-        with:
-          images: |
-            chrislusf/seaweedfs
-            ghcr.io/chrislusf/seaweedfs
-          tags: |
-            type=ref,event=tag
-          flavor: |
-            latest=false
-          labels: |
-            org.opencontainers.image.title=seaweedfs
-            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
-            org.opencontainers.image.vendor=Chris Lu
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          buildkitd-flags: "--debug"
-      -
-        name: Login to Docker Hub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      -
-        name: Login to GHCR
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ secrets.GHCR_USERNAME }}
-          password: ${{ secrets.GHCR_TOKEN }}
-      -
-        name: Build
-        uses: docker/build-push-action@v2
-        with:
-          context: ./docker
-          push: ${{ github.event_name != 'pull_request' }}
-          file: ./docker/Dockerfile.go_build
-          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
-          tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
-  build-large-release-container:
-    runs-on: [ubuntu-latest]
-
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v2
-      -
-        name: Docker meta
-        id: docker_meta
-        uses: docker/metadata-action@v3
-        with:
-          images: |
-            chrislusf/seaweedfs
-            ghcr.io/chrislusf/seaweedfs
-          tags: |
-            type=ref,event=tag,suffix=_large_disk
-          flavor: |
-            latest=false
-          labels: |
-            org.opencontainers.image.title=seaweedfs
-            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
-            org.opencontainers.image.vendor=Chris Lu
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          buildkitd-flags: "--debug"
-      -
-        name: Login to Docker Hub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      -
-        name: Login to GHCR
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ secrets.GHCR_USERNAME }}
-          password: ${{ secrets.GHCR_TOKEN }}
-      -
-        name: Build
-        uses: docker/build-push-action@v2
-        with:
-          context: ./docker
-          push: ${{ github.event_name != 'pull_request' }}
-          file: ./docker/Dockerfile.go_build_large
-          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
-          tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
@@ -1,4 +1,4 @@
-name: "go: build binary"
+name: Go

 on:
  push:
@@ -6,10 +6,6 @@ on:
  pull_request:
    branches: [ master ]

-concurrency:
-  group: ${{ github.head_ref }}/go
-  cancel-in-progress: true
-
 jobs:

  build:
@@ -0,0 +1,62 @@
+name: Release
+
+on:
+  push:
+    branches: [ master ]
+
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [linux, windows, darwin, freebsd ]
+        goarch: [amd64, arm]
+        exclude:  
+          - goarch: arm
+            goos: darwin
+          - goarch: arm
+            goos: windows
+
+    steps:
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Wait for the deletion
+      uses: jakejarvis/wait-action@master
+      with:
+        time: '30s'
+
+    - name: Set BUILD_TIME env
+      run: echo BUILD_TIME=$(date -u +%Y-%m-%d-%H-%M) >> ${GITHUB_ENV}
+
+    - name: Go Release Binaries
+      uses: wangyoucao577/go-release-action@v1.14
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        goos: ${{ matrix.goos }}
+        goarch: ${{ matrix.goarch }}
+        release_tag: dev
+        overwrite: true 
+        build_flags: -tags 5BytesOffset # optional, default is 
+        ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+        # Where to run `go build .`
+        project_path: weed
+        binary_name: weed-large-disk
+        asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
+
+    - name: Go Release Binaries
+      uses: wangyoucao577/go-release-action@v1.14
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        goos: ${{ matrix.goos }}
+        goarch: ${{ matrix.goarch }}
+        release_tag: dev
+        overwrite: true 
+        ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+        # Where to run `go build .`
+        project_path: weed
+        binary_name: weed-
+        asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
@@ -0,0 +1,47 @@
+sudo: false
+language: go
+go:
+  - 1.14.x
+  - 1.15.x
+
+before_install:
+  - export PATH=/home/travis/gopath/bin:$PATH
+
+install:
+  - export CGO_ENABLED="0"
+  - go env
+
+script:
+  - env GO111MODULE=on go test ./weed/...
+
+before_deploy:
+  - make release
+deploy:
+  provider: releases
+  skip_cleanup: true
+  api_key:
+    secure: ERL986+ncQ8lwAJUYDrQ8s2/FxF/cyNIwJIFCqspnWxQgGNNyokET9HapmlPSxjpFRF0q6L2WCg9OY3mSVRq4oI6hg1igOQ12KlLyN71XSJ3c8w0Ay5ho48TQ9l3f3Iu97mntBCe9l0R9pnT8wj1VI8YJxloXwUMG2yeTjA9aBI=
+  file:
+    - build/linux_arm.tar.gz
+    - build/linux_arm64.tar.gz
+    - build/linux_386.tar.gz
+    - build/linux_amd64.tar.gz
+    - build/linux_amd64_large_disk.tar.gz
+    - build/darwin_amd64.tar.gz
+    - build/darwin_amd64_large_disk.tar.gz
+    - build/windows_386.zip
+    - build/windows_amd64.zip
+    - build/windows_amd64_large_disk.zip
+    - build/freebsd_arm.tar.gz
+    - build/freebsd_amd64.tar.gz
+    - build/freebsd_386.tar.gz
+    - build/netbsd_arm.tar.gz
+    - build/netbsd_amd64.tar.gz
+    - build/netbsd_386.tar.gz
+    - build/openbsd_arm.tar.gz
+    - build/openbsd_amd64.tar.gz
+    - build/openbsd_386.tar.gz
+  on:
+    tags: true
+    repo: chrislusf/seaweedfs
+    go: 1.15.x
@@ -0,0 +1,144 @@
+BINARY = weed/weed
+package = github.com/chrislusf/seaweedfs/weed
+
+GO_FLAGS = #-v
+SOURCE_DIR = ./weed/
+
+appname := weed
+
+sources := $(wildcard *.go)
+
+COMMIT ?= $(shell git rev-parse --short HEAD)
+LDFLAGS ?= -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${COMMIT}
+
+build = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -ldflags "-extldflags -static $(LDFLAGS)" -o build/$(appname)$(3) $(SOURCE_DIR)
+tar = cd build && tar -cvzf $(1)_$(2).tar.gz $(appname)$(3) && rm $(appname)$(3)
+zip = cd build && zip $(1)_$(2).zip $(appname)$(3) && rm $(appname)$(3)
+
+build_large = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -tags 5BytesOffset -ldflags "-extldflags -static $(LDFLAGS)" -o build/$(appname)$(3) $(SOURCE_DIR)
+tar_large = cd build && tar -cvzf $(1)_$(2)_large_disk.tar.gz $(appname)$(3) && rm $(appname)$(3)
+zip_large = cd build && zip $(1)_$(2)_large_disk.zip $(appname)$(3) && rm $(appname)$(3)
+
+all: build
+
+.PHONY : clean deps build linux release windows_build darwin_build linux_build bsd_build clean
+
+clean:
+	go clean -i $(GO_FLAGS) $(SOURCE_DIR)
+	rm -f $(BINARY)
+	rm -rf build/
+
+deps:
+	go get $(GO_FLAGS) -d $(SOURCE_DIR)
+	rm -rf /home/travis/gopath/src/github.com/coreos/etcd/vendor/golang.org/x/net/trace
+	rm -rf /home/travis/gopath/src/go.etcd.io/etcd/vendor/golang.org/x/net/trace
+
+build: deps
+	go build $(GO_FLAGS) -ldflags "$(LDFLAGS)" -o $(BINARY) $(SOURCE_DIR)
+
+install: deps
+	go install $(GO_FLAGS) -ldflags "$(LDFLAGS)" $(SOURCE_DIR)
+
+linux: deps
+	mkdir -p linux
+	GOOS=linux GOARCH=amd64 go build $(GO_FLAGS) -ldflags "$(LDFLAGS)" -o linux/$(BINARY) $(SOURCE_DIR)
+
+release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_build 5_byte_arm64_build 5_byte_darwin_build 5_byte_windows_build
+
+##### LINUX BUILDS #####
+5_byte_linux_build:
+	$(call build_large,linux,amd64,)
+	$(call tar_large,linux,amd64)
+
+5_byte_darwin_build:
+	$(call build_large,darwin,amd64,)
+	$(call tar_large,darwin,amd64)
+
+5_byte_windows_build:
+	$(call build_large,windows,amd64,.exe)
+	$(call zip_large,windows,amd64,.exe)
+
+5_byte_arm_build: $(sources)
+	$(call build_large,linux,arm,)
+	$(call tar_large,linux,arm)
+
+5_byte_arm64_build: $(sources)
+	$(call build_large,linux,arm64,)
+	$(call tar_large,linux,arm64)
+
+linux_build: build/linux_arm.tar.gz build/linux_arm64.tar.gz build/linux_386.tar.gz build/linux_amd64.tar.gz
+
+build/linux_386.tar.gz: $(sources)
+	$(call build,linux,386,)
+	$(call tar,linux,386)
+
+build/linux_amd64.tar.gz: $(sources)
+	$(call build,linux,amd64,)
+	$(call tar,linux,amd64)
+
+build/linux_arm.tar.gz: $(sources)
+	$(call build,linux,arm,)
+	$(call tar,linux,arm)
+
+build/linux_arm64.tar.gz: $(sources)
+	$(call build,linux,arm64,)
+	$(call tar,linux,arm64)
+
+##### DARWIN (MAC) BUILDS #####
+darwin_build: build/darwin_amd64.tar.gz
+
+build/darwin_amd64.tar.gz: $(sources)
+	$(call build,darwin,amd64,)
+	$(call tar,darwin,amd64)
+
+##### WINDOWS BUILDS #####
+windows_build: build/windows_386.zip build/windows_amd64.zip
+
+build/windows_386.zip: $(sources)
+	$(call build,windows,386,.exe)
+	$(call zip,windows,386,.exe)
+
+build/windows_amd64.zip: $(sources)
+	$(call build,windows,amd64,.exe)
+	$(call zip,windows,amd64,.exe)
+
+##### BSD BUILDS #####
+bsd_build: build/freebsd_arm.tar.gz build/freebsd_386.tar.gz build/freebsd_amd64.tar.gz \
+ build/netbsd_arm.tar.gz build/netbsd_386.tar.gz build/netbsd_amd64.tar.gz \
+ build/openbsd_arm.tar.gz build/openbsd_386.tar.gz build/openbsd_amd64.tar.gz
+
+build/freebsd_386.tar.gz: $(sources)
+	$(call build,freebsd,386,)
+	$(call tar,freebsd,386)
+
+build/freebsd_amd64.tar.gz: $(sources)
+	$(call build,freebsd,amd64,)
+	$(call tar,freebsd,amd64)
+
+build/freebsd_arm.tar.gz: $(sources)
+	$(call build,freebsd,arm,)
+	$(call tar,freebsd,arm)
+
+build/netbsd_386.tar.gz: $(sources)
+	$(call build,netbsd,386,)
+	$(call tar,netbsd,386)
+
+build/netbsd_amd64.tar.gz: $(sources)
+	$(call build,netbsd,amd64,)
+	$(call tar,netbsd,amd64)
+
+build/netbsd_arm.tar.gz: $(sources)
+	$(call build,netbsd,arm,)
+	$(call tar,netbsd,arm)
+
+build/openbsd_386.tar.gz: $(sources)
+	$(call build,openbsd,386,)
+	$(call tar,openbsd,386)
+
+build/openbsd_amd64.tar.gz: $(sources)
+	$(call build,openbsd,amd64,)
+	$(call tar,openbsd,amd64)
+
+build/openbsd_arm.tar.gz: $(sources)
+	$(call build,openbsd,arm,)
+	$(call tar,openbsd,arm)
@@ -1,14 +1,10 @@
 # SeaweedFS


-[![Slack](https://img.shields.io/badge/slack-purple)](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
-[![Twitter](https://img.shields.io/twitter/follow/seaweedfs.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=seaweedfs)
-[![Build Status](https://img.shields.io/github/workflow/status/chrislusf/seaweedfs/Go)](https://github.com/chrislusf/seaweedfs/actions/workflows/go.yml)
+[![Build Status](https://travis-ci.org/chrislusf/seaweedfs.svg?branch=master)](https://travis-ci.org/chrislusf/seaweedfs)
 [![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed)
 [![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki)
-[![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
-[![SeaweedFS on Maven Central](https://img.shields.io/maven-central/v/com.github.chrislusf/seaweedfs-client)](https://search.maven.org/search?q=g:com.github.chrislusf)
-
+[![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/)

 ![SeaweedFS Logo](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/seaweedfs.png)

@@ -43,9 +39,7 @@ Your support will be really appreciated by me and other supporters!
 - [SeaweedFS on Twitter](https://twitter.com/SeaweedFS)
 - [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
 - [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
- [SeaweedFS White Paper](https://github.com/chrislusf/seaweedfs/wiki/SeaweedFS_Architecture.pdf)
- [SeaweedFS Introduction Slides 2021.5](https://docs.google.com/presentation/d/1DcxKWlINc-HNCjhYeERkpGXXm6nTCES8mi2W5G0Z4Ts/edit?usp=sharing)
- [SeaweedFS Introduction Slides 2019.3](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
+- [SeaweedFS Introduction Slides](https://www.slideshare.net/chrislusf/seaweedfs-introduction)

 Table of Contents
 =================
@@ -69,15 +63,11 @@ Table of Contents
 * [License](#license)


-## Quick Start for S3 API on Docker ##
-
-`docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
-
-## Quick Start with single binary ##
+## Quick Start ##
 * Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
 * Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.

-Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
+Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thoudsands of machines. That is it!

 ## Introduction ##

@@ -86,33 +76,17 @@ SeaweedFS is a simple and highly scalable distributed file system. There are two
 1. to store billions of files!
 2. to serve the files fast!

-SeaweedFS started as an Object Store to handle small files efficiently. 
-Instead of managing all file metadata in a central master, 
-the central master only manages volumes on volume servers, 
-and these volume servers manage files and their metadata. 
-This relieves concurrency pressure from the central master and spreads file metadata into volume servers, 
-allowing faster file access (O(1), usually just one disk read operation).
+SeaweedFS started as an Object Store to handle small files efficiently. Instead of managing all file metadata in a central master, the central master only manages volumes on volume servers, and these volume servers manage files and their metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers, allowing faster file access (O(1), usually just one disk read operation).

-There is only 40 bytes of disk storage overhead for each file's metadata. 
-It is so simple with O(1) disk reads that you are welcome to challenge the performance with your actual use cases.
+SeaweedFS can transparently integrate with the cloud. With hot data on local cluster, and warm data on the cloud with O(1) access time, SeaweedFS can achieve both fast local access time and elastic cloud storage capacity. What's more, the cloud storage access API cost is minimized. Faster and Cheaper than direct cloud storage!

-SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf). 
-Also, SeaweedFS implements erasure coding with ideas from 
-[f4: Facebook’s Warm BLOB Storage System](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-muralidhar.pdf), and has a lot of similarities with [Facebook’s Tectonic Filesystem](https://www.usenix.org/system/files/fast21-pan.pdf)
+There is only 40 bytes of disk storage overhead for each file's metadata. It is so simple with O(1) disk reads that you are welcome to challenge the performance with your actual use cases.

-On top of the object store, optional [Filer] can support directories and POSIX attributes. 
-Filer is a separate linearly-scalable stateless server with customizable metadata stores, 
-e.g., MySql, Postgres, Redis, Cassandra, HBase, Mongodb, Elastic Search, LevelDB, RocksDB, Sqlite, MemSql, TiDB, Etcd, CockroachDB, etc.
+SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf). Also, SeaweedFS implements erasure coding with ideas from [f4: Facebook’s Warm BLOB Storage System](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-muralidhar.pdf)

-For any distributed key value stores, the large values can be offloaded to SeaweedFS. 
-With the fast access speed and linearly scalable capacity, 
-SeaweedFS can work as a distributed [Key-Large-Value store][KeyLargeValueStore].
+On top of the object store, optional [Filer] can support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql, Postgres, Redis, Cassandra, HBase, Mongodb, Elastic Search, LevelDB, RocksDB, MemSql, TiDB, Etcd, CockroachDB, etc.

-SeaweedFS can transparently integrate with the cloud. 
-With hot data on local cluster, and warm data on the cloud with O(1) access time, 
-SeaweedFS can achieve both fast local access time and elastic cloud storage capacity.
-What's more, the cloud storage access API cost is minimized. 
-Faster and Cheaper than direct cloud storage!
+For any distributed key value stores, the large values can be offloaded to SeaweedFS. With the fast access speed and linearly scalable capacity, SeaweedFS can work as a distributed [Key-Large-Value store][KeyLargeValueStore].

 [Back to TOC](#table-of-contents)

@@ -128,7 +102,6 @@ Faster and Cheaper than direct cloud storage!
 * Support ETag, Accept-Range, Last-Modified, etc.
 * Support in-memory/leveldb/readonly mode tuning for memory/performance balance.
 * Support rebalancing the writable and readonly volumes.
-* [Customizable Multiple Storage Tiers][TieredStorage]: Customizable storage disk types to balance performance and cost.
 * [Transparent cloud integration][CloudTier]: unlimited capacity via tiered cloud storage for warm data.
 * [Erasure Coding for warm storage][ErasureCoding]  Rack-Aware 10.4 erasure coding reduces storage cost and increases availability.

@@ -146,8 +119,6 @@ Faster and Cheaper than direct cloud storage!
 * [WebDAV] accesses as a mapped drive on Mac and Windows, or from mobile devices.
 * [AES256-GCM Encrypted Storage][FilerDataEncryption] safely stores the encrypted data.
 * [Super Large Files][SuperLargeFiles] stores large or super large files in tens of TB.
-* [Cloud Drive][CloudDrive] mounts cloud storage to local cluster, cached for fast read and write with asynchronous write back.
-* [Gateway to Remote Object Store][GatewayToRemoteObjectStore] mirrors bucket operations to remote object storage, in addition to [Cloud Drive][CloudDrive]

 ## Kubernetes ##
 * [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs-csi-driver.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)
@@ -161,7 +132,6 @@ Faster and Cheaper than direct cloud storage!
 [Hadoop]: https://github.com/chrislusf/seaweedfs/wiki/Hadoop-Compatible-File-System
 [WebDAV]: https://github.com/chrislusf/seaweedfs/wiki/WebDAV
 [ErasureCoding]: https://github.com/chrislusf/seaweedfs/wiki/Erasure-coding-for-warm-storage
-[TieredStorage]: https://github.com/chrislusf/seaweedfs/wiki/Tiered-Storage
 [CloudTier]: https://github.com/chrislusf/seaweedfs/wiki/Cloud-Tier
 [FilerDataEncryption]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Data-Encryption
 [FilerTTL]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Stores
@@ -170,9 +140,6 @@ Faster and Cheaper than direct cloud storage!
 [ActiveActiveAsyncReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Active-Active-cross-cluster-continuous-synchronization
 [FilerStoreReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Store-Replication
 [KeyLargeValueStore]: https://github.com/chrislusf/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store
-[CloudDrive]: https://github.com/chrislusf/seaweedfs/wiki/Cloud-Drive-Architecture
-[GatewayToRemoteObjectStore]: https://github.com/chrislusf/seaweedfs/wiki/Gateway-to-Remote-Object-Storage
-

 [Back to TOC](#table-of-contents)

@@ -326,7 +293,7 @@ When requesting a file key, an optional "dataCenter" parameter can limit the ass

 [Back to TOC](#table-of-contents)

-## Object Store Architecture ##
+## Architecture ##

 Usually distributed file systems split each file into chunks, a central master keeps a mapping of filenames, chunk indices to chunk handles, and also which chunks each chunk server has.

@@ -404,7 +371,7 @@ The architectures are mostly the same. SeaweedFS aims to store and read files fa

 * SeaweedFS optimizes for small files, ensuring O(1) disk seek operation, and can also handle large files.
 * SeaweedFS statically assigns a volume id for a file. Locating file content becomes just a lookup of the volume id, which can be easily cached.
-* SeaweedFS Filer metadata store can be any well-known and proven data stores, e.g., Redis, Cassandra, HBase, Mongodb, Elastic Search, MySql, Postgres, Sqlite, MemSql, TiDB, CockroachDB, Etcd etc, and is easy to customized.
+* SeaweedFS Filer metadata store can be any well-known and proven data stores, e.g., Redis, Cassandra, HBase, Mongodb, Elastic Search, MySql, Postgres, MemSql, TiDB, CockroachDB, Etcd etc, and is easy to customized.
 * SeaweedFS Volume server also communicates directly with clients via HTTP, supporting range queries, direct uploads, etc.

 | System         | File Metadata                   | File Content Read| POSIX  | REST API | Optimized for large number of small files |
@@ -442,11 +409,11 @@ SeaweedFS has a centralized master group to look up free volumes, while Ceph use

 Same as SeaweedFS, Ceph is also based on the object store RADOS. Ceph is rather complicated with mixed reviews.

-Ceph uses CRUSH hashing to automatically manage the data placement, which is efficient to locate the data. But the data has to be placed according to the CRUSH algorithm. Any wrong configuration would cause data loss. Topology changes, such as adding new servers to increase capacity, will cause data migration with high IO cost to fit the CRUSH algorithm. SeaweedFS places data by assigning them to any writable volumes. If writes to one volume failed, just pick another volume to write. Adding more volumes are also as simple as it can be.
+Ceph uses CRUSH hashing to automatically manage the data placement, which is efficient to locate the data. But the data has to be placed according to the CRUSH algorithm. Any wrong configuration would cause data loss. SeaweedFS places data by assigning them to any writable volumes. If writes to one volume failed, just pick another volume to write. Adding more volumes are also as simple as it can be.

 SeaweedFS is optimized for small files. Small files are stored as one continuous block of content, with at most 8 unused bytes between files. Small file access is O(1) disk read.

-SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Sqlite, Mongodb, Redis, Elastic Search, Cassandra, HBase, MemSql, TiDB, CockroachCB, Etcd, to manage file directories. These stores are proven, scalable, and easier to manage.
+SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Mongodb, Redis, Elastic Search, Cassandra, HBase, MemSql, TiDB, CockroachCB, Etcd, to manage file directories. These stores are proven, scalable, and easier to manage.

 | SeaweedFS         | comparable to Ceph | advantage |
 | -------------  | ------------- | ---------------- |
@@ -592,7 +559,6 @@ The text of this page is available for modification and reuse under the terms of

 [Back to TOC](#table-of-contents)

-## Stargazers over time
-
-[![Stargazers over time](https://starchart.cc/chrislusf/seaweedfs.svg)](https://starchart.cc/chrislusf/seaweedfs)
+## Stargazers over time ##

+[![Stargazers over time](https://starcharts.herokuapp.com/chrislusf/seaweedfs.svg)](https://starcharts.herokuapp.com/chrislusf/seaweedfs)
@@ -1,4 +1,3 @@
-
 <h1 align="center">Sponsors &amp; Backers</h1>

 - [Become a backer or sponsor on Patreon](https://www.patreon.com/seaweedfs).
@@ -13,4 +12,3 @@

 - [ColorfulClouds Tech Co. Ltd.](https://caiyunai.com/)
 - [Haravan - Ecommerce Platform](https://www.haravan.com)
- PeterCxy - Creator of Shelter App
@@ -1,16 +1,11 @@
 FROM alpine

-# 'latest' or 'dev'
-ARG RELEASE=latest
-
 RUN \
    ARCH=$(if [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "64" ]; then echo "amd64"; \
         elif [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "32" ]; then echo "386"; \
         elif [ $(uname -m) == "aarch64" ]; then echo "arm64"; \
         elif [ $(uname -m) == "armv7l" ]; then echo "arm"; \
-         elif [ $(uname -m) == "armv6l" ]; then echo "arm"; \
-         elif [ $(uname -m) == "s390x" ]; then echo "s390x"; \
-         elif [ $(uname -m) == "ppc64le" ]; then echo "ppc64le"; fi;) && \
+         elif [ $(uname -m) == "armv6l" ]; then echo "arm"; fi;) && \
    echo "Building for $ARCH" 1>&2 && \
    SUPERCRONIC_SHA1SUM=$(echo $ARCH | sed 's/386/e0126b0102b9f388ecd55714358e3ad60d0cebdb/g' | sed 's/amd64/5ddf8ea26b56d4a7ff6faecdd8966610d5cb9d85/g' | sed 's/arm64/e2714c43e7781bf1579c85aa61259245f56dbba1/g' | sed 's/arm/47481c3341bc3a1ae91a728e0cc63c8e6d3791ad/g') && \
    SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.1.9/supercronic-linux-$ARCH && \
@@ -18,7 +13,7 @@ RUN \
    # Install SeaweedFS and Supercronic ( for cron job mode )
    apk add --no-cache --virtual build-dependencies --update wget curl ca-certificates && \
    apk add fuse && \
-    wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/${RELEASE} | egrep -o "chrislusf/seaweedfs/releases/download/.*/linux_$ARCH.tar.gz" | head -n 1) && \
+    wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/latest | egrep -o "chrislusf/seaweedfs/releases/download/.*/linux_$ARCH.tar.gz") && \
    tar -C /usr/bin/ -xzvf /tmp/linux_$ARCH.tar.gz && \
    curl -fsSLO "$SUPERCRONIC_URL" && \
    echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
@@ -42,8 +37,6 @@ EXPOSE 19333
 EXPOSE 9333
 # s3 server http port
 EXPOSE 8333
-# webdav server http port
-EXPOSE 7333

 RUN mkdir -p /data/filerldb2

@@ -1,43 +0,0 @@
-FROM gcc:11 as builder
-RUN mkdir -p /go/src/github.com/chrislusf/
-RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
-ARG BRANCH=${BRANCH:-master}
-RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
-RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
-  && apt-get update \
-  && apt-get install -y golang-src \
-  && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
-  && CGO_ENABLED=0 go install -ldflags "-extldflags -static ${LDFLAGS}" -compiler=gccgo -tags gccgo,noasm
-
-FROM alpine AS final
-LABEL author="Chris Lu"
-COPY --from=builder /go/bin/weed /usr/bin/
-RUN mkdir -p /etc/seaweedfs
-COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
-COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
-RUN apk add fuse # for weed mount
-
-# volume server gprc port
-EXPOSE 18080
-# volume server http port
-EXPOSE 8080
-# filer server gprc port
-EXPOSE 18888
-# filer server http port
-EXPOSE 8888
-# master server shared gprc port
-EXPOSE 19333
-# master server shared http port
-EXPOSE 9333
-# s3 server http port
-EXPOSE 8333
-# webdav server http port
-EXPOSE 7333
-
-RUN mkdir -p /data/filerldb2
-
-VOLUME /data
-
-RUN chmod +x /entrypoint.sh
-
-ENTRYPOINT ["/entrypoint.sh"]
@@ -1,16 +1,16 @@
-FROM amd64/golang:1.17-alpine as builder
-RUN apk add git g++ fuse
+FROM frolvlad/alpine-glibc as builder
+RUN apk add git go g++ fuse
 RUN mkdir -p /go/src/github.com/chrislusf/
 RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
 ARG BRANCH=${BRANCH:-master}
 RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
 RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
  && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
-  && CGO_ENABLED=0 go install -ldflags "-extldflags -static ${LDFLAGS}"
+  && go install -ldflags "${LDFLAGS}"

 FROM alpine AS final
 LABEL author="Chris Lu"
-COPY --from=builder /go/bin/weed /usr/bin/
+COPY --from=builder /root/go/bin/weed /usr/bin/
 RUN mkdir -p /etc/seaweedfs
 COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
 COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
@@ -30,8 +30,6 @@ EXPOSE 19333
 EXPOSE 9333
 # s3 server http port
 EXPOSE 8333
-# webdav server http port
-EXPOSE 7333

 RUN mkdir -p /data/filerldb2

@@ -1,16 +1,16 @@
-FROM amd64/golang:1.17-alpine as builder
-RUN apk add git g++ fuse
+FROM frolvlad/alpine-glibc as builder
+RUN apk add git go g++ fuse
 RUN mkdir -p /go/src/github.com/chrislusf/
 RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
 ARG BRANCH=${BRANCH:-master}
 RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
 RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
  && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
-  && CGO_ENABLED=0 go install -tags 5BytesOffset -ldflags "-extldflags -static ${LDFLAGS}"
+  && go install -tags 5BytesOffset -ldflags "${LDFLAGS}"

 FROM alpine AS final
 LABEL author="Chris Lu"
-COPY --from=builder /go/bin/weed /usr/bin/
+COPY --from=builder /root/go/bin/weed /usr/bin/
 RUN mkdir -p /etc/seaweedfs
 COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
 COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
@@ -30,8 +30,6 @@ EXPOSE 19333
 EXPOSE 9333
 # s3 server http port
 EXPOSE 8333
-# webdav server http port
-EXPOSE 7333

 RUN mkdir -p /data/filerldb2

@@ -20,8 +20,6 @@ EXPOSE 19333
 EXPOSE 9333
 # s3 server http port
 EXPOSE 8333
-# webdav server http port
-EXPOSE 7333

 RUN mkdir -p /data/filerldb2

@@ -1,31 +0,0 @@
-FROM ubuntu:20.04
-
-RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-        git \
-        sudo \
-        debianutils \
-        python3-pip \
-        python3-virtualenv \
-        python3-dev \
-        libevent-dev \
-        libffi-dev \
-        libxml2-dev \
-        libxslt-dev \
-        zlib1g-dev && \
-    DEBIAN_FRONTEND=noninteractive apt-get clean && \
-    rm -rf /var/lib/apt/lists/* && \
-    git clone https://github.com/ceph/s3-tests.git /opt/s3-tests
-
-WORKDIR /opt/s3-tests
-RUN ./bootstrap
-
-ENV \
-    NOSETESTS_EXCLUDE="" \
-    NOSETESTS_ATTR="" \
-    NOSETESTS_OPTIONS="" \
-    S3TEST_CONF="/s3test.conf"
-
-ENTRYPOINT ["/bin/bash", "-c"]
-CMD ["sleep 10 && exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"]
@@ -4,65 +4,22 @@ all: gen

 gen: dev

-binary:
-	export SWCOMMIT=$(shell git rev-parse --short HEAD)
-	export SWLDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
-	cd ../weed; CGO_ENABLED=0 GOOS=linux go build -ldflags "-extldflags -static $(SWLDFLAGS)"; mv weed ../docker/
-
-build: binary
+build:
+	cd ../weed; GOOS=linux go build; mv weed ../docker/
 	docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
 	rm ./weed

-s3tests_build:
-	docker build --no-cache -t chrislusf/ceph-s3-tests:local -f Dockerfile.s3tests .
-
 dev: build
-	docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
-
-dev_tls: build certstrap
-	ENV_FILE="tls.env" docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
-
-dev_mount: build
-	docker-compose -f compose/local-mount-compose.yml -p seaweedfs up
-
-run_image: build
-	docker run --rm -ti --device /dev/fuse --cap-add SYS_ADMIN --entrypoint /bin/sh chrislusf/seaweedfs:local
-
-profile_mount: build
-	docker-compose -f compose/local-mount-profile-compose.yml -p seaweedfs up
+	docker-compose -f local-dev-compose.yml -p seaweedfs up

 k8s: build
-	docker-compose -f compose/local-k8s-compose.yml -p seaweedfs up
+	docker-compose -f local-k8s-compose.yml -p seaweedfs up

 dev_registry: build
-	docker-compose -f compose/local-registry-compose.yml -p seaweedfs up
-
-dev_replicate: build
-	docker-compose -f compose/local-replicate-compose.yml -p seaweedfs up
+	docker-compose -f local-registry-compose.yml -p seaweedfs up

 cluster: build
-	docker-compose -f compose/local-cluster-compose.yml -p seaweedfs up
-
-2clusters: build
-	docker-compose -f compose/local-clusters-compose.yml -p seaweedfs up
-
-s3tests: build s3tests_build
-	docker-compose -f compose/local-s3tests-compose.yml -p seaweedfs up
-
-filer_etcd: build
-	docker stack deploy -c compose/swarm-etcd.yml fs
+	docker-compose -f local-cluster-compose.yml -p seaweedfs up

 clean:
 	rm ./weed
-
-certstrap:
-	go get github.com/square/certstrap
-	certstrap --depot-path compose/tls init --passphrase "" --common-name "SeaweedFS CA" || true
-	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name volume01.dev || true
-	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name master01.dev || true
-	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name filer01.dev || true
-	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name client01.dev || true
-	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" volume01.dev || true
-	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" master01.dev || true
-	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" filer01.dev || true
-	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" client01.dev || true
@@ -1,28 +0,0 @@
-version: '2'
-
-services:
-  server1:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 9333:9333
-      - 19333:19333
-      - 8084:8080
-      - 18084:18080
-      - 8888:8888
-      - 18888:18888
-    command: "server -ip=server1 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
-    volumes:
-      - ./master-cloud.toml:/etc/seaweedfs/master.toml
-    depends_on:
-      - server2
-  server2:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 9334:9333
-      - 19334:19333
-      - 8085:8080
-      - 18085:18080
-      - 8889:8888
-      - 18889:18888
-      - 8334:8333
-    command: "server -ip=server2 -filer -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
@@ -1,50 +0,0 @@
-version: '2'
-
-services:
-  master:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 9333:9333
-      - 19333:19333
-    command: "master -ip=master -volumeSizeLimitMB=1024"
-  volume:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 8080:8080
-      - 18080:18080
-    command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
-    depends_on:
-      - master
-  s3:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 8888:8888
-      - 18888:18888
-      - 8333:8333
-    command: '-v 1 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333'
-    volumes:
-      - ./s3.json:/etc/seaweedfs/s3.json
-    depends_on:
-      - master
-      - volume
-  minio-gateway-s3:
-    image: minio/minio
-    ports:
-      - 9000:9000
-    command: 'minio gateway s3 http://s3:8333'
-    restart: on-failure
-    environment:
-      MINIO_ACCESS_KEY: "some_access_key1"
-      MINIO_SECRET_KEY: "some_secret_key1"
-    depends_on:
-      - s3
-  minio-warp:
-    image: minio/warp
-    command: 'mixed --duration=5m --obj.size=3mb --autoterm'
-    restart: on-failure
-    environment:
-      WARP_HOST: "minio-gateway-s3:9000"
-      WARP_ACCESS_KEY: "some_access_key1"
-      WARP_SECRET_KEY: "some_secret_key1"
-    depends_on:
-      - minio-gateway-s3
@@ -1,46 +0,0 @@
-version: '2'
-
-services:
-  master:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 9333:9333
-      - 19333:19333
-    command: "master -ip=master"
-  volume:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 7455:8080
-      - 9325:9325
-    command: 'volume -mserver="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455'
-    depends_on:
-      - master
-  filer:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 8888:8888
-      - 18888:18888
-      - 9326:9326
-    command: 'filer -master="master:9333"  -metricsPort=9326'
-    tty: true
-    stdin_open: true
-    depends_on:
-      - master
-      - volume
-  mount_1:
-    image: chrislusf/seaweedfs:local
-    privileged: true
-    entrypoint: '/bin/sh -c "mkdir -p t1 && mkdir -p cache/t1 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t1 -dir=./t1 -filer.path=/c1 -volumeServerAccess=filerProxy"'
-    depends_on:
-      - master
-      - volume
-      - filer
-  mount_2:
-    image: chrislusf/seaweedfs:local
-    privileged: true
-    entrypoint: '/bin/sh -c "mkdir -p t2 && mkdir -p cache/t2 && weed -v=4  mount -filer=filer:8888 -cacheDir=./cache/t2 -dir=./t2 -filer.path=/c1"'
-    depends_on:
-      - master
-      - volume
-      - filer
-      - mount_1
@@ -1,47 +0,0 @@
-version: '2'
-
-services:
-  master:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 9333:9333
-      - 19333:19333
-    command: "master -ip=master"
-  volume:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 7455:8080
-      - 9325:9325
-    volumes:
-      - /Volumes/mobile_disk/99:/data
-    command: 'volume -mserver="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455'
-    depends_on:
-      - master
-  filer:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 8888:8888
-      - 18888:18888
-      - 9326:9326
-    volumes:
-      - /Volumes/mobile_disk/99:/data
-    command: 'filer -master="master:9333"  -metricsPort=9326'
-    tty: true
-    stdin_open: true
-    depends_on:
-      - master
-      - volume
-  mount:
-    image: chrislusf/seaweedfs:local
-    privileged: true
-    cap_add:
-      - SYS_ADMIN
-    devices:
-      - fuse
-    volumes:
-      - /Volumes/mobile_disk/99:/data
-    entrypoint: '/bin/sh -c "mkdir -p t1 && weed -v=4 mount -filer=filer:8888 -dir=./t1 -cacheCapacityMB=0 -memprofile=/data/mount.mem.pprof"'
-    depends_on:
-      - master
-      - volume
-      - filer
@@ -1,61 +0,0 @@
-version: '2'
-
-services:
-  master:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 9333:9333
-      - 19333:19333
-    command: "master -ip=master"
-  volume:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 8080:8080
-      - 18080:18080
-    command: "volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
-    depends_on:
-      - master
-  filer:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 8888:8888
-      - 18888:18888
-    command: '-v=9 filer -master="master:9333"'
-    restart: on-failure
-    volumes:
-      - ./notification.toml:/etc/seaweedfs/notification.toml
-    depends_on:
-      - master
-      - volume
-      - rabbitmq
-      - replicate
-    environment:
-      RABBIT_SERVER_URL: "amqp://guest:guest@rabbitmq:5672/"
-  replicate:
-    image: chrislusf/seaweedfs:local
-    command: '-v=9 filer.replicate'
-    restart: on-failure
-    volumes:
-      - ./notification.toml:/etc/seaweedfs/notification.toml
-      - ./replication.toml:/etc/seaweedfs/replication.toml
-    depends_on:
-      - rabbitmq
-    environment:
-      RABBIT_SERVER_URL: "amqp://guest:guest@rabbitmq:5672/"
-  s3:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 8333:8333
-    command: 's3 -filer="filer:8888"'
-    depends_on:
-      - master
-      - volume
-      - filer
-  rabbitmq:
-    image: rabbitmq:3.8.10-management-alpine
-    ports:
-      - 5672:5672
-      - 15671:15671
-      - 15672:15672
-    environment:
-      RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "-rabbit log_levels [{connection,error},{queue,debug}]"
@@ -1,45 +0,0 @@
-version: '2'
-
-services:
-  master:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 9333:9333
-      - 19333:19333
-    command: "master -ip=master -volumeSizeLimitMB=16"
-    environment:
-      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
-      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
-  volume:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 8080:8080
-      - 18080:18080
-    command: "volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
-    depends_on:
-      - master
-  s3:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 8888:8888
-      - 18888:18888
-      - 8000:8000
-    command: 'filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000'
-    volumes:
-      - ./s3.json:/etc/seaweedfs/s3.json
-    depends_on:
-      - master
-      - volume
-  s3tests:
-    image: chrislusf/ceph-s3-tests:local
-    volumes:
-      - ./s3tests.conf:/opt/s3-tests/s3tests.conf
-    environment:
-      S3TEST_CONF: "s3tests.conf"
-      NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
-      NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
-      NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
-    depends_on:
-      - master
-      - volume
-      - s3
@@ -1,30 +0,0 @@
-
-# Put this file to one of the location, with descending priority
-#    ./master.toml
-#    $HOME/.seaweedfs/master.toml
-#    /etc/seaweedfs/master.toml
-# this file is read by master
-
-[master.maintenance]
-# periodically run these scripts are the same as running them from 'weed shell'
-scripts = """
-  lock
-  ec.encode -fullPercent=95 -quietFor=1h
-  ec.rebuild -force
-  ec.balance -force
-  volume.balance -force
-  volume.fix.replication
-  unlock
-"""
-sleep_minutes = 17          # sleep minutes between each script execution
-
-# configurations for tiered cloud storage
-# old volumes are transparently moved to cloud for cost efficiency
-[storage.backend]
-	[storage.backend.s3.default]
-	enabled = true
-	aws_access_key_id     = "any"     # if empty, loads from the shared credentials file (~/.aws/credentials).
-	aws_secret_access_key = "any"     # if empty, loads from the shared credentials file (~/.aws/credentials).
-	region = "us-east-2"
-	bucket = "volume_bucket"    # an existing bucket
-	endpoint = "http://server2:8333"
@@ -1,17 +0,0 @@
-[notification.log]
-# this is only for debugging perpose and does not work with "weed filer.replicate"
-enabled = false
-
-
-[notification.gocdk_pub_sub]
-# The Go Cloud Development Kit (https://gocloud.dev).
-# PubSub API (https://godoc.org/gocloud.dev/pubsub).
-# Supports AWS SNS/SQS, Azure Service Bus, Google PubSub, NATS and RabbitMQ.
-enabled = true
-# This URL will Dial the RabbitMQ server at the URL in the environment
-# variable RABBIT_SERVER_URL and open the exchange "myexchange".
-# The exchange must have already been created by some other means, like
-# the RabbitMQ management plugin. Сreate myexchange of type fanout and myqueue then
-# create binding myexchange => myqueue
-topic_url = "rabbit://swexchange"
-sub_url = "rabbit://swqueue"
@@ -1,11 +0,0 @@
-[source.filer]
-enabled = true
-grpcAddress = "filer:18888"
-# all files under this directory tree are replicated.
-# this is not a directory on your hard drive, but on your filer.
-# i.e., all files with this "prefix" are sent to notification message queue.
-directory = "/buckets"
-
-[sink.local_incremental]
-enabled = true
-directory = "/data"
@@ -1,105 +0,0 @@
-{
-  "identities": [
-    {
-      "name": "anonymous",
-      "actions": [
-        "Read"
-      ]
-    },
-    {
-      "name": "some_admin_user",
-      "credentials": [
-        {
-          "accessKey": "some_access_key1",
-          "secretKey": "some_secret_key1"
-        }
-      ],
-      "actions": [
-        "Admin",
-        "Read",
-        "List",
-        "Tagging",
-        "Write"
-      ]
-    },
-    {
-      "name": "s3_tests",
-      "credentials": [
-        {
-          "accessKey": "ABCDEFGHIJKLMNOPQRST",
-          "secretKey": "abcdefghijklmnopqrstuvwxyzabcdefghijklmn"
-        },
-        {
-          "accessKey": "0555b35654ad1656d804",
-          "secretKey": "h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=="
-        }
-      ],
-      "actions": [
-        "Admin",
-        "Read",
-        "List",
-        "Tagging",
-        "Write"
-      ]
-    },
-    {
-      "name": "s3_tests_alt",
-      "credentials": [
-        {
-          "accessKey": "NOPQRSTUVWXYZABCDEFG",
-          "secretKey": "nopqrstuvwxyzabcdefghijklmnabcdefghijklm"
-        }
-      ],
-      "actions": [
-        "Admin",
-        "Read",
-        "List",
-        "Tagging",
-        "Write"
-      ]
-    },
-    {
-      "name": "s3_tests_tenant",
-      "credentials": [
-        {
-          "accessKey": "HIJKLMNOPQRSTUVWXYZA",
-          "secretKey": "opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab"
-        }
-      ],
-      "actions": [
-        "Admin",
-        "Read",
-        "List",
-        "Tagging",
-        "Write"
-      ]
-    },
-    {
-      "name": "some_read_only_user",
-      "credentials": [
-        {
-          "accessKey": "some_access_key2",
-          "secretKey": "some_secret_key2"
-        }
-      ],
-      "actions": [
-        "Read"
-      ]
-    },
-    {
-      "name": "some_normal_user",
-      "credentials": [
-        {
-          "accessKey": "some_access_key3",
-          "secretKey": "some_secret_key3"
-        }
-      ],
-      "actions": [
-        "Read",
-        "List",
-        "Tagging",
-        "Write"
-      ]
-    }
-  ]
-}
@@ -1,70 +0,0 @@
-[DEFAULT]
-## this section is just used for host, port and bucket_prefix
-
-# host set for rgw in vstart.sh
-host = s3
-
-# port set for rgw in vstart.sh
-port = 8000
-
-## say "False" to disable TLS
-is_secure = False
-
-[fixtures]
-## all the buckets created will start with this prefix;
-## {random} will be filled with random characters to pad
-## the prefix to 30 characters long, and avoid collisions
-bucket prefix = yournamehere-{random}-
-
-[s3 main]
-# main display_name set in vstart.sh
-display_name = M. Tester
-
-# main user_idname set in vstart.sh
-user_id = testid
-
-# main email set in vstart.sh
-email = tester@ceph.com
-
-# zonegroup api_name for bucket location
-api_name = default
-
-## main AWS access key
-access_key = 0555b35654ad1656d804
-
-## main AWS secret key
-secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==
-
-## replace with key id obtained when secret is created, or delete if KMS not tested
-#kms_keyid = 01234567-89ab-cdef-0123-456789abcdef
-
-[s3 alt]
-# alt display_name set in vstart.sh
-display_name = john.doe
-## alt email set in vstart.sh
-email = john.doe@example.com
-
-# alt user_id set in vstart.sh
-user_id = 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234
-
-# alt AWS access key set in vstart.sh
-access_key = NOPQRSTUVWXYZABCDEFG
-
-# alt AWS secret key set in vstart.sh
-secret_key = nopqrstuvwxyzabcdefghijklmnabcdefghijklm
-
-[s3 tenant]
-# tenant display_name set in vstart.sh
-display_name = testx$tenanteduser
-
-# tenant user_id set in vstart.sh
-user_id = 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-
-# tenant AWS secret key set in vstart.sh
-access_key = HIJKLMNOPQRSTUVWXYZA
-
-# tenant AWS secret key set in vstart.sh
-secret_key = opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab
-
-# tenant email set in vstart.sh
-email = tenanteduser@example.com
@@ -1,84 +0,0 @@
-# 2021-01-30 16:25:30
-version: '3.8'
-
-services:
-
-  etcd:
-    image: gasparekatapy/etcd
-    networks:
-      - net
-    deploy:
-      mode: replicated
-      replicas: 3
-
-  master:
-    image: chrislusf/seaweedfs:local
-    environment:
-      WEED_MASTER_FILER_DEFAULT: "filer:8888"
-      WEED_MASTER_SEQUENCER_TYPE: "raft"
-    ports:
-      - "9333:9333"
-      - "19333:19333"
-    networks:
-      - net
-    command:
-      - 'master'
-      - '-resumeState=true'
-      - '-ip=master'
-      - '-port=9333'
-    deploy:
-      mode: replicated
-      replicas: 1
-
-  filer:
-    image: chrislusf/seaweedfs:local
-    environment:
-      WEED_LEVELDB2_ENABLED: "false"
-      WEED_ETCD_ENABLED: "true"
-      WEED_ETCD_SERVERS: "etcd:2379"
-    ports:
-      - target: 8888
-        published: 8888
-        protocol: tcp
-        mode: host
-      - target: 18888
-        published: 18888
-        protocol: tcp
-        mode: host
-    networks:
-      - net
-    command:
-      - 'filer'
-      - '-ip=filer'
-      - '-port=8888'
-      - '-port.readonly=28888'
-      - '-master=master:9333'
-      - '-disableDirListing=true'
-    deploy:
-      mode: replicated
-      replicas: 1
-
-  volume:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - target: 8080
-        published: 8080
-        protocol: tcp
-        mode: host
-      - target: 18080
-        published: 18080
-        protocol: tcp
-        mode: host
-    networks:
-      - net
-    command:
-      - 'volume'
-      - '-mserver=master:9333'
-      - '-port=8080'
-    deploy:
-      mode: global
-
-  ###########################################################################
-
-networks:
-  net:
@@ -1,14 +0,0 @@
-WEED_GRPC_CA=/etc/seaweedfs/tls/SeaweedFS_CA.crt
-WEED_GRPC_ALLOWED_WILDCARD_DOMAIN=".dev"
-WEED_GRPC_MASTER_CERT=/etc/seaweedfs/tls/master01.dev.crt
-WEED_GRPC_MASTER_KEY=/etc/seaweedfs/tls/master01.dev.key
-WEED_GRPC_VOLUME_CERT=/etc/seaweedfs/tls/volume01.dev.crt
-WEED_GRPC_VOLUME_KEY=/etc/seaweedfs/tls/volume01.dev.key
-WEED_GRPC_FILER_CERT=/etc/seaweedfs/tls/filer01.dev.crt
-WEED_GRPC_FILER_KEY=/etc/seaweedfs/tls/filer01.dev.key
-WEED_GRPC_CLIENT_CERT=/etc/seaweedfs/tls/client01.dev.crt
-WEED_GRPC_CLIENT_KEY=/etc/seaweedfs/tls/client01.dev.key
-WEED_GRPC_MASTER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
-WEED_GRPC_VOLUME_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
-WEED_GRPC_FILER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
-WEED_GRPC_CLIENT_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
@@ -60,9 +60,9 @@ case "$1" in
  'cronjob')
 	MASTER=${WEED_MASTER-localhost:9333}
 	FIX_REPLICATION_CRON_SCHEDULE=${CRON_SCHEDULE-*/7 * * * * *}
-	echo "$FIX_REPLICATION_CRON_SCHEDULE" 'echo "lock; volume.fix.replication; unlock" | weed shell -master='$MASTER > /crontab
+	echo "$FIX_REPLICATION_CRON_SCHEDULE" 'echo "volume.fix.replication" | weed shell -master='$MASTER > /crontab
 	BALANCING_CRON_SCHEDULE=${CRON_SCHEDULE-25 * * * * *}
-	echo "$BALANCING_CRON_SCHEDULE" 'echo "lock; volume.balance -collection ALL_COLLECTIONS -force; unlock" | weed shell -master='$MASTER >> /crontab
+	echo "$BALANCING_CRON_SCHEDULE" 'echo "volume.balance -c ALL -force" | weed shell -master='$MASTER >> /crontab
 	echo "Running Crontab:"
 	cat /crontab
 	exec supercronic /crontab
@@ -19,32 +19,12 @@ services:
      - 9335:9335
      - 19335:19335
    command: "master -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m3"
-  volume1:
+  volume:
    image: chrislusf/seaweedfs:local
    ports:
      - 8080:8080
      - 18080:18080
-    command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1 -disk=ssd1'
-    depends_on:
-      - master0
-      - master1
-      - master2
-  volume2:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 8082:8082
-      - 18082:18082
-    command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1 -disk=ssd1'
-    depends_on:
-      - master0
-      - master1
-      - master2
-  volume3:
-    image: chrislusf/seaweedfs:local
-    ports:
-      - 8083:8083
-      - 18083:18083
-    command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
+    command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume -publicUrl=localhost:8080'
    depends_on:
      - master0
      - master1
@@ -59,8 +39,7 @@ services:
      - master0
      - master1
      - master2
-      - volume1
-      - volume2
+      - volume
  s3:
    image: chrislusf/seaweedfs:local
    ports:
@@ -70,6 +49,5 @@ services:
      - master0
      - master1
      - master2
-      - volume1
-      - volume2
+      - volume
      - filer
@@ -6,50 +6,33 @@ services:
    ports:
      - 9333:9333
      - 19333:19333
-    command: "-v=1 master -ip=master"
-    volumes:
-      - ./tls:/etc/seaweedfs/tls
-    env_file:
-     - ${ENV_FILE:-dev.env}
+    command: "master -ip=master"
  volume:
    image: chrislusf/seaweedfs:local
    ports:
      - 8080:8080
      - 18080:18080
-    command: "-v=1 volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
+    command: "volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
    depends_on:
      - master
-    volumes:
-      - ./tls:/etc/seaweedfs/tls
-    env_file:
-      - ${ENV_FILE:-dev.env}
  filer:
    image: chrislusf/seaweedfs:local
    ports:
-      - 8111:8111
      - 8888:8888
      - 18888:18888
-    command: '-v=1 filer -master="master:9333" -iam'
+    command: 'filer -master="master:9333"'
    depends_on:
      - master
      - volume
-    volumes:
-      - ./tls:/etc/seaweedfs/tls
-    env_file:
-      - ${ENV_FILE:-dev.env}
  s3:
    image: chrislusf/seaweedfs:local
    ports:
      - 8333:8333
-    command: '-v=1 s3 -filer="filer:8888"'
+    command: 's3 -filer="filer:8888"'
    depends_on:
      - master
      - volume
      - filer
-    volumes:
-      - ./tls:/etc/seaweedfs/tls
-    env_file:
-      - ${ENV_FILE:-dev.env}
  mount:
    image: chrislusf/seaweedfs:local
    privileged: true
@@ -57,10 +40,6 @@ services:
      - SYS_ADMIN
    mem_limit: 4096m
    command: '-v=4 mount -filer="filer:8888" -dirAutoCreate -dir=/mnt/seaweedfs -cacheCapacityMB=100 -concurrentWriters=128'
-    volumes:
-      - ./tls:/etc/seaweedfs/tls
-    env_file:
-      - ${ENV_FILE:-dev.env}
    depends_on:
      - master
      - volume
@@ -15,18 +15,24 @@ services:
    command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
    depends_on:
      - master
-  s3:
+  filer:
    image: chrislusf/seaweedfs:local
    ports:
      - 8888:8888
      - 18888:18888
-      - 8333:8333
-    command: '-v 9 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333'
-    volumes:
-      - ./s3.json:/etc/seaweedfs/s3.json
+    command: 'filer -master="master:9333"'
    depends_on:
      - master
      - volume
+  s3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8333:8333
+    command: '-v 9 s3 -filer="filer:8888"'
+    depends_on:
+      - master
+      - volume
+      - filer
  minio:
    image: minio/minio
    ports:
@@ -48,15 +48,6 @@ services:
      - master
      - volume
      - filer
-  webdav:
-    image: chrislusf/seaweedfs # use a remote image
-    ports:
-      - 7333:7333
-    command: 'webdav -filer="filer:8888"'
-    depends_on:
-      - master
-      - volume
-      - filer
  prometheus:
    image: prom/prometheus:v2.21.0
    ports:
@@ -33,12 +33,3 @@ services:
      - master
      - volume
      - filer
-  webdav:
-    image: chrislusf/seaweedfs:dev # use a remote dev image
-    ports:
-      - 7333:7333
-    command: 'webdav -filer="filer:8888"'
-    depends_on:
-      - master
-      - volume
-      - filer
@@ -1,34 +1,22 @@
 module github.com/chrislusf/seaweedfs

-go 1.17
+go 1.12

 require (
-	cloud.google.com/go v0.94.1 // indirect
-	cloud.google.com/go/pubsub v1.3.1
-	cloud.google.com/go/storage v1.16.1
-	github.com/Azure/azure-pipeline-go v0.2.3
-	github.com/Azure/azure-storage-blob-go v0.14.0
-	github.com/BurntSushi/toml v0.3.1 // indirect
-	github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect
+	cloud.google.com/go v0.44.3
+	github.com/Azure/azure-pipeline-go v0.2.2 // indirect
+	github.com/Azure/azure-storage-blob-go v0.8.0
 	github.com/OneOfOne/xxhash v1.2.2
 	github.com/Shopify/sarama v1.23.1
-	github.com/aws/aws-sdk-go v1.35.3
-	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/aws/aws-sdk-go v1.34.30
 	github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
-	github.com/bwmarrin/snowflake v0.3.0
 	github.com/cespare/xxhash v1.1.0
-	github.com/cespare/xxhash/v2 v2.1.1 // indirect
-	github.com/chrislusf/raft v1.0.7
-	github.com/colinmarc/hdfs/v2 v2.2.0
+	github.com/chrislusf/raft v1.0.4
 	github.com/coreos/go-semver v0.3.0 // indirect
-	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/disintegration/imaging v1.6.2
 	github.com/dustin/go-humanize v1.0.0
 	github.com/eapache/go-resiliency v1.2.0 // indirect
-	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
-	github.com/eapache/queue v1.1.0 // indirect
 	github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a
 	github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
@@ -36,167 +24,78 @@ require (
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
 	github.com/fclairamb/ftpserverlib v0.8.0
 	github.com/frankban/quicktest v1.7.2 // indirect
-	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/go-errors/errors v1.1.1 // indirect
 	github.com/go-redis/redis/v8 v8.4.4
 	github.com/go-sql-driver/mysql v1.5.0
-	github.com/go-stack/stack v1.8.0 // indirect
-	github.com/go-zookeeper/zk v1.0.2 // indirect
-	github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
-	github.com/golang-jwt/jwt v3.2.1+incompatible
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
-	github.com/golang/protobuf v1.5.2
-	github.com/golang/snappy v0.0.4 // indirect
+	github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6
+	github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 // indirect
+	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6
+	github.com/golang/protobuf v1.4.2
 	github.com/google/btree v1.0.0
-	github.com/google/go-cmp v0.5.6 // indirect
-	github.com/google/uuid v1.3.0
-	github.com/google/wire v0.4.0 // indirect
-	github.com/googleapis/gax-go v2.0.2+incompatible // indirect
-	github.com/googleapis/gax-go/v2 v2.1.0 // indirect
+	github.com/google/uuid v1.1.1
 	github.com/gorilla/mux v1.7.4
 	github.com/gorilla/websocket v1.4.1 // indirect
-	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
-	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
-	github.com/hashicorp/errwrap v1.0.0 // indirect
-	github.com/hashicorp/go-multierror v1.0.0 // indirect
-	github.com/hashicorp/go-uuid v1.0.2 // indirect
-	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
 	github.com/jcmturner/gofork v1.0.0 // indirect
-	github.com/jcmturner/gokrb5/v8 v8.4.1
-	github.com/jinzhu/copier v0.2.8
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/json-iterator/go v1.1.11
+	github.com/json-iterator/go v1.1.10
+	github.com/karlseguin/ccache v2.0.3+incompatible
 	github.com/karlseguin/ccache/v2 v2.0.7
-	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/klauspost/compress v1.10.9 // indirect
 	github.com/klauspost/cpuid v1.2.1 // indirect
+	github.com/klauspost/crc32 v1.2.0
 	github.com/klauspost/reedsolomon v1.9.2
-	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
 	github.com/kurin/blazer v0.5.3
-	github.com/lib/pq v1.10.0
+	github.com/lib/pq v1.2.0
+	github.com/lunixbochs/vtclean v1.0.0 // indirect
 	github.com/magiconair/properties v1.8.1 // indirect
-	github.com/mailru/easyjson v0.7.1 // indirect
-	github.com/mattn/go-ieproxy v0.0.1 // indirect
-	github.com/mattn/go-isatty v0.0.12 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/mitchellh/mapstructure v1.1.2 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
-	github.com/nats-io/jwt v1.0.1 // indirect
-	github.com/nats-io/nats.go v1.10.0 // indirect
-	github.com/nats-io/nkeys v0.2.0 // indirect
-	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect
+	github.com/mattn/go-runewidth v0.0.4 // indirect
 	github.com/olivere/elastic/v7 v7.0.19
-	github.com/pelletier/go-toml v1.7.0 // indirect
 	github.com/peterh/liner v1.1.0
 	github.com/pierrec/lz4 v2.2.7+incompatible // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/posener/complete v1.2.3
-	github.com/pquerna/cachecontrol v0.1.0
-	github.com/prometheus/client_golang v1.11.0
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.26.0 // indirect
-	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/prometheus/client_golang v1.3.0
+	github.com/rakyll/statik v0.1.7
 	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
-	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
-	github.com/seaweedfs/fuse v1.2.0
+	github.com/seaweedfs/fuse v1.0.9
 	github.com/seaweedfs/goexif v1.0.2
-	github.com/sirupsen/logrus v1.6.0 // indirect
+	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/spf13/afero v1.6.0 // indirect
-	github.com/spf13/cast v1.3.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/viper v1.4.0
-	github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71
-	github.com/stretchr/testify v1.7.0
-	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
+	github.com/stretchr/testify v1.6.1
+	github.com/syndtr/goleveldb v1.0.0
 	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
-	github.com/tidwall/gjson v1.8.1
-	github.com/tidwall/match v1.0.3
-	github.com/tidwall/pretty v1.1.0 // indirect
-	github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210824090536-16d902a3c7e5
+	github.com/tidwall/gjson v1.3.2
+	github.com/tidwall/match v1.0.1
 	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
-	github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43
 	github.com/valyala/bytebufferpool v1.0.0
 	github.com/viant/assertly v0.5.4 // indirect
 	github.com/viant/ptrie v0.3.0
 	github.com/viant/toolbox v0.33.2 // indirect
-	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
-	github.com/xdg-go/scram v1.0.2 // indirect
-	github.com/xdg-go/stringprep v1.0.2 // indirect
-	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	go.etcd.io/etcd v3.3.25+incompatible
-	go.mongodb.org/mongo-driver v1.7.0
-	go.opencensus.io v0.23.0 // indirect
-	go.opentelemetry.io/otel v0.15.0 // indirect
-	gocloud.dev v0.20.0
-	gocloud.dev/pubsub/natspubsub v0.20.0
-	gocloud.dev/pubsub/rabbitpubsub v0.20.0
-	golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 // indirect
-	golang.org/x/image v0.0.0-20200119044424-58c23975cae1
-	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
-	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
-	golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf
-	golang.org/x/text v0.3.6 // indirect
-	golang.org/x/tools v0.1.5
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/api v0.56.0
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83 // indirect
-	google.golang.org/grpc v1.40.0
-	google.golang.org/protobuf v1.27.1
-	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect
-	gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect
+	github.com/willf/bitset v1.1.10 // indirect
+	github.com/willf/bloom v2.0.3+incompatible
+	github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 // indirect
+	go.etcd.io/etcd v3.3.15+incompatible
+	go.mongodb.org/mongo-driver v1.3.2
+	gocloud.dev v0.16.0
+	gocloud.dev/pubsub/natspubsub v0.16.0
+	gocloud.dev/pubsub/rabbitpubsub v0.16.0
+	golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
+	golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb
+	golang.org/x/sync v0.0.0-20200930132711-30421366ff76 // indirect
+	golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd
+	golang.org/x/tools v0.0.0-20200103221440-774c71fcf114
+	google.golang.org/api v0.9.0
+	google.golang.org/appengine v1.6.2 // indirect
+	google.golang.org/grpc v1.29.1
+	google.golang.org/protobuf v1.24.0
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect
 	gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect
-	gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect
-	modernc.org/b v1.0.0 // indirect
-	modernc.org/cc/v3 v3.33.5 // indirect
-	modernc.org/ccgo/v3 v3.9.4 // indirect
-	modernc.org/libc v1.9.5 // indirect
-	modernc.org/mathutil v1.2.2 // indirect
-	modernc.org/memory v1.0.4 // indirect
-	modernc.org/opt v0.1.1 // indirect
-	modernc.org/sqlite v1.10.7
-	modernc.org/strutil v1.1.0 // indirect
-	modernc.org/token v1.0.0 // indirect
-)
-
-require (
-	github.com/coreos/etcd v3.3.10+incompatible // indirect
-	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
-	github.com/d4l3k/messagediff v1.2.1 // indirect
-	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
-	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
-	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
-	github.com/jcmturner/rpc/v2 v2.0.2 // indirect
-	github.com/mattn/go-runewidth v0.0.7 // indirect
-	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3 // indirect
-	github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd // indirect
-	github.com/pingcap/kvproto v0.0.0-20210806074406-317f69fb54b4 // indirect
-	github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4 // indirect
-	github.com/pingcap/parser v0.0.0-20210525032559-c37778aff307 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d // indirect
-	github.com/twmb/murmur3 v1.1.3 // indirect
-	go.etcd.io/etcd/api/v3 v3.5.0 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect
-	go.etcd.io/etcd/client/v3 v3.5.0 // indirect
-	go.uber.org/atomic v1.7.0 // indirect
-	go.uber.org/multierr v1.7.0 // indirect
-	go.uber.org/zap v1.17.0 // indirect
-	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
-	golang.org/x/mod v0.4.2 // indirect
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/karlseguin/expect.v1 v1.0.1 // indirect
 )

 // replace github.com/seaweedfs/fuse => /Users/chris/go/src/github.com/seaweedfs/fuse
 // replace github.com/chrislusf/raft => /Users/chris/go/src/github.com/chrislusf/raft
+
+replace go.etcd.io/etcd => go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547
@@ -0,0 +1,23 @@
+## SEAWEEDFS - helm chart (2.x)
+
+### info:
+* master/filer/volume are stateful sets with anti-affinity on the hostname,
+so your deployment will be spread/HA.
+* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances)
+and backup/HA memsql can provide.
+* mysql user/password are created in a k8s secret (secret-seaweedfs-db.yaml) and injected to the filer
+with ENV.
+* cert config exists and can be enabled, but not been tested.
+
+### current instances config (AIO):
+1 instance for each type (master/filer/volume/s3)
+
+instances need node labels:
+* sw-volume: true  (for volume instance, specific tag)
+* sw-backend: true (for all others, as they less resource demanding)
+
+you can update the replicas count for each node type in values.yaml,
+need to add more nodes with the corresponding label.
+
+most of the configuration are available through values.yaml
+
@@ -1,48 +0,0 @@
-## SEAWEEDFS - helm chart (2.x)
-
-### info:
-* master/filer/volume are stateful sets with anti-affinity on the hostname,
-so your deployment will be spread/HA.
-* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances)
-and backup/HA memsql can provide.
-* mysql user/password are created in a k8s secret (secret-seaweedfs-db.yaml) and injected to the filer
-with ENV.
-* cert config exists and can be enabled, but not been tested.
-
-### prerequisites
-kubernetes node have labels which help to define which node(Host) will run which pod.
-
-s3/filer/master needs the label **sw-backend=true**
-
-volume need the label **sw-volume=true**
-
-to label a node to be able to run all pod types in k8s:
-```
-kubectl label node YOUR_NODE_NAME sw-volume=true,sw-backend=true
-```
-
-on production k8s deployment you will want each pod to have a different host,
-especially the volume server & the masters, currently all pods (master/volume/filer)
-have anti-affinity rule to disallow running multiple pod type on the same host.
-if you still want to run multiple pods of the same type (master/volume/filer) on the same host
-please set/update the corresponding affinity rule in values.yaml to an empty one:
-
-```affinity: ""```
-
-### PVC - storage class ###
-
-on the volume stateful set added support for K8S PVC, currently example
-with the simple local-path-provisioner from Rancher (comes included with k3d / k3s)
-https://github.com/rancher/local-path-provisioner
-
-you can use ANY storage class you like, just update the correct storage-class
-for your deployment.
-
-### current instances config (AIO):
-1 instance for each type (master/filer+s3/volume)
-
-you can update the replicas count for each node type in values.yaml,
-need to add more nodes with the corresponding labels.
-
-most of the configuration are available through values.yaml
-
@@ -1,21 +0,0 @@
-{{- if not (or .Values.filer.s3.skipAuthSecretCreation .Values.s3.skipAuthSecretCreation) }}
-{{- $access_key_admin := randAlphaNum 16 -}}
-{{- $secret_key_admin := randAlphaNum 32 -}}
-{{- $access_key_read := randAlphaNum 16 -}}
-{{- $secret_key_read := randAlphaNum 32 -}}
-apiVersion: v1
-kind: Secret
-type: Opaque
-metadata:
-  name: seaweedfs-s3-secret
-  namespace: {{ .Release.Namespace }}
-  annotations:
-    "helm.sh/resource-policy": keep
-    "helm.sh/hook": "pre-install"
-stringData:
-  admin_access_key_id: {{ $access_key_admin }}
-  admin_secret_access_key: {{ $secret_key_admin }}
-  read_access_key_id: {{ $access_key_read }}
-  read_secret_access_key: {{ $secret_key_read }}
-  seaweedfs_s3_config: '{"identities":[{"name":"anvAdmin","credentials":[{"accessKey":"{{ $access_key_admin }}","secretKey":"{{ $secret_key_admin }}"}],"actions":["Admin","Read","Write"]},{"name":"anvReadOnly","credentials":[{"accessKey":"{{ $access_key_read }}","secretKey":"{{ $secret_key_read }}"}],"actions":["Read"]}]}'
-{{- end }}
@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "2.69"
-version: "2.69"
+appVersion: "2.21"
+version: 2.21
@@ -126,26 +126,3 @@ Inject extra environment vars in the format key:value, if populated
 {{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}}
 {{- end -}}
 {{- end -}}
-
-
-{{/* check if any PVC exists */}}
-{{- define "volume.pvc_exists" -}}
-{{- if or (or (eq .Values.volume.data.type "persistentVolumeClaim") (and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "persistentVolumeClaim") -}}
-{{- printf "true" -}}
-{{- else -}}
-{{- printf "false" -}}
-{{- end -}}
-{{- end -}}
-
-{{/* check if any HostPath exists */}}
-{{- define "volume.hostpath_exists" -}}
-{{- if or (or (eq .Values.volume.data.type "hostPath") (and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "hostPath") -}}
-{{- printf "true" -}}
-{{- else -}}
-{{- if or .Values.global.enableSecurity .Values.volume.extraVolumes -}}
-{{- printf "true" -}}
-{{- else -}}
-{{- printf "false" -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}
@@ -15,13 +15,13 @@ spec:
      backoffLimit: 2
      template:
        spec:
-          {{- if .Values.cronjob.nodeSelector }}
+          {{- with .Values.cronjob.nodeSelector }}
          nodeSelector:
-            {{ tpl .Values.cronjob.nodeSelector . | indent 12 | trim }}
+          {{- toYaml . | nindent 12 }}
          {{- end }}
-          {{- if .Values.cronjob.tolerations }}
+          {{- with .Values.cronjob.tolerations }}
          tolerations:
-            {{ tpl .Values.cronjob.tolerations . | nindent 12 | trim }}
+          {{- toYaml . | nindent 12 }}
          {{- end }}
          restartPolicy: OnFailure
          containers:
@@ -36,13 +36,10 @@ spec:
                - |
                  set -ex
                  echo -e "lock\n\
-                  volume.balance -force \
+                  volume.balance -force\
                  {{ if .Values.volume.dataCenter }} -dataCenter {{ .Values.volume.dataCenter }}{{ end }}\
                  {{ if .Values.cronjob.collection }} -collection {{ .Values.cronjob.collection }}{{ end }}\n\
-                  {{- if .Values.cronjob.enableFixReplication }}
-                  volume.fix.replication -collectionPattern={{ .Values.cronjob.collectionPattern }} \n\
-                  {{- end }}
-                  unlock\n" | \
+                  volume.fix.replication\nunlock\n" | \
                  /usr/bin/weed shell \
                  {{- if .Values.cronjob.master }}
                  -master {{ .Values.cronjob.master }} \
@@ -28,4 +28,4 @@ spec:
 {{- end }}
  selector:
    app: {{ template "seaweedfs.name" . }}
-    component: filer
+    component: filer
@@ -133,36 +133,14 @@ spec:
              -encryptVolumeData \
              {{- end }}
              -ip=${POD_IP} \
-              {{- if .Values.filer.enable_peers }}
              {{- if gt (.Values.filer.replicas | int) 1 }}
              -peers=$(echo -n "{{ range $index := until (.Values.filer.replicas | int) }}${SEAWEEDFS_FULLNAME}-filer-{{ $index }}.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}{{ if lt $index (sub ($.Values.filer.replicas | int) 1) }},{{ end }}{{ end }}" | sed "s/$HOSTNAME.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}//" | sed 's/,$//; 's/^,//'; s/,,/,/;' )  \
              {{- end }}
-              {{- end }}
-              {{- if .Values.filer.s3.enabled }}
-              -s3 \
-              -s3.port={{ .Values.filer.s3.port }} \
-              {{- if .Values.filer.s3.domainName }}
-              -s3.domainName={{ .Values.filer.s3.domainName }} \
-              {{- end }}
-              {{- if .Values.global.enableSecurity }}
-              -s3.cert.file=/usr/local/share/ca-certificates/client/tls.crt \
-              -s3.key.file=/usr/local/share/ca-certificates/client/tls.key \
-              {{- end }}
-              {{- if .Values.filer.s3.allowEmptyFolder }}
-              -s3.allowEmptyFolder={{ .Values.filer.s3.allowEmptyFolder }} \
-              {{- end }}
-              {{- if .Values.filer.s3.enableAuth }}
-              -s3.config=/etc/sw/seaweedfs_s3_config \
-              {{- end }}
-              {{- end }}
              -master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
          {{- if or (.Values.global.enableSecurity) (.Values.filer.extraVolumeMounts) }}    
          volumeMounts:
            - name: seaweedfs-filer-log-volume
              mountPath: "/logs/"
-            - mountPath: /etc/sw
-              name: config-users
-              readOnly: true
            {{- if .Values.global.enableSecurity }}
            - name: security-config
              readOnly: true
@@ -220,13 +198,6 @@ spec:
          hostPath:
            path: /storage/logs/seaweedfs/filer
            type: DirectoryOrCreate
-        - name: db-schema-config-volume
-          configMap:
-            name: seaweedfs-db-init-config
-        - name: config-users
-          secret:
-            defaultMode: 420
-            secretName: seaweedfs-s3-secret
        {{- if .Values.global.enableSecurity }}
        - name: security-config
          configMap:
@@ -90,16 +90,10 @@ spec:
              {{- if .Values.s3.allowEmptyFolder }}
              -allowEmptyFolder={{ .Values.s3.allowEmptyFolder }} \
              {{- end }}
-              {{- if .Values.s3.enableAuth }}
-              -config=/etc/sw/seaweedfs_s3_config \
-              {{- end }}
              -filer={{ template "seaweedfs.name" . }}-filer-client:{{ .Values.filer.port }}
          volumeMounts:
            - name: logs
              mountPath: "/logs/"
-            - mountPath: /etc/sw
-              name: config-users
-              readOnly: true
            {{- if .Values.global.enableSecurity }}
            - name: security-config
              readOnly: true
@@ -150,10 +144,6 @@ spec:
            {{ tpl .Values.s3.resources . | nindent 12 | trim }}
          {{- end }}
      volumes:
-        - name: config-users
-          secret:
-            defaultMode: 420
-            secretName: seaweedfs-s3-secret
        {{- if eq .Values.s3.logs.type "hostPath" }}
        - name: logs
          hostPath:
@@ -9,15 +9,15 @@ metadata:
 spec:
  ports:
  - name: "swfs-s3"
-    port: {{ if .Values.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }}
-    targetPort: {{ if .Values.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }}
+    port: {{ .Values.s3.port }}
+    targetPort: {{ .Values.s3.port }}
    protocol: TCP
-{{- if and .Values.s3.enabled .Values.s3.metricsPort }}
-  - name: "metrics"
+{{- if .Values.s3.metricsPort }}
+  - name: "swfs-s3-metrics"
    port: {{ .Values.s3.metricsPort }}
    targetPort: {{ .Values.s3.metricsPort }}
    protocol: TCP
 {{- end }}
  selector:
    app: {{ template "seaweedfs.name" . }}
-    component: {{ if .Values.s3.enabled }}s3{{ else }}filer{{ end }}
+    component: s3
@@ -23,6 +23,6 @@ spec:
    targetPort: {{ .Values.volume.metricsPort }}
    protocol: TCP
 {{- end }}
-  selector:
+selector:
    app: {{ template "seaweedfs.name" . }}
    component: volume
@@ -45,19 +45,6 @@ spec:
      priorityClassName: {{ .Values.volume.priorityClassName | quote }}
      {{- end }}
      enableServiceLinks: false
-      {{- if .Values.volume.dir_idx }}
-      initContainers:
-        - name: seaweedfs-vol-move-idx
-          image: {{ template "volume.image" . }}
-          imagePullPolicy: {{ .Values.global.pullPolicy | default "IfNotPresent" }}
-          command: [ '/bin/sh', '-c' ]
-          args: ['if ls {{ .Values.volume.dir }}/*.idx >/dev/null 2>&1; then mv {{ .Values.volume.dir }}/*.idx {{ .Values.volume.dir_idx }}/; fi;']
-          volumeMounts:
-          - name: idx
-            mountPath: {{ .Values.volume.dir_idx }}
-          - name: data
-            mountPath: {{ .Values.volume.dir }}
-      {{- end }}
      containers:
        - name: seaweedfs
          image: {{ template "volume.image" . }}
@@ -110,7 +97,7 @@ spec:
                -dataCenter={{ .Values.volume.dataCenter }} \
                {{- end }}
                -ip.bind={{ .Values.volume.ipBind }} \
-                -readMode={{ .Values.volume.readMode }} \
+                -read.redirect={{ .Values.volume.readRedirect }} \
                {{- if .Values.volume.whiteList }}
                -whiteList={{ .Values.volume.whiteList }} \
                {{- end }}
@@ -131,13 +118,9 @@ spec:
                -compactionMBps={{ .Values.volume.compactionMBps }} \
                -mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
          volumeMounts:
-            - name: data
-              mountPath: "{{ .Values.volume.dir }}/"
-            {{- if .Values.volume.dir_idx }}
-            - name: idx
-              mountPath: "{{ .Values.volume.dir_idx }}/"
-            {{- end }}
-            - name: logs
+            - name: seaweedfs-volume-storage
+              mountPath: "/data/"
+            - name: seaweedfs-volume-log-volume
              mountPath: "/logs/"
            {{- if .Values.global.enableSecurity }}
            - name: security-config
@@ -190,27 +173,15 @@ spec:
          resources:
            {{ tpl .Values.volume.resources . | nindent 12 | trim }}
          {{- end }}
-      {{- $hostpath_exists := include "volume.hostpath_exists" . -}}
-      {{- if $hostpath_exists }}
      volumes:
-        {{- if eq .Values.volume.data.type "hostPath" }}
-        - name: data
-          hostPath:
-            path: /storage/object_store/
-            type: DirectoryOrCreate
-        {{- end }}
-        {{- if and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx }}
-        - name: idx
-          hostPath:
-            path: /ssd/seaweedfs-volume-idx/
-            type: DirectoryOrCreate
-        {{- end }}
-        {{- if eq .Values.volume.logs.type "hostPath" }}
-        - name: logs
+        - name: seaweedfs-volume-log-volume
          hostPath:
            path: /storage/logs/seaweedfs/volume
            type: DirectoryOrCreate
-        {{- end }}
+        - name: seaweedfs-volume-storage
+          hostPath:
+            path: /storage/object_store/
+            type: DirectoryOrCreate
        {{- if .Values.global.enableSecurity }}
        - name: security-config
          configMap:
@@ -234,43 +205,8 @@ spec:
      {{- if .Values.volume.extraVolumes }}
        {{ tpl .Values.volume.extraVolumes . | indent 8 | trim }}
      {{- end }}
-      {{- end }}
      {{- if .Values.volume.nodeSelector }}
      nodeSelector:
        {{ tpl .Values.volume.nodeSelector . | indent 8 | trim }}
      {{- end }}
-  {{- $pvc_exists := include "volume.pvc_exists" . -}}
-  {{- if $pvc_exists }}
-  volumeClaimTemplates:
-    {{- if eq .Values.volume.data.type "persistentVolumeClaim"}}
-    - metadata:
-        name: data
-      spec:
-        accessModes: [ "ReadWriteOnce" ]
-        storageClassName: {{ .Values.volume.data.storageClass }}
-        resources:
-          requests:
-            storage: {{ .Values.volume.data.size }}
-    {{- end }}
-    {{- if and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx }}
-    - metadata:
-        name: idx
-      spec:
-        accessModes: [ "ReadWriteOnce" ]
-        storageClassName: {{ .Values.volume.idx.storageClass }}
-        resources:
-          requests:
-            storage: {{ .Values.volume.idx.size }}
-    {{- end }}
-    {{- if eq .Values.volume.logs.type "persistentVolumeClaim" }}
-    - metadata:
-        name: logs
-      spec:
-        accessModes: [ "ReadWriteOnce" ]
-        storageClassName: {{ .Values.volume.logs.storageClass }}
-        resources:
-          requests:
-            storage: {{ .Values.volume.logs.size }}
-    {{- end }}
-      {{- end }}
 {{- end }}
@@ -4,6 +4,7 @@ global:
  registry: ""
  repository: ""
  imageName: chrislusf/seaweedfs
+  # imageTag: "2.21" - started using {.Chart.appVersion}
  imagePullPolicy: IfNotPresent
  imagePullSecrets: imagepullsecret
  restartPolicy: Always
@@ -135,26 +136,8 @@ volume:
  # limit file size to avoid out of memory, default 256mb
  fileSizeLimitMB: null
  # minimum free disk space(in percents). If free disk space lower this value - all volumes marks as ReadOnly
-  minFreeSpacePercent: 7
+  minFreeSpacePercent: 1

-# can use ANY storage-class , example with local-path-provisner
-#  data:
-#    type: "persistentVolumeClaim"
-#    size: "24Ti"
-#    storageClass: "local-path-provisioner"
-  data:
-    type: "hostPath"
-    size: ""
-    storageClass: ""
-  idx:
-    type: "hostPath"
-    size: ""
-    storageClass: ""
-
-  logs:
-    type: "hostPath"
-    size: ""
-    storageClass: ""

  # limit background compaction or copying speed in mega bytes per second
  compactionMBps: "50"
@@ -174,8 +157,8 @@ volume:
  # Volume server's data center name
  dataCenter: null

-  # Redirect moved or non-local volumes. (default proxy)
-  readMode: proxy
+  # Redirect moved or non-local volumes. (default true)
+  readRedirect: true

  # Comma separated Ip addresses having write permission. No limit if empty.
  whiteList: null
@@ -246,8 +229,6 @@ filer:
  maxMB: null
  # encrypt data on volume servers
  encryptVolumeData: false
-  # enable peers sync metadata, for leveldb (localdb for filer but with sync across)
-  enable_peers: false

  # Whether proxy or redirect to volume server during file GET request
  redirectOnRead: false
@@ -330,19 +311,8 @@ filer:
    # directories under this folder will be automatically creating a separate bucket
    WEED_FILER_BUCKETS_FOLDER: "/buckets"

-  s3:
-    enabled: true
-    port: 8333
-    #allow empty folders
-    allowEmptyFolder: false
-    # Suffix of the host name, {bucket}.{domainName}
-    domainName: ""
-    # enable user & permission to s3 (need to inject to all services)
-    enableAuth: false
-    skipAuthSecretCreation: false
-
 s3:
-  enabled: false
+  enabled: true
  repository: null
  imageName: null
  imageTag: null
@@ -353,9 +323,6 @@ s3:
  loggingOverrideLevel: null
  #allow empty folders
  allowEmptyFolder: true
-  # enable user & permission to s3 (need to inject to all services)
-  enableAuth: false
-  skipAuthSecretCreation: false

  # Suffix of the host name, {bucket}.{domainName}
  domainName: ""
@@ -392,21 +359,17 @@ s3:
    storageClass: ""

 cronjob:
-  enabled: true
-  master: "seaweedfs-master:9333"
-  filer: "seaweedfs-filer-client:8888"
-  tolerations: ""
-  nodeSelector: |
-    sw-backend: "true"
-  replication:
-    enable: true
-    collectionPattern: ""
+  enabled: false
  schedule: "*/7 * * * *"
  resources: null
  # balance all volumes among volume servers
  # ALL|EACH_COLLECTION|<collection_name>
  collection: ""
-
+  master: ""
+  filer: ""
+  tolerations: ""
+  nodeSelector: |
+    sw-backend: "true"

 certificates:
  commonName: "SeaweedFS CA"
@@ -5,7 +5,7 @@

    <groupId>com.github.chrislusf</groupId>
    <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.8</version>
+    <version>1.5.6</version>

    <parent>
        <groupId>org.sonatype.oss</groupId>
@@ -17,7 +17,7 @@
        <protobuf.version>3.9.1</protobuf.version>
        <!-- follow https://github.com/grpc/grpc-java -->
        <grpc.version>1.23.0</grpc.version>
-        <guava.version>30.0-jre</guava.version>
+        <guava.version>28.0-jre</guava.version>
    </properties>

    <dependencies>
@@ -135,7 +135,7 @@
            <plugin>
                <groupId>org.sonatype.plugins</groupId>
                <artifactId>nexus-staging-maven-plugin</artifactId>
-                <version>1.6.8</version>
+                <version>1.6.7</version>
                <extensions>true</extensions>
                <configuration>
                    <serverId>ossrh</serverId>
@@ -5,7 +5,7 @@

    <groupId>com.github.chrislusf</groupId>
    <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.8</version>
+    <version>1.5.6</version>

    <parent>
        <groupId>org.sonatype.oss</groupId>
@@ -130,7 +130,7 @@
            <plugin>
                <groupId>org.sonatype.plugins</groupId>
                <artifactId>nexus-staging-maven-plugin</artifactId>
-                <version>1.6.8</version>
+                <version>1.6.7</version>
                <extensions>true</extensions>
                <configuration>
                    <serverId>ossrh</serverId>
@@ -5,7 +5,7 @@

    <groupId>com.github.chrislusf</groupId>
    <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.8</version>
+    <version>1.5.6</version>

    <parent>
        <groupId>org.sonatype.oss</groupId>
@@ -3,14 +3,13 @@ package seaweedfs.client;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

-import java.nio.Buffer;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;

 public class ByteBufferPool {

-    private static final int MIN_BUFFER_SIZE = 1 * 1024 * 1024;
+    private static final int MIN_BUFFER_SIZE = 8 * 1024 * 1024;
    private static final Logger LOG = LoggerFactory.getLogger(ByteBufferPool.class);

    private static final List<ByteBuffer> bufferList = new ArrayList<>();
@@ -35,7 +34,7 @@ public class ByteBufferPool {
    }

    public static synchronized void release(ByteBuffer obj) {
-        ((Buffer)obj).clear();
+        obj.clear();
        bufferList.add(0, obj);
    }

@@ -23,7 +23,7 @@ public class FileChunkManifest {
    }

    public static List<FilerProto.FileChunk> resolveChunkManifest(
-            final FilerClient filerClient, List<FilerProto.FileChunk> chunks) throws IOException {
+            final FilerGrpcClient filerGrpcClient, List<FilerProto.FileChunk> chunks) throws IOException {

        List<FilerProto.FileChunk> dataChunks = new ArrayList<>();

@@ -35,30 +35,30 @@ public class FileChunkManifest {

            // IsChunkManifest
            LOG.debug("fetching chunk manifest:{}", chunk);
-            byte[] data = fetchChunk(filerClient, chunk);
+            byte[] data = fetchChunk(filerGrpcClient, chunk);
            FilerProto.FileChunkManifest m = FilerProto.FileChunkManifest.newBuilder().mergeFrom(data).build();
            List<FilerProto.FileChunk> resolvedChunks = new ArrayList<>();
            for (FilerProto.FileChunk t : m.getChunksList()) {
                // avoid deprecated chunk.getFileId()
                resolvedChunks.add(t.toBuilder().setFileId(FilerClient.toFileId(t.getFid())).build());
            }
-            dataChunks.addAll(resolveChunkManifest(filerClient, resolvedChunks));
+            dataChunks.addAll(resolveChunkManifest(filerGrpcClient, resolvedChunks));
        }

        return dataChunks;
    }

-    private static byte[] fetchChunk(final FilerClient filerClient, FilerProto.FileChunk chunk) throws IOException {
+    private static byte[] fetchChunk(final FilerGrpcClient filerGrpcClient, FilerProto.FileChunk chunk) throws IOException {

        String vid = "" + chunk.getFid().getVolumeId();
-        FilerProto.Locations locations = filerClient.vidLocations.get(vid);
+        FilerProto.Locations locations = filerGrpcClient.vidLocations.get(vid);
        if (locations == null) {
            FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder();
            lookupRequest.addVolumeIds(vid);
-            FilerProto.LookupVolumeResponse lookupResponse = filerClient
+            FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient
                    .getBlockingStub().lookupVolume(lookupRequest.build());
            locations = lookupResponse.getLocationsMapMap().get(vid);
-            filerClient.vidLocations.put(vid, locations);
+            filerGrpcClient.vidLocations.put(vid, locations);
            LOG.debug("fetchChunk vid:{} locations:{}", vid, locations);
        }

@@ -74,7 +74,7 @@ public class FileChunkManifest {
        byte[] chunkData = SeaweedRead.chunkCache.getChunk(chunkView.fileId);
        if (chunkData == null) {
            LOG.debug("doFetchFullChunkData:{}", chunkView);
-            chunkData = SeaweedRead.doFetchFullChunkData(filerClient, chunkView, locations);
+            chunkData = SeaweedRead.doFetchFullChunkData(chunkView, locations);
        }
        if (chunk.getIsChunkManifest()){
            LOG.debug("chunk {} size {}", chunkView.fileId, chunkData.length);
@@ -86,7 +86,7 @@ public class FileChunkManifest {
    }

    public static List<FilerProto.FileChunk> maybeManifestize(
-            final FilerClient filerClient, List<FilerProto.FileChunk> inputChunks, String parentDirectory) throws IOException {
+            final FilerGrpcClient filerGrpcClient, List<FilerProto.FileChunk> inputChunks, String parentDirectory) throws IOException {
        // the return variable
        List<FilerProto.FileChunk> chunks = new ArrayList<>();

@@ -101,7 +101,7 @@ public class FileChunkManifest {

        int remaining = dataChunks.size();
        for (int i = 0; i + mergeFactor < dataChunks.size(); i += mergeFactor) {
-            FilerProto.FileChunk chunk = mergeIntoManifest(filerClient, dataChunks.subList(i, i + mergeFactor), parentDirectory);
+            FilerProto.FileChunk chunk = mergeIntoManifest(filerGrpcClient, dataChunks.subList(i, i + mergeFactor), parentDirectory);
            chunks.add(chunk);
            remaining -= mergeFactor;
        }
@@ -113,7 +113,7 @@ public class FileChunkManifest {
        return chunks;
    }

-    private static FilerProto.FileChunk mergeIntoManifest(final FilerClient filerClient, List<FilerProto.FileChunk> dataChunks, String parentDirectory) throws IOException {
+    private static FilerProto.FileChunk mergeIntoManifest(final FilerGrpcClient filerGrpcClient, List<FilerProto.FileChunk> dataChunks, String parentDirectory) throws IOException {
        // create and serialize the manifest
        dataChunks = FilerClient.beforeEntrySerialization(dataChunks);
        FilerProto.FileChunkManifest.Builder m = FilerProto.FileChunkManifest.newBuilder().addAllChunks(dataChunks);
@@ -127,8 +127,8 @@ public class FileChunkManifest {
        }

        FilerProto.FileChunk.Builder manifestChunk = SeaweedWrite.writeChunk(
-                filerClient.getReplication(),
-                filerClient,
+                filerGrpcClient.getReplication(),
+                filerGrpcClient,
                minOffset,
                data, 0, data.length, parentDirectory);
        manifestChunk.setIsChunkManifest(true);
@@ -4,21 +4,25 @@ import com.google.common.base.Strings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;

-public class FilerClient extends FilerGrpcClient {
+public class FilerClient {

    private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class);

+    private final FilerGrpcClient filerGrpcClient;
+
    public FilerClient(String host, int grpcPort) {
-        super(host, grpcPort-10000, grpcPort);
+        filerGrpcClient = new FilerGrpcClient(host, grpcPort);
    }

-    public FilerClient(String host, int port, int grpcPort) {
-        super(host, port, grpcPort);
+    public FilerClient(FilerGrpcClient filerGrpcClient) {
+        this.filerGrpcClient = filerGrpcClient;
    }

    public static String toFileId(FilerProto.FileId fid) {
@@ -62,39 +66,24 @@ public class FilerClient extends FilerGrpcClient {

    public static FilerProto.Entry afterEntryDeserialization(FilerProto.Entry entry) {
        if (entry.getChunksList().size() <= 0) {
-            if (entry.getContent().isEmpty()) {
-                return entry;
-            } else {
-                if (entry.getAttributes().getFileSize() <= 0) {
-                    FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
-                    FilerProto.FuseAttributes.Builder attrBuilder = entry.getAttributes().toBuilder();
-                    attrBuilder.setFileSize(entry.getContent().size());
-                    entryBuilder.setAttributes(attrBuilder);
-                    return entryBuilder.build();
-                }
-            }
            return entry;
-        } else {
-            FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
-            entryBuilder.clearChunks();
-            long fileSize = 0;
-            for (FilerProto.FileChunk chunk : entry.getChunksList()) {
-                fileSize = Math.max(fileSize, chunk.getOffset()+chunk.getSize());
-                FilerProto.FileChunk.Builder chunkBuilder = chunk.toBuilder();
-                chunkBuilder.setFileId(toFileId(chunk.getFid()));
-                String sourceFileId = toFileId(chunk.getSourceFid());
-                if (sourceFileId != null) {
-                    chunkBuilder.setSourceFileId(sourceFileId);
-                }
-                entryBuilder.addChunks(chunkBuilder);
-            }
-            if (entry.getAttributes().getFileSize() <= 0) {
-                FilerProto.FuseAttributes.Builder attrBuilder = entry.getAttributes().toBuilder();
-                attrBuilder.setFileSize(fileSize);
-                entryBuilder.setAttributes(attrBuilder);
-            }
-            return entryBuilder.build();
        }
+        String fileId = entry.getChunks(0).getFileId();
+        if (fileId != null && fileId.length() != 0) {
+            return entry;
+        }
+        FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
+        entryBuilder.clearChunks();
+        for (FilerProto.FileChunk chunk : entry.getChunksList()) {
+            FilerProto.FileChunk.Builder chunkBuilder = chunk.toBuilder();
+            chunkBuilder.setFileId(toFileId(chunk.getFid()));
+            String sourceFileId = toFileId(chunk.getSourceFid());
+            if (sourceFileId != null) {
+                chunkBuilder.setSourceFileId(sourceFileId);
+            }
+            entryBuilder.addChunks(chunkBuilder);
+        }
+        return entryBuilder.build();
    }

    public boolean mkdirs(String path, int mode) {
@@ -111,9 +100,9 @@ public class FilerClient extends FilerGrpcClient {
        if ("/".equals(path)) {
            return true;
        }
-        String[] dirAndName = SeaweedUtil.toDirAndName(path);
-        String parent = dirAndName[0];
-        String name = dirAndName[1];
+        Path pathObject = Paths.get(path);
+        String parent = pathObject.getParent().toString();
+        String name = pathObject.getFileName().toString();

        mkdirs(parent, mode, uid, gid, userName, groupNames);

@@ -132,32 +121,23 @@ public class FilerClient extends FilerGrpcClient {

    public boolean mv(String oldPath, String newPath) {

-        String[] oldDirAndName = SeaweedUtil.toDirAndName(oldPath);
-        String oldParent = oldDirAndName[0];
-        String oldName = oldDirAndName[1];
+        Path oldPathObject = Paths.get(oldPath);
+        String oldParent = oldPathObject.getParent().toString();
+        String oldName = oldPathObject.getFileName().toString();

-        String[] newDirAndName = SeaweedUtil.toDirAndName(newPath);
-        String newParent = newDirAndName[0];
-        String newName = newDirAndName[1];
+        Path newPathObject = Paths.get(newPath);
+        String newParent = newPathObject.getParent().toString();
+        String newName = newPathObject.getFileName().toString();

        return atomicRenameEntry(oldParent, oldName, newParent, newName);

    }

-    public boolean exists(String path){
-
-        String[] dirAndName = SeaweedUtil.toDirAndName(path);
-        String parent = dirAndName[0];
-        String entryName = dirAndName[1];
-
-        return lookupEntry(parent, entryName) != null;
-    }
-
    public boolean rm(String path, boolean isRecursive, boolean ignoreRecusiveError) {

-        String[] dirAndName = SeaweedUtil.toDirAndName(path);
-        String parent = dirAndName[0];
-        String name = dirAndName[1];
+        Path pathObject = Paths.get(path);
+        String parent = pathObject.getParent().toString();
+        String name = pathObject.getFileName().toString();

        return deleteEntry(
                parent,
@@ -168,43 +148,31 @@ public class FilerClient extends FilerGrpcClient {
    }

    public boolean touch(String path, int mode) {
-
        String currentUser = System.getProperty("user.name");
-
-        long now = System.currentTimeMillis() / 1000L;
-        return touch(path, now, mode, 0, 0, currentUser, new String[]{});
-
+        return touch(path, mode, 0, 0, currentUser, new String[]{});
    }

-    public boolean touch(String path, long modifiedTimeSecond, int mode, int uid, int gid, String userName, String[] groupNames) {
+    public boolean touch(String path, int mode, int uid, int gid, String userName, String[] groupNames) {

-        String[] dirAndName = SeaweedUtil.toDirAndName(path);
-        String parent = dirAndName[0];
-        String name = dirAndName[1];
+        Path pathObject = Paths.get(path);
+        String parent = pathObject.getParent().toString();
+        String name = pathObject.getFileName().toString();

        FilerProto.Entry entry = lookupEntry(parent, name);
        if (entry == null) {
            return createEntry(
                    parent,
-                    newFileEntry(name, modifiedTimeSecond, mode, uid, gid, userName, groupNames).build()
+                    newFileEntry(name, mode, uid, gid, userName, groupNames).build()
            );
        }
-        FilerProto.FuseAttributes.Builder attr = entry.getAttributes().toBuilder();
-        if (modifiedTimeSecond>0) {
-            attr.setMtime(modifiedTimeSecond);
-        }
-        if (uid>0) {
-            attr.setUid(uid);
-        }
-        if (gid>0) {
-            attr.setGid(gid);
-        }
-        if (userName!=null) {
-            attr.setUserName(userName);
-        }
-        if (groupNames!=null) {
-            attr.clearGroupName().addAllGroupName(Arrays.asList(groupNames));
-        }
+        long now = System.currentTimeMillis() / 1000L;
+        FilerProto.FuseAttributes.Builder attr = entry.getAttributes().toBuilder()
+                .setMtime(now)
+                .setUid(uid)
+                .setGid(gid)
+                .setUserName(userName)
+                .clearGroupName()
+                .addAllGroupName(Arrays.asList(groupNames));
        return updateEntry(parent, entry.toBuilder().setAttributes(attr).build());
    }

@@ -227,15 +195,17 @@ public class FilerClient extends FilerGrpcClient {
                        .addAllGroupName(Arrays.asList(groupNames)));
    }

-    public FilerProto.Entry.Builder newFileEntry(String name, long modifiedTimeSecond, int mode,
+    public FilerProto.Entry.Builder newFileEntry(String name, int mode,
                                                 int uid, int gid, String userName, String[] groupNames) {

+        long now = System.currentTimeMillis() / 1000L;
+
        return FilerProto.Entry.newBuilder()
                .setName(name)
                .setIsDirectory(false)
                .setAttributes(FilerProto.FuseAttributes.newBuilder()
-                        .setMtime(modifiedTimeSecond)
-                        .setCrtime(modifiedTimeSecond)
+                        .setMtime(now)
+                        .setCrtime(now)
                        .setUid(uid)
                        .setGid(gid)
                        .setFileMode(mode)
@@ -266,7 +236,7 @@ public class FilerClient extends FilerGrpcClient {
    }

    public List<FilerProto.Entry> listEntries(String path, String entryPrefix, String lastEntryName, int limit, boolean includeLastEntry) {
-        Iterator<FilerProto.ListEntriesResponse> iter = this.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
+        Iterator<FilerProto.ListEntriesResponse> iter = filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
                .setDirectory(path)
                .setPrefix(entryPrefix)
                .setStartFromFileName(lastEntryName)
@@ -283,7 +253,7 @@ public class FilerClient extends FilerGrpcClient {

    public FilerProto.Entry lookupEntry(String directory, String entryName) {
        try {
-            FilerProto.Entry entry = this.getBlockingStub().lookupDirectoryEntry(
+            FilerProto.Entry entry = filerGrpcClient.getBlockingStub().lookupDirectoryEntry(
                    FilerProto.LookupDirectoryEntryRequest.newBuilder()
                            .setDirectory(directory)
                            .setName(entryName)
@@ -304,7 +274,7 @@ public class FilerClient extends FilerGrpcClient {
    public boolean createEntry(String parent, FilerProto.Entry entry) {
        try {
            FilerProto.CreateEntryResponse createEntryResponse =
-                    this.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder()
+                    filerGrpcClient.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder()
                            .setDirectory(parent)
                            .setEntry(entry)
                            .build());
@@ -321,7 +291,7 @@ public class FilerClient extends FilerGrpcClient {

    public boolean updateEntry(String parent, FilerProto.Entry entry) {
        try {
-            this.getBlockingStub().updateEntry(FilerProto.UpdateEntryRequest.newBuilder()
+            filerGrpcClient.getBlockingStub().updateEntry(FilerProto.UpdateEntryRequest.newBuilder()
                    .setDirectory(parent)
                    .setEntry(entry)
                    .build());
@@ -334,7 +304,7 @@ public class FilerClient extends FilerGrpcClient {

    public boolean deleteEntry(String parent, String entryName, boolean isDeleteFileChunk, boolean isRecursive, boolean ignoreRecusiveError) {
        try {
-            this.getBlockingStub().deleteEntry(FilerProto.DeleteEntryRequest.newBuilder()
+            filerGrpcClient.getBlockingStub().deleteEntry(FilerProto.DeleteEntryRequest.newBuilder()
                    .setDirectory(parent)
                    .setName(entryName)
                    .setIsDeleteData(isDeleteFileChunk)
@@ -350,7 +320,7 @@ public class FilerClient extends FilerGrpcClient {

    public boolean atomicRenameEntry(String oldParent, String oldName, String newParent, String newName) {
        try {
-            this.getBlockingStub().atomicRenameEntry(FilerProto.AtomicRenameEntryRequest.newBuilder()
+            filerGrpcClient.getBlockingStub().atomicRenameEntry(FilerProto.AtomicRenameEntryRequest.newBuilder()
                    .setOldDirectory(oldParent)
                    .setOldName(oldName)
                    .setNewDirectory(newParent)
@@ -364,7 +334,7 @@ public class FilerClient extends FilerGrpcClient {
    }

    public Iterator<FilerProto.SubscribeMetadataResponse> watch(String prefix, String clientName, long sinceNs) {
-        return this.getBlockingStub().subscribeMetadata(FilerProto.SubscribeMetadataRequest.newBuilder()
+        return filerGrpcClient.getBlockingStub().subscribeMetadata(FilerProto.SubscribeMetadataRequest.newBuilder()
                .setPathPrefix(prefix)
                .setClientName(clientName)
                .setSinceNs(sinceNs)
@@ -9,8 +9,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

 import javax.net.ssl.SSLException;
-import java.util.HashMap;
 import java.util.Map;
+import java.util.HashMap;
 import java.util.concurrent.TimeUnit;

 public class FilerGrpcClient {
@@ -26,9 +26,6 @@ public class FilerGrpcClient {
        }
    }

-    public final int VOLUME_SERVER_ACCESS_DIRECT = 0;
-    public final int VOLUME_SERVER_ACCESS_PUBLIC_URL = 1;
-    public final int VOLUME_SERVER_ACCESS_FILER_PROXY = 2;
    public final Map<String, FilerProto.Locations> vidLocations = new HashMap<>();
    private final ManagedChannel channel;
    private final SeaweedFilerGrpc.SeaweedFilerBlockingStub blockingStub;
@@ -37,14 +34,12 @@ public class FilerGrpcClient {
    private boolean cipher = false;
    private String collection = "";
    private String replication = "";
-    private int volumeServerAccess = VOLUME_SERVER_ACCESS_DIRECT;
-    private String filerAddress;

-    public FilerGrpcClient(String host, int port, int grpcPort) {
-        this(host, port, grpcPort, sslContext);
+    public FilerGrpcClient(String host, int grpcPort) {
+        this(host, grpcPort, sslContext);
    }

-    public FilerGrpcClient(String host, int port, int grpcPort, SslContext sslContext) {
+    public FilerGrpcClient(String host, int grpcPort, SslContext sslContext) {

        this(sslContext == null ?
                ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext()
@@ -54,8 +49,6 @@ public class FilerGrpcClient {
                        .negotiationType(NegotiationType.TLS)
                        .sslContext(sslContext));

-        filerAddress = SeaweedUtil.joinHostPort(host, port);
-
        FilerProto.GetFilerConfigurationResponse filerConfigurationResponse =
                this.getBlockingStub().getFilerConfiguration(
                        FilerProto.GetFilerConfigurationRequest.newBuilder().build());
@@ -65,7 +58,7 @@ public class FilerGrpcClient {

    }

-    private FilerGrpcClient(ManagedChannelBuilder<?> channelBuilder) {
+    public FilerGrpcClient(ManagedChannelBuilder<?> channelBuilder) {
        channel = channelBuilder.build();
        blockingStub = SeaweedFilerGrpc.newBlockingStub(channel);
        asyncStub = SeaweedFilerGrpc.newStub(channel);
@@ -100,39 +93,4 @@ public class FilerGrpcClient {
        return futureStub;
    }

-    public void setAccessVolumeServerDirectly() {
-        this.volumeServerAccess = VOLUME_SERVER_ACCESS_DIRECT;
-    }
-
-    public boolean isAccessVolumeServerDirectly() {
-        return this.volumeServerAccess == VOLUME_SERVER_ACCESS_DIRECT;
-    }
-
-    public void setAccessVolumeServerByPublicUrl() {
-        this.volumeServerAccess = VOLUME_SERVER_ACCESS_PUBLIC_URL;
-    }
-
-    public boolean isAccessVolumeServerByPublicUrl() {
-        return this.volumeServerAccess == VOLUME_SERVER_ACCESS_PUBLIC_URL;
-    }
-
-    public void setAccessVolumeServerByFilerProxy() {
-        this.volumeServerAccess = VOLUME_SERVER_ACCESS_FILER_PROXY;
-    }
-
-    public boolean isAccessVolumeServerByFilerProxy() {
-        return this.volumeServerAccess == VOLUME_SERVER_ACCESS_FILER_PROXY;
-    }
-
-    public String getChunkUrl(String chunkId, String url, String publicUrl) {
-        switch (this.volumeServerAccess) {
-            case VOLUME_SERVER_ACCESS_PUBLIC_URL:
-                return String.format("http://%s/%s", publicUrl, chunkId);
-            case VOLUME_SERVER_ACCESS_FILER_PROXY:
-                return String.format("http://%s/?proxyChunkId=%s", this.filerAddress, chunkId);
-            default:
-                return String.format("http://%s/%s", url, chunkId);
-        }
-    }
-
 }
@@ -1,23 +0,0 @@
-package seaweedfs.client;
-
-import java.io.IOException;
-
-public class RemoteUtil {
-    public static boolean isInRemoteOnly(FilerProto.Entry entry) {
-        if (entry.getChunksList() == null || entry.getChunksList().isEmpty()) {
-            return entry.getRemoteEntry() != null && entry.getRemoteEntry().getRemoteSize() > 0;
-        }
-        return false;
-    }
-
-    public static FilerProto.Entry downloadRemoteEntry(FilerClient filerClient, String fullpath, FilerProto.Entry entry) throws IOException {
-        String dir = SeaweedOutputStream.getParentDirectory(fullpath);
-        String name = SeaweedOutputStream.getFileName(fullpath);
-
-        final FilerProto.DownloadToLocalResponse downloadToLocalResponse = filerClient.getBlockingStub()
-                .downloadToLocal(FilerProto.DownloadToLocalRequest.newBuilder()
-                .setDirectory(dir).setName(name).build());
-
-        return downloadToLocalResponse.getEntry();
-    }
-}
@@ -23,7 +23,7 @@ public class SeaweedRead {
    static VolumeIdCache volumeIdCache = new VolumeIdCache(4 * 1024);

    // returns bytesRead
-    public static long read(FilerClient filerClient, List<VisibleInterval> visibleIntervals,
+    public static long read(FilerGrpcClient filerGrpcClient, List<VisibleInterval> visibleIntervals,
                            final long position, final ByteBuffer buf, final long fileSize) throws IOException {

        List<ChunkView> chunkViews = viewFromVisibles(visibleIntervals, position, buf.remaining());
@@ -42,7 +42,7 @@ public class SeaweedRead {
        }

        if (lookupRequest.getVolumeIdsCount() > 0) {
-            FilerProto.LookupVolumeResponse lookupResponse = filerClient
+            FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient
                    .getBlockingStub().lookupVolume(lookupRequest.build());
            Map<String, FilerProto.Locations> vid2Locations = lookupResponse.getLocationsMapMap();
            for (Map.Entry<String, FilerProto.Locations> entry : vid2Locations.entrySet()) {
@@ -64,15 +64,14 @@ public class SeaweedRead {
                startOffset += gap;
            }

-            String volumeId = parseVolumeId(chunkView.fileId);
-            FilerProto.Locations locations = knownLocations.get(volumeId);
+            FilerProto.Locations locations = knownLocations.get(parseVolumeId(chunkView.fileId));
            if (locations == null || locations.getLocationsCount() == 0) {
                LOG.error("failed to locate {}", chunkView.fileId);
-                volumeIdCache.clearLocations(volumeId);
+                // log here!
                return 0;
            }

-            int len = readChunkView(filerClient, startOffset, buf, chunkView, locations);
+            int len = readChunkView(startOffset, buf, chunkView, locations);

            LOG.debug("read [{},{}) {} size {}", startOffset, startOffset + len, chunkView.fileId, chunkView.size);

@@ -94,12 +93,12 @@ public class SeaweedRead {
        return readCount;
    }

-    private static int readChunkView(FilerClient filerClient, long startOffset, ByteBuffer buf, ChunkView chunkView, FilerProto.Locations locations) throws IOException {
+    private static int readChunkView(long startOffset, ByteBuffer buf, ChunkView chunkView, FilerProto.Locations locations) throws IOException {

        byte[] chunkData = chunkCache.getChunk(chunkView.fileId);

        if (chunkData == null) {
-            chunkData = doFetchFullChunkData(filerClient, chunkView, locations);
+            chunkData = doFetchFullChunkData(chunkView, locations);
            chunkCache.setChunk(chunkView.fileId, chunkData);
        }

@@ -111,13 +110,13 @@ public class SeaweedRead {
        return len;
    }

-    public static byte[] doFetchFullChunkData(FilerClient filerClient, ChunkView chunkView, FilerProto.Locations locations) throws IOException {
+    public static byte[] doFetchFullChunkData(ChunkView chunkView, FilerProto.Locations locations) throws IOException {

        byte[] data = null;
        IOException lastException = null;
        for (long waitTime = 1000L; waitTime < 10 * 1000; waitTime += waitTime / 2) {
            for (FilerProto.Location location : locations.getLocationsList()) {
-                String url = filerClient.getChunkUrl(chunkView.fileId, location.getUrl(), location.getPublicUrl());
+                String url = String.format("http://%s/%s", location.getUrl(), chunkView.fileId);
                try {
                    data = doFetchOneFullChunkData(chunkView, url);
                    lastException = null;
@@ -146,7 +145,7 @@ public class SeaweedRead {

    }

-    private static byte[] doFetchOneFullChunkData(ChunkView chunkView, String url) throws IOException {
+    public static byte[] doFetchOneFullChunkData(ChunkView chunkView, String url) throws IOException {

        HttpGet request = new HttpGet(url);

@@ -222,9 +221,9 @@ public class SeaweedRead {
    }

    public static List<VisibleInterval> nonOverlappingVisibleIntervals(
-            final FilerClient filerClient, List<FilerProto.FileChunk> chunkList) throws IOException {
+            final FilerGrpcClient filerGrpcClient, List<FilerProto.FileChunk> chunkList) throws IOException {

-        chunkList = FileChunkManifest.resolveChunkManifest(filerClient, chunkList);
+        chunkList = FileChunkManifest.resolveChunkManifest(filerGrpcClient, chunkList);

        FilerProto.FileChunk[] chunks = chunkList.toArray(new FilerProto.FileChunk[0]);
        Arrays.sort(chunks, new Comparator<FilerProto.FileChunk>() {
@@ -27,30 +27,4 @@ public class SeaweedUtil {
    public static CloseableHttpClient getClosableHttpClient() {
        return httpClient;
    }
-
-    public static String[] toDirAndName(String fullpath) {
-        if (fullpath == null) {
-            return new String[]{"/", ""};
-        }
-        if (fullpath.endsWith("/")) {
-            fullpath = fullpath.substring(0, fullpath.length() - 1);
-        }
-        if (fullpath.length() == 0) {
-            return new String[]{"/", ""};
-        }
-        int sep = fullpath.lastIndexOf("/");
-        String parent = sep == 0 ? "/" : fullpath.substring(0, sep);
-        String name = fullpath.substring(sep + 1);
-        return new String[]{parent, name};
-    }
-
-    public static String joinHostPort(String host, int port) {
-        if (host.startsWith("[") && host.endsWith("]")) {
-            return host + ":" + port;
-        }
-        if (host.indexOf(':')>=0) {
-            return "[" + host + "]:" + port;
-        }
-        return host + ":" + port;
-    }
 }
@@ -1,6 +1,5 @@
 package seaweedfs.client;

-import com.google.common.base.Strings;
 import com.google.protobuf.ByteString;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPost;
@@ -24,46 +23,41 @@ public class SeaweedWrite {

    public static void writeData(FilerProto.Entry.Builder entry,
                                 final String replication,
-                                 final FilerClient filerClient,
+                                 final FilerGrpcClient filerGrpcClient,
                                 final long offset,
                                 final byte[] bytes,
                                 final long bytesOffset, final long bytesLength,
                                 final String path) throws IOException {
        FilerProto.FileChunk.Builder chunkBuilder = writeChunk(
-                replication, filerClient, offset, bytes, bytesOffset, bytesLength, path);
+                replication, filerGrpcClient, offset, bytes, bytesOffset, bytesLength, path);
        synchronized (entry) {
            entry.addChunks(chunkBuilder);
        }
    }

    public static FilerProto.FileChunk.Builder writeChunk(final String replication,
-                                                          final FilerClient filerClient,
+                                                          final FilerGrpcClient filerGrpcClient,
                                                          final long offset,
                                                          final byte[] bytes,
                                                          final long bytesOffset,
                                                          final long bytesLength,
                                                          final String path) throws IOException {
-        FilerProto.AssignVolumeResponse response = filerClient.getBlockingStub().assignVolume(
+        FilerProto.AssignVolumeResponse response = filerGrpcClient.getBlockingStub().assignVolume(
                FilerProto.AssignVolumeRequest.newBuilder()
-                        .setCollection(filerClient.getCollection())
-                        .setReplication(Strings.isNullOrEmpty(replication) ? filerClient.getReplication() : replication)
+                        .setCollection(filerGrpcClient.getCollection())
+                        .setReplication(replication == null ? filerGrpcClient.getReplication() : replication)
                        .setDataCenter("")
                        .setTtlSec(0)
                        .setPath(path)
                        .build());
-
-        if (!Strings.isNullOrEmpty(response.getError())) {
-            throw new IOException(response.getError());
-        }
-
        String fileId = response.getFileId();
+        String url = response.getUrl();
        String auth = response.getAuth();
-
-        String targetUrl = filerClient.getChunkUrl(fileId, response.getLocation().getUrl(), response.getLocation().getPublicUrl());
+        String targetUrl = String.format("http://%s/%s", url, fileId);

        ByteString cipherKeyString = com.google.protobuf.ByteString.EMPTY;
        byte[] cipherKey = null;
-        if (filerClient.isCipher()) {
+        if (filerGrpcClient.isCipher()) {
            cipherKey = genCipherKey();
            cipherKeyString = ByteString.copyFrom(cipherKey);
        }
@@ -81,15 +75,15 @@ public class SeaweedWrite {
                .setCipherKey(cipherKeyString);
    }

-    public static void writeMeta(final FilerClient filerClient,
+    public static void writeMeta(final FilerGrpcClient filerGrpcClient,
                                 final String parentDirectory,
                                 final FilerProto.Entry.Builder entry) throws IOException {

        synchronized (entry) {
-            List<FilerProto.FileChunk> chunks = FileChunkManifest.maybeManifestize(filerClient, entry.getChunksList(), parentDirectory);
+            List<FilerProto.FileChunk> chunks = FileChunkManifest.maybeManifestize(filerGrpcClient, entry.getChunksList(), parentDirectory);
            entry.clearChunks();
            entry.addAllChunks(chunks);
-            filerClient.getBlockingStub().createEntry(
+            filerGrpcClient.getBlockingStub().createEntry(
                    FilerProto.CreateEntryRequest.newBuilder()
                            .setDirectory(parentDirectory)
                            .setEntry(entry)
@@ -26,13 +26,6 @@ public class VolumeIdCache {
        return this.cache.getIfPresent(volumeId);
    }

-    public void clearLocations(String volumeId) {
-        if (this.cache == null) {
-            return;
-        }
-        this.cache.invalidate(volumeId);
-    }
-
    public void setLocations(String volumeId, FilerProto.Locations locations) {
        if (this.cache == null) {
            return;
@@ -67,8 +67,6 @@ service SeaweedFiler {
    rpc KvPut (KvPutRequest) returns (KvPutResponse) {
    }

-    rpc DownloadToLocal (DownloadToLocalRequest) returns (DownloadToLocalResponse) {
-    }
 }

 //////////////////////////////////////////////////
@@ -94,13 +92,6 @@ message ListEntriesResponse {
    Entry entry = 1;
 }

-message RemoteEntry {
-    string storage_name = 1;
-    int64 last_local_sync_ts_ns = 2;
-    string remote_e_tag = 3;
-    int64 remote_mtime = 4;
-    int64 remote_size = 5;
-}
 message Entry {
    string name = 1;
    bool is_directory = 2;
@@ -110,8 +101,6 @@ message Entry {
    bytes hard_link_id = 7;
    int32 hard_link_counter = 8; // only exists in hard link meta data
    bytes content = 9; // if not empty, the file content
-
-    RemoteEntry remote_entry = 10;
 }

 message FullEntry {
@@ -167,7 +156,6 @@ message FuseAttributes {
    repeated string group_name = 12; // for hdfs
    string symlink_target = 13;
    bytes md5 = 14;
-    string disk_type = 15;
 }

 message CreateEntryRequest {
@@ -219,7 +207,6 @@ message AtomicRenameEntryRequest {
    string old_name = 2;
    string new_directory = 3;
    string new_name = 4;
-    repeated int32 signatures = 5;
 }

 message AtomicRenameEntryResponse {
@@ -233,17 +220,17 @@ message AssignVolumeRequest {
    string data_center = 5;
    string path = 6;
    string rack = 7;
-    string disk_type = 8;
 }

 message AssignVolumeResponse {
    string file_id = 1;
+    string url = 2;
+    string public_url = 3;
    int32 count = 4;
    string auth = 5;
    string collection = 6;
    string replication = 7;
    string error = 8;
-    Location location = 9;
 }

 message LookupVolumeRequest {
@@ -257,7 +244,6 @@ message Locations {
 message Location {
    string url = 1;
    string public_url = 2;
-    uint32 grpc_port = 3;
 }
 message LookupVolumeResponse {
    map<string, Locations> locations_map = 1;
@@ -284,9 +270,11 @@ message StatisticsRequest {
    string replication = 1;
    string collection = 2;
    string ttl = 3;
-    string disk_type = 4;
 }
 message StatisticsResponse {
+    string replication = 1;
+    string collection = 2;
+    string ttl = 3;
    uint64 total_size = 4;
    uint64 used_size = 5;
    uint64 file_count = 6;
@@ -304,8 +292,6 @@ message GetFilerConfigurationResponse {
    int32 signature = 8;
    string metrics_address = 9;
    int32 metrics_interval_sec = 10;
-    string version = 11;
-    string cluster_id = 12;
 }

 message SubscribeMetadataRequest {
@@ -313,7 +299,6 @@ message SubscribeMetadataRequest {
    string path_prefix = 2;
    int64 since_ns = 3;
    int32 signature = 4;
-    repeated string path_prefixes = 6;
 }
 message SubscribeMetadataResponse {
    string directory = 1;
@@ -338,7 +323,6 @@ message KeepConnectedResponse {
 message LocateBrokerRequest {
    string resource = 1;
 }
-
 message LocateBrokerResponse {
    bool found = 1;
    // if found, send the exact address
@@ -350,9 +334,7 @@ message LocateBrokerResponse {
    repeated Resource resources = 2;
 }

-/////////////////////////
 // Key-Value operations
-/////////////////////////
 message KvGetRequest {
    bytes key = 1;
 }
@@ -368,9 +350,7 @@ message KvPutResponse {
    string error = 1;
 }

-/////////////////////////
 // path-based configurations
-/////////////////////////
 message FilerConf {
    int32 version = 1;
    message PathConf {
@@ -378,21 +358,14 @@ message FilerConf {
        string collection = 2;
        string replication = 3;
        string ttl = 4;
-        string disk_type = 5;
+        enum DiskType {
+            NONE = 0;
+            HDD = 1;
+            SSD = 2;
+        }
+        DiskType disk_type = 5;
        bool fsync = 6;
        uint32 volume_growth_count = 7;
-        bool read_only = 8;
    }
    repeated PathConf locations = 2;
 }
-
-/////////////////////////
-// Remote Storage related
-/////////////////////////
-message DownloadToLocalRequest {
-    string directory = 1;
-    string name = 2;
-}
-message DownloadToLocalResponse {
-    Entry entry = 1;
-}
@@ -16,17 +16,8 @@ public class SeaweedFilerTest {
        filerClient.mkdirs("/new_folder", 0755);
        filerClient.touch("/new_folder/new_empty_file", 0755);
        filerClient.touch("/new_folder/new_empty_file2", 0755);
-        if(!filerClient.exists("/new_folder/new_empty_file")){
-            System.out.println("/new_folder/new_empty_file should exists");
-        }
-
        filerClient.rm("/new_folder/new_empty_file", false, true);
        filerClient.rm("/new_folder", true, true);
-        if(filerClient.exists("/new_folder/new_empty_file")){
-            System.out.println("/new_folder/new_empty_file should not exists");
-        }
-        if(!filerClient.exists("/")){
-            System.out.println("/ should exists");
-        }
+
    }
 }
@@ -11,13 +11,13 @@
        <dependency>
            <groupId>com.github.chrislusf</groupId>
            <artifactId>seaweedfs-client</artifactId>
-            <version>1.6.8</version>
+            <version>1.5.6</version>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>com.github.chrislusf</groupId>
            <artifactId>seaweedfs-hadoop2-client</artifactId>
-            <version>1.6.8</version>
+            <version>1.5.6</version>
            <scope>compile</scope>
        </dependency>
        <dependency>
@@ -1,47 +0,0 @@
-package com.seaweedfs.examples;
-
-import seaweedfs.client.FilerClient;
-import seaweedfs.client.SeaweedInputStream;
-import seaweedfs.client.SeaweedOutputStream;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
-
-public class ExampleWriteFile {
-
-    public static void main(String[] args) throws IOException {
-
-        FilerClient filerClient = new FilerClient("localhost", 18888);
-
-        SeaweedInputStream seaweedInputStream = new SeaweedInputStream(filerClient, "/test.zip");
-        unZipFiles(filerClient, seaweedInputStream);
-
-    }
-
-    public static void unZipFiles(FilerClient filerClient, InputStream is) throws IOException {
-        ZipInputStream zin = new ZipInputStream(is);
-        ZipEntry ze;
-        while ((ze = zin.getNextEntry()) != null) {
-
-            String filename = ze.getName();
-            if (filename.indexOf("/") >= 0) {
-                filename = filename.substring(filename.lastIndexOf("/") + 1);
-            }
-            if (filename.length()==0) {
-                continue;
-            }
-
-            SeaweedOutputStream seaweedOutputStream = new SeaweedOutputStream(filerClient, "/test/"+filename);
-            byte[] bytesIn = new byte[16 * 1024];
-            int read = 0;
-            while ((read = zin.read(bytesIn))!=-1) {
-                seaweedOutputStream.write(bytesIn,0,read);
-            }
-            seaweedOutputStream.close();
-
-            System.out.println(ze.getName());
-        }
-    }
-}
@@ -1,22 +0,0 @@
-package com.seaweedfs.examples;
-
-import com.google.common.io.Files;
-import seaweedfs.client.FilerClient;
-import seaweedfs.client.SeaweedOutputStream;
-
-import java.io.File;
-import java.io.IOException;
-
-public class ExampleWriteFile2 {
-
-    public static void main(String[] args) throws IOException {
-
-        FilerClient filerClient = new FilerClient("localhost", 18888);
-
-        SeaweedOutputStream seaweedOutputStream = new SeaweedOutputStream(filerClient, "/test/1");
-        Files.copy(new File("/etc/resolv.conf"), seaweedOutputStream);
-        seaweedOutputStream.close();
-
-    }
-
-}
@@ -1,25 +0,0 @@
-package com.seaweedfs.examples;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-
-import java.io.*;
-
-public class HdfsCopyFile {
-    public static void main(String[] args) throws IOException {
-        Configuration configuration = new Configuration();
-
-        configuration.set("fs.defaultFS", "seaweedfs://localhost:8888");
-        configuration.set("fs.seaweedfs.impl", "seaweed.hdfs.SeaweedFileSystem");
-
-        FileSystem fs = FileSystem.get(configuration);
-        String source = "/Users/chris/tmp/test.zip";
-        String destination = "/buckets/spark/test01.zip";
-        InputStream in = new BufferedInputStream(new FileInputStream(source));
-
-        OutputStream out = fs.create(new Path(destination));
-        IOUtils.copyBytes(in, out, 4096, true);
-    }
-}
@@ -1,7 +1,8 @@
 package com.seaweedfs.examples;

+import seaweed.hdfs.SeaweedInputStream;
 import seaweedfs.client.FilerClient;
-import seaweedfs.client.SeaweedInputStream;
+import seaweedfs.client.FilerGrpcClient;

 import java.io.FileInputStream;
 import java.io.IOException;
@@ -9,11 +10,12 @@ import java.io.InputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;

-public class ExampleReadFile {
+public class UnzipFile {

    public static void main(String[] args) throws IOException {

-        FilerClient filerClient = new FilerClient("localhost", 18888);
+        FilerGrpcClient filerGrpcClient = new FilerGrpcClient("localhost", 18888);
+        FilerClient filerClient = new FilerClient(filerGrpcClient);

        long startTime = System.currentTimeMillis();
        parseZip("/Users/chris/tmp/test.zip");
@@ -23,7 +25,11 @@ public class ExampleReadFile {
        long localProcessTime = startTime2 - startTime;

        SeaweedInputStream seaweedInputStream = new SeaweedInputStream(
-                filerClient, "/test.zip");
+                filerGrpcClient,
+                new org.apache.hadoop.fs.FileSystem.Statistics(""),
+                "/",
+                filerClient.lookupEntry("/", "test.zip")
+        );
        parseZip(seaweedInputStream);

        long swProcessTime = System.currentTimeMillis() - startTime2;
@@ -7,7 +7,7 @@ import java.io.IOException;
 import java.util.Date;
 import java.util.Iterator;

-public class ExampleWatchFileChanges {
+public class WatchFiles {

    public static void main(String[] args) throws IOException {

--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Chris Lu	a9e6db1a8e	Merge branch 'master' into ftp	2021-01-21 00:47:27 -08:00
Chris Lu	f0455dee68	WIP	2020-12-26 23:46:20 -08:00