2.66

fix https://github.com/chrislusf/seaweedfs/issues/2311

.github/workflows/binary_test.yml

@@ -0,0 +1,50 @@
+name: "go: test building cross-platform binary"
+
+on:
+  pull_request:
+  workflow_dispatch: []
+
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [linux, windows, darwin, freebsd, netbsd, openbsd]
+        goarch: [amd64, arm, arm64, 386]
+        exclude:
+          - goarch: arm
+            goos: darwin
+          - goarch: 386
+            goos: darwin
+          - goarch: arm
+            goos: windows
+          - goarch: arm64
+            goos: windows
+
+    concurrency:
+      group: ${{ github.head_ref }}/binary_test/${{ matrix.goos }}/${{ matrix.goarch }}
+      cancel-in-progress: true
+
+    steps:
+
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.13
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Get dependencies
+        run: |
+          cd weed; go get -v -t -d ./...
+          if [ -f Gopkg.toml ]; then
+              curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+              dep ensure
+          fi
+
+      - name: Build
+        run: cd weed; GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -v .

.github/workflows/cleanup.yml

@@ -1,4 +1,4 @@
-name: Cleanup
+name: "chore: cleanup"
 
 on:
   push:

.github/workflows/container_latest.yml

@@ -1,4 +1,5 @@
-name: Build Latest Containers
+name: "docker: build latest containers"
+
 on:
   push:
     branches:
@@ -8,6 +9,7 @@ on:
 jobs:
   build-latest:
     runs-on: [ubuntu-latest]
+
     steps:
       -
         name: Checkout
@@ -15,7 +17,7 @@ jobs:
       -
         name: Docker meta
         id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v2
+        uses: docker/metadata-action@v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -55,12 +57,13 @@ jobs:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}
           file: ./docker/Dockerfile
-          platforms: linux/amd64
+          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}
 
   build-dev:
     runs-on: [ubuntu-latest]
+
     steps:
       -
         name: Checkout
@@ -68,7 +71,7 @@ jobs:
       -
         name: Docker meta
         id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v2
+        uses: docker/metadata-action@v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -109,6 +112,6 @@ jobs:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}
           file: ./docker/Dockerfile.go_build
-          platforms: linux/amd64
+          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/container_release.yml

@@ -1,4 +1,5 @@
-name: Build Release Containers
+name: "docker: build release containers"
+
 on:
   push:
     tags:
@@ -8,6 +9,7 @@ on:
 jobs:
   build-default:
     runs-on: [ubuntu-latest]
+
     steps:
       -
         name: Checkout
@@ -15,7 +17,7 @@ jobs:
       -
         name: Docker meta
         id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v2
+        uses: docker/metadata-action@v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -58,11 +60,12 @@ jobs:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}
           file: ./docker/Dockerfile.go_build
-          platforms: linux/amd64
+          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}
   build-large:
     runs-on: [ubuntu-latest]
+
     steps:
       -
         name: Checkout
@@ -70,7 +73,7 @@ jobs:
       -
         name: Docker meta
         id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v2
+        uses: docker/metadata-action@v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -113,6 +116,6 @@ jobs:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}
           file: ./docker/Dockerfile.go_build_large
-          platforms: linux/amd64
+          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/container_test.yml

@@ -0,0 +1,53 @@
+name: "docker: test building container images"
+
+on:
+  pull_request:
+  workflow_dispatch: []
+
+concurrency:
+  group: ${{ github.head_ref }}/container_test
+  cancel-in-progress: true
+
+jobs:
+  build-test:
+    runs-on: [ubuntu-latest]
+    strategy:
+      matrix:
+        platform: [ linux ]
+        arch: [ amd64, arm, arm64, 386 ]
+
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            chrislusf/seaweedfs
+            ghcr.io/chrislusf/seaweedfs
+          tags: |
+            type=raw,value=latest
+          labels: |
+            org.opencontainers.image.title=seaweedfs
+            org.opencontainers.image.vendor=Chris Lu
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          buildkitd-flags: "--debug"
+      -
+        name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./docker
+          push: false
+          file: ./docker/Dockerfile
+          platforms: ${{ matrix.platform }}/${{ matrix.arch }}
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/go.yml

@@ -1,4 +1,4 @@
-name: Go
+name: "go: test building binary"
 
 on:
   push:
@@ -6,6 +6,10 @@ on:
   pull_request:
     branches: [ master ]
 
+concurrency:
+  group: ${{ github.head_ref }}/go
+  cancel-in-progress: true
+
 jobs:
 
   build:

.github/workflows/release.yml

@@ -1,4 +1,4 @@
-name: Build Dev Binaries
+name: "go: build dev binaries"
 
 on:
   push:
@@ -11,13 +11,17 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        goos: [linux, windows, darwin, freebsd ]
-        goarch: [amd64, arm]
+        goos: [linux, windows, darwin, freebsd, netbsd, openbsd]
+        goarch: [amd64, arm, arm64, 386]
         exclude:
           - goarch: arm
             goos: darwin
+          - goarch: 386
+            goos: darwin
           - goarch: arm
             goos: windows
+          - goarch: arm64
+            goos: windows
 
     steps:
 
@@ -33,9 +37,8 @@ jobs:
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries
-        uses: wangyoucao577/go-release-action@v1.17
+        uses: wangyoucao577/go-release-action@v1.20
         with:
-          goversion: 1.17
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
           goarch: ${{ matrix.goarch }}
@@ -50,9 +53,8 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries
-        uses: wangyoucao577/go-release-action@v1.17
+        uses: wangyoucao577/go-release-action@v1.20
         with:
-          goversion: 1.17
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
           goarch: ${{ matrix.goarch }}

.github/workflows/release_binaries.yml

@@ -1,6 +1,6 @@
 # This is a basic workflow to help you get started with Actions
 
-name: Build Versioned Releases
+name: "go: build versioned binaries"
 
 on:
   release:
@@ -15,22 +15,25 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        goos: [linux, windows, darwin, freebsd ]
-        goarch: [amd64, arm]
+        goos: [linux, windows, darwin, freebsd, netbsd, openbsd]
+        goarch: [amd64, arm, arm64, 386]
         exclude:
           - goarch: arm
             goos: darwin
+          - goarch: 386
+            goos: darwin
           - goarch: arm
             goos: windows
+          - goarch: arm64
+            goos: windows
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v2
       - name: Go Release Binaries
-        uses: wangyoucao577/go-release-action@v1.19
+        uses: wangyoucao577/go-release-action@v1.20
         with:
-          goversion: 1.17
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
           goarch: ${{ matrix.goarch }}
@@ -43,9 +46,8 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@v1.19
+        uses: wangyoucao577/go-release-action@v1.20
         with:
-          goversion: 1.17
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
           goarch: ${{ matrix.goarch }}

.travis.yml.disabled

@@ -1,46 +0,0 @@
-sudo: false
-language: go
-go:
-  - 1.17.x
-
-before_install:
-  - export PATH=/home/travis/gopath/bin:$PATH
-
-install:
-  - export CGO_ENABLED="0"
-  - go env
-
-script:
-  - env GO111MODULE=on go test ./weed/...
-
-before_deploy:
-  - make release
-deploy:
-  provider: releases
-  skip_cleanup: true
-  api_key:
-    secure: ERL986+ncQ8lwAJUYDrQ8s2/FxF/cyNIwJIFCqspnWxQgGNNyokET9HapmlPSxjpFRF0q6L2WCg9OY3mSVRq4oI6hg1igOQ12KlLyN71XSJ3c8w0Ay5ho48TQ9l3f3Iu97mntBCe9l0R9pnT8wj1VI8YJxloXwUMG2yeTjA9aBI=
-  file:
-    - build/linux_arm.tar.gz
-    - build/linux_arm64.tar.gz
-    - build/linux_386.tar.gz
-    - build/linux_amd64.tar.gz
-    - build/linux_amd64_large_disk.tar.gz
-    - build/darwin_amd64.tar.gz
-    - build/darwin_amd64_large_disk.tar.gz
-    - build/windows_386.zip
-    - build/windows_amd64.zip
-    - build/windows_amd64_large_disk.zip
-    - build/freebsd_arm.tar.gz
-    - build/freebsd_amd64.tar.gz
-    - build/freebsd_386.tar.gz
-    - build/netbsd_arm.tar.gz
-    - build/netbsd_amd64.tar.gz
-    - build/netbsd_386.tar.gz
-    - build/openbsd_arm.tar.gz
-    - build/openbsd_amd64.tar.gz
-    - build/openbsd_386.tar.gz
-  on:
-    tags: true
-    repo: chrislusf/seaweedfs
-    go: 1.17.x

Makefile

@@ -1,147 +0,0 @@
-BINARY = weed/weed
-package = github.com/chrislusf/seaweedfs/weed
-
-GO_FLAGS = #-v
-SOURCE_DIR = ./weed/
-
-appname := weed
-
-sources := $(wildcard *.go)
-
-COMMIT ?= $(shell git rev-parse --short HEAD)
-LDFLAGS ?= -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${COMMIT}
-
-build = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -ldflags "-extldflags -static $(LDFLAGS)" -o build/$(appname)$(3) $(SOURCE_DIR)
-tar = cd build && tar -cvzf $(1)_$(2).tar.gz $(appname)$(3) && rm $(appname)$(3)
-zip = cd build && zip $(1)_$(2).zip $(appname)$(3) && rm $(appname)$(3)
-
-build_large = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -tags 5BytesOffset -ldflags "-extldflags -static $(LDFLAGS)" -o build/$(appname)$(3) $(SOURCE_DIR)
-tar_large = cd build && tar -cvzf $(1)_$(2)_large_disk.tar.gz $(appname)$(3) && rm $(appname)$(3)
-zip_large = cd build && zip $(1)_$(2)_large_disk.zip $(appname)$(3) && rm $(appname)$(3)
-
-all: build
-
-.PHONY : clean deps build linux release windows_build darwin_build linux_build bsd_build clean
-
-clean:
-	go clean -i $(GO_FLAGS) $(SOURCE_DIR)
-	rm -f $(BINARY)
-	rm -rf build/
-
-deps:
-	go get $(GO_FLAGS) -d $(SOURCE_DIR)
-	rm -rf /home/travis/gopath/src/github.com/coreos/etcd/vendor/golang.org/x/net/trace
-	rm -rf /home/travis/gopath/src/go.etcd.io/etcd/vendor/golang.org/x/net/trace
-
-build: deps
-	go build $(GO_FLAGS) -ldflags "$(LDFLAGS)" -o $(BINARY) $(SOURCE_DIR)
-
-gccgo_build: deps
-	go build $(GO_FLAGS) -ldflags "$(LDFLAGS)" -compiler=gccgo -tags gccgo,noasm -o $(BINARY) $(SOURCE_DIR)
-
-install: deps
-	go install $(GO_FLAGS) -ldflags "$(LDFLAGS)" $(SOURCE_DIR)
-
-linux: deps
-	mkdir -p linux
-	GOOS=linux GOARCH=amd64 go build $(GO_FLAGS) -ldflags "$(LDFLAGS)" -o linux/$(BINARY) $(SOURCE_DIR)
-
-release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_build 5_byte_arm64_build 5_byte_darwin_build 5_byte_windows_build
-
-##### LINUX BUILDS #####
-5_byte_linux_build:
-	$(call build_large,linux,amd64,)
-	$(call tar_large,linux,amd64)
-
-5_byte_darwin_build:
-	$(call build_large,darwin,amd64,)
-	$(call tar_large,darwin,amd64)
-
-5_byte_windows_build:
-	$(call build_large,windows,amd64,.exe)
-	$(call zip_large,windows,amd64,.exe)
-
-5_byte_arm_build: $(sources)
-	$(call build_large,linux,arm,)
-	$(call tar_large,linux,arm)
-
-5_byte_arm64_build: $(sources)
-	$(call build_large,linux,arm64,)
-	$(call tar_large,linux,arm64)
-
-linux_build: build/linux_arm.tar.gz build/linux_arm64.tar.gz build/linux_386.tar.gz build/linux_amd64.tar.gz
-
-build/linux_386.tar.gz: $(sources)
-	$(call build,linux,386,)
-	$(call tar,linux,386)
-
-build/linux_amd64.tar.gz: $(sources)
-	$(call build,linux,amd64,)
-	$(call tar,linux,amd64)
-
-build/linux_arm.tar.gz: $(sources)
-	$(call build,linux,arm,)
-	$(call tar,linux,arm)
-
-build/linux_arm64.tar.gz: $(sources)
-	$(call build,linux,arm64,)
-	$(call tar,linux,arm64)
-
-##### DARWIN (MAC) BUILDS #####
-darwin_build: build/darwin_amd64.tar.gz
-
-build/darwin_amd64.tar.gz: $(sources)
-	$(call build,darwin,amd64,)
-	$(call tar,darwin,amd64)
-
-##### WINDOWS BUILDS #####
-windows_build: build/windows_386.zip build/windows_amd64.zip
-
-build/windows_386.zip: $(sources)
-	$(call build,windows,386,.exe)
-	$(call zip,windows,386,.exe)
-
-build/windows_amd64.zip: $(sources)
-	$(call build,windows,amd64,.exe)
-	$(call zip,windows,amd64,.exe)
-
-##### BSD BUILDS #####
-bsd_build: build/freebsd_arm.tar.gz build/freebsd_386.tar.gz build/freebsd_amd64.tar.gz \
- build/netbsd_arm.tar.gz build/netbsd_386.tar.gz build/netbsd_amd64.tar.gz \
- build/openbsd_arm.tar.gz build/openbsd_386.tar.gz build/openbsd_amd64.tar.gz
-
-build/freebsd_386.tar.gz: $(sources)
-	$(call build,freebsd,386,)
-	$(call tar,freebsd,386)
-
-build/freebsd_amd64.tar.gz: $(sources)
-	$(call build,freebsd,amd64,)
-	$(call tar,freebsd,amd64)
-
-build/freebsd_arm.tar.gz: $(sources)
-	$(call build,freebsd,arm,)
-	$(call tar,freebsd,arm)
-
-build/netbsd_386.tar.gz: $(sources)
-	$(call build,netbsd,386,)
-	$(call tar,netbsd,386)
-
-build/netbsd_amd64.tar.gz: $(sources)
-	$(call build,netbsd,amd64,)
-	$(call tar,netbsd,amd64)
-
-build/netbsd_arm.tar.gz: $(sources)
-	$(call build,netbsd,arm,)
-	$(call tar,netbsd,arm)
-
-build/openbsd_386.tar.gz: $(sources)
-	$(call build,openbsd,386,)
-	$(call tar,openbsd,386)
-
-build/openbsd_amd64.tar.gz: $(sources)
-	$(call build,openbsd,amd64,)
-	$(call tar,openbsd,amd64)
-
-build/openbsd_arm.tar.gz: $(sources)
-	$(call build,openbsd,arm,)
-	$(call tar,openbsd,arm)

README.md

@@ -146,7 +146,7 @@ Faster and Cheaper than direct cloud storage!
 * [WebDAV] accesses as a mapped drive on Mac and Windows, or from mobile devices.
 * [AES256-GCM Encrypted Storage][FilerDataEncryption] safely stores the encrypted data.
 * [Super Large Files][SuperLargeFiles] stores large or super large files in tens of TB.
-* [Cloud Drive][CloudDrive] mount cloud data to local cluster for fast read and write with asynchronous write back.
+* [Cloud Drive][CloudDrive] mounts cloud storage to local cluster, cached for fast read and write with asynchronous write back.
 
 ## Kubernetes ##
 * [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs-csi-driver.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)

docker/Dockerfile

@@ -8,7 +8,9 @@ RUN \
          elif [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "32" ]; then echo "386"; \
          elif [ $(uname -m) == "aarch64" ]; then echo "arm64"; \
          elif [ $(uname -m) == "armv7l" ]; then echo "arm"; \
-         elif [ $(uname -m) == "armv6l" ]; then echo "arm"; fi;) && \
+         elif [ $(uname -m) == "armv6l" ]; then echo "arm"; \
+         elif [ $(uname -m) == "s390x" ]; then echo "s390x"; \
+         elif [ $(uname -m) == "ppc64le" ]; then echo "ppc64le"; fi;) && \
     echo "Building for $ARCH" 1>&2 && \
     SUPERCRONIC_SHA1SUM=$(echo $ARCH | sed 's/386/e0126b0102b9f388ecd55714358e3ad60d0cebdb/g' | sed 's/amd64/5ddf8ea26b56d4a7ff6faecdd8966610d5cb9d85/g' | sed 's/arm64/e2714c43e7781bf1579c85aa61259245f56dbba1/g' | sed 's/arm/47481c3341bc3a1ae91a728e0cc63c8e6d3791ad/g') && \
     SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.1.9/supercronic-linux-$ARCH && \

go.mod

@@ -3,24 +3,25 @@ module github.com/chrislusf/seaweedfs
 go 1.17
 
 require (
-	cloud.google.com/go v0.58.0 // indirect
+	cloud.google.com/go v0.94.1 // indirect
 	cloud.google.com/go/pubsub v1.3.1
-	cloud.google.com/go/storage v1.9.0
-	github.com/Azure/azure-pipeline-go v0.2.2 // indirect
-	github.com/Azure/azure-storage-blob-go v0.9.0
+	cloud.google.com/go/storage v1.16.1
+	github.com/Azure/azure-pipeline-go v0.2.3
+	github.com/Azure/azure-storage-blob-go v0.14.0
 	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect
 	github.com/OneOfOne/xxhash v1.2.2
 	github.com/Shopify/sarama v1.23.1
-	github.com/aws/aws-sdk-go v1.34.30
+	github.com/aws/aws-sdk-go v1.35.3
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/cespare/xxhash v1.1.0
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/chrislusf/raft v1.0.7
+	github.com/colinmarc/hdfs/v2 v2.2.0
 	github.com/coreos/go-semver v0.3.0 // indirect
-	github.com/coreos/go-systemd/v22 v22.0.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/disintegration/imaging v1.6.2
@@ -42,31 +43,29 @@ require (
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/go-zookeeper/zk v1.0.2 // indirect
 	github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
-	github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 // indirect
 	github.com/golang-jwt/jwt v3.2.1+incompatible
-	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e
-	github.com/golang/protobuf v1.4.3
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
+	github.com/golang/protobuf v1.5.2
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.0.0
-	github.com/google/go-cmp v0.5.5 // indirect
-	github.com/google/uuid v1.1.1
+	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/google/uuid v1.3.0
 	github.com/google/wire v0.4.0 // indirect
 	github.com/googleapis/gax-go v2.0.2+incompatible // indirect
-	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
+	github.com/googleapis/gax-go/v2 v2.1.0 // indirect
 	github.com/gorilla/mux v1.7.4
 	github.com/gorilla/websocket v1.4.1 // indirect
-	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
-	github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-multierror v1.0.0 // indirect
-	github.com/hashicorp/go-uuid v1.0.1 // indirect
+	github.com/hashicorp/go-uuid v1.0.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/jcmturner/gofork v1.0.0 // indirect
+	github.com/jcmturner/gokrb5/v8 v8.4.1
 	github.com/jinzhu/copier v0.2.8
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.11
-	github.com/jstemmer/go-junit-report v0.9.1 // indirect
 	github.com/karlseguin/ccache/v2 v2.0.7
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/klauspost/compress v1.10.9 // indirect
@@ -79,7 +78,6 @@ require (
 	github.com/mailru/easyjson v0.7.1 // indirect
 	github.com/mattn/go-ieproxy v0.0.1 // indirect
 	github.com/mattn/go-isatty v0.0.12 // indirect
-	github.com/mattn/go-runewidth v0.0.4 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/mitchellh/mapstructure v1.1.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -105,66 +103,55 @@ require (
 	github.com/seaweedfs/fuse v1.2.0
 	github.com/seaweedfs/goexif v1.0.2
 	github.com/sirupsen/logrus v1.6.0 // indirect
-	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/afero v1.6.0 // indirect
 	github.com/spf13/cast v1.3.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/pflag v1.0.3 // indirect
 	github.com/spf13/viper v1.4.0
 	github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71
-	github.com/stretchr/testify v1.6.1
-	github.com/syndtr/goleveldb v1.0.0
+	github.com/stretchr/testify v1.7.0
+	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
 	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
 	github.com/tidwall/gjson v1.8.1
 	github.com/tidwall/match v1.0.3
 	github.com/tidwall/pretty v1.1.0 // indirect
+	github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210824090536-16d902a3c7e5
 	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
+	github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43
 	github.com/valyala/bytebufferpool v1.0.0
 	github.com/viant/assertly v0.5.4 // indirect
 	github.com/viant/ptrie v0.3.0
 	github.com/viant/toolbox v0.33.2 // indirect
-	github.com/willf/bitset v1.1.10 // indirect
-	github.com/willf/bloom v2.0.3+incompatible
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.0.2 // indirect
 	github.com/xdg-go/stringprep v1.0.2 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	go.etcd.io/etcd v3.3.15+incompatible
+	go.etcd.io/etcd v3.3.25+incompatible
 	go.mongodb.org/mongo-driver v1.7.0
-	go.opencensus.io v0.22.4 // indirect
+	go.opencensus.io v0.23.0 // indirect
 	go.opentelemetry.io/otel v0.15.0 // indirect
-	go.uber.org/atomic v1.6.0 // indirect
-	go.uber.org/multierr v1.5.0 // indirect
-	go.uber.org/zap v1.14.1 // indirect
 	gocloud.dev v0.20.0
 	gocloud.dev/pubsub/natspubsub v0.20.0
 	gocloud.dev/pubsub/rabbitpubsub v0.20.0
-	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect
+	golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 // indirect
 	golang.org/x/image v0.0.0-20200119044424-58c23975cae1
-	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
-	golang.org/x/mod v0.3.0 // indirect
 	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
-	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
-	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a // indirect
-	golang.org/x/sys v0.0.0-20210817142637-7d9622a276b7
+	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
+	golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf
 	golang.org/x/text v0.3.6 // indirect
-	golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78
+	golang.org/x/tools v0.1.5
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/api v0.26.0
-	google.golang.org/appengine v1.6.6 // indirect
-	google.golang.org/genproto v0.0.0-20200608115520-7c474a2e3482 // indirect
-	google.golang.org/grpc v1.29.1
-	google.golang.org/protobuf v1.26.0-rc.1
+	google.golang.org/api v0.56.0
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83 // indirect
+	google.golang.org/grpc v1.40.0
+	google.golang.org/protobuf v1.27.1
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect
 	gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect
 	gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect
 	gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect
 	gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect
-	gopkg.in/yaml.v2 v2.3.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
-	honnef.co/go/tools v0.0.1-2020.1.4 // indirect
 	modernc.org/b v1.0.0 // indirect
 	modernc.org/cc/v3 v3.33.5 // indirect
 	modernc.org/ccgo/v3 v3.9.4 // indirect
@@ -177,7 +164,39 @@ require (
 	modernc.org/token v1.0.0 // indirect
 )
 
+require (
+	github.com/coreos/etcd v3.3.10+incompatible // indirect
+	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
+	github.com/d4l3k/messagediff v1.2.1 // indirect
+	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
+	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
+	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
+	github.com/jcmturner/rpc/v2 v2.0.2 // indirect
+	github.com/mattn/go-runewidth v0.0.7 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3 // indirect
+	github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd // indirect
+	github.com/pingcap/kvproto v0.0.0-20210806074406-317f69fb54b4 // indirect
+	github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4 // indirect
+	github.com/pingcap/parser v0.0.0-20210525032559-c37778aff307 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d // indirect
+	github.com/twmb/murmur3 v1.1.3 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.0 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect
+	go.etcd.io/etcd/client/v3 v3.5.0 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.7.0 // indirect
+	go.uber.org/zap v1.17.0 // indirect
+	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
+	golang.org/x/mod v0.4.2 // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+)
+
 // replace github.com/seaweedfs/fuse => /Users/chris/go/src/github.com/seaweedfs/fuse
 // replace github.com/chrislusf/raft => /Users/chris/go/src/github.com/chrislusf/raft
-
-replace go.etcd.io/etcd => go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547

k8s/helm_charts2/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "2.64"
-version: "2.64"
+appVersion: "2.66"
+version: "2.66"

other/java/client/src/main/java/seaweedfs/client/FilerClient.java

@@ -59,24 +59,39 @@ public class FilerClient extends FilerGrpcClient {
 
     public static FilerProto.Entry afterEntryDeserialization(FilerProto.Entry entry) {
         if (entry.getChunksList().size() <= 0) {
-            return entry;
-        }
-        String fileId = entry.getChunks(0).getFileId();
-        if (fileId != null && fileId.length() != 0) {
-            return entry;
-        }
-        FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
-        entryBuilder.clearChunks();
-        for (FilerProto.FileChunk chunk : entry.getChunksList()) {
-            FilerProto.FileChunk.Builder chunkBuilder = chunk.toBuilder();
-            chunkBuilder.setFileId(toFileId(chunk.getFid()));
-            String sourceFileId = toFileId(chunk.getSourceFid());
-            if (sourceFileId != null) {
-                chunkBuilder.setSourceFileId(sourceFileId);
+            if (entry.getContent().isEmpty()) {
+                return entry;
+            } else {
+                if (entry.getAttributes().getFileSize() <= 0) {
+                    FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
+                    FilerProto.FuseAttributes.Builder attrBuilder = entry.getAttributes().toBuilder();
+                    attrBuilder.setFileSize(entry.getContent().size());
+                    entryBuilder.setAttributes(attrBuilder);
+                    return entryBuilder.build();
+                }
             }
-            entryBuilder.addChunks(chunkBuilder);
+            return entry;
+        } else {
+            FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
+            entryBuilder.clearChunks();
+            long fileSize = 0;
+            for (FilerProto.FileChunk chunk : entry.getChunksList()) {
+                fileSize = Math.max(fileSize, chunk.getOffset()+chunk.getSize());
+                FilerProto.FileChunk.Builder chunkBuilder = chunk.toBuilder();
+                chunkBuilder.setFileId(toFileId(chunk.getFid()));
+                String sourceFileId = toFileId(chunk.getSourceFid());
+                if (sourceFileId != null) {
+                    chunkBuilder.setSourceFileId(sourceFileId);
+                }
+                entryBuilder.addChunks(chunkBuilder);
+            }
+            if (entry.getAttributes().getFileSize() <= 0) {
+                FilerProto.FuseAttributes.Builder attrBuilder = entry.getAttributes().toBuilder();
+                attrBuilder.setFileSize(fileSize);
+                entryBuilder.setAttributes(attrBuilder);
+            }
+            return entryBuilder.build();
         }
-        return entryBuilder.build();
     }
 
     public boolean mkdirs(String path, int mode) {

other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java

@@ -64,10 +64,11 @@ public class SeaweedRead {
                 startOffset += gap;
             }
 
-            FilerProto.Locations locations = knownLocations.get(parseVolumeId(chunkView.fileId));
+            String volumeId = parseVolumeId(chunkView.fileId);
+            FilerProto.Locations locations = knownLocations.get(volumeId);
             if (locations == null || locations.getLocationsCount() == 0) {
                 LOG.error("failed to locate {}", chunkView.fileId);
-                // log here!
+                volumeIdCache.clearLocations(volumeId);
                 return 0;
             }
 

other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java

@@ -26,6 +26,13 @@ public class VolumeIdCache {
         return this.cache.getIfPresent(volumeId);
     }
 
+    public void clearLocations(String volumeId) {
+        if (this.cache == null) {
+            return;
+        }
+        this.cache.invalidate(volumeId);
+    }
+
     public void setLocations(String volumeId, FilerProto.Locations locations) {
         if (this.cache == null) {
             return;

other/java/client/src/main/proto/filer.proto

@@ -305,6 +305,7 @@ message GetFilerConfigurationResponse {
     string metrics_address = 9;
     int32 metrics_interval_sec = 10;
     string version = 11;
+    string cluster_id = 12;
 }
 
 message SubscribeMetadataRequest {
@@ -312,6 +313,7 @@ message SubscribeMetadataRequest {
     string path_prefix = 2;
     int64 since_ns = 3;
     int32 signature = 4;
+    repeated string path_prefixes = 6;
 }
 message SubscribeMetadataResponse {
     string directory = 1;
@@ -336,6 +338,7 @@ message KeepConnectedResponse {
 message LocateBrokerRequest {
     string resource = 1;
 }
+
 message LocateBrokerResponse {
     bool found = 1;
     // if found, send the exact address
@@ -386,24 +389,6 @@ message FilerConf {
 /////////////////////////
 // Remote Storage related
 /////////////////////////
-message RemoteConf {
-    string type = 1;
-    string name = 2;
-    string s3_access_key = 4;
-    string s3_secret_key = 5;
-    string s3_region = 6;
-    string s3_endpoint = 7;
-    string gcs_google_application_credentials = 10;
-}
-
-message RemoteStorageMapping {
-    map<string,RemoteStorageLocation> mappings = 1;
-}
-message RemoteStorageLocation {
-    string name = 1;
-    string bucket = 2;
-    string path = 3;
-}
 message DownloadToLocalRequest {
     string directory = 1;
     string name = 2;

unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go

@@ -72,8 +72,7 @@ func startGenerateMetadata() {
 
 func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) {
 
-	tailErr := pb.FollowMetadata(*tailFiler, grpc.WithInsecure(),	"tail",
-		*dir, 0, 0, eachEntryFunc, false)
+	tailErr := pb.FollowMetadata(*tailFiler, grpc.WithInsecure(), "tail", *dir, nil, 0, 0, eachEntryFunc, false)
 
 	if tailErr != nil {
 		fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)

weed/Makefile

@@ -38,10 +38,14 @@ debug_filer_copy:
 	go build -gcflags="all=-N -l"
 	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 filer.backup -filer=localhost:8888 -filerProxy -timeAgo=10h
 
-debug_filer_remote_sync:
+debug_filer_remote_sync_dir:
 	go build -gcflags="all=-N -l"
 	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 filer.remote.sync -filer="localhost:8888" -dir=/buckets/b2 -timeAgo=1h
 
+debug_filer_remote_sync_buckets:
+	go build -gcflags="all=-N -l"
+	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 filer.remote.sync -filer="localhost:8888" -createBucketAt=cloud1 -timeAgo=1h
+
 debug_master_follower:
 	go build -gcflags="all=-N -l"
 	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 master.follower

weed/command/filer_backup.go

@@ -113,6 +113,6 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
 	})
 
 	return pb.FollowMetadata(sourceFiler, grpcDialOption, "backup_"+dataSink.GetName(),
-		sourcePath, startFrom.UnixNano(), 0, processEventFnWithOffset, false)
+		sourcePath, nil, startFrom.UnixNano(), 0, processEventFnWithOffset, false)
 
 }

weed/command/filer_cat.go

@@ -108,6 +108,11 @@ func runFilerCat(cmd *Command, args []string) bool {
 			return err
 		}
 
+		if len(respLookupEntry.Entry.Content) > 0 {
+			_, err = writer.Write(respLookupEntry.Entry.Content)
+			return err
+		}
+
 		filerCat.filerClient = client
 
 		return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)

weed/command/filer_meta_backup.go

@@ -196,7 +196,7 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
 	})
 
 	return pb.FollowMetadata(*metaBackup.filerAddress, metaBackup.grpcDialOption, "meta_backup",
-		*metaBackup.filerDirectory, startTime.UnixNano(), 0, processEventFnWithOffset, false)
+		*metaBackup.filerDirectory, nil, startTime.UnixNano(), 0, processEventFnWithOffset, false)
 
 }
 

weed/command/filer_meta_tail.go

@@ -104,7 +104,7 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
 	}
 
 	tailErr := pb.FollowMetadata(*tailFiler, grpcDialOption, "tail",
-		*tailTarget, time.Now().Add(-*tailStart).UnixNano(), 0,
+		*tailTarget, nil, time.Now().Add(-*tailStart).UnixNano(), 0,
 		func(resp *filer_pb.SubscribeMetadataResponse) error {
 			if !shouldPrint(resp) {
 				return nil

weed/command/filer_remote_sync.go

@@ -3,11 +3,10 @@ package command
 import (
 	"context"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/remote_storage"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
 	"github.com/chrislusf/seaweedfs/weed/replication/source"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/util"
@@ -16,12 +15,18 @@ import (
 )
 
 type RemoteSyncOptions struct {
-	filerAddress       *string
-	grpcDialOption     grpc.DialOption
-	readChunkFromFiler *bool
-	debug              *bool
-	timeAgo            *time.Duration
-	dir                *string
+	filerAddress             *string
+	grpcDialOption           grpc.DialOption
+	readChunkFromFiler       *bool
+	debug                    *bool
+	timeAgo                  *time.Duration
+	dir                      *string
+	createBucketAt           *string
+	createBucketRandomSuffix *bool
+
+	mappings    *remote_pb.RemoteStorageMapping
+	remoteConfs map[string]*remote_pb.RemoteConf
+	bucketsDir  string
 }
 
 const (
@@ -47,19 +52,29 @@ func init() {
 	cmdFilerRemoteSynchronize.Run = runFilerRemoteSynchronize // break init cycle
 	remoteSyncOptions.filerAddress = cmdFilerRemoteSynchronize.Flag.String("filer", "localhost:8888", "filer of the SeaweedFS cluster")
 	remoteSyncOptions.dir = cmdFilerRemoteSynchronize.Flag.String("dir", "/", "a mounted directory on filer")
+	remoteSyncOptions.createBucketAt = cmdFilerRemoteSynchronize.Flag.String("createBucketAt", "", "one remote storage name to create new buckets in")
+	remoteSyncOptions.createBucketRandomSuffix = cmdFilerRemoteSynchronize.Flag.Bool("createBucketWithRandomSuffix", false, "add randomized suffix to bucket name to avoid conflicts")
 	remoteSyncOptions.readChunkFromFiler = cmdFilerRemoteSynchronize.Flag.Bool("filerProxy", false, "read file chunks from filer instead of volume servers")
 	remoteSyncOptions.debug = cmdFilerRemoteSynchronize.Flag.Bool("debug", false, "debug mode to print out filer updated remote files")
 	remoteSyncOptions.timeAgo = cmdFilerRemoteSynchronize.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
 }
 
 var cmdFilerRemoteSynchronize = &Command{
-	UsageLine: "filer.remote.sync -filer=<filerHost>:<filerPort> -dir=/mount/s3_on_cloud",
-	Short:     "resumable continuously write back updates to remote storage if the directory is mounted to the remote storage",
-	Long: `resumable continuously write back updates to remote storage if the directory is mounted to the remote storage
+	UsageLine: "filer.remote.sync -dir=/mount/s3_on_cloud or -createBucketAt=clound1",
+	Short:     "resumable continuously write back updates to remote storage",
+	Long: `resumable continuously write back updates to remote storage
 
 	filer.remote.sync listens on filer update events. 
 	If any mounted remote file is updated, it will fetch the updated content,
 	and write to the remote storage.
+
+	There are two modes:
+	1)Write back one mounted folder to remote storage
+		weed filer.remote.sync -dir=/mount/s3_on_cloud
+	2)Watch /buckets folder and write back all changes.
+	  Any new buckets will be created in this remote storage.
+		weed filer.remote.sync -createBucketAt=cloud1
+
 `,
 }
 
@@ -72,13 +87,6 @@ func runFilerRemoteSynchronize(cmd *Command, args []string) bool {
 	dir := *remoteSyncOptions.dir
 	filerAddress := *remoteSyncOptions.filerAddress
 
-	// read filer remote storage mount mappings
-	_, _, remoteStorageMountLocation, storageConf, detectErr := filer.DetectMountInfo(grpcDialOption, filerAddress, dir)
-	if detectErr != nil {
-		fmt.Printf("read mount info: %v", detectErr)
-		return false
-	}
-
 	filerSource := &source.FilerSource{}
 	filerSource.DoInitialize(
 		filerAddress,
@@ -87,152 +95,42 @@ func runFilerRemoteSynchronize(cmd *Command, args []string) bool {
 		*remoteSyncOptions.readChunkFromFiler,
 	)
 
-	fmt.Printf("synchronize %s to remote storage...\n", dir)
-	util.RetryForever("filer.remote.sync "+dir, func() error {
-		return followUpdatesAndUploadToRemote(&remoteSyncOptions, filerSource, dir, storageConf, remoteStorageMountLocation)
-	}, func(err error) bool {
-		if err != nil {
-			glog.Errorf("synchronize %s: %v", dir, err)
-		}
-		return true
-	})
+	storageName := *remoteSyncOptions.createBucketAt
+	if storageName != "" {
+
+		remoteSyncOptions.bucketsDir = "/buckets"
+		// check buckets again
+		remoteSyncOptions.WithFilerClient(func(filerClient filer_pb.SeaweedFilerClient) error {
+			resp, err := filerClient.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+			if err != nil {
+				return err
+			}
+			remoteSyncOptions.bucketsDir = resp.DirBuckets
+			return nil
+		})
+
+		fmt.Printf("synchronize %s, default new bucket creation in %s ...\n", remoteSyncOptions.bucketsDir, storageName)
+		util.RetryForever("filer.remote.sync buckets "+storageName, func() error {
+			return remoteSyncOptions.followBucketUpdatesAndUploadToRemote(filerSource)
+		}, func(err error) bool {
+			if err != nil {
+				glog.Errorf("synchronize %s to %s: %v", remoteSyncOptions.bucketsDir, storageName, err)
+			}
+			return true
+		})
+	}
+
+	if dir != "" {
+		fmt.Printf("synchronize %s to remote storage...\n", dir)
+		util.RetryForever("filer.remote.sync "+dir, func() error {
+			return followUpdatesAndUploadToRemote(&remoteSyncOptions, filerSource, dir)
+		}, func(err error) bool {
+			if err != nil {
+				glog.Errorf("synchronize %s: %v", dir, err)
+			}
+			return true
+		})
+	}
 
 	return true
 }
-
-func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *source.FilerSource, mountedDir string, remoteStorage *filer_pb.RemoteConf, remoteStorageMountLocation *filer_pb.RemoteStorageLocation) error {
-
-	dirHash := util.HashStringToLong(mountedDir)
-
-	// 1. specified by timeAgo
-	// 2. last offset timestamp for this directory
-	// 3. directory creation time
-	var lastOffsetTs time.Time
-	if *option.timeAgo == 0 {
-		mountedDirEntry, err := filer_pb.GetEntry(option, util.FullPath(mountedDir))
-		if err != nil {
-			return fmt.Errorf("lookup %s: %v", mountedDir, err)
-		}
-
-		lastOffsetTsNs, err := getOffset(option.grpcDialOption, *option.filerAddress, RemoteSyncKeyPrefix, int32(dirHash))
-		if err == nil && mountedDirEntry.Attributes.Crtime < lastOffsetTsNs/1000000 {
-			lastOffsetTs = time.Unix(0, lastOffsetTsNs)
-			glog.V(0).Infof("resume from %v", lastOffsetTs)
-		} else {
-			lastOffsetTs = time.Unix(mountedDirEntry.Attributes.Crtime, 0)
-		}
-	} else {
-		lastOffsetTs = time.Now().Add(-*option.timeAgo)
-	}
-
-	client, err := remote_storage.GetRemoteStorage(remoteStorage)
-	if err != nil {
-		return err
-	}
-
-	eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
-		message := resp.EventNotification
-		if message.OldEntry == nil && message.NewEntry == nil {
-			return nil
-		}
-		if message.OldEntry == nil && message.NewEntry != nil {
-			if !filer.HasData(message.NewEntry) {
-				return nil
-			}
-			glog.V(2).Infof("create: %+v", resp)
-			if !shouldSendToRemote(message.NewEntry) {
-				glog.V(2).Infof("skipping creating: %+v", resp)
-				return nil
-			}
-			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
-			if message.NewEntry.IsDirectory {
-				glog.V(0).Infof("mkdir  %s", remote_storage.FormatLocation(dest))
-				return client.WriteDirectory(dest, message.NewEntry)
-			}
-			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
-			reader := filer.NewFileReader(filerSource, message.NewEntry)
-			remoteEntry, writeErr := client.WriteFile(dest, message.NewEntry, reader)
-			if writeErr != nil {
-				return writeErr
-			}
-			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
-		}
-		if message.OldEntry != nil && message.NewEntry == nil {
-			glog.V(2).Infof("delete: %+v", resp)
-			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
-			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(dest))
-			return client.DeleteFile(dest)
-		}
-		if message.OldEntry != nil && message.NewEntry != nil {
-			oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
-			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
-			if !shouldSendToRemote(message.NewEntry) {
-				glog.V(2).Infof("skipping updating: %+v", resp)
-				return nil
-			}
-			if message.NewEntry.IsDirectory {
-				return client.WriteDirectory(dest, message.NewEntry)
-			}
-			if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name {
-				if filer.IsSameData(message.OldEntry, message.NewEntry) {
-					glog.V(2).Infof("update meta: %+v", resp)
-					return client.UpdateFileMetadata(dest, message.OldEntry, message.NewEntry)
-				}
-			}
-			glog.V(2).Infof("update: %+v", resp)
-			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest))
-			if err := client.DeleteFile(oldDest); err != nil {
-				return err
-			}
-			reader := filer.NewFileReader(filerSource, message.NewEntry)
-			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
-			remoteEntry, writeErr := client.WriteFile(dest, message.NewEntry, reader)
-			if writeErr != nil {
-				return writeErr
-			}
-			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
-		}
-
-		return nil
-	}
-
-	processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
-		lastTime := time.Unix(0, lastTsNs)
-		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
-		return setOffset(option.grpcDialOption, *option.filerAddress, RemoteSyncKeyPrefix, int32(dirHash), lastTsNs)
-	})
-
-	return pb.FollowMetadata(*option.filerAddress, option.grpcDialOption,
-		"filer.remote.sync", mountedDir, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
-}
-
-func toRemoteStorageLocation(mountDir, sourcePath util.FullPath, remoteMountLocation *filer_pb.RemoteStorageLocation) *filer_pb.RemoteStorageLocation {
-	source := string(sourcePath[len(mountDir):])
-	dest := util.FullPath(remoteMountLocation.Path).Child(source)
-	return &filer_pb.RemoteStorageLocation{
-		Name:   remoteMountLocation.Name,
-		Bucket: remoteMountLocation.Bucket,
-		Path:   string(dest),
-	}
-}
-
-func shouldSendToRemote(entry *filer_pb.Entry) bool {
-	if entry.RemoteEntry == nil {
-		return true
-	}
-	if entry.RemoteEntry.LastLocalSyncTsNs/1e9 < entry.Attributes.Mtime {
-		return true
-	}
-	return false
-}
-
-func updateLocalEntry(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.Entry, remoteEntry *filer_pb.RemoteEntry) error {
-	entry.RemoteEntry = remoteEntry
-	return filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-		_, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
-			Directory: dir,
-			Entry:     entry,
-		})
-		return err
-	})
-}

weed/command/filer_remote_sync_buckets.go

@@ -0,0 +1,367 @@
+package command
+
+import (
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
+	"github.com/chrislusf/seaweedfs/weed/remote_storage"
+	"github.com/chrislusf/seaweedfs/weed/replication/source"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/golang/protobuf/proto"
+	"math"
+	"math/rand"
+	"strings"
+	"time"
+)
+
+func (option *RemoteSyncOptions) followBucketUpdatesAndUploadToRemote(filerSource *source.FilerSource) error {
+
+	// read filer remote storage mount mappings
+	if detectErr := option.collectRemoteStorageConf(); detectErr != nil {
+		return fmt.Errorf("read mount info: %v", detectErr)
+	}
+
+	eachEntryFunc, err := option.makeBucketedEventProcessor(filerSource)
+	if err != nil {
+		return err
+	}
+
+	processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
+		lastTime := time.Unix(0, lastTsNs)
+		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
+		return remote_storage.SetSyncOffset(option.grpcDialOption, *option.filerAddress, option.bucketsDir, lastTsNs)
+	})
+
+	lastOffsetTs := collectLastSyncOffset(option, option.bucketsDir)
+
+	return pb.FollowMetadata(*option.filerAddress, option.grpcDialOption, "filer.remote.sync",
+		option.bucketsDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
+}
+
+func (option *RemoteSyncOptions) makeBucketedEventProcessor(filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
+
+	handleCreateBucket := func(entry *filer_pb.Entry) error {
+		if !entry.IsDirectory {
+			return nil
+		}
+		if entry.RemoteEntry != nil {
+			// this directory is imported from "remote.mount.buckets" or "remote.mount"
+			return nil
+		}
+		remoteConf, found := option.remoteConfs[*option.createBucketAt]
+		if !found {
+			return fmt.Errorf("un-configured remote storage %s", *option.createBucketAt)
+		}
+
+		client, err := remote_storage.GetRemoteStorage(remoteConf)
+		if err != nil {
+			return err
+		}
+
+		bucketName := strings.ToLower(entry.Name)
+		if *option.createBucketRandomSuffix {
+			// https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
+			if len(bucketName)+5 > 63 {
+				bucketName = bucketName[:58]
+			}
+			bucketName = fmt.Sprintf("%s-%4d", bucketName, rand.Uint32()%10000)
+		}
+
+		glog.V(0).Infof("create bucket %s", bucketName)
+		if err := client.CreateBucket(bucketName); err != nil {
+			return err
+		}
+
+		bucketPath := util.FullPath(option.bucketsDir).Child(entry.Name)
+		remoteLocation := &remote_pb.RemoteStorageLocation{
+			Name:   *option.createBucketAt,
+			Bucket: bucketName,
+			Path:   "/",
+		}
+
+		// need to add new mapping here before getting upates from metadata tailing
+		option.mappings.Mappings[string(bucketPath)] = remoteLocation
+
+		return filer.InsertMountMapping(option, string(bucketPath), remoteLocation)
+
+	}
+	handleDeleteBucket := func(entry *filer_pb.Entry) error {
+		if !entry.IsDirectory {
+			return nil
+		}
+
+		client, remoteStorageMountLocation, err := option.findRemoteStorageClient(entry.Name)
+		if err != nil {
+			return err
+		}
+
+		glog.V(0).Infof("delete remote bucket %s", remoteStorageMountLocation.Bucket)
+		if err := client.DeleteBucket(remoteStorageMountLocation.Bucket); err != nil {
+			return err
+		}
+
+		bucketPath := util.FullPath(option.bucketsDir).Child(entry.Name)
+
+		return filer.DeleteMountMapping(option, string(bucketPath))
+	}
+
+	handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error {
+		message := resp.EventNotification
+		if message.NewEntry != nil {
+			// update
+			if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
+				newMappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
+				if readErr != nil {
+					return fmt.Errorf("unmarshal mappings: %v", readErr)
+				}
+				option.mappings = newMappings
+			}
+			if strings.HasSuffix(message.NewEntry.Name, filer.REMOTE_STORAGE_CONF_SUFFIX) {
+				conf := &remote_pb.RemoteConf{}
+				if err := proto.Unmarshal(message.NewEntry.Content, conf); err != nil {
+					return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err)
+				}
+				option.remoteConfs[conf.Name] = conf
+			}
+		} else if message.OldEntry != nil {
+			// deletion
+			if strings.HasSuffix(message.OldEntry.Name, filer.REMOTE_STORAGE_CONF_SUFFIX) {
+				conf := &remote_pb.RemoteConf{}
+				if err := proto.Unmarshal(message.OldEntry.Content, conf); err != nil {
+					return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.OldEntry.Name, err)
+				}
+				delete(option.remoteConfs, conf.Name)
+			}
+		}
+
+		return nil
+	}
+
+	eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
+		message := resp.EventNotification
+		if strings.HasPrefix(resp.Directory, filer.DirectoryEtcRemote) {
+			return handleEtcRemoteChanges(resp)
+		}
+
+		if message.OldEntry == nil && message.NewEntry == nil {
+			return nil
+		}
+		if message.OldEntry == nil && message.NewEntry != nil {
+			if message.NewParentPath == option.bucketsDir {
+				return handleCreateBucket(message.NewEntry)
+			}
+			if !filer.HasData(message.NewEntry) {
+				return nil
+			}
+			bucket, remoteStorageMountLocation, remoteStorage, ok := option.detectBucketInfo(message.NewParentPath)
+			if !ok {
+				return nil
+			}
+			client, err := remote_storage.GetRemoteStorage(remoteStorage)
+			if err != nil {
+				return err
+			}
+			glog.V(2).Infof("create: %+v", resp)
+			if !shouldSendToRemote(message.NewEntry) {
+				glog.V(2).Infof("skipping creating: %+v", resp)
+				return nil
+			}
+			dest := toRemoteStorageLocation(bucket, util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
+			if message.NewEntry.IsDirectory {
+				glog.V(0).Infof("mkdir  %s", remote_storage.FormatLocation(dest))
+				return client.WriteDirectory(dest, message.NewEntry)
+			}
+			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
+			reader := filer.NewFileReader(filerSource, message.NewEntry)
+			remoteEntry, writeErr := client.WriteFile(dest, message.NewEntry, reader)
+			if writeErr != nil {
+				return writeErr
+			}
+			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
+		}
+		if message.OldEntry != nil && message.NewEntry == nil {
+			if resp.Directory == option.bucketsDir {
+				return handleDeleteBucket(message.OldEntry)
+			}
+			bucket, remoteStorageMountLocation, remoteStorage, ok := option.detectBucketInfo(resp.Directory)
+			if !ok {
+				return nil
+			}
+			client, err := remote_storage.GetRemoteStorage(remoteStorage)
+			if err != nil {
+				return err
+			}
+			glog.V(2).Infof("delete: %+v", resp)
+			dest := toRemoteStorageLocation(bucket, util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
+			if message.OldEntry.IsDirectory {
+				glog.V(0).Infof("rmdir  %s", remote_storage.FormatLocation(dest))
+				return client.RemoveDirectory(dest)
+			}
+			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(dest))
+			return client.DeleteFile(dest)
+		}
+		if message.OldEntry != nil && message.NewEntry != nil {
+			if resp.Directory == option.bucketsDir {
+				if message.NewParentPath == option.bucketsDir {
+					if message.OldEntry.Name == message.NewEntry.Name {
+						return nil
+					}
+					if err := handleCreateBucket(message.NewEntry); err != nil {
+						return err
+					}
+					if err := handleDeleteBucket(message.OldEntry); err != nil {
+						return err
+					}
+				}
+			}
+			oldBucket, oldRemoteStorageMountLocation, oldRemoteStorage, oldOk := option.detectBucketInfo(resp.Directory)
+			newBucket, newRemoteStorageMountLocation, newRemoteStorage, newOk := option.detectBucketInfo(message.NewParentPath)
+			if oldOk && newOk {
+				if !shouldSendToRemote(message.NewEntry) {
+					glog.V(2).Infof("skipping updating: %+v", resp)
+					return nil
+				}
+				client, err := remote_storage.GetRemoteStorage(oldRemoteStorage)
+				if err != nil {
+					return err
+				}
+				if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name {
+					// update the same entry
+					if message.NewEntry.IsDirectory {
+						// update directory property
+						return nil
+					}
+					if filer.IsSameData(message.OldEntry, message.NewEntry) {
+						glog.V(2).Infof("update meta: %+v", resp)
+						oldDest := toRemoteStorageLocation(oldBucket, util.NewFullPath(resp.Directory, message.OldEntry.Name), oldRemoteStorageMountLocation)
+						return client.UpdateFileMetadata(oldDest, message.OldEntry, message.NewEntry)
+					} else {
+						newDest := toRemoteStorageLocation(newBucket, util.NewFullPath(message.NewParentPath, message.NewEntry.Name), newRemoteStorageMountLocation)
+						reader := filer.NewFileReader(filerSource, message.NewEntry)
+						glog.V(0).Infof("create %s", remote_storage.FormatLocation(newDest))
+						remoteEntry, writeErr := client.WriteFile(newDest, message.NewEntry, reader)
+						if writeErr != nil {
+							return writeErr
+						}
+						return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
+					}
+				}
+			}
+
+			// the following is entry rename
+			if oldOk {
+				client, err := remote_storage.GetRemoteStorage(oldRemoteStorage)
+				if err != nil {
+					return err
+				}
+				oldDest := toRemoteStorageLocation(oldBucket, util.NewFullPath(resp.Directory, message.OldEntry.Name), oldRemoteStorageMountLocation)
+				if message.OldEntry.IsDirectory {
+					return client.RemoveDirectory(oldDest)
+				}
+				glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest))
+				if err := client.DeleteFile(oldDest); err != nil {
+					return err
+				}
+			}
+			if newOk {
+				if !shouldSendToRemote(message.NewEntry) {
+					glog.V(2).Infof("skipping updating: %+v", resp)
+					return nil
+				}
+				client, err := remote_storage.GetRemoteStorage(newRemoteStorage)
+				if err != nil {
+					return err
+				}
+				newDest := toRemoteStorageLocation(newBucket, util.NewFullPath(message.NewParentPath, message.NewEntry.Name), newRemoteStorageMountLocation)
+				if message.NewEntry.IsDirectory {
+					return client.WriteDirectory(newDest, message.NewEntry)
+				}
+				reader := filer.NewFileReader(filerSource, message.NewEntry)
+				glog.V(0).Infof("create %s", remote_storage.FormatLocation(newDest))
+				remoteEntry, writeErr := client.WriteFile(newDest, message.NewEntry, reader)
+				if writeErr != nil {
+					return writeErr
+				}
+				return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
+			}
+		}
+
+		return nil
+	}
+	return eachEntryFunc, nil
+}
+
+func (option *RemoteSyncOptions) findRemoteStorageClient(bucketName string) (client remote_storage.RemoteStorageClient, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, err error) {
+	bucket := util.FullPath(option.bucketsDir).Child(bucketName)
+
+	var isMounted bool
+	remoteStorageMountLocation, isMounted = option.mappings.Mappings[string(bucket)]
+	if !isMounted {
+		return nil, remoteStorageMountLocation, fmt.Errorf("%s is not mounted", bucket)
+	}
+	remoteConf, hasClient := option.remoteConfs[remoteStorageMountLocation.Name]
+	if !hasClient {
+		return nil, remoteStorageMountLocation, fmt.Errorf("%s mounted to un-configured %+v", bucket, remoteStorageMountLocation)
+	}
+
+	client, err = remote_storage.GetRemoteStorage(remoteConf)
+	if err != nil {
+		return nil, remoteStorageMountLocation, err
+	}
+	return client, remoteStorageMountLocation, nil
+}
+
+func (option *RemoteSyncOptions) detectBucketInfo(actualDir string) (bucket util.FullPath, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, remoteConf *remote_pb.RemoteConf, ok bool) {
+	bucket, ok = extractBucketPath(option.bucketsDir, actualDir)
+	if !ok {
+		return "", nil, nil, false
+	}
+	var isMounted bool
+	remoteStorageMountLocation, isMounted = option.mappings.Mappings[string(bucket)]
+	if !isMounted {
+		glog.Warningf("%s is not mounted", bucket)
+		return "", nil, nil, false
+	}
+	var hasClient bool
+	remoteConf, hasClient = option.remoteConfs[remoteStorageMountLocation.Name]
+	if !hasClient {
+		glog.Warningf("%s mounted to un-configured %+v", bucket, remoteStorageMountLocation)
+		return "", nil, nil, false
+	}
+	return bucket, remoteStorageMountLocation, remoteConf, true
+}
+
+func extractBucketPath(bucketsDir, dir string) (util.FullPath, bool) {
+	if !strings.HasPrefix(dir, bucketsDir+"/") {
+		return "", false
+	}
+	parts := strings.SplitN(dir[len(bucketsDir)+1:], "/", 2)
+	return util.FullPath(bucketsDir).Child(parts[0]), true
+}
+
+func (option *RemoteSyncOptions) collectRemoteStorageConf() (err error) {
+
+	if mappings, err := filer.ReadMountMappings(option.grpcDialOption, *option.filerAddress); err != nil {
+		return err
+	} else {
+		option.mappings = mappings
+	}
+
+	option.remoteConfs = make(map[string]*remote_pb.RemoteConf)
+	err = filer_pb.List(option, filer.DirectoryEtcRemote, "", func(entry *filer_pb.Entry, isLast bool) error {
+		if !strings.HasSuffix(entry.Name, filer.REMOTE_STORAGE_CONF_SUFFIX) {
+			return nil
+		}
+		conf := &remote_pb.RemoteConf{}
+		if err := proto.Unmarshal(entry.Content, conf); err != nil {
+			return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, entry.Name, err)
+		}
+		option.remoteConfs[conf.Name] = conf
+		return nil
+	}, "", false, math.MaxUint32)
+
+	return
+}

weed/command/filer_remote_sync_dir.go

@@ -0,0 +1,221 @@
+package command
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
+	"github.com/chrislusf/seaweedfs/weed/remote_storage"
+	"github.com/chrislusf/seaweedfs/weed/replication/source"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/golang/protobuf/proto"
+	"os"
+	"strings"
+	"time"
+)
+
+func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *source.FilerSource, mountedDir string) error {
+
+	// read filer remote storage mount mappings
+	_, _, remoteStorageMountLocation, remoteStorage, detectErr := filer.DetectMountInfo(option.grpcDialOption, *option.filerAddress, mountedDir)
+	if detectErr != nil {
+		return fmt.Errorf("read mount info: %v", detectErr)
+	}
+
+	eachEntryFunc, err := makeEventProcessor(remoteStorage, mountedDir, remoteStorageMountLocation, filerSource)
+	if err != nil {
+		return err
+	}
+
+	processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
+		lastTime := time.Unix(0, lastTsNs)
+		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
+		return remote_storage.SetSyncOffset(option.grpcDialOption, *option.filerAddress, mountedDir, lastTsNs)
+	})
+
+	lastOffsetTs := collectLastSyncOffset(option, mountedDir)
+
+	return pb.FollowMetadata(*option.filerAddress, option.grpcDialOption, "filer.remote.sync",
+		mountedDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
+}
+
+func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
+	client, err := remote_storage.GetRemoteStorage(remoteStorage)
+	if err != nil {
+		return nil, err
+	}
+
+	handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error {
+		message := resp.EventNotification
+		if message.NewEntry == nil {
+			return nil
+		}
+		if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
+			mappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
+			if readErr != nil {
+				return fmt.Errorf("unmarshal mappings: %v", readErr)
+			}
+			if remoteLoc, found := mappings.Mappings[mountedDir]; found {
+				if remoteStorageMountLocation.Bucket != remoteLoc.Bucket || remoteStorageMountLocation.Path != remoteLoc.Path {
+					glog.Fatalf("Unexpected mount changes %+v => %+v", remoteStorageMountLocation, remoteLoc)
+				}
+			} else {
+				glog.V(0).Infof("unmounted %s exiting ...", mountedDir)
+				os.Exit(0)
+			}
+		}
+		if message.NewEntry.Name == remoteStorage.Name+filer.REMOTE_STORAGE_CONF_SUFFIX {
+			conf := &remote_pb.RemoteConf{}
+			if err := proto.Unmarshal(message.NewEntry.Content, conf); err != nil {
+				return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err)
+			}
+			remoteStorage = conf
+			if newClient, err := remote_storage.GetRemoteStorage(remoteStorage); err == nil {
+				client = newClient
+			} else {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
+		message := resp.EventNotification
+		if strings.HasPrefix(resp.Directory, filer.DirectoryEtcRemote) {
+			return handleEtcRemoteChanges(resp)
+		}
+
+		if message.OldEntry == nil && message.NewEntry == nil {
+			return nil
+		}
+		if message.OldEntry == nil && message.NewEntry != nil {
+			if !filer.HasData(message.NewEntry) {
+				return nil
+			}
+			glog.V(2).Infof("create: %+v", resp)
+			if !shouldSendToRemote(message.NewEntry) {
+				glog.V(2).Infof("skipping creating: %+v", resp)
+				return nil
+			}
+			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
+			if message.NewEntry.IsDirectory {
+				glog.V(0).Infof("mkdir  %s", remote_storage.FormatLocation(dest))
+				return client.WriteDirectory(dest, message.NewEntry)
+			}
+			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
+			reader := filer.NewFileReader(filerSource, message.NewEntry)
+			remoteEntry, writeErr := client.WriteFile(dest, message.NewEntry, reader)
+			if writeErr != nil {
+				return writeErr
+			}
+			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
+		}
+		if message.OldEntry != nil && message.NewEntry == nil {
+			glog.V(2).Infof("delete: %+v", resp)
+			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
+			if message.OldEntry.IsDirectory {
+				glog.V(0).Infof("rmdir  %s", remote_storage.FormatLocation(dest))
+				return client.RemoveDirectory(dest)
+			}
+			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(dest))
+			return client.DeleteFile(dest)
+		}
+		if message.OldEntry != nil && message.NewEntry != nil {
+			oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
+			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
+			if !shouldSendToRemote(message.NewEntry) {
+				glog.V(2).Infof("skipping updating: %+v", resp)
+				return nil
+			}
+			if message.NewEntry.IsDirectory {
+				return client.WriteDirectory(dest, message.NewEntry)
+			}
+			if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name {
+				if filer.IsSameData(message.OldEntry, message.NewEntry) {
+					glog.V(2).Infof("update meta: %+v", resp)
+					return client.UpdateFileMetadata(dest, message.OldEntry, message.NewEntry)
+				}
+			}
+			glog.V(2).Infof("update: %+v", resp)
+			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest))
+			if err := client.DeleteFile(oldDest); err != nil {
+				return err
+			}
+			reader := filer.NewFileReader(filerSource, message.NewEntry)
+			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
+			remoteEntry, writeErr := client.WriteFile(dest, message.NewEntry, reader)
+			if writeErr != nil {
+				return writeErr
+			}
+			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
+		}
+
+		return nil
+	}
+	return eachEntryFunc, nil
+}
+
+func collectLastSyncOffset(option *RemoteSyncOptions, mountedDir string) time.Time {
+	// 1. specified by timeAgo
+	// 2. last offset timestamp for this directory
+	// 3. directory creation time
+	var lastOffsetTs time.Time
+	if *option.timeAgo == 0 {
+		mountedDirEntry, err := filer_pb.GetEntry(option, util.FullPath(mountedDir))
+		if err != nil {
+			glog.V(0).Infof("get mounted directory %s: %v", mountedDir, err)
+			return time.Now()
+		}
+
+		lastOffsetTsNs, err := remote_storage.GetSyncOffset(option.grpcDialOption, *option.filerAddress, mountedDir)
+		if mountedDirEntry != nil {
+			if err == nil && mountedDirEntry.Attributes.Crtime < lastOffsetTsNs/1000000 {
+				lastOffsetTs = time.Unix(0, lastOffsetTsNs)
+				glog.V(0).Infof("resume from %v", lastOffsetTs)
+			} else {
+				lastOffsetTs = time.Unix(mountedDirEntry.Attributes.Crtime, 0)
+			}
+		} else {
+			lastOffsetTs = time.Now()
+		}
+	} else {
+		lastOffsetTs = time.Now().Add(-*option.timeAgo)
+	}
+	return lastOffsetTs
+}
+
+func toRemoteStorageLocation(mountDir, sourcePath util.FullPath, remoteMountLocation *remote_pb.RemoteStorageLocation) *remote_pb.RemoteStorageLocation {
+	source := string(sourcePath[len(mountDir):])
+	dest := util.FullPath(remoteMountLocation.Path).Child(source)
+	return &remote_pb.RemoteStorageLocation{
+		Name:   remoteMountLocation.Name,
+		Bucket: remoteMountLocation.Bucket,
+		Path:   string(dest),
+	}
+}
+
+func shouldSendToRemote(entry *filer_pb.Entry) bool {
+	if entry.RemoteEntry == nil {
+		return true
+	}
+	if entry.RemoteEntry.RemoteMtime < entry.Attributes.Mtime {
+		return true
+	}
+	return false
+}
+
+func updateLocalEntry(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.Entry, remoteEntry *filer_pb.RemoteEntry) error {
+	remoteEntry.LastLocalSyncTsNs = time.Now().UnixNano()
+	entry.RemoteEntry = remoteEntry
+	return filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		_, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
+			Directory: dir,
+			Entry:     entry,
+		})
+		return err
+	})
+}

weed/command/filer_sync.go

@@ -171,7 +171,7 @@ func doSubscribeFilerMetaChanges(grpcDialOption grpc.DialOption, sourceFiler, so
 	})
 
 	return pb.FollowMetadata(sourceFiler, grpcDialOption, "syncTo_"+targetFiler,
-		sourcePath, sourceFilerOffsetTsNs, targetFilerSignature, processEventFnWithOffset, false)
+		sourcePath, nil, sourceFilerOffsetTsNs, targetFilerSignature, processEventFnWithOffset, false)
 
 }
 

weed/command/filer_sync_std.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 package command

weed/command/imports.go

@@ -3,8 +3,10 @@ package command
 import (
 	_ "net/http/pprof"
 
-	_ "github.com/chrislusf/seaweedfs/weed/remote_storage/s3"
+	_ "github.com/chrislusf/seaweedfs/weed/remote_storage/azure"
 	_ "github.com/chrislusf/seaweedfs/weed/remote_storage/gcs"
+	_ "github.com/chrislusf/seaweedfs/weed/remote_storage/hdfs"
+	_ "github.com/chrislusf/seaweedfs/weed/remote_storage/s3"
 
 	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/azuresink"
 	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/b2sink"
@@ -28,4 +30,5 @@ import (
 	_ "github.com/chrislusf/seaweedfs/weed/filer/redis"
 	_ "github.com/chrislusf/seaweedfs/weed/filer/redis2"
 	_ "github.com/chrislusf/seaweedfs/weed/filer/sqlite"
+	_ "github.com/chrislusf/seaweedfs/weed/filer/tikv"
 )

weed/command/mount_notsupported.go

@@ -1,6 +1,5 @@
-// +build !linux
-// +build !darwin
-// +build !freebsd
+//go:build !linux && !darwin && !freebsd
+// +build !linux,!darwin,!freebsd
 
 package command
 

weed/command/mount_std.go

@@ -1,3 +1,4 @@
+//go:build linux || darwin || freebsd
 // +build linux darwin freebsd
 
 package command

weed/command/scaffold/filer.toml

@@ -230,3 +230,11 @@ location = "/tmp/"
 address = "localhost:6379"
 password = ""
 database = 1
+
+[tikv]
+enabled = false
+# If you have many pd address, use ',' split then:
+#   pdaddrs = "pdhost1:2379, pdhost2:2379, pdhost3:2379"
+pdaddrs = "localhost:2379"
+# Concurrency for TiKV delete range
+deleterange_concurrency = 1

weed/filer/etcd/etcd_store.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 	"time"
 
-	"go.etcd.io/etcd/clientv3"
+	"go.etcd.io/etcd/client/v3"
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/glog"

weed/filer/read_remote.go

@@ -2,8 +2,8 @@ package filer
 
 import (
 	"context"
-	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -11,21 +11,8 @@ func (entry *Entry) IsInRemoteOnly() bool {
 	return len(entry.Chunks) == 0 && entry.Remote != nil && entry.Remote.RemoteSize > 0
 }
 
-func (f *Filer) ReadRemote(entry *Entry, offset int64, size int64) (data []byte, err error) {
-	client, _, found := f.RemoteStorage.GetRemoteStorageClient(entry.Remote.StorageName)
-	if !found {
-		return nil, fmt.Errorf("remote storage %v not found", entry.Remote.StorageName)
-	}
-
-	mountDir, remoteLoation := f.RemoteStorage.FindMountDirectory(entry.FullPath)
-
-	sourceLoc := MapFullPathToRemoteStorageLocation(mountDir, remoteLoation, entry.FullPath)
-
-	return client.ReadFile(sourceLoc, offset, size)
-}
-
-func MapFullPathToRemoteStorageLocation(localMountedDir util.FullPath, remoteMountedLocation *filer_pb.RemoteStorageLocation, fp util.FullPath) *filer_pb.RemoteStorageLocation {
-	remoteLocation := &filer_pb.RemoteStorageLocation{
+func MapFullPathToRemoteStorageLocation(localMountedDir util.FullPath, remoteMountedLocation *remote_pb.RemoteStorageLocation, fp util.FullPath) *remote_pb.RemoteStorageLocation {
+	remoteLocation := &remote_pb.RemoteStorageLocation{
 		Name:   remoteMountedLocation.Name,
 		Bucket: remoteMountedLocation.Bucket,
 		Path:   remoteMountedLocation.Path,
@@ -34,11 +21,11 @@ func MapFullPathToRemoteStorageLocation(localMountedDir util.FullPath, remoteMou
 	return remoteLocation
 }
 
-func MapRemoteStorageLocationPathToFullPath(localMountedDir util.FullPath, remoteMountedLocation *filer_pb.RemoteStorageLocation, remoteLocationPath string)(fp util.FullPath)  {
+func MapRemoteStorageLocationPathToFullPath(localMountedDir util.FullPath, remoteMountedLocation *remote_pb.RemoteStorageLocation, remoteLocationPath string) (fp util.FullPath) {
 	return localMountedDir.Child(remoteLocationPath[len(remoteMountedLocation.Path):])
 }
 
-func DownloadToLocal(filerClient filer_pb.FilerClient, remoteConf *filer_pb.RemoteConf, remoteLocation *filer_pb.RemoteStorageLocation, parent util.FullPath, entry *filer_pb.Entry) error {
+func DownloadToLocal(filerClient filer_pb.FilerClient, remoteConf *remote_pb.RemoteConf, remoteLocation *remote_pb.RemoteStorageLocation, parent util.FullPath, entry *filer_pb.Entry) error {
 	return filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 		_, err := client.DownloadToLocal(context.Background(), &filer_pb.DownloadToLocalRequest{
 			Directory: string(parent),

weed/filer/remote_mapping.go

@@ -0,0 +1,121 @@
+package filer
+
+import (
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
+	"github.com/golang/protobuf/proto"
+	"google.golang.org/grpc"
+)
+
+func ReadMountMappings(grpcDialOption grpc.DialOption, filerAddress string) (mappings *remote_pb.RemoteStorageMapping, readErr error) {
+	var oldContent []byte
+	if readErr = pb.WithFilerClient(filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		oldContent, readErr = ReadInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE)
+		return readErr
+	}); readErr != nil {
+		return nil, readErr
+	}
+
+	mappings, readErr = UnmarshalRemoteStorageMappings(oldContent)
+	if readErr != nil {
+		return nil, fmt.Errorf("unmarshal mappings: %v", readErr)
+	}
+
+	return
+}
+
+func InsertMountMapping(filerClient filer_pb.FilerClient, dir string, remoteStorageLocation *remote_pb.RemoteStorageLocation) (err error) {
+
+	// read current mapping
+	var oldContent, newContent []byte
+	err = filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		oldContent, err = ReadInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE)
+		return err
+	})
+	if err != nil {
+		if err != filer_pb.ErrNotFound {
+			return fmt.Errorf("read existing mapping: %v", err)
+		}
+	}
+
+	// add new mapping
+	newContent, err = addRemoteStorageMapping(oldContent, dir, remoteStorageLocation)
+	if err != nil {
+		return fmt.Errorf("add mapping %s~%s: %v", dir, remoteStorageLocation, err)
+	}
+
+	// save back
+	err = filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		return SaveInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE, newContent)
+	})
+	if err != nil {
+		return fmt.Errorf("save mapping: %v", err)
+	}
+
+	return nil
+}
+
+func DeleteMountMapping(filerClient filer_pb.FilerClient, dir string) (err error) {
+
+	// read current mapping
+	var oldContent, newContent []byte
+	err = filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		oldContent, err = ReadInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE)
+		return err
+	})
+	if err != nil {
+		if err != filer_pb.ErrNotFound {
+			return fmt.Errorf("read existing mapping: %v", err)
+		}
+	}
+
+	// add new mapping
+	newContent, err = removeRemoteStorageMapping(oldContent, dir)
+	if err != nil {
+		return fmt.Errorf("delete mount %s: %v", dir, err)
+	}
+
+	// save back
+	err = filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		return SaveInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE, newContent)
+	})
+	if err != nil {
+		return fmt.Errorf("save mapping: %v", err)
+	}
+
+	return nil
+}
+
+func addRemoteStorageMapping(oldContent []byte, dir string, storageLocation *remote_pb.RemoteStorageLocation) (newContent []byte, err error) {
+	mappings, unmarshalErr := UnmarshalRemoteStorageMappings(oldContent)
+	if unmarshalErr != nil {
+		// skip
+	}
+
+	// set the new mapping
+	mappings.Mappings[dir] = storageLocation
+
+	if newContent, err = proto.Marshal(mappings); err != nil {
+		return oldContent, fmt.Errorf("marshal mappings: %v", err)
+	}
+
+	return
+}
+
+func removeRemoteStorageMapping(oldContent []byte, dir string) (newContent []byte, err error) {
+	mappings, unmarshalErr := UnmarshalRemoteStorageMappings(oldContent)
+	if unmarshalErr != nil {
+		return nil, unmarshalErr
+	}
+
+	// set the new mapping
+	delete(mappings.Mappings, dir)
+
+	if newContent, err = proto.Marshal(mappings); err != nil {
+		return oldContent, fmt.Errorf("marshal mappings: %v", err)
+	}
+
+	return
+}

weed/filer/remote_storage.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
 	"github.com/chrislusf/seaweedfs/weed/remote_storage"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/golang/protobuf/proto"
@@ -21,13 +22,13 @@ const REMOTE_STORAGE_MOUNT_FILE = "mount.mapping"
 
 type FilerRemoteStorage struct {
 	rules             ptrie.Trie
-	storageNameToConf map[string]*filer_pb.RemoteConf
+	storageNameToConf map[string]*remote_pb.RemoteConf
 }
 
 func NewFilerRemoteStorage() (rs *FilerRemoteStorage) {
 	rs = &FilerRemoteStorage{
 		rules:             ptrie.New(),
-		storageNameToConf: make(map[string]*filer_pb.RemoteConf),
+		storageNameToConf: make(map[string]*remote_pb.RemoteConf),
 	}
 	return rs
 }
@@ -56,7 +57,7 @@ func (rs *FilerRemoteStorage) LoadRemoteStorageConfigurationsAndMapping(filer *F
 		if !strings.HasSuffix(entry.Name(), REMOTE_STORAGE_CONF_SUFFIX) {
 			return nil
 		}
-		conf := &filer_pb.RemoteConf{}
+		conf := &remote_pb.RemoteConf{}
 		if err := proto.Unmarshal(entry.Content, conf); err != nil {
 			return fmt.Errorf("unmarshal %s/%s: %v", DirectoryEtcRemote, entry.Name(), err)
 		}
@@ -66,7 +67,7 @@ func (rs *FilerRemoteStorage) LoadRemoteStorageConfigurationsAndMapping(filer *F
 }
 
 func (rs *FilerRemoteStorage) loadRemoteStorageMountMapping(data []byte) (err error) {
-	mappings := &filer_pb.RemoteStorageMapping{}
+	mappings := &remote_pb.RemoteStorageMapping{}
 	if err := proto.Unmarshal(data, mappings); err != nil {
 		return fmt.Errorf("unmarshal %s/%s: %v", DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE, err)
 	}
@@ -76,23 +77,23 @@ func (rs *FilerRemoteStorage) loadRemoteStorageMountMapping(data []byte) (err er
 	return nil
 }
 
-func (rs *FilerRemoteStorage) mapDirectoryToRemoteStorage(dir util.FullPath, loc *filer_pb.RemoteStorageLocation) {
+func (rs *FilerRemoteStorage) mapDirectoryToRemoteStorage(dir util.FullPath, loc *remote_pb.RemoteStorageLocation) {
 	rs.rules.Put([]byte(dir+"/"), loc)
 }
 
-func (rs *FilerRemoteStorage) FindMountDirectory(p util.FullPath) (mountDir util.FullPath, remoteLocation *filer_pb.RemoteStorageLocation) {
+func (rs *FilerRemoteStorage) FindMountDirectory(p util.FullPath) (mountDir util.FullPath, remoteLocation *remote_pb.RemoteStorageLocation) {
 	rs.rules.MatchPrefix([]byte(p), func(key []byte, value interface{}) bool {
 		mountDir = util.FullPath(string(key[:len(key)-1]))
-		remoteLocation = value.(*filer_pb.RemoteStorageLocation)
+		remoteLocation = value.(*remote_pb.RemoteStorageLocation)
 		return true
 	})
 	return
 }
 
-func (rs *FilerRemoteStorage) FindRemoteStorageClient(p util.FullPath) (client remote_storage.RemoteStorageClient, remoteConf *filer_pb.RemoteConf, found bool) {
-	var storageLocation *filer_pb.RemoteStorageLocation
+func (rs *FilerRemoteStorage) FindRemoteStorageClient(p util.FullPath) (client remote_storage.RemoteStorageClient, remoteConf *remote_pb.RemoteConf, found bool) {
+	var storageLocation *remote_pb.RemoteStorageLocation
 	rs.rules.MatchPrefix([]byte(p), func(key []byte, value interface{}) bool {
-		storageLocation = value.(*filer_pb.RemoteStorageLocation)
+		storageLocation = value.(*remote_pb.RemoteStorageLocation)
 		return true
 	})
 
@@ -104,7 +105,7 @@ func (rs *FilerRemoteStorage) FindRemoteStorageClient(p util.FullPath) (client r
 	return rs.GetRemoteStorageClient(storageLocation.Name)
 }
 
-func (rs *FilerRemoteStorage) GetRemoteStorageClient(storageName string) (client remote_storage.RemoteStorageClient, remoteConf *filer_pb.RemoteConf, found bool) {
+func (rs *FilerRemoteStorage) GetRemoteStorageClient(storageName string) (client remote_storage.RemoteStorageClient, remoteConf *remote_pb.RemoteConf, found bool) {
 	remoteConf, found = rs.storageNameToConf[storageName]
 	if !found {
 		return
@@ -118,9 +119,9 @@ func (rs *FilerRemoteStorage) GetRemoteStorageClient(storageName string) (client
 	return
 }
 
-func UnmarshalRemoteStorageMappings(oldContent []byte) (mappings *filer_pb.RemoteStorageMapping, err error) {
-	mappings = &filer_pb.RemoteStorageMapping{
-		Mappings: make(map[string]*filer_pb.RemoteStorageLocation),
+func UnmarshalRemoteStorageMappings(oldContent []byte) (mappings *remote_pb.RemoteStorageMapping, err error) {
+	mappings = &remote_pb.RemoteStorageMapping{
+		Mappings: make(map[string]*remote_pb.RemoteStorageLocation),
 	}
 	if len(oldContent) > 0 {
 		if err = proto.Unmarshal(oldContent, mappings); err != nil {
@@ -130,56 +131,7 @@ func UnmarshalRemoteStorageMappings(oldContent []byte) (mappings *filer_pb.Remot
 	return
 }
 
-func AddRemoteStorageMapping(oldContent []byte, dir string, storageLocation *filer_pb.RemoteStorageLocation) (newContent []byte, err error) {
-	mappings, unmarshalErr := UnmarshalRemoteStorageMappings(oldContent)
-	if unmarshalErr != nil {
-		// skip
-	}
-
-	// set the new mapping
-	mappings.Mappings[dir] = storageLocation
-
-	if newContent, err = proto.Marshal(mappings); err != nil {
-		return oldContent, fmt.Errorf("marshal mappings: %v", err)
-	}
-
-	return
-}
-
-func RemoveRemoteStorageMapping(oldContent []byte, dir string) (newContent []byte, err error) {
-	mappings, unmarshalErr := UnmarshalRemoteStorageMappings(oldContent)
-	if unmarshalErr != nil {
-		return nil, unmarshalErr
-	}
-
-	// set the new mapping
-	delete(mappings.Mappings, dir)
-
-	if newContent, err = proto.Marshal(mappings); err != nil {
-		return oldContent, fmt.Errorf("marshal mappings: %v", err)
-	}
-
-	return
-}
-
-func ReadMountMappings(grpcDialOption grpc.DialOption, filerAddress string) (mappings *filer_pb.RemoteStorageMapping, readErr error) {
-	var oldContent []byte
-	if readErr = pb.WithFilerClient(filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
-		oldContent, readErr = ReadInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE)
-		return readErr
-	}); readErr != nil {
-		return nil, readErr
-	}
-
-	mappings, readErr = UnmarshalRemoteStorageMappings(oldContent)
-	if readErr != nil {
-		return nil, fmt.Errorf("unmarshal mappings: %v", readErr)
-	}
-
-	return
-}
-
-func ReadRemoteStorageConf(grpcDialOption grpc.DialOption, filerAddress string, storageName string) (conf *filer_pb.RemoteConf, readErr error) {
+func ReadRemoteStorageConf(grpcDialOption grpc.DialOption, filerAddress string, storageName string) (conf *remote_pb.RemoteConf, readErr error) {
 	var oldContent []byte
 	if readErr = pb.WithFilerClient(filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		oldContent, readErr = ReadInsideFiler(client, DirectoryEtcRemote, storageName+REMOTE_STORAGE_CONF_SUFFIX)
@@ -189,7 +141,7 @@ func ReadRemoteStorageConf(grpcDialOption grpc.DialOption, filerAddress string,
 	}
 
 	// unmarshal storage configuration
-	conf = &filer_pb.RemoteConf{}
+	conf = &remote_pb.RemoteConf{}
 	if unMarshalErr := proto.Unmarshal(oldContent, conf); unMarshalErr != nil {
 		readErr = fmt.Errorf("unmarshal %s/%s: %v", DirectoryEtcRemote, storageName+REMOTE_STORAGE_CONF_SUFFIX, unMarshalErr)
 		return
@@ -198,7 +150,7 @@ func ReadRemoteStorageConf(grpcDialOption grpc.DialOption, filerAddress string,
 	return
 }
 
-func DetectMountInfo(grpcDialOption grpc.DialOption, filerAddress string, dir string) (*filer_pb.RemoteStorageMapping, string, *filer_pb.RemoteStorageLocation, *filer_pb.RemoteConf, error) {
+func DetectMountInfo(grpcDialOption grpc.DialOption, filerAddress string, dir string) (*remote_pb.RemoteStorageMapping, string, *remote_pb.RemoteStorageLocation, *remote_pb.RemoteConf, error) {
 
 	mappings, listErr := ReadMountMappings(grpcDialOption, filerAddress)
 	if listErr != nil {
@@ -209,7 +161,7 @@ func DetectMountInfo(grpcDialOption grpc.DialOption, filerAddress string, dir st
 	}
 
 	var localMountedDir string
-	var remoteStorageMountedLocation *filer_pb.RemoteStorageLocation
+	var remoteStorageMountedLocation *remote_pb.RemoteStorageLocation
 	for k, loc := range mappings.Mappings {
 		if strings.HasPrefix(dir, k) {
 			localMountedDir, remoteStorageMountedLocation = k, loc

weed/filer/remote_storage_test.go

@@ -1,20 +1,20 @@
 package filer
 
 import (
-	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
 	"github.com/stretchr/testify/assert"
 	"testing"
 )
 
 func TestFilerRemoteStorage_FindRemoteStorageClient(t *testing.T) {
-	conf := &filer_pb.RemoteConf{
+	conf := &remote_pb.RemoteConf{
 		Name: "s7",
 		Type: "s3",
 	}
 	rs := NewFilerRemoteStorage()
 	rs.storageNameToConf[conf.Name] = conf
 
-	rs.mapDirectoryToRemoteStorage("/a/b/c", &filer_pb.RemoteStorageLocation{
+	rs.mapDirectoryToRemoteStorage("/a/b/c", &remote_pb.RemoteStorageLocation{
 		Name:   "s7",
 		Bucket: "some",
 		Path:   "/dir",

weed/filer/sqlite/sqlite_store.go

@@ -1,3 +1,4 @@
+//go:build linux || darwin || windows
 // +build linux darwin windows
 
 // limited GOOS due to modernc.org/libc/unistd

weed/filer/sqlite/sqlite_store_unsupported.go

@@ -1,3 +1,4 @@
+//go:build !linux && !darwin && !windows && !s390 && !ppc64le && !mips64
 // +build !linux,!darwin,!windows,!s390,!ppc64le,!mips64
 
 // limited GOOS due to modernc.org/libc/unistd

weed/filer/stream.go

@@ -7,6 +7,7 @@ import (
 	"math"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -126,14 +127,14 @@ func ReadAll(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk)
 
 // ----------------  ChunkStreamReader ----------------------------------
 type ChunkStreamReader struct {
-	chunkViews         []*ChunkView
-	totalSize          int64
-	logicOffset        int64
-	buffer             []byte
-	bufferOffset       int64
-	bufferPos          int
-	nextChunkViewIndex int
-	lookupFileId       wdclient.LookupFileIdFunctionType
+	chunkViews   []*ChunkView
+	totalSize    int64
+	logicOffset  int64
+	buffer       []byte
+	bufferOffset int64
+	bufferLock   sync.Mutex
+	chunk        string
+	lookupFileId wdclient.LookupFileIdFunctionType
 }
 
 var _ = io.ReadSeeker(&ChunkStreamReader{})
@@ -175,27 +176,29 @@ func NewChunkStreamReader(filerClient filer_pb.FilerClient, chunks []*filer_pb.F
 }
 
 func (c *ChunkStreamReader) ReadAt(p []byte, off int64) (n int, err error) {
+	c.bufferLock.Lock()
+	defer c.bufferLock.Unlock()
 	if err = c.prepareBufferFor(off); err != nil {
 		return
 	}
 	c.logicOffset = off
-	return c.Read(p)
+	return c.doRead(p)
 }
 
 func (c *ChunkStreamReader) Read(p []byte) (n int, err error) {
+	c.bufferLock.Lock()
+	defer c.bufferLock.Unlock()
+	return c.doRead(p)
+}
+
+func (c *ChunkStreamReader) doRead(p []byte) (n int, err error) {
+	// fmt.Printf("do read [%d,%d) at %s[%d,%d)\n", c.logicOffset, c.logicOffset+int64(len(p)), c.chunk, c.bufferOffset, c.bufferOffset+int64(len(c.buffer)))
 	for n < len(p) {
-		if c.isBufferEmpty() {
-			if c.nextChunkViewIndex >= len(c.chunkViews) {
-				return n, io.EOF
-			}
-			chunkView := c.chunkViews[c.nextChunkViewIndex]
-			if err = c.fetchChunkToBuffer(chunkView); err != nil {
-				return
-			}
-			c.nextChunkViewIndex++
+		// println("read", c.logicOffset)
+		if err = c.prepareBufferFor(c.logicOffset); err != nil {
+			return
 		}
-		t := copy(p[n:], c.buffer[c.bufferPos:])
-		c.bufferPos += t
+		t := copy(p[n:], c.buffer[c.logicOffset-c.bufferOffset:])
 		n += t
 		c.logicOffset += int64(t)
 	}
@@ -203,10 +206,12 @@ func (c *ChunkStreamReader) Read(p []byte) (n int, err error) {
 }
 
 func (c *ChunkStreamReader) isBufferEmpty() bool {
-	return len(c.buffer) <= c.bufferPos
+	return len(c.buffer) <= int(c.logicOffset-c.bufferOffset)
 }
 
 func (c *ChunkStreamReader) Seek(offset int64, whence int) (int64, error) {
+	c.bufferLock.Lock()
+	defer c.bufferLock.Unlock()
 
 	var err error
 	switch whence {
@@ -226,48 +231,59 @@ func (c *ChunkStreamReader) Seek(offset int64, whence int) (int64, error) {
 
 }
 
+func insideChunk(offset int64, chunk *ChunkView) bool {
+	return chunk.LogicOffset <= offset && offset < chunk.LogicOffset+int64(chunk.Size)
+}
+
 func (c *ChunkStreamReader) prepareBufferFor(offset int64) (err error) {
 	// stay in the same chunk
-	if !c.isBufferEmpty() {
-		if c.bufferOffset <= offset && offset < c.bufferOffset+int64(len(c.buffer)) {
-			c.bufferPos = int(offset - c.bufferOffset)
-			return nil
-		}
+	if c.bufferOffset <= offset && offset < c.bufferOffset+int64(len(c.buffer)) {
+		return nil
 	}
 
+	// fmt.Printf("fetch for offset %d\n", offset)
+
 	// need to seek to a different chunk
 	currentChunkIndex := sort.Search(len(c.chunkViews), func(i int) bool {
 		return offset < c.chunkViews[i].LogicOffset
 	})
 	if currentChunkIndex == len(c.chunkViews) {
 		// not found
-		if c.chunkViews[0].LogicOffset <= offset {
+		if insideChunk(offset, c.chunkViews[0]) {
+			// fmt.Printf("select0 chunk %d %s\n", currentChunkIndex, c.chunkViews[currentChunkIndex].FileId)
 			currentChunkIndex = 0
-		} else if c.chunkViews[len(c.chunkViews)-1].LogicOffset <= offset {
-			currentChunkIndex = len(c.chunkViews) -1
+		} else if insideChunk(offset, c.chunkViews[len(c.chunkViews)-1]) {
+			currentChunkIndex = len(c.chunkViews) - 1
+			// fmt.Printf("select last chunk %d %s\n", currentChunkIndex, c.chunkViews[currentChunkIndex].FileId)
 		} else {
 			return io.EOF
 		}
 	} else if currentChunkIndex > 0 {
-		if c.chunkViews[currentChunkIndex-1].LogicOffset <= offset {
+		if insideChunk(offset, c.chunkViews[currentChunkIndex]) {
+			// good hit
+		} else if insideChunk(offset, c.chunkViews[currentChunkIndex-1]) {
 			currentChunkIndex -= 1
+			// fmt.Printf("select -1 chunk %d %s\n", currentChunkIndex, c.chunkViews[currentChunkIndex].FileId)
 		} else {
+			// glog.Fatalf("unexpected1 offset %d", offset)
 			return fmt.Errorf("unexpected1 offset %d", offset)
 		}
 	} else {
+		// glog.Fatalf("unexpected2 offset %d", offset)
 		return fmt.Errorf("unexpected2 offset %d", offset)
 	}
 
 	// positioning within the new chunk
 	chunk := c.chunkViews[currentChunkIndex]
-	if chunk.LogicOffset <= offset && offset < chunk.LogicOffset+int64(chunk.Size) {
+	if insideChunk(offset, chunk) {
 		if c.isBufferEmpty() || c.bufferOffset != chunk.LogicOffset {
 			if err = c.fetchChunkToBuffer(chunk); err != nil {
 				return
 			}
-			c.nextChunkViewIndex = currentChunkIndex + 1
 		}
-		c.bufferPos = int(offset - c.bufferOffset)
+	} else {
+		// glog.Fatalf("unexpected3 offset %d in %s [%d,%d)", offset, chunk.FileId, chunk.LogicOffset, chunk.LogicOffset+int64(chunk.Size))
+		return fmt.Errorf("unexpected3 offset %d in %s [%d,%d)", offset, chunk.FileId, chunk.LogicOffset, chunk.LogicOffset+int64(chunk.Size))
 	}
 	return
 }
@@ -298,10 +314,10 @@ func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
 		return err
 	}
 	c.buffer = buffer.Bytes()
-	c.bufferPos = 0
 	c.bufferOffset = chunkView.LogicOffset
+	c.chunk = chunkView.FileId
 
-	// glog.V(0).Infof("read %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size))
+	// glog.V(0).Infof("fetched %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size))
 
 	return nil
 }

weed/filer/tikv/tikv.go

@@ -0,0 +1,5 @@
+package tikv
+
+/*
+ * This empty file is let go build can work without tikv tag
+ */

weed/filer/tikv/tikv_store.go

@@ -0,0 +1,389 @@
+//go:build tikv
+// +build tikv
+
+package tikv
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha1"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/tikv/client-go/v2/tikv"
+	"github.com/tikv/client-go/v2/txnkv"
+)
+
+var (
+	_ filer.FilerStore = ((*TikvStore)(nil))
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &TikvStore{})
+}
+
+type TikvStore struct {
+	client                 *tikv.KVStore
+	deleteRangeConcurrency int
+}
+
+// Basic APIs
+func (store *TikvStore) GetName() string {
+	return "tikv"
+}
+
+func (store *TikvStore) Initialize(config util.Configuration, prefix string) error {
+	pdAddrs := []string{}
+	pdAddrsStr := config.GetString(prefix + "pdaddrs")
+	for _, item := range strings.Split(pdAddrsStr, ",") {
+		pdAddrs = append(pdAddrs, strings.TrimSpace(item))
+	}
+	drc := config.GetInt(prefix + "deleterange_concurrency")
+	if drc <= 0 {
+		drc = 1
+	}
+	store.deleteRangeConcurrency = drc
+	return store.initialize(pdAddrs)
+}
+
+func (store *TikvStore) initialize(pdAddrs []string) error {
+	client, err := tikv.NewTxnClient(pdAddrs)
+	store.client = client
+	return err
+}
+
+func (store *TikvStore) Shutdown() {
+	err := store.client.Close()
+	if err != nil {
+		glog.V(0).Infof("Shutdown TiKV client got error: %v", err)
+	}
+}
+
+// ~ Basic APIs
+
+// Entry APIs
+func (store *TikvStore) InsertEntry(ctx context.Context, entry *filer.Entry) error {
+	dir, name := entry.DirAndName()
+	key := generateKey(dir, name)
+
+	value, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
+	}
+	txn, err := store.getTxn(ctx)
+	if err != nil {
+		return err
+	}
+	err = txn.RunInTxn(func(txn *txnkv.KVTxn) error {
+		return txn.Set(key, value)
+	})
+	if err != nil {
+		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
+	}
+	return nil
+}
+
+func (store *TikvStore) UpdateEntry(ctx context.Context, entry *filer.Entry) error {
+	return store.InsertEntry(ctx, entry)
+}
+
+func (store *TikvStore) FindEntry(ctx context.Context, path util.FullPath) (*filer.Entry, error) {
+	dir, name := path.DirAndName()
+	key := generateKey(dir, name)
+
+	txn, err := store.getTxn(ctx)
+	if err != nil {
+		return nil, err
+	}
+	var value []byte = nil
+	err = txn.RunInTxn(func(txn *txnkv.KVTxn) error {
+		val, err := txn.Get(context.TODO(), key)
+		if err == nil {
+			value = val
+		}
+		return err
+	})
+
+	if isNotExists(err) || value == nil {
+		return nil, filer_pb.ErrNotFound
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("get %s : %v", path, err)
+	}
+
+	entry := &filer.Entry{
+		FullPath: path,
+	}
+	err = entry.DecodeAttributesAndChunks(value)
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+	return entry, nil
+}
+
+func (store *TikvStore) DeleteEntry(ctx context.Context, path util.FullPath) error {
+	dir, name := path.DirAndName()
+	key := generateKey(dir, name)
+
+	txn, err := store.getTxn(ctx)
+	if err != nil {
+		return err
+	}
+
+	err = txn.RunInTxn(func(txn *txnkv.KVTxn) error {
+		return txn.Delete(key)
+	})
+	if err != nil {
+		return fmt.Errorf("delete %s : %v", path, err)
+	}
+	return nil
+}
+
+// ~ Entry APIs
+
+// Directory APIs
+func (store *TikvStore) DeleteFolderChildren(ctx context.Context, path util.FullPath) error {
+	directoryPrefix := genDirectoryKeyPrefix(path, "")
+
+	txn, err := store.getTxn(ctx)
+	if err != nil {
+		return err
+	}
+	var (
+		startKey []byte = nil
+		endKey   []byte = nil
+	)
+	err = txn.RunInTxn(func(txn *txnkv.KVTxn) error {
+		iter, err := txn.Iter(directoryPrefix, nil)
+		if err != nil {
+			return err
+		}
+		defer iter.Close()
+		for iter.Valid() {
+			key := iter.Key()
+			endKey = key
+			if !bytes.HasPrefix(key, directoryPrefix) {
+				break
+			}
+			if startKey == nil {
+				startKey = key
+			}
+
+			err = iter.Next()
+			if err != nil {
+				return err
+			}
+		}
+		// Only one Key matched just delete it.
+		if startKey != nil && bytes.Equal(startKey, endKey) {
+			return txn.Delete(startKey)
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("delete %s : %v", path, err)
+	}
+
+	if startKey != nil && endKey != nil && !bytes.Equal(startKey, endKey) {
+		// has startKey and endKey and they are not equals, so use delete range
+		_, err = store.client.DeleteRange(context.Background(), startKey, endKey, store.deleteRangeConcurrency)
+		if err != nil {
+			return fmt.Errorf("delete %s : %v", path, err)
+		}
+	}
+	return err
+}
+
+func (store *TikvStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (string, error) {
+	return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc)
+}
+
+func (store *TikvStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (string, error) {
+	lastFileName := ""
+	directoryPrefix := genDirectoryKeyPrefix(dirPath, prefix)
+	lastFileStart := directoryPrefix
+	if startFileName != "" {
+		lastFileStart = genDirectoryKeyPrefix(dirPath, startFileName)
+	}
+
+	txn, err := store.getTxn(ctx)
+	if err != nil {
+		return lastFileName, err
+	}
+	err = txn.RunInTxn(func(txn *txnkv.KVTxn) error {
+		iter, err := txn.Iter(lastFileStart, nil)
+		if err != nil {
+			return err
+		}
+		defer iter.Close()
+		i := int64(0)
+		first := true
+		for iter.Valid() {
+			if first {
+				first = false
+				if !includeStartFile {
+					if iter.Valid() {
+						// Check first item is lastFileStart
+						if bytes.Equal(iter.Key(), lastFileStart) {
+							// Is lastFileStart and not include start file, just
+							// ignore it.
+							err = iter.Next()
+							if err != nil {
+								return err
+							}
+							continue
+						}
+					}
+				}
+			}
+			// Check for limitation
+			if limit > 0 {
+				i++
+				if i > limit {
+					break
+				}
+			}
+			// Validate key prefix
+			key := iter.Key()
+			if !bytes.HasPrefix(key, directoryPrefix) {
+				break
+			}
+			value := iter.Value()
+
+			// Start process
+			fileName := getNameFromKey(key)
+			if fileName != "" {
+				// Got file name, then generate the Entry
+				entry := &filer.Entry{
+					FullPath: util.NewFullPath(string(dirPath), fileName),
+				}
+				// Update lastFileName
+				lastFileName = fileName
+				// Check for decode value.
+				if decodeErr := entry.DecodeAttributesAndChunks(value); decodeErr != nil {
+					// Got error just return the error
+					glog.V(0).Infof("list %s : %v", entry.FullPath, err)
+					return err
+				}
+				// Run for each callback if return false just break the iteration
+				if !eachEntryFunc(entry) {
+					break
+				}
+			}
+			// End process
+
+			err = iter.Next()
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return lastFileName, fmt.Errorf("prefix list %s : %v", dirPath, err)
+	}
+	return lastFileName, nil
+}
+
+// ~ Directory APIs
+
+// Transaction Related APIs
+func (store *TikvStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	tx, err := store.client.Begin()
+	if err != nil {
+		return ctx, err
+	}
+	return context.WithValue(ctx, "tx", tx), nil
+}
+
+func (store *TikvStore) CommitTransaction(ctx context.Context) error {
+	if tx, ok := ctx.Value("tx").(*txnkv.KVTxn); ok {
+		return tx.Commit(context.Background())
+	}
+	return nil
+}
+
+func (store *TikvStore) RollbackTransaction(ctx context.Context) error {
+	if tx, ok := ctx.Value("tx").(*txnkv.KVTxn); ok {
+		return tx.Rollback()
+	}
+	return nil
+}
+
+// ~ Transaction Related APIs
+
+// Transaction Wrapper
+type TxnWrapper struct {
+	*txnkv.KVTxn
+	inContext bool
+}
+
+func (w *TxnWrapper) RunInTxn(f func(txn *txnkv.KVTxn) error) error {
+	err := f(w.KVTxn)
+	if !w.inContext {
+		if err != nil {
+			w.KVTxn.Rollback()
+			return err
+		}
+		w.KVTxn.Commit(context.Background())
+		return nil
+	}
+	return err
+}
+
+func (store *TikvStore) getTxn(ctx context.Context) (*TxnWrapper, error) {
+	if tx, ok := ctx.Value("tx").(*txnkv.KVTxn); ok {
+		return &TxnWrapper{tx, true}, nil
+	}
+	txn, err := store.client.Begin()
+	if err != nil {
+		return nil, err
+	}
+	return &TxnWrapper{txn, false}, nil
+}
+
+// ~ Transaction Wrapper
+
+// Encoding Functions
+func hashToBytes(dir string) []byte {
+	h := sha1.New()
+	io.WriteString(h, dir)
+	b := h.Sum(nil)
+	return b
+}
+
+func generateKey(dirPath, fileName string) []byte {
+	key := hashToBytes(dirPath)
+	key = append(key, []byte(fileName)...)
+	return key
+}
+
+func getNameFromKey(key []byte) string {
+	return string(key[sha1.Size:])
+}
+
+func genDirectoryKeyPrefix(fullpath util.FullPath, startFileName string) (keyPrefix []byte) {
+	keyPrefix = hashToBytes(string(fullpath))
+	if len(startFileName) > 0 {
+		keyPrefix = append(keyPrefix, []byte(startFileName)...)
+	}
+	return keyPrefix
+}
+
+func isNotExists(err error) bool {
+	if err == nil {
+		return false
+	}
+	if err.Error() == "not exist" {
+		return true
+	}
+	return false
+}
+
+// ~ Encoding Functions

weed/filer/tikv/tikv_store_kv.go

@@ -0,0 +1,50 @@
+//go:build tikv
+// +build tikv
+
+package tikv
+
+import (
+	"context"
+
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/tikv/client-go/v2/txnkv"
+)
+
+func (store *TikvStore) KvPut(ctx context.Context, key []byte, value []byte) error {
+	tw, err := store.getTxn(ctx)
+	if err != nil {
+		return err
+	}
+	return tw.RunInTxn(func(txn *txnkv.KVTxn) error {
+		return txn.Set(key, value)
+	})
+}
+
+func (store *TikvStore) KvGet(ctx context.Context, key []byte) ([]byte, error) {
+	tw, err := store.getTxn(ctx)
+	if err != nil {
+		return nil, err
+	}
+	var data []byte = nil
+	err = tw.RunInTxn(func(txn *txnkv.KVTxn) error {
+		val, err := txn.Get(context.TODO(), key)
+		if err == nil {
+			data = val
+		}
+		return err
+	})
+	if isNotExists(err) {
+		return data, filer.ErrKvNotFound
+	}
+	return data, err
+}
+
+func (store *TikvStore) KvDelete(ctx context.Context, key []byte) error {
+	tw, err := store.getTxn(ctx)
+	if err != nil {
+		return err
+	}
+	return tw.RunInTxn(func(txn *txnkv.KVTxn) error {
+		return txn.Delete(key)
+	})
+}

weed/filesys/meta_cache/meta_cache_subscribe.go

@@ -39,11 +39,9 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil
 		err := mc.AtomicUpdateEntryFromFiler(context.Background(), oldPath, newEntry)
 		if err == nil {
 			if message.OldEntry != nil && message.NewEntry != nil {
-				if message.OldEntry.Name == message.NewEntry.Name {
-					// no need to invalidate
-				} else {
-					oldKey := util.NewFullPath(resp.Directory, message.OldEntry.Name)
-					mc.invalidateFunc(oldKey)
+				oldKey := util.NewFullPath(resp.Directory, message.OldEntry.Name)
+				mc.invalidateFunc(oldKey)
+				if message.OldEntry.Name != message.NewEntry.Name {
 					newKey := util.NewFullPath(dir, message.NewEntry.Name)
 					mc.invalidateFunc(newKey)
 				}

weed/pb/Makefile

@@ -6,6 +6,7 @@ gen:
 	protoc master.proto --go_out=plugins=grpc:./master_pb --go_opt=paths=source_relative
 	protoc volume_server.proto --go_out=plugins=grpc:./volume_server_pb --go_opt=paths=source_relative
 	protoc filer.proto --go_out=plugins=grpc:./filer_pb --go_opt=paths=source_relative
+	protoc remote.proto --go_out=plugins=grpc:./remote_pb --go_opt=paths=source_relative
 	protoc iam.proto --go_out=plugins=grpc:./iam_pb --go_opt=paths=source_relative
 	protoc messaging.proto --go_out=plugins=grpc:./messaging_pb --go_opt=paths=source_relative
 	# protoc filer.proto --java_out=../../other/java/client/src/main/java

weed/pb/filer.proto

@@ -305,6 +305,7 @@ message GetFilerConfigurationResponse {
     string metrics_address = 9;
     int32 metrics_interval_sec = 10;
     string version = 11;
+    string cluster_id = 12;
 }
 
 message SubscribeMetadataRequest {
@@ -312,6 +313,7 @@ message SubscribeMetadataRequest {
     string path_prefix = 2;
     int64 since_ns = 3;
     int32 signature = 4;
+    repeated string path_prefixes = 6;
 }
 message SubscribeMetadataResponse {
     string directory = 1;
@@ -336,6 +338,7 @@ message KeepConnectedResponse {
 message LocateBrokerRequest {
     string resource = 1;
 }
+
 message LocateBrokerResponse {
     bool found = 1;
     // if found, send the exact address
@@ -386,24 +389,6 @@ message FilerConf {
 /////////////////////////
 // Remote Storage related
 /////////////////////////
-message RemoteConf {
-    string type = 1;
-    string name = 2;
-    string s3_access_key = 4;
-    string s3_secret_key = 5;
-    string s3_region = 6;
-    string s3_endpoint = 7;
-    string gcs_google_application_credentials = 10;
-}
-
-message RemoteStorageMapping {
-    map<string,RemoteStorageLocation> mappings = 1;
-}
-message RemoteStorageLocation {
-    string name = 1;
-    string bucket = 2;
-    string path = 3;
-}
 message DownloadToLocalRequest {
     string directory = 1;
     string name = 2;

weed/pb/filer_pb/filer.pb.go

@@ -2404,6 +2404,7 @@ type GetFilerConfigurationResponse struct {
 	MetricsAddress     string   `protobuf:"bytes,9,opt,name=metrics_address,json=metricsAddress,proto3" json:"metrics_address,omitempty"`
 	MetricsIntervalSec int32    `protobuf:"varint,10,opt,name=metrics_interval_sec,json=metricsIntervalSec,proto3" json:"metrics_interval_sec,omitempty"`
 	Version            string   `protobuf:"bytes,11,opt,name=version,proto3" json:"version,omitempty"`
+	ClusterId          string   `protobuf:"bytes,12,opt,name=cluster_id,json=clusterId,proto3" json:"cluster_id,omitempty"`
 }
 
 func (x *GetFilerConfigurationResponse) Reset() {
@@ -2508,15 +2509,23 @@ func (x *GetFilerConfigurationResponse) GetVersion() string {
 	return ""
 }
 
+func (x *GetFilerConfigurationResponse) GetClusterId() string {
+	if x != nil {
+		return x.ClusterId
+	}
+	return ""
+}
+
 type SubscribeMetadataRequest struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	ClientName string `protobuf:"bytes,1,opt,name=client_name,json=clientName,proto3" json:"client_name,omitempty"`
-	PathPrefix string `protobuf:"bytes,2,opt,name=path_prefix,json=pathPrefix,proto3" json:"path_prefix,omitempty"`
-	SinceNs    int64  `protobuf:"varint,3,opt,name=since_ns,json=sinceNs,proto3" json:"since_ns,omitempty"`
-	Signature  int32  `protobuf:"varint,4,opt,name=signature,proto3" json:"signature,omitempty"`
+	ClientName   string   `protobuf:"bytes,1,opt,name=client_name,json=clientName,proto3" json:"client_name,omitempty"`
+	PathPrefix   string   `protobuf:"bytes,2,opt,name=path_prefix,json=pathPrefix,proto3" json:"path_prefix,omitempty"`
+	SinceNs      int64    `protobuf:"varint,3,opt,name=since_ns,json=sinceNs,proto3" json:"since_ns,omitempty"`
+	Signature    int32    `protobuf:"varint,4,opt,name=signature,proto3" json:"signature,omitempty"`
+	PathPrefixes []string `protobuf:"bytes,6,rep,name=path_prefixes,json=pathPrefixes,proto3" json:"path_prefixes,omitempty"`
 }
 
 func (x *SubscribeMetadataRequest) Reset() {
@@ -2579,6 +2588,13 @@ func (x *SubscribeMetadataRequest) GetSignature() int32 {
 	return 0
 }
 
+func (x *SubscribeMetadataRequest) GetPathPrefixes() []string {
+	if x != nil {
+		return x.PathPrefixes
+	}
+	return nil
+}
+
 type SubscribeMetadataResponse struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -3176,211 +3192,6 @@ func (x *FilerConf) GetLocations() []*FilerConf_PathConf {
 /////////////////////////
 // Remote Storage related
 /////////////////////////
-type RemoteConf struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Type                            string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"`
-	Name                            string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
-	S3AccessKey                     string `protobuf:"bytes,4,opt,name=s3_access_key,json=s3AccessKey,proto3" json:"s3_access_key,omitempty"`
-	S3SecretKey                     string `protobuf:"bytes,5,opt,name=s3_secret_key,json=s3SecretKey,proto3" json:"s3_secret_key,omitempty"`
-	S3Region                        string `protobuf:"bytes,6,opt,name=s3_region,json=s3Region,proto3" json:"s3_region,omitempty"`
-	S3Endpoint                      string `protobuf:"bytes,7,opt,name=s3_endpoint,json=s3Endpoint,proto3" json:"s3_endpoint,omitempty"`
-	GcsGoogleApplicationCredentials string `protobuf:"bytes,10,opt,name=gcs_google_application_credentials,json=gcsGoogleApplicationCredentials,proto3" json:"gcs_google_application_credentials,omitempty"`
-}
-
-func (x *RemoteConf) Reset() {
-	*x = RemoteConf{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_filer_proto_msgTypes[49]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *RemoteConf) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*RemoteConf) ProtoMessage() {}
-
-func (x *RemoteConf) ProtoReflect() protoreflect.Message {
-	mi := &file_filer_proto_msgTypes[49]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use RemoteConf.ProtoReflect.Descriptor instead.
-func (*RemoteConf) Descriptor() ([]byte, []int) {
-	return file_filer_proto_rawDescGZIP(), []int{49}
-}
-
-func (x *RemoteConf) GetType() string {
-	if x != nil {
-		return x.Type
-	}
-	return ""
-}
-
-func (x *RemoteConf) GetName() string {
-	if x != nil {
-		return x.Name
-	}
-	return ""
-}
-
-func (x *RemoteConf) GetS3AccessKey() string {
-	if x != nil {
-		return x.S3AccessKey
-	}
-	return ""
-}
-
-func (x *RemoteConf) GetS3SecretKey() string {
-	if x != nil {
-		return x.S3SecretKey
-	}
-	return ""
-}
-
-func (x *RemoteConf) GetS3Region() string {
-	if x != nil {
-		return x.S3Region
-	}
-	return ""
-}
-
-func (x *RemoteConf) GetS3Endpoint() string {
-	if x != nil {
-		return x.S3Endpoint
-	}
-	return ""
-}
-
-func (x *RemoteConf) GetGcsGoogleApplicationCredentials() string {
-	if x != nil {
-		return x.GcsGoogleApplicationCredentials
-	}
-	return ""
-}
-
-type RemoteStorageMapping struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Mappings map[string]*RemoteStorageLocation `protobuf:"bytes,1,rep,name=mappings,proto3" json:"mappings,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
-}
-
-func (x *RemoteStorageMapping) Reset() {
-	*x = RemoteStorageMapping{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_filer_proto_msgTypes[50]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *RemoteStorageMapping) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*RemoteStorageMapping) ProtoMessage() {}
-
-func (x *RemoteStorageMapping) ProtoReflect() protoreflect.Message {
-	mi := &file_filer_proto_msgTypes[50]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use RemoteStorageMapping.ProtoReflect.Descriptor instead.
-func (*RemoteStorageMapping) Descriptor() ([]byte, []int) {
-	return file_filer_proto_rawDescGZIP(), []int{50}
-}
-
-func (x *RemoteStorageMapping) GetMappings() map[string]*RemoteStorageLocation {
-	if x != nil {
-		return x.Mappings
-	}
-	return nil
-}
-
-type RemoteStorageLocation struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Name   string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"`
-	Path   string `protobuf:"bytes,3,opt,name=path,proto3" json:"path,omitempty"`
-}
-
-func (x *RemoteStorageLocation) Reset() {
-	*x = RemoteStorageLocation{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_filer_proto_msgTypes[51]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *RemoteStorageLocation) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*RemoteStorageLocation) ProtoMessage() {}
-
-func (x *RemoteStorageLocation) ProtoReflect() protoreflect.Message {
-	mi := &file_filer_proto_msgTypes[51]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use RemoteStorageLocation.ProtoReflect.Descriptor instead.
-func (*RemoteStorageLocation) Descriptor() ([]byte, []int) {
-	return file_filer_proto_rawDescGZIP(), []int{51}
-}
-
-func (x *RemoteStorageLocation) GetName() string {
-	if x != nil {
-		return x.Name
-	}
-	return ""
-}
-
-func (x *RemoteStorageLocation) GetBucket() string {
-	if x != nil {
-		return x.Bucket
-	}
-	return ""
-}
-
-func (x *RemoteStorageLocation) GetPath() string {
-	if x != nil {
-		return x.Path
-	}
-	return ""
-}
-
 type DownloadToLocalRequest struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -3393,7 +3204,7 @@ type DownloadToLocalRequest struct {
 func (x *DownloadToLocalRequest) Reset() {
 	*x = DownloadToLocalRequest{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_filer_proto_msgTypes[52]
+		mi := &file_filer_proto_msgTypes[49]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -3406,7 +3217,7 @@ func (x *DownloadToLocalRequest) String() string {
 func (*DownloadToLocalRequest) ProtoMessage() {}
 
 func (x *DownloadToLocalRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_filer_proto_msgTypes[52]
+	mi := &file_filer_proto_msgTypes[49]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -3419,7 +3230,7 @@ func (x *DownloadToLocalRequest) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use DownloadToLocalRequest.ProtoReflect.Descriptor instead.
 func (*DownloadToLocalRequest) Descriptor() ([]byte, []int) {
-	return file_filer_proto_rawDescGZIP(), []int{52}
+	return file_filer_proto_rawDescGZIP(), []int{49}
 }
 
 func (x *DownloadToLocalRequest) GetDirectory() string {
@@ -3447,7 +3258,7 @@ type DownloadToLocalResponse struct {
 func (x *DownloadToLocalResponse) Reset() {
 	*x = DownloadToLocalResponse{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_filer_proto_msgTypes[53]
+		mi := &file_filer_proto_msgTypes[50]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -3460,7 +3271,7 @@ func (x *DownloadToLocalResponse) String() string {
 func (*DownloadToLocalResponse) ProtoMessage() {}
 
 func (x *DownloadToLocalResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_filer_proto_msgTypes[53]
+	mi := &file_filer_proto_msgTypes[50]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -3473,7 +3284,7 @@ func (x *DownloadToLocalResponse) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use DownloadToLocalResponse.ProtoReflect.Descriptor instead.
 func (*DownloadToLocalResponse) Descriptor() ([]byte, []int) {
-	return file_filer_proto_rawDescGZIP(), []int{53}
+	return file_filer_proto_rawDescGZIP(), []int{50}
 }
 
 func (x *DownloadToLocalResponse) GetEntry() *Entry {
@@ -3497,7 +3308,7 @@ type LocateBrokerResponse_Resource struct {
 func (x *LocateBrokerResponse_Resource) Reset() {
 	*x = LocateBrokerResponse_Resource{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_filer_proto_msgTypes[56]
+		mi := &file_filer_proto_msgTypes[53]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -3510,7 +3321,7 @@ func (x *LocateBrokerResponse_Resource) String() string {
 func (*LocateBrokerResponse_Resource) ProtoMessage() {}
 
 func (x *LocateBrokerResponse_Resource) ProtoReflect() protoreflect.Message {
-	mi := &file_filer_proto_msgTypes[56]
+	mi := &file_filer_proto_msgTypes[53]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -3558,7 +3369,7 @@ type FilerConf_PathConf struct {
 func (x *FilerConf_PathConf) Reset() {
 	*x = FilerConf_PathConf{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_filer_proto_msgTypes[57]
+		mi := &file_filer_proto_msgTypes[54]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -3571,7 +3382,7 @@ func (x *FilerConf_PathConf) String() string {
 func (*FilerConf_PathConf) ProtoMessage() {}
 
 func (x *FilerConf_PathConf) ProtoReflect() protoreflect.Message {
-	mi := &file_filer_proto_msgTypes[57]
+	mi := &file_filer_proto_msgTypes[54]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -3955,7 +3766,7 @@ var file_filer_proto_rawDesc = []byte{
 	0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04,
 	0x52, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x1e, 0x0a, 0x1c, 0x47,
 	0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61,
-	0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xde, 0x02, 0x0a, 0x1d,
+	0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xfd, 0x02, 0x0a, 0x1d,
 	0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72,
 	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a,
 	0x07, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07,
@@ -3977,126 +3788,96 @@ var file_filer_proto_rawDesc = []byte{
 	0x72, 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12,
 	0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x53,
 	0x65, 0x63, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20,
-	0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x95, 0x01, 0x0a,
-	0x18, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61,
-	0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69,
-	0x65, 0x6e, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a,
-	0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x61,
-	0x74, 0x68, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x0a, 0x70, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x19, 0x0a, 0x08, 0x73,
-	0x69, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x73,
-	0x69, 0x6e, 0x63, 0x65, 0x4e, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74,
-	0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61,
-	0x74, 0x75, 0x72, 0x65, 0x22, 0x9a, 0x01, 0x0a, 0x19, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69,
-	0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
-	0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18,
-	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79,
-	0x12, 0x4a, 0x0a, 0x12, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69,
-	0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x66,
-	0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x74,
-	0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x11, 0x65, 0x76, 0x65, 0x6e, 0x74,
-	0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x13, 0x0a, 0x05,
-	0x74, 0x73, 0x5f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x74, 0x73, 0x4e,
-	0x73, 0x22, 0x61, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x13, 0x0a,
-	0x05, 0x74, 0x73, 0x5f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x74, 0x73,
-	0x4e, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f,
-	0x6b, 0x65, 0x79, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10,
-	0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x48, 0x61, 0x73, 0x68,
-	0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04,
-	0x64, 0x61, 0x74, 0x61, 0x22, 0x65, 0x0a, 0x14, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e,
-	0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04,
-	0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65,
-	0x12, 0x1b, 0x0a, 0x09, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20,
-	0x01, 0x28, 0x0d, 0x52, 0x08, 0x67, 0x72, 0x70, 0x63, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1c, 0x0a,
-	0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09,
-	0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x22, 0x17, 0x0a, 0x15, 0x4b,
-	0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70,
-	0x6f, 0x6e, 0x73, 0x65, 0x22, 0x31, 0x0a, 0x13, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, 0x72,
-	0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72,
-	0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72,
-	0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0xcd, 0x01, 0x0a, 0x14, 0x4c, 0x6f, 0x63, 0x61,
-	0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52,
-	0x05, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x12, 0x45, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72,
-	0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x66, 0x69, 0x6c, 0x65,
-	0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65,
-	0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72,
-	0x63, 0x65, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x1a, 0x58, 0x0a,
-	0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x67, 0x72, 0x70,
-	0x63, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x0d, 0x67, 0x72, 0x70, 0x63, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73,
-	0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x75,
-	0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72,
-	0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x20, 0x0a, 0x0c, 0x4b, 0x76, 0x47, 0x65, 0x74,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01,
-	0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x3b, 0x0a, 0x0d, 0x4b, 0x76, 0x47,
-	0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61,
-	0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65,
-	0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x36, 0x0a, 0x0c, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20,
-	0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75,
-	0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x25,
-	0x0a, 0x0d, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
-	0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05,
-	0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xeb, 0x02, 0x0a, 0x09, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43,
-	0x6f, 0x6e, 0x66, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01,
-	0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x3a, 0x0a,
-	0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b,
-	0x32, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x6c, 0x65,
-	0x72, 0x43, 0x6f, 0x6e, 0x66, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x52, 0x09,
-	0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x87, 0x02, 0x0a, 0x08, 0x50, 0x61,
-	0x74, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x27, 0x0a, 0x0f, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69,
-	0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12,
-	0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20,
-	0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12,
-	0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f,
-	0x6e, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03,
-	0x74, 0x74, 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65,
-	0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65,
-	0x12, 0x14, 0x0a, 0x05, 0x66, 0x73, 0x79, 0x6e, 0x63, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52,
-	0x05, 0x66, 0x73, 0x79, 0x6e, 0x63, 0x12, 0x2e, 0x0a, 0x13, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65,
-	0x5f, 0x67, 0x72, 0x6f, 0x77, 0x74, 0x68, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x07, 0x20,
-	0x01, 0x28, 0x0d, 0x52, 0x11, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x47, 0x72, 0x6f, 0x77, 0x74,
-	0x68, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x6f,
-	0x6e, 0x6c, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x72, 0x65, 0x61, 0x64, 0x4f,
-	0x6e, 0x6c, 0x79, 0x22, 0x87, 0x02, 0x0a, 0x0a, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f,
-	0x6e, 0x66, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x22, 0x0a, 0x0d, 0x73, 0x33,
-	0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x0b, 0x73, 0x33, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x22,
-	0x0a, 0x0d, 0x73, 0x33, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18,
-	0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x33, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b,
-	0x65, 0x79, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x33, 0x5f, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18,
-	0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x33, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12,
-	0x1f, 0x0a, 0x0b, 0x73, 0x33, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x07,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x33, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74,
-	0x12, 0x4b, 0x0a, 0x22, 0x67, 0x63, 0x73, 0x5f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x5f, 0x61,
-	0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x72, 0x65, 0x64, 0x65,
-	0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1f, 0x67, 0x63,
-	0x73, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69,
-	0x6f, 0x6e, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x22, 0xbe, 0x01,
-	0x0a, 0x14, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d,
-	0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x48, 0x0a, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e,
-	0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72,
-	0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67,
-	0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67,
-	0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73,
-	0x1a, 0x5c, 0x0a, 0x0d, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72,
-	0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03,
-	0x6b, 0x65, 0x79, 0x12, 0x35, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01,
-	0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65,
-	0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74,
-	0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x57,
-	0x0a, 0x15, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c,
-	0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18,
-	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x62,
-	0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63,
-	0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x4a, 0x0a, 0x16, 0x44, 0x6f, 0x77, 0x6e, 0x6c,
+	0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1d, 0x0a, 0x0a,
+	0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x22, 0xba, 0x01, 0x0a, 0x18,
+	0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74,
+	0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65,
+	0x6e, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63,
+	0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x61, 0x74,
+	0x68, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a,
+	0x70, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x69,
+	0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x73, 0x69,
+	0x6e, 0x63, 0x65, 0x4e, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75,
+	0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74,
+	0x75, 0x72, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x70, 0x61, 0x74, 0x68, 0x5f, 0x70, 0x72, 0x65, 0x66,
+	0x69, 0x78, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x70, 0x61, 0x74, 0x68,
+	0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x22, 0x9a, 0x01, 0x0a, 0x19, 0x53, 0x75, 0x62,
+	0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74,
+	0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63,
+	0x74, 0x6f, 0x72, 0x79, 0x12, 0x4a, 0x0a, 0x12, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x6e, 0x6f,
+	0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b,
+	0x32, 0x1b, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x76, 0x65, 0x6e,
+	0x74, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x11, 0x65,
+	0x76, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e,
+	0x12, 0x13, 0x0a, 0x05, 0x74, 0x73, 0x5f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52,
+	0x04, 0x74, 0x73, 0x4e, 0x73, 0x22, 0x61, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72,
+	0x79, 0x12, 0x13, 0x0a, 0x05, 0x74, 0x73, 0x5f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03,
+	0x52, 0x04, 0x74, 0x73, 0x4e, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74,
+	0x69, 0x6f, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x05, 0x52, 0x10, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79,
+	0x48, 0x61, 0x73, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01,
+	0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x65, 0x0a, 0x14, 0x4b, 0x65, 0x65, 0x70,
+	0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04,
+	0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x70, 0x6f, 0x72,
+	0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x67, 0x72, 0x70, 0x63, 0x50, 0x6f, 0x72,
+	0x74, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x03,
+	0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x22,
+	0x17, 0x0a, 0x15, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x31, 0x0a, 0x13, 0x4c, 0x6f, 0x63, 0x61,
+	0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
+	0x1a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0xcd, 0x01, 0x0a, 0x14,
+	0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x12, 0x45, 0x0a, 0x09, 0x72, 0x65,
+	0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e,
+	0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42,
+	0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65,
+	0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
+	0x73, 0x1a, 0x58, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x25, 0x0a,
+	0x0e, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x67, 0x72, 0x70, 0x63, 0x41, 0x64, 0x64, 0x72, 0x65,
+	0x73, 0x73, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
+	0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x72, 0x65,
+	0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x20, 0x0a, 0x0c, 0x4b,
+	0x76, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b,
+	0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x3b, 0x0a,
+	0x0d, 0x4b, 0x76, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14,
+	0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76,
+	0x61, 0x6c, 0x75, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x36, 0x0a, 0x0c, 0x4b, 0x76,
+	0x50, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65,
+	0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05,
+	0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c,
+	0x75, 0x65, 0x22, 0x25, 0x0a, 0x0d, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xeb, 0x02, 0x0a, 0x09, 0x46, 0x69,
+	0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69,
+	0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f,
+	0x6e, 0x12, 0x3a, 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02,
+	0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
+	0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f,
+	0x6e, 0x66, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x87, 0x02,
+	0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x27, 0x0a, 0x0f, 0x6c, 0x6f,
+	0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x65,
+	0x66, 0x69, 0x78, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f,
+	0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74,
+	0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69,
+	0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63,
+	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x04, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f,
+	0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b,
+	0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x73, 0x79, 0x6e, 0x63, 0x18, 0x06, 0x20,
+	0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x73, 0x79, 0x6e, 0x63, 0x12, 0x2e, 0x0a, 0x13, 0x76, 0x6f,
+	0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x67, 0x72, 0x6f, 0x77, 0x74, 0x68, 0x5f, 0x63, 0x6f, 0x75, 0x6e,
+	0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x11, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x47,
+	0x72, 0x6f, 0x77, 0x74, 0x68, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65,
+	0x61, 0x64, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x72,
+	0x65, 0x61, 0x64, 0x4f, 0x6e, 0x6c, 0x79, 0x22, 0x4a, 0x0a, 0x16, 0x44, 0x6f, 0x77, 0x6e, 0x6c,
 	0x6f, 0x61, 0x64, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
 	0x74, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01,
 	0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12,
@@ -4233,7 +4014,7 @@ func file_filer_proto_rawDescGZIP() []byte {
 	return file_filer_proto_rawDescData
 }
 
-var file_filer_proto_msgTypes = make([]protoimpl.MessageInfo, 59)
+var file_filer_proto_msgTypes = make([]protoimpl.MessageInfo, 55)
 var file_filer_proto_goTypes = []interface{}{
 	(*LookupDirectoryEntryRequest)(nil),   // 0: filer_pb.LookupDirectoryEntryRequest
 	(*LookupDirectoryEntryResponse)(nil),  // 1: filer_pb.LookupDirectoryEntryResponse
@@ -4284,23 +4065,19 @@ var file_filer_proto_goTypes = []interface{}{
 	(*KvPutRequest)(nil),                  // 46: filer_pb.KvPutRequest
 	(*KvPutResponse)(nil),                 // 47: filer_pb.KvPutResponse
 	(*FilerConf)(nil),                     // 48: filer_pb.FilerConf
-	(*RemoteConf)(nil),                    // 49: filer_pb.RemoteConf
-	(*RemoteStorageMapping)(nil),          // 50: filer_pb.RemoteStorageMapping
-	(*RemoteStorageLocation)(nil),         // 51: filer_pb.RemoteStorageLocation
-	(*DownloadToLocalRequest)(nil),        // 52: filer_pb.DownloadToLocalRequest
-	(*DownloadToLocalResponse)(nil),       // 53: filer_pb.DownloadToLocalResponse
-	nil,                                   // 54: filer_pb.Entry.ExtendedEntry
-	nil,                                   // 55: filer_pb.LookupVolumeResponse.LocationsMapEntry
-	(*LocateBrokerResponse_Resource)(nil), // 56: filer_pb.LocateBrokerResponse.Resource
-	(*FilerConf_PathConf)(nil),            // 57: filer_pb.FilerConf.PathConf
-	nil,                                   // 58: filer_pb.RemoteStorageMapping.MappingsEntry
+	(*DownloadToLocalRequest)(nil),        // 49: filer_pb.DownloadToLocalRequest
+	(*DownloadToLocalResponse)(nil),       // 50: filer_pb.DownloadToLocalResponse
+	nil,                                   // 51: filer_pb.Entry.ExtendedEntry
+	nil,                                   // 52: filer_pb.LookupVolumeResponse.LocationsMapEntry
+	(*LocateBrokerResponse_Resource)(nil), // 53: filer_pb.LocateBrokerResponse.Resource
+	(*FilerConf_PathConf)(nil),            // 54: filer_pb.FilerConf.PathConf
 }
 var file_filer_proto_depIdxs = []int32{
 	5,  // 0: filer_pb.LookupDirectoryEntryResponse.entry:type_name -> filer_pb.Entry
 	5,  // 1: filer_pb.ListEntriesResponse.entry:type_name -> filer_pb.Entry
 	8,  // 2: filer_pb.Entry.chunks:type_name -> filer_pb.FileChunk
 	11, // 3: filer_pb.Entry.attributes:type_name -> filer_pb.FuseAttributes
-	54, // 4: filer_pb.Entry.extended:type_name -> filer_pb.Entry.ExtendedEntry
+	51, // 4: filer_pb.Entry.extended:type_name -> filer_pb.Entry.ExtendedEntry
 	4,  // 5: filer_pb.Entry.remote_entry:type_name -> filer_pb.RemoteEntry
 	5,  // 6: filer_pb.FullEntry.entry:type_name -> filer_pb.Entry
 	5,  // 7: filer_pb.EventNotification.old_entry:type_name -> filer_pb.Entry
@@ -4312,60 +4089,58 @@ var file_filer_proto_depIdxs = []int32{
 	5,  // 13: filer_pb.UpdateEntryRequest.entry:type_name -> filer_pb.Entry
 	8,  // 14: filer_pb.AppendToEntryRequest.chunks:type_name -> filer_pb.FileChunk
 	26, // 15: filer_pb.Locations.locations:type_name -> filer_pb.Location
-	55, // 16: filer_pb.LookupVolumeResponse.locations_map:type_name -> filer_pb.LookupVolumeResponse.LocationsMapEntry
+	52, // 16: filer_pb.LookupVolumeResponse.locations_map:type_name -> filer_pb.LookupVolumeResponse.LocationsMapEntry
 	28, // 17: filer_pb.CollectionListResponse.collections:type_name -> filer_pb.Collection
 	7,  // 18: filer_pb.SubscribeMetadataResponse.event_notification:type_name -> filer_pb.EventNotification
-	56, // 19: filer_pb.LocateBrokerResponse.resources:type_name -> filer_pb.LocateBrokerResponse.Resource
-	57, // 20: filer_pb.FilerConf.locations:type_name -> filer_pb.FilerConf.PathConf
-	58, // 21: filer_pb.RemoteStorageMapping.mappings:type_name -> filer_pb.RemoteStorageMapping.MappingsEntry
-	5,  // 22: filer_pb.DownloadToLocalResponse.entry:type_name -> filer_pb.Entry
-	25, // 23: filer_pb.LookupVolumeResponse.LocationsMapEntry.value:type_name -> filer_pb.Locations
-	51, // 24: filer_pb.RemoteStorageMapping.MappingsEntry.value:type_name -> filer_pb.RemoteStorageLocation
-	0,  // 25: filer_pb.SeaweedFiler.LookupDirectoryEntry:input_type -> filer_pb.LookupDirectoryEntryRequest
-	2,  // 26: filer_pb.SeaweedFiler.ListEntries:input_type -> filer_pb.ListEntriesRequest
-	12, // 27: filer_pb.SeaweedFiler.CreateEntry:input_type -> filer_pb.CreateEntryRequest
-	14, // 28: filer_pb.SeaweedFiler.UpdateEntry:input_type -> filer_pb.UpdateEntryRequest
-	16, // 29: filer_pb.SeaweedFiler.AppendToEntry:input_type -> filer_pb.AppendToEntryRequest
-	18, // 30: filer_pb.SeaweedFiler.DeleteEntry:input_type -> filer_pb.DeleteEntryRequest
-	20, // 31: filer_pb.SeaweedFiler.AtomicRenameEntry:input_type -> filer_pb.AtomicRenameEntryRequest
-	22, // 32: filer_pb.SeaweedFiler.AssignVolume:input_type -> filer_pb.AssignVolumeRequest
-	24, // 33: filer_pb.SeaweedFiler.LookupVolume:input_type -> filer_pb.LookupVolumeRequest
-	29, // 34: filer_pb.SeaweedFiler.CollectionList:input_type -> filer_pb.CollectionListRequest
-	31, // 35: filer_pb.SeaweedFiler.DeleteCollection:input_type -> filer_pb.DeleteCollectionRequest
-	33, // 36: filer_pb.SeaweedFiler.Statistics:input_type -> filer_pb.StatisticsRequest
-	35, // 37: filer_pb.SeaweedFiler.GetFilerConfiguration:input_type -> filer_pb.GetFilerConfigurationRequest
-	37, // 38: filer_pb.SeaweedFiler.SubscribeMetadata:input_type -> filer_pb.SubscribeMetadataRequest
-	37, // 39: filer_pb.SeaweedFiler.SubscribeLocalMetadata:input_type -> filer_pb.SubscribeMetadataRequest
-	40, // 40: filer_pb.SeaweedFiler.KeepConnected:input_type -> filer_pb.KeepConnectedRequest
-	42, // 41: filer_pb.SeaweedFiler.LocateBroker:input_type -> filer_pb.LocateBrokerRequest
-	44, // 42: filer_pb.SeaweedFiler.KvGet:input_type -> filer_pb.KvGetRequest
-	46, // 43: filer_pb.SeaweedFiler.KvPut:input_type -> filer_pb.KvPutRequest
-	52, // 44: filer_pb.SeaweedFiler.DownloadToLocal:input_type -> filer_pb.DownloadToLocalRequest
-	1,  // 45: filer_pb.SeaweedFiler.LookupDirectoryEntry:output_type -> filer_pb.LookupDirectoryEntryResponse
-	3,  // 46: filer_pb.SeaweedFiler.ListEntries:output_type -> filer_pb.ListEntriesResponse
-	13, // 47: filer_pb.SeaweedFiler.CreateEntry:output_type -> filer_pb.CreateEntryResponse
-	15, // 48: filer_pb.SeaweedFiler.UpdateEntry:output_type -> filer_pb.UpdateEntryResponse
-	17, // 49: filer_pb.SeaweedFiler.AppendToEntry:output_type -> filer_pb.AppendToEntryResponse
-	19, // 50: filer_pb.SeaweedFiler.DeleteEntry:output_type -> filer_pb.DeleteEntryResponse
-	21, // 51: filer_pb.SeaweedFiler.AtomicRenameEntry:output_type -> filer_pb.AtomicRenameEntryResponse
-	23, // 52: filer_pb.SeaweedFiler.AssignVolume:output_type -> filer_pb.AssignVolumeResponse
-	27, // 53: filer_pb.SeaweedFiler.LookupVolume:output_type -> filer_pb.LookupVolumeResponse
-	30, // 54: filer_pb.SeaweedFiler.CollectionList:output_type -> filer_pb.CollectionListResponse
-	32, // 55: filer_pb.SeaweedFiler.DeleteCollection:output_type -> filer_pb.DeleteCollectionResponse
-	34, // 56: filer_pb.SeaweedFiler.Statistics:output_type -> filer_pb.StatisticsResponse
-	36, // 57: filer_pb.SeaweedFiler.GetFilerConfiguration:output_type -> filer_pb.GetFilerConfigurationResponse
-	38, // 58: filer_pb.SeaweedFiler.SubscribeMetadata:output_type -> filer_pb.SubscribeMetadataResponse
-	38, // 59: filer_pb.SeaweedFiler.SubscribeLocalMetadata:output_type -> filer_pb.SubscribeMetadataResponse
-	41, // 60: filer_pb.SeaweedFiler.KeepConnected:output_type -> filer_pb.KeepConnectedResponse
-	43, // 61: filer_pb.SeaweedFiler.LocateBroker:output_type -> filer_pb.LocateBrokerResponse
-	45, // 62: filer_pb.SeaweedFiler.KvGet:output_type -> filer_pb.KvGetResponse
-	47, // 63: filer_pb.SeaweedFiler.KvPut:output_type -> filer_pb.KvPutResponse
-	53, // 64: filer_pb.SeaweedFiler.DownloadToLocal:output_type -> filer_pb.DownloadToLocalResponse
-	45, // [45:65] is the sub-list for method output_type
-	25, // [25:45] is the sub-list for method input_type
-	25, // [25:25] is the sub-list for extension type_name
-	25, // [25:25] is the sub-list for extension extendee
-	0,  // [0:25] is the sub-list for field type_name
+	53, // 19: filer_pb.LocateBrokerResponse.resources:type_name -> filer_pb.LocateBrokerResponse.Resource
+	54, // 20: filer_pb.FilerConf.locations:type_name -> filer_pb.FilerConf.PathConf
+	5,  // 21: filer_pb.DownloadToLocalResponse.entry:type_name -> filer_pb.Entry
+	25, // 22: filer_pb.LookupVolumeResponse.LocationsMapEntry.value:type_name -> filer_pb.Locations
+	0,  // 23: filer_pb.SeaweedFiler.LookupDirectoryEntry:input_type -> filer_pb.LookupDirectoryEntryRequest
+	2,  // 24: filer_pb.SeaweedFiler.ListEntries:input_type -> filer_pb.ListEntriesRequest
+	12, // 25: filer_pb.SeaweedFiler.CreateEntry:input_type -> filer_pb.CreateEntryRequest
+	14, // 26: filer_pb.SeaweedFiler.UpdateEntry:input_type -> filer_pb.UpdateEntryRequest
+	16, // 27: filer_pb.SeaweedFiler.AppendToEntry:input_type -> filer_pb.AppendToEntryRequest
+	18, // 28: filer_pb.SeaweedFiler.DeleteEntry:input_type -> filer_pb.DeleteEntryRequest
+	20, // 29: filer_pb.SeaweedFiler.AtomicRenameEntry:input_type -> filer_pb.AtomicRenameEntryRequest
+	22, // 30: filer_pb.SeaweedFiler.AssignVolume:input_type -> filer_pb.AssignVolumeRequest
+	24, // 31: filer_pb.SeaweedFiler.LookupVolume:input_type -> filer_pb.LookupVolumeRequest
+	29, // 32: filer_pb.SeaweedFiler.CollectionList:input_type -> filer_pb.CollectionListRequest
+	31, // 33: filer_pb.SeaweedFiler.DeleteCollection:input_type -> filer_pb.DeleteCollectionRequest
+	33, // 34: filer_pb.SeaweedFiler.Statistics:input_type -> filer_pb.StatisticsRequest
+	35, // 35: filer_pb.SeaweedFiler.GetFilerConfiguration:input_type -> filer_pb.GetFilerConfigurationRequest
+	37, // 36: filer_pb.SeaweedFiler.SubscribeMetadata:input_type -> filer_pb.SubscribeMetadataRequest
+	37, // 37: filer_pb.SeaweedFiler.SubscribeLocalMetadata:input_type -> filer_pb.SubscribeMetadataRequest
+	40, // 38: filer_pb.SeaweedFiler.KeepConnected:input_type -> filer_pb.KeepConnectedRequest
+	42, // 39: filer_pb.SeaweedFiler.LocateBroker:input_type -> filer_pb.LocateBrokerRequest
+	44, // 40: filer_pb.SeaweedFiler.KvGet:input_type -> filer_pb.KvGetRequest
+	46, // 41: filer_pb.SeaweedFiler.KvPut:input_type -> filer_pb.KvPutRequest
+	49, // 42: filer_pb.SeaweedFiler.DownloadToLocal:input_type -> filer_pb.DownloadToLocalRequest
+	1,  // 43: filer_pb.SeaweedFiler.LookupDirectoryEntry:output_type -> filer_pb.LookupDirectoryEntryResponse
+	3,  // 44: filer_pb.SeaweedFiler.ListEntries:output_type -> filer_pb.ListEntriesResponse
+	13, // 45: filer_pb.SeaweedFiler.CreateEntry:output_type -> filer_pb.CreateEntryResponse
+	15, // 46: filer_pb.SeaweedFiler.UpdateEntry:output_type -> filer_pb.UpdateEntryResponse
+	17, // 47: filer_pb.SeaweedFiler.AppendToEntry:output_type -> filer_pb.AppendToEntryResponse
+	19, // 48: filer_pb.SeaweedFiler.DeleteEntry:output_type -> filer_pb.DeleteEntryResponse
+	21, // 49: filer_pb.SeaweedFiler.AtomicRenameEntry:output_type -> filer_pb.AtomicRenameEntryResponse
+	23, // 50: filer_pb.SeaweedFiler.AssignVolume:output_type -> filer_pb.AssignVolumeResponse
+	27, // 51: filer_pb.SeaweedFiler.LookupVolume:output_type -> filer_pb.LookupVolumeResponse
+	30, // 52: filer_pb.SeaweedFiler.CollectionList:output_type -> filer_pb.CollectionListResponse
+	32, // 53: filer_pb.SeaweedFiler.DeleteCollection:output_type -> filer_pb.DeleteCollectionResponse
+	34, // 54: filer_pb.SeaweedFiler.Statistics:output_type -> filer_pb.StatisticsResponse
+	36, // 55: filer_pb.SeaweedFiler.GetFilerConfiguration:output_type -> filer_pb.GetFilerConfigurationResponse
+	38, // 56: filer_pb.SeaweedFiler.SubscribeMetadata:output_type -> filer_pb.SubscribeMetadataResponse
+	38, // 57: filer_pb.SeaweedFiler.SubscribeLocalMetadata:output_type -> filer_pb.SubscribeMetadataResponse
+	41, // 58: filer_pb.SeaweedFiler.KeepConnected:output_type -> filer_pb.KeepConnectedResponse
+	43, // 59: filer_pb.SeaweedFiler.LocateBroker:output_type -> filer_pb.LocateBrokerResponse
+	45, // 60: filer_pb.SeaweedFiler.KvGet:output_type -> filer_pb.KvGetResponse
+	47, // 61: filer_pb.SeaweedFiler.KvPut:output_type -> filer_pb.KvPutResponse
+	50, // 62: filer_pb.SeaweedFiler.DownloadToLocal:output_type -> filer_pb.DownloadToLocalResponse
+	43, // [43:63] is the sub-list for method output_type
+	23, // [23:43] is the sub-list for method input_type
+	23, // [23:23] is the sub-list for extension type_name
+	23, // [23:23] is the sub-list for extension extendee
+	0,  // [0:23] is the sub-list for field type_name
 }
 
 func init() { file_filer_proto_init() }
@@ -4963,42 +4738,6 @@ func file_filer_proto_init() {
 			}
 		}
 		file_filer_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*RemoteConf); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_filer_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*RemoteStorageMapping); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_filer_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*RemoteStorageLocation); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_filer_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*DownloadToLocalRequest); i {
 			case 0:
 				return &v.state
@@ -5010,7 +4749,7 @@ func file_filer_proto_init() {
 				return nil
 			}
 		}
-		file_filer_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} {
+		file_filer_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*DownloadToLocalResponse); i {
 			case 0:
 				return &v.state
@@ -5022,7 +4761,7 @@ func file_filer_proto_init() {
 				return nil
 			}
 		}
-		file_filer_proto_msgTypes[56].Exporter = func(v interface{}, i int) interface{} {
+		file_filer_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*LocateBrokerResponse_Resource); i {
 			case 0:
 				return &v.state
@@ -5034,7 +4773,7 @@ func file_filer_proto_init() {
 				return nil
 			}
 		}
-		file_filer_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} {
+		file_filer_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*FilerConf_PathConf); i {
 			case 0:
 				return &v.state
@@ -5053,7 +4792,7 @@ func file_filer_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_filer_proto_rawDesc,
 			NumEnums:      0,
-			NumMessages:   59,
+			NumMessages:   55,
 			NumExtensions: 0,
 			NumServices:   1,
 		},

weed/pb/filer_pb/filer_pb_helper.go

@@ -151,7 +151,3 @@ func (fp *FilerConf_PathConf) Key() interface{} {
 	key, _ := proto.Marshal(fp)
 	return string(key)
 }
-func (fp *RemoteStorageLocation) Key() interface{} {
-	key, _ := proto.Marshal(fp)
-	return string(key)
-}

weed/pb/filer_pb_tail.go

@@ -12,12 +12,12 @@ import (
 
 type ProcessMetadataFunc func(resp *filer_pb.SubscribeMetadataResponse) error
 
-func FollowMetadata(filerAddress string, grpcDialOption grpc.DialOption,
-	clientName string, pathPrefix string, lastTsNs int64, selfSignature int32,
+func FollowMetadata(filerAddress string, grpcDialOption grpc.DialOption, clientName string,
+	pathPrefix string, additionalPathPrefixes []string, lastTsNs int64, selfSignature int32,
 	processEventFn ProcessMetadataFunc, fatalOnError bool) error {
 
-	err := WithFilerClient(filerAddress, grpcDialOption, makeFunc(
-		clientName, pathPrefix, lastTsNs, selfSignature, processEventFn, fatalOnError))
+	err := WithFilerClient(filerAddress, grpcDialOption, makeFunc(clientName,
+		pathPrefix, additionalPathPrefixes, lastTsNs, selfSignature, processEventFn, fatalOnError))
 	if err != nil {
 		return fmt.Errorf("subscribing filer meta change: %v", err)
 	}
@@ -28,8 +28,8 @@ func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient,
 	clientName string, pathPrefix string, lastTsNs int64, selfSignature int32,
 	processEventFn ProcessMetadataFunc, fatalOnError bool) error {
 
-	err := filerClient.WithFilerClient(makeFunc(
-		clientName, pathPrefix, lastTsNs, selfSignature, processEventFn, fatalOnError))
+	err := filerClient.WithFilerClient(makeFunc(clientName,
+		pathPrefix, nil, lastTsNs, selfSignature, processEventFn, fatalOnError))
 	if err != nil {
 		return fmt.Errorf("subscribing filer meta change: %v", err)
 	}
@@ -37,16 +37,17 @@ func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient,
 	return nil
 }
 
-func makeFunc(clientName string, pathPrefix string, lastTsNs int64, selfSignature int32,
+func makeFunc(clientName string, pathPrefix string, additionalPathPrefixes []string, lastTsNs int64, selfSignature int32,
 	processEventFn ProcessMetadataFunc, fatalOnError bool) func(client filer_pb.SeaweedFilerClient) error {
 	return func(client filer_pb.SeaweedFilerClient) error {
 		ctx, cancel := context.WithCancel(context.Background())
 		defer cancel()
 		stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
-			ClientName: clientName,
-			PathPrefix: pathPrefix,
-			SinceNs:    lastTsNs,
-			Signature:  selfSignature,
+			ClientName:   clientName,
+			PathPrefix:   pathPrefix,
+			PathPrefixes: additionalPathPrefixes,
+			SinceNs:      lastTsNs,
+			Signature:    selfSignature,
 		})
 		if err != nil {
 			return fmt.Errorf("subscribe: %v", err)

weed/pb/remote.proto

@@ -0,0 +1,74 @@
+syntax = "proto3";
+
+package remote_pb;
+
+option go_package = "github.com/chrislusf/seaweedfs/weed/pb/remote_pb";
+option java_package = "seaweedfs.client";
+option java_outer_classname = "FilerProto";
+
+/////////////////////////
+// Remote Storage related
+/////////////////////////
+message RemoteConf {
+  string type = 1;
+  string name = 2;
+  string s3_access_key = 4;
+  string s3_secret_key = 5;
+  string s3_region = 6;
+  string s3_endpoint = 7;
+  string s3_storage_class = 8;
+  bool s3_force_path_style = 9;
+  bool s3_v4_signature = 11;
+
+  string gcs_google_application_credentials = 10;
+  string gcs_project_id = 12;
+
+  string azure_account_name = 15;
+  string azure_account_key = 16;
+
+  string backblaze_key_id = 20;
+  string backblaze_application_key = 21;
+  string backblaze_endpoint = 22;
+
+  string aliyun_access_key = 25;
+  string aliyun_secret_key = 26;
+  string aliyun_endpoint = 27;
+  string aliyun_region = 28;
+
+  string tencent_secret_id = 30;
+  string tencent_secret_key = 31;
+  string tencent_endpoint = 32;
+
+  string baidu_access_key = 35;
+  string baidu_secret_key = 36;
+  string baidu_endpoint = 37;
+  string baidu_region = 38;
+
+  string wasabi_access_key = 40;
+  string wasabi_secret_key = 41;
+  string wasabi_endpoint = 42;
+  string wasabi_region = 43;
+
+  repeated string hdfs_namenodes = 50;
+  string hdfs_username = 51;
+  string hdfs_service_principal_name = 52;
+  string hdfs_data_transfer_protection = 53;
+
+  string filebase_access_key = 60;
+  string filebase_secret_key = 61;
+  string filebase_endpoint = 62;
+
+  string storj_access_key = 65;
+  string storj_secret_key = 66;
+  string storj_endpoint = 67;
+
+}
+
+message RemoteStorageMapping {
+  map<string,RemoteStorageLocation> mappings = 1;
+}
+message RemoteStorageLocation {
+  string name = 1;
+  string bucket = 2;
+  string path = 3;
+}

weed/pb/remote_pb/remote.pb.go

@@ -0,0 +1,739 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.12.3
+// source: remote.proto
+
+package remote_pb
+
+import (
+	proto "github.com/golang/protobuf/proto"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
+
+/////////////////////////
+// Remote Storage related
+/////////////////////////
+type RemoteConf struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Type                            string   `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"`
+	Name                            string   `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
+	S3AccessKey                     string   `protobuf:"bytes,4,opt,name=s3_access_key,json=s3AccessKey,proto3" json:"s3_access_key,omitempty"`
+	S3SecretKey                     string   `protobuf:"bytes,5,opt,name=s3_secret_key,json=s3SecretKey,proto3" json:"s3_secret_key,omitempty"`
+	S3Region                        string   `protobuf:"bytes,6,opt,name=s3_region,json=s3Region,proto3" json:"s3_region,omitempty"`
+	S3Endpoint                      string   `protobuf:"bytes,7,opt,name=s3_endpoint,json=s3Endpoint,proto3" json:"s3_endpoint,omitempty"`
+	S3StorageClass                  string   `protobuf:"bytes,8,opt,name=s3_storage_class,json=s3StorageClass,proto3" json:"s3_storage_class,omitempty"`
+	S3ForcePathStyle                bool     `protobuf:"varint,9,opt,name=s3_force_path_style,json=s3ForcePathStyle,proto3" json:"s3_force_path_style,omitempty"`
+	S3V4Signature                   bool     `protobuf:"varint,11,opt,name=s3_v4_signature,json=s3V4Signature,proto3" json:"s3_v4_signature,omitempty"`
+	GcsGoogleApplicationCredentials string   `protobuf:"bytes,10,opt,name=gcs_google_application_credentials,json=gcsGoogleApplicationCredentials,proto3" json:"gcs_google_application_credentials,omitempty"`
+	GcsProjectId                    string   `protobuf:"bytes,12,opt,name=gcs_project_id,json=gcsProjectId,proto3" json:"gcs_project_id,omitempty"`
+	AzureAccountName                string   `protobuf:"bytes,15,opt,name=azure_account_name,json=azureAccountName,proto3" json:"azure_account_name,omitempty"`
+	AzureAccountKey                 string   `protobuf:"bytes,16,opt,name=azure_account_key,json=azureAccountKey,proto3" json:"azure_account_key,omitempty"`
+	BackblazeKeyId                  string   `protobuf:"bytes,20,opt,name=backblaze_key_id,json=backblazeKeyId,proto3" json:"backblaze_key_id,omitempty"`
+	BackblazeApplicationKey         string   `protobuf:"bytes,21,opt,name=backblaze_application_key,json=backblazeApplicationKey,proto3" json:"backblaze_application_key,omitempty"`
+	BackblazeEndpoint               string   `protobuf:"bytes,22,opt,name=backblaze_endpoint,json=backblazeEndpoint,proto3" json:"backblaze_endpoint,omitempty"`
+	AliyunAccessKey                 string   `protobuf:"bytes,25,opt,name=aliyun_access_key,json=aliyunAccessKey,proto3" json:"aliyun_access_key,omitempty"`
+	AliyunSecretKey                 string   `protobuf:"bytes,26,opt,name=aliyun_secret_key,json=aliyunSecretKey,proto3" json:"aliyun_secret_key,omitempty"`
+	AliyunEndpoint                  string   `protobuf:"bytes,27,opt,name=aliyun_endpoint,json=aliyunEndpoint,proto3" json:"aliyun_endpoint,omitempty"`
+	AliyunRegion                    string   `protobuf:"bytes,28,opt,name=aliyun_region,json=aliyunRegion,proto3" json:"aliyun_region,omitempty"`
+	TencentSecretId                 string   `protobuf:"bytes,30,opt,name=tencent_secret_id,json=tencentSecretId,proto3" json:"tencent_secret_id,omitempty"`
+	TencentSecretKey                string   `protobuf:"bytes,31,opt,name=tencent_secret_key,json=tencentSecretKey,proto3" json:"tencent_secret_key,omitempty"`
+	TencentEndpoint                 string   `protobuf:"bytes,32,opt,name=tencent_endpoint,json=tencentEndpoint,proto3" json:"tencent_endpoint,omitempty"`
+	BaiduAccessKey                  string   `protobuf:"bytes,35,opt,name=baidu_access_key,json=baiduAccessKey,proto3" json:"baidu_access_key,omitempty"`
+	BaiduSecretKey                  string   `protobuf:"bytes,36,opt,name=baidu_secret_key,json=baiduSecretKey,proto3" json:"baidu_secret_key,omitempty"`
+	BaiduEndpoint                   string   `protobuf:"bytes,37,opt,name=baidu_endpoint,json=baiduEndpoint,proto3" json:"baidu_endpoint,omitempty"`
+	BaiduRegion                     string   `protobuf:"bytes,38,opt,name=baidu_region,json=baiduRegion,proto3" json:"baidu_region,omitempty"`
+	WasabiAccessKey                 string   `protobuf:"bytes,40,opt,name=wasabi_access_key,json=wasabiAccessKey,proto3" json:"wasabi_access_key,omitempty"`
+	WasabiSecretKey                 string   `protobuf:"bytes,41,opt,name=wasabi_secret_key,json=wasabiSecretKey,proto3" json:"wasabi_secret_key,omitempty"`
+	WasabiEndpoint                  string   `protobuf:"bytes,42,opt,name=wasabi_endpoint,json=wasabiEndpoint,proto3" json:"wasabi_endpoint,omitempty"`
+	WasabiRegion                    string   `protobuf:"bytes,43,opt,name=wasabi_region,json=wasabiRegion,proto3" json:"wasabi_region,omitempty"`
+	HdfsNamenodes                   []string `protobuf:"bytes,50,rep,name=hdfs_namenodes,json=hdfsNamenodes,proto3" json:"hdfs_namenodes,omitempty"`
+	HdfsUsername                    string   `protobuf:"bytes,51,opt,name=hdfs_username,json=hdfsUsername,proto3" json:"hdfs_username,omitempty"`
+	HdfsServicePrincipalName        string   `protobuf:"bytes,52,opt,name=hdfs_service_principal_name,json=hdfsServicePrincipalName,proto3" json:"hdfs_service_principal_name,omitempty"`
+	HdfsDataTransferProtection      string   `protobuf:"bytes,53,opt,name=hdfs_data_transfer_protection,json=hdfsDataTransferProtection,proto3" json:"hdfs_data_transfer_protection,omitempty"`
+	FilebaseAccessKey               string   `protobuf:"bytes,60,opt,name=filebase_access_key,json=filebaseAccessKey,proto3" json:"filebase_access_key,omitempty"`
+	FilebaseSecretKey               string   `protobuf:"bytes,61,opt,name=filebase_secret_key,json=filebaseSecretKey,proto3" json:"filebase_secret_key,omitempty"`
+	FilebaseEndpoint                string   `protobuf:"bytes,62,opt,name=filebase_endpoint,json=filebaseEndpoint,proto3" json:"filebase_endpoint,omitempty"`
+	StorjAccessKey                  string   `protobuf:"bytes,65,opt,name=storj_access_key,json=storjAccessKey,proto3" json:"storj_access_key,omitempty"`
+	StorjSecretKey                  string   `protobuf:"bytes,66,opt,name=storj_secret_key,json=storjSecretKey,proto3" json:"storj_secret_key,omitempty"`
+	StorjEndpoint                   string   `protobuf:"bytes,67,opt,name=storj_endpoint,json=storjEndpoint,proto3" json:"storj_endpoint,omitempty"`
+}
+
+func (x *RemoteConf) Reset() {
+	*x = RemoteConf{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_remote_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *RemoteConf) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RemoteConf) ProtoMessage() {}
+
+func (x *RemoteConf) ProtoReflect() protoreflect.Message {
+	mi := &file_remote_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RemoteConf.ProtoReflect.Descriptor instead.
+func (*RemoteConf) Descriptor() ([]byte, []int) {
+	return file_remote_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *RemoteConf) GetType() string {
+	if x != nil {
+		return x.Type
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetS3AccessKey() string {
+	if x != nil {
+		return x.S3AccessKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetS3SecretKey() string {
+	if x != nil {
+		return x.S3SecretKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetS3Region() string {
+	if x != nil {
+		return x.S3Region
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetS3Endpoint() string {
+	if x != nil {
+		return x.S3Endpoint
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetS3StorageClass() string {
+	if x != nil {
+		return x.S3StorageClass
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetS3ForcePathStyle() bool {
+	if x != nil {
+		return x.S3ForcePathStyle
+	}
+	return false
+}
+
+func (x *RemoteConf) GetS3V4Signature() bool {
+	if x != nil {
+		return x.S3V4Signature
+	}
+	return false
+}
+
+func (x *RemoteConf) GetGcsGoogleApplicationCredentials() string {
+	if x != nil {
+		return x.GcsGoogleApplicationCredentials
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetGcsProjectId() string {
+	if x != nil {
+		return x.GcsProjectId
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetAzureAccountName() string {
+	if x != nil {
+		return x.AzureAccountName
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetAzureAccountKey() string {
+	if x != nil {
+		return x.AzureAccountKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetBackblazeKeyId() string {
+	if x != nil {
+		return x.BackblazeKeyId
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetBackblazeApplicationKey() string {
+	if x != nil {
+		return x.BackblazeApplicationKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetBackblazeEndpoint() string {
+	if x != nil {
+		return x.BackblazeEndpoint
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetAliyunAccessKey() string {
+	if x != nil {
+		return x.AliyunAccessKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetAliyunSecretKey() string {
+	if x != nil {
+		return x.AliyunSecretKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetAliyunEndpoint() string {
+	if x != nil {
+		return x.AliyunEndpoint
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetAliyunRegion() string {
+	if x != nil {
+		return x.AliyunRegion
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetTencentSecretId() string {
+	if x != nil {
+		return x.TencentSecretId
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetTencentSecretKey() string {
+	if x != nil {
+		return x.TencentSecretKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetTencentEndpoint() string {
+	if x != nil {
+		return x.TencentEndpoint
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetBaiduAccessKey() string {
+	if x != nil {
+		return x.BaiduAccessKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetBaiduSecretKey() string {
+	if x != nil {
+		return x.BaiduSecretKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetBaiduEndpoint() string {
+	if x != nil {
+		return x.BaiduEndpoint
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetBaiduRegion() string {
+	if x != nil {
+		return x.BaiduRegion
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetWasabiAccessKey() string {
+	if x != nil {
+		return x.WasabiAccessKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetWasabiSecretKey() string {
+	if x != nil {
+		return x.WasabiSecretKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetWasabiEndpoint() string {
+	if x != nil {
+		return x.WasabiEndpoint
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetWasabiRegion() string {
+	if x != nil {
+		return x.WasabiRegion
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetHdfsNamenodes() []string {
+	if x != nil {
+		return x.HdfsNamenodes
+	}
+	return nil
+}
+
+func (x *RemoteConf) GetHdfsUsername() string {
+	if x != nil {
+		return x.HdfsUsername
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetHdfsServicePrincipalName() string {
+	if x != nil {
+		return x.HdfsServicePrincipalName
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetHdfsDataTransferProtection() string {
+	if x != nil {
+		return x.HdfsDataTransferProtection
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetFilebaseAccessKey() string {
+	if x != nil {
+		return x.FilebaseAccessKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetFilebaseSecretKey() string {
+	if x != nil {
+		return x.FilebaseSecretKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetFilebaseEndpoint() string {
+	if x != nil {
+		return x.FilebaseEndpoint
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetStorjAccessKey() string {
+	if x != nil {
+		return x.StorjAccessKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetStorjSecretKey() string {
+	if x != nil {
+		return x.StorjSecretKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetStorjEndpoint() string {
+	if x != nil {
+		return x.StorjEndpoint
+	}
+	return ""
+}
+
+type RemoteStorageMapping struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Mappings map[string]*RemoteStorageLocation `protobuf:"bytes,1,rep,name=mappings,proto3" json:"mappings,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+}
+
+func (x *RemoteStorageMapping) Reset() {
+	*x = RemoteStorageMapping{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_remote_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *RemoteStorageMapping) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RemoteStorageMapping) ProtoMessage() {}
+
+func (x *RemoteStorageMapping) ProtoReflect() protoreflect.Message {
+	mi := &file_remote_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RemoteStorageMapping.ProtoReflect.Descriptor instead.
+func (*RemoteStorageMapping) Descriptor() ([]byte, []int) {
+	return file_remote_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *RemoteStorageMapping) GetMappings() map[string]*RemoteStorageLocation {
+	if x != nil {
+		return x.Mappings
+	}
+	return nil
+}
+
+type RemoteStorageLocation struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Name   string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"`
+	Path   string `protobuf:"bytes,3,opt,name=path,proto3" json:"path,omitempty"`
+}
+
+func (x *RemoteStorageLocation) Reset() {
+	*x = RemoteStorageLocation{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_remote_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *RemoteStorageLocation) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RemoteStorageLocation) ProtoMessage() {}
+
+func (x *RemoteStorageLocation) ProtoReflect() protoreflect.Message {
+	mi := &file_remote_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RemoteStorageLocation.ProtoReflect.Descriptor instead.
+func (*RemoteStorageLocation) Descriptor() ([]byte, []int) {
+	return file_remote_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *RemoteStorageLocation) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
+}
+
+func (x *RemoteStorageLocation) GetBucket() string {
+	if x != nil {
+		return x.Bucket
+	}
+	return ""
+}
+
+func (x *RemoteStorageLocation) GetPath() string {
+	if x != nil {
+		return x.Path
+	}
+	return ""
+}
+
+var File_remote_proto protoreflect.FileDescriptor
+
+var file_remote_proto_rawDesc = []byte{
+	0x0a, 0x0c, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09,
+	0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x22, 0xe2, 0x0d, 0x0a, 0x0a, 0x52, 0x65,
+	0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04,
+	0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65,
+	0x12, 0x22, 0x0a, 0x0d, 0x73, 0x33, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65,
+	0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x33, 0x41, 0x63, 0x63, 0x65, 0x73,
+	0x73, 0x4b, 0x65, 0x79, 0x12, 0x22, 0x0a, 0x0d, 0x73, 0x33, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65,
+	0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x33, 0x53,
+	0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x33, 0x5f, 0x72,
+	0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x33, 0x52,
+	0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x33, 0x5f, 0x65, 0x6e, 0x64, 0x70,
+	0x6f, 0x69, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x33, 0x45, 0x6e,
+	0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x33, 0x5f, 0x73, 0x74, 0x6f,
+	0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x0e, 0x73, 0x33, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73,
+	0x12, 0x2d, 0x0a, 0x13, 0x73, 0x33, 0x5f, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x70, 0x61, 0x74,
+	0x68, 0x5f, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x73,
+	0x33, 0x46, 0x6f, 0x72, 0x63, 0x65, 0x50, 0x61, 0x74, 0x68, 0x53, 0x74, 0x79, 0x6c, 0x65, 0x12,
+	0x26, 0x0a, 0x0f, 0x73, 0x33, 0x5f, 0x76, 0x34, 0x5f, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75,
+	0x72, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x73, 0x33, 0x56, 0x34, 0x53, 0x69,
+	0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x4b, 0x0a, 0x22, 0x67, 0x63, 0x73, 0x5f, 0x67,
+	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x5f, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f,
+	0x6e, 0x5f, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x0a, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x1f, 0x67, 0x63, 0x73, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x41, 0x70,
+	0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74,
+	0x69, 0x61, 0x6c, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x67, 0x63, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x6a,
+	0x65, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x67, 0x63,
+	0x73, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x64, 0x12, 0x2c, 0x0a, 0x12, 0x61, 0x7a,
+	0x75, 0x72, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65,
+	0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x41, 0x63, 0x63,
+	0x6f, 0x75, 0x6e, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x61, 0x7a, 0x75, 0x72,
+	0x65, 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x10, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e,
+	0x74, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x10, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a,
+	0x65, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x14, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e,
+	0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x4b, 0x65, 0x79, 0x49, 0x64, 0x12, 0x3a,
+	0x0a, 0x19, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x5f, 0x61, 0x70, 0x70, 0x6c,
+	0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x15, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x17, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x41, 0x70, 0x70, 0x6c,
+	0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x2d, 0x0a, 0x12, 0x62, 0x61,
+	0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74,
+	0x18, 0x16, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a,
+	0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2a, 0x0a, 0x11, 0x61, 0x6c, 0x69,
+	0x79, 0x75, 0x6e, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x19,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x41, 0x63, 0x63, 0x65,
+	0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x11, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x5f,
+	0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x0f, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65,
+	0x79, 0x12, 0x27, 0x0a, 0x0f, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x5f, 0x65, 0x6e, 0x64, 0x70,
+	0x6f, 0x69, 0x6e, 0x74, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x61, 0x6c, 0x69, 0x79,
+	0x75, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x61, 0x6c,
+	0x69, 0x79, 0x75, 0x6e, 0x5f, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x1c, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x0c, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12,
+	0x2a, 0x0a, 0x11, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65,
+	0x74, 0x5f, 0x69, 0x64, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x65, 0x6e, 0x63,
+	0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x49, 0x64, 0x12, 0x2c, 0x0a, 0x12, 0x74,
+	0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65,
+	0x79, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74,
+	0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x10, 0x74, 0x65, 0x6e,
+	0x63, 0x65, 0x6e, 0x74, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x20, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x45, 0x6e, 0x64, 0x70,
+	0x6f, 0x69, 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x62, 0x61, 0x69, 0x64, 0x75, 0x5f, 0x61, 0x63,
+	0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x23, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e,
+	0x62, 0x61, 0x69, 0x64, 0x75, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x28,
+	0x0a, 0x10, 0x62, 0x61, 0x69, 0x64, 0x75, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b,
+	0x65, 0x79, 0x18, 0x24, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x62, 0x61, 0x69, 0x64, 0x75, 0x53,
+	0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x62, 0x61, 0x69, 0x64,
+	0x75, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x25, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x0d, 0x62, 0x61, 0x69, 0x64, 0x75, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12,
+	0x21, 0x0a, 0x0c, 0x62, 0x61, 0x69, 0x64, 0x75, 0x5f, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18,
+	0x26, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x62, 0x61, 0x69, 0x64, 0x75, 0x52, 0x65, 0x67, 0x69,
+	0x6f, 0x6e, 0x12, 0x2a, 0x0a, 0x11, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, 0x61, 0x63, 0x63,
+	0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x28, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x77,
+	0x61, 0x73, 0x61, 0x62, 0x69, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x2a,
+	0x0a, 0x11, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f,
+	0x6b, 0x65, 0x79, 0x18, 0x29, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x77, 0x61, 0x73, 0x61, 0x62,
+	0x69, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x0f, 0x77, 0x61,
+	0x73, 0x61, 0x62, 0x69, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x2a, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x0e, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x45, 0x6e, 0x64, 0x70, 0x6f,
+	0x69, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, 0x72, 0x65,
+	0x67, 0x69, 0x6f, 0x6e, 0x18, 0x2b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x77, 0x61, 0x73, 0x61,
+	0x62, 0x69, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x68, 0x64, 0x66, 0x73,
+	0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x32, 0x20, 0x03, 0x28, 0x09,
+	0x52, 0x0d, 0x68, 0x64, 0x66, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x12,
+	0x23, 0x0a, 0x0d, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65,
+	0x18, 0x33, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x68, 0x64, 0x66, 0x73, 0x55, 0x73, 0x65, 0x72,
+	0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3d, 0x0a, 0x1b, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x73, 0x65, 0x72,
+	0x76, 0x69, 0x63, 0x65, 0x5f, 0x70, 0x72, 0x69, 0x6e, 0x63, 0x69, 0x70, 0x61, 0x6c, 0x5f, 0x6e,
+	0x61, 0x6d, 0x65, 0x18, 0x34, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x68, 0x64, 0x66, 0x73, 0x53,
+	0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x69, 0x6e, 0x63, 0x69, 0x70, 0x61, 0x6c, 0x4e,
+	0x61, 0x6d, 0x65, 0x12, 0x41, 0x0a, 0x1d, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x64, 0x61, 0x74, 0x61,
+	0x5f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x65, 0x63,
+	0x74, 0x69, 0x6f, 0x6e, 0x18, 0x35, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x68, 0x64, 0x66, 0x73,
+	0x44, 0x61, 0x74, 0x61, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74,
+	0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2e, 0x0a, 0x13, 0x66, 0x69, 0x6c, 0x65, 0x62, 0x61,
+	0x73, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x3c, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x11, 0x66, 0x69, 0x6c, 0x65, 0x62, 0x61, 0x73, 0x65, 0x41, 0x63, 0x63,
+	0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x2e, 0x0a, 0x13, 0x66, 0x69, 0x6c, 0x65, 0x62, 0x61,
+	0x73, 0x65, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x3d, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x11, 0x66, 0x69, 0x6c, 0x65, 0x62, 0x61, 0x73, 0x65, 0x53, 0x65, 0x63,
+	0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x2b, 0x0a, 0x11, 0x66, 0x69, 0x6c, 0x65, 0x62, 0x61,
+	0x73, 0x65, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x3e, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x10, 0x66, 0x69, 0x6c, 0x65, 0x62, 0x61, 0x73, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f,
+	0x69, 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x5f, 0x61, 0x63, 0x63,
+	0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x41, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73,
+	0x74, 0x6f, 0x72, 0x6a, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a,
+	0x10, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65,
+	0x79, 0x18, 0x42, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x53, 0x65,
+	0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x74, 0x6f, 0x72, 0x6a,
+	0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x43, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x0d, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x22, 0xc0,
+	0x01, 0x0a, 0x14, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65,
+	0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69,
+	0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x72, 0x65, 0x6d, 0x6f,
+	0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72,
+	0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x70, 0x70, 0x69,
+	0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e,
+	0x67, 0x73, 0x1a, 0x5d, 0x0a, 0x0d, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e,
+	0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x36, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62,
+	0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f,
+	0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38,
+	0x01, 0x22, 0x57, 0x0a, 0x15, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61,
+	0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61,
+	0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16,
+	0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06,
+	0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x42, 0x50, 0x0a, 0x10, 0x73, 0x65,
+	0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a,
+	0x46, 0x69, 0x6c, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68,
+	0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66,
+	0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f,
+	0x70, 0x62, 0x2f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_remote_proto_rawDescOnce sync.Once
+	file_remote_proto_rawDescData = file_remote_proto_rawDesc
+)
+
+func file_remote_proto_rawDescGZIP() []byte {
+	file_remote_proto_rawDescOnce.Do(func() {
+		file_remote_proto_rawDescData = protoimpl.X.CompressGZIP(file_remote_proto_rawDescData)
+	})
+	return file_remote_proto_rawDescData
+}
+
+var file_remote_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
+var file_remote_proto_goTypes = []interface{}{
+	(*RemoteConf)(nil),            // 0: remote_pb.RemoteConf
+	(*RemoteStorageMapping)(nil),  // 1: remote_pb.RemoteStorageMapping
+	(*RemoteStorageLocation)(nil), // 2: remote_pb.RemoteStorageLocation
+	nil,                           // 3: remote_pb.RemoteStorageMapping.MappingsEntry
+}
+var file_remote_proto_depIdxs = []int32{
+	3, // 0: remote_pb.RemoteStorageMapping.mappings:type_name -> remote_pb.RemoteStorageMapping.MappingsEntry
+	2, // 1: remote_pb.RemoteStorageMapping.MappingsEntry.value:type_name -> remote_pb.RemoteStorageLocation
+	2, // [2:2] is the sub-list for method output_type
+	2, // [2:2] is the sub-list for method input_type
+	2, // [2:2] is the sub-list for extension type_name
+	2, // [2:2] is the sub-list for extension extendee
+	0, // [0:2] is the sub-list for field type_name
+}
+
+func init() { file_remote_proto_init() }
+func file_remote_proto_init() {
+	if File_remote_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_remote_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RemoteConf); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_remote_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RemoteStorageMapping); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_remote_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RemoteStorageLocation); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_remote_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   4,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_remote_proto_goTypes,
+		DependencyIndexes: file_remote_proto_depIdxs,
+		MessageInfos:      file_remote_proto_msgTypes,
+	}.Build()
+	File_remote_proto = out.File
+	file_remote_proto_rawDesc = nil
+	file_remote_proto_goTypes = nil
+	file_remote_proto_depIdxs = nil
+}

weed/pb/remote_pb/remote_pb_helper.go

@@ -0,0 +1,8 @@
+package remote_pb
+
+import "github.com/golang/protobuf/proto"
+
+func (fp *RemoteStorageLocation) Key() interface{} {
+	key, _ := proto.Marshal(fp)
+	return string(key)
+}

weed/pb/volume_server.proto

@@ -3,6 +3,8 @@ syntax = "proto3";
 package volume_server_pb;
 option go_package = "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb";
 
+import "remote.proto";
+
 //////////////////////////////////////////////////
 
 service VolumeServer {
@@ -260,6 +262,7 @@ message CopyFileRequest {
 }
 message CopyFileResponse {
     bytes file_content = 1;
+    int64 modified_ts_ns = 2;
 }
 
 message ReadNeedleBlobRequest {
@@ -473,14 +476,8 @@ message FetchAndWriteNeedleRequest {
     int64 offset = 4;
     int64 size = 5;
     // remote conf
-    string remote_type = 6;
-    string remote_name = 7;
-    string s3_access_key = 8;
-    string s3_secret_key = 9;
-    string s3_region = 10;
-    string s3_endpoint = 11;
-    string remote_bucket = 12;
-    string remote_path = 13;
+    remote_pb.RemoteConf remote_conf = 15;
+    remote_pb.RemoteStorageLocation remote_location = 16;
 }
 message FetchAndWriteNeedleResponse {
 }

weed/remote_storage/azure/azure_highlevel.go

@@ -0,0 +1,120 @@
+package azure
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"github.com/Azure/azure-pipeline-go/pipeline"
+	. "github.com/Azure/azure-storage-blob-go/azblob"
+	"io"
+	"sync"
+)
+
+// copied from https://github.com/Azure/azure-storage-blob-go/blob/master/azblob/highlevel.go#L73:6
+// uploadReaderAtToBlockBlob was not public
+
+// uploadReaderAtToBlockBlob uploads a buffer in blocks to a block blob.
+func uploadReaderAtToBlockBlob(ctx context.Context, reader io.ReaderAt, readerSize int64,
+	blockBlobURL BlockBlobURL, o UploadToBlockBlobOptions) (CommonResponse, error) {
+	if o.BlockSize == 0 {
+		// If bufferSize > (BlockBlobMaxStageBlockBytes * BlockBlobMaxBlocks), then error
+		if readerSize > BlockBlobMaxStageBlockBytes*BlockBlobMaxBlocks {
+			return nil, errors.New("buffer is too large to upload to a block blob")
+		}
+		// If bufferSize <= BlockBlobMaxUploadBlobBytes, then Upload should be used with just 1 I/O request
+		if readerSize <= BlockBlobMaxUploadBlobBytes {
+			o.BlockSize = BlockBlobMaxUploadBlobBytes // Default if unspecified
+		} else {
+			o.BlockSize = readerSize / BlockBlobMaxBlocks   // buffer / max blocks = block size to use all 50,000 blocks
+			if o.BlockSize < BlobDefaultDownloadBlockSize { // If the block size is smaller than 4MB, round up to 4MB
+				o.BlockSize = BlobDefaultDownloadBlockSize
+			}
+			// StageBlock will be called with blockSize blocks and a Parallelism of (BufferSize / BlockSize).
+		}
+	}
+
+	if readerSize <= BlockBlobMaxUploadBlobBytes {
+		// If the size can fit in 1 Upload call, do it this way
+		var body io.ReadSeeker = io.NewSectionReader(reader, 0, readerSize)
+		if o.Progress != nil {
+			body = pipeline.NewRequestBodyProgress(body, o.Progress)
+		}
+		return blockBlobURL.Upload(ctx, body, o.BlobHTTPHeaders, o.Metadata, o.AccessConditions, o.BlobAccessTier, o.BlobTagsMap, o.ClientProvidedKeyOptions)
+	}
+
+	var numBlocks = uint16(((readerSize - 1) / o.BlockSize) + 1)
+
+	blockIDList := make([]string, numBlocks) // Base-64 encoded block IDs
+	progress := int64(0)
+	progressLock := &sync.Mutex{}
+
+	err := DoBatchTransfer(ctx, BatchTransferOptions{
+		OperationName: "uploadReaderAtToBlockBlob",
+		TransferSize:  readerSize,
+		ChunkSize:     o.BlockSize,
+		Parallelism:   o.Parallelism,
+		Operation: func(offset int64, count int64, ctx context.Context) error {
+			// This function is called once per block.
+			// It is passed this block's offset within the buffer and its count of bytes
+			// Prepare to read the proper block/section of the buffer
+			var body io.ReadSeeker = io.NewSectionReader(reader, offset, count)
+			blockNum := offset / o.BlockSize
+			if o.Progress != nil {
+				blockProgress := int64(0)
+				body = pipeline.NewRequestBodyProgress(body,
+					func(bytesTransferred int64) {
+						diff := bytesTransferred - blockProgress
+						blockProgress = bytesTransferred
+						progressLock.Lock() // 1 goroutine at a time gets a progress report
+						progress += diff
+						o.Progress(progress)
+						progressLock.Unlock()
+					})
+			}
+
+			// Block IDs are unique values to avoid issue if 2+ clients are uploading blocks
+			// at the same time causing PutBlockList to get a mix of blocks from all the clients.
+			blockIDList[blockNum] = base64.StdEncoding.EncodeToString(newUUID().bytes())
+			_, err := blockBlobURL.StageBlock(ctx, blockIDList[blockNum], body, o.AccessConditions.LeaseAccessConditions, nil, o.ClientProvidedKeyOptions)
+			return err
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	// All put blocks were successful, call Put Block List to finalize the blob
+	return blockBlobURL.CommitBlockList(ctx, blockIDList, o.BlobHTTPHeaders, o.Metadata, o.AccessConditions, o.BlobAccessTier, o.BlobTagsMap, o.ClientProvidedKeyOptions)
+}
+
+// The UUID reserved variants.
+const (
+	reservedNCS       byte = 0x80
+	reservedRFC4122   byte = 0x40
+	reservedMicrosoft byte = 0x20
+	reservedFuture    byte = 0x00
+)
+
+type uuid [16]byte
+
+// NewUUID returns a new uuid using RFC 4122 algorithm.
+func newUUID() (u uuid) {
+	u = uuid{}
+	// Set all bits to randomly (or pseudo-randomly) chosen values.
+	rand.Read(u[:])
+	u[8] = (u[8] | reservedRFC4122) & 0x7F // u.setVariant(ReservedRFC4122)
+
+	var version byte = 4
+	u[6] = (u[6] & 0xF) | (version << 4) // u.setVersion(4)
+	return
+}
+
+// String returns an unparsed version of the generated UUID sequence.
+func (u uuid) String() string {
+	return fmt.Sprintf("%x-%x-%x-%x-%x", u[0:4], u[4:6], u[6:8], u[8:10], u[10:])
+}
+
+func (u uuid) bytes() []byte {
+	return u[:]
+}

weed/remote_storage/azure/azure_storage_client.go

@@ -0,0 +1,252 @@
+package azure
+
+import (
+	"context"
+	"fmt"
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
+	"github.com/chrislusf/seaweedfs/weed/remote_storage"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"io"
+	"io/ioutil"
+	"net/url"
+	"os"
+	"reflect"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["azure"] = new(azureRemoteStorageMaker)
+}
+
+type azureRemoteStorageMaker struct{}
+
+func (s azureRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s azureRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+
+	client := &azureRemoteStorageClient{
+		conf: conf,
+	}
+
+	accountName, accountKey := conf.AzureAccountName, conf.AzureAccountKey
+	if len(accountName) == 0 || len(accountKey) == 0 {
+		accountName, accountKey = os.Getenv("AZURE_STORAGE_ACCOUNT"), os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+		if len(accountName) == 0 || len(accountKey) == 0 {
+			return nil, fmt.Errorf("either AZURE_STORAGE_ACCOUNT or AZURE_STORAGE_ACCESS_KEY environment variable is not set")
+		}
+	}
+
+	// Use your Storage account's name and key to create a credential object.
+	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	if err != nil {
+		return nil, fmt.Errorf("invalid Azure credential with account name:%s: %v", accountName, err)
+	}
+
+	// Create a request pipeline that is used to process HTTP(S) requests and responses.
+	p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
+
+	// Create an ServiceURL object that wraps the service URL and a request pipeline.
+	u, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", accountName))
+	client.serviceURL = azblob.NewServiceURL(*u, p)
+
+	return client, nil
+}
+
+type azureRemoteStorageClient struct {
+	conf       *remote_pb.RemoteConf
+	serviceURL azblob.ServiceURL
+}
+
+var _ = remote_storage.RemoteStorageClient(&azureRemoteStorageClient{})
+
+func (az *azureRemoteStorageClient) Traverse(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {
+
+	pathKey := loc.Path[1:]
+	containerURL := az.serviceURL.NewContainerURL(loc.Bucket)
+
+	// List the container that we have created above
+	for marker := (azblob.Marker{}); marker.NotDone(); {
+		// Get a result segment starting with the blob indicated by the current Marker.
+		listBlob, err := containerURL.ListBlobsFlatSegment(context.Background(), marker, azblob.ListBlobsSegmentOptions{
+			Prefix: pathKey,
+		})
+		if err != nil {
+			return fmt.Errorf("azure traverse %s%s: %v", loc.Bucket, loc.Path, err)
+		}
+
+		// ListBlobs returns the start of the next segment; you MUST use this to get
+		// the next segment (after processing the current result segment).
+		marker = listBlob.NextMarker
+
+		// Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute)
+		for _, blobInfo := range listBlob.Segment.BlobItems {
+			key := blobInfo.Name
+			key = "/" + key
+			dir, name := util.FullPath(key).DirAndName()
+			err = visitFn(dir, name, false, &filer_pb.RemoteEntry{
+				RemoteMtime: blobInfo.Properties.LastModified.Unix(),
+				RemoteSize:  *blobInfo.Properties.ContentLength,
+				RemoteETag:  string(blobInfo.Properties.Etag),
+				StorageName: az.conf.Name,
+			})
+			if err != nil {
+				return fmt.Errorf("azure processing %s%s: %v", loc.Bucket, loc.Path, err)
+			}
+		}
+	}
+
+	return
+}
+func (az *azureRemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) {
+
+	key := loc.Path[1:]
+	containerURL := az.serviceURL.NewContainerURL(loc.Bucket)
+	blobURL := containerURL.NewBlockBlobURL(key)
+
+	downloadResponse, readErr := blobURL.Download(context.Background(), offset, size, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+	if readErr != nil {
+		return nil, readErr
+	}
+	// NOTE: automatically retries are performed if the connection fails
+	bodyStream := downloadResponse.Body(azblob.RetryReaderOptions{MaxRetryRequests: 20})
+	defer bodyStream.Close()
+
+	data, err = ioutil.ReadAll(bodyStream)
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to download file %s%s: %v", loc.Bucket, loc.Path, err)
+	}
+
+	return
+}
+
+func (az *azureRemoteStorageClient) WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) {
+	return nil
+}
+
+func (az *azureRemoteStorageClient) RemoveDirectory(loc *remote_pb.RemoteStorageLocation) (err error) {
+	return nil
+}
+
+func (az *azureRemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) {
+
+	key := loc.Path[1:]
+	containerURL := az.serviceURL.NewContainerURL(loc.Bucket)
+	blobURL := containerURL.NewBlockBlobURL(key)
+
+	readerAt, ok := reader.(io.ReaderAt)
+	if !ok {
+		return nil, fmt.Errorf("unexpected reader: readerAt expected")
+	}
+	fileSize := int64(filer.FileSize(entry))
+
+	_, err = uploadReaderAtToBlockBlob(context.Background(), readerAt, fileSize, blobURL, azblob.UploadToBlockBlobOptions{
+		BlockSize:   4 * 1024 * 1024,
+		Parallelism: 16})
+	if err != nil {
+		return nil, fmt.Errorf("azure upload to %s%s: %v", loc.Bucket, loc.Path, err)
+	}
+
+	metadata := toMetadata(entry.Extended)
+	if len(metadata) > 0 {
+		_, err = blobURL.SetMetadata(context.Background(), metadata, azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+		if err != nil {
+			return nil, fmt.Errorf("azure set metadata on %s%s: %v", loc.Bucket, loc.Path, err)
+		}
+	}
+
+	// read back the remote entry
+	return az.readFileRemoteEntry(loc)
+
+}
+
+func (az *azureRemoteStorageClient) readFileRemoteEntry(loc *remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
+	key := loc.Path[1:]
+	containerURL := az.serviceURL.NewContainerURL(loc.Bucket)
+	blobURL := containerURL.NewBlockBlobURL(key)
+
+	attr, err := blobURL.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return &filer_pb.RemoteEntry{
+		RemoteMtime: attr.LastModified().Unix(),
+		RemoteSize:  attr.ContentLength(),
+		RemoteETag:  string(attr.ETag()),
+		StorageName: az.conf.Name,
+	}, nil
+
+}
+
+func toMetadata(attributes map[string][]byte) map[string]string {
+	metadata := make(map[string]string)
+	for k, v := range attributes {
+		metadata[k] = string(v)
+	}
+	return metadata
+}
+
+func (az *azureRemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error) {
+	if reflect.DeepEqual(oldEntry.Extended, newEntry.Extended) {
+		return nil
+	}
+	metadata := toMetadata(newEntry.Extended)
+
+	key := loc.Path[1:]
+	containerURL := az.serviceURL.NewContainerURL(loc.Bucket)
+
+	_, err = containerURL.NewBlobURL(key).SetMetadata(context.Background(), metadata, azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+
+	return
+}
+
+func (az *azureRemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) {
+	key := loc.Path[1:]
+	containerURL := az.serviceURL.NewContainerURL(loc.Bucket)
+	if _, err = containerURL.NewBlobURL(key).Delete(context.Background(),
+		azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{}); err != nil {
+		return fmt.Errorf("azure delete %s%s: %v", loc.Bucket, loc.Path, err)
+	}
+	return
+}
+
+func (az *azureRemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket, err error) {
+	ctx := context.Background()
+	for containerMarker := (azblob.Marker{}); containerMarker.NotDone(); {
+		listContainer, err := az.serviceURL.ListContainersSegment(ctx, containerMarker, azblob.ListContainersSegmentOptions{})
+		if err == nil {
+			for _, v := range listContainer.ContainerItems {
+				buckets = append(buckets, &remote_storage.Bucket{
+					Name:      v.Name,
+					CreatedAt: v.Properties.LastModified,
+				})
+			}
+		} else {
+			return buckets, err
+		}
+		containerMarker = listContainer.NextMarker
+	}
+	return
+}
+
+func (az *azureRemoteStorageClient) CreateBucket(name string) (err error) {
+	containerURL := az.serviceURL.NewContainerURL(name)
+	if _, err = containerURL.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessNone); err != nil {
+		return fmt.Errorf("create bucket %s: %v", name, err)
+	}
+	return
+}
+
+func (az *azureRemoteStorageClient) DeleteBucket(name string) (err error) {
+	containerURL := az.serviceURL.NewContainerURL(name)
+	if _, err = containerURL.Delete(context.Background(), azblob.ContainerAccessConditions{}); err != nil {
+		return fmt.Errorf("delete bucket %s: %v", name, err)
+	}
+	return
+}

weed/remote_storage/gcs/gcs_storage_client.go

@@ -1,10 +1,12 @@
-package s3
+package gcs
 
 import (
 	"cloud.google.com/go/storage"
 	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
 	"github.com/chrislusf/seaweedfs/weed/remote_storage"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"google.golang.org/api/iterator"
@@ -21,7 +23,11 @@ func init() {
 
 type gcsRemoteStorageMaker struct{}
 
-func (s gcsRemoteStorageMaker) Make(conf *filer_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+func (s gcsRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s gcsRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
 	client := &gcsRemoteStorageClient{
 		conf: conf,
 	}
@@ -36,6 +42,15 @@ func (s gcsRemoteStorageMaker) Make(conf *filer_pb.RemoteConf) (remote_storage.R
 		}
 	}
 
+	projectID := conf.GcsProjectId
+	if projectID == "" {
+		found := false
+		projectID, found = os.LookupEnv("GOOGLE_CLOUD_PROJECT")
+		if !found {
+			glog.Warningf("need to specific GOOGLE_CLOUD_PROJECT env variable")
+		}
+	}
+
 	googleApplicationCredentials = util.ResolvePath(googleApplicationCredentials)
 
 	c, err := storage.NewClient(context.Background(), option.WithCredentialsFile(googleApplicationCredentials))
@@ -44,17 +59,19 @@ func (s gcsRemoteStorageMaker) Make(conf *filer_pb.RemoteConf) (remote_storage.R
 	}
 
 	client.client = c
+	client.projectID = projectID
 	return client, nil
 }
 
 type gcsRemoteStorageClient struct {
-	conf *filer_pb.RemoteConf
-	client        *storage.Client
+	conf      *remote_pb.RemoteConf
+	client    *storage.Client
+	projectID string
 }
 
 var _ = remote_storage.RemoteStorageClient(&gcsRemoteStorageClient{})
 
-func (gcs *gcsRemoteStorageClient) Traverse(loc *filer_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {
+func (gcs *gcsRemoteStorageClient) Traverse(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {
 
 	pathKey := loc.Path[1:]
 
@@ -86,7 +103,7 @@ func (gcs *gcsRemoteStorageClient) Traverse(loc *filer_pb.RemoteStorageLocation,
 	}
 	return
 }
-func (gcs *gcsRemoteStorageClient) ReadFile(loc *filer_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) {
+func (gcs *gcsRemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) {
 
 	key := loc.Path[1:]
 	rangeReader, readErr := gcs.client.Bucket(loc.Bucket).Object(key).NewRangeReader(context.Background(), offset, size)
@@ -102,38 +119,34 @@ func (gcs *gcsRemoteStorageClient) ReadFile(loc *filer_pb.RemoteStorageLocation,
 	return
 }
 
-func (gcs *gcsRemoteStorageClient) WriteDirectory(loc *filer_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) {
+func (gcs *gcsRemoteStorageClient) WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) {
 	return nil
 }
 
-func (gcs *gcsRemoteStorageClient) WriteFile(loc *filer_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) {
+func (gcs *gcsRemoteStorageClient) RemoveDirectory(loc *remote_pb.RemoteStorageLocation) (err error) {
+	return nil
+}
+
+func (gcs *gcsRemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) {
 
 	key := loc.Path[1:]
 
+	metadata := toMetadata(entry.Extended)
 	wc := gcs.client.Bucket(loc.Bucket).Object(key).NewWriter(context.Background())
+	wc.Metadata = metadata
 	if _, err = io.Copy(wc, reader); err != nil {
-		wc.Close()
 		return nil, fmt.Errorf("upload to gcs %s/%s%s: %v", loc.Name, loc.Bucket, loc.Path, err)
 	}
 	if err = wc.Close(); err != nil {
 		return nil, fmt.Errorf("close gcs %s/%s%s: %v", loc.Name, loc.Bucket, loc.Path, err)
 	}
 
-	metadata := toMetadata(entry.Extended)
-	if len(metadata) > 0 {
-		if _, err = gcs.client.Bucket(loc.Bucket).Object(key).Update(context.Background(), storage.ObjectAttrsToUpdate{
-			Metadata:           metadata,
-		}); err != nil {
-			return nil, fmt.Errorf("update metadata gcs %s/%s%s: %v", loc.Name, loc.Bucket, loc.Path, err)
-		}
-	}
-
 	// read back the remote entry
 	return gcs.readFileRemoteEntry(loc)
 
 }
 
-func (gcs *gcsRemoteStorageClient) readFileRemoteEntry(loc *filer_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
+func (gcs *gcsRemoteStorageClient) readFileRemoteEntry(loc *remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
 	key := loc.Path[1:]
 	attr, err := gcs.client.Bucket(loc.Bucket).Object(key).Attrs(context.Background())
 
@@ -158,7 +171,7 @@ func toMetadata(attributes map[string][]byte) map[string]string {
 	return metadata
 }
 
-func (gcs *gcsRemoteStorageClient) UpdateFileMetadata(loc *filer_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error) {
+func (gcs *gcsRemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error) {
 	if reflect.DeepEqual(oldEntry.Extended, newEntry.Extended) {
 		return nil
 	}
@@ -168,7 +181,7 @@ func (gcs *gcsRemoteStorageClient) UpdateFileMetadata(loc *filer_pb.RemoteStorag
 
 	if len(metadata) > 0 {
 		_, err = gcs.client.Bucket(loc.Bucket).Object(key).Update(context.Background(), storage.ObjectAttrsToUpdate{
-			Metadata:           metadata,
+			Metadata: metadata,
 		})
 	} else {
 		// no way to delete the metadata yet
@@ -176,10 +189,50 @@ func (gcs *gcsRemoteStorageClient) UpdateFileMetadata(loc *filer_pb.RemoteStorag
 
 	return
 }
-func (gcs *gcsRemoteStorageClient) DeleteFile(loc *filer_pb.RemoteStorageLocation) (err error) {
+func (gcs *gcsRemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) {
 	key := loc.Path[1:]
 	if err = gcs.client.Bucket(loc.Bucket).Object(key).Delete(context.Background()); err != nil {
 		return fmt.Errorf("gcs delete %s%s: %v", loc.Bucket, key, err)
 	}
 	return
 }
+
+func (gcs *gcsRemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket, err error) {
+	if gcs.projectID == "" {
+		return nil, fmt.Errorf("gcs project id or GOOGLE_CLOUD_PROJECT env variable not set")
+	}
+	iter := gcs.client.Buckets(context.Background(), gcs.projectID)
+	for {
+		b, err := iter.Next()
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			return buckets, err
+		}
+		buckets = append(buckets, &remote_storage.Bucket{
+			Name:      b.Name,
+			CreatedAt: b.Created,
+		})
+	}
+	return
+}
+
+func (gcs *gcsRemoteStorageClient) CreateBucket(name string) (err error) {
+	if gcs.projectID == "" {
+		return fmt.Errorf("gcs project id or GOOGLE_CLOUD_PROJECT env variable not set")
+	}
+	err = gcs.client.Bucket(name).Create(context.Background(), gcs.projectID, &storage.BucketAttrs{})
+	if err != nil {
+		return fmt.Errorf("create bucket %s: %v", name, err)
+	}
+	return
+}
+
+func (gcs *gcsRemoteStorageClient) DeleteBucket(name string) (err error) {
+	err = gcs.client.Bucket(name).Delete(context.Background())
+	if err != nil {
+		return fmt.Errorf("delete bucket %s: %v", name, err)
+	}
+	return
+}

weed/remote_storage/hdfs/hdfs_kerberos.go

@@ -0,0 +1,55 @@
+package hdfs
+
+import (
+	"fmt"
+	"os"
+	"os/user"
+	"strings"
+
+	krb "github.com/jcmturner/gokrb5/v8/client"
+	"github.com/jcmturner/gokrb5/v8/config"
+	"github.com/jcmturner/gokrb5/v8/credentials"
+)
+
+// copy-paste from https://github.com/colinmarc/hdfs/blob/master/cmd/hdfs/kerberos.go
+func getKerberosClient() (*krb.Client, error) {
+	configPath := os.Getenv("KRB5_CONFIG")
+	if configPath == "" {
+		configPath = "/etc/krb5.conf"
+	}
+
+	cfg, err := config.Load(configPath)
+	if err != nil {
+		return nil, err
+	}
+
+	// Determine the ccache location from the environment, falling back to the
+	// default location.
+	ccachePath := os.Getenv("KRB5CCNAME")
+	if strings.Contains(ccachePath, ":") {
+		if strings.HasPrefix(ccachePath, "FILE:") {
+			ccachePath = strings.SplitN(ccachePath, ":", 2)[1]
+		} else {
+			return nil, fmt.Errorf("unusable ccache: %s", ccachePath)
+		}
+	} else if ccachePath == "" {
+		u, err := user.Current()
+		if err != nil {
+			return nil, err
+		}
+
+		ccachePath = fmt.Sprintf("/tmp/krb5cc_%s", u.Uid)
+	}
+
+	ccache, err := credentials.LoadCCache(ccachePath)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := krb.NewFromCCache(ccache, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return client, nil
+}

weed/remote_storage/hdfs/hdfs_storage_client.go

@@ -0,0 +1,191 @@
+package hdfs
+
+import (
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
+	"github.com/chrislusf/seaweedfs/weed/remote_storage"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/colinmarc/hdfs/v2"
+	"io"
+	"os"
+	"path"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["hdfs"] = new(hdfsRemoteStorageMaker)
+}
+
+type hdfsRemoteStorageMaker struct{}
+
+func (s hdfsRemoteStorageMaker) HasBucket() bool {
+	return false
+}
+
+func (s hdfsRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	client := &hdfsRemoteStorageClient{
+		conf: conf,
+	}
+
+	options := hdfs.ClientOptions{
+		Addresses:           conf.HdfsNamenodes,
+		UseDatanodeHostname: false,
+	}
+
+	if conf.HdfsServicePrincipalName != "" {
+		var err error
+		options.KerberosClient, err = getKerberosClient()
+		if err != nil {
+			return nil, fmt.Errorf("get kerberos authentication: %s", err)
+		}
+		options.KerberosServicePrincipleName = conf.HdfsServicePrincipalName
+
+		if conf.HdfsDataTransferProtection != "" {
+			options.DataTransferProtection = conf.HdfsDataTransferProtection
+		}
+	} else {
+		options.User = conf.HdfsUsername
+	}
+
+	c, err := hdfs.NewClient(options)
+	if err != nil {
+		return nil, err
+	}
+
+	client.client = c
+	return client, nil
+}
+
+type hdfsRemoteStorageClient struct {
+	conf   *remote_pb.RemoteConf
+	client *hdfs.Client
+}
+
+var _ = remote_storage.RemoteStorageClient(&hdfsRemoteStorageClient{})
+
+func (c *hdfsRemoteStorageClient) Traverse(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {
+
+	return remote_storage.TraverseBfs(func(parentDir util.FullPath, visitFn remote_storage.VisitFunc) error {
+		children, err := c.client.ReadDir(string(parentDir))
+		if err != nil {
+			return err
+		}
+		for _, child := range children {
+			if err := visitFn(string(parentDir), child.Name(), child.IsDir(), &filer_pb.RemoteEntry{
+				StorageName:       c.conf.Name,
+				LastLocalSyncTsNs: 0,
+				RemoteETag:        "",
+				RemoteMtime:       child.ModTime().Unix(),
+				RemoteSize:        child.Size(),
+			}); err != nil {
+				return nil
+			}
+		}
+		return nil
+	}, util.FullPath(loc.Path), visitFn)
+
+}
+func (c *hdfsRemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) {
+
+	f, err := c.client.Open(loc.Path)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+	data = make([]byte, size)
+	_, err = f.ReadAt(data, offset)
+
+	return
+
+}
+
+func (c *hdfsRemoteStorageClient) WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) {
+	return c.client.MkdirAll(loc.Path, os.FileMode(entry.Attributes.FileMode))
+}
+
+func (c *hdfsRemoteStorageClient) RemoveDirectory(loc *remote_pb.RemoteStorageLocation) (err error) {
+	return c.client.RemoveAll(loc.Path)
+}
+
+func (c *hdfsRemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) {
+
+	dirname := path.Dir(loc.Path)
+
+	// ensure parent directory
+	if err = c.client.MkdirAll(dirname, 0755); err != nil {
+		return
+	}
+
+	// remove existing file
+	info, err := c.client.Stat(loc.Path)
+	if err == nil {
+		err = c.client.Remove(loc.Path)
+		if err != nil {
+			return
+		}
+	}
+
+	// create new file
+	out, err := c.client.Create(loc.Path)
+	if err != nil {
+		return
+	}
+
+	cleanup := func() {
+		if removeErr := c.client.Remove(loc.Path); removeErr != nil {
+			glog.Errorf("clean up %s%s: %v", loc.Name, loc.Path, removeErr)
+		}
+	}
+
+	if _, err = io.Copy(out, reader); err != nil {
+		cleanup()
+		return
+	}
+
+	if err = out.Close(); err != nil {
+		cleanup()
+		return
+	}
+
+	info, err = c.client.Stat(loc.Path)
+	if err != nil {
+		return
+	}
+
+	return &filer_pb.RemoteEntry{
+		RemoteMtime: info.ModTime().Unix(),
+		RemoteSize:  info.Size(),
+		RemoteETag:  "",
+		StorageName: c.conf.Name,
+	}, nil
+
+}
+
+func (c *hdfsRemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) error {
+	if oldEntry.Attributes.FileMode != newEntry.Attributes.FileMode {
+		if err := c.client.Chmod(loc.Path, os.FileMode(newEntry.Attributes.FileMode)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *hdfsRemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) {
+	if err = c.client.Remove(loc.Path); err != nil {
+		return fmt.Errorf("hdfs delete %s: %v", loc.Path, err)
+	}
+	return
+}
+
+func (c *hdfsRemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket, err error) {
+	return
+}
+
+func (c *hdfsRemoteStorageClient) CreateBucket(name string) (err error) {
+	return
+}
+
+func (c *hdfsRemoteStorageClient) DeleteBucket(name string) (err error) {
+	return
+}

weed/remote_storage/remote_storage.go

@@ -3,13 +3,28 @@ package remote_storage
 import (
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
+	"github.com/golang/protobuf/proto"
 	"io"
+	"sort"
 	"strings"
 	"sync"
+	"time"
 )
 
-func ParseLocation(remote string) (loc *filer_pb.RemoteStorageLocation) {
-	loc = &filer_pb.RemoteStorageLocation{}
+func ParseLocationName(remote string) (locationName string) {
+	if strings.HasSuffix(string(remote), "/") {
+		remote = remote[:len(remote)-1]
+	}
+	parts := strings.SplitN(string(remote), "/", 2)
+	if len(parts) >= 1 {
+		return parts[0]
+	}
+	return
+}
+
+func parseBucketLocation(remote string) (loc *remote_pb.RemoteStorageLocation) {
+	loc = &remote_pb.RemoteStorageLocation{}
 	if strings.HasSuffix(string(remote), "/") {
 		remote = remote[:len(remote)-1]
 	}
@@ -27,32 +42,98 @@ func ParseLocation(remote string) (loc *filer_pb.RemoteStorageLocation) {
 	return
 }
 
-func FormatLocation(loc *filer_pb.RemoteStorageLocation) string {
+func parseNoBucketLocation(remote string) (loc *remote_pb.RemoteStorageLocation) {
+	loc = &remote_pb.RemoteStorageLocation{}
+	if strings.HasSuffix(string(remote), "/") {
+		remote = remote[:len(remote)-1]
+	}
+	parts := strings.SplitN(string(remote), "/", 2)
+	if len(parts) >= 1 {
+		loc.Name = parts[0]
+	}
+	loc.Path = string(remote[len(loc.Name):])
+	if loc.Path == "" {
+		loc.Path = "/"
+	}
+	return
+}
+
+func FormatLocation(loc *remote_pb.RemoteStorageLocation) string {
+	if loc.Bucket == "" {
+		return fmt.Sprintf("%s%s", loc.Name, loc.Path)
+	}
 	return fmt.Sprintf("%s/%s%s", loc.Name, loc.Bucket, loc.Path)
 }
 
 type VisitFunc func(dir string, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error
 
+type Bucket struct {
+	Name      string
+	CreatedAt time.Time
+}
+
 type RemoteStorageClient interface {
-	Traverse(loc *filer_pb.RemoteStorageLocation, visitFn VisitFunc) error
-	ReadFile(loc *filer_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error)
-	WriteDirectory(loc *filer_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error)
-	WriteFile(loc *filer_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error)
-	UpdateFileMetadata(loc *filer_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error)
-	DeleteFile(loc *filer_pb.RemoteStorageLocation) (err error)
+	Traverse(loc *remote_pb.RemoteStorageLocation, visitFn VisitFunc) error
+	ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error)
+	WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error)
+	RemoveDirectory(loc *remote_pb.RemoteStorageLocation) (err error)
+	WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error)
+	UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error)
+	DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error)
+	ListBuckets() ([]*Bucket, error)
+	CreateBucket(name string) (err error)
+	DeleteBucket(name string) (err error)
 }
 
 type RemoteStorageClientMaker interface {
-	Make(remoteConf *filer_pb.RemoteConf) (RemoteStorageClient, error)
+	Make(remoteConf *remote_pb.RemoteConf) (RemoteStorageClient, error)
+	HasBucket() bool
+}
+
+type CachedRemoteStorageClient struct {
+	*remote_pb.RemoteConf
+	RemoteStorageClient
 }
 
 var (
 	RemoteStorageClientMakers = make(map[string]RemoteStorageClientMaker)
-	remoteStorageClients      = make(map[string]RemoteStorageClient)
+	remoteStorageClients      = make(map[string]CachedRemoteStorageClient)
 	remoteStorageClientsLock  sync.Mutex
 )
 
-func makeRemoteStorageClient(remoteConf *filer_pb.RemoteConf) (RemoteStorageClient, error) {
+func GetAllRemoteStorageNames() string {
+	var storageNames []string
+	for k := range RemoteStorageClientMakers {
+		storageNames = append(storageNames, k)
+	}
+	sort.Strings(storageNames)
+	return strings.Join(storageNames, "|")
+}
+
+func GetRemoteStorageNamesHasBucket() string {
+	var storageNames []string
+	for k, m := range RemoteStorageClientMakers {
+		if m.HasBucket() {
+			storageNames = append(storageNames, k)
+		}
+	}
+	sort.Strings(storageNames)
+	return strings.Join(storageNames, "|")
+}
+
+func ParseRemoteLocation(remoteConfType string, remote string) (remoteStorageLocation *remote_pb.RemoteStorageLocation, err error) {
+	maker, found := RemoteStorageClientMakers[remoteConfType]
+	if !found {
+		return nil, fmt.Errorf("remote storage type %s not found", remoteConfType)
+	}
+
+	if !maker.HasBucket() {
+		return parseNoBucketLocation(remote), nil
+	}
+	return parseBucketLocation(remote), nil
+}
+
+func makeRemoteStorageClient(remoteConf *remote_pb.RemoteConf) (RemoteStorageClient, error) {
 	maker, found := RemoteStorageClientMakers[remoteConf.Type]
 	if !found {
 		return nil, fmt.Errorf("remote storage type %s not found", remoteConf.Type)
@@ -60,13 +141,13 @@ func makeRemoteStorageClient(remoteConf *filer_pb.RemoteConf) (RemoteStorageClie
 	return maker.Make(remoteConf)
 }
 
-func GetRemoteStorage(remoteConf *filer_pb.RemoteConf) (RemoteStorageClient, error) {
+func GetRemoteStorage(remoteConf *remote_pb.RemoteConf) (RemoteStorageClient, error) {
 	remoteStorageClientsLock.Lock()
 	defer remoteStorageClientsLock.Unlock()
 
 	existingRemoteStorageClient, found := remoteStorageClients[remoteConf.Name]
-	if found {
-		return existingRemoteStorageClient, nil
+	if found && proto.Equal(existingRemoteStorageClient.RemoteConf, remoteConf) {
+		return existingRemoteStorageClient.RemoteStorageClient, nil
 	}
 
 	newRemoteStorageClient, err := makeRemoteStorageClient(remoteConf)
@@ -74,7 +155,10 @@ func GetRemoteStorage(remoteConf *filer_pb.RemoteConf) (RemoteStorageClient, err
 		return nil, fmt.Errorf("make remote storage client %s: %v", remoteConf.Name, err)
 	}
 
-	remoteStorageClients[remoteConf.Name] = newRemoteStorageClient
+	remoteStorageClients[remoteConf.Name] = CachedRemoteStorageClient{
+		RemoteConf:          remoteConf,
+		RemoteStorageClient: newRemoteStorageClient,
+	}
 
 	return newRemoteStorageClient, nil
 }

weed/remote_storage/s3/aliyun.go

@@ -0,0 +1,49 @@
+package s3
+
+import (
+	"fmt"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
+	"github.com/chrislusf/seaweedfs/weed/remote_storage"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"os"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["aliyun"] = new(AliyunRemoteStorageMaker)
+}
+
+type AliyunRemoteStorageMaker struct{}
+
+func (s AliyunRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s AliyunRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	client := &s3RemoteStorageClient{
+		conf: conf,
+	}
+	accessKey := util.Nvl(conf.AliyunAccessKey, os.Getenv("ALICLOUD_ACCESS_KEY_ID"))
+	secretKey := util.Nvl(conf.AliyunSecretKey, os.Getenv("ALICLOUD_ACCESS_KEY_SECRET"))
+
+	config := &aws.Config{
+		Endpoint:                      aws.String(conf.AliyunEndpoint),
+		Region:                        aws.String(conf.AliyunRegion),
+		S3ForcePathStyle:              aws.Bool(false),
+		S3DisableContentMD5Validation: aws.Bool(true),
+	}
+	if accessKey != "" && secretKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "")
+	}
+
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return nil, fmt.Errorf("create aliyun session: %v", err)
+	}
+	sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
+	client.conn = s3.New(sess)
+	return client, nil
+}

weed/remote_storage/s3/backblaze.go

@@ -0,0 +1,44 @@
+package s3
+
+import (
+	"fmt"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
+	"github.com/chrislusf/seaweedfs/weed/remote_storage"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["b2"] = new(BackBlazeRemoteStorageMaker)
+}
+
+type BackBlazeRemoteStorageMaker struct{}
+
+func (s BackBlazeRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s BackBlazeRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	client := &s3RemoteStorageClient{
+		conf: conf,
+	}
+	config := &aws.Config{
+		Endpoint:                      aws.String(conf.BackblazeEndpoint),
+		Region:                        aws.String("us-west-002"),
+		S3ForcePathStyle:              aws.Bool(true),
+		S3DisableContentMD5Validation: aws.Bool(true),
+	}
+	if conf.BackblazeKeyId != "" && conf.BackblazeApplicationKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(conf.BackblazeKeyId, conf.BackblazeApplicationKey, "")
+	}
+
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return nil, fmt.Errorf("create backblaze session: %v", err)
+	}
+	sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
+	client.conn = s3.New(sess)
+	return client, nil
+}

weed/remote_storage/s3/baidu.go

@@ -0,0 +1,49 @@
+package s3
+
+import (
+	"fmt"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
+	"github.com/chrislusf/seaweedfs/weed/remote_storage"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"os"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["baidu"] = new(BaiduRemoteStorageMaker)
+}
+
+type BaiduRemoteStorageMaker struct{}
+
+func (s BaiduRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s BaiduRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	client := &s3RemoteStorageClient{
+		conf: conf,
+	}
+	accessKey := util.Nvl(conf.BaiduAccessKey, os.Getenv("BDCLOUD_ACCESS_KEY"))
+	secretKey := util.Nvl(conf.BaiduSecretKey, os.Getenv("BDCLOUD_SECRET_KEY"))
+
+	config := &aws.Config{
+		Endpoint:                      aws.String(conf.BaiduEndpoint),
+		Region:                        aws.String(conf.BaiduRegion),
+		S3ForcePathStyle:              aws.Bool(true),
+		S3DisableContentMD5Validation: aws.Bool(true),
+	}
+	if accessKey != "" && secretKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "")
+	}
+
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return nil, fmt.Errorf("create baidu session: %v", err)
+	}
+	sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
+	client.conn = s3.New(sess)
+	return client, nil
+}

weed/remote_storage/s3/filebase.go

@@ -0,0 +1,51 @@
+package s3
+
+import (
+	"fmt"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
+	"github.com/chrislusf/seaweedfs/weed/remote_storage"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"os"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["filebase"] = new(FilebaseRemoteStorageMaker)
+}
+
+type FilebaseRemoteStorageMaker struct{}
+
+func (s FilebaseRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s FilebaseRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	client := &s3RemoteStorageClient{
+		conf: conf,
+	}
+	accessKey := util.Nvl(conf.FilebaseAccessKey, os.Getenv("AWS_ACCESS_KEY_ID"))
+	secretKey := util.Nvl(conf.FilebaseSecretKey, os.Getenv("AWS_SECRET_ACCESS_KEY"))
+
+	config := &aws.Config{
+		Endpoint:                      aws.String(conf.FilebaseEndpoint),
+		Region:                        aws.String("us-east-1"),
+		S3ForcePathStyle:              aws.Bool(true),
+		S3DisableContentMD5Validation: aws.Bool(true),
+	}
+	if accessKey != "" && secretKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "")
+	}
+
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return nil, fmt.Errorf("create filebase session: %v", err)
+	}
+	sess.Handlers.Sign.PushBackNamed(v4.SignRequestHandler)
+	sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
+	client.conn = s3.New(sess)
+	return client, nil
+}

weed/remote_storage/s3/s3_storage_client.go

@@ -4,12 +4,15 @@ import (
 	"fmt"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
+	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3iface"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
 	"github.com/chrislusf/seaweedfs/weed/remote_storage"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"io"
@@ -22,14 +25,19 @@ func init() {
 
 type s3RemoteStorageMaker struct{}
 
-func (s s3RemoteStorageMaker) Make(conf *filer_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+func (s s3RemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s s3RemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
 	client := &s3RemoteStorageClient{
 		conf: conf,
 	}
 	config := &aws.Config{
-		Region:           aws.String(conf.S3Region),
-		Endpoint:         aws.String(conf.S3Endpoint),
-		S3ForcePathStyle: aws.Bool(true),
+		Region:                        aws.String(conf.S3Region),
+		Endpoint:                      aws.String(conf.S3Endpoint),
+		S3ForcePathStyle:              aws.Bool(conf.S3ForcePathStyle),
+		S3DisableContentMD5Validation: aws.Bool(true),
 	}
 	if conf.S3AccessKey != "" && conf.S3SecretKey != "" {
 		config.Credentials = credentials.NewStaticCredentials(conf.S3AccessKey, conf.S3SecretKey, "")
@@ -39,18 +47,25 @@ func (s s3RemoteStorageMaker) Make(conf *filer_pb.RemoteConf) (remote_storage.Re
 	if err != nil {
 		return nil, fmt.Errorf("create aws session: %v", err)
 	}
+	if conf.S3V4Signature {
+		sess.Handlers.Sign.PushBackNamed(v4.SignRequestHandler)
+	}
+	sess.Handlers.Build.PushBack(func(r *request.Request) {
+		r.HTTPRequest.Header.Set("User-Agent", "SeaweedFS/"+util.VERSION_NUMBER)
+	})
+	sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
 	client.conn = s3.New(sess)
 	return client, nil
 }
 
 type s3RemoteStorageClient struct {
-	conf *filer_pb.RemoteConf
+	conf *remote_pb.RemoteConf
 	conn s3iface.S3API
 }
 
 var _ = remote_storage.RemoteStorageClient(&s3RemoteStorageClient{})
 
-func (s *s3RemoteStorageClient) Traverse(remote *filer_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {
+func (s *s3RemoteStorageClient) Traverse(remote *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {
 
 	pathKey := remote.Path[1:]
 
@@ -92,7 +107,7 @@ func (s *s3RemoteStorageClient) Traverse(remote *filer_pb.RemoteStorageLocation,
 	}
 	return
 }
-func (s *s3RemoteStorageClient) ReadFile(loc *filer_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) {
+func (s *s3RemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) {
 	downloader := s3manager.NewDownloaderWithClient(s.conn, func(u *s3manager.Downloader) {
 		u.PartSize = int64(4 * 1024 * 1024)
 		u.Concurrency = 1
@@ -113,11 +128,15 @@ func (s *s3RemoteStorageClient) ReadFile(loc *filer_pb.RemoteStorageLocation, of
 	return writerAt.Bytes(), nil
 }
 
-func (s *s3RemoteStorageClient) WriteDirectory(loc *filer_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) {
+func (s *s3RemoteStorageClient) WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) {
 	return nil
 }
 
-func (s *s3RemoteStorageClient) WriteFile(loc *filer_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) {
+func (s *s3RemoteStorageClient) RemoveDirectory(loc *remote_pb.RemoteStorageLocation) (err error) {
+	return nil
+}
+
+func (s *s3RemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) {
 
 	fileSize := int64(filer.FileSize(entry))
 
@@ -129,7 +148,7 @@ func (s *s3RemoteStorageClient) WriteFile(loc *filer_pb.RemoteStorageLocation, e
 	// Create an uploader with the session and custom options
 	uploader := s3manager.NewUploaderWithClient(s.conn, func(u *s3manager.Uploader) {
 		u.PartSize = partSize
-		u.Concurrency = 5
+		u.Concurrency = 1
 	})
 
 	// process tagging
@@ -143,15 +162,16 @@ func (s *s3RemoteStorageClient) WriteFile(loc *filer_pb.RemoteStorageLocation, e
 
 	// Upload the file to S3.
 	_, err = uploader.Upload(&s3manager.UploadInput{
-		Bucket:               aws.String(loc.Bucket),
-		Key:                  aws.String(loc.Path[1:]),
-		Body:                 reader,
-		Tagging:              aws.String(tags),
+		Bucket:       aws.String(loc.Bucket),
+		Key:          aws.String(loc.Path[1:]),
+		Body:         reader,
+		Tagging:      aws.String(tags),
+		StorageClass: aws.String(s.conf.S3StorageClass),
 	})
 
 	//in case it fails to upload
 	if err != nil {
-		return nil, fmt.Errorf("upload to s3 %s/%s%s: %v", loc.Name, loc.Bucket, loc.Path, err)
+		return nil, fmt.Errorf("upload to %s/%s%s: %v", loc.Name, loc.Bucket, loc.Path, err)
 	}
 
 	// read back the remote entry
@@ -170,7 +190,7 @@ func toTagging(attributes map[string][]byte) *s3.Tagging {
 	return tagging
 }
 
-func (s *s3RemoteStorageClient) readFileRemoteEntry(loc *filer_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
+func (s *s3RemoteStorageClient) readFileRemoteEntry(loc *remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
 	resp, err := s.conn.HeadObject(&s3.HeadObjectInput{
 		Bucket: aws.String(loc.Bucket),
 		Key:    aws.String(loc.Path[1:]),
@@ -188,7 +208,7 @@ func (s *s3RemoteStorageClient) readFileRemoteEntry(loc *filer_pb.RemoteStorageL
 
 }
 
-func (s *s3RemoteStorageClient) UpdateFileMetadata(loc *filer_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error) {
+func (s *s3RemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error) {
 	if reflect.DeepEqual(oldEntry.Extended, newEntry.Extended) {
 		return nil
 	}
@@ -207,10 +227,52 @@ func (s *s3RemoteStorageClient) UpdateFileMetadata(loc *filer_pb.RemoteStorageLo
 	}
 	return
 }
-func (s *s3RemoteStorageClient) DeleteFile(loc *filer_pb.RemoteStorageLocation) (err error) {
+func (s *s3RemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) {
 	_, err = s.conn.DeleteObject(&s3.DeleteObjectInput{
 		Bucket: aws.String(loc.Bucket),
 		Key:    aws.String(loc.Path[1:]),
 	})
 	return
 }
+
+func (s *s3RemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket, err error) {
+	resp, err := s.conn.ListBuckets(&s3.ListBucketsInput{})
+	if err != nil {
+		return nil, fmt.Errorf("list buckets: %v", err)
+	}
+	for _, b := range resp.Buckets {
+		buckets = append(buckets, &remote_storage.Bucket{
+			Name:      *b.Name,
+			CreatedAt: *b.CreationDate,
+		})
+	}
+	return
+}
+
+func (s *s3RemoteStorageClient) CreateBucket(name string) (err error) {
+	_, err = s.conn.CreateBucket(&s3.CreateBucketInput{
+		ACL:                        nil,
+		Bucket:                     aws.String(name),
+		CreateBucketConfiguration:  nil,
+		GrantFullControl:           nil,
+		GrantRead:                  nil,
+		GrantReadACP:               nil,
+		GrantWrite:                 nil,
+		GrantWriteACP:              nil,
+		ObjectLockEnabledForBucket: nil,
+	})
+	if err != nil {
+		return fmt.Errorf("%s create bucket %s: %v", s.conf.Name, name, err)
+	}
+	return
+}
+
+func (s *s3RemoteStorageClient) DeleteBucket(name string) (err error) {
+	_, err = s.conn.DeleteBucket(&s3.DeleteBucketInput{
+		Bucket:                     aws.String(name),
+	})
+	if err != nil {
+		return fmt.Errorf("delete bucket %s: %v", name, err)
+	}
+	return
+}

weed/remote_storage/s3/storj.go

@@ -0,0 +1,49 @@
+package s3
+
+import (
+	"fmt"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
+	"github.com/chrislusf/seaweedfs/weed/remote_storage"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"os"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["storj"] = new(StorjRemoteStorageMaker)
+}
+
+type StorjRemoteStorageMaker struct{}
+
+func (s StorjRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s StorjRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	client := &s3RemoteStorageClient{
+		conf: conf,
+	}
+	accessKey := util.Nvl(conf.StorjAccessKey, os.Getenv("AWS_ACCESS_KEY_ID"))
+	secretKey := util.Nvl(conf.StorjSecretKey, os.Getenv("AWS_SECRET_ACCESS_KEY"))
+
+	config := &aws.Config{
+		Endpoint:                      aws.String(conf.StorjEndpoint),
+		Region:                        aws.String("us-west-2"),
+		S3ForcePathStyle:              aws.Bool(true),
+		S3DisableContentMD5Validation: aws.Bool(true),
+	}
+	if accessKey != "" && secretKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "")
+	}
+
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return nil, fmt.Errorf("create storj session: %v", err)
+	}
+	sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
+	client.conn = s3.New(sess)
+	return client, nil
+}

weed/remote_storage/s3/tencent.go

@@ -0,0 +1,49 @@
+package s3
+
+import (
+	"fmt"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
+	"github.com/chrislusf/seaweedfs/weed/remote_storage"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"os"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["tencent"] = new(TencentRemoteStorageMaker)
+}
+
+type TencentRemoteStorageMaker struct{}
+
+func (s TencentRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s TencentRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	client := &s3RemoteStorageClient{
+		conf: conf,
+	}
+	accessKey := util.Nvl(conf.TencentSecretId, os.Getenv("COS_SECRETID"))
+	secretKey := util.Nvl(conf.TencentSecretKey, os.Getenv("COS_SECRETKEY"))
+
+	config := &aws.Config{
+		Endpoint:                      aws.String(conf.TencentEndpoint),
+		Region:                        aws.String("us-west-2"),
+		S3ForcePathStyle:              aws.Bool(true),
+		S3DisableContentMD5Validation: aws.Bool(true),
+	}
+	if accessKey != "" && secretKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "")
+	}
+
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return nil, fmt.Errorf("create tencent session: %v", err)
+	}
+	sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
+	client.conn = s3.New(sess)
+	return client, nil
+}