2.47

fix https://github.com/chrislusf/seaweedfs/issues/2062

.github/workflows/release.yml

@@ -33,13 +33,15 @@ jobs:
         run: echo BUILD_TIME=$(date -u +%Y-%m-%d-%H-%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries
-        uses: wangyoucao577/go-release-action@v1.14
+        uses: wangyoucao577/go-release-action@v1.17
         with:
+          goversion: 1.16
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
           goarch: ${{ matrix.goarch }}
           release_tag: dev
           overwrite: true
+          pre_command: export CGO_ENABLED=0
           build_flags: -tags 5BytesOffset # optional, default is
           ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
           # Where to run `go build .`
@@ -48,15 +50,17 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries
-        uses: wangyoucao577/go-release-action@v1.14
+        uses: wangyoucao577/go-release-action@v1.17
         with:
+          goversion: 1.16
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
           goarch: ${{ matrix.goarch }}
           release_tag: dev
           overwrite: true
+          pre_command: export CGO_ENABLED=0
           ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
-          binary_name: weed-
+          binary_name: weed
           asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"

.travis.yml

@@ -1,7 +1,6 @@
 sudo: false
 language: go
 go:
-  - 1.15.x
   - 1.16.x
 
 before_install:

README.md

@@ -7,6 +7,7 @@
 [![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed)
 [![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki)
 [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
+[![SeaweedFS on Maven Central](https://img.shields.io/maven-central/v/com.github.chrislusf/seaweedfs-client)](https://search.maven.org/search?q=g:com.github.chrislusf)
 
 
 ![SeaweedFS Logo](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/seaweedfs.png)
@@ -42,6 +43,7 @@ Your support will be really appreciated by me and other supporters!
 - [SeaweedFS on Twitter](https://twitter.com/SeaweedFS)
 - [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
 - [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
+- [SeaweedFS White Paper](https://github.com/chrislusf/seaweedfs/wiki/SeaweedFS_Architecture.pdf)
 - [SeaweedFS Introduction Slides](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
 
 Table of Contents
@@ -66,11 +68,15 @@ Table of Contents
 * [License](#license)
 
 
-## Quick Start ##
+## Quick Start with single binary ##
 * Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
 * Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
 
-Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thoudsands of machines. That is it!
+Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
+
+## Quick Start for S3 API on Docker ##
+
+`docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
 
 ## Introduction ##
 

docker/Dockerfile

@@ -1,6 +1,7 @@
 FROM alpine
 
-ARG RELEASE=latest # 'latest' or 'dev'
+# 'latest' or 'dev'
+ARG RELEASE=latest
 
 RUN \
     ARCH=$(if [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "64" ]; then echo "amd64"; \

docker/Dockerfile.go_build

@@ -1,16 +1,16 @@
-FROM frolvlad/alpine-glibc as builder
-RUN apk add git go g++ fuse
+FROM amd64/golang:1.16-alpine as builder
+RUN apk add git g++ fuse
 RUN mkdir -p /go/src/github.com/chrislusf/
 RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
 ARG BRANCH=${BRANCH:-master}
 RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
 RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
   && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
-  && go install -ldflags "${LDFLAGS}"
+  && CGO_ENABLED=0 go install -ldflags "-extldflags -static ${LDFLAGS}"
 
 FROM alpine AS final
 LABEL author="Chris Lu"
-COPY --from=builder /root/go/bin/weed /usr/bin/
+COPY --from=builder /go/bin/weed /usr/bin/
 RUN mkdir -p /etc/seaweedfs
 COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
 COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh

docker/Dockerfile.go_build_large

@@ -1,16 +1,16 @@
-FROM frolvlad/alpine-glibc as builder
-RUN apk add git go g++ fuse
+FROM amd64/golang:1.16-alpine as builder
+RUN apk add git g++ fuse
 RUN mkdir -p /go/src/github.com/chrislusf/
 RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
 ARG BRANCH=${BRANCH:-master}
 RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
 RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
   && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
-  && go install -tags 5BytesOffset -ldflags "${LDFLAGS}"
+  && CGO_ENABLED=0 go install -tags 5BytesOffset -ldflags "-extldflags -static ${LDFLAGS}"
 
 FROM alpine AS final
 LABEL author="Chris Lu"
-COPY --from=builder /root/go/bin/weed /usr/bin/
+COPY --from=builder /go/bin/weed /usr/bin/
 RUN mkdir -p /etc/seaweedfs
 COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
 COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh

docker/Dockerfile.s3tests

@@ -28,4 +28,4 @@ ENV \
     S3TEST_CONF="/s3test.conf"
 
 ENTRYPOINT ["/bin/bash", "-c"]
-CMD ["exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"]
+CMD ["sleep 10 && exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"]

docker/Makefile

@@ -4,8 +4,12 @@ all: gen
 
 gen: dev
 
-build:
-	cd ../weed; GOOS=linux go build; mv weed ../docker/
+binary:
+	export SWCOMMIT=$(shell git rev-parse --short HEAD)
+	export SWLDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
+	cd ../weed; CGO_ENABLED=0 GOOS=linux go build -ldflags "-extldflags -static $(SWLDFLAGS)"; mv weed ../docker/
+
+build: binary
 	docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
 	rm ./weed
 
@@ -15,9 +19,15 @@ s3tests_build:
 dev: build
 	docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
 
+dev_tls: build certstrap
+	ENV_FILE="tls.env" docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
+
 dev_mount: build
 	docker-compose -f compose/local-mount-compose.yml -p seaweedfs up
 
+profile_mount: build
+	docker-compose -f compose/local-mount-profile-compose.yml -p seaweedfs up
+
 k8s: build
 	docker-compose -f compose/local-k8s-compose.yml -p seaweedfs up
 
@@ -41,3 +51,15 @@ filer_etcd: build
 
 clean:
 	rm ./weed
+
+certstrap:
+	go get github.com/square/certstrap
+	certstrap --depot-path compose/tls init --passphrase "" --common-name "SeaweedFS CA" || true
+	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name volume01.dev || true
+	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name master01.dev || true
+	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name filer01.dev || true
+	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name client01.dev || true
+	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" volume01.dev || true
+	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" master01.dev || true
+	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" filer01.dev || true
+	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" client01.dev || true

docker/compose/local-dev-compose.yml

@@ -6,33 +6,50 @@ services:
     ports:
       - 9333:9333
       - 19333:19333
-    command: "master -ip=master"
+    command: "-v=1 master -ip=master"
+    volumes:
+      - ./tls:/etc/seaweedfs/tls
+    env_file:
+     - ${ENV_FILE:-dev.env}
   volume:
     image: chrislusf/seaweedfs:local
     ports:
       - 8080:8080
       - 18080:18080
-    command: "volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
+    command: "-v=1 volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
     depends_on:
       - master
+    volumes:
+      - ./tls:/etc/seaweedfs/tls
+    env_file:
+      - ${ENV_FILE:-dev.env}
   filer:
     image: chrislusf/seaweedfs:local
     ports:
+      - 8111:8111
       - 8888:8888
       - 18888:18888
-    command: 'filer -master="master:9333"'
+    command: '-v=1 filer -master="master:9333" -iam'
     depends_on:
       - master
       - volume
+    volumes:
+      - ./tls:/etc/seaweedfs/tls
+    env_file:
+      - ${ENV_FILE:-dev.env}
   s3:
     image: chrislusf/seaweedfs:local
     ports:
       - 8333:8333
-    command: 's3 -filer="filer:8888"'
+    command: '-v=1 s3 -filer="filer:8888"'
     depends_on:
       - master
       - volume
       - filer
+    volumes:
+      - ./tls:/etc/seaweedfs/tls
+    env_file:
+      - ${ENV_FILE:-dev.env}
   mount:
     image: chrislusf/seaweedfs:local
     privileged: true
@@ -40,6 +57,10 @@ services:
       - SYS_ADMIN
     mem_limit: 4096m
     command: '-v=4 mount -filer="filer:8888" -dirAutoCreate -dir=/mnt/seaweedfs -cacheCapacityMB=100 -concurrentWriters=128'
+    volumes:
+      - ./tls:/etc/seaweedfs/tls
+    env_file:
+      - ${ENV_FILE:-dev.env}
     depends_on:
       - master
       - volume

docker/compose/local-minio-gateway-compose.yml

@@ -0,0 +1,50 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=master -volumeSizeLimitMB=1024"
+  volume:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8080:8080
+      - 18080:18080
+    command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
+    depends_on:
+      - master
+  s3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8888:8888
+      - 18888:18888
+      - 8333:8333
+    command: '-v 1 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333'
+    volumes:
+      - ./s3.json:/etc/seaweedfs/s3.json
+    depends_on:
+      - master
+      - volume
+  minio-gateway-s3:
+    image: minio/minio
+    ports:
+      - 9000:9000
+    command: 'minio gateway s3 http://s3:8333'
+    restart: on-failure
+    environment:
+      MINIO_ACCESS_KEY: "some_access_key1"
+      MINIO_SECRET_KEY: "some_secret_key1"
+    depends_on:
+      - s3
+  minio-warp:
+    image: minio/warp
+    command: 'mixed --duration=5m --obj.size=3mb --autoterm'
+    restart: on-failure
+    environment:
+      WARP_HOST: "minio-gateway-s3:9000"
+      WARP_ACCESS_KEY: "some_access_key1"
+      WARP_SECRET_KEY: "some_secret_key1"
+    depends_on:
+      - minio-gateway-s3

docker/compose/local-mount-compose.yml

@@ -38,7 +38,7 @@ services:
   mount_2:
     image: chrislusf/seaweedfs:local
     privileged: true
-    entrypoint: '/bin/sh -c "mkdir -p t2 && mkdir -p cache/t2 && weed -v=4  mount -filer=filer:8888 -cacheDir=./cache/t2 -dir=./t2 -filer.path=/c1 -volumeServerAcess=publicUrl"'
+    entrypoint: '/bin/sh -c "mkdir -p t2 && mkdir -p cache/t2 && weed -v=4  mount -filer=filer:8888 -cacheDir=./cache/t2 -dir=./t2 -filer.path=/c1"'
     depends_on:
       - master
       - volume

docker/compose/local-mount-profile-compose.yml

@@ -0,0 +1,47 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=master"
+  volume:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 7455:8080
+      - 9325:9325
+    volumes:
+      - /Volumes/mobile_disk/99:/data
+    command: 'volume -mserver="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455'
+    depends_on:
+      - master
+  filer:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8888:8888
+      - 18888:18888
+      - 9326:9326
+    volumes:
+      - /Volumes/mobile_disk/99:/data
+    command: 'filer -master="master:9333"  -metricsPort=9326'
+    tty: true
+    stdin_open: true
+    depends_on:
+      - master
+      - volume
+  mount:
+    image: chrislusf/seaweedfs:local
+    privileged: true
+    cap_add:
+      - SYS_ADMIN
+    devices:
+      - fuse
+    volumes:
+      - /Volumes/mobile_disk/99:/data
+    entrypoint: '/bin/sh -c "mkdir -p t1 && weed mount -filer=filer:8888 -dir=./t1 -cacheCapacityMB=0 -memprofile=/data/mount.mem.pprof"'
+    depends_on:
+      - master
+      - volume
+      - filer

docker/compose/local-registry-compose.yml

@@ -15,24 +15,18 @@ services:
     command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
     depends_on:
       - master
-  filer:
+  s3:
     image: chrislusf/seaweedfs:local
     ports:
       - 8888:8888
       - 18888:18888
-    command: 'filer -master="master:9333"'
-    depends_on:
-      - master
-      - volume
-  s3:
-    image: chrislusf/seaweedfs:local
-    ports:
       - 8333:8333
-    command: '-v 9 s3 -filer="filer:8888"'
+    command: '-v 9 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333'
+    volumes:
+      - ./s3.json:/etc/seaweedfs/s3.json
     depends_on:
       - master
       - volume
-      - filer
   minio:
     image: minio/minio
     ports:

docker/compose/local-s3tests-compose.yml

@@ -38,7 +38,7 @@ services:
       S3TEST_CONF: "s3tests.conf"
       NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
       NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
-      NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_delimiter_not_skip_special|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_head_zero_bytes|object_write_cache_control|object_write_expires|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifmatch_failed|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_authenticated|object_raw_response_headers|object_raw_authenticated_bucket_acl|object_raw_authenticated_object_acl|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_not_expired|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_create_exists|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|bucket_create_special_key_names|object_copy_zero_size|object_copy_verify_contenttype|object_copy_to_itself|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
+      NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_write_cache_control|object_write_expires|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifmatch_failed|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
     depends_on:
       - master
       - volume

docker/compose/tls.env

@@ -0,0 +1,14 @@
+WEED_GRPC_CA=/etc/seaweedfs/tls/SeaweedFS_CA.crt
+WEED_GRPC_ALLOWED_WILDCARD_DOMAIN=".dev"
+WEED_GRPC_MASTER_CERT=/etc/seaweedfs/tls/master01.dev.crt
+WEED_GRPC_MASTER_KEY=/etc/seaweedfs/tls/master01.dev.key
+WEED_GRPC_VOLUME_CERT=/etc/seaweedfs/tls/volume01.dev.crt
+WEED_GRPC_VOLUME_KEY=/etc/seaweedfs/tls/volume01.dev.key
+WEED_GRPC_FILER_CERT=/etc/seaweedfs/tls/filer01.dev.crt
+WEED_GRPC_FILER_KEY=/etc/seaweedfs/tls/filer01.dev.key
+WEED_GRPC_CLIENT_CERT=/etc/seaweedfs/tls/client01.dev.crt
+WEED_GRPC_CLIENT_KEY=/etc/seaweedfs/tls/client01.dev.key
+WEED_GRPC_MASTER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
+WEED_GRPC_VOLUME_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
+WEED_GRPC_FILER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
+WEED_GRPC_CLIENT_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"

docker/entrypoint.sh

@@ -60,9 +60,9 @@ case "$1" in
   'cronjob')
 	MASTER=${WEED_MASTER-localhost:9333}
 	FIX_REPLICATION_CRON_SCHEDULE=${CRON_SCHEDULE-*/7 * * * * *}
-	echo "$FIX_REPLICATION_CRON_SCHEDULE" 'echo "volume.fix.replication" | weed shell -master='$MASTER > /crontab
+	echo "$FIX_REPLICATION_CRON_SCHEDULE" 'echo "lock; volume.fix.replication; unlock" | weed shell -master='$MASTER > /crontab
 	BALANCING_CRON_SCHEDULE=${CRON_SCHEDULE-25 * * * * *}
-	echo "$BALANCING_CRON_SCHEDULE" 'echo "volume.balance -c ALL -force" | weed shell -master='$MASTER >> /crontab
+	echo "$BALANCING_CRON_SCHEDULE" 'echo "lock; volume.balance -collection ALL_COLLECTIONS -force; unlock" | weed shell -master='$MASTER >> /crontab
 	echo "Running Crontab:"
 	cat /crontab
 	exec supercronic /crontab

go.mod

@@ -1,6 +1,6 @@
 module github.com/chrislusf/seaweedfs
 
-go 1.12
+go 1.16
 
 require (
 	cloud.google.com/go v0.58.0 // indirect
@@ -13,8 +13,9 @@ require (
 	github.com/Shopify/sarama v1.23.1
 	github.com/aws/aws-sdk-go v1.34.30
 	github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
+	github.com/bwmarrin/snowflake v0.3.0
 	github.com/cespare/xxhash v1.1.0
-	github.com/chrislusf/raft v1.0.4
+	github.com/chrislusf/raft v1.0.6
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/disintegration/imaging v1.6.2
@@ -38,8 +39,10 @@ require (
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/mux v1.7.4
 	github.com/gorilla/websocket v1.4.1 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
 	github.com/jcmturner/gofork v1.0.0 // indirect
+	github.com/jinzhu/copier v0.2.8
 	github.com/json-iterator/go v1.1.10
 	github.com/karlseguin/ccache v2.0.3+incompatible // indirect
 	github.com/karlseguin/ccache/v2 v2.0.7
@@ -48,7 +51,7 @@ require (
 	github.com/klauspost/crc32 v1.2.0
 	github.com/klauspost/reedsolomon v1.9.2
 	github.com/kurin/blazer v0.5.3
-	github.com/lib/pq v1.2.0
+	github.com/lib/pq v1.10.0
 	github.com/lunixbochs/vtclean v1.0.0 // indirect
 	github.com/magiconair/properties v1.8.1 // indirect
 	github.com/mattn/go-colorable v0.1.2 // indirect
@@ -57,9 +60,8 @@ require (
 	github.com/peterh/liner v1.1.0
 	github.com/pierrec/lz4 v2.2.7+incompatible // indirect
 	github.com/prometheus/client_golang v1.3.0
-	github.com/rakyll/statik v0.1.7
 	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
-	github.com/seaweedfs/fuse v1.1.1
+	github.com/seaweedfs/fuse v1.1.6
 	github.com/seaweedfs/goexif v1.0.2
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
@@ -73,7 +75,6 @@ require (
 	github.com/tidwall/match v1.0.1
 	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
 	github.com/valyala/bytebufferpool v1.0.0
-	github.com/valyala/fasthttp v1.20.0
 	github.com/viant/assertly v0.5.4 // indirect
 	github.com/viant/ptrie v0.3.0
 	github.com/viant/toolbox v0.33.2 // indirect

go.sum

@@ -141,6 +141,8 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 h1:fUmDBbSvv1uOzo/t8WaxZMVb7BxJ8JECo5lGoR9c5bA=
 github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72/go.mod h1:OEE5igu/CDjGegM1Jn6ZMo7R6LlV/JChAkjfQQIRLpg=
+github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
+github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4=
@@ -153,6 +155,10 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chrislusf/raft v1.0.4 h1:THhbsVik2hxdE0/VXX834f64Wn9RzgVPp+E+XCWZdKM=
 github.com/chrislusf/raft v1.0.4/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68=
+github.com/chrislusf/raft v1.0.5 h1:g8GxKCSStfm0/bGBDpNEbmEXL6MJkpXX+NI0ksbX5D4=
+github.com/chrislusf/raft v1.0.5/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68=
+github.com/chrislusf/raft v1.0.6 h1:wunb85WWhMKhNRn7EmdIw35D4Lmew0ZJv8oYDizR/+Y=
+github.com/chrislusf/raft v1.0.6/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -433,6 +439,8 @@ github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03 h1:FUwcHNlEqkqLjL
 github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
 github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
 github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
+github.com/jinzhu/copier v0.2.8 h1:N8MbL5niMwE3P4dOwurJixz5rMkKfujmMRFmAanSzWE=
+github.com/jinzhu/copier v0.2.8/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -496,6 +504,8 @@ github.com/kurin/blazer v0.5.3/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E=
+github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
 github.com/lunixbochs/vtclean v1.0.0 h1:xu2sLAri4lGiovBDQKxl5mrXyESr3gUr5m5SM5+LVb8=
@@ -681,6 +691,14 @@ github.com/seaweedfs/fuse v1.1.0 h1:cL1qPHFNtFv0UuJTLjKKgWDzfJ4iZzTa4Y7ipc2acGw=
 github.com/seaweedfs/fuse v1.1.0/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8=
 github.com/seaweedfs/fuse v1.1.1 h1:WD51YFJcBViOx8I89jeqPD+vAKl4EowzBy9GUw0plb0=
 github.com/seaweedfs/fuse v1.1.1/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8=
+github.com/seaweedfs/fuse v1.1.3 h1:0DddotXwSRGbYG2kynoJyr8GHCy30Z2SpdhP3vdyijY=
+github.com/seaweedfs/fuse v1.1.3/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8=
+github.com/seaweedfs/fuse v1.1.4 h1:YYqkK86agMhXRSwR+wFbRI8ikMgk3kL6PNTna1MAHyQ=
+github.com/seaweedfs/fuse v1.1.4/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8=
+github.com/seaweedfs/fuse v1.1.5 h1:wyuRh/mDvrvt8ZLDS7YdPSe6nczniSx4sQFs/Jonveo=
+github.com/seaweedfs/fuse v1.1.5/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8=
+github.com/seaweedfs/fuse v1.1.6 h1:kvCqaIsCEaYOBw5r8kJPUs9GcbwlIKcScnkPLT7HLuQ=
+github.com/seaweedfs/fuse v1.1.6/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8=
 github.com/seaweedfs/goexif v1.0.2 h1:p+rTXYdQ2mgxd+1JaTrQ9N8DvYuw9UH9xgYmJ+Bb29E=
 github.com/seaweedfs/goexif v1.0.2/go.mod h1:MrKs5LK0HXdffrdCZrW3OIMegL2xXpC6ThLyXMyjdrk=
 github.com/secsy/goftp v0.0.0-20190720192957-f31499d7c79a h1:C6IhVTxNkhlb0tlCB6JfHOUv1f0xHPK7V8X4HlJZEJw=

k8s/seaweedfs/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "2.30"
-version: 2.30
+appVersion: "2.47"
+version: 2.47

k8s/seaweedfs/templates/volume-service.yaml

@@ -23,6 +23,6 @@ spec:
     targetPort: {{ .Values.volume.metricsPort }}
     protocol: TCP
 {{- end }}
-selector:
+  selector:
     app: {{ template "seaweedfs.name" . }}
     component: volume

k8s/seaweedfs/values.yaml

@@ -4,7 +4,7 @@ global:
   registry: ""
   repository: ""
   imageName: chrislusf/seaweedfs
-  # imageTag: "2.30" - started using {.Chart.appVersion}
+  # imageTag: "2.47" - started using {.Chart.appVersion}
   imagePullPolicy: IfNotPresent
   imagePullSecrets: imagepullsecret
   restartPolicy: Always

other/java/client/pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.2</version>
+    <version>1.6.4</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -17,7 +17,7 @@
         <protobuf.version>3.9.1</protobuf.version>
         <!-- follow https://github.com/grpc/grpc-java -->
         <grpc.version>1.23.0</grpc.version>
-        <guava.version>28.0-jre</guava.version>
+        <guava.version>30.0-jre</guava.version>
     </properties>
 
     <dependencies>

other/java/client/pom.xml.deploy

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.2</version>
+    <version>1.6.4</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/pom_debug.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.2</version>
+    <version>1.6.4</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/src/main/java/seaweedfs/client/ByteBufferPool.java

@@ -3,6 +3,7 @@ package seaweedfs.client;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.nio.Buffer;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
@@ -34,7 +35,7 @@ public class ByteBufferPool {
     }
 
     public static synchronized void release(ByteBuffer obj) {
-        obj.clear();
+        ((Buffer)obj).clear();
         bufferList.add(0, obj);
     }
 

other/java/client/src/main/java/seaweedfs/client/FilerClient.java

@@ -94,7 +94,7 @@ public class FilerClient extends FilerGrpcClient {
             return true;
         }
         File pathFile = new File(path);
-        String parent = pathFile.getParent();
+        String parent = pathFile.getParent().replace('\\','/');
         String name = pathFile.getName();
 
         mkdirs(parent, mode, uid, gid, userName, groupNames);
@@ -115,21 +115,33 @@ public class FilerClient extends FilerGrpcClient {
     public boolean mv(String oldPath, String newPath) {
 
         File oldPathFile = new File(oldPath);
-        String oldParent = oldPathFile.getParent();
+        String oldParent = oldPathFile.getParent().replace('\\','/');
         String oldName = oldPathFile.getName();
 
         File newPathFile = new File(newPath);
-        String newParent = newPathFile.getParent();
+        String newParent = newPathFile.getParent().replace('\\','/');
         String newName = newPathFile.getName();
 
         return atomicRenameEntry(oldParent, oldName, newParent, newName);
 
     }
 
+    public boolean exists(String path){
+        File pathFile = new File(path);
+        String parent = pathFile.getParent();
+        String entryName = pathFile.getName();
+        if(parent == null) {
+            parent = path;
+            entryName  ="";
+        }
+        return lookupEntry(parent, entryName) != null;
+
+    }
+
     public boolean rm(String path, boolean isRecursive, boolean ignoreRecusiveError) {
 
         File pathFile = new File(path);
-        String parent = pathFile.getParent();
+        String parent = pathFile.getParent().replace('\\','/');
         String name = pathFile.getName();
 
         return deleteEntry(
@@ -142,30 +154,40 @@ public class FilerClient extends FilerGrpcClient {
 
     public boolean touch(String path, int mode) {
         String currentUser = System.getProperty("user.name");
-        return touch(path, mode, 0, 0, currentUser, new String[]{});
+
+        long now = System.currentTimeMillis() / 1000L;
+        return touch(path, now, mode, 0, 0, currentUser, new String[]{});
     }
 
-    public boolean touch(String path, int mode, int uid, int gid, String userName, String[] groupNames) {
+    public boolean touch(String path, long modifiedTimeSecond, int mode, int uid, int gid, String userName, String[] groupNames) {
 
         File pathFile = new File(path);
-        String parent = pathFile.getParent();
+        String parent = pathFile.getParent().replace('\\','/');
         String name = pathFile.getName();
 
         FilerProto.Entry entry = lookupEntry(parent, name);
         if (entry == null) {
             return createEntry(
                     parent,
-                    newFileEntry(name, mode, uid, gid, userName, groupNames).build()
+                    newFileEntry(name, modifiedTimeSecond, mode, uid, gid, userName, groupNames).build()
             );
         }
-        long now = System.currentTimeMillis() / 1000L;
-        FilerProto.FuseAttributes.Builder attr = entry.getAttributes().toBuilder()
-                .setMtime(now)
-                .setUid(uid)
-                .setGid(gid)
-                .setUserName(userName)
-                .clearGroupName()
-                .addAllGroupName(Arrays.asList(groupNames));
+        FilerProto.FuseAttributes.Builder attr = entry.getAttributes().toBuilder();
+        if (modifiedTimeSecond>0) {
+            attr.setMtime(modifiedTimeSecond);
+        }
+        if (uid>0) {
+            attr.setUid(uid);
+        }
+        if (gid>0) {
+            attr.setGid(gid);
+        }
+        if (userName!=null) {
+            attr.setUserName(userName);
+        }
+        if (groupNames!=null) {
+            attr.clearGroupName().addAllGroupName(Arrays.asList(groupNames));
+        }
         return updateEntry(parent, entry.toBuilder().setAttributes(attr).build());
     }
 
@@ -188,17 +210,15 @@ public class FilerClient extends FilerGrpcClient {
                         .addAllGroupName(Arrays.asList(groupNames)));
     }
 
-    public FilerProto.Entry.Builder newFileEntry(String name, int mode,
+    public FilerProto.Entry.Builder newFileEntry(String name, long modifiedTimeSecond, int mode,
                                                  int uid, int gid, String userName, String[] groupNames) {
 
-        long now = System.currentTimeMillis() / 1000L;
-
         return FilerProto.Entry.newBuilder()
                 .setName(name)
                 .setIsDirectory(false)
                 .setAttributes(FilerProto.FuseAttributes.newBuilder()
-                        .setMtime(now)
-                        .setCrtime(now)
+                        .setMtime(modifiedTimeSecond)
+                        .setCrtime(modifiedTimeSecond)
                         .setUid(uid)
                         .setGid(gid)
                         .setFileMode(mode)

other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java

@@ -6,6 +6,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.EOFException;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
@@ -34,6 +35,10 @@ public class SeaweedInputStream extends InputStream {
         this.entry = filerClient.lookupEntry(
                 SeaweedOutputStream.getParentDirectory(fullpath),
                 SeaweedOutputStream.getFileName(fullpath));
+        if(entry == null){
+            throw new FileNotFoundException();
+        }
+
         this.contentLength = SeaweedRead.fileSize(entry);
 
         this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerClient, entry.getChunksList());

other/java/client/src/test/java/seaweedfs/client/SeaweedFilerTest.java

@@ -16,8 +16,17 @@ public class SeaweedFilerTest {
         filerClient.mkdirs("/new_folder", 0755);
         filerClient.touch("/new_folder/new_empty_file", 0755);
         filerClient.touch("/new_folder/new_empty_file2", 0755);
+        if(!filerClient.exists("/new_folder/new_empty_file")){
+            System.out.println("/new_folder/new_empty_file should exists");
+        }
+
         filerClient.rm("/new_folder/new_empty_file", false, true);
         filerClient.rm("/new_folder", true, true);
-
+        if(filerClient.exists("/new_folder/new_empty_file")){
+            System.out.println("/new_folder/new_empty_file should not exists");
+        }
+        if(!filerClient.exists("/")){
+            System.out.println("/ should exists");
+        }
     }
 }

other/java/examples/pom.xml

@@ -11,13 +11,13 @@
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-client</artifactId>
-            <version>1.6.2</version>
+            <version>1.6.4</version>
             <scope>compile</scope>
         </dependency>
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-hadoop2-client</artifactId>
-            <version>1.6.2</version>
+            <version>1.6.4</version>
             <scope>compile</scope>
         </dependency>
         <dependency>

other/java/examples/src/main/java/com/seaweedfs/examples/ExampleWriteFile2.java

@@ -0,0 +1,22 @@
+package com.seaweedfs.examples;
+
+import com.google.common.io.Files;
+import seaweedfs.client.FilerClient;
+import seaweedfs.client.SeaweedOutputStream;
+
+import java.io.File;
+import java.io.IOException;
+
+public class ExampleWriteFile2 {
+
+    public static void main(String[] args) throws IOException {
+
+        FilerClient filerClient = new FilerClient("localhost", 18888);
+
+        SeaweedOutputStream seaweedOutputStream = new SeaweedOutputStream(filerClient, "/test/1");
+        Files.copy(new File("/etc/resolv.conf"), seaweedOutputStream);
+        seaweedOutputStream.close();
+
+    }
+
+}

other/java/hdfs-over-ftp/pom.xml

@@ -0,0 +1,120 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>hdfs-over-ftp</groupId>
+    <artifactId>hdfs-over-ftp</artifactId>
+    <version>1.0</version>
+
+    <parent>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>2.4.3</version>
+    </parent>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.springfox</groupId>
+            <artifactId>springfox-swagger2</artifactId>
+            <version>2.9.2</version>
+        </dependency>
+        <dependency>
+            <groupId>io.springfox</groupId>
+            <artifactId>springfox-swagger-ui</artifactId>
+            <version>2.9.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <version>3.2.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-client</artifactId>
+            <version>3.2.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.ftpserver</groupId>
+            <artifactId>ftpserver-core</artifactId>
+            <version>1.1.1</version>
+        </dependency>
+        <dependency>
+            <groupId>com.github.chrislusf</groupId>
+            <artifactId>seaweedfs-hadoop3-client</artifactId>
+            <version>1.6.2</version>
+        </dependency>
+    </dependencies>
+
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.1</version>
+                <configuration>
+                    <source>1.8</source>
+                    <target>1.8</target>
+                    <encoding>UTF-8</encoding>
+                    <compilerArguments>
+                        <verbose />
+                        <bootclasspath>${java.home}/lib/rt.jar</bootclasspath>
+                    </compilerArguments>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <version>2.6</version>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <mainClass>org.apache.hadoop.seaweed.ftp.ApplicationServer</mainClass>
+                            <addClasspath>true</addClasspath>
+                            <classpathPrefix>lib/</classpathPrefix>
+                        </manifest>
+                        <manifestEntries>
+                            <Class-Path>./</Class-Path>
+                        </manifestEntries>
+                    </archive>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <appendAssemblyId>false</appendAssemblyId>
+                    <descriptors>
+                        <descriptor>src/main/resources/assembly.xml</descriptor>
+                    </descriptors>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

other/java/hdfs-over-ftp/src/main/java/org/apache/hadoop/seaweed/ftp/ApplicationServer.java

@@ -0,0 +1,14 @@
+package org.apache.hadoop.seaweed.ftp;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+
+@SpringBootApplication
+public class ApplicationServer {
+
+    public static void main(String[] args) {
+        SpringApplication.run(ApplicationServer.class, args);
+    }
+
+}

other/java/hdfs-over-ftp/src/main/java/org/apache/hadoop/seaweed/ftp/config/SwaggerConfig.java

@@ -0,0 +1,27 @@
+package org.apache.hadoop.seaweed.ftp.config;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import springfox.documentation.builders.ApiInfoBuilder;
+import springfox.documentation.builders.PathSelectors;
+import springfox.documentation.builders.RequestHandlerSelectors;
+import springfox.documentation.spi.DocumentationType;
+import springfox.documentation.spring.web.plugins.Docket;
+import springfox.documentation.swagger2.annotations.EnableSwagger2;
+
+@Configuration
+@EnableSwagger2
+public class SwaggerConfig {
+    @Bean
+    public Docket createRestApi() {
+        return new Docket(DocumentationType.SWAGGER_2)
+                .pathMapping("/")
+                .select()
+                .apis(RequestHandlerSelectors.basePackage("org.apache.hadoop.seaweed.ftp"))
+                .paths(PathSelectors.any())
+                .build().apiInfo(new ApiInfoBuilder()
+                        .title("FTP API Doc")
+                        .version("1.0")
+                        .build());
+    }
+}

other/java/hdfs-over-ftp/src/main/java/org/apache/hadoop/seaweed/ftp/controller/FtpManagerController.java

@@ -0,0 +1,71 @@
+package org.apache.hadoop.seaweed.ftp.controller;
+
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.apache.hadoop.seaweed.ftp.service.HFtpService;
+import org.apache.hadoop.seaweed.ftp.controller.vo.Result;
+import org.apache.log4j.Logger;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@RestController
+@RequestMapping("/manager")
+@Api(tags = "FTP操作管理")
+public class FtpManagerController {
+
+    private static Logger log = Logger.getLogger(FtpManagerController.class);
+
+    @Autowired
+    private HFtpService hdfsOverFtpServer;
+
+    @GetMapping("/status")
+    @ApiOperation("查看FTP服务状态")
+    public Result status() {
+        Map map = new HashMap<>();
+        try {
+            boolean status = hdfsOverFtpServer.statusServer();
+            map.put("is_running", status);
+            return new Result(true, map, "FTP 服务状态获取成功");
+        }catch (Exception e) {
+            log.error(e);
+            map.put("is_running", false);
+            return new Result(true, map, "FTP 服务状态获取成功");
+        }
+    }
+
+    @PutMapping("/start")
+    @ApiOperation("启动FTP服务")
+    public Result start() {
+        try {
+            boolean status = hdfsOverFtpServer.statusServer();
+            if(!status) {
+                hdfsOverFtpServer.startServer();
+            }
+            return new Result(true, "FTP 服务启动成功");
+        }catch (Exception e) {
+            log.error(e);
+            return new Result(false, "FTP 服务启动失败");
+        }
+    }
+
+    @PutMapping("/stop")
+    @ApiOperation("停止FTP服务")
+    public Result stop() {
+        try {
+            boolean status = hdfsOverFtpServer.statusServer();
+            if(status) {
+                hdfsOverFtpServer.stopServer();
+            }
+            return new Result(true, "FTP 服务停止成功");
+        }catch (Exception e) {
+            log.error(e);
+            return new Result(false, "FTP 服务停止失败");
+        }
+    }
+}

other/java/hdfs-over-ftp/src/main/java/org/apache/hadoop/seaweed/ftp/controller/UserController.java

@@ -0,0 +1,98 @@
+package org.apache.hadoop.seaweed.ftp.controller;
+
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.apache.ftpserver.ftplet.User;
+import org.apache.ftpserver.usermanager.Md5PasswordEncryptor;
+import org.apache.ftpserver.usermanager.UserFactory;
+import org.apache.hadoop.seaweed.ftp.controller.vo.FtpUser;
+import org.apache.hadoop.seaweed.ftp.controller.vo.Result;
+import org.apache.hadoop.seaweed.ftp.users.HdfsUserManager;
+import org.apache.log4j.Logger;
+import org.springframework.web.bind.annotation.*;
+
+import java.io.File;
+
+@RestController
+@RequestMapping("/user")
+@Api(tags = "FTP用户管理")
+public class UserController {
+
+    private static Logger log = Logger.getLogger(UserController.class);
+
+    /***
+     * {
+     *     "name": "test",
+     *     "password": "test",
+     *     "homeDirectory": "/buckets/test/"
+     * }
+     * @param ftpUser
+     * @return
+     */
+    @PostMapping("/add")
+    @ApiOperation("新增/编辑用户")
+    public Result add(@RequestBody FtpUser ftpUser) {
+        try {
+            HdfsUserManager userManagerFactory = new HdfsUserManager();
+            userManagerFactory.setFile(new File(System.getProperty("user.dir") + File.separator + "users.properties"));
+            userManagerFactory.setPasswordEncryptor(new Md5PasswordEncryptor());
+
+            UserFactory userFactory = new UserFactory();
+            userFactory.setHomeDirectory(ftpUser.getHomeDirectory());
+            userFactory.setName(ftpUser.getName());
+            userFactory.setPassword(ftpUser.getPassword());
+            userFactory.setEnabled(ftpUser.isEnabled());
+            userFactory.setMaxIdleTime(ftpUser.getMaxIdleTime());
+
+            User user = userFactory.createUser();
+            userManagerFactory.save(user, ftpUser.isRenamePush());
+            return new Result(true, "新建用户成功");
+        }catch (Exception e) {
+            log.error(e);
+            return new Result(false, "新建用户失败");
+        }
+    }
+
+    @DeleteMapping("/delete/{user}")
+    @ApiOperation("删除用户")
+    public Result delete(@PathVariable(value = "user") String user) {
+        try {
+            HdfsUserManager userManagerFactory = new HdfsUserManager();
+            userManagerFactory.setFile(new File(System.getProperty("user.dir") + File.separator + "users.properties"));
+            userManagerFactory.delete(user);
+            return new Result(true, "删除用户成功");
+        }catch (Exception e) {
+            log.error(e);
+            return new Result(false, "删除用户失败");
+        }
+    }
+
+    @GetMapping("/show/{userName}")
+    @ApiOperation("查看用户")
+    public Result show(@PathVariable(value = "userName") String userName) {
+        try {
+            HdfsUserManager userManagerFactory = new HdfsUserManager();
+            userManagerFactory.setFile(new File(System.getProperty("user.dir") + File.separator + "users.properties"));
+            User user = userManagerFactory.getUserByName(userName);
+            FtpUser ftpUser = new FtpUser(user.getHomeDirectory(), user.getPassword(), user.getEnabled(), user.getName(), user.getMaxIdleTime(), HdfsUserManager.getUserRenamePush(userName));
+            return new Result(true, ftpUser, "获取用户信息成功");
+        }catch (Exception e) {
+            log.error(e);
+            return new Result(false, "获取用户信息失败");
+        }
+    }
+
+    @GetMapping("/list")
+    @ApiOperation("列举用户")
+    public Result list() {
+        try {
+            HdfsUserManager userManagerFactory = new HdfsUserManager();
+            userManagerFactory.setFile(new File(System.getProperty("user.dir") + File.separator + "users.properties"));
+            String[] allUserNames = userManagerFactory.getAllUserNames();
+            return new Result(true, allUserNames, "列举用户成功");
+        }catch (Exception e) {
+            log.error(e);
+            return new Result(false, "列举用户失败");
+        }
+    }
+}

other/java/hdfs-over-ftp/src/main/java/org/apache/hadoop/seaweed/ftp/controller/vo/FtpUser.java

@@ -0,0 +1,71 @@
+package org.apache.hadoop.seaweed.ftp.controller.vo;
+
+public class FtpUser {
+
+    private String homeDirectory;
+    private String password;
+    private boolean enabled;
+    private String name;
+    private int maxIdleTime;
+    private boolean renamePush;
+
+    public FtpUser() {
+    }
+
+    public FtpUser(String homeDirectory, String password, boolean enabled, String name, int maxIdleTime, boolean renamePush) {
+        this.homeDirectory = homeDirectory;
+        this.password = password;
+        this.enabled = enabled;
+        this.name = name;
+        this.maxIdleTime = maxIdleTime;
+        this.renamePush = renamePush;
+    }
+
+    public String getHomeDirectory() {
+        return homeDirectory;
+    }
+
+    public void setHomeDirectory(String homeDirectory) {
+        this.homeDirectory = homeDirectory;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public boolean isEnabled() {
+        return enabled;
+    }
+
+    public void setEnabled(boolean enabled) {
+        this.enabled = enabled;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public int getMaxIdleTime() {
+        return maxIdleTime;
+    }
+
+    public void setMaxIdleTime(int maxIdleTime) {
+        this.maxIdleTime = maxIdleTime;
+    }
+
+    public boolean isRenamePush() {
+        return renamePush;
+    }
+
+    public void setRenamePush(boolean renamePush) {
+        this.renamePush = renamePush;
+    }
+}

other/java/hdfs-over-ftp/src/main/java/org/apache/hadoop/seaweed/ftp/controller/vo/Result.java

@@ -0,0 +1,43 @@
+package org.apache.hadoop.seaweed.ftp.controller.vo;
+
+public class Result {
+
+    private boolean status;
+    private Object data;
+    private String message;
+
+    public Result(boolean status, String message) {
+        this.status = status;
+        this.message = message;
+    }
+
+    public Result(boolean status, Object data, String message) {
+        this.status = status;
+        this.message = message;
+        this.data = data;
+    }
+
+    public boolean isStatus() {
+        return status;
+    }
+
+    public void setStatus(boolean status) {
+        this.status = status;
+    }
+
+    public String getMessage() {
+        return message;
+    }
+
+    public void setMessage(String message) {
+        this.message = message;
+    }
+
+    public Object getData() {
+        return data;
+    }
+
+    public void setData(Object data) {
+        this.data = data;
+    }
+}

other/java/hdfs-over-ftp/src/main/java/org/apache/hadoop/seaweed/ftp/service/HFtpService.java

@@ -0,0 +1,102 @@
+package org.apache.hadoop.seaweed.ftp.service;
+
+import org.apache.ftpserver.DataConnectionConfiguration;
+import org.apache.ftpserver.DataConnectionConfigurationFactory;
+import org.apache.ftpserver.FtpServer;
+import org.apache.ftpserver.FtpServerFactory;
+import org.apache.ftpserver.command.CommandFactoryFactory;
+import org.apache.ftpserver.listener.ListenerFactory;
+import org.apache.hadoop.seaweed.ftp.service.filesystem.HdfsFileSystemManager;
+import org.apache.hadoop.seaweed.ftp.service.filesystem.HdfsOverFtpSystem;
+import org.apache.hadoop.seaweed.ftp.users.HdfsUserManager;
+import org.apache.log4j.Logger;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.io.File;
+
+/**
+ * reference: https://github.com/iponweb/hdfs-over-ftp
+ */
+@Component
+public class HFtpService {
+
+	private static Logger log = Logger.getLogger(HFtpService.class);
+
+	@Value("${ftp.port}")
+	private int port = 0;
+
+	@Value("${ftp.passive-address}")
+	private String passiveAddress;
+
+	@Value("${ftp.passive-ports}")
+	private String passivePorts;
+
+	@Value("${hdfs.uri}")
+	private String hdfsUri;
+
+	@Value("${seaweedFs.enable}")
+	private boolean seaweedFsEnable;
+
+	@Value("${seaweedFs.access}")
+	private String seaweedFsAccess;
+
+	@Value("${seaweedFs.replication}")
+	private String seaweedFsReplication;
+
+	private FtpServer ftpServer = null;
+
+	public void startServer() throws Exception {
+		log.info("Starting HDFS-Over-Ftp server. port: " + port + " passive-address: " + passiveAddress + " passive-ports: " + passivePorts + " hdfs-uri: " + hdfsUri);
+
+		HdfsOverFtpSystem.setHdfsUri(hdfsUri);
+		HdfsOverFtpSystem.setSeaweedFsEnable(seaweedFsEnable);
+		HdfsOverFtpSystem.setSeaweedFsAccess(seaweedFsAccess);
+		HdfsOverFtpSystem.setSeaweedFsReplication(seaweedFsReplication);
+
+		FtpServerFactory server = new FtpServerFactory();
+		server.setFileSystem(new HdfsFileSystemManager());
+
+		ListenerFactory factory = new ListenerFactory();
+		factory.setPort(port);
+
+		DataConnectionConfigurationFactory dccFactory = new DataConnectionConfigurationFactory();
+		dccFactory.setPassiveAddress("0.0.0.0");
+		dccFactory.setPassivePorts(passivePorts);
+		dccFactory.setPassiveExternalAddress(passiveAddress);
+		DataConnectionConfiguration dcc = dccFactory.createDataConnectionConfiguration();
+		factory.setDataConnectionConfiguration(dcc);
+
+		server.addListener("default", factory.createListener());
+
+		HdfsUserManager userManager = new HdfsUserManager();
+		final File file = loadResource("/users.properties");
+		userManager.setFile(file);
+		server.setUserManager(userManager);
+
+		CommandFactoryFactory cmFact = new CommandFactoryFactory();
+		cmFact.setUseDefaultCommands(true);
+		server.setCommandFactory(cmFact.createCommandFactory());
+
+		// start the server
+		ftpServer = server.createServer();
+		ftpServer.start();
+	}
+
+	public void stopServer() {
+		log.info("Stopping Hdfs-Over-Ftp server. port: " + port + " passive-address: " + passiveAddress + " passive-ports: " + passivePorts + " hdfs-uri: " + hdfsUri);
+		ftpServer.stop();
+	}
+
+	public boolean statusServer() {
+		try {
+			return !ftpServer.isStopped();
+		}catch (Exception e) {
+			return false;
+		}
+	}
+
+	private static File loadResource(String resourceName) {
+		return new File(System.getProperty("user.dir") + resourceName);
+	}
+}

other/java/hdfs-over-ftp/src/main/java/org/apache/hadoop/seaweed/ftp/service/filesystem/HdfsFileObject.java

@@ -0,0 +1,333 @@
+package org.apache.hadoop.seaweed.ftp.service.filesystem;
+
+import org.apache.ftpserver.ftplet.FtpFile;
+import org.apache.ftpserver.ftplet.User;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.seaweed.ftp.users.HdfsUser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * This class implements all actions to HDFS
+ */
+public class HdfsFileObject implements FtpFile {
+
+	private final Logger log = LoggerFactory.getLogger(HdfsFileObject.class);
+
+	private Path homePath;
+	private Path path;
+	private Path fullPath;
+	private HdfsUser user;
+
+	/**
+	 * Constructs HdfsFileObject from path
+	 *
+	 * @param path path to represent object
+	 * @param user accessor of the object
+	 */
+	public HdfsFileObject(String homePath, String path, User user) {
+		this.homePath = new Path(homePath);
+		this.path = new Path(path);
+		this.fullPath = new Path(homePath + path);
+		this.user = (HdfsUser) user;
+	}
+
+	public String getAbsolutePath() {
+		// strip the last '/' if necessary
+		String fullName = path.toString();
+		int filelen = fullName.length();
+		if ((filelen != 1) && (fullName.charAt(filelen - 1) == '/')) {
+			fullName = fullName.substring(0, filelen - 1);
+		}
+
+		return fullName;
+	}
+
+	public String getName() {
+		return path.getName();
+	}
+
+	/**
+	 * HDFS has no hidden objects
+	 *
+	 * @return always false
+	 */
+	public boolean isHidden() {
+		return false;
+	}
+
+	/**
+	 * Checks if the object is a directory
+	 *
+	 * @return true if the object is a directory
+	 */
+	public boolean isDirectory() {
+		try {
+			log.debug("is directory? : " + fullPath);
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			FileStatus fs = dfs.getFileStatus(fullPath);
+			return fs.isDir();
+		} catch (IOException e) {
+			log.debug(fullPath + " is not dir", e);
+			return false;
+		}
+	}
+
+	/**
+	 * Checks if the object is a file
+	 *
+	 * @return true if the object is a file
+	 */
+	public boolean isFile() {
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			return dfs.isFile(fullPath);
+		} catch (IOException e) {
+			log.debug(fullPath + " is not file", e);
+			return false;
+		}
+	}
+
+	/**
+	 * Checks if the object does exist
+	 *
+	 * @return true if the object does exist
+	 */
+	public boolean doesExist() {
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			dfs.getFileStatus(fullPath);
+			return true;
+		} catch (IOException e) {
+			//   log.debug(path + " does not exist", e);
+			return false;
+		}
+	}
+
+	public boolean isReadable() {
+		return true;
+	}
+
+	public boolean isWritable() {
+		return true;
+	}
+
+	public boolean isRemovable() {
+		return true;
+	}
+
+	/**
+	 * Get owner of the object
+	 *
+	 * @return owner of the object
+	 */
+	public String getOwnerName() {
+		return "root";
+		/*
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			FileStatus fs = dfs.getFileStatus(fullPath);
+			String owner = fs.getOwner();
+			if(owner.length() == 0) {
+				return "root";
+			}
+			return owner;
+		} catch (IOException e) {
+			e.printStackTrace();
+			return null;
+		}
+		 */
+	}
+
+	/**
+	 * Get group of the object
+	 *
+	 * @return group of the object
+	 */
+	public String getGroupName() {
+		return "root";
+		/*
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			FileStatus fs = dfs.getFileStatus(fullPath);
+			String group = fs.getGroup();
+			if(group.length() == 0) {
+				return "root";
+			}
+			return group;
+		} catch (IOException e) {
+			e.printStackTrace();
+			return null;
+		}
+		 */
+	}
+
+	/**
+	 * Get link count
+	 *
+	 * @return 3 is for a directory and 1 is for a file
+	 */
+	public int getLinkCount() {
+		return isDirectory() ? 3 : 1;
+	}
+
+	/**
+	 * Get last modification date
+	 *
+	 * @return last modification date as a long
+	 */
+	public long getLastModified() {
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			FileStatus fs = dfs.getFileStatus(fullPath);
+			return fs.getModificationTime();
+		} catch (IOException e) {
+			e.printStackTrace();
+			return 0;
+		}
+	}
+
+	public boolean setLastModified(long l) {
+		return false;
+	}
+
+	/**
+	 * Get a size of the object
+	 *
+	 * @return size of the object in bytes
+	 */
+	public long getSize() {
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			FileStatus fs = dfs.getFileStatus(fullPath);
+			log.debug("getSize(): " + fullPath + " : " + fs.getLen());
+			return fs.getLen();
+		} catch (IOException e) {
+			e.printStackTrace();
+			return 0;
+		}
+	}
+
+	public Object getPhysicalFile() {
+		return null;
+	}
+
+	/**
+	 * Create a new dir from the object
+	 *
+	 * @return true if dir is created
+	 */
+	public boolean mkdir() {
+		try {
+			FileSystem fs = HdfsOverFtpSystem.getDfs();
+			fs.mkdirs(fullPath);
+//			fs.setOwner(path, user.getName(), user.getMainGroup());
+			return true;
+		} catch (IOException e) {
+			e.printStackTrace();
+			return false;
+		}
+	}
+
+	/**
+	 * Delete object from the HDFS filesystem
+	 *
+	 * @return true if the object is deleted
+	 */
+	public boolean delete() {
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			dfs.delete(fullPath, true);
+			return true;
+		} catch (IOException e) {
+			e.printStackTrace();
+			return false;
+		}
+	}
+
+	public boolean move(FtpFile ftpFile) {
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			dfs.rename(fullPath, new Path(fullPath.getParent() + File.separator + ftpFile.getName()));
+			return true;
+		} catch (IOException e) {
+			e.printStackTrace();
+			return false;
+		}
+	}
+
+
+	/**
+	 * List files of the directory
+	 *
+	 * @return List of files in the directory
+	 */
+	public List<FtpFile> listFiles() {
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			FileStatus fileStats[] = dfs.listStatus(fullPath);
+
+			// get the virtual name of the base directory
+			String virtualFileStr = getAbsolutePath();
+			if (virtualFileStr.charAt(virtualFileStr.length() - 1) != '/') {
+				virtualFileStr += '/';
+			}
+
+			FtpFile[] virtualFiles = new FtpFile[fileStats.length];
+			for (int i = 0; i < fileStats.length; i++) {
+				File fileObj = new File(fileStats[i].getPath().toString());
+				String fileName = virtualFileStr + fileObj.getName();
+				virtualFiles[i] = new HdfsFileObject(homePath.toString(), fileName, user);
+			}
+			return Collections.unmodifiableList(Arrays.asList(virtualFiles));
+		} catch (IOException e) {
+			log.debug("", e);
+			return null;
+		}
+	}
+
+	/**
+	 * Creates output stream to write to the object
+	 *
+	 * @param l is not used here
+	 * @return OutputStream
+	 * @throws IOException
+	 */
+	public OutputStream createOutputStream(long l) {
+		try {
+			FileSystem fs = HdfsOverFtpSystem.getDfs();
+			FSDataOutputStream out = fs.create(fullPath);
+//			fs.setOwner(fullPath, user.getName(), user.getMainGroup());
+			return out;
+		} catch (IOException e) {
+			e.printStackTrace();
+			return null;
+		}
+	}
+
+	/**
+	 * Creates input stream to read from the object
+	 *
+	 * @param l is not used here
+	 * @return OutputStream
+	 * @throws IOException
+	 */
+	public InputStream createInputStream(long l) {
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			FSDataInputStream in = dfs.open(fullPath);
+			return in;
+		} catch (IOException e) {
+			e.printStackTrace();
+			return null;
+		}
+	}
+}

other/java/hdfs-over-ftp/src/main/java/org/apache/hadoop/seaweed/ftp/service/filesystem/HdfsFileSystemManager.java

@@ -0,0 +1,14 @@
+package org.apache.hadoop.seaweed.ftp.service.filesystem;
+
+import org.apache.ftpserver.ftplet.FileSystemFactory;
+import org.apache.ftpserver.ftplet.FileSystemView;
+import org.apache.ftpserver.ftplet.User;
+
+/**
+ * Impelented FileSystemManager to use HdfsFileSystemView
+ */
+public class HdfsFileSystemManager implements FileSystemFactory {
+	public FileSystemView createFileSystemView(User user) {
+		return new HdfsFileSystemView(user);
+	}
+}

other/java/hdfs-over-ftp/src/main/java/org/apache/hadoop/seaweed/ftp/service/filesystem/HdfsFileSystemView.java

@@ -0,0 +1,104 @@
+package org.apache.hadoop.seaweed.ftp.service.filesystem;
+
+import org.apache.ftpserver.ftplet.FileSystemView;
+import org.apache.ftpserver.ftplet.FtpFile;
+import org.apache.ftpserver.ftplet.User;
+import org.apache.hadoop.fs.Path;
+
+import java.io.File;
+
+/**
+ * Implemented FileSystemView to use HdfsFileObject
+ */
+public class HdfsFileSystemView implements FileSystemView {
+
+	private String homePath;
+	private String currPath = File.separator;
+	private User user;
+
+	/**
+	 * Constructor - set the user object.
+	 */
+	protected HdfsFileSystemView(User user) {
+		if (user == null) {
+			throw new IllegalArgumentException("user can not be null");
+		}
+		if (user.getHomeDirectory() == null) {
+			throw new IllegalArgumentException(
+					"User home directory can not be null");
+		}
+
+		this.homePath = user.getHomeDirectory();
+		this.user = user;
+	}
+
+	public FtpFile getHomeDirectory() {
+		return new HdfsFileObject(homePath, File.separator, user);
+	}
+
+	public FtpFile getWorkingDirectory() {
+		FtpFile fileObj;
+		if (currPath.equals(File.separator)) {
+			fileObj = new HdfsFileObject(homePath, File.separator, user);
+		} else {
+			fileObj = new HdfsFileObject(homePath, currPath, user);
+
+		}
+		return fileObj;
+	}
+
+	public boolean changeWorkingDirectory(String dir) {
+
+		Path path;
+		if (dir.startsWith(File.separator) || new Path(currPath).equals(new Path(dir))) {
+			path = new Path(dir);
+		} else if (currPath.length() > 1) {
+			path = new Path(currPath + File.separator + dir);
+		} else {
+			if(dir.startsWith("/")) {
+				path = new Path(dir);
+			}else {
+				path = new Path(File.separator + dir);
+			}
+		}
+
+		// 防止退回根目录
+		if (path.getName().equals("..")) {
+			path = new Path(File.separator);
+		}
+
+		HdfsFileObject file = new HdfsFileObject(homePath, path.toString(), user);
+		if (file.isDirectory()) {
+			currPath = path.toString();
+			return true;
+		} else {
+			return false;
+		}
+	}
+
+	public FtpFile getFile(String file) {
+		String path;
+		if (file.startsWith(File.separator)) {
+			path = file;
+		} else if (currPath.length() > 1) {
+			path = currPath + File.separator + file;
+		} else {
+			path = File.separator + file;
+		}
+		return new HdfsFileObject(homePath, path, user);
+	}
+
+	/**
+	 * Is the file content random accessible?
+	 */
+	public boolean isRandomAccessible() {
+		return true;
+	}
+
+	/**
+	 * Dispose file system view - does nothing.
+	 */
+	public void dispose() {
+	}
+
+}

other/java/hdfs-over-ftp/src/main/java/org/apache/hadoop/seaweed/ftp/service/filesystem/HdfsOverFtpSystem.java

@@ -0,0 +1,72 @@
+package org.apache.hadoop.seaweed.ftp.service.filesystem;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * Class to store DFS connection
+ */
+public class HdfsOverFtpSystem {
+
+	private static FileSystem fs = null;
+
+	private static String hdfsUri;
+
+	private static boolean seaweedFsEnable;
+
+	private static String seaweedFsAccess;
+
+	private static String seaweedFsReplication;
+
+	private final static Logger log = LoggerFactory.getLogger(HdfsOverFtpSystem.class);
+
+	private static void hdfsInit() throws IOException {
+		Configuration configuration = new Configuration();
+
+		configuration.set("fs.defaultFS", hdfsUri);
+		if(seaweedFsEnable) {
+			configuration.set("fs.seaweedfs.impl", "seaweed.hdfs.SeaweedFileSystem");
+			configuration.set("fs.seaweed.volume.server.access", seaweedFsAccess);
+			configuration.set("fs.seaweed.replication", seaweedFsReplication);
+		}
+		fs = FileSystem.get(configuration);
+		log.info("HDFS load success");
+	}
+
+	/**
+	 * Get dfs
+	 *
+	 * @return dfs
+	 * @throws IOException
+	 */
+	public static FileSystem getDfs() throws IOException {
+		if (fs == null) {
+			hdfsInit();
+		}
+		return fs;
+	}
+
+	public static void setHdfsUri(String hdfsUri) {
+		HdfsOverFtpSystem.hdfsUri = hdfsUri;
+	}
+
+	public static String getHdfsUri() {
+		return hdfsUri;
+	}
+
+	public static void setSeaweedFsEnable(boolean seaweedFsEnable) {
+		HdfsOverFtpSystem.seaweedFsEnable = seaweedFsEnable;
+	}
+
+	public static void setSeaweedFsAccess(String seaweedFsAccess) {
+		HdfsOverFtpSystem.seaweedFsAccess = seaweedFsAccess;
+	}
+
+	public static void setSeaweedFsReplication(String seaweedFsReplication) {
+		HdfsOverFtpSystem.seaweedFsReplication = seaweedFsReplication;
+	}
+}

other/java/hdfs-over-ftp/src/main/java/org/apache/hadoop/seaweed/ftp/users/HdfsUser.java

@@ -0,0 +1,239 @@
+package org.apache.hadoop.seaweed.ftp.users;
+
+import org.apache.ftpserver.ftplet.Authority;
+import org.apache.ftpserver.ftplet.AuthorizationRequest;
+import org.apache.ftpserver.ftplet.User;
+import org.apache.log4j.Logger;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class HdfsUser implements User, Serializable {
+
+	private static final long serialVersionUID = -47371353779731294L;
+
+	private String name = null;
+
+	private String password = null;
+
+	private int maxIdleTimeSec = 0; // no limit
+
+	private String homeDir = null;
+
+	private boolean isEnabled = true;
+
+	private List<? extends Authority> authorities = new ArrayList<Authority>();
+
+	private ArrayList<String> groups = new ArrayList<String>();
+
+	private Logger log = Logger.getLogger(HdfsUser.class);
+
+	/**
+	 * Default constructor.
+	 */
+	public HdfsUser() {
+	}
+
+	/**
+	 * Copy constructor.
+	 */
+	public HdfsUser(User user) {
+		name = user.getName();
+		password = user.getPassword();
+		authorities = user.getAuthorities();
+		maxIdleTimeSec = user.getMaxIdleTime();
+		homeDir = user.getHomeDirectory();
+		isEnabled = user.getEnabled();
+	}
+
+	public ArrayList<String> getGroups() {
+		return groups;
+	}
+
+	/**
+	 * Get the main group of the user
+	 *
+	 * @return main group of the user
+	 */
+	public String getMainGroup() {
+		if (groups.size() > 0) {
+			return groups.get(0);
+		} else {
+			log.error("User " + name + " is not a memer of any group");
+			return "error";
+		}
+	}
+
+	/**
+	 * Checks if user is a member of the group
+	 *
+	 * @param group to check
+	 * @return true if the user id a member of the group
+	 */
+	public boolean isGroupMember(String group) {
+		for (String userGroup : groups) {
+			if (userGroup.equals(group)) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	/**
+	 * Set users' groups
+	 *
+	 * @param groups to set
+	 */
+	public void setGroups(ArrayList<String> groups) {
+		if (groups.size() < 1) {
+			log.error("User " + name + " is not a memer of any group");
+		}
+		this.groups = groups;
+	}
+
+	/**
+	 * Get the user name.
+	 */
+	public String getName() {
+		return name;
+	}
+
+	/**
+	 * Set user name.
+	 */
+	public void setName(String name) {
+		this.name = name;
+	}
+
+	/**
+	 * Get the user password.
+	 */
+	public String getPassword() {
+		return password;
+	}
+
+	/**
+	 * Set user password.
+	 */
+	public void setPassword(String pass) {
+		password = pass;
+	}
+
+	public List<Authority> getAuthorities() {
+		if (authorities != null) {
+			return Collections.unmodifiableList(authorities);
+		} else {
+			return null;
+		}
+	}
+
+	public void setAuthorities(List<Authority> authorities) {
+		if (authorities != null) {
+			this.authorities = Collections.unmodifiableList(authorities);
+		} else {
+			this.authorities = null;
+		}
+	}
+
+	/**
+	 * Get the maximum idle time in second.
+	 */
+	public int getMaxIdleTime() {
+		return maxIdleTimeSec;
+	}
+
+	/**
+	 * Set the maximum idle time in second.
+	 */
+	public void setMaxIdleTime(int idleSec) {
+		maxIdleTimeSec = idleSec;
+		if (maxIdleTimeSec < 0) {
+			maxIdleTimeSec = 0;
+		}
+	}
+
+	/**
+	 * Get the user enable status.
+	 */
+	public boolean getEnabled() {
+		return isEnabled;
+	}
+
+	/**
+	 * Set the user enable status.
+	 */
+	public void setEnabled(boolean enb) {
+		isEnabled = enb;
+	}
+
+	/**
+	 * Get the user home directory.
+	 */
+	public String getHomeDirectory() {
+		return homeDir;
+	}
+
+	/**
+	 * Set the user home directory.
+	 */
+	public void setHomeDirectory(String home) {
+		homeDir = home;
+	}
+
+	/**
+	 * String representation.
+	 */
+	public String toString() {
+		return name;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	public AuthorizationRequest authorize(AuthorizationRequest request) {
+		List<Authority> authorities = getAuthorities();
+
+		// check for no authorities at all
+		if (authorities == null) {
+			return null;
+		}
+
+		boolean someoneCouldAuthorize = false;
+		for (Authority authority : authorities) {
+			if (authority.canAuthorize(request)) {
+				someoneCouldAuthorize = true;
+
+				request = authority.authorize(request);
+
+				// authorization failed, return null
+				if (request == null) {
+					return null;
+				}
+			}
+
+		}
+
+		if (someoneCouldAuthorize) {
+			return request;
+		} else {
+			return null;
+		}
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	public List<Authority> getAuthorities(Class<? extends Authority> clazz) {
+		List<Authority> selected = new ArrayList<Authority>();
+
+		for (Authority authority : authorities) {
+			if (authority.getClass().equals(clazz)) {
+				selected.add(authority);
+			}
+		}
+
+		return selected;
+	}
+}

other/java/hdfs-over-ftp/src/main/java/org/apache/hadoop/seaweed/ftp/users/HdfsUserManager.java

@@ -0,0 +1,453 @@
+package org.apache.hadoop.seaweed.ftp.users;
+
+import org.apache.ftpserver.FtpServerConfigurationException;
+import org.apache.ftpserver.ftplet.*;
+import org.apache.ftpserver.usermanager.*;
+import org.apache.ftpserver.usermanager.impl.*;
+import org.apache.ftpserver.util.BaseProperties;
+import org.apache.ftpserver.util.IoUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.*;
+
+public class HdfsUserManager extends AbstractUserManager {
+
+	private final Logger LOG = LoggerFactory
+			.getLogger(HdfsUserManager.class);
+
+	private final static String DEPRECATED_PREFIX = "FtpServer.user.";
+
+	private final static String PREFIX = "ftpserver.user.";
+
+	private static BaseProperties userDataProp;
+
+	private File userDataFile = new File("users.conf");
+
+	private boolean isConfigured = false;
+
+	private PasswordEncryptor passwordEncryptor = new Md5PasswordEncryptor();
+
+
+	/**
+	 * Retrieve the file used to load and store users
+	 *
+	 * @return The file
+	 */
+	public File getFile() {
+		return userDataFile;
+	}
+
+	/**
+	 * Set the file used to store and read users. Must be set before
+	 * {@link #configure()} is called.
+	 *
+	 * @param propFile A file containing users
+	 */
+	public void setFile(File propFile) {
+		if (isConfigured) {
+			throw new IllegalStateException("Must be called before configure()");
+		}
+
+		this.userDataFile = propFile;
+	}
+
+
+	/**
+	 * Retrieve the password encryptor used for this user manager
+	 *
+	 * @return The password encryptor. Default to {@link Md5PasswordEncryptor}
+	 *         if no other has been provided
+	 */
+	public PasswordEncryptor getPasswordEncryptor() {
+		return passwordEncryptor;
+	}
+
+
+	/**
+	 * Set the password encryptor to use for this user manager
+	 *
+	 * @param passwordEncryptor The password encryptor
+	 */
+	public void setPasswordEncryptor(PasswordEncryptor passwordEncryptor) {
+		this.passwordEncryptor = passwordEncryptor;
+	}
+
+
+	/**
+	 * Lazy init the user manager
+	 */
+	private void lazyInit() {
+		if (!isConfigured) {
+			configure();
+		}
+	}
+
+	/**
+	 * Configure user manager.
+	 */
+	public void configure() {
+		isConfigured = true;
+		try {
+			userDataProp = new BaseProperties();
+
+			if (userDataFile != null && userDataFile.exists()) {
+				FileInputStream fis = null;
+				try {
+					fis = new FileInputStream(userDataFile);
+					userDataProp.load(fis);
+				} finally {
+					IoUtils.close(fis);
+				}
+			}
+		} catch (IOException e) {
+			throw new FtpServerConfigurationException(
+					"Error loading user data file : "
+							+ userDataFile.getAbsolutePath(), e);
+		}
+
+		convertDeprecatedPropertyNames();
+	}
+
+	private void convertDeprecatedPropertyNames() {
+		Enumeration<?> keys = userDataProp.propertyNames();
+
+		boolean doSave = false;
+
+		while (keys.hasMoreElements()) {
+			String key = (String) keys.nextElement();
+
+			if (key.startsWith(DEPRECATED_PREFIX)) {
+				String newKey = PREFIX
+						+ key.substring(DEPRECATED_PREFIX.length());
+				userDataProp.setProperty(newKey, userDataProp.getProperty(key));
+				userDataProp.remove(key);
+
+				doSave = true;
+			}
+		}
+
+		if (doSave) {
+			try {
+				saveUserData();
+			} catch (FtpException e) {
+				throw new FtpServerConfigurationException(
+						"Failed to save updated user data", e);
+			}
+		}
+	}
+
+	public synchronized void save(User usr, boolean renamePush) throws FtpException {
+		lazyInit();
+		userDataProp.setProperty(PREFIX + usr.getName() + ".rename.push", renamePush);
+		save(usr);
+	}
+
+	/**
+	 * Save user data. Store the properties.
+	 */
+	public synchronized void save(User usr) throws FtpException {
+		lazyInit();
+
+		// null value check
+		if (usr.getName() == null) {
+			throw new NullPointerException("User name is null.");
+		}
+		String thisPrefix = PREFIX + usr.getName() + '.';
+
+		// set other properties
+		userDataProp.setProperty(thisPrefix + ATTR_PASSWORD, getPassword(usr));
+
+		String home = usr.getHomeDirectory();
+		if (home == null) {
+			home = "/";
+		}
+		userDataProp.setProperty(thisPrefix + ATTR_HOME, home);
+		userDataProp.setProperty(thisPrefix + ATTR_ENABLE, usr.getEnabled());
+		userDataProp.setProperty(thisPrefix + ATTR_WRITE_PERM, usr
+				.authorize(new WriteRequest()) != null);
+		userDataProp.setProperty(thisPrefix + ATTR_MAX_IDLE_TIME, usr
+				.getMaxIdleTime());
+
+		TransferRateRequest transferRateRequest = new TransferRateRequest();
+		transferRateRequest = (TransferRateRequest) usr
+				.authorize(transferRateRequest);
+
+		if (transferRateRequest != null) {
+			userDataProp.setProperty(thisPrefix + ATTR_MAX_UPLOAD_RATE,
+					transferRateRequest.getMaxUploadRate());
+			userDataProp.setProperty(thisPrefix + ATTR_MAX_DOWNLOAD_RATE,
+					transferRateRequest.getMaxDownloadRate());
+		} else {
+			userDataProp.remove(thisPrefix + ATTR_MAX_UPLOAD_RATE);
+			userDataProp.remove(thisPrefix + ATTR_MAX_DOWNLOAD_RATE);
+		}
+
+		// request that always will succeed
+		ConcurrentLoginRequest concurrentLoginRequest = new ConcurrentLoginRequest(
+				0, 0);
+		concurrentLoginRequest = (ConcurrentLoginRequest) usr
+				.authorize(concurrentLoginRequest);
+
+		if (concurrentLoginRequest != null) {
+			userDataProp.setProperty(thisPrefix + ATTR_MAX_LOGIN_NUMBER,
+					concurrentLoginRequest.getMaxConcurrentLogins());
+			userDataProp.setProperty(thisPrefix + ATTR_MAX_LOGIN_PER_IP,
+					concurrentLoginRequest.getMaxConcurrentLoginsPerIP());
+		} else {
+			userDataProp.remove(thisPrefix + ATTR_MAX_LOGIN_NUMBER);
+			userDataProp.remove(thisPrefix + ATTR_MAX_LOGIN_PER_IP);
+		}
+
+		saveUserData();
+	}
+
+	/**
+	 * @throws FtpException
+	 */
+	private void saveUserData() throws FtpException {
+		File dir = userDataFile.getAbsoluteFile().getParentFile();
+		if (dir != null && !dir.exists() && !dir.mkdirs()) {
+			String dirName = dir.getAbsolutePath();
+			throw new FtpServerConfigurationException(
+					"Cannot create directory for user data file : " + dirName);
+		}
+
+		// save user data
+		FileOutputStream fos = null;
+		try {
+			fos = new FileOutputStream(userDataFile);
+			userDataProp.store(fos, "Generated file - don't edit (please)");
+		} catch (IOException ex) {
+			LOG.error("Failed saving user data", ex);
+			throw new FtpException("Failed saving user data", ex);
+		} finally {
+			IoUtils.close(fos);
+		}
+	}
+
+
+	public synchronized void list() throws FtpException {
+		lazyInit();
+
+		Map dataMap = new HashMap();
+		Enumeration<String> propNames = (Enumeration<String>) userDataProp.propertyNames();
+		ArrayList<String> a = Collections.list(propNames);
+		a.remove("i18nMap");//去除i18nMap
+		for(String attrName : a){
+//			dataMap.put(attrName, propNames.);
+		}
+
+	}
+
+	/**
+	 * Delete an user. Removes all this user entries from the properties. After
+	 * removing the corresponding from the properties, save the data.
+	 */
+	public synchronized void delete(String usrName) throws FtpException {
+		lazyInit();
+
+		// remove entries from properties
+		String thisPrefix = PREFIX + usrName + '.';
+		Enumeration<?> propNames = userDataProp.propertyNames();
+		ArrayList<String> remKeys = new ArrayList<String>();
+		while (propNames.hasMoreElements()) {
+			String thisKey = propNames.nextElement().toString();
+			if (thisKey.startsWith(thisPrefix)) {
+				remKeys.add(thisKey);
+			}
+		}
+		Iterator<String> remKeysIt = remKeys.iterator();
+		while (remKeysIt.hasNext()) {
+			userDataProp.remove(remKeysIt.next());
+		}
+
+		saveUserData();
+	}
+
+	/**
+	 * Get user password. Returns the encrypted value.
+	 * <p/>
+	 * <pre>
+	 * If the password value is not null
+	 *    password = new password
+	 * else
+	 *   if user does exist
+	 *     password = old password
+	 *   else
+	 *     password = &quot;&quot;
+	 * </pre>
+	 */
+	private String getPassword(User usr) {
+		String name = usr.getName();
+		String password = usr.getPassword();
+
+		if (password != null) {
+			password = passwordEncryptor.encrypt(password);
+		} else {
+			String blankPassword = passwordEncryptor.encrypt("");
+
+			if (doesExist(name)) {
+				String key = PREFIX + name + '.' + ATTR_PASSWORD;
+				password = userDataProp.getProperty(key, blankPassword);
+			} else {
+				password = blankPassword;
+			}
+		}
+		return password;
+	}
+
+	/**
+	 * Get all user names.
+	 */
+	public synchronized String[] getAllUserNames() {
+		lazyInit();
+
+		// get all user names
+		String suffix = '.' + ATTR_HOME;
+		ArrayList<String> ulst = new ArrayList<String>();
+		Enumeration<?> allKeys = userDataProp.propertyNames();
+		int prefixlen = PREFIX.length();
+		int suffixlen = suffix.length();
+		while (allKeys.hasMoreElements()) {
+			String key = (String) allKeys.nextElement();
+			if (key.endsWith(suffix)) {
+				String name = key.substring(prefixlen);
+				int endIndex = name.length() - suffixlen;
+				name = name.substring(0, endIndex);
+				ulst.add(name);
+			}
+		}
+
+		Collections.sort(ulst);
+		return ulst.toArray(new String[0]);
+	}
+
+	private ArrayList<String> parseGroups(String groupsLine) {
+		String groupsArray[] = groupsLine.split(",");
+		return new ArrayList(Arrays.asList(groupsArray));
+	}
+
+	public static synchronized boolean getUserRenamePush(String userName) {
+		return userDataProp.getBoolean(PREFIX + userName + ".rename.push", false);
+	}
+
+	/**
+	 * Load user data.
+	 */
+	public synchronized User getUserByName(String userName) {
+		lazyInit();
+
+		if (!doesExist(userName)) {
+			return null;
+		}
+
+		String baseKey = PREFIX + userName + '.';
+		HdfsUser user = new HdfsUser();
+		user.setName(userName);
+		user.setEnabled(userDataProp.getBoolean(baseKey + ATTR_ENABLE, true));
+		user.setHomeDirectory(userDataProp
+				.getProperty(baseKey + ATTR_HOME, "/"));
+
+//		user.setGroups(parseGroups(userDataProp
+//				.getProperty(baseKey + "groups")));
+
+		List<Authority> authorities = new ArrayList<Authority>();
+
+		if (userDataProp.getBoolean(baseKey + ATTR_WRITE_PERM, false)) {
+			authorities.add(new WritePermission());
+		}
+
+		int maxLogin = userDataProp.getInteger(baseKey + ATTR_MAX_LOGIN_NUMBER,
+				0);
+		int maxLoginPerIP = userDataProp.getInteger(baseKey
+				+ ATTR_MAX_LOGIN_PER_IP, 0);
+
+		authorities.add(new ConcurrentLoginPermission(maxLogin, maxLoginPerIP));
+
+		int uploadRate = userDataProp.getInteger(
+				baseKey + ATTR_MAX_UPLOAD_RATE, 0);
+		int downloadRate = userDataProp.getInteger(baseKey
+				+ ATTR_MAX_DOWNLOAD_RATE, 0);
+
+		authorities.add(new TransferRatePermission(downloadRate, uploadRate));
+
+		user.setAuthorities(authorities);
+
+		user.setMaxIdleTime(userDataProp.getInteger(baseKey
+				+ ATTR_MAX_IDLE_TIME, 0));
+
+		return user;
+	}
+
+	/**
+	 * User existance check
+	 */
+	public synchronized boolean doesExist(String name) {
+		lazyInit();
+
+		String key = PREFIX + name + '.' + ATTR_HOME;
+		return userDataProp.containsKey(key);
+	}
+
+	/**
+	 * User authenticate method
+	 */
+	public synchronized User authenticate(Authentication authentication)
+			throws AuthenticationFailedException {
+		lazyInit();
+
+		if (authentication instanceof UsernamePasswordAuthentication) {
+			UsernamePasswordAuthentication upauth = (UsernamePasswordAuthentication) authentication;
+
+			String user = upauth.getUsername();
+			String password = upauth.getPassword();
+
+			if (user == null) {
+				throw new AuthenticationFailedException("Authentication failed");
+			}
+
+			if (password == null) {
+				password = "";
+			}
+
+			String storedPassword = userDataProp.getProperty(PREFIX + user + '.'
+					+ ATTR_PASSWORD);
+
+			if (storedPassword == null) {
+				// user does not exist
+				throw new AuthenticationFailedException("Authentication failed");
+			}
+
+			if (passwordEncryptor.matches(password, storedPassword)) {
+				return getUserByName(user);
+			} else {
+				throw new AuthenticationFailedException("Authentication failed");
+			}
+
+		} else if (authentication instanceof AnonymousAuthentication) {
+			if (doesExist("anonymous")) {
+				return getUserByName("anonymous");
+			} else {
+				throw new AuthenticationFailedException("Authentication failed");
+			}
+		} else {
+			throw new IllegalArgumentException(
+					"Authentication not supported by this user manager");
+		}
+	}
+
+	/**
+	 * Close the user manager - remove existing entries.
+	 */
+	public synchronized void dispose() {
+		if (userDataProp != null) {
+			userDataProp.clear();
+			userDataProp = null;
+		}
+	}
+}

other/java/hdfs-over-ftp/src/main/resources/application.yml

@@ -0,0 +1,15 @@
+server:
+  port: 8080
+
+ftp:
+  port: 2222
+  passive-address: localhost
+  passive-ports: 30000-30999
+
+hdfs:
+  uri: seaweedfs://localhost:8888
+
+seaweedFs:
+  enable: true
+  access: direct # direct/filerProxy/publicUrl
+  replication: "000"

other/java/hdfs-over-ftp/src/main/resources/assembly.xml

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
+
+    <id>package</id>
+    <formats>
+        <!-- 指定打包格式，支持的打包格式有zip、tar、tar.gz (or tgz)、tar.bz2 (or tbz2)、jar、dir、war，可以同时指定多个打包格式 -->
+        <format>tar.gz</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+
+    <fileSets>
+        <fileSet>
+            <directory>src/main/resources</directory>
+            <outputDirectory>/</outputDirectory>
+            <includes>
+                <include>application.yml</include>
+                <include>logback-spring.xml</include>
+                <include>users.properties</include>
+                <include>kafka-producer.properties</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>${project.build.directory}</directory>
+            <outputDirectory>/</outputDirectory>
+            <includes>
+                <include>*.jar</include>
+            </includes>
+        </fileSet>
+    </fileSets>
+    <dependencySets>
+        <dependencySet>
+            <useProjectArtifact>false</useProjectArtifact>
+            <outputDirectory>lib</outputDirectory>
+            <scope>runtime</scope>
+            <unpack>false</unpack>
+        </dependencySet>
+    </dependencySets>
+</assembly>

other/java/hdfs-over-ftp/src/main/resources/logback-spring.xml

@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<configuration>
+    <!--定义日志文件的存储地址 勿在 LogBack 的配置中使用相对路径-->
+    <property name="LOG_HOME" value="${user.dir}/logs/" />
+
+    <!-- 控制台输出 -->
+    <appender name="Stdout" class="ch.qos.logback.core.ConsoleAppender">
+        <!-- 日志输出编码 -->
+        <layout class="ch.qos.logback.classic.PatternLayout">
+            <!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度%msg：日志消息，%n是换行符-->
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n
+            </pattern>
+        </layout>
+    </appender>
+
+    <!-- 按照每天生成日志文件 -->
+    <appender name="RollingFile"
+              class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <File>${LOG_HOME}/fileLog.log</File>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>${LOG_HOME}/fileLog.log.%d.%i</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100 MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+        </rollingPolicy>
+        <encoder>
+            <pattern>
+                %d %p (%file:%line\)- %m%n
+            </pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
+
+    <!-- 日志输出级别 -->
+    <root level="info">
+        <appender-ref ref="Stdout" />
+        <appender-ref ref="RollingFile" />
+    </root>
+
+</configuration>

other/java/hdfs-over-ftp/users.properties

@@ -0,0 +1,12 @@
+#Generated file - don't edit (please)
+#Thu Mar 11 19:11:12 CST 2021
+ftpserver.user.test.idletime=0
+ftpserver.user.test.maxloginperip=0
+ftpserver.user.test.userpassword=44664D4D827C740293D2AA244FB60445
+ftpserver.user.test.enableflag=true
+ftpserver.user.test.maxloginnumber=0
+ftpserver.user.test.rename.push=true
+ftpserver.user.test.homedirectory=/buckets/test/
+ftpserver.user.test.downloadrate=0
+ftpserver.user.test.writepermission=true
+ftpserver.user.test.uploadrate=0

other/java/hdfs2/dependency-reduced-pom.xml

@@ -301,7 +301,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.6.2</seaweedfs.client.version>
+    <seaweedfs.client.version>1.6.4</seaweedfs.client.version>
     <hadoop.version>2.9.2</hadoop.version>
   </properties>
 </project>

other/java/hdfs2/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.6.2</seaweedfs.client.version>
+        <seaweedfs.client.version>1.6.4</seaweedfs.client.version>
         <hadoop.version>2.9.2</hadoop.version>
     </properties>
 

other/java/hdfs3/dependency-reduced-pom.xml

@@ -309,7 +309,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.6.2</seaweedfs.client.version>
+    <seaweedfs.client.version>1.6.4</seaweedfs.client.version>
     <hadoop.version>3.1.1</hadoop.version>
   </properties>
 </project>

other/java/hdfs3/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.6.2</seaweedfs.client.version>
+        <seaweedfs.client.version>1.6.4</seaweedfs.client.version>
         <hadoop.version>3.1.1</hadoop.version>
     </properties>
 

test/random_access/pom.xml

@@ -8,7 +8,7 @@
     <version>1.0-SNAPSHOT</version>
 
     <properties>
-        <guava.version>28.0-jre</guava.version>
+        <guava.version>30.0-jre</guava.version>
     </properties>
 
     <dependencies>

unmaintained/repeated_vacuum/repeated_vacuum.go

@@ -52,7 +52,7 @@ func main() {
 }
 
 func genFile(grpcDialOption grpc.DialOption, i int) (*operation.AssignResult, string) {
-	assignResult, err := operation.Assign(*master, grpcDialOption, &operation.VolumeAssignRequest{
+	assignResult, err := operation.Assign(func() string { return *master }, grpcDialOption, &operation.VolumeAssignRequest{
 		Count:       1,
 		Replication: *replication,
 	})

weed/Makefile

@@ -16,7 +16,7 @@ debug_shell:
 
 debug_mount:
 	go build -gcflags="all=-N -l"
-	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- mount -dir=~/tmp/mm -cacheCapacityMB=0 -filer.path=/buckets
+	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 mount -dir=~/tmp/mm -cacheCapacityMB=0 -filer.path=/buckets
 
 debug_server:
 	go build -gcflags="all=-N -l"

weed/command/benchmark.go

@@ -305,7 +305,7 @@ func readFiles(fileIdLineChan chan string, s *stat) {
 		}
 		var bytes []byte
 		for _, url := range urls {
-			bytes, _, err = util.FastGet(url)
+			bytes, _, err = util.Get(url)
 			if err == nil {
 				break
 			}

weed/command/command.go

@@ -22,9 +22,11 @@ var Commands = []*Command{
 	cmdFilerReplicate,
 	cmdFilerSynchronize,
 	cmdFix,
+	cmdGateway,
 	cmdMaster,
 	cmdMount,
 	cmdS3,
+	cmdIam,
 	cmdMsgBroker,
 	cmdScaffold,
 	cmdServer,

weed/command/export.go

@@ -215,7 +215,7 @@ func runExport(cmd *Command, args []string) bool {
 
 	err = storage.ScanVolumeFile(util.ResolvePath(*export.dir), *export.collection, vid, storage.NeedleMapInMemory, volumeFileScanner)
 	if err != nil && err != io.EOF {
-		glog.Fatalf("Export Volume File [ERROR] %s\n", err)
+		glog.Errorf("Export Volume File [ERROR] %s\n", err)
 	}
 	return true
 }

weed/command/filer.go

@@ -25,6 +25,8 @@ var (
 	filerS3Options     S3Options
 	filerStartWebDav   *bool
 	filerWebDavOptions WebDavOption
+	filerStartIam      *bool
+	filerIamOptions    IamOptions
 )
 
 type FilerOptions struct {
@@ -47,6 +49,7 @@ type FilerOptions struct {
 	metricsHttpPort         *int
 	saveToFilerLimit        *int
 	defaultLevelDbDirectory *string
+	concurrentUploadLimitMB *int
 }
 
 func init() {
@@ -54,12 +57,12 @@ func init() {
 	f.masters = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers")
 	f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this default collection")
 	f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address")
-	f.bindIp = cmdFiler.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
+	f.bindIp = cmdFiler.Flag.String("ip.bind", "", "ip address to bind to")
 	f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
 	f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
 	f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
 	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
-	f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit")
+	f.maxMB = cmdFiler.Flag.Int("maxMB", 4, "split files larger than the limit")
 	f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
 	f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
 	f.rack = cmdFiler.Flag.String("rack", "", "prefer to write to volumes in this rack")
@@ -69,6 +72,7 @@ func init() {
 	f.metricsHttpPort = cmdFiler.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
 	f.saveToFilerLimit = cmdFiler.Flag.Int("saveToFilerLimit", 0, "files smaller than this limit will be saved in filer store")
 	f.defaultLevelDbDirectory = cmdFiler.Flag.String("defaultStoreDir", ".", "if filer.toml is empty, use an embedded filer store in the directory")
+	f.concurrentUploadLimitMB = cmdFiler.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size")
 
 	// start s3 on filer
 	filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
@@ -89,6 +93,10 @@ func init() {
 	filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
 	filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
 	filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 1000, "local cache capacity in MB")
+
+	// start iam on filer
+	filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
+	filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port")
 }
 
 var cmdFiler = &Command{
@@ -106,7 +114,7 @@ var cmdFiler = &Command{
 	GET /path/to/
 
 	The configuration file "filer.toml" is read from ".", "$HOME/.seaweedfs/", "/usr/local/etc/seaweedfs/", or "/etc/seaweedfs/", in that order.
-	If the "filer.toml" is not found, an embedded filer store will be craeted under "-defaultStoreDir".
+	If the "filer.toml" is not found, an embedded filer store will be created under "-defaultStoreDir".
 
 	The example filer.toml configuration file can be generated by "weed scaffold -config=filer"
 
@@ -119,22 +127,33 @@ func runFiler(cmd *Command, args []string) bool {
 
 	go stats_collect.StartMetricsServer(*f.metricsHttpPort)
 
+	filerAddress := fmt.Sprintf("%s:%d", *f.ip, *f.port)
+	startDelay := time.Duration(2)
 	if *filerStartS3 {
-		filerAddress := fmt.Sprintf("%s:%d", *f.ip, *f.port)
 		filerS3Options.filer = &filerAddress
 		go func() {
-			time.Sleep(2 * time.Second)
+			time.Sleep(startDelay * time.Second)
 			filerS3Options.startS3Server()
 		}()
+		startDelay++
 	}
 
 	if *filerStartWebDav {
-		filerAddress := fmt.Sprintf("%s:%d", *f.ip, *f.port)
 		filerWebDavOptions.filer = &filerAddress
 		go func() {
-			time.Sleep(2 * time.Second)
+			time.Sleep(startDelay * time.Second)
 			filerWebDavOptions.startWebDav()
 		}()
+		startDelay++
+	}
+
+	if *filerStartIam {
+		filerIamOptions.filer = &filerAddress
+		filerIamOptions.masters = f.masters
+		go func() {
+			time.Sleep(startDelay * time.Second)
+			filerIamOptions.startIamServer()
+		}()
 	}
 
 	f.startFiler()
@@ -159,21 +178,22 @@ func (fo *FilerOptions) startFiler() {
 	}
 
 	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
-		Masters:            strings.Split(*fo.masters, ","),
-		Collection:         *fo.collection,
-		DefaultReplication: *fo.defaultReplicaPlacement,
-		DisableDirListing:  *fo.disableDirListing,
-		MaxMB:              *fo.maxMB,
-		DirListingLimit:    *fo.dirListingLimit,
-		DataCenter:         *fo.dataCenter,
-		Rack:               *fo.rack,
-		DefaultLevelDbDir:  defaultLevelDbDirectory,
-		DisableHttp:        *fo.disableHttp,
-		Host:               *fo.ip,
-		Port:               uint32(*fo.port),
-		Cipher:             *fo.cipher,
-		SaveToFilerLimit:   *fo.saveToFilerLimit,
-		Filers:             peers,
+		Masters:               strings.Split(*fo.masters, ","),
+		Collection:            *fo.collection,
+		DefaultReplication:    *fo.defaultReplicaPlacement,
+		DisableDirListing:     *fo.disableDirListing,
+		MaxMB:                 *fo.maxMB,
+		DirListingLimit:       *fo.dirListingLimit,
+		DataCenter:            *fo.dataCenter,
+		Rack:                  *fo.rack,
+		DefaultLevelDbDir:     defaultLevelDbDirectory,
+		DisableHttp:           *fo.disableHttp,
+		Host:                  *fo.ip,
+		Port:                  uint32(*fo.port),
+		Cipher:                *fo.cipher,
+		SaveToFilerLimit:      int64(*fo.saveToFilerLimit),
+		Filers:                peers,
+		ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
 	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)

weed/command/filer_copy.go

@@ -56,7 +56,7 @@ func init() {
 	copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name")
 	copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
 	copy.diskType = cmdCopy.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
-	copy.maxMB = cmdCopy.Flag.Int("maxMB", 32, "split files larger than the limit")
+	copy.maxMB = cmdCopy.Flag.Int("maxMB", 4, "split files larger than the limit")
 	copy.concurrenctFiles = cmdCopy.Flag.Int("c", 8, "concurrent file copy goroutines")
 	copy.concurrenctChunks = cmdCopy.Flag.Int("concurrentChunks", 8, "concurrent chunk copy goroutines for each file")
 }
@@ -207,16 +207,6 @@ func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan Fi
 	}
 
 	mode := fi.Mode()
-	if mode.IsDir() {
-		files, _ := ioutil.ReadDir(fileOrDir)
-		for _, subFileOrDir := range files {
-			if err = genFileCopyTask(fileOrDir+"/"+subFileOrDir.Name(), destPath+fi.Name()+"/", fileCopyTaskChan); err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
 	uid, gid := util.GetFileUidGid(fi)
 
 	fileCopyTaskChan <- FileCopyTask{
@@ -228,6 +218,16 @@ func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan Fi
 		gid:                gid,
 	}
 
+	if mode.IsDir() {
+		files, _ := ioutil.ReadDir(fileOrDir)
+		println("checking directory", fileOrDir)
+		for _, subFileOrDir := range files {
+			if err = genFileCopyTask(fileOrDir+"/"+subFileOrDir.Name(), destPath+fi.Name()+"/", fileCopyTaskChan); err != nil {
+				return err
+			}
+		}
+	}
+
 	return nil
 }
 
@@ -293,38 +293,42 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
 
 	// upload the file content
 	fileName := filepath.Base(f.Name())
-	mimeType := detectMimeType(f)
-	data, err := ioutil.ReadAll(f)
-	if err != nil {
-		return err
-	}
+	var mimeType string
 
 	var chunks []*filer_pb.FileChunk
 	var assignResult *filer_pb.AssignVolumeResponse
 	var assignError error
 
-	if task.fileSize > 0 {
+	if task.fileMode&os.ModeDir == 0 && task.fileSize > 0 {
+
+		mimeType = detectMimeType(f)
+		data, err := ioutil.ReadAll(f)
+		if err != nil {
+			return err
+		}
 
 		// assign a volume
-		err := pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err = util.Retry("assignVolume", func() error {
+			return pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 
-			request := &filer_pb.AssignVolumeRequest{
-				Count:       1,
-				Replication: *worker.options.replication,
-				Collection:  *worker.options.collection,
-				TtlSec:      worker.options.ttlSec,
-				DiskType:    *worker.options.diskType,
-				Path:        task.destinationUrlPath,
-			}
+				request := &filer_pb.AssignVolumeRequest{
+					Count:       1,
+					Replication: *worker.options.replication,
+					Collection:  *worker.options.collection,
+					TtlSec:      worker.options.ttlSec,
+					DiskType:    *worker.options.diskType,
+					Path:        task.destinationUrlPath,
+				}
 
-			assignResult, assignError = client.AssignVolume(context.Background(), request)
-			if assignError != nil {
-				return fmt.Errorf("assign volume failure %v: %v", request, assignError)
-			}
-			if assignResult.Error != "" {
-				return fmt.Errorf("assign volume failure %v: %v", request, assignResult.Error)
-			}
-			return nil
+				assignResult, assignError = client.AssignVolume(context.Background(), request)
+				if assignError != nil {
+					return fmt.Errorf("assign volume failure %v: %v", request, assignError)
+				}
+				if assignResult.Error != "" {
+					return fmt.Errorf("assign volume failure %v: %v", request, assignResult.Error)
+				}
+				return nil
+			})
 		})
 		if err != nil {
 			return fmt.Errorf("Failed to assign from %v: %v\n", worker.options.masters, err)
@@ -402,31 +406,30 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 			// assign a volume
 			var assignResult *filer_pb.AssignVolumeResponse
 			var assignError error
-			err := pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
-				request := &filer_pb.AssignVolumeRequest{
-					Count:       1,
-					Replication: *worker.options.replication,
-					Collection:  *worker.options.collection,
-					TtlSec:      worker.options.ttlSec,
-					DiskType:    *worker.options.diskType,
-					Path:        task.destinationUrlPath + fileName,
-				}
+			err := util.Retry("assignVolume", func() error {
+				return pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+					request := &filer_pb.AssignVolumeRequest{
+						Count:       1,
+						Replication: *worker.options.replication,
+						Collection:  *worker.options.collection,
+						TtlSec:      worker.options.ttlSec,
+						DiskType:    *worker.options.diskType,
+						Path:        task.destinationUrlPath + fileName,
+					}
 
-				assignResult, assignError = client.AssignVolume(context.Background(), request)
-				if assignError != nil {
-					return fmt.Errorf("assign volume failure %v: %v", request, assignError)
-				}
-				if assignResult.Error != "" {
-					return fmt.Errorf("assign volume failure %v: %v", request, assignResult.Error)
-				}
-				return nil
+					assignResult, assignError = client.AssignVolume(context.Background(), request)
+					if assignError != nil {
+						return fmt.Errorf("assign volume failure %v: %v", request, assignError)
+					}
+					if assignResult.Error != "" {
+						return fmt.Errorf("assign volume failure %v: %v", request, assignResult.Error)
+					}
+					return nil
+				})
 			})
 			if err != nil {
 				fmt.Printf("Failed to assign from %v: %v\n", worker.options.masters, err)
 			}
-			if err != nil {
-				fmt.Printf("Failed to assign from %v: %v\n", worker.options.masters, err)
-			}
 
 			targetUrl := "http://" + assignResult.Url + "/" + assignResult.FileId
 			if collection == "" {

weed/command/gateway.go

@@ -0,0 +1,93 @@
+package command
+
+import (
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+var (
+	gatewayOptions GatewayOptions
+)
+
+type GatewayOptions struct {
+	masters *string
+	filers  *string
+	bindIp  *string
+	port    *int
+	maxMB   *int
+}
+
+func init() {
+	cmdGateway.Run = runGateway // break init cycle
+	gatewayOptions.masters = cmdGateway.Flag.String("master", "localhost:9333", "comma-separated master servers")
+	gatewayOptions.filers = cmdGateway.Flag.String("filer", "localhost:8888", "comma-separated filer servers")
+	gatewayOptions.bindIp = cmdGateway.Flag.String("ip.bind", "localhost", "ip address to bind to")
+	gatewayOptions.port = cmdGateway.Flag.Int("port", 5647, "gateway http listen port")
+	gatewayOptions.maxMB = cmdGateway.Flag.Int("maxMB", 4, "split files larger than the limit")
+}
+
+var cmdGateway = &Command{
+	UsageLine: "gateway -port=8888 -master=<ip:port>[,<ip:port>]* -filer=<ip:port>[,<ip:port>]*",
+	Short:     "start a gateway server that points to a list of master servers or a list of filers",
+	Long: `start a gateway server which accepts REST operation to write any blobs, files, or topic messages.
+
+	POST /blobs/
+		upload the blob and return a chunk id
+	DELETE /blobs/<chunk_id>
+		delete a chunk id
+
+	/*
+	POST /files/path/to/a/file
+		save /path/to/a/file on filer 
+	DELETE /files/path/to/a/file
+		delete /path/to/a/file on filer 
+
+	POST /topics/topicName
+		save on filer to /topics/topicName/<ds>/ts.json
+	*/
+`,
+}
+
+func runGateway(cmd *Command, args []string) bool {
+
+	util.LoadConfiguration("security", false)
+
+	gatewayOptions.startGateway()
+
+	return true
+}
+
+func (gw *GatewayOptions) startGateway() {
+
+	defaultMux := http.NewServeMux()
+
+	_, gws_err := weed_server.NewGatewayServer(defaultMux, &weed_server.GatewayOption{
+		Masters: strings.Split(*gw.masters, ","),
+		Filers:  strings.Split(*gw.filers, ","),
+		MaxMB:   *gw.maxMB,
+	})
+	if gws_err != nil {
+		glog.Fatalf("Gateway startup error: %v", gws_err)
+	}
+
+	glog.V(0).Infof("Start Seaweed Gateway %s at %s:%d", util.Version(), *gw.bindIp, *gw.port)
+	gatewayListener, e := util.NewListener(
+		*gw.bindIp+":"+strconv.Itoa(*gw.port),
+		time.Duration(10)*time.Second,
+	)
+	if e != nil {
+		glog.Fatalf("Filer listener error: %v", e)
+	}
+
+	httpS := &http.Server{Handler: defaultMux}
+	if err := httpS.Serve(gatewayListener); err != nil {
+		glog.Fatalf("Gateway Fail to serve: %v", e)
+	}
+
+}

weed/command/iam.go

@@ -0,0 +1,97 @@
+package command
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/iamapi"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/gorilla/mux"
+	"time"
+)
+
+var (
+	iamStandaloneOptions IamOptions
+)
+
+type IamOptions struct {
+	filer   *string
+	masters *string
+	port    *int
+}
+
+func init() {
+	cmdIam.Run = runIam // break init cycle
+	iamStandaloneOptions.filer = cmdIam.Flag.String("filer", "localhost:8888", "filer server address")
+	iamStandaloneOptions.masters = cmdIam.Flag.String("master", "localhost:9333", "comma-separated master servers")
+	iamStandaloneOptions.port = cmdIam.Flag.Int("port", 8111, "iam server http listen port")
+}
+
+var cmdIam = &Command{
+	UsageLine: "iam [-port=8111] [-filer=<ip:port>] [-masters=<ip:port>,<ip:port>]",
+	Short:     "start a iam API compatible server",
+	Long:      "start a iam API compatible server.",
+}
+
+func runIam(cmd *Command, args []string) bool {
+	return iamStandaloneOptions.startIamServer()
+}
+
+func (iamopt *IamOptions) startIamServer() bool {
+	filerGrpcAddress, err := pb.ParseServerToGrpcAddress(*iamopt.filer)
+	if err != nil {
+		glog.Fatal(err)
+		return false
+	}
+
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+	for {
+		err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+			if err != nil {
+				return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
+			}
+			glog.V(0).Infof("IAM read filer configuration: %s", resp)
+			return nil
+		})
+		if err != nil {
+			glog.V(0).Infof("wait to connect to filer %s grpc address %s", *iamopt.filer, filerGrpcAddress)
+			time.Sleep(time.Second)
+		} else {
+			glog.V(0).Infof("connected to filer %s grpc address %s", *iamopt.filer, filerGrpcAddress)
+			break
+		}
+	}
+
+	router := mux.NewRouter().SkipClean(true)
+	_, iamApiServer_err := iamapi.NewIamApiServer(router, &iamapi.IamServerOption{
+		Filer:            *iamopt.filer,
+		Port:             *iamopt.port,
+		FilerGrpcAddress: filerGrpcAddress,
+		GrpcDialOption:   grpcDialOption,
+	})
+	glog.V(0).Info("NewIamApiServer created")
+	if iamApiServer_err != nil {
+		glog.Fatalf("IAM API Server startup error: %v", iamApiServer_err)
+	}
+
+	httpS := &http.Server{Handler: router}
+
+	listenAddress := fmt.Sprintf(":%d", *iamopt.port)
+	iamApiListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
+	if err != nil {
+		glog.Fatalf("IAM API Server listener on %s error: %v", listenAddress, err)
+	}
+
+	glog.V(0).Infof("Start Seaweed IAM API Server %s at http port %d", util.Version(), *iamopt.port)
+	if err = httpS.Serve(iamApiListener); err != nil {
+		glog.Fatalf("IAM API Server Fail to serve: %v", err)
+	}
+
+	return true
+}

weed/command/master.go

@@ -6,7 +6,6 @@ import (
 	"google.golang.org/grpc/reflection"
 	"net/http"
 	"os"
-	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -48,8 +47,8 @@ type MasterOptions struct {
 func init() {
 	cmdMaster.Run = runMaster // break init cycle
 	m.port = cmdMaster.Flag.Int("port", 9333, "http listen port")
-	m.ip = cmdMaster.Flag.String("ip", util.DetectedHostAddress(), "master <ip>|<server> address")
-	m.ipBind = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
+	m.ip = cmdMaster.Flag.String("ip", util.DetectedHostAddress(), "master <ip>|<server> address, also used as identifier")
+	m.ipBind = cmdMaster.Flag.String("ip.bind", "", "ip address to bind to")
 	m.metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
 	m.peers = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095")
 	m.volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
@@ -86,7 +85,6 @@ func runMaster(cmd *Command, args []string) bool {
 	util.LoadConfiguration("security", false)
 	util.LoadConfiguration("master", false)
 
-	runtime.GOMAXPROCS(runtime.NumCPU())
 	grace.SetupProfiling(*masterCpuProfile, *masterMemProfile)
 
 	parent, _ := util.FullPath(*m.metaFolder).DirAndName()
@@ -138,7 +136,6 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 	if err != nil {
 		glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
 	}
-	// Create your protocol servers.
 	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
 	master_pb.RegisterSeaweedServer(grpcS, ms)
 	protobuf.RegisterRaftServer(grpcS, raftServer)

weed/command/mount.go

@@ -25,6 +25,7 @@ type MountOptions struct {
 	volumeServerAccess *string
 	uidMap             *string
 	gidMap             *string
+	readOnly           *bool
 }
 
 var (
@@ -45,7 +46,7 @@ func init() {
 	mountOptions.diskType = cmdMount.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
 	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files")
-	mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 128, "limit concurrent goroutine writers if not 0")
+	mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers if not 0")
 	mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data")
 	mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 1000, "local file chunk cache capacity in MB (0 will disable cache)")
 	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
@@ -55,6 +56,7 @@ func init() {
 	mountOptions.volumeServerAccess = cmdMount.Flag.String("volumeServerAccess", "direct", "access volume servers by [direct|publicUrl|filerProxy]")
 	mountOptions.uidMap = cmdMount.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated <local_uid>:<filer_uid>")
 	mountOptions.gidMap = cmdMount.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated <local_gid>:<filer_gid>")
+	mountOptions.readOnly = cmdMount.Flag.Bool("readOnly", false, "read only")
 
 	mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
 	mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")

weed/command/mount_std.go

@@ -63,16 +63,23 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 	// try to connect to filer, filerBucketsPath may be useful later
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 	var cipher bool
-	err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
-		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+	for i := 0; i < 10; i++ {
+		err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+			if err != nil {
+				return fmt.Errorf("get filer grpc address %s configuration: %v", filerGrpcAddress, err)
+			}
+			cipher = resp.Cipher
+			return nil
+		})
 		if err != nil {
-			return fmt.Errorf("get filer grpc address %s configuration: %v", filerGrpcAddress, err)
+			glog.V(0).Infof("failed to talk to filer %s: %v", filerGrpcAddress, err)
+			glog.V(0).Infof("wait for %d seconds ...", i+1)
+			time.Sleep(time.Duration(i+1) * time.Second)
 		}
-		cipher = resp.Cipher
-		return nil
-	})
+	}
 	if err != nil {
-		glog.Infof("failed to talk to filer %s: %v", filerGrpcAddress, err)
+		glog.Errorf("failed to talk to filer %s: %v", filerGrpcAddress, err)
 		return true
 	}
 
@@ -142,8 +149,6 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		fuse.Subtype("seaweedfs"),
 		// fuse.NoAppleDouble(), // include .DS_Store, otherwise can not delete non-empty folders
 		fuse.NoAppleXattr(),
-		fuse.NoBrowse(),
-		fuse.AutoXattr(),
 		fuse.ExclCreate(),
 		fuse.DaemonTimeout("3600"),
 		fuse.AllowSUID(),
@@ -162,6 +167,9 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 	if *option.nonempty {
 		options = append(options, fuse.AllowNonEmptyMount())
 	}
+	if *option.readOnly {
+		options = append(options, fuse.ReadOnly())
+	}
 
 	// find mount point
 	mountRoot := filerMountRootPath
@@ -186,7 +194,6 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		CacheDir:           *option.cacheDir,
 		CacheSizeMB:        *option.cacheSizeMB,
 		DataCenter:         *option.dataCenter,
-		EntryCacheTtl:      3 * time.Second,
 		MountUid:           uid,
 		MountGid:           gid,
 		MountMode:          mountMode,

weed/command/scaffold.go

@@ -103,9 +103,9 @@ dir = "./filerrdb"					# directory to store rocksdb files
 
 [mysql]  # or memsql, tidb
 # CREATE TABLE IF NOT EXISTS filemeta (
-#   dirhash     BIGINT         COMMENT 'first 64 bits of MD5 hash value of directory field',
-#   name        VARCHAR(1000)  COMMENT 'directory or file name',
-#   directory   TEXT           COMMENT 'full path to parent directory',
+#   dirhash     BIGINT               COMMENT 'first 64 bits of MD5 hash value of directory field',
+#   name        VARCHAR(1000) BINARY COMMENT 'directory or file name',
+#   directory   TEXT                 COMMENT 'full path to parent directory',
 #   meta        LONGBLOB,
 #   PRIMARY KEY (dirhash, name)
 # ) DEFAULT CHARSET=utf8;
@@ -120,13 +120,16 @@ connection_max_idle = 2
 connection_max_open = 100
 connection_max_lifetime_seconds = 0
 interpolateParams = false
+# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
+enableUpsert = true
+upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
 
 [mysql2]  # or memsql, tidb
 enabled = false
 createTable = """
   CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` (
     dirhash BIGINT,
-    name VARCHAR(1000),
+    name VARCHAR(1000) BINARY,
     directory TEXT,
     meta LONGBLOB,
     PRIMARY KEY (dirhash, name)
@@ -141,6 +144,9 @@ connection_max_idle = 2
 connection_max_open = 100
 connection_max_lifetime_seconds = 0
 interpolateParams = false
+# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
+enableUpsert = true
+upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
 
 [postgres] # or cockroachdb, YugabyteDB
 # CREATE TABLE IF NOT EXISTS filemeta (
@@ -161,6 +167,9 @@ sslmode = "disable"
 connection_max_idle = 100
 connection_max_open = 100
 connection_max_lifetime_seconds = 0
+# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
+enableUpsert = true
+upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
 
 [postgres2]
 enabled = false
@@ -183,6 +192,9 @@ sslmode = "disable"
 connection_max_idle = 100
 connection_max_open = 100
 connection_max_lifetime_seconds = 0
+# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
+enableUpsert = true
+upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
 
 [cassandra]
 # CREATE TABLE filemeta (
@@ -269,7 +281,7 @@ index.max_result_window = 10000
 #     Make sure they are not the same if using the same store type!
 # 4. Set enabled to true
 #
-# The following is just using cassandra as an example
+# The following is just using redis as an example
 ##########################
 [redis2.tmp]
 enabled = false
@@ -440,22 +452,28 @@ expires_after_seconds = 10           # seconds
 # the host name is not checked, so the PERM files can be shared.
 [grpc]
 ca = ""
+# Set wildcard domain for enable TLS authentication by common names
+allowed_wildcard_domain = "" # .mycompany.com
 
 [grpc.volume]
 cert = ""
 key  = ""
+allowed_commonNames = ""	# comma-separated SSL certificate common names
 
 [grpc.master]
 cert = ""
 key  = ""
+allowed_commonNames = ""	# comma-separated SSL certificate common names
 
 [grpc.filer]
 cert = ""
 key  = ""
+allowed_commonNames = ""	# comma-separated SSL certificate common names
 
 [grpc.msg_broker]
 cert = ""
 key  = ""
+allowed_commonNames = ""	# comma-separated SSL certificate common names
 
 # use this for any place needs a grpc client
 # i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
@@ -463,7 +481,6 @@ key  = ""
 cert = ""
 key  = ""
 
-
 # volume server https options
 # Note: work in progress!
 #     this does not work with other clients, e.g., "weed filer|mount" etc, yet.
@@ -501,7 +518,7 @@ default = "localhost:8888"    # used by maintenance scripts if the scripts needs
 
 
 [master.sequencer]
-type = "raft"     # Choose [raft|etcd] type for storing the file id sequence
+type = "raft"     # Choose [raft|etcd|snowflake] type for storing the file id sequence
 # when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence
 # example : http://127.0.0.1:2379,http://127.0.0.1:2389
 sequencer_etcd_urls = "http://127.0.0.1:2379"

weed/command/server.go

@@ -2,9 +2,8 @@ package command
 
 import (
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/util/grace"
 	"os"
-	"runtime"
-	"runtime/pprof"
 	"strings"
 	"time"
 
@@ -16,6 +15,7 @@ import (
 
 type ServerOptions struct {
 	cpuprofile *string
+	memprofile *string
 	v          VolumeServerOptions
 }
 
@@ -49,8 +49,8 @@ var cmdServer = &Command{
 }
 
 var (
-	serverIp                  = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
-	serverBindIp              = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
+	serverIp                  = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
+	serverBindIp              = cmdServer.Flag.String("ip.bind", "", "ip address to bind to")
 	serverTimeout             = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
 	serverDataCenter          = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
 	serverRack                = cmdServer.Flag.String("rack", "", "current volume server's rack name")
@@ -58,7 +58,8 @@ var (
 	serverDisableHttp         = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
 	volumeDataFolders         = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
 	volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.")
-	volumeMinFreeSpacePercent = cmdServer.Flag.String("volume.minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly.")
+	volumeMinFreeSpacePercent = cmdServer.Flag.String("volume.minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly (deprecated, use minFreeSpace instead).")
+	volumeMinFreeSpace        = cmdServer.Flag.String("volume.minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.")
 	serverMetricsHttpPort     = cmdServer.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
 
 	// pulseSeconds              = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
@@ -76,6 +77,7 @@ var (
 
 func init() {
 	serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")
+	serverOptions.memprofile = cmdServer.Flag.String("memprofile", "", "memory profile output file")
 
 	masterOptions.port = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
 	masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
@@ -93,11 +95,12 @@ func init() {
 	filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
 	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
 	filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
-	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit")
+	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 4, "split files larger than the limit")
 	filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
 	filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers")
 	filerOptions.peers = cmdServer.Flag.String("filer.peers", "", "all filers sharing the same filer store in comma separated ip:port list")
 	filerOptions.saveToFilerLimit = cmdServer.Flag.Int("filer.saveToFilerLimit", 0, "Small files smaller than this limit can be cached in filer store.")
+	filerOptions.concurrentUploadLimitMB = cmdServer.Flag.Int("filer.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
 
 	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
 	serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
@@ -107,6 +110,7 @@ func init() {
 	serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
 	serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
 	serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")
+	serverOptions.v.concurrentUploadLimitMB = cmdServer.Flag.Int("volume.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
 	serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
 	serverOptions.v.preStopSeconds = cmdServer.Flag.Int("volume.preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
 	serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
@@ -138,14 +142,7 @@ func runServer(cmd *Command, args []string) bool {
 	util.LoadConfiguration("security", false)
 	util.LoadConfiguration("master", false)
 
-	if *serverOptions.cpuprofile != "" {
-		f, err := os.Create(*serverOptions.cpuprofile)
-		if err != nil {
-			glog.Fatal(err)
-		}
-		pprof.StartCPUProfile(f)
-		defer pprof.StopCPUProfile()
-	}
+	grace.SetupProfiling(*serverOptions.cpuprofile, *serverOptions.memprofile)
 
 	if *isStartingS3 {
 		*isStartingFiler = true
@@ -192,7 +189,6 @@ func runServer(cmd *Command, args []string) bool {
 	webdavOptions.filer = &filerAddress
 	msgBrokerOptions.filer = &filerAddress
 
-	runtime.GOMAXPROCS(runtime.NumCPU())
 	go stats_collect.StartMetricsServer(*serverMetricsHttpPort)
 
 	folders := strings.Split(*volumeDataFolders, ",")
@@ -249,8 +245,8 @@ func runServer(cmd *Command, args []string) bool {
 
 	// start volume server
 	if *isStartingVolumeServer {
-		go serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption, *volumeMinFreeSpacePercent)
-
+		minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent)
+		go serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption, minFreeSpaces)
 	}
 
 	if *isStartingMasterServer {
@@ -258,6 +254,4 @@ func runServer(cmd *Command, args []string) bool {
 	}
 
 	select {}
-
-	return true
 }

weed/command/upload.go

@@ -43,7 +43,7 @@ func init() {
 	upload.dataCenter = cmdUpload.Flag.String("dataCenter", "", "optional data center name")
 	upload.diskType = cmdUpload.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 	upload.ttl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
-	upload.maxMB = cmdUpload.Flag.Int("maxMB", 32, "split files larger than the limit")
+	upload.maxMB = cmdUpload.Flag.Int("maxMB", 4, "split files larger than the limit")
 	upload.usePublicUrl = cmdUpload.Flag.Bool("usePublicUrl", false, "upload to public url from volume server")
 }
 

weed/command/volume.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	httppprof "net/http/pprof"
 	"os"
-	"runtime"
 	"runtime/pprof"
 	"strconv"
 	"strings"
@@ -36,31 +35,31 @@ var (
 )
 
 type VolumeServerOptions struct {
-	port                  *int
-	publicPort            *int
-	folders               []string
-	folderMaxLimits       []int
-	idxFolder             *string
-	ip                    *string
-	publicUrl             *string
-	bindIp                *string
-	masters               *string
-	idleConnectionTimeout *int
-	dataCenter            *string
-	rack                  *string
-	whiteList             []string
-	indexType             *string
-	diskType              *string
-	fixJpgOrientation     *bool
-	readRedirect          *bool
-	cpuProfile            *string
-	memProfile            *string
-	compactionMBPerSecond *int
-	fileSizeLimitMB       *int
-	minFreeSpacePercents  []float32
-	pprof                 *bool
-	preStopSeconds        *int
-	metricsHttpPort       *int
+	port                    *int
+	publicPort              *int
+	folders                 []string
+	folderMaxLimits         []int
+	idxFolder               *string
+	ip                      *string
+	publicUrl               *string
+	bindIp                  *string
+	masters                 *string
+	idleConnectionTimeout   *int
+	dataCenter              *string
+	rack                    *string
+	whiteList               []string
+	indexType               *string
+	diskType                *string
+	fixJpgOrientation       *bool
+	readRedirect            *bool
+	cpuProfile              *string
+	memProfile              *string
+	compactionMBPerSecond   *int
+	fileSizeLimitMB         *int
+	concurrentUploadLimitMB *int
+	pprof                   *bool
+	preStopSeconds          *int
+	metricsHttpPort         *int
 	// pulseSeconds          *int
 	enableTcp *bool
 }
@@ -69,9 +68,9 @@ func init() {
 	cmdVolume.Run = runVolume // break init cycle
 	v.port = cmdVolume.Flag.Int("port", 8080, "http listen port")
 	v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public")
-	v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
+	v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
 	v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
-	v.bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
+	v.bindIp = cmdVolume.Flag.String("ip.bind", "", "ip address to bind to")
 	v.masters = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
 	v.preStopSeconds = cmdVolume.Flag.Int("preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
 	// v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
@@ -86,6 +85,7 @@ func init() {
 	v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
 	v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
 	v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
+	v.concurrentUploadLimitMB = cmdVolume.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size")
 	v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
 	v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
 	v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files")
@@ -104,15 +104,14 @@ var (
 	volumeFolders         = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
 	maxVolumeCounts       = cmdVolume.Flag.String("max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.")
 	volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
-	minFreeSpacePercent   = cmdVolume.Flag.String("minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly.")
+	minFreeSpacePercent   = cmdVolume.Flag.String("minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly (deprecated, use minFreeSpace instead).")
+	minFreeSpace          = cmdVolume.Flag.String("minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.")
 )
 
 func runVolume(cmd *Command, args []string) bool {
 
 	util.LoadConfiguration("security", false)
 
-	runtime.GOMAXPROCS(runtime.NumCPU())
-
 	// If --pprof is set we assume the caller wants to be able to collect
 	// cpu and memory profiles via go tool pprof
 	if !*v.pprof {
@@ -121,12 +120,13 @@ func runVolume(cmd *Command, args []string) bool {
 
 	go stats_collect.StartMetricsServer(*v.metricsHttpPort)
 
-	v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, *minFreeSpacePercent)
+	minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent)
+	v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, minFreeSpaces)
 
 	return true
 }
 
-func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption, minFreeSpacePercent string) {
+func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption string, minFreeSpaces []util.MinFreeSpace) {
 
 	// Set multiple folders and each folder's max volume count limit'
 	v.folders = strings.Split(volumeFolders, ",")
@@ -154,22 +154,13 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 		glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(v.folders), len(v.folderMaxLimits))
 	}
 
-	// set minFreeSpacePercent
-	minFreeSpacePercentStrings := strings.Split(minFreeSpacePercent, ",")
-	for _, freeString := range minFreeSpacePercentStrings {
-		if value, e := strconv.ParseFloat(freeString, 32); e == nil {
-			v.minFreeSpacePercents = append(v.minFreeSpacePercents, float32(value))
-		} else {
-			glog.Fatalf("The value specified in -minFreeSpacePercent not a valid value %s", freeString)
-		}
-	}
-	if len(v.minFreeSpacePercents) == 1 && len(v.folders) > 1 {
+	if len(minFreeSpaces) == 1 && len(v.folders) > 1 {
 		for i := 0; i < len(v.folders)-1; i++ {
-			v.minFreeSpacePercents = append(v.minFreeSpacePercents, v.minFreeSpacePercents[0])
+			minFreeSpaces = append(minFreeSpaces, minFreeSpaces[0])
 		}
 	}
-	if len(v.folders) != len(v.minFreeSpacePercents) {
-		glog.Fatalf("%d directories by -dir, but only %d minFreeSpacePercent is set by -minFreeSpacePercent", len(v.folders), len(v.minFreeSpacePercents))
+	if len(v.folders) != len(minFreeSpaces) {
+		glog.Fatalf("%d directories by -dir, but only %d minFreeSpacePercent is set by -minFreeSpacePercent", len(v.folders), len(minFreeSpaces))
 	}
 
 	// set disk types
@@ -232,7 +223,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 
 	volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
 		*v.ip, *v.port, *v.publicUrl,
-		v.folders, v.folderMaxLimits, v.minFreeSpacePercents, diskTypes,
+		v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes,
 		*v.idxFolder,
 		volumeNeedleMapKind,
 		strings.Split(masters, ","), 5, *v.dataCenter, *v.rack,
@@ -240,6 +231,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 		*v.fixJpgOrientation, *v.readRedirect,
 		*v.compactionMBPerSecond,
 		*v.fileSizeLimitMB,
+		int64(*v.concurrentUploadLimitMB)*1024*1024,
 	)
 	// starting grpc server
 	grpcS := v.startGrpcService(volumeServer)

weed/filer/abstract_sql/abstract_sql_store.go

@@ -13,15 +13,15 @@ import (
 )
 
 type SqlGenerator interface {
-	GetSqlInsert(bucket string) string
-	GetSqlUpdate(bucket string) string
-	GetSqlFind(bucket string) string
-	GetSqlDelete(bucket string) string
-	GetSqlDeleteFolderChildren(bucket string) string
-	GetSqlListExclusive(bucket string) string
-	GetSqlListInclusive(bucket string) string
-	GetSqlCreateTable(bucket string) string
-	GetSqlDropTable(bucket string) string
+	GetSqlInsert(tableName string) string
+	GetSqlUpdate(tableName string) string
+	GetSqlFind(tableName string) string
+	GetSqlDelete(tableName string) string
+	GetSqlDeleteFolderChildren(tableName string) string
+	GetSqlListExclusive(tableName string) string
+	GetSqlListInclusive(tableName string) string
+	GetSqlCreateTable(tableName string) string
+	GetSqlDropTable(tableName string) string
 }
 
 type AbstractSqlStore struct {
@@ -32,6 +32,29 @@ type AbstractSqlStore struct {
 	dbsLock            sync.Mutex
 }
 
+func (store *AbstractSqlStore) OnBucketCreation(bucket string) {
+	store.dbsLock.Lock()
+	defer store.dbsLock.Unlock()
+
+	store.CreateTable(context.Background(), bucket)
+
+	if store.dbs == nil {
+		return
+	}
+	store.dbs[bucket] = true
+}
+func (store *AbstractSqlStore) OnBucketDeletion(bucket string) {
+	store.dbsLock.Lock()
+	defer store.dbsLock.Unlock()
+
+	store.deleteTable(context.Background(), bucket)
+
+	if store.dbs == nil {
+		return
+	}
+	delete(store.dbs, bucket)
+}
+
 const (
 	DEFAULT_TABLE = "filemeta"
 )
@@ -253,7 +276,9 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat
 		}
 	}
 
-	res, err := db.ExecContext(ctx, store.GetSqlDeleteFolderChildren(bucket), util.HashStringToLong(string(shortPath)), fullpath)
+	glog.V(4).Infof("delete %s SQL %s %d", string(shortPath), store.GetSqlDeleteFolderChildren(bucket), util.HashStringToLong(string(shortPath)))
+
+	res, err := db.ExecContext(ctx, store.GetSqlDeleteFolderChildren(bucket), util.HashStringToLong(string(shortPath)), string(shortPath))
 	if err != nil {
 		return fmt.Errorf("deleteFolderChildren %s: %s", fullpath, err)
 	}

weed/filer/filechunk_manifest.go

@@ -97,20 +97,20 @@ func fetchChunk(lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string,
 func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) ([]byte, error) {
 
 	var err error
-	var buffer bytes.Buffer
 	var shouldRetry bool
+	receivedData := make([]byte, 0, size)
 
 	for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
 		for _, urlString := range urlStrings {
-			shouldRetry, err = util.FastReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
-				buffer.Write(data)
+			receivedData = receivedData[:0]
+			shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
+				receivedData = append(receivedData, data...)
 			})
 			if !shouldRetry {
 				break
 			}
 			if err != nil {
 				glog.V(0).Infof("read %s failed, err: %v", urlString, err)
-				buffer.Reset()
 			} else {
 				break
 			}
@@ -123,7 +123,8 @@ func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool
 		}
 	}
 
-	return buffer.Bytes(), err
+	return receivedData, err
+
 }
 
 func MaybeManifestize(saveFunc SaveDataAsChunkFunctionType, inputChunks []*filer_pb.FileChunk) (chunks []*filer_pb.FileChunk, err error) {

weed/filer/filechunks.go

@@ -2,7 +2,6 @@ package filer
 
 import (
 	"bytes"
-	"encoding/hex"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 	"math"
@@ -43,12 +42,11 @@ func ETagEntry(entry *Entry) (etag string) {
 
 func ETagChunks(chunks []*filer_pb.FileChunk) (etag string) {
 	if len(chunks) == 1 {
-		return chunks[0].ETag
+		return fmt.Sprintf("%x", util.Base64Md5ToBytes(chunks[0].ETag))
 	}
 	md5_digests := [][]byte{}
 	for _, c := range chunks {
-		md5_decoded, _ := hex.DecodeString(c.ETag)
-		md5_digests = append(md5_digests, md5_decoded)
+		md5_digests = append(md5_digests, util.Base64Md5ToBytes(c.ETag))
 	}
 	return fmt.Sprintf("%x-%d", util.Md5(bytes.Join(md5_digests, nil)), len(chunks))
 }

weed/filer/filer.go

@@ -241,12 +241,12 @@ func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err er
 	if oldEntry != nil {
 		entry.Attr.Crtime = oldEntry.Attr.Crtime
 		if oldEntry.IsDirectory() && !entry.IsDirectory() {
-			glog.Errorf("existing %s is a directory", entry.FullPath)
-			return fmt.Errorf("existing %s is a directory", entry.FullPath)
+			glog.Errorf("existing %s is a directory", oldEntry.FullPath)
+			return fmt.Errorf("existing %s is a directory", oldEntry.FullPath)
 		}
 		if !oldEntry.IsDirectory() && entry.IsDirectory() {
-			glog.Errorf("existing %s is a file", entry.FullPath)
-			return fmt.Errorf("existing %s is a file", entry.FullPath)
+			glog.Errorf("existing %s is a file", oldEntry.FullPath)
+			return fmt.Errorf("existing %s is a file", oldEntry.FullPath)
 		}
 	}
 	return f.Store.UpdateEntry(ctx, entry)

weed/filer/filer_buckets.go

@@ -29,7 +29,7 @@ func (f *Filer) LoadBuckets() {
 
 	limit := int64(math.MaxInt32)
 
-	entries, _, err := f.ListDirectoryEntries(context.Background(), util.FullPath(f.DirBucketsPath), "", false, limit, "", "")
+	entries, _, err := f.ListDirectoryEntries(context.Background(), util.FullPath(f.DirBucketsPath), "", false, limit, "", "", "")
 
 	if err != nil {
 		glog.V(1).Infof("no buckets found: %v", err)

weed/filer/filer_conf.go

@@ -18,6 +18,7 @@ const (
 	FilerConfName         = "filer.conf"
 	IamConfigDirecotry    = "/etc/iam"
 	IamIdentityFile       = "identity.json"
+	IamPoliciesFile       = "policies.json"
 )
 
 type FilerConf struct {

weed/filer/filer_delete_entry.go

@@ -11,6 +11,10 @@ import (
 
 type HardLinkId []byte
 
+const (
+	MsgFailDelNonEmptyFolder = "fail to delete non-empty folder"
+)
+
 func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isFromOtherCluster bool, signatures []int32) (err error) {
 	if p == "/" {
 		return nil
@@ -69,7 +73,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
 	includeLastFile := false
 	if !isDeletingBucket {
 		for {
-			entries, _, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "")
+			entries, _, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "", "")
 			if err != nil {
 				glog.Errorf("list folder %s: %v", entry.FullPath, err)
 				return nil, nil, fmt.Errorf("list folder %s: %v", entry.FullPath, err)
@@ -77,7 +81,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
 			if lastFileName == "" && !isRecursive && len(entries) > 0 {
 				// only for first iteration in the loop
 				glog.Errorf("deleting a folder %s has children: %+v ...", entry.FullPath, entries[0].Name())
-				return nil, nil, fmt.Errorf("fail to delete non-empty folder: %s", entry.FullPath)
+				return nil, nil, fmt.Errorf("%s: %s", MsgFailDelNonEmptyFolder, entry.FullPath)
 			}
 
 			for _, sub := range entries {

weed/filer/filer_notify.go

@@ -116,13 +116,13 @@ func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, eachLogEntryFn func(
 	sizeBuf := make([]byte, 4)
 	startTsNs := startTime.UnixNano()
 
-	dayEntries, _, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366, "", "")
+	dayEntries, _, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366, "", "", "")
 	if listDayErr != nil {
 		return lastTsNs, fmt.Errorf("fail to list log by day: %v", listDayErr)
 	}
 	for _, dayEntry := range dayEntries {
 		// println("checking day", dayEntry.FullPath)
-		hourMinuteEntries, _, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60, "", "")
+		hourMinuteEntries, _, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60, "", "", "")
 		if listHourMinuteErr != nil {
 			return lastTsNs, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr)
 		}

weed/filer/filer_on_meta_event.go

@@ -11,6 +11,28 @@ import (
 
 // onMetadataChangeEvent is triggered after filer processed change events from local or remote filers
 func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse) {
+	f.maybeReloadFilerConfiguration(event)
+	f.onBucketEvents(event)
+}
+
+func (f *Filer) onBucketEvents(event *filer_pb.SubscribeMetadataResponse) {
+	message := event.EventNotification
+	for _, sig := range message.Signatures {
+		if sig == f.Signature {
+			return
+		}
+	}
+	if f.DirBucketsPath == event.Directory {
+		if message.OldEntry == nil && message.NewEntry != nil {
+			f.Store.OnBucketCreation(message.NewEntry.Name)
+		}
+		if message.OldEntry != nil && message.NewEntry == nil {
+			f.Store.OnBucketDeletion(message.OldEntry.Name)
+		}
+	}
+}
+
+func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataResponse) {
 	if DirectoryEtcSeaweedFS != event.Directory {
 		if DirectoryEtcSeaweedFS != event.EventNotification.NewParentPath {
 			return
@@ -26,7 +48,6 @@ func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse)
 	if entry.Name == FilerConfName {
 		f.reloadFilerConfiguration(entry)
 	}
-
 }
 
 func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) {

weed/filer/filer_search.go

@@ -20,9 +20,9 @@ func splitPattern(pattern string) (prefix string, restPattern string) {
 }
 
 // For now, prefix and namePattern are mutually exclusive
-func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string) (entries []*Entry, hasMore bool, err error) {
+func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string, namePatternExclude string) (entries []*Entry, hasMore bool, err error) {
 
-	_, err = f.StreamListDirectoryEntries(ctx, p, startFileName, inclusive, limit+1, prefix, namePattern, func(entry *Entry) bool {
+	_, err = f.StreamListDirectoryEntries(ctx, p, startFileName, inclusive, limit+1, prefix, namePattern, namePatternExclude, func(entry *Entry) bool {
 		entries = append(entries, entry)
 		return true
 	})
@@ -36,7 +36,7 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start
 }
 
 // For now, prefix and namePattern are mutually exclusive
-func (f *Filer) StreamListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) {
+func (f *Filer) StreamListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string, namePatternExclude string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) {
 	if strings.HasSuffix(string(p), "/") && len(p) > 1 {
 		p = p[0 : len(p)-1]
 	}
@@ -47,30 +47,38 @@ func (f *Filer) StreamListDirectoryEntries(ctx context.Context, p util.FullPath,
 	}
 	var missedCount int64
 
-	missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, limit, prefix, restNamePattern, eachEntryFunc)
+	missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, limit, prefix, restNamePattern, namePatternExclude, eachEntryFunc)
 
 	for missedCount > 0 && err == nil {
-		missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, missedCount, prefix, restNamePattern, eachEntryFunc)
+		missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, missedCount, prefix, restNamePattern, namePatternExclude, eachEntryFunc)
 	}
 
 	return
 }
 
-func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix, restNamePattern string, eachEntryFunc ListEachEntryFunc) (missedCount int64, lastFileName string, err error) {
+func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix, restNamePattern string, namePatternExclude string, eachEntryFunc ListEachEntryFunc) (missedCount int64, lastFileName string, err error) {
 
-	if len(restNamePattern) == 0 {
+	if len(restNamePattern) == 0 && len(namePatternExclude) == 0 {
 		lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix, eachEntryFunc)
 		return 0, lastFileName, err
 	}
 
 	lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix, func(entry *Entry) bool {
-		nameToTest := strings.ToLower(entry.Name())
-		if matched, matchErr := filepath.Match(restNamePattern, nameToTest[len(prefix):]); matchErr == nil && matched {
-			if !eachEntryFunc(entry) {
-				return false
+		nameToTest := entry.Name()
+		if len(namePatternExclude) > 0 {
+			if matched, matchErr := filepath.Match(namePatternExclude, nameToTest); matchErr == nil && matched {
+				missedCount++
+				return true
 			}
-		} else {
-			missedCount++
+		}
+		if len(restNamePattern) > 0 {
+			if matched, matchErr := filepath.Match(restNamePattern, nameToTest[len(prefix):]); matchErr == nil && !matched {
+				missedCount++
+				return true
+			}
+		}
+		if !eachEntryFunc(entry) {
+			return false
 		}
 		return true
 	})

weed/filer/filerstore.go

@@ -39,3 +39,8 @@ type FilerStore interface {
 
 	Shutdown()
 }
+
+type BucketAware interface {
+	OnBucketCreation(bucket string)
+	OnBucketDeletion(bucket string)
+}

weed/filer/filerstore_wrapper.go

@@ -21,6 +21,8 @@ type VirtualFilerStore interface {
 	DeleteHardLink(ctx context.Context, hardLinkId HardLinkId) error
 	DeleteOneEntry(ctx context.Context, entry *Entry) error
 	AddPathSpecificStore(path string, storeId string, store FilerStore)
+	OnBucketCreation(bucket string)
+	OnBucketDeletion(bucket string)
 }
 
 type FilerStoreWrapper struct {
@@ -40,6 +42,27 @@ func NewFilerStoreWrapper(store FilerStore) *FilerStoreWrapper {
 	}
 }
 
+func (fsw *FilerStoreWrapper) OnBucketCreation(bucket string) {
+	for _, store := range fsw.storeIdToStore {
+		if ba, ok := store.(BucketAware); ok {
+			ba.OnBucketCreation(bucket)
+		}
+	}
+	if ba, ok := fsw.defaultStore.(BucketAware); ok {
+		ba.OnBucketCreation(bucket)
+	}
+}
+func (fsw *FilerStoreWrapper) OnBucketDeletion(bucket string) {
+	for _, store := range fsw.storeIdToStore {
+		if ba, ok := store.(BucketAware); ok {
+			ba.OnBucketDeletion(bucket)
+		}
+	}
+	if ba, ok := fsw.defaultStore.(BucketAware); ok {
+		ba.OnBucketDeletion(bucket)
+	}
+}
+
 func (fsw *FilerStoreWrapper) AddPathSpecificStore(path string, storeId string, store FilerStore) {
 	fsw.storeIdToStore[storeId] = NewFilerStorePathTranlator(path, store)
 	err := fsw.pathToStore.Put([]byte(path), storeId)
@@ -126,8 +149,8 @@ func (fsw *FilerStoreWrapper) FindEntry(ctx context.Context, fp util.FullPath) (
 		stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "find").Observe(time.Since(start).Seconds())
 	}()
 
-	glog.V(4).Infof("FindEntry %s", fp)
 	entry, err = actualStore.FindEntry(ctx, fp)
+	// glog.V(4).Infof("FindEntry %s: %v", fp, err)
 	if err != nil {
 		return nil, err
 	}

weed/filer/leveldb/leveldb_store.go

@@ -8,6 +8,7 @@ import (
 	leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
+	"os"
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -38,6 +39,7 @@ func (store *LevelDBStore) Initialize(configuration weed_util.Configuration, pre
 
 func (store *LevelDBStore) initialize(dir string) (err error) {
 	glog.Infof("filer store dir: %s", dir)
+	os.MkdirAll(dir, 0755)
 	if err := weed_util.TestFolderWritable(dir); err != nil {
 		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
 	}

weed/filer/leveldb/leveldb_store_test.go

@@ -51,14 +51,14 @@ func TestCreateAndFind(t *testing.T) {
 	}
 
 	// checking one upper directory
-	entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "")
+	entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "", "")
 	if len(entries) != 1 {
 		t.Errorf("list entries count: %v", len(entries))
 		return
 	}
 
 	// checking one upper directory
-	entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
+	entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "", "")
 	if len(entries) != 1 {
 		t.Errorf("list entries count: %v", len(entries))
 		return
@@ -77,7 +77,7 @@ func TestEmptyRoot(t *testing.T) {
 	ctx := context.Background()
 
 	// checking one upper directory
-	entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
+	entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "", "")
 	if err != nil {
 		t.Errorf("list entries: %v", err)
 		return

weed/filer/leveldb2/leveldb2_store.go

@@ -38,6 +38,7 @@ func (store *LevelDB2Store) Initialize(configuration weed_util.Configuration, pr
 
 func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
 	glog.Infof("filer store leveldb2 dir: %s", dir)
+	os.MkdirAll(dir, 0755)
 	if err := weed_util.TestFolderWritable(dir); err != nil {
 		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
 	}

weed/filer/leveldb2/leveldb2_store_test.go

@@ -49,14 +49,14 @@ func TestCreateAndFind(t *testing.T) {
 	}
 
 	// checking one upper directory
-	entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "")
+	entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "", "")
 	if len(entries) != 1 {
 		t.Errorf("list entries count: %v", len(entries))
 		return
 	}
 
 	// checking one upper directory
-	entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
+	entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "", "")
 	if len(entries) != 1 {
 		t.Errorf("list entries count: %v", len(entries))
 		return
@@ -75,7 +75,7 @@ func TestEmptyRoot(t *testing.T) {
 	ctx := context.Background()
 
 	// checking one upper directory
-	entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
+	entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "", "")
 	if err != nil {
 		t.Errorf("list entries: %v", err)
 		return

weed/filer/leveldb3/leveldb3_store.go

@@ -45,6 +45,7 @@ func (store *LevelDB3Store) Initialize(configuration weed_util.Configuration, pr
 
 func (store *LevelDB3Store) initialize(dir string) (err error) {
 	glog.Infof("filer store leveldb3 dir: %s", dir)
+	os.MkdirAll(dir, 0755)
 	if err := weed_util.TestFolderWritable(dir); err != nil {
 		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
 	}

weed/filer/leveldb3/leveldb3_store_test.go

@@ -49,14 +49,14 @@ func TestCreateAndFind(t *testing.T) {
 	}
 
 	// checking one upper directory
-	entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "")
+	entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "", "")
 	if len(entries) != 1 {
 		t.Errorf("list entries count: %v", len(entries))
 		return
 	}
 
 	// checking one upper directory
-	entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
+	entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "", "")
 	if len(entries) != 1 {
 		t.Errorf("list entries count: %v", len(entries))
 		return
@@ -75,7 +75,7 @@ func TestEmptyRoot(t *testing.T) {
 	ctx := context.Background()
 
 	// checking one upper directory
-	entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
+	entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "", "")
 	if err != nil {
 		t.Errorf("list entries: %v", err)
 		return

weed/filer/meta_aggregator.go

@@ -69,6 +69,7 @@ func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self string, peer string
 		peerSignature, err = ma.readFilerStoreSignature(peer)
 	}
 
+	// when filer store is not shared by multiple filers
 	if peerSignature != f.Signature {
 		if prevTsNs, err := ma.readOffset(f, peer, peerSignature); err == nil {
 			lastTsNs = prevTsNs

weed/filer/mysql/mysql_sql_gen.go

@@ -2,6 +2,7 @@ package mysql
 
 import (
 	"fmt"
+
 	"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
 	_ "github.com/go-sql-driver/mysql"
 )
@@ -9,44 +10,49 @@ import (
 type SqlGenMysql struct {
 	CreateTableSqlTemplate string
 	DropTableSqlTemplate   string
+	UpsertQueryTemplate    string
 }
 
 var (
 	_ = abstract_sql.SqlGenerator(&SqlGenMysql{})
 )
 
-func (gen *SqlGenMysql) GetSqlInsert(bucket string) string {
-	return fmt.Sprintf("INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?)", bucket)
+func (gen *SqlGenMysql) GetSqlInsert(tableName string) string {
+	if gen.UpsertQueryTemplate != "" {
+		return fmt.Sprintf(gen.UpsertQueryTemplate, tableName)
+	} else {
+		return fmt.Sprintf("INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?)", tableName)
+	}
 }
 
-func (gen *SqlGenMysql) GetSqlUpdate(bucket string) string {
-	return fmt.Sprintf("UPDATE `%s` SET meta=? WHERE dirhash=? AND name=? AND directory=?", bucket)
+func (gen *SqlGenMysql) GetSqlUpdate(tableName string) string {
+	return fmt.Sprintf("UPDATE `%s` SET meta=? WHERE dirhash=? AND name=? AND directory=?", tableName)
 }
 
-func (gen *SqlGenMysql) GetSqlFind(bucket string) string {
-	return fmt.Sprintf("SELECT meta FROM `%s` WHERE dirhash=? AND name=? AND directory=?", bucket)
+func (gen *SqlGenMysql) GetSqlFind(tableName string) string {
+	return fmt.Sprintf("SELECT meta FROM `%s` WHERE dirhash=? AND name=? AND directory=?", tableName)
 }
 
-func (gen *SqlGenMysql) GetSqlDelete(bucket string) string {
-	return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND name=? AND directory=?", bucket)
+func (gen *SqlGenMysql) GetSqlDelete(tableName string) string {
+	return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND name=? AND directory=?", tableName)
 }
 
-func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(bucket string) string {
-	return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND directory=?", bucket)
+func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(tableName string) string {
+	return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND directory=?", tableName)
 }
 
-func (gen *SqlGenMysql) GetSqlListExclusive(bucket string) string {
-	return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", bucket)
+func (gen *SqlGenMysql) GetSqlListExclusive(tableName string) string {
+	return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", tableName)
 }
 
-func (gen *SqlGenMysql) GetSqlListInclusive(bucket string) string {
-	return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", bucket)
+func (gen *SqlGenMysql) GetSqlListInclusive(tableName string) string {
+	return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", tableName)
 }
 
-func (gen *SqlGenMysql) GetSqlCreateTable(bucket string) string {
-	return fmt.Sprintf(gen.CreateTableSqlTemplate, bucket)
+func (gen *SqlGenMysql) GetSqlCreateTable(tableName string) string {
+	return fmt.Sprintf(gen.CreateTableSqlTemplate, tableName)
 }
 
-func (gen *SqlGenMysql) GetSqlDropTable(bucket string) string {
-	return fmt.Sprintf(gen.DropTableSqlTemplate, bucket)
+func (gen *SqlGenMysql) GetSqlDropTable(tableName string) string {
+	return fmt.Sprintf(gen.DropTableSqlTemplate, tableName)
 }

weed/filer/mysql/mysql_store.go

@@ -3,9 +3,10 @@ package mysql
 import (
 	"database/sql"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/filer"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/weed/filer"
+
 	"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	_ "github.com/go-sql-driver/mysql"
@@ -29,6 +30,8 @@ func (store *MysqlStore) GetName() string {
 
 func (store *MysqlStore) Initialize(configuration util.Configuration, prefix string) (err error) {
 	return store.initialize(
+		configuration.GetString(prefix+"upsertQuery"),
+		configuration.GetBool(prefix+"enableUpsert"),
 		configuration.GetString(prefix+"username"),
 		configuration.GetString(prefix+"password"),
 		configuration.GetString(prefix+"hostname"),
@@ -41,13 +44,17 @@ func (store *MysqlStore) Initialize(configuration util.Configuration, prefix str
 	)
 }
 
-func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen,
+func (store *MysqlStore) initialize(upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database string, maxIdle, maxOpen,
 	maxLifetimeSeconds int, interpolateParams bool) (err error) {
 
 	store.SupportBucketTable = false
+	if !enableUpsert {
+		upsertQuery = ""
+	}
 	store.SqlGenerator = &SqlGenMysql{
 		CreateTableSqlTemplate: "",
 		DropTableSqlTemplate:   "drop table `%s`",
+		UpsertQueryTemplate:    upsertQuery,
 	}
 
 	sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)

weed/filer/mysql2/mysql2_store.go

@@ -32,6 +32,8 @@ func (store *MysqlStore2) GetName() string {
 func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
 	return store.initialize(
 		configuration.GetString(prefix+"createTable"),
+		configuration.GetString(prefix+"upsertQuery"),
+		configuration.GetBool(prefix+"enableUpsert"),
 		configuration.GetString(prefix+"username"),
 		configuration.GetString(prefix+"password"),
 		configuration.GetString(prefix+"hostname"),
@@ -44,13 +46,17 @@ func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix st
 	)
 }
 
-func (store *MysqlStore2) initialize(createTable, user, password, hostname string, port int, database string, maxIdle, maxOpen,
+func (store *MysqlStore2) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database string, maxIdle, maxOpen,
 	maxLifetimeSeconds int, interpolateParams bool) (err error) {
 
 	store.SupportBucketTable = true
+	if !enableUpsert {
+		upsertQuery = ""
+	}
 	store.SqlGenerator = &mysql.SqlGenMysql{
 		CreateTableSqlTemplate: createTable,
 		DropTableSqlTemplate:   "drop table `%s`",
+		UpsertQueryTemplate:    upsertQuery,
 	}
 
 	sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)

weed/filer/postgres/postgres_sql_gen.go

@@ -10,44 +10,49 @@ import (
 type SqlGenPostgres struct {
 	CreateTableSqlTemplate string
 	DropTableSqlTemplate   string
+	UpsertQueryTemplate    string
 }
 
 var (
 	_ = abstract_sql.SqlGenerator(&SqlGenPostgres{})
 )
 
-func (gen *SqlGenPostgres) GetSqlInsert(bucket string) string {
-	return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, bucket)
+func (gen *SqlGenPostgres) GetSqlInsert(tableName string) string {
+	if gen.UpsertQueryTemplate != "" {
+		return fmt.Sprintf(gen.UpsertQueryTemplate, tableName)
+	} else {
+		return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, tableName)
+	}
 }
 
-func (gen *SqlGenPostgres) GetSqlUpdate(bucket string) string {
-	return fmt.Sprintf(`UPDATE "%s" SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4`, bucket)
+func (gen *SqlGenPostgres) GetSqlUpdate(tableName string) string {
+	return fmt.Sprintf(`UPDATE "%s" SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4`, tableName)
 }
 
-func (gen *SqlGenPostgres) GetSqlFind(bucket string) string {
-	return fmt.Sprintf(`SELECT meta FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, bucket)
+func (gen *SqlGenPostgres) GetSqlFind(tableName string) string {
+	return fmt.Sprintf(`SELECT meta FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, tableName)
 }
 
-func (gen *SqlGenPostgres) GetSqlDelete(bucket string) string {
-	return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, bucket)
+func (gen *SqlGenPostgres) GetSqlDelete(tableName string) string {
+	return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, tableName)
 }
 
-func (gen *SqlGenPostgres) GetSqlDeleteFolderChildren(bucket string) string {
-	return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND directory=$2`, bucket)
+func (gen *SqlGenPostgres) GetSqlDeleteFolderChildren(tableName string) string {
+	return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND directory=$2`, tableName)
 }
 
-func (gen *SqlGenPostgres) GetSqlListExclusive(bucket string) string {
-	return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, bucket)
+func (gen *SqlGenPostgres) GetSqlListExclusive(tableName string) string {
+	return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, tableName)
 }
 
-func (gen *SqlGenPostgres) GetSqlListInclusive(bucket string) string {
-	return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>=$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, bucket)
+func (gen *SqlGenPostgres) GetSqlListInclusive(tableName string) string {
+	return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>=$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, tableName)
 }
 
-func (gen *SqlGenPostgres) GetSqlCreateTable(bucket string) string {
-	return fmt.Sprintf(gen.CreateTableSqlTemplate, bucket)
+func (gen *SqlGenPostgres) GetSqlCreateTable(tableName string) string {
+	return fmt.Sprintf(gen.CreateTableSqlTemplate, tableName)
 }
 
-func (gen *SqlGenPostgres) GetSqlDropTable(bucket string) string {
-	return fmt.Sprintf(gen.DropTableSqlTemplate, bucket)
+func (gen *SqlGenPostgres) GetSqlDropTable(tableName string) string {
+	return fmt.Sprintf(gen.DropTableSqlTemplate, tableName)
 }

weed/filer/postgres/postgres_store.go

@@ -29,6 +29,8 @@ func (store *PostgresStore) GetName() string {
 
 func (store *PostgresStore) Initialize(configuration util.Configuration, prefix string) (err error) {
 	return store.initialize(
+		configuration.GetString(prefix+"upsertQuery"),
+		configuration.GetBool(prefix+"enableUpsert"),
 		configuration.GetString(prefix+"username"),
 		configuration.GetString(prefix+"password"),
 		configuration.GetString(prefix+"hostname"),
@@ -42,12 +44,16 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix
 	)
 }
 
-func (store *PostgresStore) initialize(user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
+func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
 
 	store.SupportBucketTable = false
+	if !enableUpsert {
+		upsertQuery = ""
+	}
 	store.SqlGenerator = &SqlGenPostgres{
 		CreateTableSqlTemplate: "",
 		DropTableSqlTemplate:   `drop table "%s"`,
+		UpsertQueryTemplate:    upsertQuery,
 	}
 
 	sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode)

weed/filer/postgres2/postgres2_store.go

@@ -32,6 +32,8 @@ func (store *PostgresStore2) GetName() string {
 func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
 	return store.initialize(
 		configuration.GetString(prefix+"createTable"),
+		configuration.GetString(prefix+"upsertQuery"),
+		configuration.GetBool(prefix+"enableUpsert"),
 		configuration.GetString(prefix+"username"),
 		configuration.GetString(prefix+"password"),
 		configuration.GetString(prefix+"hostname"),
@@ -45,12 +47,16 @@ func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix
 	)
 }
 
-func (store *PostgresStore2) initialize(createTable, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
+func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
 
 	store.SupportBucketTable = true
+	if !enableUpsert {
+		upsertQuery = ""
+	}
 	store.SqlGenerator = &postgres.SqlGenPostgres{
 		CreateTableSqlTemplate: createTable,
 		DropTableSqlTemplate:   `drop table "%s"`,
+		UpsertQueryTemplate:    upsertQuery,
 	}
 
 	sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode)

weed/filer/read_write.go

@@ -35,14 +35,18 @@ func ReadContent(filerAddress string, dir, name string) ([]byte, error) {
 
 	target := fmt.Sprintf("http://%s%s/%s", filerAddress, dir, name)
 
-	data, _, err := util.FastGet(target)
+	data, _, err := util.Get(target)
 
 	return data, err
 }
 
 func SaveAs(host string, port int, dir, name string, contentType string, byteBuffer *bytes.Buffer) error {
-
-	target := fmt.Sprintf("http://%s:%d%s/%s", host, port, dir, name)
+	var target string
+	if port == 0 {
+		target = fmt.Sprintf("http://%s%s/%s", host, dir, name)
+	} else {
+		target = fmt.Sprintf("http://%s:%d%s/%s", host, port, dir, name)
+	}
 
 	// set the HTTP method, url, and request body
 	req, err := http.NewRequest(http.MethodPut, target, byteBuffer)