2.85

Merge pull request #2564 from chrislusf/dependabot/maven/other/java/client/com.google.protobuf-protobuf-java-3.16.1
Bump protobuf-java from 3.9.1 to 3.16.1 in /other/java/client
2026-10-02 20:53:39 +00:00 · 2022-01-09 19:30:23 -08:00 · 2022-01-07 15:16:21 -08:00 · 2022-01-07 14:52:16 -08:00 · 2022-01-07 22:37:10 +00:00 · 2022-01-06 09:52:28 -08:00
239 changed files with 9042 additions and 5419 deletions
@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+github: chrislusf
+patreon: seaweedfs
@@ -501,7 +501,7 @@ git clone https://github.com/chrislusf/seaweedfs.git
 Step 3: download, compile, and install the project by executing the following command

 ```bash
-make install
+cd seaweedfs/weed && make install
 ```

 Once this is done, you will find the executable "weed" in your `$GOPATH/bin` directory
@@ -1,57 +0,0 @@
-FROM alpine
-
-# 'latest' or 'dev'
-ARG RELEASE=latest
-
-RUN \
-    ARCH=$(if [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "64" ]; then echo "amd64"; \
-         elif [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "32" ]; then echo "386"; \
-         elif [ $(uname -m) == "aarch64" ]; then echo "arm64"; \
-         elif [ $(uname -m) == "armv7l" ]; then echo "arm"; \
-         elif [ $(uname -m) == "armv6l" ]; then echo "arm"; \
-         elif [ $(uname -m) == "s390x" ]; then echo "s390x"; \
-         elif [ $(uname -m) == "ppc64le" ]; then echo "ppc64le"; fi;) && \
-    echo "Building for $ARCH" 1>&2 && \
-    SUPERCRONIC_SHA1SUM=$(echo $ARCH | sed 's/386/e0126b0102b9f388ecd55714358e3ad60d0cebdb/g' | sed 's/amd64/5ddf8ea26b56d4a7ff6faecdd8966610d5cb9d85/g' | sed 's/arm64/e2714c43e7781bf1579c85aa61259245f56dbba1/g' | sed 's/arm/47481c3341bc3a1ae91a728e0cc63c8e6d3791ad/g') && \
-    SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.1.9/supercronic-linux-$ARCH && \
-    SUPERCRONIC=supercronic-linux-$ARCH && \
-    # Install SeaweedFS and Supercronic ( for cron job mode )
-    apk add --no-cache --virtual build-dependencies --update wget curl ca-certificates && \
-    apk add fuse && \
-    wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/${RELEASE} | egrep -o "chrislusf/seaweedfs/releases/download/.*/linux_$ARCH.tar.gz" | head -n 1) && \
-    tar -C /usr/bin/ -xzvf /tmp/linux_$ARCH.tar.gz && \
-    curl -fsSLO "$SUPERCRONIC_URL" && \
-    echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
-    chmod +x "$SUPERCRONIC" && \
-    mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" && \
-    ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic && \
-    apk del build-dependencies && \
-    rm -rf /tmp/*
-
-# volume server gprc port
-EXPOSE 18080
-# volume server http port
-EXPOSE 8080
-# filer server gprc port
-EXPOSE 18888
-# filer server http port
-EXPOSE 8888
-# master server shared gprc port
-EXPOSE 19333
-# master server shared http port
-EXPOSE 9333
-# s3 server http port
-EXPOSE 8333
-# webdav server http port
-EXPOSE 7333
-
-RUN mkdir -p /data/filerldb2
-
-VOLUME /data
-WORKDIR /data
-
-COPY filer.toml /etc/seaweedfs/filer.toml
-COPY entrypoint.sh /entrypoint.sh
-RUN chmod +x /entrypoint.sh
-
-ENTRYPOINT ["/entrypoint.sh"]
@@ -46,6 +46,9 @@ dev_registry: build
 dev_replicate: build
 	docker-compose -f compose/local-replicate-compose.yml -p seaweedfs up

+dev_auditlog: build
+	docker-compose -f compose/local-auditlog-compose.yml -p seaweedfs up
+
 cluster: build
 	docker-compose -f compose/local-cluster-compose.yml -p seaweedfs up

@@ -0,0 +1,4 @@
+{
+    "fluent_port": 24224,
+    "fluent_host": "fluent"
+}
@@ -0,0 +1,36 @@
+version: '2'
+
+services:
+  s3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8333:8333
+      - 9333:9333
+      - 19333:19333
+      - 8084:8080
+      - 18084:18080
+      - 8888:8888
+      - 18888:18888
+      - 8000:8000
+    command: "server -ip=s3 -filer -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.auditLogConfig=/etc/seaweedfs/fluent.json -volume.max=0 -master.volumeSizeLimitMB=8 -volume.preStopSeconds=1"
+    volumes:
+      - ./fluent.json:/etc/seaweedfs/fluent.json
+      - ./s3.json:/etc/seaweedfs/s3.json
+    depends_on:
+      - fluent
+  fluent:
+    image: fluent/fluentd:v1.14
+    ports:
+      - 24224:24224
+  #s3tests:
+  #  image: chrislusf/ceph-s3-tests:local
+  #  volumes:
+  #    - ./s3tests.conf:/opt/s3-tests/s3tests.conf
+  #  environment:
+  #    S3TEST_CONF: "s3tests.conf"
+  #    NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
+  #    NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
+  #    NOSETESTS_EXCLUDE: "(get_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
+  #  depends_on:
+  #    - s3
+  #    - fluent
@@ -38,7 +38,7 @@ services:
      S3TEST_CONF: "s3tests.conf"
      NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
      NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
-      NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
+      NOSETESTS_EXCLUDE: "(get_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
    depends_on:
      - master
      - volume
@@ -57,16 +57,6 @@ case "$1" in
  	exec /usr/bin/weed s3 $ARGS $@
 	;;

-  'cronjob')
-	MASTER=${WEED_MASTER-localhost:9333}
-	FIX_REPLICATION_CRON_SCHEDULE=${CRON_SCHEDULE-*/7 * * * * *}
-	echo "$FIX_REPLICATION_CRON_SCHEDULE" 'echo "lock; volume.fix.replication; unlock" | weed shell -master='$MASTER > /crontab
-	BALANCING_CRON_SCHEDULE=${CRON_SCHEDULE-25 * * * * *}
-	echo "$BALANCING_CRON_SCHEDULE" 'echo "lock; volume.balance -collection ALL_COLLECTIONS -force; unlock" | weed shell -master='$MASTER >> /crontab
-	echo "Running Crontab:"
-	cat /crontab
-	exec supercronic /crontab
-	;;
  *)
  	exec /usr/bin/weed $@
 	;;
@@ -28,16 +28,6 @@ services:
    depends_on:
      - master
      - volume
-  cronjob:
-    image: chrislusf/seaweedfs # use a remote image
-    command: 'cronjob'
-    environment:
-      # Run re-replication every 2 minutes
-      CRON_SCHEDULE: '*/2 * * * * *' # Default: '*/5 * * * * *'
-      WEED_MASTER: master:9333 # Default: localhost:9333
-    depends_on:
-      - master
-      - volume
  s3:
    image: chrislusf/seaweedfs # use a remote image
    ports:
@@ -67,7 +67,7 @@ require (
 	github.com/json-iterator/go v1.1.11
 	github.com/karlseguin/ccache/v2 v2.0.7
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.10.9 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
 	github.com/klauspost/cpuid v1.2.1 // indirect
 	github.com/klauspost/reedsolomon v1.9.2
 	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
@@ -100,7 +100,7 @@ require (
 	github.com/prometheus/procfs v0.6.0 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
-	github.com/seaweedfs/fuse v1.2.0
+	github.com/seaweedfs/fuse v1.2.1
 	github.com/seaweedfs/goexif v1.0.2
 	github.com/sirupsen/logrus v1.6.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
@@ -126,7 +126,7 @@ require (
 	github.com/xdg-go/stringprep v1.0.2 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
 	go.etcd.io/etcd/client/v3 v3.5.0
-	go.mongodb.org/mongo-driver v1.7.0
+	go.mongodb.org/mongo-driver v1.8.0
 	go.opencensus.io v0.23.0 // indirect
 	go.opentelemetry.io/otel v0.15.0 // indirect
 	gocloud.dev v0.20.0
@@ -136,9 +136,9 @@ require (
 	golang.org/x/image v0.0.0-20200119044424-58c23975cae1
 	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
 	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
-	golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365
-	golang.org/x/text v0.3.6 // indirect
-	golang.org/x/tools v0.1.5
+	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/api v0.57.0
 	google.golang.org/appengine v1.6.7 // indirect
@@ -165,15 +165,21 @@ require (

 require (
 	cloud.google.com/go/kms v1.0.0 // indirect
+	github.com/apache/thrift v0.15.0 // indirect
 	github.com/d4l3k/messagediff v1.2.1 // indirect
+	github.com/fluent/fluent-logger-golang v1.8.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/gorilla/websocket v1.4.2 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.2 // indirect
 	github.com/mattn/go-runewidth v0.0.7 // indirect
 	github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
+	github.com/miekg/dns v1.1.44 // indirect
+	github.com/philhofer/fwd v1.1.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/tinylib/msgp v1.1.6 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.0 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
@@ -185,5 +191,5 @@ require (
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )

-// replace github.com/seaweedfs/fuse => /Users/chris/go/src/github.com/seaweedfs/fuse
-// replace github.com/chrislusf/raft => /Users/chris/go/src/github.com/chrislusf/raft
+// replace github.com/seaweedfs/fuse => /Users/chrislu/go/src/github.com/seaweedfs/fuse
+// replace github.com/chrislusf/raft => /Users/chrislu/go/src/github.com/chrislusf/raft
@@ -124,6 +124,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
+github.com/apache/thrift v0.15.0 h1:aGvdaR0v1t9XLgjtBYwxcBvBOTMqClzwE26CHOgjW1Y=
+github.com/apache/thrift v0.15.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@@ -234,6 +236,8 @@ github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+ne
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fclairamb/ftpserverlib v0.8.0 h1:ZsWUQ8Vg3Y8LIWRUAzVnFXY982Yztz2odDdK/UVJtik=
 github.com/fclairamb/ftpserverlib v0.8.0/go.mod h1:xF4cy07oCHA9ZorKehsFGqA/1UHYaonmqHK2g3P1X8U=
+github.com/fluent/fluent-logger-golang v1.8.0 h1:K/fUDqUAItNcdf/Rq7aA2d1apwqsNgNzzInlXZTwK28=
+github.com/fluent/fluent-logger-golang v1.8.0/go.mod h1:2/HCT/jTy78yGyeNGQLGQsjF3zzzAuy6Xlk6FCMV5eU=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
@@ -427,6 +431,8 @@ github.com/gorilla/sessions v1.2.0 h1:S7P+1Hm5V/AT9cjEcUD5uDaQSX0OE577aCXgoaKpYb
 github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
 github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg=
@@ -520,6 +526,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.10.9 h1:pPRt1Z78crspaHISkpSSHjDlx+Tt9suHe519dsI0vF4=
 github.com/klauspost/compress v1.10.9/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w=
 github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/reedsolomon v1.9.2 h1:E9CMS2Pqbv+C7tsrYad4YC9MfhnMVWhMRsTi7U0UB18=
@@ -572,6 +580,8 @@ github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/miekg/dns v1.1.44 h1:4rpqcegYPVkvIeVhITrKP1sRR3KjfRc1nrOPMUZmLyc=
+github.com/miekg/dns v1.1.44/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -652,6 +662,8 @@ github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAv
 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
 github.com/peterh/liner v1.1.0 h1:f+aAedNJA6uk7+6rXsYBnhdo4Xux7ESLe+kcuVUF5os=
 github.com/peterh/liner v1.1.0/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0=
+github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ=
+github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
 github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
@@ -718,6 +730,8 @@ github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/seaweedfs/fuse v1.2.0 h1:m4Ar3I0TY/Tzyb80lfJiXUKFcMBHqA5e/cZwxQtilZo=
 github.com/seaweedfs/fuse v1.2.0/go.mod h1:iwbDQv5BZACY54r6AO/6xsLNuMaYcBKSkLTZVfmK594=
+github.com/seaweedfs/fuse v1.2.1 h1:920YH3+PnIqpNLo2lsZDTUJHUPLXnxgQGEXoVb/yMno=
+github.com/seaweedfs/fuse v1.2.1/go.mod h1:iwbDQv5BZACY54r6AO/6xsLNuMaYcBKSkLTZVfmK594=
 github.com/seaweedfs/goexif v1.0.2 h1:p+rTXYdQ2mgxd+1JaTrQ9N8DvYuw9UH9xgYmJ+Bb29E=
 github.com/seaweedfs/goexif v1.0.2/go.mod h1:MrKs5LK0HXdffrdCZrW3OIMegL2xXpC6ThLyXMyjdrk=
 github.com/secsy/goftp v0.0.0-20190720192957-f31499d7c79a h1:C6IhVTxNkhlb0tlCB6JfHOUv1f0xHPK7V8X4HlJZEJw=
@@ -789,6 +803,8 @@ github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8=
 github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw=
+github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 h1:6iRwZdrFUzbcVYZwa8dXTIILGIxmmhjyUPJEcwzPGaU=
@@ -837,6 +853,8 @@ go.etcd.io/etcd/client/v3 v3.5.0 h1:62Eh0XOro+rDwkrypAGDfgmNh5Joq+z+W9HZdlXMzek=
 go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
 go.mongodb.org/mongo-driver v1.7.0 h1:hHrvOBWlWB2c7+8Gh/Xi5jj82AgidK/t7KVXBZ+IyUA=
 go.mongodb.org/mongo-driver v1.7.0/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8=
+go.mongodb.org/mongo-driver v1.8.0 h1:R/P/JJzu8LJvJ1lDfph9GLNIKQxEtIHFfnUUUve35zY=
+go.mongodb.org/mongo-driver v1.8.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
 go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
@@ -983,6 +1001,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c=
 golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1093,6 +1112,8 @@ golang.org/x/sys v0.0.0-20210817142637-7d9622a276b7/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365 h1:6wSTsvPddg9gc/mVEEyk9oOAoxn+bT4Z9q1zx+4RwA4=
 golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1104,6 +1125,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1166,6 +1189,7 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
@@ -1179,6 +1203,8 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 h1:BonxutuHCTL0rBDnZlKjpGIQFTjyUVTexFOdWkB6Fg0=
+golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "2.80"
-version: "2.80"
+appVersion: "2.85"
+version: "2.85"
@@ -113,21 +113,6 @@ Inject extra environment vars in the format key:value, if populated
 {{- end -}}
 {{- end -}}

-{{/* Return the proper cronjob image */}}
-{{- define "cronjob.image" -}}
-{{- if .Values.cronjob.imageOverride -}}
-{{- $imageOverride := .Values.cronjob.imageOverride -}}
-{{- printf "%s" $imageOverride -}}
-{{- else -}}
-{{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}}
-{{- $repositoryName := .Values.image.repository | toString -}}
-{{- $name := .Values.global.imageName | toString -}}
-{{- $tag := .Chart.AppVersion | toString -}}
-{{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}}
-{{- end -}}
-{{- end -}}
-
-
 {{/* check if any PVC exists */}}
 {{- define "volume.pvc_exists" -}}
 {{- if or (or (eq .Values.volume.data.type "persistentVolumeClaim") (and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "persistentVolumeClaim") -}}
@@ -1,58 +0,0 @@
-{{- if .Values.cronjob }}
-{{- if .Values.cronjob.enabled }}
-apiVersion: batch/v1beta1
-kind: CronJob
-metadata:
-  name: {{ include "seaweedfs.fullname" . }}-cronjob
-spec:
-  schedule: "{{ .Values.cronjob.schedule }}"
-  startingDeadlineSeconds: 200
-  concurrencyPolicy: Forbid
-  failedJobsHistoryLimit: 2
-  successfulJobsHistoryLimit: 2
-  jobTemplate:
-    spec:
-      backoffLimit: 2
-      template:
-        spec:
-          {{- if .Values.cronjob.nodeSelector }}
-          nodeSelector:
-            {{ tpl .Values.cronjob.nodeSelector . | indent 12 | trim }}
-          {{- end }}
-          {{- if .Values.cronjob.tolerations }}
-          tolerations:
-            {{ tpl .Values.cronjob.tolerations . | nindent 12 | trim }}
-          {{- end }}
-          restartPolicy: OnFailure
-          containers:
-            - name: shell
-              image: {{ template "cronjob.image" . }}
-              imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }}
-              resources:
-              {{- toYaml .Values.cronjob.resources| nindent 16 }}
-              command:
-                - sh
-                - -c
-                - |
-                  set -ex
-                  echo -e "lock\n\
-                  volume.balance -force \
-                  {{ if .Values.volume.dataCenter }} -dataCenter {{ .Values.volume.dataCenter }}{{ end }}\
-                  {{ if .Values.cronjob.collection }} -collection {{ .Values.cronjob.collection }}{{ end }}\n\
-                  {{- if .Values.cronjob.enableFixReplication }}
-                  volume.fix.replication -collectionPattern={{ .Values.cronjob.collectionPattern }} \n\
-                  {{- end }}
-                  unlock\n" | \
-                  /usr/bin/weed shell \
-                  {{- if .Values.cronjob.master }}
-                  -master {{ .Values.cronjob.master }} \
-                  {{- else }}
-                  -master {{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}.svc:{{ .Values.master.port }} \
-                  {{- end }}
-                  {{- if .Values.cronjob.filer }}
-                  -filer {{ .Values.cronjob.filer }}
-                  {{- else }}
-                  -filer {{ template "seaweedfs.name" . }}-filer.{{ .Release.Namespace }}.svc:{{ .Values.filer.port }}
-                  {{- end }}
-{{- end }}
-{{- end }}
@@ -149,6 +149,9 @@ spec:
              {{- if .Values.filer.s3.enableAuth }}
              -s3.config=/etc/sw/seaweedfs_s3_config \
              {{- end }}
+              {{- if .Values.filer.s3.auditLogConfig }}
+              -s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \
+              {{- end }}
              {{- end }}
              -master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
          {{- if or (.Values.global.enableSecurity) (.Values.filer.extraVolumeMounts) }}    
@@ -93,6 +93,9 @@ spec:
              {{- if .Values.s3.enableAuth }}
              -config=/etc/sw/seaweedfs_s3_config \
              {{- end }}
+              {{- if .Values.s3.auditLogConfig }}
+              -auditLogConfig=/etc/sw/s3_auditLogConfig.json \
+              {{- end }}
              -filer={{ template "seaweedfs.name" . }}-filer-client:{{ .Values.filer.port }}
          volumeMounts:
            - name: logs
@@ -18,4 +18,12 @@ stringData:
  read_access_key_id: {{ $access_key_read }}
  read_secret_access_key: {{ $secret_key_read }}
  seaweedfs_s3_config: '{"identities":[{"name":"anvAdmin","credentials":[{"accessKey":"{{ $access_key_admin }}","secretKey":"{{ $secret_key_admin }}"}],"actions":["Admin","Read","Write"]},{"name":"anvReadOnly","credentials":[{"accessKey":"{{ $access_key_read }}","secretKey":"{{ $secret_key_read }}"}],"actions":["Read"]}]}'
+  {{- if .Values.filer.s3.auditLogConfig }}
+  filer_s3_auditLogConfig.json: |
+    {{ toJson .Values.filer.s3.auditLogConfig | nindent 4 }}
+  {{- end }}
+  {{- if .Values.s3.auditLogConfig }}
+  s3_auditLogConfig.json: |
+    {{ toJson .Values.s3.auditLogConfig | nindent 4 }}
+  {{- end }}
 {{- end }}
@@ -273,9 +273,6 @@ filer:
  # Limit sub dir listing size (default 100000)
  dirListLimit: 100000

-  # Turn off directory listing
-  disableDirListing: false
-
  # Disable http request, only gRpc operations are allowed
  disableHttp: false

@@ -378,6 +375,7 @@ filer:
    # enable user & permission to s3 (need to inject to all services)
    enableAuth: false
    skipAuthSecretCreation: false
+    auditLogConfig: {}

 s3:
  enabled: false
@@ -394,6 +392,7 @@ s3:
  # enable user & permission to s3 (need to inject to all services)
  enableAuth: false
  skipAuthSecretCreation: false
+  auditLogConfig: {}

  # Suffix of the host name, {bucket}.{domainName}
  domainName: ""
@@ -429,22 +428,6 @@ s3:
    size: ""
    storageClass: ""

-cronjob:
-  enabled: true
-  master: "seaweedfs-master:9333"
-  filer: "seaweedfs-filer-client:8888"
-  tolerations: ""
-  nodeSelector: |
-    sw-backend: "true"
-  replication:
-    enable: true
-    collectionPattern: ""
-  schedule: "*/7 * * * *"
-  resources: null
-  # balance all volumes among volume servers
-  # ALL|EACH_COLLECTION|<collection_name>
-  collection: ""
-

 certificates:
  commonName: "SeaweedFS CA"
@@ -5,7 +5,7 @@

    <groupId>com.github.chrislusf</groupId>
    <artifactId>seaweedfs-client</artifactId>
-    <version>1.7.0</version>
+    <version>2.85</version>

    <parent>
        <groupId>org.sonatype.oss</groupId>
@@ -14,7 +14,7 @@
    </parent>

    <properties>
-        <protobuf.version>3.9.1</protobuf.version>
+        <protobuf.version>3.16.1</protobuf.version>
        <!-- follow https://github.com/grpc/grpc-java -->
        <grpc.version>1.23.0</grpc.version>
        <guava.version>30.0-jre</guava.version>
@@ -5,7 +5,7 @@

    <groupId>com.github.chrislusf</groupId>
    <artifactId>seaweedfs-client</artifactId>
-    <version>1.7.0</version>
+    <version>2.85</version>

    <parent>
        <groupId>org.sonatype.oss</groupId>
@@ -5,7 +5,7 @@

    <groupId>com.github.chrislusf</groupId>
    <artifactId>seaweedfs-client</artifactId>
-    <version>1.7.0</version>
+    <version>2.85</version>

    <parent>
        <groupId>org.sonatype.oss</groupId>
@@ -368,6 +368,7 @@ public class FilerClient extends FilerGrpcClient {
                .setPathPrefix(prefix)
                .setClientName(clientName)
                .setSinceNs(sinceNs)
+                .setClientId(this.randomClientId)
                .build()
        );
    }
@@ -11,6 +11,7 @@ import org.slf4j.LoggerFactory;
 import javax.net.ssl.SSLException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Random;
 import java.util.concurrent.TimeUnit;

 public class FilerGrpcClient {
@@ -30,6 +31,7 @@ public class FilerGrpcClient {
    public final int VOLUME_SERVER_ACCESS_PUBLIC_URL = 1;
    public final int VOLUME_SERVER_ACCESS_FILER_PROXY = 2;
    public final Map<String, FilerProto.Locations> vidLocations = new HashMap<>();
+    protected int randomClientId;
    private final ManagedChannel channel;
    private final SeaweedFilerGrpc.SeaweedFilerBlockingStub blockingStub;
    private final SeaweedFilerGrpc.SeaweedFilerStub asyncStub;
@@ -62,6 +64,7 @@ public class FilerGrpcClient {
        cipher = filerConfigurationResponse.getCipher();
        collection = filerConfigurationResponse.getCollection();
        replication = filerConfigurationResponse.getReplication();
+        randomClientId = new Random().nextInt();

    }

@@ -29,11 +29,31 @@ public class SeaweedWrite {
                                 final byte[] bytes,
                                 final long bytesOffset, final long bytesLength,
                                 final String path) throws IOException {
-        FilerProto.FileChunk.Builder chunkBuilder = writeChunk(
-                replication, filerClient, offset, bytes, bytesOffset, bytesLength, path);
-        synchronized (entry) {
-            entry.addChunks(chunkBuilder);
+
+        IOException lastException = null;
+        for (long waitTime = 1000L; waitTime < 10 * 1000; waitTime += waitTime / 2) {
+            try {
+                FilerProto.FileChunk.Builder chunkBuilder = writeChunk(
+                        replication, filerClient, offset, bytes, bytesOffset, bytesLength, path);
+                lastException = null;
+                synchronized (entry) {
+                    entry.addChunks(chunkBuilder);
+                }
+                break;
+            } catch (IOException ioe) {
+                LOG.debug("writeData:{}", ioe);
+                lastException = ioe;
+            }
+            try {
+                Thread.sleep(waitTime);
+            } catch (InterruptedException e) {
+            }
        }
+
+        if (lastException != null) {
+            throw lastException;
+        }
+
    }

    public static FilerProto.FileChunk.Builder writeChunk(final String replication,
@@ -247,6 +247,7 @@ message AssignVolumeRequest {
    string data_center = 5;
    string path = 6;
    string rack = 7;
+    string data_node = 9;
    string disk_type = 8;
 }

@@ -328,6 +329,7 @@ message SubscribeMetadataRequest {
    int64 since_ns = 3;
    int32 signature = 4;
    repeated string path_prefixes = 6;
+    int32 client_id = 7;
 }
 message SubscribeMetadataResponse {
    string directory = 1;
@@ -396,6 +398,9 @@ message FilerConf {
        bool fsync = 6;
        uint32 volume_growth_count = 7;
        bool read_only = 8;
+        string data_center = 9;
+        string rack = 10;
+        string data_node = 11;
    }
    repeated PathConf locations = 2;
 }
@@ -11,13 +11,13 @@
        <dependency>
            <groupId>com.github.chrislusf</groupId>
            <artifactId>seaweedfs-client</artifactId>
-            <version>1.7.0</version>
+            <version>2.85</version>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>com.github.chrislusf</groupId>
            <artifactId>seaweedfs-hadoop2-client</artifactId>
-            <version>1.7.0</version>
+            <version>2.85</version>
            <scope>compile</scope>
        </dependency>
        <dependency>
@@ -29,4 +29,4 @@
    </dependencies>


-</project>
+</project>
@@ -301,7 +301,7 @@
    </snapshotRepository>
  </distributionManagement>
  <properties>
-    <seaweedfs.client.version>1.7.0</seaweedfs.client.version>
+    <seaweedfs.client.version>2.85</seaweedfs.client.version>
    <hadoop.version>2.9.2</hadoop.version>
  </properties>
 </project>
@@ -5,7 +5,7 @@
    <modelVersion>4.0.0</modelVersion>

    <properties>
-        <seaweedfs.client.version>1.7.0</seaweedfs.client.version>
+        <seaweedfs.client.version>2.85</seaweedfs.client.version>
        <hadoop.version>2.9.2</hadoop.version>
    </properties>

@@ -309,7 +309,7 @@
    </snapshotRepository>
  </distributionManagement>
  <properties>
-    <seaweedfs.client.version>1.7.0</seaweedfs.client.version>
+    <seaweedfs.client.version>2.85</seaweedfs.client.version>
    <hadoop.version>3.1.1</hadoop.version>
  </properties>
 </project>
@@ -5,7 +5,7 @@
    <modelVersion>4.0.0</modelVersion>

    <properties>
-        <seaweedfs.client.version>1.7.0</seaweedfs.client.version>
+        <seaweedfs.client.version>2.85</seaweedfs.client.version>
        <hadoop.version>3.1.1</hadoop.version>
    </properties>

@@ -0,0 +1,2 @@
+/s3-tests
+/tmp
@@ -0,0 +1,11 @@
+# the tests only support python 3.6, not newer
+FROM ubuntu:latest
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y git-core sudo tzdata
+RUN git clone https://github.com/ceph/s3-tests.git
+WORKDIR s3-tests
+
+# we pin a certain commit
+RUN git checkout 9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e
+
+RUN ./bootstrap
@@ -0,0 +1,13 @@
+# Running S3 Compatibility tests against SeaweedFS
+
+This is using [the tests from CephFS](https://github.com/ceph/s3-tests).
+
+## Prerequisites
+
+- have Docker installed
+- this has been executed on Mac. On Linux, the hostname in `s3tests.conf`  needs to be adjusted.
+
+## Running tests
+
+- `./prepare.sh` to build the docker image
+- `./run.sh` to execute all tests
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -ex
+
+docker build  --progress=plain  -t s3tests .
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -ex
+
+killall -9 weed || echo "already stopped"
+rm -Rf tmp
+mkdir tmp
+docker stop s3test-instance || echo "already stopped"
+
+ulimit -n 10000
+../../../weed/weed server -filer -s3 -volume.max 0   -master.volumeSizeLimitMB 5 -dir "$(pwd)/tmp" 1>&2>weed.log &
+
+until $(curl --output /dev/null --silent --head --fail http://127.0.0.1:9333); do
+    printf '.'
+    sleep 5
+done
+sleep 3
+
+rm -Rf logs-full.txt logs-summary.txt
+# docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v `pwd`/s3tests.conf:/s3-tests/s3tests.conf -it s3tests    ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py:test_get_obj_tagging -v  -a 'resource=object,!bucket-policy,!versioning,!encryption'
+docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v `pwd`/s3tests.conf:/s3-tests/s3tests.conf -it s3tests    ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py -v  -a 'resource=object,!bucket-policy,!versioning,!encryption' | sed -n -e '/botocore.hooks/!p;//q' | tee logs-summary.txt
+
+docker stop s3test-instance || echo "already stopped"
+killall -9 weed
@@ -0,0 +1,109 @@
+[DEFAULT]
+## this section is just used for host, port and bucket_prefix
+
+# host set for rgw in vstart.sh
+host = host.docker.internal
+
+# port set for rgw in vstart.sh
+port = 8333
+
+## say "False" to disable TLS
+is_secure = False
+
+## say "False" to disable SSL Verify
+ssl_verify = False
+
+[fixtures]
+## all the buckets created will start with this prefix;
+## {random} will be filled with random characters to pad
+## the prefix to 30 characters long, and avoid collisions
+bucket prefix = yournamehere-{random}-
+
+[s3 main]
+# main display_name set in vstart.sh
+display_name = M. Tester
+
+# main user_idname set in vstart.sh
+user_id = testid
+
+# main email set in vstart.sh
+email = tester@ceph.com
+
+# zonegroup api_name for bucket location
+api_name = default
+
+## main AWS access key
+access_key = 0555b35654ad1656d804
+
+## main AWS secret key
+secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==
+
+## replace with key id obtained when secret is created, or delete if KMS not tested
+#kms_keyid = 01234567-89ab-cdef-0123-456789abcdef
+
+[s3 alt]
+# alt display_name set in vstart.sh
+display_name = john.doe
+## alt email set in vstart.sh
+email = john.doe@example.com
+
+# alt user_id set in vstart.sh
+user_id = 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234
+
+# alt AWS access key set in vstart.sh
+access_key = NOPQRSTUVWXYZABCDEFG
+
+# alt AWS secret key set in vstart.sh
+secret_key = nopqrstuvwxyzabcdefghijklmnabcdefghijklm
+
+[s3 tenant]
+# tenant display_name set in vstart.sh
+display_name = testx$tenanteduser
+
+# tenant user_id set in vstart.sh
+user_id = 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+
+# tenant AWS secret key set in vstart.sh
+access_key = HIJKLMNOPQRSTUVWXYZA
+
+# tenant AWS secret key set in vstart.sh
+secret_key = opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab
+
+# tenant email set in vstart.sh
+email = tenanteduser@example.com
+
+#following section needs to be added for all sts-tests
+[iam]
+#used for iam operations in sts-tests
+#email from vstart.sh
+email = s3@example.com
+
+#user_id from vstart.sh
+user_id = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+
+#access_key from vstart.sh
+access_key = ABCDEFGHIJKLMNOPQRST
+
+#secret_key vstart.sh
+secret_key = abcdefghijklmnopqrstuvwxyzabcdefghijklmn
+
+#display_name from vstart.sh
+display_name = youruseridhere
+
+#following section needs to be added when you want to run Assume Role With Webidentity test
+[webidentity]
+#used for assume role with web identity test in sts-tests
+#all parameters will be obtained from ceph/qa/tasks/keycloak.py
+token=<access_token>
+
+aud=<obtained after introspecting token>
+
+sub=<obtained after introspecting token>
+
+azp=<obtained after introspecting token>
+
+user_token=<access token for a user, with attribute Department=[Engineering, Marketing>]
+
+thumbprint=<obtained from x509 certificate>
+
+KC_REALM=<name of the realm>
@@ -122,7 +122,7 @@ type needleState struct {

 func getVolumeFiles(v uint32, addr pb.ServerAddress) (map[types.NeedleId]needleState, int64, error) {
 	var idxFile *bytes.Reader
-	err := operation.WithVolumeServerClient(addr, grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error {
+	err := operation.WithVolumeServerClient(false, addr, grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error {
 		ctx, cancel := context.WithCancel(context.Background())
 		defer cancel()
 		copyFileClient, err := vs.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
@@ -180,7 +180,7 @@ func getVolumeFiles(v uint32, addr pb.ServerAddress) (map[types.NeedleId]needleS

 func getNeedleFileId(v uint32, nid types.NeedleId, addr pb.ServerAddress) (string, error) {
 	var id string
-	err := operation.WithVolumeServerClient(addr, grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error {
+	err := operation.WithVolumeServerClient(false, addr, grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error {
 		resp, err := vs.VolumeNeedleStatus(context.Background(), &volume_server_pb.VolumeNeedleStatusRequest{
 			VolumeId: v,
 			NeedleId: uint64(nid),
@@ -51,7 +51,7 @@ func main() {
 }

 func startGenerateMetadata() {
-	pb.WithFilerClient(pb.ServerAddress(*tailFiler), grpc.WithInsecure(), func(client filer_pb.SeaweedFilerClient) error {
+	pb.WithFilerClient(false, pb.ServerAddress(*tailFiler), grpc.WithInsecure(), func(client filer_pb.SeaweedFilerClient) error {

 		for i := 0; i < *n; i++ {
 			name := fmt.Sprintf("file%d", i)
@@ -77,7 +77,7 @@ func startGenerateMetadata() {

 func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) {

-	tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithInsecure(), "tail", *dir, nil, 0, 0, eachEntryFunc, false)
+	tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithInsecure(), "tail", 0, *dir, nil, 0, 0, eachEntryFunc, false)

 	if tailErr != nil {
 		fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)
@@ -15,9 +15,9 @@ import (
 )

 var (
-	volumeServer       = flag.String("volumeServer", "localhost:8080", "a volume server")
-	volumeId         = flag.Int("volumeId", -1, "a volume id to stream read")
-	grpcDialOption   grpc.DialOption
+	volumeServer   = flag.String("volumeServer", "localhost:8080", "a volume server")
+	volumeId       = flag.Int("volumeId", -1, "a volume id to stream read")
+	grpcDialOption grpc.DialOption
 )

 func main() {
@@ -33,11 +33,11 @@ func main() {
 		return nil
 	}

-	err := operation.WithVolumeServerClient(pb.ServerAddress(*volumeServer), grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error {
+	err := operation.WithVolumeServerClient(true, pb.ServerAddress(*volumeServer), grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error {
 		ctx, cancel := context.WithCancel(context.Background())
 		defer cancel()
 		copyFileClient, err := vs.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{
-			VolumeIds:           []uint32{vid},
+			VolumeIds: []uint32{vid},
 		})
 		if err != nil {
 			return err
@@ -61,4 +61,3 @@ func main() {
 	}

 }
-
@@ -84,6 +84,7 @@ func init() {
 	filerS3Options.tlsPrivateKey = cmdFiler.Flag.String("s3.key.file", "", "path to the TLS private key file")
 	filerS3Options.tlsCertificate = cmdFiler.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
 	filerS3Options.config = cmdFiler.Flag.String("s3.config", "", "path to the config file")
+	filerS3Options.auditLogConfig = cmdFiler.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
 	filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")

 	// start webdav on filer
@@ -137,6 +138,7 @@ func runFiler(cmd *Command, args []string) bool {
 	startDelay := time.Duration(2)
 	if *filerStartS3 {
 		filerS3Options.filer = &filerAddress
+		filerS3Options.bindIp = f.bindIp
 		go func() {
 			time.Sleep(startDelay * time.Second)
 			filerS3Options.startS3Server()
@@ -54,8 +54,10 @@ func runFilerBackup(cmd *Command, args []string) bool {

 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")

+	clientId := util.RandomInt32()
+
 	for {
-		err := doFilerBackup(grpcDialOption, &filerBackupOptions)
+		err := doFilerBackup(grpcDialOption, &filerBackupOptions, clientId)
 		if err != nil {
 			glog.Errorf("backup from %s: %v", *filerBackupOptions.filer, err)
 			time.Sleep(1747 * time.Millisecond)
@@ -69,7 +71,7 @@ const (
 	BackupKeyPrefix = "backup."
 )

-func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions) error {
+func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions, clientId int32) error {

 	// find data sink
 	config := util.GetViper()
@@ -112,7 +114,7 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
 		return setOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId), lastTsNs)
 	})

-	return pb.FollowMetadata(sourceFiler, grpcDialOption, "backup_"+dataSink.GetName(),
+	return pb.FollowMetadata(sourceFiler, grpcDialOption, "backup_"+dataSink.GetName(), clientId,
 		sourcePath, nil, startFrom.UnixNano(), 0, processEventFnWithOffset, false)

 }
@@ -97,7 +97,7 @@ func runFilerCat(cmd *Command, args []string) bool {
 		writer = f
 	}

-	pb.WithFilerClient(filerCat.filerAddress, filerCat.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	pb.WithFilerClient(false, filerCat.filerAddress, filerCat.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {

 		request := &filer_pb.LookupDirectoryEntryRequest{
 			Name:      name,
@@ -172,7 +172,7 @@ func runCopy(cmd *Command, args []string) bool {
 }

 func readFilerConfiguration(grpcDialOption grpc.DialOption, filerGrpcAddress pb.ServerAddress) (masters []string, collection, replication string, dirBuckets string, maxMB uint32, cipher bool, err error) {
-	err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	err = pb.WithGrpcFilerClient(false, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 		if err != nil {
 			return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
@@ -302,7 +302,7 @@ func (worker *FileCopyWorker) checkExistingFileFirst(task FileCopyTask, f *os.Fi
 		return
 	}

-	err = pb.WithGrpcFilerClient(worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	err = pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {

 		request := &filer_pb.LookupDirectoryEntryRequest{
 			Directory: task.destinationUrlPath,
@@ -344,7 +344,7 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err

 		err = util.Retry("upload", func() error {
 			// assign a volume
-			assignErr := pb.WithGrpcFilerClient(worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+			assignErr := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {

 				request := &filer_pb.AssignVolumeRequest{
 					Count:       1,
@@ -404,7 +404,7 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err

 	}

-	if err := pb.WithGrpcFilerClient(worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if err := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
 			Directory: task.destinationUrlPath,
 			Entry: &filer_pb.Entry{
@@ -461,7 +461,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 			var assignResult *filer_pb.AssignVolumeResponse
 			var assignError error
 			err := util.Retry("assignVolume", func() error {
-				return pb.WithGrpcFilerClient(worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+				return pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 					request := &filer_pb.AssignVolumeRequest{
 						Count:       1,
 						Replication: *worker.options.replication,
@@ -540,7 +540,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 		return fmt.Errorf("create manifest: %v", manifestErr)
 	}

-	if err := pb.WithGrpcFilerClient(worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if err := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
 			Directory: task.destinationUrlPath,
 			Entry: &filer_pb.Entry{
@@ -598,7 +598,7 @@ func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, off
 	var fileId, host string
 	var auth security.EncodedJwt

-	if flushErr := pb.WithGrpcFilerClient(worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if flushErr := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {

 		ctx := context.Background()

@@ -27,7 +27,8 @@ type FilerMetaBackupOptions struct {
 	restart           *bool
 	backupFilerConfig *string

-	store filer.FilerStore
+	store    filer.FilerStore
+	clientId int32
 }

 func init() {
@@ -36,6 +37,7 @@ func init() {
 	metaBackup.filerDirectory = cmdFilerMetaBackup.Flag.String("filerDir", "/", "a folder on the filer")
 	metaBackup.restart = cmdFilerMetaBackup.Flag.Bool("restart", false, "copy the full metadata before async incremental backup")
 	metaBackup.backupFilerConfig = cmdFilerMetaBackup.Flag.String("config", "", "path to filer.toml specifying backup filer store")
+	metaBackup.clientId = util.RandomInt32()
 }

 var cmdFilerMetaBackup = &Command{
@@ -195,7 +197,7 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
 		return metaBackup.setOffset(lastTime)
 	})

-	return pb.FollowMetadata(pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, "meta_backup",
+	return pb.FollowMetadata(pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, "meta_backup", metaBackup.clientId,
 		*metaBackup.filerDirectory, nil, startTime.UnixNano(), 0, processEventFnWithOffset, false)

 }
@@ -222,9 +224,9 @@ func (metaBackup *FilerMetaBackupOptions) setOffset(lastWriteTime time.Time) err

 var _ = filer_pb.FilerClient(&FilerMetaBackupOptions{})

-func (metaBackup *FilerMetaBackupOptions) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
+func (metaBackup *FilerMetaBackupOptions) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {

-	return pb.WithFilerClient(pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	return pb.WithFilerClient(streamingMode, pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		return fn(client)
 	})

@@ -6,7 +6,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/golang/protobuf/jsonpb"
 	jsoniter "github.com/json-iterator/go"
-	"github.com/olivere/elastic/v7"
+	elastic "github.com/olivere/elastic/v7"
 	"os"
 	"path/filepath"
 	"strings"
@@ -48,6 +48,7 @@ func runFilerMetaTail(cmd *Command, args []string) bool {

 	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+	clientId := util.RandomInt32()

 	var filterFunc func(dir, fname string) bool
 	if *tailPattern != "" {
@@ -105,7 +106,7 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
 		}
 	}

-	tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpcDialOption, "tail",
+	tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpcDialOption, "tail", clientId,
 		*tailTarget, nil, time.Now().Add(-*tailStart).UnixNano(), 0,
 		func(resp *filer_pb.SubscribeMetadataResponse) error {
 			if !shouldPrint(resp) {
@@ -28,12 +28,13 @@ type RemoteGatewayOptions struct {
 	mappings    *remote_pb.RemoteStorageMapping
 	remoteConfs map[string]*remote_pb.RemoteConf
 	bucketsDir  string
+	clientId    int32
 }

 var _ = filer_pb.FilerClient(&RemoteGatewayOptions{})

-func (option *RemoteGatewayOptions) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
-	return pb.WithFilerClient(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+func (option *RemoteGatewayOptions) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
+	return pb.WithFilerClient(streamingMode, pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		return fn(client)
 	})
 }
@@ -54,6 +55,7 @@ func init() {
 	remoteGatewayOptions.timeAgo = cmdFilerRemoteGateway.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
 	remoteGatewayOptions.include = cmdFilerRemoteGateway.Flag.String("include", "", "pattens of new bucket names, e.g., s3*")
 	remoteGatewayOptions.exclude = cmdFilerRemoteGateway.Flag.String("exclude", "", "pattens of new bucket names, e.g., local*")
+	remoteGatewayOptions.clientId = util.RandomInt32()
 }

 var cmdFilerRemoteGateway = &Command{
@@ -87,7 +89,7 @@ func runFilerRemoteGateway(cmd *Command, args []string) bool {

 	remoteGatewayOptions.bucketsDir = "/buckets"
 	// check buckets again
-	remoteGatewayOptions.WithFilerClient(func(filerClient filer_pb.SeaweedFilerClient) error {
+	remoteGatewayOptions.WithFilerClient(false, func(filerClient filer_pb.SeaweedFilerClient) error {
 		resp, err := filerClient.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 		if err != nil {
 			return err
@@ -38,7 +38,7 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo

 	lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, *option.timeAgo)

-	return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync",
+	return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync", option.clientId,
 		option.bucketsDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
 }

@@ -86,12 +86,26 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 				return nil
 			}
 		}
-		if *option.createBucketRandomSuffix {
-			// https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
-			if len(bucketName)+5 > 63 {
-				bucketName = bucketName[:58]
+
+		bucketPath := util.FullPath(option.bucketsDir).Child(entry.Name)
+		remoteLocation, found := option.mappings.Mappings[string(bucketPath)]
+		if !found {
+			if *option.createBucketRandomSuffix {
+				// https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
+				if len(bucketName)+5 > 63 {
+					bucketName = bucketName[:58]
+				}
+				bucketName = fmt.Sprintf("%s-%04d", bucketName, rand.Uint32()%10000)
 			}
-			bucketName = fmt.Sprintf("%s-%4d", bucketName, rand.Uint32()%10000)
+			remoteLocation = &remote_pb.RemoteStorageLocation{
+				Name:   *option.createBucketAt,
+				Bucket: bucketName,
+				Path:   "/",
+			}
+			// need to add new mapping here before getting updates from metadata tailing
+			option.mappings.Mappings[string(bucketPath)] = remoteLocation
+		} else {
+			bucketName = remoteLocation.Bucket
 		}

 		glog.V(0).Infof("create bucket %s", bucketName)
@@ -99,16 +113,6 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 			return fmt.Errorf("create bucket %s in %s: %v", bucketName, remoteConf.Name, err)
 		}

-		bucketPath := util.FullPath(option.bucketsDir).Child(entry.Name)
-		remoteLocation := &remote_pb.RemoteStorageLocation{
-			Name:   *option.createBucketAt,
-			Bucket: bucketName,
-			Path:   "/",
-		}
-
-		// need to add new mapping here before getting upates from metadata tailing
-		option.mappings.Mappings[string(bucketPath)] = remoteLocation
-
 		return filer.InsertMountMapping(option, string(bucketPath), remoteLocation)

 	}
@@ -177,6 +181,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 			if message.NewParentPath == option.bucketsDir {
 				return handleCreateBucket(message.NewEntry)
 			}
+			if strings.HasPrefix(message.NewParentPath, option.bucketsDir) && strings.Contains(message.NewParentPath, "/.uploads/") {
+				return nil
+			}
 			if !filer.HasData(message.NewEntry) {
 				return nil
 			}
@@ -18,12 +18,13 @@ type RemoteSyncOptions struct {
 	readChunkFromFiler *bool
 	timeAgo            *time.Duration
 	dir                *string
+	clientId           int32
 }

 var _ = filer_pb.FilerClient(&RemoteSyncOptions{})

-func (option *RemoteSyncOptions) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
-	return pb.WithFilerClient(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+func (option *RemoteSyncOptions) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
+	return pb.WithFilerClient(streamingMode, pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		return fn(client)
 	})
 }
@@ -40,7 +41,8 @@ func init() {
 	remoteSyncOptions.filerAddress = cmdFilerRemoteSynchronize.Flag.String("filer", "localhost:8888", "filer of the SeaweedFS cluster")
 	remoteSyncOptions.dir = cmdFilerRemoteSynchronize.Flag.String("dir", "", "a mounted directory on filer")
 	remoteSyncOptions.readChunkFromFiler = cmdFilerRemoteSynchronize.Flag.Bool("filerProxy", false, "read file chunks from filer instead of volume servers")
-	remoteSyncOptions.timeAgo = cmdFilerRemoteSynchronize.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
+	remoteSyncOptions.timeAgo = cmdFilerRemoteSynchronize.Flag.Duration("timeAgo", 0, "start time before now, skipping previous metadata changes. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
+	remoteSyncOptions.clientId = util.RandomInt32()
 }

 var cmdFilerRemoteSynchronize = &Command{
@@ -54,6 +56,11 @@ var cmdFilerRemoteSynchronize = &Command{

 		weed filer.remote.sync -dir=/mount/s3_on_cloud

+	The metadata sync starting time is determined with the following priority order:
+	1. specified by timeAgo
+	2. last sync timestamp for this directory
+	3. directory creation time
+
 `,
 }

@@ -40,7 +40,7 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour

 	lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, *option.timeAgo)

-	return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync",
+	return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync", option.clientId,
 		mountedDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
 }

@@ -227,7 +227,7 @@ func shouldSendToRemote(entry *filer_pb.Entry) bool {
 func updateLocalEntry(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.Entry, remoteEntry *filer_pb.RemoteEntry) error {
 	remoteEntry.LastLocalSyncTsNs = time.Now().UnixNano()
 	entry.RemoteEntry = remoteEntry
-	return filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	return filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		_, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
 			Directory: dir,
 			Entry:     entry,
@@ -37,6 +37,7 @@ type SyncOptions struct {
 	bDebug          *bool
 	aProxyByFiler   *bool
 	bProxyByFiler   *bool
+	clientId        int32
 }

 var (
@@ -66,6 +67,7 @@ func init() {
 	syncOptions.bDebug = cmdFilerSynchronize.Flag.Bool("b.debug", false, "debug mode to print out filer B received files")
 	syncCpuProfile = cmdFilerSynchronize.Flag.String("cpuprofile", "", "cpu profile output file")
 	syncMemProfile = cmdFilerSynchronize.Flag.String("memprofile", "", "memory profile output file")
+	syncOptions.clientId = util.RandomInt32()
 }

 var cmdFilerSynchronize = &Command{
@@ -97,7 +99,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 	filerB := pb.ServerAddress(*syncOptions.filerB)
 	go func() {
 		for {
-			err := doSubscribeFilerMetaChanges(grpcDialOption, filerA, *syncOptions.aPath, *syncOptions.aProxyByFiler, filerB,
+			err := doSubscribeFilerMetaChanges(syncOptions.clientId, grpcDialOption, filerA, *syncOptions.aPath, *syncOptions.aProxyByFiler, filerB,
 				*syncOptions.bPath, *syncOptions.bReplication, *syncOptions.bCollection, *syncOptions.bTtlSec, *syncOptions.bProxyByFiler, *syncOptions.bDiskType, *syncOptions.bDebug)
 			if err != nil {
 				glog.Errorf("sync from %s to %s: %v", *syncOptions.filerA, *syncOptions.filerB, err)
@@ -109,7 +111,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 	if !*syncOptions.isActivePassive {
 		go func() {
 			for {
-				err := doSubscribeFilerMetaChanges(grpcDialOption, filerB, *syncOptions.bPath, *syncOptions.bProxyByFiler, filerA,
+				err := doSubscribeFilerMetaChanges(syncOptions.clientId, grpcDialOption, filerB, *syncOptions.bPath, *syncOptions.bProxyByFiler, filerA,
 					*syncOptions.aPath, *syncOptions.aReplication, *syncOptions.aCollection, *syncOptions.aTtlSec, *syncOptions.aProxyByFiler, *syncOptions.aDiskType, *syncOptions.aDebug)
 				if err != nil {
 					glog.Errorf("sync from %s to %s: %v", *syncOptions.filerB, *syncOptions.filerA, err)
@@ -124,7 +126,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 	return true
 }

-func doSubscribeFilerMetaChanges(grpcDialOption grpc.DialOption, sourceFiler pb.ServerAddress, sourcePath string, sourceReadChunkFromFiler bool, targetFiler pb.ServerAddress, targetPath string,
+func doSubscribeFilerMetaChanges(clientId int32, grpcDialOption grpc.DialOption, sourceFiler pb.ServerAddress, sourcePath string, sourceReadChunkFromFiler bool, targetFiler pb.ServerAddress, targetPath string,
 	replicationStr, collection string, ttlSec int, sinkWriteChunkByFiler bool, diskType string, debug bool) error {

 	// read source filer signature
@@ -172,7 +174,7 @@ func doSubscribeFilerMetaChanges(grpcDialOption grpc.DialOption, sourceFiler pb.
 		return setOffset(grpcDialOption, targetFiler, SyncKeyPrefix, sourceFilerSignature, lastTsNs)
 	})

-	return pb.FollowMetadata(sourceFiler, grpcDialOption, "syncTo_"+string(targetFiler),
+	return pb.FollowMetadata(sourceFiler, grpcDialOption, "syncTo_"+string(targetFiler), clientId,
 		sourcePath, nil, sourceFilerOffsetTsNs, targetFilerSignature, processEventFnWithOffset, false)

 }
@@ -183,7 +185,7 @@ const (

 func getOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32) (lastOffsetTsNs int64, readErr error) {

-	readErr = pb.WithFilerClient(filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	readErr = pb.WithFilerClient(false, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		syncKey := []byte(signaturePrefix + "____")
 		util.Uint32toBytes(syncKey[len(signaturePrefix):len(signaturePrefix)+4], uint32(signature))

@@ -209,7 +211,7 @@ func getOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signature
 }

 func setOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32, offsetTsNs int64) error {
-	return pb.WithFilerClient(filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	return pb.WithFilerClient(false, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {

 		syncKey := []byte(signaturePrefix + "____")
 		util.Uint32toBytes(syncKey[len(signaturePrefix):len(signaturePrefix)+4], uint32(signature))
@@ -1,9 +1,12 @@
 package command

 import (
+	"fmt"
+	"io/ioutil"
 	"os"
 	"path"
 	"strconv"
+	"strings"

 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
@@ -29,7 +32,7 @@ var cmdFix = &Command{
 var (
 	fixVolumePath       = cmdFix.Flag.String("dir", ".", "data directory to store files")
 	fixVolumeCollection = cmdFix.Flag.String("collection", "", "the volume collection name")
-	fixVolumeId         = cmdFix.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
+	fixVolumeId         = cmdFix.Flag.Int("volumeId", 0, "an optional volume id.")
 )

 type VolumeFileScanner4Fix struct {
@@ -60,15 +63,52 @@ func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *needle.Needle, offset int64

 func runFix(cmd *Command, args []string) bool {

-	if *fixVolumeId == -1 {
+	dir := util.ResolvePath(*fixVolumePath)
+	if *fixVolumeId != 0 {
+		doFixOneVolume(dir, *fixVolumeCollection, needle.VolumeId(*fixVolumeId))
+		return true
+	}
+
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		fmt.Println(err)
 		return false
 	}

-	baseFileName := strconv.Itoa(*fixVolumeId)
-	if *fixVolumeCollection != "" {
-		baseFileName = *fixVolumeCollection + "_" + baseFileName
+	for _, file := range files {
+		if !strings.HasSuffix(file.Name(), ".dat") {
+			continue
+		}
+		if *fixVolumeCollection != "" {
+			if !strings.HasPrefix(file.Name(), *fixVolumeCollection+"_") {
+				continue
+			}
+		}
+		baseFileName := file.Name()[:len(file.Name())-4]
+		collection, volumeIdStr := "", baseFileName
+		if sepIndex := strings.LastIndex(baseFileName, "_"); sepIndex > 0 {
+			collection = baseFileName[:sepIndex]
+			volumeIdStr = baseFileName[sepIndex+1:]
+		}
+		volumeId, parseErr := strconv.ParseInt(volumeIdStr, 10, 64)
+		if parseErr != nil {
+			fmt.Printf("Failed to parse volume id from %s: %v\n", baseFileName, parseErr)
+			return false
+		}
+		doFixOneVolume(dir, collection, needle.VolumeId(volumeId))
 	}
-	indexFileName := path.Join(util.ResolvePath(*fixVolumePath), baseFileName+".idx")
+
+	return true
+}
+
+func doFixOneVolume(dir, collection string, volumeId needle.VolumeId) {
+
+	baseFileName := strconv.Itoa(int(volumeId))
+	if collection != "" {
+		baseFileName = collection + "_" + baseFileName
+	}
+
+	indexFileName := path.Join(dir, baseFileName+".idx")

 	nm := needle_map.NewMemDb()
 	defer nm.Close()
@@ -78,7 +118,7 @@ func runFix(cmd *Command, args []string) bool {
 		nm: nm,
 	}

-	if err := storage.ScanVolumeFile(util.ResolvePath(*fixVolumePath), *fixVolumeCollection, vid, storage.NeedleMapInMemory, scanner); err != nil {
+	if err := storage.ScanVolumeFile(dir, collection, vid, storage.NeedleMapInMemory, scanner); err != nil {
 		glog.Fatalf("scan .dat File: %v", err)
 		os.Remove(indexFileName)
 	}
@@ -87,6 +127,4 @@ func runFix(cmd *Command, args []string) bool {
 		glog.Fatalf("save to .idx File: %v", err)
 		os.Remove(indexFileName)
 	}
-
-	return true
 }
@@ -48,7 +48,7 @@ func (iamopt *IamOptions) startIamServer() bool {
 	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 	for {
-		err := pb.WithGrpcFilerClient(filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 			if err != nil {
 				return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
@@ -87,7 +87,7 @@ func startMasterFollower(masterOptions MasterOptions) {
 	var err error
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.master")
 	for i := 0; i < 10; i++ {
-		err = pb.WithOneOfGrpcMasterClients(masters, grpcDialOption, func(client master_pb.SeaweedClient) error {
+		err = pb.WithOneOfGrpcMasterClients(false, masters, grpcDialOption, func(client master_pb.SeaweedClient) error {
 			resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
 			if err != nil {
 				return fmt.Errorf("get master grpc address %v configuration: %v", masters, err)
@@ -78,7 +78,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 	var cipher bool
 	var err error
 	for i := 0; i < 10; i++ {
-		err = pb.WithOneOfGrpcFilerClients(filerAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err = pb.WithOneOfGrpcFilerClients(false, filerAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 			if err != nil {
 				return fmt.Errorf("get filer grpc address %v configuration: %v", filerAddresses, err)
@@ -173,11 +173,11 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		fuse.DaemonTimeout("3600"),
 		fuse.AllowSUID(),
 		fuse.DefaultPermissions(),
-		fuse.MaxReadahead(1024 * 128),
+		fuse.MaxReadahead(1024 * 512),
 		fuse.AsyncRead(),
-		fuse.WritebackCache(),
-		fuse.MaxBackground(128),
-		fuse.CongestionThreshold(128),
+		// fuse.WritebackCache(),
+		// fuse.MaxBackground(1024),
+		// fuse.CongestionThreshold(1024),
 	}

 	options = append(options, osSpecificMountOptions()...)
@@ -68,7 +68,7 @@ func (msgBrokerOpt *MessageBrokerOptions) startQueueServer() bool {
 	cipher := false

 	for {
-		err := pb.WithGrpcFilerClient(filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 			if err != nil {
 				return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
@@ -3,6 +3,7 @@ package command
 import (
 	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 	"net/http"
 	"time"

@@ -24,6 +25,7 @@ var (

 type S3Options struct {
 	filer            *string
+	bindIp           *string
 	port             *int
 	config           *string
 	domainName       *string
@@ -31,14 +33,17 @@ type S3Options struct {
 	tlsCertificate   *string
 	metricsHttpPort  *int
 	allowEmptyFolder *bool
+	auditLogConfig   *string
 }

 func init() {
 	cmdS3.Run = runS3 // break init cycle
 	s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address")
+	s3StandaloneOptions.bindIp = cmdS3.Flag.String("ip.bind", "", "ip address to bind to")
 	s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port")
 	s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
 	s3StandaloneOptions.config = cmdS3.Flag.String("config", "", "path to the config file")
+	s3StandaloneOptions.auditLogConfig = cmdS3.Flag.String("auditLogConfig", "", "path to the audit log config file")
 	s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
 	s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
 	s3StandaloneOptions.metricsHttpPort = cmdS3.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
@@ -148,7 +153,7 @@ func (s3opt *S3Options) startS3Server() bool {
 	var metricsIntervalSec int

 	for {
-		err := pb.WithGrpcFilerClient(filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 			if err != nil {
 				return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
@@ -186,12 +191,19 @@ func (s3opt *S3Options) startS3Server() bool {

 	httpS := &http.Server{Handler: router}

-	listenAddress := fmt.Sprintf(":%d", *s3opt.port)
+	listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.port)
 	s3ApiListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
 	if err != nil {
 		glog.Fatalf("S3 API Server listener on %s error: %v", listenAddress, err)
 	}

+	if len(*s3opt.auditLogConfig) > 0 {
+		s3err.InitAuditLog(*s3opt.auditLogConfig)
+		if s3err.Logger != nil {
+			defer s3err.Logger.Close()
+		}
+	}
+
 	if *s3opt.tlsPrivateKey != "" {
 		glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
 		if err = httpS.ServeTLS(s3ApiListener, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil {
@@ -199,6 +199,14 @@ address = "localhost:6379"
 password = ""
 database = 0

+[redis3_sentinel]
+enabled = false
+addresses = ["172.22.12.7:26379","172.22.12.8:26379","172.22.12.9:26379"]
+masterName = "master"
+username = ""
+password = ""
+database = 0
+
 [redis_cluster3] # beta
 enabled = false
 addresses = [
@@ -4,17 +4,46 @@
 #    /etc/seaweedfs/security.toml
 # this file is read by master, volume server, and filer

-# the jwt signing key is read by master and volume server.
-# a jwt defaults to expire after 10 seconds.
+# this jwt signing key is read by master and volume server, and it is used for write operations:
+# - the Master server generates the JWT, which can be used to write a certain file on a volume server
+# - the Volume server validates the JWT on writing
+# the jwt defaults to expire after 10 seconds.
 [jwt.signing]
 key = ""
 expires_after_seconds = 10           # seconds

-# jwt for read is only supported with master+volume setup. Filer does not support this mode.
+# by default, if the signing key above is set, the Volume UI over HTTP is disabled.
+# by setting ui.access to true, you can re-enable the Volume UI. Despite
+# some information leakage (as the UI is not authenticated), this should not
+# pose a security risk.
+[access]
+ui = false
+
+# this jwt signing key is read by master and volume server, and it is used for read operations:
+# - the Master server generates the JWT, which can be used to read a certain file on a volume server
+# - the Volume server validates the JWT on reading
+# NOTE: jwt for read is only supported with master+volume setup. Filer does not support this mode.
 [jwt.signing.read]
 key = ""
 expires_after_seconds = 10           # seconds

+
+# If this JWT key is configured, Filer only accepts writes over HTTP if they are signed with this JWT:
+# - f.e. the S3 API Shim generates the JWT
+# - the Filer server validates the JWT on writing
+# the jwt defaults to expire after 10 seconds.
+[jwt.filer_signing]
+key = ""
+expires_after_seconds = 10           # seconds
+
+# If this JWT key is configured, Filer only accepts reads over HTTP if they are signed with this JWT:
+# - f.e. the S3 API Shim generates the JWT
+# - the Filer server validates the JWT on writing
+# the jwt defaults to expire after 10 seconds.
+[jwt.filer_signing.read]
+key = ""
+expires_after_seconds = 10           # seconds
+
 # all grpc tls authentications are mutual
 # the values for the following ca, cert, and key are paths to the PERM files.
 # the host name is not checked, so the PERM files can be shared.
@@ -131,6 +131,7 @@ func init() {
 	s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
 	s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
 	s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file")
+	s3Options.auditLogConfig = cmdServer.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
 	s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")

 	webdavOptions.port = cmdServer.Flag.Int("webdav.port", 7333, "webdav server http listen port")
@@ -179,6 +180,7 @@ func runServer(cmd *Command, args []string) bool {
 	filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToAddresses()
 	filerOptions.ip = serverIp
 	filerOptions.bindIp = serverBindIp
+	s3Options.bindIp = serverBindIp
 	serverOptions.v.ip = serverIp
 	serverOptions.v.bindIp = serverBindIp
 	serverOptions.v.masters = pb.ServerAddresses(*masterOptions.peers).ToAddresses()
@@ -130,7 +130,7 @@ func runUpload(cmd *Command, args []string) bool {
 }

 func readMasterConfiguration(grpcDialOption grpc.DialOption, masterAddress pb.ServerAddress) (replication string, err error) {
-	err = pb.WithMasterClient(masterAddress, grpcDialOption, func(client master_pb.SeaweedClient) error {
+	err = pb.WithMasterClient(false, masterAddress, grpcDialOption, func(client master_pb.SeaweedClient) error {
 		resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
 		if err != nil {
 			return fmt.Errorf("get master %s configuration: %v", masterAddress, err)
@@ -85,7 +85,7 @@ func (wo *WebDavOption) startWebDav() bool {
 	var cipher bool
 	// connect to filer
 	for {
-		err := pb.WithGrpcFilerClient(filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 			if err != nil {
 				return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
@@ -101,6 +101,15 @@ func fetchChunk(lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string,
 	return retriedFetchChunkData(urlStrings, cipherKey, isGzipped, true, 0, 0)
 }

+func fetchChunkRange(lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool, offset int64, size int) ([]byte, error) {
+	urlStrings, err := lookupFileIdFn(fileId)
+	if err != nil {
+		glog.Errorf("operation LookupFileId %s failed, err: %v", fileId, err)
+		return nil, err
+	}
+	return retriedFetchChunkData(urlStrings, cipherKey, isGzipped, false, offset, size)
+}
+
 func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) ([]byte, error) {

 	var err error
@@ -79,9 +79,9 @@ func (f *Filer) AggregateFromPeers(self pb.ServerAddress) {

 }

-func (f *Filer) ListExistingPeerUpdates() (existingNodes []*master_pb.ClusterNodeUpdate){
+func (f *Filer) ListExistingPeerUpdates() (existingNodes []*master_pb.ClusterNodeUpdate) {

-	if grpcErr := pb.WithMasterClient(f.MasterClient.GetMaster(), f.GrpcDialOption, func(client master_pb.SeaweedClient) error {
+	if grpcErr := pb.WithMasterClient(false, f.MasterClient.GetMaster(), f.GrpcDialOption, func(client master_pb.SeaweedClient) error {
 		resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
 			ClientType: cluster.FilerType,
 		})
@@ -306,14 +306,19 @@ func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, e

 func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (expiredCount int64, lastFileName string, err error) {
 	lastFileName, err = f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix, func(entry *Entry) bool {
-		if entry.TtlSec > 0 {
-			if entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
-				f.Store.DeleteOneEntry(ctx, entry)
-				expiredCount++
-				return true
+		select {
+		case <-ctx.Done():
+			return false
+		default:
+			if entry.TtlSec > 0 {
+				if entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
+					f.Store.DeleteOneEntry(ctx, entry)
+					expiredCount++
+					return true
+				}
 			}
+			return eachEntryFunc(entry)
 		}
-		return eachEntryFunc(entry)
 	})
 	if err != nil {
 		return expiredCount, lastFileName, err
@@ -32,7 +32,7 @@ type FilerConf struct {

 func ReadFilerConf(filerGrpcAddress pb.ServerAddress, grpcDialOption grpc.DialOption, masterClient *wdclient.MasterClient) (*FilerConf, error) {
 	var buf bytes.Buffer
-	if err := pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if err := pb.WithGrpcFilerClient(false, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		if masterClient != nil {
 			return ReadEntry(masterClient, client, DirectoryEtcSeaweedFS, FilerConfName, &buf)
 		} else {
@@ -159,16 +159,15 @@ func mergePathConf(a, b *filer_pb.FilerConf_PathConf) {
 	a.Collection = util.Nvl(b.Collection, a.Collection)
 	a.Replication = util.Nvl(b.Replication, a.Replication)
 	a.Ttl = util.Nvl(b.Ttl, a.Ttl)
-	if b.DiskType != "" {
-		a.DiskType = b.DiskType
-	}
+	a.DiskType = util.Nvl(b.DiskType, a.DiskType)
 	a.Fsync = b.Fsync || a.Fsync
 	if b.VolumeGrowthCount > 0 {
 		a.VolumeGrowthCount = b.VolumeGrowthCount
 	}
-	if b.ReadOnly {
-		a.ReadOnly = b.ReadOnly
-	}
+	a.ReadOnly = b.ReadOnly || a.ReadOnly
+	a.DataCenter = util.Nvl(b.DataCenter, a.DataCenter)
+	a.Rack = util.Nvl(b.Rack, a.Rack)
+	a.DataNode = util.Nvl(b.DataNode, a.DataNode)
 }

 func (fc *FilerConf) ToProto() *filer_pb.FilerConf {
@@ -139,7 +139,7 @@ func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shou

 func (f *Filer) doDeleteCollection(collectionName string) (err error) {

-	return f.MasterClient.WithClient(func(client master_pb.SeaweedClient) error {
+	return f.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
 		_, err := client.CollectionDelete(context.Background(), &master_pb.CollectionDeleteRequest{
 			Name: collectionName,
 		})
@@ -25,7 +25,7 @@ type MetaAggregator struct {
 	isLeader        bool
 	grpcDialOption  grpc.DialOption
 	MetaLogBuffer   *log_buffer.LogBuffer
-	peerStatues     map[pb.ServerAddress]struct{}
+	peerStatues     map[pb.ServerAddress]int
 	peerStatuesLock sync.Mutex
 	// notifying clients
 	ListenersLock sync.Mutex
@@ -39,7 +39,7 @@ func NewMetaAggregator(filer *Filer, self pb.ServerAddress, grpcDialOption grpc.
 		filer:          filer,
 		self:           self,
 		grpcDialOption: grpcDialOption,
-		peerStatues:    make(map[pb.ServerAddress]struct{}),
+		peerStatues:    make(map[pb.ServerAddress]int),
 	}
 	t.ListenersCond = sync.NewCond(&t.ListenersLock)
 	t.MetaLogBuffer = log_buffer.NewLogBuffer("aggr", LogFlushInterval, nil, func() {
@@ -56,27 +56,40 @@ func (ma *MetaAggregator) OnPeerUpdate(update *master_pb.ClusterNodeUpdate) {
 	address := pb.ServerAddress(update.Address)
 	if update.IsAdd {
 		// every filer should subscribe to a new filer
-		ma.setActive(address, true)
-		go ma.subscribeToOneFiler(ma.filer, ma.self, address)
+		if ma.setActive(address, true) {
+			go ma.subscribeToOneFiler(ma.filer, ma.self, address)
+		}
 	} else {
 		ma.setActive(address, false)
 	}
 }

-func (ma *MetaAggregator) setActive(address pb.ServerAddress, isActive bool) {
+func (ma *MetaAggregator) setActive(address pb.ServerAddress, isActive bool) (notDuplicated bool) {
 	ma.peerStatuesLock.Lock()
 	defer ma.peerStatuesLock.Unlock()
 	if isActive {
-		ma.peerStatues[address] = struct{}{}
+		if _, found := ma.peerStatues[address]; found {
+			ma.peerStatues[address] += 1
+		} else {
+			ma.peerStatues[address] = 1
+			notDuplicated = true
+		}
 	} else {
-		delete(ma.peerStatues, address)
+		if _, found := ma.peerStatues[address]; found {
+			ma.peerStatues[address] -= 1
+		}
+		if ma.peerStatues[address] <= 0 {
+			delete(ma.peerStatues, address)
+		}
 	}
+	return
 }
-func (ma *MetaAggregator) isActive(address pb.ServerAddress)(isActive bool) {
+func (ma *MetaAggregator) isActive(address pb.ServerAddress) (isActive bool) {
 	ma.peerStatuesLock.Lock()
 	defer ma.peerStatuesLock.Unlock()
-	_, isActive = ma.peerStatues[address]
-	return
+	var count int
+	count, isActive = ma.peerStatues[address]
+	return count > 0 && isActive
 }

 func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress) {
@@ -152,7 +165,7 @@ func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self pb.ServerAddress, p

 	for {
 		glog.V(4).Infof("subscribing remote %s meta change: %v", peer, time.Unix(0, lastTsNs))
-		err := pb.WithFilerClient(peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err := pb.WithFilerClient(false, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
 			stream, err := client.SubscribeLocalMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
@@ -194,7 +207,7 @@ func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self pb.ServerAddress, p
 }

 func (ma *MetaAggregator) readFilerStoreSignature(peer pb.ServerAddress) (sig int32, err error) {
-	err = pb.WithFilerClient(peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	err = pb.WithFilerClient(false, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 		if err != nil {
 			return err
@@ -193,6 +193,10 @@ func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, dirPath uti
 	optLimit := int64(limit)
 	opts := &options.FindOptions{Limit: &optLimit, Sort: bson.M{"name": 1}}
 	cur, err := store.connect.Database(store.database).Collection(store.collectionName).Find(ctx, where, opts)
+	if err != nil {
+		return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err)
+	}
+
 	for cur.Next(ctx) {
 		var data Model
 		err := cur.Decode(&data)
@@ -26,7 +26,7 @@ func MapRemoteStorageLocationPathToFullPath(localMountedDir util.FullPath, remot
 }

 func CacheRemoteObjectToLocalCluster(filerClient filer_pb.FilerClient, remoteConf *remote_pb.RemoteConf, remoteLocation *remote_pb.RemoteStorageLocation, parent util.FullPath, entry *filer_pb.Entry) error {
-	return filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	return filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		_, err := client.CacheRemoteObjectToLocalCluster(context.Background(), &filer_pb.CacheRemoteObjectToLocalClusterRequest{
 			Directory: string(parent),
 			Name:      entry.Name,
@@ -26,6 +26,7 @@ type ChunkReadAt struct {
 	chunkCache      chunk_cache.ChunkCache
 	lastChunkFileId string
 	lastChunkData   []byte
+	readerPattern   *ReaderPattern
 }

 var _ = io.ReaderAt(&ChunkReadAt{})
@@ -43,7 +44,7 @@ func LookupFn(filerClient filer_pb.FilerClient) wdclient.LookupFileIdFunctionTyp

 		if !found {
 			util.Retry("lookup volume "+vid, func() error {
-				err = filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+				err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 					resp, err := client.LookupVolume(context.Background(), &filer_pb.LookupVolumeRequest{
 						VolumeIds: []string{vid},
 					})
@@ -88,10 +89,11 @@ func LookupFn(filerClient filer_pb.FilerClient) wdclient.LookupFileIdFunctionTyp
 func NewChunkReaderAtFromClient(lookupFn wdclient.LookupFileIdFunctionType, chunkViews []*ChunkView, chunkCache chunk_cache.ChunkCache, fileSize int64) *ChunkReadAt {

 	return &ChunkReadAt{
-		chunkViews:   chunkViews,
-		lookupFileId: lookupFn,
-		chunkCache:   chunkCache,
-		fileSize:     fileSize,
+		chunkViews:    chunkViews,
+		lookupFileId:  lookupFn,
+		chunkCache:    chunkCache,
+		fileSize:      fileSize,
+		readerPattern: NewReaderPattern(),
 	}
 }

@@ -103,6 +105,8 @@ func (c *ChunkReadAt) Close() error {

 func (c *ChunkReadAt) ReadAt(p []byte, offset int64) (n int, err error) {

+	c.readerPattern.MonitorReadAt(offset, len(p))
+
 	c.readerLock.Lock()
 	defer c.readerLock.Unlock()

@@ -171,10 +175,19 @@ func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, err error) {

 func (c *ChunkReadAt) readChunkSlice(chunkView *ChunkView, nextChunkViews *ChunkView, offset, length uint64) ([]byte, error) {

-	chunkSlice := c.chunkCache.GetChunkSlice(chunkView.FileId, offset, length)
+	var chunkSlice []byte
+	if chunkView.LogicOffset == 0 {
+		chunkSlice = c.chunkCache.GetChunkSlice(chunkView.FileId, offset, length)
+	}
 	if len(chunkSlice) > 0 {
 		return chunkSlice, nil
 	}
+	if c.lookupFileId == nil {
+		return nil, nil
+	}
+	if c.readerPattern.IsRandomMode() {
+		return c.doFetchRangeChunkData(chunkView, offset, length)
+	}
 	chunkData, err := c.readFromWholeChunkData(chunkView, nextChunkViews)
 	if err != nil {
 		return nil, err
@@ -217,7 +230,10 @@ func (c *ChunkReadAt) readOneWholeChunk(chunkView *ChunkView) (interface{}, erro

 		glog.V(4).Infof("readFromWholeChunkData %s offset %d [%d,%d) size at least %d", chunkView.FileId, chunkView.Offset, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size), chunkView.ChunkSize)

-		data := c.chunkCache.GetChunk(chunkView.FileId, chunkView.ChunkSize)
+		var data []byte
+		if chunkView.LogicOffset == 0 {
+			data = c.chunkCache.GetChunk(chunkView.FileId, chunkView.ChunkSize)
+		}
 		if data != nil {
 			glog.V(4).Infof("cache hit %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset-chunkView.Offset, chunkView.LogicOffset-chunkView.Offset+int64(len(data)))
 		} else {
@@ -226,7 +242,10 @@ func (c *ChunkReadAt) readOneWholeChunk(chunkView *ChunkView) (interface{}, erro
 			if err != nil {
 				return data, err
 			}
-			c.chunkCache.SetChunk(chunkView.FileId, data)
+			if chunkView.LogicOffset == 0 {
+				// only cache the first chunk
+				c.chunkCache.SetChunk(chunkView.FileId, data)
+			}
 		}
 		return data, err
 	})
@@ -243,3 +262,15 @@ func (c *ChunkReadAt) doFetchFullChunkData(chunkView *ChunkView) ([]byte, error)
 	return data, err

 }
+
+func (c *ChunkReadAt) doFetchRangeChunkData(chunkView *ChunkView, offset, length uint64) ([]byte, error) {
+
+	glog.V(4).Infof("+ doFetchFullChunkData %s", chunkView.FileId)
+
+	data, err := fetchChunkRange(c.lookupFileId, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(length))
+
+	glog.V(4).Infof("- doFetchFullChunkData %s", chunkView.FileId)
+
+	return data, err
+
+}
@@ -22,7 +22,7 @@ func (m *mockChunkCache) GetChunk(fileId string, minSize uint64) (data []byte) {
 }

 func (m *mockChunkCache) GetChunkSlice(fileId string, offset, length uint64) []byte {
-	return nil
+	return m.GetChunk(fileId, length)
 }

 func (m *mockChunkCache) SetChunk(fileId string, data []byte) {
@@ -64,11 +64,12 @@ func TestReaderAt(t *testing.T) {
 	}

 	readerAt := &ChunkReadAt{
-		chunkViews:   ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
-		lookupFileId: nil,
-		readerLock:   sync.Mutex{},
-		fileSize:     10,
-		chunkCache:   &mockChunkCache{},
+		chunkViews:    ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
+		lookupFileId:  nil,
+		readerLock:    sync.Mutex{},
+		fileSize:      10,
+		chunkCache:    &mockChunkCache{},
+		readerPattern: NewReaderPattern(),
 	}

 	testReadAt(t, readerAt, 0, 10, 10, io.EOF)
@@ -114,11 +115,12 @@ func TestReaderAt0(t *testing.T) {
 	}

 	readerAt := &ChunkReadAt{
-		chunkViews:   ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
-		lookupFileId: nil,
-		readerLock:   sync.Mutex{},
-		fileSize:     10,
-		chunkCache:   &mockChunkCache{},
+		chunkViews:    ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
+		lookupFileId:  nil,
+		readerLock:    sync.Mutex{},
+		fileSize:      10,
+		chunkCache:    &mockChunkCache{},
+		readerPattern: NewReaderPattern(),
 	}

 	testReadAt(t, readerAt, 0, 10, 10, io.EOF)
@@ -142,11 +144,12 @@ func TestReaderAt1(t *testing.T) {
 	}

 	readerAt := &ChunkReadAt{
-		chunkViews:   ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
-		lookupFileId: nil,
-		readerLock:   sync.Mutex{},
-		fileSize:     20,
-		chunkCache:   &mockChunkCache{},
+		chunkViews:    ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
+		lookupFileId:  nil,
+		readerLock:    sync.Mutex{},
+		fileSize:      20,
+		chunkCache:    &mockChunkCache{},
+		readerPattern: NewReaderPattern(),
 	}

 	testReadAt(t, readerAt, 0, 20, 20, io.EOF)
@@ -0,0 +1,36 @@
+package filer
+
+type ReaderPattern struct {
+	isStreaming    bool
+	lastReadOffset int64
+}
+
+// For streaming read: only cache the first chunk
+// For random read: only fetch the requested range, instead of the whole chunk
+
+func NewReaderPattern() *ReaderPattern {
+	return &ReaderPattern{
+		isStreaming:    true,
+		lastReadOffset: -1,
+	}
+}
+
+func (rp *ReaderPattern) MonitorReadAt(offset int64, size int) {
+	if rp.lastReadOffset > offset {
+		rp.isStreaming = false
+	}
+	if rp.lastReadOffset == -1 {
+		if offset != 0 {
+			rp.isStreaming = false
+		}
+	}
+	rp.lastReadOffset = offset
+}
+
+func (rp *ReaderPattern) IsStreamingMode() bool {
+	return rp.isStreaming
+}
+
+func (rp *ReaderPattern) IsRandomMode() bool {
+	return !rp.isStreaming
+}
@@ -0,0 +1,49 @@
+package redis3
+
+import (
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/go-redis/redis/v8"
+	"github.com/go-redsync/redsync/v4"
+	"github.com/go-redsync/redsync/v4/redis/goredis/v8"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &Redis3SentinelStore{})
+}
+
+type Redis3SentinelStore struct {
+	UniversalRedis3Store
+}
+
+func (store *Redis3SentinelStore) GetName() string {
+	return "redis3_sentinel"
+}
+
+func (store *Redis3SentinelStore) Initialize(configuration util.Configuration, prefix string) (err error) {
+	return store.initialize(
+		configuration.GetStringSlice(prefix+"addresses"),
+		configuration.GetString(prefix+"masterName"),
+		configuration.GetString(prefix+"username"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetInt(prefix+"database"),
+	)
+}
+
+func (store *Redis3SentinelStore) initialize(addresses []string, masterName string, username string, password string, database int) (err error) {
+	store.Client = redis.NewFailoverClient(&redis.FailoverOptions{
+		MasterName:      masterName,
+		SentinelAddrs:   addresses,
+		Username:        username,
+		Password:        password,
+		DB:              database,
+		MinRetryBackoff: time.Millisecond * 100,
+		MaxRetryBackoff: time.Minute * 1,
+		ReadTimeout:     time.Second * 30,
+		WriteTimeout:    time.Second * 5,
+	})
+	store.redsync = redsync.New(goredis.NewPool(store.Client))
+	return
+}
@@ -11,7 +11,7 @@ import (

 func ReadMountMappings(grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress) (mappings *remote_pb.RemoteStorageMapping, readErr error) {
 	var oldContent []byte
-	if readErr = pb.WithFilerClient(filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if readErr = pb.WithFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		oldContent, readErr = ReadInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE)
 		return readErr
 	}); readErr != nil {
@@ -30,7 +30,7 @@ func InsertMountMapping(filerClient filer_pb.FilerClient, dir string, remoteStor

 	// read current mapping
 	var oldContent, newContent []byte
-	err = filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		oldContent, err = ReadInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE)
 		return err
 	})
@@ -47,7 +47,7 @@ func InsertMountMapping(filerClient filer_pb.FilerClient, dir string, remoteStor
 	}

 	// save back
-	err = filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		return SaveInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE, newContent)
 	})
 	if err != nil {
@@ -61,7 +61,7 @@ func DeleteMountMapping(filerClient filer_pb.FilerClient, dir string) (err error

 	// read current mapping
 	var oldContent, newContent []byte
-	err = filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		oldContent, err = ReadInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE)
 		return err
 	})
@@ -78,7 +78,7 @@ func DeleteMountMapping(filerClient filer_pb.FilerClient, dir string) (err error
 	}

 	// save back
-	err = filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		return SaveInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE, newContent)
 	})
 	if err != nil {
@@ -133,7 +133,7 @@ func UnmarshalRemoteStorageMappings(oldContent []byte) (mappings *remote_pb.Remo

 func ReadRemoteStorageConf(grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress, storageName string) (conf *remote_pb.RemoteConf, readErr error) {
 	var oldContent []byte
-	if readErr = pb.WithFilerClient(filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if readErr = pb.WithFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		oldContent, readErr = ReadInsideFiler(client, DirectoryEtcRemote, storageName+REMOTE_STORAGE_CONF_SUFFIX)
 		return readErr
 	}); readErr != nil {
@@ -161,7 +161,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
 		},
 	}
 	file.dirtyMetadata = true
-	fh := dir.wfs.AcquireHandle(file, req.Uid, req.Gid, req.Flags&fuse.OpenWriteOnly > 0)
+	fh := dir.wfs.AcquireHandle(file, req.Uid, req.Gid)
 	return file, fh, nil

 }
@@ -201,7 +201,7 @@ func (dir *Dir) doCreateEntry(name string, mode os.FileMode, uid, gid uint32, ex
 	}
 	glog.V(1).Infof("create %s/%s", dirFullPath, name)

-	err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err := dir.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

 		dir.wfs.mapPbIdFromLocalToFiler(request.Entry)
 		defer dir.wfs.mapPbIdFromFilerToLocal(request.Entry)
@@ -242,7 +242,7 @@ func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, err

 	dirFullPath := dir.FullPath()

-	err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err := dir.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

 		dir.wfs.mapPbIdFromLocalToFiler(newEntry)
 		defer dir.wfs.mapPbIdFromFilerToLocal(newEntry)
@@ -566,7 +566,7 @@ func (dir *Dir) saveEntry(entry *filer_pb.Entry) error {

 	parentDir, name := util.FullPath(dir.FullPath()).DirAndName()

-	return dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	return dir.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

 		dir.wfs.mapPbIdFromLocalToFiler(entry)
 		defer dir.wfs.mapPbIdFromFilerToLocal(entry)
@@ -68,7 +68,7 @@ func (dir *Dir) Link(ctx context.Context, req *fuse.LinkRequest, old fs.Node) (f
 	}

 	// apply changes to the filer, and also apply to local metaCache
-	err = dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err = dir.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

 		dir.wfs.mapPbIdFromLocalToFiler(request.Entry)
 		defer dir.wfs.mapPbIdFromFilerToLocal(request.Entry)
@@ -121,7 +121,7 @@ func (dir *Dir) Symlink(ctx context.Context, req *fuse.SymlinkRequest) (fs.Node,
 		Signatures: []int32{dir.wfs.signature},
 	}

-	err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err := dir.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

 		dir.wfs.mapPbIdFromLocalToFiler(request.Entry)
 		defer dir.wfs.mapPbIdFromFilerToLocal(request.Entry)
@@ -22,7 +22,7 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
 	glog.V(4).Infof("dir Rename %s => %s", oldPath, newPath)

 	// update remote filer
-	err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err := dir.wfs.WithFilerClient(true, func(client filer_pb.SeaweedFilerClient) error {
 		ctx, cancel := context.WithCancel(context.Background())
 		defer cancel()

@@ -111,9 +111,10 @@ func (dir *Dir) handleRenameResponse(ctx context.Context, resp *filer_pb.StreamR
 		// change file handle
 		inodeId := oldPath.AsInode()
 		dir.wfs.handlesLock.Lock()
-		if existingHandle, found := dir.wfs.handles[inodeId]; found && existingHandle == nil {
+		if existingHandle, found := dir.wfs.handles[inodeId]; found && existingHandle != nil {
 			glog.V(4).Infof("opened file handle %s => %s", oldPath, newPath)
 			delete(dir.wfs.handles, inodeId)
+			existingHandle.f.entry.Name = newName
 			dir.wfs.handles[newPath.AsInode()] = existingHandle
 		}
 		dir.wfs.handlesLock.Unlock()
@@ -1,10 +0,0 @@
-package filesys
-
-type DirtyPages interface {
-	AddPage(offset int64, data []byte)
-	FlushData() error
-	ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64)
-	GetStorageOptions() (collection, replication string)
-	SetWriteOnly(writeOnly bool)
-	GetWriteOnly() (writeOnly bool)
-}
@@ -3,6 +3,7 @@ package filesys
 import (
 	"bytes"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filesys/page_writer"
 	"io"
 	"sync"
 	"time"
@@ -12,9 +13,8 @@ import (
 )

 type ContinuousDirtyPages struct {
-	intervals      *ContinuousIntervals
+	intervals      *page_writer.ContinuousIntervals
 	f              *File
-	writeOnly      bool
 	writeWaitGroup sync.WaitGroup
 	chunkAddLock   sync.Mutex
 	lastErr        error
@@ -22,11 +22,10 @@ type ContinuousDirtyPages struct {
 	replication    string
 }

-func newContinuousDirtyPages(file *File, writeOnly bool) *ContinuousDirtyPages {
+func newContinuousDirtyPages(file *File) *ContinuousDirtyPages {
 	dirtyPages := &ContinuousDirtyPages{
-		intervals: &ContinuousIntervals{},
+		intervals: &page_writer.ContinuousIntervals{},
 		f:         file,
-		writeOnly: writeOnly,
 	}
 	return dirtyPages
 }
@@ -107,7 +106,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64,
 		defer pages.writeWaitGroup.Done()

 		reader = io.LimitReader(reader, size)
-		chunk, collection, replication, err := pages.f.wfs.saveDataAsChunk(pages.f.fullpath(), pages.writeOnly)(reader, pages.f.Name, offset)
+		chunk, collection, replication, err := pages.f.wfs.saveDataAsChunk(pages.f.fullpath())(reader, pages.f.Name, offset)
 		if err != nil {
 			glog.V(0).Infof("%s saveToStorage [%d,%d): %v", pages.f.fullpath(), offset, offset+size, err)
 			pages.lastErr = err
@@ -128,19 +127,6 @@ func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64,
 	}
 }

-func max(x, y int64) int64 {
-	if x > y {
-		return x
-	}
-	return y
-}
-func min(x, y int64) int64 {
-	if x < y {
-		return x
-	}
-	return y
-}
-
 func (pages *ContinuousDirtyPages) ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) {
 	return pages.intervals.ReadDataAt(data, startOffset)
 }
@@ -148,13 +134,5 @@ func (pages *ContinuousDirtyPages) ReadDirtyDataAt(data []byte, startOffset int6
 func (pages *ContinuousDirtyPages) GetStorageOptions() (collection, replication string) {
 	return pages.collection, pages.replication
 }
-
-func (pages *ContinuousDirtyPages) SetWriteOnly(writeOnly bool) {
-	if pages.writeOnly {
-		pages.writeOnly = writeOnly
-	}
-}
-
-func (pages *ContinuousDirtyPages) GetWriteOnly() (writeOnly bool) {
-	return pages.writeOnly
+func (pages ContinuousDirtyPages) Destroy() {
 }
@@ -0,0 +1,106 @@
+package filesys
+
+import (
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filesys/page_writer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"io"
+	"sync"
+	"time"
+)
+
+type StreamDirtyPages struct {
+	f              *File
+	writeWaitGroup sync.WaitGroup
+	pageAddLock    sync.Mutex
+	chunkAddLock   sync.Mutex
+	lastErr        error
+	collection     string
+	replication    string
+	chunkedStream  *page_writer.ChunkedStreamWriter
+}
+
+func newStreamDirtyPages(file *File, chunkSize int64) *StreamDirtyPages {
+
+	dirtyPages := &StreamDirtyPages{
+		f:             file,
+		chunkedStream: page_writer.NewChunkedStreamWriter(chunkSize),
+	}
+
+	dirtyPages.chunkedStream.SetSaveToStorageFunction(dirtyPages.saveChunkedFileIntevalToStorage)
+
+	return dirtyPages
+}
+
+func (pages *StreamDirtyPages) AddPage(offset int64, data []byte) {
+
+	pages.pageAddLock.Lock()
+	defer pages.pageAddLock.Unlock()
+
+	glog.V(4).Infof("%v stream AddPage [%d, %d)", pages.f.fullpath(), offset, offset+int64(len(data)))
+	if _, err := pages.chunkedStream.WriteAt(data, offset); err != nil {
+		pages.lastErr = err
+	}
+
+	return
+}
+
+func (pages *StreamDirtyPages) FlushData() error {
+	pages.saveChunkedFileToStorage()
+	pages.writeWaitGroup.Wait()
+	if pages.lastErr != nil {
+		return fmt.Errorf("flush data: %v", pages.lastErr)
+	}
+	pages.chunkedStream.Reset()
+	return nil
+}
+
+func (pages *StreamDirtyPages) ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) {
+	return pages.chunkedStream.ReadDataAt(data, startOffset)
+}
+
+func (pages *StreamDirtyPages) GetStorageOptions() (collection, replication string) {
+	return pages.collection, pages.replication
+}
+
+func (pages *StreamDirtyPages) saveChunkedFileToStorage() {
+
+	pages.chunkedStream.FlushAll()
+
+}
+
+func (pages *StreamDirtyPages) saveChunkedFileIntevalToStorage(reader io.Reader, offset int64, size int64, cleanupFn func()) {
+
+	mtime := time.Now().UnixNano()
+	pages.writeWaitGroup.Add(1)
+	writer := func() {
+		defer pages.writeWaitGroup.Done()
+		defer cleanupFn()
+
+		chunk, collection, replication, err := pages.f.wfs.saveDataAsChunk(pages.f.fullpath())(reader, pages.f.Name, offset)
+		if err != nil {
+			glog.V(0).Infof("%s saveToStorage [%d,%d): %v", pages.f.fullpath(), offset, offset+size, err)
+			pages.lastErr = err
+			return
+		}
+		chunk.Mtime = mtime
+		pages.collection, pages.replication = collection, replication
+		pages.chunkAddLock.Lock()
+		pages.f.addChunks([]*filer_pb.FileChunk{chunk})
+		glog.V(3).Infof("%s saveToStorage %s [%d,%d)", pages.f.fullpath(), chunk.FileId, offset, offset+size)
+		pages.chunkAddLock.Unlock()
+
+	}
+
+	if pages.f.wfs.concurrentWriters != nil {
+		pages.f.wfs.concurrentWriters.Execute(writer)
+	} else {
+		go writer()
+	}
+
+}
+
+func (pages StreamDirtyPages) Destroy() {
+	pages.chunkedStream.Reset()
+}
@@ -2,6 +2,7 @@ package filesys

 import (
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filesys/page_writer"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"io"
@@ -11,24 +12,21 @@ import (
 )

 type TempFileDirtyPages struct {
-	f                *File
-	tf               *os.File
-	writtenIntervals *WrittenContinuousIntervals
-	writeOnly        bool
-	writeWaitGroup   sync.WaitGroup
-	pageAddLock      sync.Mutex
-	chunkAddLock     sync.Mutex
-	lastErr          error
-	collection       string
-	replication      string
+	f              *File
+	writeWaitGroup sync.WaitGroup
+	pageAddLock    sync.Mutex
+	chunkAddLock   sync.Mutex
+	lastErr        error
+	collection     string
+	replication    string
+	chunkedFile    *page_writer.ChunkedFileWriter
 }

-func newTempFileDirtyPages(file *File, writeOnly bool) *TempFileDirtyPages {
+func newTempFileDirtyPages(file *File, chunkSize int64) *TempFileDirtyPages {

 	tempFile := &TempFileDirtyPages{
-		f:                file,
-		writeOnly:        writeOnly,
-		writtenIntervals: &WrittenContinuousIntervals{},
+		f:           file,
+		chunkedFile: page_writer.NewChunkedFileWriter(file.wfs.option.getTempFilePageDir(), chunkSize),
 	}

 	return tempFile
@@ -39,85 +37,49 @@ func (pages *TempFileDirtyPages) AddPage(offset int64, data []byte) {
 	pages.pageAddLock.Lock()
 	defer pages.pageAddLock.Unlock()

-	if pages.tf == nil {
-		tf, err := os.CreateTemp(pages.f.wfs.option.getTempFilePageDir(), "")
-		if err != nil {
-			glog.Errorf("create temp file: %v", err)
-			pages.lastErr = err
-			return
-		}
-		pages.tf = tf
-		pages.writtenIntervals.tempFile = tf
-		pages.writtenIntervals.lastOffset = 0
-	}
-
-	writtenOffset := pages.writtenIntervals.lastOffset
-	dataSize := int64(len(data))
-
-	// glog.V(4).Infof("%s AddPage %v at %d [%d,%d)", pages.f.fullpath(), pages.tf.Name(), writtenOffset, offset, offset+dataSize)
-
-	if _, err := pages.tf.WriteAt(data, writtenOffset); err != nil {
+	glog.V(4).Infof("%v tempfile AddPage [%d, %d)", pages.f.fullpath(), offset, offset+int64(len(data)))
+	if _, err := pages.chunkedFile.WriteAt(data, offset); err != nil {
 		pages.lastErr = err
-	} else {
-		pages.writtenIntervals.AddInterval(writtenOffset, len(data), offset)
-		pages.writtenIntervals.lastOffset += dataSize
 	}

-	// pages.writtenIntervals.debug()
-
 	return
 }

 func (pages *TempFileDirtyPages) FlushData() error {
-
-	pages.saveExistingPagesToStorage()
+	pages.saveChunkedFileToStorage()
 	pages.writeWaitGroup.Wait()
 	if pages.lastErr != nil {
 		return fmt.Errorf("flush data: %v", pages.lastErr)
 	}
-	pages.pageAddLock.Lock()
-	defer pages.pageAddLock.Unlock()
-	if pages.tf != nil {
-
-		pages.writtenIntervals.tempFile = nil
-		pages.writtenIntervals.lists = nil
-
-		pages.tf.Close()
-		os.Remove(pages.tf.Name())
-		pages.tf = nil
-	}
+	pages.chunkedFile.Reset()
 	return nil
 }

-func (pages *TempFileDirtyPages) saveExistingPagesToStorage() {
+func (pages *TempFileDirtyPages) ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) {
+	return pages.chunkedFile.ReadDataAt(data, startOffset)
+}

-	pageSize := pages.f.wfs.option.ChunkSizeLimit
+func (pages *TempFileDirtyPages) GetStorageOptions() (collection, replication string) {
+	return pages.collection, pages.replication
+}

-	// glog.V(4).Infof("%v saveExistingPagesToStorage %d lists", pages.f.Name, len(pages.writtenIntervals.lists))
+func (pages *TempFileDirtyPages) saveChunkedFileToStorage() {

-	for _, list := range pages.writtenIntervals.lists {
-		listStopOffset := list.Offset() + list.Size()
-		for uploadedOffset := int64(0); uploadedOffset < listStopOffset; uploadedOffset += pageSize {
-			start, stop := max(list.Offset(), uploadedOffset), min(listStopOffset, uploadedOffset+pageSize)
-			if start >= stop {
-				continue
-			}
-			// glog.V(4).Infof("uploading %v [%d,%d) %d/%d", pages.f.Name, start, stop, i, len(pages.writtenIntervals.lists))
-			pages.saveToStorage(list.ToReader(start, stop), start, stop-start)
-		}
-	}
+	pages.chunkedFile.ProcessEachInterval(func(file *os.File, logicChunkIndex page_writer.LogicChunkIndex, interval *page_writer.ChunkWrittenInterval) {
+		reader := page_writer.NewFileIntervalReader(pages.chunkedFile, logicChunkIndex, interval)
+		pages.saveChunkedFileIntevalToStorage(reader, int64(logicChunkIndex)*pages.chunkedFile.ChunkSize+interval.StartOffset, interval.Size())
+	})

 }

-func (pages *TempFileDirtyPages) saveToStorage(reader io.Reader, offset int64, size int64) {
+func (pages *TempFileDirtyPages) saveChunkedFileIntevalToStorage(reader io.Reader, offset int64, size int64) {

 	mtime := time.Now().UnixNano()
 	pages.writeWaitGroup.Add(1)
 	writer := func() {
 		defer pages.writeWaitGroup.Done()

-		reader = io.LimitReader(reader, size)
-		chunk, collection, replication, err := pages.f.wfs.saveDataAsChunk(pages.f.fullpath(), pages.writeOnly)(reader, pages.f.Name, offset)
+		chunk, collection, replication, err := pages.f.wfs.saveDataAsChunk(pages.f.fullpath())(reader, pages.f.Name, offset)
 		if err != nil {
 			glog.V(0).Infof("%s saveToStorage [%d,%d): %v", pages.f.fullpath(), offset, offset+size, err)
 			pages.lastErr = err
@@ -136,22 +98,9 @@ func (pages *TempFileDirtyPages) saveToStorage(reader io.Reader, offset int64, s
 	} else {
 		go writer()
 	}
+
 }

-func (pages *TempFileDirtyPages) ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) {
-	return pages.writtenIntervals.ReadDataAt(data, startOffset)
-}
-
-func (pages *TempFileDirtyPages) GetStorageOptions() (collection, replication string) {
-	return pages.collection, pages.replication
-}
-
-func (pages *TempFileDirtyPages) SetWriteOnly(writeOnly bool) {
-	if pages.writeOnly {
-		pages.writeOnly = writeOnly
-	}
-}
-
-func (pages *TempFileDirtyPages) GetWriteOnly() (writeOnly bool) {
-	return pages.writeOnly
+func (pages TempFileDirtyPages) Destroy() {
+	pages.chunkedFile.Reset()
 }
@@ -1,289 +0,0 @@
-package filesys
-
-import (
-	"io"
-	"log"
-	"os"
-)
-
-type WrittenIntervalNode struct {
-	DataOffset int64
-	TempOffset int64
-	Size       int64
-	Next       *WrittenIntervalNode
-}
-
-type WrittenIntervalLinkedList struct {
-	tempFile *os.File
-	Head     *WrittenIntervalNode
-	Tail     *WrittenIntervalNode
-}
-
-type WrittenContinuousIntervals struct {
-	tempFile   *os.File
-	lastOffset int64
-	lists      []*WrittenIntervalLinkedList
-}
-
-func (list *WrittenIntervalLinkedList) Offset() int64 {
-	return list.Head.DataOffset
-}
-func (list *WrittenIntervalLinkedList) Size() int64 {
-	return list.Tail.DataOffset + list.Tail.Size - list.Head.DataOffset
-}
-func (list *WrittenIntervalLinkedList) addNodeToTail(node *WrittenIntervalNode) {
-	// glog.V(4).Infof("add to tail [%d,%d) + [%d,%d) => [%d,%d)", list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, node.Offset+node.Size, list.Head.Offset, node.Offset+node.Size)
-	if list.Tail.TempOffset+list.Tail.Size == node.TempOffset {
-		// already connected
-		list.Tail.Size += node.Size
-	} else {
-		list.Tail.Next = node
-		list.Tail = node
-	}
-}
-func (list *WrittenIntervalLinkedList) addNodeToHead(node *WrittenIntervalNode) {
-	// glog.V(4).Infof("add to head [%d,%d) + [%d,%d) => [%d,%d)", node.Offset, node.Offset+node.Size, list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, list.Tail.Offset+list.Tail.Size)
-	node.Next = list.Head
-	list.Head = node
-}
-
-func (list *WrittenIntervalLinkedList) ReadData(buf []byte, start, stop int64) {
-	t := list.Head
-	for {
-
-		nodeStart, nodeStop := max(start, t.DataOffset), min(stop, t.DataOffset+t.Size)
-		if nodeStart < nodeStop {
-			// glog.V(4).Infof("copying start=%d stop=%d t=[%d,%d) => bufSize=%d nodeStart=%d, nodeStop=%d", start, stop, t.DataOffset, t.DataOffset+t.Size, len(buf), nodeStart, nodeStop)
-			list.tempFile.ReadAt(buf[nodeStart-start:nodeStop-start], t.TempOffset+nodeStart-t.DataOffset)
-		}
-
-		if t.Next == nil {
-			break
-		}
-		t = t.Next
-	}
-}
-
-func (c *WrittenContinuousIntervals) TotalSize() (total int64) {
-	for _, list := range c.lists {
-		total += list.Size()
-	}
-	return
-}
-
-func (list *WrittenIntervalLinkedList) subList(start, stop int64) *WrittenIntervalLinkedList {
-	var nodes []*WrittenIntervalNode
-	for t := list.Head; t != nil; t = t.Next {
-		nodeStart, nodeStop := max(start, t.DataOffset), min(stop, t.DataOffset+t.Size)
-		if nodeStart >= nodeStop {
-			// skip non overlapping WrittenIntervalNode
-			continue
-		}
-		nodes = append(nodes, &WrittenIntervalNode{
-			TempOffset: t.TempOffset + nodeStart - t.DataOffset,
-			DataOffset: nodeStart,
-			Size:       nodeStop - nodeStart,
-			Next:       nil,
-		})
-	}
-	for i := 1; i < len(nodes); i++ {
-		nodes[i-1].Next = nodes[i]
-	}
-	return &WrittenIntervalLinkedList{
-		tempFile: list.tempFile,
-		Head:     nodes[0],
-		Tail:     nodes[len(nodes)-1],
-	}
-}
-
-func (c *WrittenContinuousIntervals) debug() {
-	log.Printf("++")
-	for _, l := range c.lists {
-		log.Printf("++++")
-		for t := l.Head; ; t = t.Next {
-			log.Printf("[%d,%d) => [%d,%d) %d", t.DataOffset, t.DataOffset+t.Size, t.TempOffset, t.TempOffset+t.Size, t.Size)
-			if t.Next == nil {
-				break
-			}
-		}
-		log.Printf("----")
-	}
-	log.Printf("--")
-}
-
-func (c *WrittenContinuousIntervals) AddInterval(tempOffset int64, dataSize int, dataOffset int64) {
-
-	interval := &WrittenIntervalNode{DataOffset: dataOffset, TempOffset: tempOffset, Size: int64(dataSize)}
-
-	// append to the tail and return
-	if len(c.lists) == 1 {
-		lastSpan := c.lists[0]
-		if lastSpan.Tail.DataOffset+lastSpan.Tail.Size == dataOffset {
-			lastSpan.addNodeToTail(interval)
-			return
-		}
-	}
-
-	var newLists []*WrittenIntervalLinkedList
-	for _, list := range c.lists {
-		// if list is to the left of new interval, add to the new list
-		if list.Tail.DataOffset+list.Tail.Size <= interval.DataOffset {
-			newLists = append(newLists, list)
-		}
-		// if list is to the right of new interval, add to the new list
-		if interval.DataOffset+interval.Size <= list.Head.DataOffset {
-			newLists = append(newLists, list)
-		}
-		// if new interval overwrite the right part of the list
-		if list.Head.DataOffset < interval.DataOffset && interval.DataOffset < list.Tail.DataOffset+list.Tail.Size {
-			// create a new list of the left part of existing list
-			newLists = append(newLists, list.subList(list.Offset(), interval.DataOffset))
-		}
-		// if new interval overwrite the left part of the list
-		if list.Head.DataOffset < interval.DataOffset+interval.Size && interval.DataOffset+interval.Size < list.Tail.DataOffset+list.Tail.Size {
-			// create a new list of the right part of existing list
-			newLists = append(newLists, list.subList(interval.DataOffset+interval.Size, list.Tail.DataOffset+list.Tail.Size))
-		}
-		// skip anything that is fully overwritten by the new interval
-	}
-
-	c.lists = newLists
-	// add the new interval to the lists, connecting neighbor lists
-	var prevList, nextList *WrittenIntervalLinkedList
-
-	for _, list := range c.lists {
-		if list.Head.DataOffset == interval.DataOffset+interval.Size {
-			nextList = list
-			break
-		}
-	}
-
-	for _, list := range c.lists {
-		if list.Head.DataOffset+list.Size() == dataOffset {
-			list.addNodeToTail(interval)
-			prevList = list
-			break
-		}
-	}
-
-	if prevList != nil && nextList != nil {
-		// glog.V(4).Infof("connecting [%d,%d) + [%d,%d) => [%d,%d)", prevList.Head.Offset, prevList.Tail.Offset+prevList.Tail.Size, nextList.Head.Offset, nextList.Tail.Offset+nextList.Tail.Size, prevList.Head.Offset, nextList.Tail.Offset+nextList.Tail.Size)
-		prevList.Tail.Next = nextList.Head
-		prevList.Tail = nextList.Tail
-		c.removeList(nextList)
-	} else if nextList != nil {
-		// add to head was not done when checking
-		nextList.addNodeToHead(interval)
-	}
-	if prevList == nil && nextList == nil {
-		c.lists = append(c.lists, &WrittenIntervalLinkedList{
-			tempFile: c.tempFile,
-			Head:     interval,
-			Tail:     interval,
-		})
-	}
-
-	return
-}
-
-func (c *WrittenContinuousIntervals) RemoveLargestIntervalLinkedList() *WrittenIntervalLinkedList {
-	var maxSize int64
-	maxIndex := -1
-	for k, list := range c.lists {
-		if maxSize <= list.Size() {
-			maxSize = list.Size()
-			maxIndex = k
-		}
-	}
-	if maxSize <= 0 {
-		return nil
-	}
-
-	t := c.lists[maxIndex]
-	t.tempFile = c.tempFile
-	c.lists = append(c.lists[0:maxIndex], c.lists[maxIndex+1:]...)
-	return t
-
-}
-
-func (c *WrittenContinuousIntervals) removeList(target *WrittenIntervalLinkedList) {
-	index := -1
-	for k, list := range c.lists {
-		if list.Offset() == target.Offset() {
-			index = k
-		}
-	}
-	if index < 0 {
-		return
-	}
-
-	c.lists = append(c.lists[0:index], c.lists[index+1:]...)
-
-}
-
-func (c *WrittenContinuousIntervals) ReadDataAt(data []byte, startOffset int64) (maxStop int64) {
-	for _, list := range c.lists {
-		start := max(startOffset, list.Offset())
-		stop := min(startOffset+int64(len(data)), list.Offset()+list.Size())
-		if start < stop {
-			list.ReadData(data[start-startOffset:], start, stop)
-			maxStop = max(maxStop, stop)
-		}
-	}
-	return
-}
-
-func (l *WrittenIntervalLinkedList) ToReader(start int64, stop int64) io.Reader {
-	// TODO: optimize this to avoid another loop
-	var readers []io.Reader
-	for t := l.Head; ; t = t.Next {
-		startOffset, stopOffset := max(t.DataOffset, start), min(t.DataOffset+t.Size, stop)
-		if startOffset < stopOffset {
-			// glog.V(4).Infof("ToReader read [%d,%d) from [%d,%d) %d", t.DataOffset, t.DataOffset+t.Size, t.TempOffset, t.TempOffset+t.Size, t.Size)
-			readers = append(readers, newFileSectionReader(l.tempFile, startOffset-t.DataOffset+t.TempOffset, startOffset, stopOffset-startOffset))
-		}
-		if t.Next == nil {
-			break
-		}
-	}
-	if len(readers) == 1 {
-		return readers[0]
-	}
-	return io.MultiReader(readers...)
-}
-
-type FileSectionReader struct {
-	file            *os.File
-	tempStartOffset int64
-	Offset          int64
-	dataStart       int64
-	dataStop        int64
-}
-
-var _ = io.Reader(&FileSectionReader{})
-
-func newFileSectionReader(tempfile *os.File, offset int64, dataOffset int64, size int64) *FileSectionReader {
-	return &FileSectionReader{
-		file:            tempfile,
-		tempStartOffset: offset,
-		Offset:          offset,
-		dataStart:       dataOffset,
-		dataStop:        dataOffset + size,
-	}
-}
-
-func (f *FileSectionReader) Read(p []byte) (n int, err error) {
-	remaining := (f.dataStop - f.dataStart) - (f.Offset - f.tempStartOffset)
-	if remaining <= 0 {
-		return 0, io.EOF
-	}
-	dataLen := min(remaining, int64(len(p)))
-	// glog.V(4).Infof("reading [%d,%d) from %v [%d,%d)/[%d,%d) %d", f.Offset-f.tempStartOffset+f.dataStart, f.Offset-f.tempStartOffset+f.dataStart+dataLen, f.file.Name(), f.Offset, f.Offset+dataLen, f.tempStartOffset, f.tempStartOffset+f.dataStop-f.dataStart, f.dataStop-f.dataStart)
-	n, err = f.file.ReadAt(p[:dataLen], f.Offset)
-	if n > 0 {
-		f.Offset += int64(n)
-	} else {
-		err = io.EOF
-	}
-	return
-}
@@ -96,8 +96,9 @@ func (file *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp
 func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {

 	glog.V(4).Infof("file %v open %+v", file.fullpath(), req)
+	// resp.Flags |= fuse.OpenDirectIO

-	handle := file.wfs.AcquireHandle(file, req.Uid, req.Gid, req.Flags&fuse.OpenWriteOnly > 0)
+	handle := file.wfs.AcquireHandle(file, req.Uid, req.Gid)

 	resp.Handle = fuse.HandleID(handle.handle)

@@ -331,7 +332,7 @@ func (file *File) addChunks(chunks []*filer_pb.FileChunk) {
 }

 func (file *File) saveEntry(entry *filer_pb.Entry) error {
-	return file.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	return file.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

 		file.wfs.mapPbIdFromLocalToFiler(entry)
 		defer file.wfs.mapPbIdFromFilerToLocal(entry)
@@ -362,7 +363,7 @@ func (file *File) getEntry() *filer_pb.Entry {
 }

 func (file *File) downloadRemoteEntry(entry *filer_pb.Entry) (*filer_pb.Entry, error) {
-	err := file.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err := file.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

 		request := &filer_pb.CacheRemoteObjectToLocalClusterRequest{
 			Directory: file.dir.FullPath(),
@@ -20,12 +20,13 @@ import (

 type FileHandle struct {
 	// cache file has been written to
-	dirtyPages     DirtyPages
+	dirtyPages     *PageWriter
 	entryViewCache []filer.VisibleInterval
 	reader         io.ReaderAt
 	contentType    string
 	handle         uint64
 	sync.Mutex
+	sync.WaitGroup

 	f         *File
 	RequestId fuse.RequestID // unique ID for request
@@ -36,11 +37,11 @@ type FileHandle struct {
 	isDeleted bool
 }

-func newFileHandle(file *File, uid, gid uint32, writeOnly bool) *FileHandle {
+func newFileHandle(file *File, uid, gid uint32) *FileHandle {
 	fh := &FileHandle{
 		f: file,
 		// dirtyPages: newContinuousDirtyPages(file, writeOnly),
-		dirtyPages: newTempFileDirtyPages(file, writeOnly),
+		dirtyPages: newPageWriter(file, file.wfs.option.ChunkSizeLimit),
 		Uid:        uid,
 		Gid:        gid,
 	}
@@ -62,10 +63,14 @@ var _ = fs.HandleReleaser(&FileHandle{})

 func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {

-	glog.V(4).Infof("%s read fh %d: [%d,%d) size %d resp.Data cap=%d", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size), req.Size, cap(resp.Data))
+	fh.Add(1)
+	defer fh.Done()
+
 	fh.Lock()
 	defer fh.Unlock()

+	glog.V(4).Infof("%s read fh %d: [%d,%d) size %d resp.Data cap=%d", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size), req.Size, cap(resp.Data))
+
 	if req.Size <= 0 {
 		return nil
 	}
@@ -168,12 +173,17 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {
 // Write to the file handle
 func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error {

+	fh.Add(1)
+	defer fh.Done()
+
+	fh.dirtyPages.writerPattern.MonitorWriteAt(req.Offset, len(req.Data))
+
 	fh.Lock()
 	defer fh.Unlock()

 	// write the request to volume servers
 	data := req.Data
-	if len(data) <= 512 {
+	if len(data) <= 512 && req.Offset == 0 {
 		// fuse message cacheable size
 		data = make([]byte, len(req.Data))
 		copy(data, req.Data)
@@ -207,8 +217,7 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err

 	glog.V(4).Infof("Release %v fh %d open=%d", fh.f.fullpath(), fh.handle, fh.f.isOpen)

-	fh.Lock()
-	defer fh.Unlock()
+	fh.Wait()

 	fh.f.wfs.handlesLock.Lock()
 	fh.f.isOpen--
@@ -220,6 +229,7 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err
 		fh.reader = nil

 		fh.f.wfs.ReleaseHandle(fh.f.fullpath(), fuse.HandleID(fh.handle))
+		fh.dirtyPages.Destroy()
 	}

 	if fh.f.isOpen < 0 {
@@ -240,6 +250,9 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 		return nil
 	}

+	fh.Add(1)
+	defer fh.Done()
+
 	fh.Lock()
 	defer fh.Unlock()

@@ -248,7 +261,6 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 		return err
 	}

-	glog.V(4).Infof("Flush %v fh %d success", fh.f.fullpath(), fh.handle)
 	return nil
 }

@@ -266,7 +278,7 @@ func (fh *FileHandle) doFlush(ctx context.Context, header fuse.Header) error {
 		return nil
 	}

-	err := fh.f.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err := fh.f.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

 		entry := fh.f.getEntry()
 		if entry == nil {
@@ -303,7 +315,7 @@ func (fh *FileHandle) doFlush(ctx context.Context, header fuse.Header) error {
 		manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(entry.Chunks)

 		chunks, _ := filer.CompactFileChunks(fh.f.wfs.LookupFn(), nonManifestChunks)
-		chunks, manifestErr := filer.MaybeManifestize(fh.f.wfs.saveDataAsChunk(fh.f.fullpath(), fh.dirtyPages.GetWriteOnly()), chunks)
+		chunks, manifestErr := filer.MaybeManifestize(fh.f.wfs.saveDataAsChunk(fh.f.fullpath()), chunks)
 		if manifestErr != nil {
 			// not good, but should be ok
 			glog.V(0).Infof("MaybeManifestize: %v", manifestErr)
@@ -58,7 +58,7 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil
 	}

 	util.RetryForever("followMetaUpdates", func() error {
-		return pb.WithFilerClientFollowMetadata(client, "mount", dir, &lastTsNs, selfSignature, processEventFn, true)
+		return pb.WithFilerClientFollowMetadata(client, "mount", selfSignature, dir, &lastTsNs, selfSignature, processEventFn, true)
 	}, func(err error) bool {
 		glog.Errorf("follow metadata updates: %v", err)
 		return true
@@ -0,0 +1,115 @@
+package filesys
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/filesys/page_writer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+)
+
+type PageWriter struct {
+	f             *File
+	collection    string
+	replication   string
+	chunkSize     int64
+	writerPattern *WriterPattern
+
+	randomWriter page_writer.DirtyPages
+	streamWriter page_writer.DirtyPages
+}
+
+var (
+	_ = page_writer.DirtyPages(&PageWriter{})
+)
+
+func newPageWriter(file *File, chunkSize int64) *PageWriter {
+	pw := &PageWriter{
+		f:             file,
+		chunkSize:     chunkSize,
+		writerPattern: NewWriterPattern(chunkSize),
+		randomWriter:  newTempFileDirtyPages(file, chunkSize),
+		streamWriter:  newStreamDirtyPages(file, chunkSize),
+		//streamWriter: newContinuousDirtyPages(file),
+		//streamWriter:  nil,
+	}
+	return pw
+}
+
+func (pw *PageWriter) AddPage(offset int64, data []byte) {
+
+	glog.V(4).Infof("%v AddPage [%d, %d) streaming:%v", pw.f.fullpath(), offset, offset+int64(len(data)), pw.writerPattern.IsStreamingMode())
+
+	chunkIndex := offset / pw.chunkSize
+	for i := chunkIndex; len(data) > 0; i++ {
+		writeSize := min(int64(len(data)), (i+1)*pw.chunkSize-offset)
+		pw.addToOneChunk(i, offset, data[:writeSize])
+		offset += writeSize
+		data = data[writeSize:]
+	}
+}
+
+func (pw *PageWriter) addToOneChunk(chunkIndex, offset int64, data []byte) {
+	if chunkIndex > 0 {
+		if pw.writerPattern.IsStreamingMode() && pw.streamWriter != nil {
+			pw.streamWriter.AddPage(offset, data)
+			return
+		}
+	}
+	pw.randomWriter.AddPage(offset, data)
+}
+
+func (pw *PageWriter) FlushData() error {
+	pw.writerPattern.Reset()
+	if pw.streamWriter != nil {
+		if err := pw.streamWriter.FlushData(); err != nil {
+			return err
+		}
+	}
+	return pw.randomWriter.FlushData()
+}
+
+func (pw *PageWriter) ReadDirtyDataAt(data []byte, offset int64) (maxStop int64) {
+	glog.V(4).Infof("ReadDirtyDataAt %v [%d, %d)", pw.f.fullpath(), offset, offset+int64(len(data)))
+
+	chunkIndex := offset / pw.chunkSize
+	for i := chunkIndex; len(data) > 0; i++ {
+		readSize := min(int64(len(data)), (i+1)*pw.chunkSize-offset)
+
+		if pw.streamWriter != nil {
+			m1 := pw.streamWriter.ReadDirtyDataAt(data[:readSize], offset)
+			maxStop = max(maxStop, m1)
+		}
+		m2 := pw.randomWriter.ReadDirtyDataAt(data[:readSize], offset)
+		maxStop = max(maxStop, m2)
+
+		offset += readSize
+		data = data[readSize:]
+	}
+
+	return
+}
+
+func (pw *PageWriter) GetStorageOptions() (collection, replication string) {
+	if pw.writerPattern.IsStreamingMode() && pw.streamWriter != nil {
+		return pw.streamWriter.GetStorageOptions()
+	}
+	return pw.randomWriter.GetStorageOptions()
+}
+
+func (pw *PageWriter) Destroy() {
+	if pw.streamWriter != nil {
+		pw.streamWriter.Destroy()
+	}
+	pw.randomWriter.Destroy()
+}
+
+func max(x, y int64) int64 {
+	if x > y {
+		return x
+	}
+	return y
+}
+func min(x, y int64) int64 {
+	if x < y {
+		return x
+	}
+	return y
+}
@@ -0,0 +1,109 @@
+package page_writer
+
+import "math"
+
+// ChunkWrittenInterval mark one written interval within one page chunk
+type ChunkWrittenInterval struct {
+	StartOffset int64
+	stopOffset  int64
+	prev        *ChunkWrittenInterval
+	next        *ChunkWrittenInterval
+}
+
+func (interval *ChunkWrittenInterval) Size() int64 {
+	return interval.stopOffset - interval.StartOffset
+}
+
+func (interval *ChunkWrittenInterval) isComplete(chunkSize int64) bool {
+	return interval.stopOffset-interval.StartOffset == chunkSize
+}
+
+// ChunkWrittenIntervalList mark written intervals within one page chunk
+type ChunkWrittenIntervalList struct {
+	head *ChunkWrittenInterval
+	tail *ChunkWrittenInterval
+}
+
+func newChunkWrittenIntervalList() *ChunkWrittenIntervalList {
+	list := &ChunkWrittenIntervalList{
+		head: &ChunkWrittenInterval{
+			StartOffset: -1,
+			stopOffset:  -1,
+		},
+		tail: &ChunkWrittenInterval{
+			StartOffset: math.MaxInt64,
+			stopOffset:  math.MaxInt64,
+		},
+	}
+	list.head.next = list.tail
+	list.tail.prev = list.head
+	return list
+}
+
+func (list *ChunkWrittenIntervalList) MarkWritten(startOffset, stopOffset int64) {
+	interval := &ChunkWrittenInterval{
+		StartOffset: startOffset,
+		stopOffset:  stopOffset,
+	}
+	list.addInterval(interval)
+}
+
+func (list *ChunkWrittenIntervalList) IsComplete(chunkSize int64) bool {
+	return list.size() == 1 && list.head.next.isComplete(chunkSize)
+}
+
+func (list *ChunkWrittenIntervalList) addInterval(interval *ChunkWrittenInterval) {
+
+	p := list.head
+	for ; p.next != nil && p.next.StartOffset <= interval.StartOffset; p = p.next {
+	}
+	q := list.tail
+	for ; q.prev != nil && q.prev.stopOffset >= interval.stopOffset; q = q.prev {
+	}
+
+	if interval.StartOffset <= p.stopOffset && q.StartOffset <= interval.stopOffset {
+		// merge p and q together
+		p.stopOffset = q.stopOffset
+		unlinkNodesBetween(p, q.next)
+		return
+	}
+	if interval.StartOffset <= p.stopOffset {
+		// merge new interval into p
+		p.stopOffset = interval.stopOffset
+		unlinkNodesBetween(p, q)
+		return
+	}
+	if q.StartOffset <= interval.stopOffset {
+		// merge new interval into q
+		q.StartOffset = interval.StartOffset
+		unlinkNodesBetween(p, q)
+		return
+	}
+
+	// add the new interval between p and q
+	unlinkNodesBetween(p, q)
+	p.next = interval
+	interval.prev = p
+	q.prev = interval
+	interval.next = q
+
+}
+
+// unlinkNodesBetween remove all nodes after start and before stop, exclusive
+func unlinkNodesBetween(start *ChunkWrittenInterval, stop *ChunkWrittenInterval) {
+	if start.next == stop {
+		return
+	}
+	start.next.prev = nil
+	start.next = stop
+	stop.prev.next = nil
+	stop.prev = start
+}
+
+func (list *ChunkWrittenIntervalList) size() int {
+	var count int
+	for t := list.head; t != nil; t = t.next {
+		count++
+	}
+	return count - 2
+}
@@ -0,0 +1,49 @@
+package page_writer
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func Test_PageChunkWrittenIntervalList(t *testing.T) {
+	list := newChunkWrittenIntervalList()
+
+	assert.Equal(t, 0, list.size(), "empty list")
+
+	list.MarkWritten(0, 5)
+	assert.Equal(t, 1, list.size(), "one interval")
+
+	list.MarkWritten(0, 5)
+	assert.Equal(t, 1, list.size(), "duplicated interval2")
+
+	list.MarkWritten(95, 100)
+	assert.Equal(t, 2, list.size(), "two intervals")
+
+	list.MarkWritten(50, 60)
+	assert.Equal(t, 3, list.size(), "three intervals")
+
+	list.MarkWritten(50, 55)
+	assert.Equal(t, 3, list.size(), "three intervals merge")
+
+	list.MarkWritten(40, 50)
+	assert.Equal(t, 3, list.size(), "three intervals grow forward")
+
+	list.MarkWritten(50, 65)
+	assert.Equal(t, 3, list.size(), "three intervals grow backward")
+
+	list.MarkWritten(70, 80)
+	assert.Equal(t, 4, list.size(), "four intervals")
+
+	list.MarkWritten(60, 70)
+	assert.Equal(t, 3, list.size(), "three intervals merged")
+
+	list.MarkWritten(59, 71)
+	assert.Equal(t, 3, list.size(), "covered three intervals")
+
+	list.MarkWritten(5, 59)
+	assert.Equal(t, 2, list.size(), "covered two intervals")
+
+	list.MarkWritten(70, 99)
+	assert.Equal(t, 1, list.size(), "covered one intervals")
+
+}
@@ -0,0 +1,160 @@
+package page_writer
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"io"
+	"os"
+	"sync"
+)
+
+type LogicChunkIndex int
+type ActualChunkIndex int
+
+// ChunkedFileWriter assumes the write requests will come in within chunks
+type ChunkedFileWriter struct {
+	dir                     string
+	file                    *os.File
+	logicToActualChunkIndex map[LogicChunkIndex]ActualChunkIndex
+	chunkUsages             []*ChunkWrittenIntervalList
+	ChunkSize               int64
+	sync.Mutex
+}
+
+var _ = io.WriterAt(&ChunkedFileWriter{})
+
+func NewChunkedFileWriter(dir string, chunkSize int64) *ChunkedFileWriter {
+	return &ChunkedFileWriter{
+		dir:                     dir,
+		file:                    nil,
+		logicToActualChunkIndex: make(map[LogicChunkIndex]ActualChunkIndex),
+		ChunkSize:               chunkSize,
+	}
+}
+
+func (cw *ChunkedFileWriter) WriteAt(p []byte, off int64) (n int, err error) {
+	cw.Lock()
+	defer cw.Unlock()
+
+	if cw.file == nil {
+		cw.file, err = os.CreateTemp(cw.dir, "")
+		if err != nil {
+			glog.Errorf("create temp file: %v", err)
+			return
+		}
+	}
+
+	actualOffset, chunkUsage := cw.toActualWriteOffset(off)
+	n, err = cw.file.WriteAt(p, actualOffset)
+	if err == nil {
+		startOffset := off % cw.ChunkSize
+		chunkUsage.MarkWritten(startOffset, startOffset+int64(n))
+	}
+	return
+}
+
+func (cw *ChunkedFileWriter) ReadDataAt(p []byte, off int64) (maxStop int64) {
+	cw.Lock()
+	defer cw.Unlock()
+
+	if cw.file == nil {
+		return
+	}
+
+	logicChunkIndex := off / cw.ChunkSize
+	actualChunkIndex, chunkUsage := cw.toActualReadOffset(off)
+	if chunkUsage != nil {
+		for t := chunkUsage.head.next; t != chunkUsage.tail; t = t.next {
+			logicStart := max(off, logicChunkIndex*cw.ChunkSize+t.StartOffset)
+			logicStop := min(off+int64(len(p)), logicChunkIndex*cw.ChunkSize+t.stopOffset)
+			if logicStart < logicStop {
+				actualStart := logicStart - logicChunkIndex*cw.ChunkSize + int64(actualChunkIndex)*cw.ChunkSize
+				_, err := cw.file.ReadAt(p[logicStart-off:logicStop-off], actualStart)
+				if err != nil {
+					glog.Errorf("reading temp file: %v", err)
+					break
+				}
+				maxStop = max(maxStop, logicStop)
+			}
+		}
+	}
+	return
+}
+
+func (cw *ChunkedFileWriter) toActualWriteOffset(logicOffset int64) (actualOffset int64, chunkUsage *ChunkWrittenIntervalList) {
+	logicChunkIndex := LogicChunkIndex(logicOffset / cw.ChunkSize)
+	offsetRemainder := logicOffset % cw.ChunkSize
+	existingActualChunkIndex, found := cw.logicToActualChunkIndex[logicChunkIndex]
+	if found {
+		return int64(existingActualChunkIndex)*cw.ChunkSize + offsetRemainder, cw.chunkUsages[existingActualChunkIndex]
+	}
+	cw.logicToActualChunkIndex[logicChunkIndex] = ActualChunkIndex(len(cw.chunkUsages))
+	chunkUsage = newChunkWrittenIntervalList()
+	cw.chunkUsages = append(cw.chunkUsages, chunkUsage)
+	return int64(len(cw.chunkUsages)-1)*cw.ChunkSize + offsetRemainder, chunkUsage
+}
+
+func (cw *ChunkedFileWriter) toActualReadOffset(logicOffset int64) (actualChunkIndex ActualChunkIndex, chunkUsage *ChunkWrittenIntervalList) {
+	logicChunkIndex := LogicChunkIndex(logicOffset / cw.ChunkSize)
+	existingActualChunkIndex, found := cw.logicToActualChunkIndex[logicChunkIndex]
+	if found {
+		return existingActualChunkIndex, cw.chunkUsages[existingActualChunkIndex]
+	}
+	return 0, nil
+}
+
+func (cw *ChunkedFileWriter) ProcessEachInterval(process func(file *os.File, logicChunkIndex LogicChunkIndex, interval *ChunkWrittenInterval)) {
+	for logicChunkIndex, actualChunkIndex := range cw.logicToActualChunkIndex {
+		chunkUsage := cw.chunkUsages[actualChunkIndex]
+		for t := chunkUsage.head.next; t != chunkUsage.tail; t = t.next {
+			process(cw.file, logicChunkIndex, t)
+		}
+	}
+}
+
+// Reset releases used resources
+func (cw *ChunkedFileWriter) Reset() {
+	if cw.file != nil {
+		cw.file.Close()
+		os.Remove(cw.file.Name())
+		cw.file = nil
+	}
+	cw.logicToActualChunkIndex = make(map[LogicChunkIndex]ActualChunkIndex)
+	cw.chunkUsages = cw.chunkUsages[:0]
+}
+
+type FileIntervalReader struct {
+	f           *os.File
+	startOffset int64
+	stopOffset  int64
+	position    int64
+}
+
+var _ = io.Reader(&FileIntervalReader{})
+
+func NewFileIntervalReader(cw *ChunkedFileWriter, logicChunkIndex LogicChunkIndex, interval *ChunkWrittenInterval) *FileIntervalReader {
+	actualChunkIndex, found := cw.logicToActualChunkIndex[logicChunkIndex]
+	if !found {
+		// this should never happen
+		return nil
+	}
+	return &FileIntervalReader{
+		f:           cw.file,
+		startOffset: int64(actualChunkIndex)*cw.ChunkSize + interval.StartOffset,
+		stopOffset:  int64(actualChunkIndex)*cw.ChunkSize + interval.stopOffset,
+		position:    0,
+	}
+}
+
+func (fr *FileIntervalReader) Read(p []byte) (n int, err error) {
+	readSize := minInt(len(p), int(fr.stopOffset-fr.startOffset-fr.position))
+	n, err = fr.f.ReadAt(p[:readSize], fr.startOffset+fr.position)
+	if err == nil || err == io.EOF {
+		fr.position += int64(n)
+		if fr.stopOffset-fr.startOffset-fr.position == 0 {
+			// return a tiny bit faster
+			err = io.EOF
+			return
+		}
+	}
+	return
+}
@@ -0,0 +1,60 @@
+package page_writer
+
+import (
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+)
+
+func TestChunkedFileWriter_toActualOffset(t *testing.T) {
+	cw := NewChunkedFileWriter("", 16)
+
+	writeToFile(cw, 50, 60)
+	writeToFile(cw, 60, 64)
+
+	writeToFile(cw, 32, 40)
+	writeToFile(cw, 42, 48)
+
+	writeToFile(cw, 48, 50)
+
+	assert.Equal(t, 1, cw.chunkUsages[0].size(), "fully covered")
+	assert.Equal(t, 2, cw.chunkUsages[1].size(), "2 intervals")
+
+}
+
+func writeToFile(cw *ChunkedFileWriter, startOffset int64, stopOffset int64) {
+
+	_, chunkUsage := cw.toActualWriteOffset(startOffset)
+
+	// skip doing actual writing
+
+	innerOffset := startOffset % cw.ChunkSize
+	chunkUsage.MarkWritten(innerOffset, innerOffset+stopOffset-startOffset)
+
+}
+
+func TestWriteChunkedFile(t *testing.T) {
+	x := NewChunkedFileWriter(os.TempDir(), 20)
+	defer x.Reset()
+	y := NewChunkedFileWriter(os.TempDir(), 12)
+	defer y.Reset()
+
+	batchSize := 4
+	buf := make([]byte, batchSize)
+	for i := 0; i < 256; i++ {
+		for x := 0; x < batchSize; x++ {
+			buf[x] = byte(i)
+		}
+		x.WriteAt(buf, int64(i*batchSize))
+		y.WriteAt(buf, int64((255-i)*batchSize))
+	}
+
+	a := make([]byte, 1)
+	b := make([]byte, 1)
+	for i := 0; i < 256*batchSize; i++ {
+		x.ReadDataAt(a, int64(i))
+		y.ReadDataAt(b, int64(256*batchSize-1-i))
+		assert.Equal(t, a[0], b[0], "same read")
+	}
+
+}
@@ -0,0 +1,119 @@
+package page_writer
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/util/mem"
+	"io"
+	"sync"
+	"sync/atomic"
+)
+
+type SaveToStorageFunc func(reader io.Reader, offset int64, size int64, cleanupFn func())
+
+// ChunkedStreamWriter assumes the write requests will come in within chunks and in streaming mode
+type ChunkedStreamWriter struct {
+	activeChunks     map[LogicChunkIndex]*MemChunk
+	activeChunksLock sync.Mutex
+	ChunkSize        int64
+	saveToStorageFn  SaveToStorageFunc
+	sync.Mutex
+}
+
+type MemChunk struct {
+	buf   []byte
+	usage *ChunkWrittenIntervalList
+}
+
+var _ = io.WriterAt(&ChunkedStreamWriter{})
+
+func NewChunkedStreamWriter(chunkSize int64) *ChunkedStreamWriter {
+	return &ChunkedStreamWriter{
+		ChunkSize:    chunkSize,
+		activeChunks: make(map[LogicChunkIndex]*MemChunk),
+	}
+}
+
+func (cw *ChunkedStreamWriter) SetSaveToStorageFunction(saveToStorageFn SaveToStorageFunc) {
+	cw.saveToStorageFn = saveToStorageFn
+}
+
+func (cw *ChunkedStreamWriter) WriteAt(p []byte, off int64) (n int, err error) {
+	cw.Lock()
+	defer cw.Unlock()
+
+	logicChunkIndex := LogicChunkIndex(off / cw.ChunkSize)
+	offsetRemainder := off % cw.ChunkSize
+
+	memChunk, found := cw.activeChunks[logicChunkIndex]
+	if !found {
+		memChunk = &MemChunk{
+			buf:   mem.Allocate(int(cw.ChunkSize)),
+			usage: newChunkWrittenIntervalList(),
+		}
+		cw.activeChunks[logicChunkIndex] = memChunk
+	}
+	n = copy(memChunk.buf[offsetRemainder:], p)
+	memChunk.usage.MarkWritten(offsetRemainder, offsetRemainder+int64(n))
+	if memChunk.usage.IsComplete(cw.ChunkSize) {
+		if cw.saveToStorageFn != nil {
+			cw.saveOneChunk(memChunk, logicChunkIndex)
+			delete(cw.activeChunks, logicChunkIndex)
+		}
+	}
+
+	return
+}
+
+func (cw *ChunkedStreamWriter) ReadDataAt(p []byte, off int64) (maxStop int64) {
+	cw.Lock()
+	defer cw.Unlock()
+
+	logicChunkIndex := LogicChunkIndex(off / cw.ChunkSize)
+	memChunkBaseOffset := int64(logicChunkIndex) * cw.ChunkSize
+	memChunk, found := cw.activeChunks[logicChunkIndex]
+	if !found {
+		return
+	}
+
+	for t := memChunk.usage.head.next; t != memChunk.usage.tail; t = t.next {
+		logicStart := max(off, int64(logicChunkIndex)*cw.ChunkSize+t.StartOffset)
+		logicStop := min(off+int64(len(p)), memChunkBaseOffset+t.stopOffset)
+		if logicStart < logicStop {
+			copy(p[logicStart-off:logicStop-off], memChunk.buf[logicStart-memChunkBaseOffset:logicStop-memChunkBaseOffset])
+			maxStop = max(maxStop, logicStop)
+		}
+	}
+	return
+}
+
+func (cw *ChunkedStreamWriter) FlushAll() {
+	cw.Lock()
+	defer cw.Unlock()
+	for logicChunkIndex, memChunk := range cw.activeChunks {
+		if cw.saveToStorageFn != nil {
+			cw.saveOneChunk(memChunk, logicChunkIndex)
+			delete(cw.activeChunks, logicChunkIndex)
+		}
+	}
+}
+
+func (cw *ChunkedStreamWriter) saveOneChunk(memChunk *MemChunk, logicChunkIndex LogicChunkIndex) {
+	var referenceCounter = int32(memChunk.usage.size())
+	for t := memChunk.usage.head.next; t != memChunk.usage.tail; t = t.next {
+		reader := util.NewBytesReader(memChunk.buf[t.StartOffset:t.stopOffset])
+		cw.saveToStorageFn(reader, int64(logicChunkIndex)*cw.ChunkSize+t.StartOffset, t.Size(), func() {
+			atomic.AddInt32(&referenceCounter, -1)
+			if atomic.LoadInt32(&referenceCounter) == 0 {
+				mem.Free(memChunk.buf)
+			}
+		})
+	}
+}
+
+// Reset releases used resources
+func (cw *ChunkedStreamWriter) Reset() {
+	for t, memChunk := range cw.activeChunks {
+		mem.Free(memChunk.buf)
+		delete(cw.activeChunks, t)
+	}
+}
@@ -0,0 +1,33 @@
+package page_writer
+
+import (
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+)
+
+func TestWriteChunkedStream(t *testing.T) {
+	x := NewChunkedStreamWriter(20)
+	defer x.Reset()
+	y := NewChunkedFileWriter(os.TempDir(), 12)
+	defer y.Reset()
+
+	batchSize := 4
+	buf := make([]byte, batchSize)
+	for i := 0; i < 256; i++ {
+		for x := 0; x < batchSize; x++ {
+			buf[x] = byte(i)
+		}
+		x.WriteAt(buf, int64(i*batchSize))
+		y.WriteAt(buf, int64((255-i)*batchSize))
+	}
+
+	a := make([]byte, 1)
+	b := make([]byte, 1)
+	for i := 0; i < 256*batchSize; i++ {
+		x.ReadDataAt(a, int64(i))
+		y.ReadDataAt(b, int64(256*batchSize-1-i))
+		assert.Equal(t, a[0], b[0], "same read")
+	}
+
+}
@@ -1,4 +1,4 @@
-package filesys
+package page_writer

 import (
 	"io"
@@ -1,4 +1,4 @@
-package filesys
+package page_writer

 import (
 	"bytes"
@@ -0,0 +1,28 @@
+package page_writer
+
+type DirtyPages interface {
+	AddPage(offset int64, data []byte)
+	FlushData() error
+	ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64)
+	GetStorageOptions() (collection, replication string)
+	Destroy()
+}
+
+func max(x, y int64) int64 {
+	if x > y {
+		return x
+	}
+	return y
+}
+func min(x, y int64) int64 {
+	if x < y {
+		return x
+	}
+	return y
+}
+func minInt(x, y int) int {
+	if x < y {
+		return x
+	}
+	return y
+}
@@ -0,0 +1,44 @@
+package filesys
+
+type WriterPattern struct {
+	isStreaming     bool
+	lastWriteOffset int64
+	chunkSize       int64
+}
+
+// For streaming write: only cache the first chunk
+// For random write: fall back to temp file approach
+// writes can only change from streaming mode to non-streaming mode
+
+func NewWriterPattern(chunkSize int64) *WriterPattern {
+	return &WriterPattern{
+		isStreaming:     true,
+		lastWriteOffset: -1,
+		chunkSize:       chunkSize,
+	}
+}
+
+func (rp *WriterPattern) MonitorWriteAt(offset int64, size int) {
+	if rp.lastWriteOffset > offset {
+		rp.isStreaming = false
+	}
+	if rp.lastWriteOffset == -1 {
+		if offset != 0 {
+			rp.isStreaming = false
+		}
+	}
+	rp.lastWriteOffset = offset
+}
+
+func (rp *WriterPattern) IsStreamingMode() bool {
+	return rp.isStreaming
+}
+
+func (rp *WriterPattern) IsRandomMode() bool {
+	return !rp.isStreaming
+}
+
+func (rp *WriterPattern) Reset() {
+	rp.isStreaming = true
+	rp.lastWriteOffset = -1
+}
@@ -148,7 +148,7 @@ func (wfs *WFS) Root() (fs.Node, error) {
 	return wfs.root, nil
 }

-func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32, writeOnly bool) (fileHandle *FileHandle) {
+func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHandle) {

 	fullpath := file.fullpath()
 	glog.V(4).Infof("AcquireHandle %s uid=%d gid=%d", fullpath, uid, gid)
@@ -160,7 +160,6 @@ func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32, writeOnly bool) (file
 	if found && existingHandle != nil && existingHandle.f.isOpen > 0 {
 		existingHandle.f.isOpen++
 		wfs.handlesLock.Unlock()
-		existingHandle.dirtyPages.SetWriteOnly(writeOnly)
 		glog.V(4).Infof("Reuse AcquiredHandle %s open %d", fullpath, existingHandle.f.isOpen)
 		return existingHandle
 	}
@@ -168,7 +167,7 @@ func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32, writeOnly bool) (file

 	entry, _ := file.maybeLoadEntry(context.Background())
 	file.entry = entry
-	fileHandle = newFileHandle(file, uid, gid, writeOnly)
+	fileHandle = newFileHandle(file, uid, gid)

 	wfs.handlesLock.Lock()
 	file.isOpen++
@@ -198,7 +197,7 @@ func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.

 	if wfs.stats.lastChecked < time.Now().Unix()-20 {

-		err := wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

 			request := &filer_pb.StatisticsRequest{
 				Collection:  wfs.option.Collection,
@@ -11,7 +11,7 @@ import (

 var _ = filer_pb.FilerClient(&WFS{})

-func (wfs *WFS) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) (err error) {
+func (wfs *WFS) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) (err error) {

 	return util.Retry("filer grpc", func() error {

@@ -20,7 +20,7 @@ func (wfs *WFS) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) (err
 		for x := 0; x < n; x++ {

 			filerGrpcAddress := wfs.option.FilerAddresses[i].ToGrpcAddress()
-			err = pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
+			err = pb.WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {
 				client := filer_pb.NewSeaweedFilerClient(grpcConnection)
 				return fn(client)
 			}, filerGrpcAddress, wfs.option.GrpcDialOption)
@@ -13,13 +13,13 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 )

-func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath, writeOnly bool) filer.SaveDataAsChunkFunctionType {
+func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath) filer.SaveDataAsChunkFunctionType {

 	return func(reader io.Reader, filename string, offset int64) (chunk *filer_pb.FileChunk, collection, replication string, err error) {
 		var fileId, host string
 		var auth security.EncodedJwt

-		if err := wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		if err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 			return util.Retry("assignVolume", func() error {
 				request := &filer_pb.AssignVolumeRequest{
 					Count:       1,
@@ -74,7 +74,7 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath, writeOnly bool) filer.Sa
 			return nil, "", "", fmt.Errorf("upload result: %v", uploadResult.Error)
 		}

-		if !writeOnly {
+		if offset == 0 {
 			wfs.chunkCache.SetChunk(fileId, data)
 		}

--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
chrislu	19555385f7	2.85	2022-01-09 19:30:23 -08:00
Chris LuandGitHub	3b1dc85c6f	Merge pull request #2564 from chrislusf/dependabot/maven/other/java/client/com.google.protobuf-protobuf-java-3.16.1 Bump protobuf-java from 3.9.1 to 3.16.1 in /other/java/client	2022-01-07 15:16:21 -08:00
chrislu	110d5a5233	support fixing a collection of volumes, or volumes under one directory	2022-01-07 14:52:16 -08:00
dependabot[bot]andGitHub	46237ea31a	Bump protobuf-java from 3.9.1 to 3.16.1 in /other/java/client Bumps [protobuf-java](https://github.com/protocolbuffers/protobuf) from 3.9.1 to 3.16.1. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/master/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v3.9.1...v3.16.1) --- updated-dependencies: - dependency-name: com.google.protobuf:protobuf-java dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com>	2022-01-07 22:37:10 +00:00
chrislu	60dc450091	skip fixing read only volumes fix https://github.com/chrislusf/seaweedfs/issues/2562	2022-01-06 09:52:28 -08:00
chrislu	3df8f96117	avoid changing inode	2022-01-06 01:36:11 -08:00
chrislu	67b0645808	mount: need to change entry name after renaming	2022-01-05 21:27:41 -08:00
chrislu	4de060daa6	mount: skip special character in the filenames fix https://github.com/chrislusf/seaweedfs/issues/2559	2022-01-05 03:57:24 -08:00
chrislu	e76105e2ab	fix auth permission checking	2022-01-03 21:05:20 -08:00
chrislu	a7887166cf	wildcard prefix to restrict access to directories in s3 bucket https://github.com/chrislusf/seaweedfs/discussions/2551	2022-01-03 15:39:36 -08:00
chrislu	5799a20f71	2.84	2022-01-02 17:05:19 -08:00
Chris LuandGitHub	077f831749	Merge pull request #2550 from chrislusf/metadata_follow_with_client_id add client id for all metadata listening clients	2022-01-02 01:11:27 -08:00
Chris LuandGitHub	42c849e0df	Merge branch 'master' into metadata_follow_with_client_id	2022-01-02 01:07:30 -08:00
Chris LuandGitHub	9b94177380	Merge pull request #2543 from skurfuerst/seaweedfs-158 FEATURE: add JWT to HTTP endpoints of Filer and use them in S3 Client	2022-01-01 22:34:13 -08:00
Sebastian Kurfuerst	99abddf376	FEATURE: add test setup for running the CephFS S3 compatibility suite	2021-12-31 22:07:49 +01:00
Sebastian Kurfuerst	c35660175d	BUGFIX: ensure Authorization header is only added once	2021-12-31 22:06:18 +01:00
Sebastian Kurfuerst	1cd3b6b4e1	BUGFIX: security.toml contained wrong keys	2021-12-31 22:05:41 +01:00
Sebastian Kurfuerst	10404c4275	FEATURE: add JWT to HTTP endpoints of Filer and use them in S3 Client - one JWT for reading and one for writing, analogous to how the JWT between Master and Volume Server works - I did not implement IP `whiteList` parameter on the filer Additionally, because http_util.DownloadFile now sets the JWT, the `download` command should now work when `jwt.signing.read` is configured. By looking at the code, I think this case did not work before. ## Docs to be adjusted after a release Page `Amazon-S3-API`: ``` # Authentication with Filer You can use mTLS for the gRPC connection between S3-API-Proxy and the filer, as explained in [Security-Configuration](Security-Configuration) - controlled by the `grpc.` configuration in `security.toml`. Starting with version XX, it is also possible to authenticate the HTTP operations between the S3-API-Proxy and the Filer (especially uploading new files). This is configured by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. With both configurations (gRPC and JWT), it is possible to have Filer and S3 communicate in fully authenticated fashion; so Filer will reject any unauthenticated communication. ``` Page `Security Overview`: ``` The following items are not covered, yet: - master server http REST services Starting with version XX, the Filer HTTP REST services can be secured with a JWT, by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. ... Before version XX: "weed filer -disableHttp", disable http operations, only gRPC operations are allowed. This works with "weed mount" by FUSE. It does not work* with the [S3 Gateway](Amazon S3 API), as this does HTTP calls to the Filer. Starting with version XX: secured by JWT, by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. This now works with the [S3 Gateway](Amazon S3 API). ... # Securing Filer HTTP with JWT To enable JWT-based access control for the Filer, 1. generate `security.toml` file by `weed scaffold -config=security` 2. set `filer_jwt.signing.key` to a secret string - and optionally filer_jwt.signing.read.key` as well to a secret string 3. copy the same `security.toml` file to the filers and all S3 proxies. If `filer_jwt.signing.key` is configured: When sending upload/update/delete HTTP operations to a filer server, the request header `Authorization` should be the JWT string (`Authorization: Bearer [JwtToken]`). The operation is authorized after the filer validates the JWT with `filer_jwt.signing.key`. If `filer_jwt.signing.read.key` is configured: When sending GET or HEAD requests to a filer server, the request header `Authorization` should be the JWT string (`Authorization: Bearer [JwtToken]`). The operation is authorized after the filer validates the JWT with `filer_jwt.signing.read.key`. The S3 API Gateway reads the above JWT keys and sends authenticated HTTP requests to the filer. ``` Page `Security Configuration`: ``` (update scaffold file) ... [filer_jwt.signing] key = "blahblahblahblah" [filer_jwt.signing.read] key = "blahblahblahblah" ``` Resolves: #158	2021-12-30 14:45:27 +01:00
chrislu	34742be029	remove duplicated metadata subscription in filer https://github.com/chrislusf/seaweedfs/issues/2545	2021-12-30 01:51:52 -08:00
chrislu	5c87fcc6d2	add client id for all metadata listening clients	2021-12-30 00:23:57 -08:00
chrislu	fb434318e3	dynamically adjust connection timeout better fix for https://github.com/chrislusf/seaweedfs/issues/2541	2021-12-29 22:44:39 -08:00
chrislu	5788bf2270	s3: increase timeout limit https://github.com/chrislusf/seaweedfs/issues/2541	2021-12-29 22:21:02 -08:00
Sebastian Kurfuerst	fcc09cef6f	Refactor: pass in claim type into security.DecodeJwt	2021-12-29 12:40:41 +01:00
Sebastian Kurfuerst	d156d410ef	rename security.GenJwt to security.GenJwtForVolumeServer	2021-12-29 12:39:41 +01:00
Sebastian Kurfuerst	eda4c43a08	fix typo in error message	2021-12-29 12:38:14 +01:00
chrislu	d351541757	upgrade fuse version	2021-12-28 18:18:54 -08:00
chrislu	498661e3bb	mount: remove limits on number of parallel requests	2021-12-28 17:41:01 -08:00
chrislu	9a00c17555	reader: avoid wrong pattern detection due to lock waiting	2021-12-28 16:30:33 -08:00
chrislu	f7a6f6b4c0	if this is enabled, there are some "bus error" with git clone	2021-12-28 13:44:52 -08:00
chrislu	0da2dfd640	fuse: change to direct io mode before and after: chrislu$ time dd if=/dev/random of=/Users/chrislu/tmp/mm/testfile bs=131072 count=8192 8192+0 records in 8192+0 records out 1073741824 bytes transferred in 4.534068 secs (236816430 bytes/sec) dd if=/dev/random of=/Users/chrislu/tmp/mm/testfile bs=131072 count=8192 0.01s user 3.86s system 84% cpu 4.561 total chrislu$ time dd if=/dev/random of=/Users/chrislu/tmp/mm/testfile bs=131072 count=8192 8192+0 records in 8192+0 records out 1073741824 bytes transferred in 3.824072 secs (280784948 bytes/sec) dd if=/dev/random of=/Users/chrislu/tmp/mm/testfile bs=131072 count=8192 0.01s user 3.22s system 83% cpu 3.857 total	2021-12-28 12:22:56 -08:00
chrislu	3fd4da34a4	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2021-12-28 01:10:38 -08:00
chrislu	2422556456	monitor write pattern: avoid timing due to locking	2021-12-28 01:10:35 -08:00
Chris LuandGitHub	6c8caea09c	Merge pull request #2539 from ck789987/filer-list-use-context filer list entries use context to break job	2021-12-27 23:47:51 -08:00
chenkai	47c30e3add	filer list entries use context to break job	2021-12-28 15:03:41 +08:00
chrislu	80db8b13d8	bug: cleanup function was called twice	2021-12-27 20:53:02 -08:00
chrislu	67b723f74e	Filer Server API support fsync fix https://github.com/chrislusf/seaweedfs/issues/2528	2021-12-26 17:28:47 -08:00
chrislu	9f9ef1340c	use streaming mode for long poll grpc calls streaming mode would create separate grpc connections for each call. this is to ensure the long poll connections are properly closed.	2021-12-26 00:15:03 -08:00
chrislu	c935b9669e	2.83	2021-12-25 01:01:34 -08:00
chrislu	eb4ad2546f	use proper chunk size limit option	2021-12-24 22:52:18 -08:00
chrislu	41bbf320bb	use 2MB chunk size. cache size is the wrong option	2021-12-24 22:50:19 -08:00
chrislu	982ea85d81	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2021-12-24 22:40:07 -08:00
chrislu	083d8e9ece	add stream writer this should improve streaming write performance, which is common in many cases, e.g., copying large files. This is additional to improved random read write operations: https://github.com/chrislusf/seaweedfs/wiki/FUSE-Mount/_compare/3e69d193805c79802f4f8f6cc63269b7a9a911f3...19084d87918f297cac15e2471c19306176e0771f	2021-12-24 22:38:22 -08:00
Chris LuandGitHub	e7a6a2733b	Merge pull request #2534 from skurfuerst/add-ui-access-to-security-toml	2021-12-24 09:57:00 -08:00
Sebastian Kurfürst	6db49100d6	BUGFIX: add access.ui setting to scaffolded security.toml ... The property is read here: https://github.com/chrislusf/seaweedfs/blob/b70cb3e0b2a75543f0410d4c09f42aa95fcf2ee2/weed/server/volume_server.go#L69	2021-12-24 13:59:04 +01:00
chrislu	255a1c7dcd	refactor type names	2021-12-23 18:23:18 -08:00
chrislu	f77ca41769	refactor	2021-12-23 17:48:34 -08:00
chrislu	1d36884845	rename files	2021-12-23 17:47:58 -08:00
chrislu	2d1a1f5e03	rename variables and functions	2021-12-23 17:35:57 -08:00
chrislu	7bf48ee135	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2021-12-23 17:23:26 -08:00
chrislu	6de331b014	clean up	2021-12-23 17:23:21 -08:00
chrislu	032df784ed	chunked file works now	2021-12-23 17:17:32 -08:00
Chris LuandGitHub	6ead7758ce	Merge pull request #2532 from kmlebedev/fix_CVE fix cves	2021-12-23 12:00:40 -08:00
Chris LuandGitHub	d264f9449f	Merge pull request #2533 from banjiaojuhao/filer_add-datacenter-rack-datanode-for-path-specific-configuration	2021-12-23 09:46:56 -08:00
banjiaojuhao	083bf3a137	filer server: add "datacenter, rack and datanode" for path specific configuration	2021-12-23 23:25:05 +08:00
Konstantin Lebedev	791ad774a2	CVE-2020-13949 CVE-2020-27813 CVE-2019-19794 CVE-2021-38561	2021-12-23 13:57:41 +05:00
chrislu	c2aad1c7ff	detect non streaming mode on first write request	2021-12-22 17:20:44 -08:00
chrislu	b541e39a2c	fix tests	2021-12-22 16:17:30 -08:00
chrislu	2bc6fa90ff	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2021-12-22 16:05:43 -08:00
chrislu	0ec7bc6710	detect non streaming mode on the first read	2021-12-22 16:05:38 -08:00
chrislu	4c1368d621	fix test	2021-12-22 16:05:08 -08:00
Chris LuandGitHub	4e73705533	Merge pull request #2530 from banjiaojuhao/filer-upload-file-to-node filer server: allow upload file to specific dataNode	2021-12-22 12:49:15 -08:00
banjiaojuhao	08336be92e	filer server: allow upload file to specific dataNode	2021-12-22 21:57:26 +08:00
chrislu	7b78fc72b0	add page chunk interval list	2021-12-22 02:53:33 -08:00
chrislu	b7cd52636b	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2021-12-21 22:24:43 -08:00
chrislu	3981d65b68	remove println	2021-12-21 22:24:38 -08:00
chrislu	9a73319b45	mount: different write strategy for streaming write and random write	2021-12-21 17:28:55 -08:00
Chris LuandGitHub	b0665a15f4	Merge pull request #2527 from banjiaojuhao/master-assign-by-datanode	2021-12-21 08:56:51 -08:00
banjiaojuhao	dda6b90d25	assign fileId according to DataNode with empty DataCenter and Rack	2021-12-21 17:28:33 +08:00
chrislu	4b8dcff448	reverting default admin scripts fix https://github.com/chrislusf/seaweedfs/issues/2525 this new default value was introduced in 2.80 this affects production environments, e.g., EC is not desired, volume balancing is not preferred, etc.	2021-12-20 13:34:57 -08:00
chrislu	b21a67bbe6	add writer pattern object for later use	2021-12-20 11:53:48 -08:00
chrislu	4fd29dad86	remove writeOnly flag	2021-12-20 01:11:43 -08:00
chrislu	bc96682760	refactor, change file locations	2021-12-20 01:02:23 -08:00
chrislu	866c2657f0	avoid FUSE cache only for the first 512 bytes	2021-12-19 23:13:36 -08:00
chrislu	0cb9036f66	mount: only cache the first chunk on stream read	2021-12-19 23:06:03 -08:00
chrislu	a152f17937	mount: improve read performance on random reads	2021-12-19 22:43:14 -08:00
chrislu	85c526c583	s3: bind to a specific IP fix https://github.com/chrislusf/seaweedfs/issues/2516	2021-12-17 11:34:37 -08:00
Chris LuandGitHub	e526a299dc	Update FUNDING.yml	2021-12-16 11:07:53 -08:00
Chris LuandGitHub	182fb9e3ed	Create FUNDING.yml	2021-12-16 11:02:32 -08:00
chrislu	5eacff9d4f	log message adds server name address https://github.com/chrislusf/seaweedfs/issues/2514#issuecomment-995925733	2021-12-16 10:46:26 -08:00
chrislu	50ddd8c8e2	remove debug messages fix https://github.com/chrislusf/seaweedfs/issues/2514	2021-12-16 00:58:15 -08:00
chrislu	8329cf86a7	Merge branch 'content_disposition_download'	2021-12-15 13:19:40 -08:00
chrislu	7210558c7b	s3: pass through s3 presigned headers fix https://github.com/chrislusf/seaweedfs/discussions/2502	2021-12-15 13:18:53 -08:00
Chris LuandGitHub	c63fff9a10	Merge pull request #2510 from kmlebedev/helm_auditLogConfig helm add auditLogConfig option	2021-12-15 11:28:02 -08:00
chrislu	bf4d7affc0	gateway to remote object store: skip replicating multipart upload part files fix https://github.com/chrislusf/seaweedfs/issues/2509	2021-12-14 19:48:31 -08:00
chrislu	8dca561bd1	adjust the name to be consistent comes from https://github.com/chrislusf/seaweedfs/pull/2511	2021-12-14 19:39:08 -08:00
Chris LuandGitHub	f0492ac039	Merge pull request #2511 from ErikCoder07/grafana_dashboard feat: Added a volume heartbeat graph to grafana dashboard	2021-12-14 19:35:14 -08:00
ErikCoder07	d8ffeb4d0e	minor changes: added the makefile directory in the Installation Guide section	2021-12-14 21:25:16 -03:00
ErikCoder07	560a057941	feat: Added a volume heartbeat graph to grafana dashboard In the current dashboard there is no way to know the status of volume servers, as prometheus only accepts push information, if any server volume goes offline, the last information sent will be read continuously by grafana. Added a graph that calculates the delta between the information of the last push timestamp with the timestamp of 15 seconds ago, if the timestamp has not been changed, server state will be 0, if volume server sent any information the server state will be 1. -This graph will have to be modified if the time interval of pushs is modified, a suggestion for improvement is to send the interval in seconds along with the metrics. -Only local tests were run, maybe if the latency between pushgateway and volume server is too high, the delta calculation could be affected.	2021-12-14 20:24:00 -03:00
Konstantin Lebedev	761ffdc405	helm add auditLogConfig option	2021-12-14 18:05:22 +05:00
chrislu	316f326464	add more help message	2021-12-13 13:14:36 -08:00
Chris LuandGitHub	adfc1e0559	Merge pull request #2508 from kmlebedev/s3StatusOffAudit disable audit log for s3 statusHandler	2021-12-13 10:19:06 -08:00
Konstantin Lebedev	969f513265	disable audit log for s3 statusHandler	2021-12-13 16:08:19 +05:00
Chris LuandGitHub	ab0c6a84bf	Merge pull request #2503 from kmlebedev/audit_log_nonblocking Audit log force async	2021-12-13 00:47:16 -08:00
Konstantin Lebedev	34779e8f38	force enable asynchronous I/O sending events to Fluentd	2021-12-13 13:39:39 +05:00
chrislu	c3b73ec23b	2.82	2021-12-12 23:25:24 -08:00
chrislu	7bf14452f7	s3: handle Response-Content-Disposition	2021-12-12 23:05:40 -08:00
chrislu	94e5c0a454	skip purging from ec volumes	2021-12-12 22:55:27 -08:00
chrislu	a2d3f89c7b	add lock messages	2021-12-10 13:24:38 -08:00
chrislu	991a3dca0d	rename file	2021-12-10 13:11:00 -08:00
kmlebedev	4f98553ba9	audit log SignatureVersion	2021-12-10 19:40:32 +05:00
chrislu	2d6fcdf83a	add delay if need to resume from disk	2021-12-09 13:13:05 -08:00
Chris LuandGitHub	faf84d8fb6	Merge pull request #2500 from paochiang/rename	2021-12-09 08:52:35 -08:00
liubaojiang	c7abc34dbb	rename operation returns the correct old parent path	2021-12-10 00:02:57 +08:00
Konstantin Lebedev	98251fe16a	non blocking audit log	2021-12-09 19:47:16 +05:00
chrislu	40ee9b4d7b	sleep between the gap of each execution related to https://github.com/chrislusf/seaweedfs/issues/2476	2021-12-07 11:44:08 -08:00
Chris LuandGitHub	2ba08afed1	Merge pull request #2498 from kmlebedev/s3_audit_log	2021-12-07 09:35:48 -08:00
Konstantin Lebedev	10678cde81	audit log config	2021-12-07 18:20:52 +05:00
Konstantin Lebedev	4ec8715f20	audit log	2021-12-07 12:15:48 +05:00
Konstantin Lebedev	3ac48cd540	audit log	2021-12-07 12:13:21 +05:00
chrislu	7f0a97c7b6	shell: volume.list supports different verbosity level	2021-12-05 21:54:40 -08:00
chrislu	5ea9715721	2.81 also sync java client version to SeaweedFS version	2021-12-05 18:05:24 -08:00
chrislu	df9a72b87f	fix compilation	2021-12-05 17:24:13 -08:00
chrislu	c7c60d1f8a	Java: add retry on write	2021-12-05 17:15:01 -08:00
chrislu	e6c026db65	volume.fix.replication: fix misplaced volumes fix https://github.com/chrislusf/seaweedfs/issues/2416	2021-12-05 16:56:25 -08:00
chrislu	53e2dee177	[volume.check.disk] was using the wrong source and target locations fix https://github.com/chrislusf/seaweedfs/issues/2268	2021-12-05 14:32:04 -08:00
chrislu	c146c76d10	avoid creating the same bucket with a different randomized name related to https://github.com/chrislusf/seaweedfs/issues/2492	2021-12-05 13:06:41 -08:00
chrislu	42d97a3442	adjust randomized bucket name	2021-12-05 12:36:58 -08:00
chrislu	488afa5002	volume: load volume can optionally be skipped, if ec volume exists fix https://github.com/chrislusf/seaweedfs/issues/2489	2021-12-05 02:28:52 -08:00
chrislu	73e3e77904	deprecate all cronjob module related code related to https://github.com/chrislusf/seaweedfs/issues/2483	2021-12-05 01:20:47 -08:00
chrislu	59e58c4b23	volume: fix loading old volume format fix https://github.com/chrislusf/seaweedfs/issues/2487	2021-12-05 01:06:01 -08:00
chrislu	b70cb3e0b2	upgrade protoc to 3.17.3 $brew install protobuf $ protoc --version libprotoc 3.17.3 $ go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26 $ go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1	2021-12-05 00:42:25 -08:00
chrislu	f7229084a9	upgrade go.mongodb.org/mongo-driver go.mongodb.org/mongo-driver@v1.7.0: retracted by module author: Contains data race bug in background connection establishment. go: to switch to the latest unretracted version	2021-12-04 23:48:55 -08:00
chrislu	5c3b783310	s3: copy object API needs to escape special characters fix https://github.com/chrislusf/seaweedfs/issues/2482	2021-12-04 23:24:53 -08:00
Chris LuandGitHub	9b0da7cd7d	Merge pull request #2488 from Bl1tz23/master MongodbStore.ListDirectoryEntries panics on Find method failure	2021-12-03 22:45:03 -08:00
Bl1tz23	83bffca7e6	fix MongodbStore.ListDirectoryEntries panic on find failure	2021-12-03 13:58:37 +03:00
Chris LuandGitHub	75e4013439	Merge pull request #2486 from t4nmoy/fix-filer-retry	2021-12-02 21:58:33 -08:00
Tanmoy Majumdar	ea09fb477a	return ' shouldRetry=true' so that filer can retry the failed chunk	2021-12-03 11:54:20 +06:00
Chris Lu	689f5513a9	redis3 supports sentinel	2021-11-29 01:09:51 -08:00