2.39

adjust help message
go fmt
2026-10-02 20:53:39 +00:00 · 2021-04-11 19:47:11 -07:00 · 2021-04-11 00:26:28 -07:00 · 2021-04-10 23:48:18 -07:00 · 2021-04-10 23:47:47 -07:00 · 2021-04-10 02:36:53 -07:00
123 changed files with 5642 additions and 1487 deletions
@@ -7,6 +7,7 @@
 [![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed)
 [![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki)
 [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
+[![SeaweedFS on Maven Central](https://img.shields.io/maven-central/v/com.github.chrislusf/seaweedfs-client)](https://search.maven.org/search?q=g:com.github.chrislusf)


 ![SeaweedFS Logo](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/seaweedfs.png)
@@ -42,6 +43,7 @@ Your support will be really appreciated by me and other supporters!
 - [SeaweedFS on Twitter](https://twitter.com/SeaweedFS)
 - [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
 - [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
+- [SeaweedFS White Paper](https://github.com/chrislusf/seaweedfs/wiki/SeaweedFS_Architecture.pdf)
 - [SeaweedFS Introduction Slides](https://www.slideshare.net/chrislusf/seaweedfs-introduction)

 Table of Contents
@@ -70,7 +72,7 @@ Table of Contents
 * Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
 * Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.

-Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thoudsands of machines. That is it!
+Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!

 ## Introduction ##

@@ -0,0 +1,50 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=master -volumeSizeLimitMB=1024"
+  volume:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8080:8080
+      - 18080:18080
+    command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
+    depends_on:
+      - master
+  s3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8888:8888
+      - 18888:18888
+      - 8333:8333
+    command: '-v 1 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333'
+    volumes:
+      - ./s3.json:/etc/seaweedfs/s3.json
+    depends_on:
+      - master
+      - volume
+  minio-gateway-s3:
+    image: minio/minio
+    ports:
+      - 9000:9000
+    command: 'minio gateway s3 http://s3:8333'
+    restart: on-failure
+    environment:
+      MINIO_ACCESS_KEY: "some_access_key1"
+      MINIO_SECRET_KEY: "some_secret_key1"
+    depends_on:
+      - s3
+  minio-warp:
+    image: minio/warp
+    command: 'mixed --duration=5m --obj.size=3mb --autoterm'
+    restart: on-failure
+    environment:
+      WARP_HOST: "minio-gateway-s3:9000"
+      WARP_ACCESS_KEY: "some_access_key1"
+      WARP_SECRET_KEY: "some_secret_key1"
+    depends_on:
+      - minio-gateway-s3
@@ -15,24 +15,18 @@ services:
    command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
    depends_on:
      - master
-  filer:
+  s3:
    image: chrislusf/seaweedfs:local
    ports:
      - 8888:8888
      - 18888:18888
-    command: 'filer -master="master:9333"'
-    depends_on:
-      - master
-      - volume
-  s3:
-    image: chrislusf/seaweedfs:local
-    ports:
      - 8333:8333
-    command: '-v 9 s3 -filer="filer:8888"'
+    command: '-v 9 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333'
+    volumes:
+      - ./s3.json:/etc/seaweedfs/s3.json
    depends_on:
      - master
      - volume
-      - filer
  minio:
    image: minio/minio
    ports:
@@ -38,7 +38,7 @@ services:
      S3TEST_CONF: "s3tests.conf"
      NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
      NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
-      NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_write_cache_control|object_write_expires|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifmatch_failed|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
+      NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_write_cache_control|object_write_expires|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifmatch_failed|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
    depends_on:
      - master
      - volume
@@ -13,8 +13,9 @@ require (
 	github.com/Shopify/sarama v1.23.1
 	github.com/aws/aws-sdk-go v1.34.30
 	github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
+	github.com/bwmarrin/snowflake v0.3.0
 	github.com/cespare/xxhash v1.1.0
-	github.com/chrislusf/raft v1.0.4
+	github.com/chrislusf/raft v1.0.6
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/disintegration/imaging v1.6.2
@@ -38,7 +39,7 @@ require (
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/mux v1.7.4
 	github.com/gorilla/websocket v1.4.1 // indirect
-	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
 	github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
 	github.com/jcmturner/gofork v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.10
@@ -88,7 +89,7 @@ require (
 	gocloud.dev/pubsub/rabbitpubsub v0.20.0
 	golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
 	golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb
-	golang.org/x/sync v0.0.0-20200930132711-30421366ff76 // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd
 	golang.org/x/tools v0.0.0-20200608174601-1b747fd94509
 	google.golang.org/api v0.26.0
@@ -141,6 +141,8 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 h1:fUmDBbSvv1uOzo/t8WaxZMVb7BxJ8JECo5lGoR9c5bA=
 github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72/go.mod h1:OEE5igu/CDjGegM1Jn6ZMo7R6LlV/JChAkjfQQIRLpg=
+github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
+github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4=
@@ -153,6 +155,10 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chrislusf/raft v1.0.4 h1:THhbsVik2hxdE0/VXX834f64Wn9RzgVPp+E+XCWZdKM=
 github.com/chrislusf/raft v1.0.4/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68=
+github.com/chrislusf/raft v1.0.5 h1:g8GxKCSStfm0/bGBDpNEbmEXL6MJkpXX+NI0ksbX5D4=
+github.com/chrislusf/raft v1.0.5/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68=
+github.com/chrislusf/raft v1.0.6 h1:wunb85WWhMKhNRn7EmdIw35D4Lmew0ZJv8oYDizR/+Y=
+github.com/chrislusf/raft v1.0.6/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -959,6 +965,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200930132711-30421366ff76 h1:JnxiSYT3Nm0BT2a8CyvYyM6cnrWpidecD1UuSYbhKm0=
 golang.org/x/sync v0.0.0-20200930132711-30421366ff76/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "2.32"
-version: 2.32
+appVersion: "2.39"
+version: 2.39
@@ -4,7 +4,7 @@ global:
  registry: ""
  repository: ""
  imageName: chrislusf/seaweedfs
-  # imageTag: "2.32" - started using {.Chart.appVersion}
+  # imageTag: "2.39" - started using {.Chart.appVersion}
  imagePullPolicy: IfNotPresent
  imagePullSecrets: imagepullsecret
  restartPolicy: Always
@@ -5,7 +5,7 @@

    <groupId>com.github.chrislusf</groupId>
    <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.2</version>
+    <version>1.6.4</version>

    <parent>
        <groupId>org.sonatype.oss</groupId>
@@ -17,7 +17,7 @@
        <protobuf.version>3.9.1</protobuf.version>
        <!-- follow https://github.com/grpc/grpc-java -->
        <grpc.version>1.23.0</grpc.version>
-        <guava.version>28.0-jre</guava.version>
+        <guava.version>30.0-jre</guava.version>
    </properties>

    <dependencies>
@@ -5,7 +5,7 @@

    <groupId>com.github.chrislusf</groupId>
    <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.2</version>
+    <version>1.6.4</version>

    <parent>
        <groupId>org.sonatype.oss</groupId>
@@ -5,7 +5,7 @@

    <groupId>com.github.chrislusf</groupId>
    <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.2</version>
+    <version>1.6.4</version>

    <parent>
        <groupId>org.sonatype.oss</groupId>
@@ -3,6 +3,7 @@ package seaweedfs.client;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

+import java.nio.Buffer;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
@@ -34,7 +35,7 @@ public class ByteBufferPool {
    }

    public static synchronized void release(ByteBuffer obj) {
-        obj.clear();
+        ((Buffer)obj).clear();
        bufferList.add(0, obj);
    }

@@ -94,7 +94,7 @@ public class FilerClient extends FilerGrpcClient {
            return true;
        }
        File pathFile = new File(path);
-        String parent = pathFile.getParent();
+        String parent = pathFile.getParent().replace('\\','/');
        String name = pathFile.getName();

        mkdirs(parent, mode, uid, gid, userName, groupNames);
@@ -115,11 +115,11 @@ public class FilerClient extends FilerGrpcClient {
    public boolean mv(String oldPath, String newPath) {

        File oldPathFile = new File(oldPath);
-        String oldParent = oldPathFile.getParent();
+        String oldParent = oldPathFile.getParent().replace('\\','/');
        String oldName = oldPathFile.getName();

        File newPathFile = new File(newPath);
-        String newParent = newPathFile.getParent();
+        String newParent = newPathFile.getParent().replace('\\','/');
        String newName = newPathFile.getName();

        return atomicRenameEntry(oldParent, oldName, newParent, newName);
@@ -129,7 +129,7 @@ public class FilerClient extends FilerGrpcClient {
    public boolean rm(String path, boolean isRecursive, boolean ignoreRecusiveError) {

        File pathFile = new File(path);
-        String parent = pathFile.getParent();
+        String parent = pathFile.getParent().replace('\\','/');
        String name = pathFile.getName();

        return deleteEntry(
@@ -148,7 +148,7 @@ public class FilerClient extends FilerGrpcClient {
    public boolean touch(String path, int mode, int uid, int gid, String userName, String[] groupNames) {

        File pathFile = new File(path);
-        String parent = pathFile.getParent();
+        String parent = pathFile.getParent().replace('\\','/');
        String name = pathFile.getName();

        FilerProto.Entry entry = lookupEntry(parent, name);
@@ -11,13 +11,13 @@
        <dependency>
            <groupId>com.github.chrislusf</groupId>
            <artifactId>seaweedfs-client</artifactId>
-            <version>1.6.2</version>
+            <version>1.6.4</version>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>com.github.chrislusf</groupId>
            <artifactId>seaweedfs-hadoop2-client</artifactId>
-            <version>1.6.2</version>
+            <version>1.6.4</version>
            <scope>compile</scope>
        </dependency>
        <dependency>
@@ -0,0 +1,120 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>hdfs-over-ftp</groupId>
+    <artifactId>hdfs-over-ftp</artifactId>
+    <version>1.0</version>
+
+    <parent>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>2.4.3</version>
+    </parent>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.springfox</groupId>
+            <artifactId>springfox-swagger2</artifactId>
+            <version>2.9.2</version>
+        </dependency>
+        <dependency>
+            <groupId>io.springfox</groupId>
+            <artifactId>springfox-swagger-ui</artifactId>
+            <version>2.9.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <version>3.2.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-client</artifactId>
+            <version>3.2.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.ftpserver</groupId>
+            <artifactId>ftpserver-core</artifactId>
+            <version>1.1.1</version>
+        </dependency>
+        <dependency>
+            <groupId>com.github.chrislusf</groupId>
+            <artifactId>seaweedfs-hadoop3-client</artifactId>
+            <version>1.6.2</version>
+        </dependency>
+    </dependencies>
+
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.1</version>
+                <configuration>
+                    <source>1.8</source>
+                    <target>1.8</target>
+                    <encoding>UTF-8</encoding>
+                    <compilerArguments>
+                        <verbose />
+                        <bootclasspath>${java.home}/lib/rt.jar</bootclasspath>
+                    </compilerArguments>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <version>2.6</version>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <mainClass>org.apache.hadoop.seaweed.ftp.ApplicationServer</mainClass>
+                            <addClasspath>true</addClasspath>
+                            <classpathPrefix>lib/</classpathPrefix>
+                        </manifest>
+                        <manifestEntries>
+                            <Class-Path>./</Class-Path>
+                        </manifestEntries>
+                    </archive>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <appendAssemblyId>false</appendAssemblyId>
+                    <descriptors>
+                        <descriptor>src/main/resources/assembly.xml</descriptor>
+                    </descriptors>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
@@ -0,0 +1,14 @@
+package org.apache.hadoop.seaweed.ftp;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+
+@SpringBootApplication
+public class ApplicationServer {
+
+    public static void main(String[] args) {
+        SpringApplication.run(ApplicationServer.class, args);
+    }
+
+}
@@ -0,0 +1,27 @@
+package org.apache.hadoop.seaweed.ftp.config;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import springfox.documentation.builders.ApiInfoBuilder;
+import springfox.documentation.builders.PathSelectors;
+import springfox.documentation.builders.RequestHandlerSelectors;
+import springfox.documentation.spi.DocumentationType;
+import springfox.documentation.spring.web.plugins.Docket;
+import springfox.documentation.swagger2.annotations.EnableSwagger2;
+
+@Configuration
+@EnableSwagger2
+public class SwaggerConfig {
+    @Bean
+    public Docket createRestApi() {
+        return new Docket(DocumentationType.SWAGGER_2)
+                .pathMapping("/")
+                .select()
+                .apis(RequestHandlerSelectors.basePackage("org.apache.hadoop.seaweed.ftp"))
+                .paths(PathSelectors.any())
+                .build().apiInfo(new ApiInfoBuilder()
+                        .title("FTP API Doc")
+                        .version("1.0")
+                        .build());
+    }
+}
@@ -0,0 +1,71 @@
+package org.apache.hadoop.seaweed.ftp.controller;
+
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.apache.hadoop.seaweed.ftp.service.HFtpService;
+import org.apache.hadoop.seaweed.ftp.controller.vo.Result;
+import org.apache.log4j.Logger;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@RestController
+@RequestMapping("/manager")
+@Api(tags = "FTP操作管理")
+public class FtpManagerController {
+
+    private static Logger log = Logger.getLogger(FtpManagerController.class);
+
+    @Autowired
+    private HFtpService hdfsOverFtpServer;
+
+    @GetMapping("/status")
+    @ApiOperation("查看FTP服务状态")
+    public Result status() {
+        Map map = new HashMap<>();
+        try {
+            boolean status = hdfsOverFtpServer.statusServer();
+            map.put("is_running", status);
+            return new Result(true, map, "FTP 服务状态获取成功");
+        }catch (Exception e) {
+            log.error(e);
+            map.put("is_running", false);
+            return new Result(true, map, "FTP 服务状态获取成功");
+        }
+    }
+
+    @PutMapping("/start")
+    @ApiOperation("启动FTP服务")
+    public Result start() {
+        try {
+            boolean status = hdfsOverFtpServer.statusServer();
+            if(!status) {
+                hdfsOverFtpServer.startServer();
+            }
+            return new Result(true, "FTP 服务启动成功");
+        }catch (Exception e) {
+            log.error(e);
+            return new Result(false, "FTP 服务启动失败");
+        }
+    }
+
+    @PutMapping("/stop")
+    @ApiOperation("停止FTP服务")
+    public Result stop() {
+        try {
+            boolean status = hdfsOverFtpServer.statusServer();
+            if(status) {
+                hdfsOverFtpServer.stopServer();
+            }
+            return new Result(true, "FTP 服务停止成功");
+        }catch (Exception e) {
+            log.error(e);
+            return new Result(false, "FTP 服务停止失败");
+        }
+    }
+}
@@ -0,0 +1,98 @@
+package org.apache.hadoop.seaweed.ftp.controller;
+
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.apache.ftpserver.ftplet.User;
+import org.apache.ftpserver.usermanager.Md5PasswordEncryptor;
+import org.apache.ftpserver.usermanager.UserFactory;
+import org.apache.hadoop.seaweed.ftp.controller.vo.FtpUser;
+import org.apache.hadoop.seaweed.ftp.controller.vo.Result;
+import org.apache.hadoop.seaweed.ftp.users.HdfsUserManager;
+import org.apache.log4j.Logger;
+import org.springframework.web.bind.annotation.*;
+
+import java.io.File;
+
+@RestController
+@RequestMapping("/user")
+@Api(tags = "FTP用户管理")
+public class UserController {
+
+    private static Logger log = Logger.getLogger(UserController.class);
+
+    /***
+     * {
+     *     "name": "test",
+     *     "password": "test",
+     *     "homeDirectory": "/buckets/test/"
+     * }
+     * @param ftpUser
+     * @return
+     */
+    @PostMapping("/add")
+    @ApiOperation("新增/编辑用户")
+    public Result add(@RequestBody FtpUser ftpUser) {
+        try {
+            HdfsUserManager userManagerFactory = new HdfsUserManager();
+            userManagerFactory.setFile(new File(System.getProperty("user.dir") + File.separator + "users.properties"));
+            userManagerFactory.setPasswordEncryptor(new Md5PasswordEncryptor());
+
+            UserFactory userFactory = new UserFactory();
+            userFactory.setHomeDirectory(ftpUser.getHomeDirectory());
+            userFactory.setName(ftpUser.getName());
+            userFactory.setPassword(ftpUser.getPassword());
+            userFactory.setEnabled(ftpUser.isEnabled());
+            userFactory.setMaxIdleTime(ftpUser.getMaxIdleTime());
+
+            User user = userFactory.createUser();
+            userManagerFactory.save(user, ftpUser.isRenamePush());
+            return new Result(true, "新建用户成功");
+        }catch (Exception e) {
+            log.error(e);
+            return new Result(false, "新建用户失败");
+        }
+    }
+
+    @DeleteMapping("/delete/{user}")
+    @ApiOperation("删除用户")
+    public Result delete(@PathVariable(value = "user") String user) {
+        try {
+            HdfsUserManager userManagerFactory = new HdfsUserManager();
+            userManagerFactory.setFile(new File(System.getProperty("user.dir") + File.separator + "users.properties"));
+            userManagerFactory.delete(user);
+            return new Result(true, "删除用户成功");
+        }catch (Exception e) {
+            log.error(e);
+            return new Result(false, "删除用户失败");
+        }
+    }
+
+    @GetMapping("/show/{userName}")
+    @ApiOperation("查看用户")
+    public Result show(@PathVariable(value = "userName") String userName) {
+        try {
+            HdfsUserManager userManagerFactory = new HdfsUserManager();
+            userManagerFactory.setFile(new File(System.getProperty("user.dir") + File.separator + "users.properties"));
+            User user = userManagerFactory.getUserByName(userName);
+            FtpUser ftpUser = new FtpUser(user.getHomeDirectory(), user.getPassword(), user.getEnabled(), user.getName(), user.getMaxIdleTime(), HdfsUserManager.getUserRenamePush(userName));
+            return new Result(true, ftpUser, "获取用户信息成功");
+        }catch (Exception e) {
+            log.error(e);
+            return new Result(false, "获取用户信息失败");
+        }
+    }
+
+    @GetMapping("/list")
+    @ApiOperation("列举用户")
+    public Result list() {
+        try {
+            HdfsUserManager userManagerFactory = new HdfsUserManager();
+            userManagerFactory.setFile(new File(System.getProperty("user.dir") + File.separator + "users.properties"));
+            String[] allUserNames = userManagerFactory.getAllUserNames();
+            return new Result(true, allUserNames, "列举用户成功");
+        }catch (Exception e) {
+            log.error(e);
+            return new Result(false, "列举用户失败");
+        }
+    }
+}
@@ -0,0 +1,71 @@
+package org.apache.hadoop.seaweed.ftp.controller.vo;
+
+public class FtpUser {
+
+    private String homeDirectory;
+    private String password;
+    private boolean enabled;
+    private String name;
+    private int maxIdleTime;
+    private boolean renamePush;
+
+    public FtpUser() {
+    }
+
+    public FtpUser(String homeDirectory, String password, boolean enabled, String name, int maxIdleTime, boolean renamePush) {
+        this.homeDirectory = homeDirectory;
+        this.password = password;
+        this.enabled = enabled;
+        this.name = name;
+        this.maxIdleTime = maxIdleTime;
+        this.renamePush = renamePush;
+    }
+
+    public String getHomeDirectory() {
+        return homeDirectory;
+    }
+
+    public void setHomeDirectory(String homeDirectory) {
+        this.homeDirectory = homeDirectory;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public boolean isEnabled() {
+        return enabled;
+    }
+
+    public void setEnabled(boolean enabled) {
+        this.enabled = enabled;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public int getMaxIdleTime() {
+        return maxIdleTime;
+    }
+
+    public void setMaxIdleTime(int maxIdleTime) {
+        this.maxIdleTime = maxIdleTime;
+    }
+
+    public boolean isRenamePush() {
+        return renamePush;
+    }
+
+    public void setRenamePush(boolean renamePush) {
+        this.renamePush = renamePush;
+    }
+}
@@ -0,0 +1,43 @@
+package org.apache.hadoop.seaweed.ftp.controller.vo;
+
+public class Result {
+
+    private boolean status;
+    private Object data;
+    private String message;
+
+    public Result(boolean status, String message) {
+        this.status = status;
+        this.message = message;
+    }
+
+    public Result(boolean status, Object data, String message) {
+        this.status = status;
+        this.message = message;
+        this.data = data;
+    }
+
+    public boolean isStatus() {
+        return status;
+    }
+
+    public void setStatus(boolean status) {
+        this.status = status;
+    }
+
+    public String getMessage() {
+        return message;
+    }
+
+    public void setMessage(String message) {
+        this.message = message;
+    }
+
+    public Object getData() {
+        return data;
+    }
+
+    public void setData(Object data) {
+        this.data = data;
+    }
+}
@@ -0,0 +1,102 @@
+package org.apache.hadoop.seaweed.ftp.service;
+
+import org.apache.ftpserver.DataConnectionConfiguration;
+import org.apache.ftpserver.DataConnectionConfigurationFactory;
+import org.apache.ftpserver.FtpServer;
+import org.apache.ftpserver.FtpServerFactory;
+import org.apache.ftpserver.command.CommandFactoryFactory;
+import org.apache.ftpserver.listener.ListenerFactory;
+import org.apache.hadoop.seaweed.ftp.service.filesystem.HdfsFileSystemManager;
+import org.apache.hadoop.seaweed.ftp.service.filesystem.HdfsOverFtpSystem;
+import org.apache.hadoop.seaweed.ftp.users.HdfsUserManager;
+import org.apache.log4j.Logger;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.io.File;
+
+/**
+ * reference: https://github.com/iponweb/hdfs-over-ftp
+ */
+@Component
+public class HFtpService {
+
+	private static Logger log = Logger.getLogger(HFtpService.class);
+
+	@Value("${ftp.port}")
+	private int port = 0;
+
+	@Value("${ftp.passive-address}")
+	private String passiveAddress;
+
+	@Value("${ftp.passive-ports}")
+	private String passivePorts;
+
+	@Value("${hdfs.uri}")
+	private String hdfsUri;
+
+	@Value("${seaweedFs.enable}")
+	private boolean seaweedFsEnable;
+
+	@Value("${seaweedFs.access}")
+	private String seaweedFsAccess;
+
+	@Value("${seaweedFs.replication}")
+	private String seaweedFsReplication;
+
+	private FtpServer ftpServer = null;
+
+	public void startServer() throws Exception {
+		log.info("Starting HDFS-Over-Ftp server. port: " + port + " passive-address: " + passiveAddress + " passive-ports: " + passivePorts + " hdfs-uri: " + hdfsUri);
+
+		HdfsOverFtpSystem.setHdfsUri(hdfsUri);
+		HdfsOverFtpSystem.setSeaweedFsEnable(seaweedFsEnable);
+		HdfsOverFtpSystem.setSeaweedFsAccess(seaweedFsAccess);
+		HdfsOverFtpSystem.setSeaweedFsReplication(seaweedFsReplication);
+
+		FtpServerFactory server = new FtpServerFactory();
+		server.setFileSystem(new HdfsFileSystemManager());
+
+		ListenerFactory factory = new ListenerFactory();
+		factory.setPort(port);
+
+		DataConnectionConfigurationFactory dccFactory = new DataConnectionConfigurationFactory();
+		dccFactory.setPassiveAddress("0.0.0.0");
+		dccFactory.setPassivePorts(passivePorts);
+		dccFactory.setPassiveExternalAddress(passiveAddress);
+		DataConnectionConfiguration dcc = dccFactory.createDataConnectionConfiguration();
+		factory.setDataConnectionConfiguration(dcc);
+
+		server.addListener("default", factory.createListener());
+
+		HdfsUserManager userManager = new HdfsUserManager();
+		final File file = loadResource("/users.properties");
+		userManager.setFile(file);
+		server.setUserManager(userManager);
+
+		CommandFactoryFactory cmFact = new CommandFactoryFactory();
+		cmFact.setUseDefaultCommands(true);
+		server.setCommandFactory(cmFact.createCommandFactory());
+
+		// start the server
+		ftpServer = server.createServer();
+		ftpServer.start();
+	}
+
+	public void stopServer() {
+		log.info("Stopping Hdfs-Over-Ftp server. port: " + port + " passive-address: " + passiveAddress + " passive-ports: " + passivePorts + " hdfs-uri: " + hdfsUri);
+		ftpServer.stop();
+	}
+
+	public boolean statusServer() {
+		try {
+			return !ftpServer.isStopped();
+		}catch (Exception e) {
+			return false;
+		}
+	}
+
+	private static File loadResource(String resourceName) {
+		return new File(System.getProperty("user.dir") + resourceName);
+	}
+}
@@ -0,0 +1,333 @@
+package org.apache.hadoop.seaweed.ftp.service.filesystem;
+
+import org.apache.ftpserver.ftplet.FtpFile;
+import org.apache.ftpserver.ftplet.User;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.seaweed.ftp.users.HdfsUser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * This class implements all actions to HDFS
+ */
+public class HdfsFileObject implements FtpFile {
+
+	private final Logger log = LoggerFactory.getLogger(HdfsFileObject.class);
+
+	private Path homePath;
+	private Path path;
+	private Path fullPath;
+	private HdfsUser user;
+
+	/**
+	 * Constructs HdfsFileObject from path
+	 *
+	 * @param path path to represent object
+	 * @param user accessor of the object
+	 */
+	public HdfsFileObject(String homePath, String path, User user) {
+		this.homePath = new Path(homePath);
+		this.path = new Path(path);
+		this.fullPath = new Path(homePath + path);
+		this.user = (HdfsUser) user;
+	}
+
+	public String getAbsolutePath() {
+		// strip the last '/' if necessary
+		String fullName = path.toString();
+		int filelen = fullName.length();
+		if ((filelen != 1) && (fullName.charAt(filelen - 1) == '/')) {
+			fullName = fullName.substring(0, filelen - 1);
+		}
+
+		return fullName;
+	}
+
+	public String getName() {
+		return path.getName();
+	}
+
+	/**
+	 * HDFS has no hidden objects
+	 *
+	 * @return always false
+	 */
+	public boolean isHidden() {
+		return false;
+	}
+
+	/**
+	 * Checks if the object is a directory
+	 *
+	 * @return true if the object is a directory
+	 */
+	public boolean isDirectory() {
+		try {
+			log.debug("is directory? : " + fullPath);
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			FileStatus fs = dfs.getFileStatus(fullPath);
+			return fs.isDir();
+		} catch (IOException e) {
+			log.debug(fullPath + " is not dir", e);
+			return false;
+		}
+	}
+
+	/**
+	 * Checks if the object is a file
+	 *
+	 * @return true if the object is a file
+	 */
+	public boolean isFile() {
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			return dfs.isFile(fullPath);
+		} catch (IOException e) {
+			log.debug(fullPath + " is not file", e);
+			return false;
+		}
+	}
+
+	/**
+	 * Checks if the object does exist
+	 *
+	 * @return true if the object does exist
+	 */
+	public boolean doesExist() {
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			dfs.getFileStatus(fullPath);
+			return true;
+		} catch (IOException e) {
+			//   log.debug(path + " does not exist", e);
+			return false;
+		}
+	}
+
+	public boolean isReadable() {
+		return true;
+	}
+
+	public boolean isWritable() {
+		return true;
+	}
+
+	public boolean isRemovable() {
+		return true;
+	}
+
+	/**
+	 * Get owner of the object
+	 *
+	 * @return owner of the object
+	 */
+	public String getOwnerName() {
+		return "root";
+		/*
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			FileStatus fs = dfs.getFileStatus(fullPath);
+			String owner = fs.getOwner();
+			if(owner.length() == 0) {
+				return "root";
+			}
+			return owner;
+		} catch (IOException e) {
+			e.printStackTrace();
+			return null;
+		}
+		 */
+	}
+
+	/**
+	 * Get group of the object
+	 *
+	 * @return group of the object
+	 */
+	public String getGroupName() {
+		return "root";
+		/*
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			FileStatus fs = dfs.getFileStatus(fullPath);
+			String group = fs.getGroup();
+			if(group.length() == 0) {
+				return "root";
+			}
+			return group;
+		} catch (IOException e) {
+			e.printStackTrace();
+			return null;
+		}
+		 */
+	}
+
+	/**
+	 * Get link count
+	 *
+	 * @return 3 is for a directory and 1 is for a file
+	 */
+	public int getLinkCount() {
+		return isDirectory() ? 3 : 1;
+	}
+
+	/**
+	 * Get last modification date
+	 *
+	 * @return last modification date as a long
+	 */
+	public long getLastModified() {
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			FileStatus fs = dfs.getFileStatus(fullPath);
+			return fs.getModificationTime();
+		} catch (IOException e) {
+			e.printStackTrace();
+			return 0;
+		}
+	}
+
+	public boolean setLastModified(long l) {
+		return false;
+	}
+
+	/**
+	 * Get a size of the object
+	 *
+	 * @return size of the object in bytes
+	 */
+	public long getSize() {
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			FileStatus fs = dfs.getFileStatus(fullPath);
+			log.debug("getSize(): " + fullPath + " : " + fs.getLen());
+			return fs.getLen();
+		} catch (IOException e) {
+			e.printStackTrace();
+			return 0;
+		}
+	}
+
+	public Object getPhysicalFile() {
+		return null;
+	}
+
+	/**
+	 * Create a new dir from the object
+	 *
+	 * @return true if dir is created
+	 */
+	public boolean mkdir() {
+		try {
+			FileSystem fs = HdfsOverFtpSystem.getDfs();
+			fs.mkdirs(fullPath);
+//			fs.setOwner(path, user.getName(), user.getMainGroup());
+			return true;
+		} catch (IOException e) {
+			e.printStackTrace();
+			return false;
+		}
+	}
+
+	/**
+	 * Delete object from the HDFS filesystem
+	 *
+	 * @return true if the object is deleted
+	 */
+	public boolean delete() {
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			dfs.delete(fullPath, true);
+			return true;
+		} catch (IOException e) {
+			e.printStackTrace();
+			return false;
+		}
+	}
+
+	public boolean move(FtpFile ftpFile) {
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			dfs.rename(fullPath, new Path(fullPath.getParent() + File.separator + ftpFile.getName()));
+			return true;
+		} catch (IOException e) {
+			e.printStackTrace();
+			return false;
+		}
+	}
+
+
+	/**
+	 * List files of the directory
+	 *
+	 * @return List of files in the directory
+	 */
+	public List<FtpFile> listFiles() {
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			FileStatus fileStats[] = dfs.listStatus(fullPath);
+
+			// get the virtual name of the base directory
+			String virtualFileStr = getAbsolutePath();
+			if (virtualFileStr.charAt(virtualFileStr.length() - 1) != '/') {
+				virtualFileStr += '/';
+			}
+
+			FtpFile[] virtualFiles = new FtpFile[fileStats.length];
+			for (int i = 0; i < fileStats.length; i++) {
+				File fileObj = new File(fileStats[i].getPath().toString());
+				String fileName = virtualFileStr + fileObj.getName();
+				virtualFiles[i] = new HdfsFileObject(homePath.toString(), fileName, user);
+			}
+			return Collections.unmodifiableList(Arrays.asList(virtualFiles));
+		} catch (IOException e) {
+			log.debug("", e);
+			return null;
+		}
+	}
+
+	/**
+	 * Creates output stream to write to the object
+	 *
+	 * @param l is not used here
+	 * @return OutputStream
+	 * @throws IOException
+	 */
+	public OutputStream createOutputStream(long l) {
+		try {
+			FileSystem fs = HdfsOverFtpSystem.getDfs();
+			FSDataOutputStream out = fs.create(fullPath);
+//			fs.setOwner(fullPath, user.getName(), user.getMainGroup());
+			return out;
+		} catch (IOException e) {
+			e.printStackTrace();
+			return null;
+		}
+	}
+
+	/**
+	 * Creates input stream to read from the object
+	 *
+	 * @param l is not used here
+	 * @return OutputStream
+	 * @throws IOException
+	 */
+	public InputStream createInputStream(long l) {
+		try {
+			FileSystem dfs = HdfsOverFtpSystem.getDfs();
+			FSDataInputStream in = dfs.open(fullPath);
+			return in;
+		} catch (IOException e) {
+			e.printStackTrace();
+			return null;
+		}
+	}
+}
@@ -0,0 +1,14 @@
+package org.apache.hadoop.seaweed.ftp.service.filesystem;
+
+import org.apache.ftpserver.ftplet.FileSystemFactory;
+import org.apache.ftpserver.ftplet.FileSystemView;
+import org.apache.ftpserver.ftplet.User;
+
+/**
+ * Impelented FileSystemManager to use HdfsFileSystemView
+ */
+public class HdfsFileSystemManager implements FileSystemFactory {
+	public FileSystemView createFileSystemView(User user) {
+		return new HdfsFileSystemView(user);
+	}
+}
@@ -0,0 +1,104 @@
+package org.apache.hadoop.seaweed.ftp.service.filesystem;
+
+import org.apache.ftpserver.ftplet.FileSystemView;
+import org.apache.ftpserver.ftplet.FtpFile;
+import org.apache.ftpserver.ftplet.User;
+import org.apache.hadoop.fs.Path;
+
+import java.io.File;
+
+/**
+ * Implemented FileSystemView to use HdfsFileObject
+ */
+public class HdfsFileSystemView implements FileSystemView {
+
+	private String homePath;
+	private String currPath = File.separator;
+	private User user;
+
+	/**
+	 * Constructor - set the user object.
+	 */
+	protected HdfsFileSystemView(User user) {
+		if (user == null) {
+			throw new IllegalArgumentException("user can not be null");
+		}
+		if (user.getHomeDirectory() == null) {
+			throw new IllegalArgumentException(
+					"User home directory can not be null");
+		}
+
+		this.homePath = user.getHomeDirectory();
+		this.user = user;
+	}
+
+	public FtpFile getHomeDirectory() {
+		return new HdfsFileObject(homePath, File.separator, user);
+	}
+
+	public FtpFile getWorkingDirectory() {
+		FtpFile fileObj;
+		if (currPath.equals(File.separator)) {
+			fileObj = new HdfsFileObject(homePath, File.separator, user);
+		} else {
+			fileObj = new HdfsFileObject(homePath, currPath, user);
+
+		}
+		return fileObj;
+	}
+
+	public boolean changeWorkingDirectory(String dir) {
+
+		Path path;
+		if (dir.startsWith(File.separator) || new Path(currPath).equals(new Path(dir))) {
+			path = new Path(dir);
+		} else if (currPath.length() > 1) {
+			path = new Path(currPath + File.separator + dir);
+		} else {
+			if(dir.startsWith("/")) {
+				path = new Path(dir);
+			}else {
+				path = new Path(File.separator + dir);
+			}
+		}
+
+		// 防止退回根目录
+		if (path.getName().equals("..")) {
+			path = new Path(File.separator);
+		}
+
+		HdfsFileObject file = new HdfsFileObject(homePath, path.toString(), user);
+		if (file.isDirectory()) {
+			currPath = path.toString();
+			return true;
+		} else {
+			return false;
+		}
+	}
+
+	public FtpFile getFile(String file) {
+		String path;
+		if (file.startsWith(File.separator)) {
+			path = file;
+		} else if (currPath.length() > 1) {
+			path = currPath + File.separator + file;
+		} else {
+			path = File.separator + file;
+		}
+		return new HdfsFileObject(homePath, path, user);
+	}
+
+	/**
+	 * Is the file content random accessible?
+	 */
+	public boolean isRandomAccessible() {
+		return true;
+	}
+
+	/**
+	 * Dispose file system view - does nothing.
+	 */
+	public void dispose() {
+	}
+
+}
@@ -0,0 +1,72 @@
+package org.apache.hadoop.seaweed.ftp.service.filesystem;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * Class to store DFS connection
+ */
+public class HdfsOverFtpSystem {
+
+	private static FileSystem fs = null;
+
+	private static String hdfsUri;
+
+	private static boolean seaweedFsEnable;
+
+	private static String seaweedFsAccess;
+
+	private static String seaweedFsReplication;
+
+	private final static Logger log = LoggerFactory.getLogger(HdfsOverFtpSystem.class);
+
+	private static void hdfsInit() throws IOException {
+		Configuration configuration = new Configuration();
+
+		configuration.set("fs.defaultFS", hdfsUri);
+		if(seaweedFsEnable) {
+			configuration.set("fs.seaweedfs.impl", "seaweed.hdfs.SeaweedFileSystem");
+			configuration.set("fs.seaweed.volume.server.access", seaweedFsAccess);
+			configuration.set("fs.seaweed.replication", seaweedFsReplication);
+		}
+		fs = FileSystem.get(configuration);
+		log.info("HDFS load success");
+	}
+
+	/**
+	 * Get dfs
+	 *
+	 * @return dfs
+	 * @throws IOException
+	 */
+	public static FileSystem getDfs() throws IOException {
+		if (fs == null) {
+			hdfsInit();
+		}
+		return fs;
+	}
+
+	public static void setHdfsUri(String hdfsUri) {
+		HdfsOverFtpSystem.hdfsUri = hdfsUri;
+	}
+
+	public static String getHdfsUri() {
+		return hdfsUri;
+	}
+
+	public static void setSeaweedFsEnable(boolean seaweedFsEnable) {
+		HdfsOverFtpSystem.seaweedFsEnable = seaweedFsEnable;
+	}
+
+	public static void setSeaweedFsAccess(String seaweedFsAccess) {
+		HdfsOverFtpSystem.seaweedFsAccess = seaweedFsAccess;
+	}
+
+	public static void setSeaweedFsReplication(String seaweedFsReplication) {
+		HdfsOverFtpSystem.seaweedFsReplication = seaweedFsReplication;
+	}
+}
@@ -0,0 +1,239 @@
+package org.apache.hadoop.seaweed.ftp.users;
+
+import org.apache.ftpserver.ftplet.Authority;
+import org.apache.ftpserver.ftplet.AuthorizationRequest;
+import org.apache.ftpserver.ftplet.User;
+import org.apache.log4j.Logger;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class HdfsUser implements User, Serializable {
+
+	private static final long serialVersionUID = -47371353779731294L;
+
+	private String name = null;
+
+	private String password = null;
+
+	private int maxIdleTimeSec = 0; // no limit
+
+	private String homeDir = null;
+
+	private boolean isEnabled = true;
+
+	private List<? extends Authority> authorities = new ArrayList<Authority>();
+
+	private ArrayList<String> groups = new ArrayList<String>();
+
+	private Logger log = Logger.getLogger(HdfsUser.class);
+
+	/**
+	 * Default constructor.
+	 */
+	public HdfsUser() {
+	}
+
+	/**
+	 * Copy constructor.
+	 */
+	public HdfsUser(User user) {
+		name = user.getName();
+		password = user.getPassword();
+		authorities = user.getAuthorities();
+		maxIdleTimeSec = user.getMaxIdleTime();
+		homeDir = user.getHomeDirectory();
+		isEnabled = user.getEnabled();
+	}
+
+	public ArrayList<String> getGroups() {
+		return groups;
+	}
+
+	/**
+	 * Get the main group of the user
+	 *
+	 * @return main group of the user
+	 */
+	public String getMainGroup() {
+		if (groups.size() > 0) {
+			return groups.get(0);
+		} else {
+			log.error("User " + name + " is not a memer of any group");
+			return "error";
+		}
+	}
+
+	/**
+	 * Checks if user is a member of the group
+	 *
+	 * @param group to check
+	 * @return true if the user id a member of the group
+	 */
+	public boolean isGroupMember(String group) {
+		for (String userGroup : groups) {
+			if (userGroup.equals(group)) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	/**
+	 * Set users' groups
+	 *
+	 * @param groups to set
+	 */
+	public void setGroups(ArrayList<String> groups) {
+		if (groups.size() < 1) {
+			log.error("User " + name + " is not a memer of any group");
+		}
+		this.groups = groups;
+	}
+
+	/**
+	 * Get the user name.
+	 */
+	public String getName() {
+		return name;
+	}
+
+	/**
+	 * Set user name.
+	 */
+	public void setName(String name) {
+		this.name = name;
+	}
+
+	/**
+	 * Get the user password.
+	 */
+	public String getPassword() {
+		return password;
+	}
+
+	/**
+	 * Set user password.
+	 */
+	public void setPassword(String pass) {
+		password = pass;
+	}
+
+	public List<Authority> getAuthorities() {
+		if (authorities != null) {
+			return Collections.unmodifiableList(authorities);
+		} else {
+			return null;
+		}
+	}
+
+	public void setAuthorities(List<Authority> authorities) {
+		if (authorities != null) {
+			this.authorities = Collections.unmodifiableList(authorities);
+		} else {
+			this.authorities = null;
+		}
+	}
+
+	/**
+	 * Get the maximum idle time in second.
+	 */
+	public int getMaxIdleTime() {
+		return maxIdleTimeSec;
+	}
+
+	/**
+	 * Set the maximum idle time in second.
+	 */
+	public void setMaxIdleTime(int idleSec) {
+		maxIdleTimeSec = idleSec;
+		if (maxIdleTimeSec < 0) {
+			maxIdleTimeSec = 0;
+		}
+	}
+
+	/**
+	 * Get the user enable status.
+	 */
+	public boolean getEnabled() {
+		return isEnabled;
+	}
+
+	/**
+	 * Set the user enable status.
+	 */
+	public void setEnabled(boolean enb) {
+		isEnabled = enb;
+	}
+
+	/**
+	 * Get the user home directory.
+	 */
+	public String getHomeDirectory() {
+		return homeDir;
+	}
+
+	/**
+	 * Set the user home directory.
+	 */
+	public void setHomeDirectory(String home) {
+		homeDir = home;
+	}
+
+	/**
+	 * String representation.
+	 */
+	public String toString() {
+		return name;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	public AuthorizationRequest authorize(AuthorizationRequest request) {
+		List<Authority> authorities = getAuthorities();
+
+		// check for no authorities at all
+		if (authorities == null) {
+			return null;
+		}
+
+		boolean someoneCouldAuthorize = false;
+		for (Authority authority : authorities) {
+			if (authority.canAuthorize(request)) {
+				someoneCouldAuthorize = true;
+
+				request = authority.authorize(request);
+
+				// authorization failed, return null
+				if (request == null) {
+					return null;
+				}
+			}
+
+		}
+
+		if (someoneCouldAuthorize) {
+			return request;
+		} else {
+			return null;
+		}
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	public List<Authority> getAuthorities(Class<? extends Authority> clazz) {
+		List<Authority> selected = new ArrayList<Authority>();
+
+		for (Authority authority : authorities) {
+			if (authority.getClass().equals(clazz)) {
+				selected.add(authority);
+			}
+		}
+
+		return selected;
+	}
+}
@@ -0,0 +1,453 @@
+package org.apache.hadoop.seaweed.ftp.users;
+
+import org.apache.ftpserver.FtpServerConfigurationException;
+import org.apache.ftpserver.ftplet.*;
+import org.apache.ftpserver.usermanager.*;
+import org.apache.ftpserver.usermanager.impl.*;
+import org.apache.ftpserver.util.BaseProperties;
+import org.apache.ftpserver.util.IoUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.*;
+
+public class HdfsUserManager extends AbstractUserManager {
+
+	private final Logger LOG = LoggerFactory
+			.getLogger(HdfsUserManager.class);
+
+	private final static String DEPRECATED_PREFIX = "FtpServer.user.";
+
+	private final static String PREFIX = "ftpserver.user.";
+
+	private static BaseProperties userDataProp;
+
+	private File userDataFile = new File("users.conf");
+
+	private boolean isConfigured = false;
+
+	private PasswordEncryptor passwordEncryptor = new Md5PasswordEncryptor();
+
+
+	/**
+	 * Retrieve the file used to load and store users
+	 *
+	 * @return The file
+	 */
+	public File getFile() {
+		return userDataFile;
+	}
+
+	/**
+	 * Set the file used to store and read users. Must be set before
+	 * {@link #configure()} is called.
+	 *
+	 * @param propFile A file containing users
+	 */
+	public void setFile(File propFile) {
+		if (isConfigured) {
+			throw new IllegalStateException("Must be called before configure()");
+		}
+
+		this.userDataFile = propFile;
+	}
+
+
+	/**
+	 * Retrieve the password encryptor used for this user manager
+	 *
+	 * @return The password encryptor. Default to {@link Md5PasswordEncryptor}
+	 *         if no other has been provided
+	 */
+	public PasswordEncryptor getPasswordEncryptor() {
+		return passwordEncryptor;
+	}
+
+
+	/**
+	 * Set the password encryptor to use for this user manager
+	 *
+	 * @param passwordEncryptor The password encryptor
+	 */
+	public void setPasswordEncryptor(PasswordEncryptor passwordEncryptor) {
+		this.passwordEncryptor = passwordEncryptor;
+	}
+
+
+	/**
+	 * Lazy init the user manager
+	 */
+	private void lazyInit() {
+		if (!isConfigured) {
+			configure();
+		}
+	}
+
+	/**
+	 * Configure user manager.
+	 */
+	public void configure() {
+		isConfigured = true;
+		try {
+			userDataProp = new BaseProperties();
+
+			if (userDataFile != null && userDataFile.exists()) {
+				FileInputStream fis = null;
+				try {
+					fis = new FileInputStream(userDataFile);
+					userDataProp.load(fis);
+				} finally {
+					IoUtils.close(fis);
+				}
+			}
+		} catch (IOException e) {
+			throw new FtpServerConfigurationException(
+					"Error loading user data file : "
+							+ userDataFile.getAbsolutePath(), e);
+		}
+
+		convertDeprecatedPropertyNames();
+	}
+
+	private void convertDeprecatedPropertyNames() {
+		Enumeration<?> keys = userDataProp.propertyNames();
+
+		boolean doSave = false;
+
+		while (keys.hasMoreElements()) {
+			String key = (String) keys.nextElement();
+
+			if (key.startsWith(DEPRECATED_PREFIX)) {
+				String newKey = PREFIX
+						+ key.substring(DEPRECATED_PREFIX.length());
+				userDataProp.setProperty(newKey, userDataProp.getProperty(key));
+				userDataProp.remove(key);
+
+				doSave = true;
+			}
+		}
+
+		if (doSave) {
+			try {
+				saveUserData();
+			} catch (FtpException e) {
+				throw new FtpServerConfigurationException(
+						"Failed to save updated user data", e);
+			}
+		}
+	}
+
+	public synchronized void save(User usr, boolean renamePush) throws FtpException {
+		lazyInit();
+		userDataProp.setProperty(PREFIX + usr.getName() + ".rename.push", renamePush);
+		save(usr);
+	}
+
+	/**
+	 * Save user data. Store the properties.
+	 */
+	public synchronized void save(User usr) throws FtpException {
+		lazyInit();
+
+		// null value check
+		if (usr.getName() == null) {
+			throw new NullPointerException("User name is null.");
+		}
+		String thisPrefix = PREFIX + usr.getName() + '.';
+
+		// set other properties
+		userDataProp.setProperty(thisPrefix + ATTR_PASSWORD, getPassword(usr));
+
+		String home = usr.getHomeDirectory();
+		if (home == null) {
+			home = "/";
+		}
+		userDataProp.setProperty(thisPrefix + ATTR_HOME, home);
+		userDataProp.setProperty(thisPrefix + ATTR_ENABLE, usr.getEnabled());
+		userDataProp.setProperty(thisPrefix + ATTR_WRITE_PERM, usr
+				.authorize(new WriteRequest()) != null);
+		userDataProp.setProperty(thisPrefix + ATTR_MAX_IDLE_TIME, usr
+				.getMaxIdleTime());
+
+		TransferRateRequest transferRateRequest = new TransferRateRequest();
+		transferRateRequest = (TransferRateRequest) usr
+				.authorize(transferRateRequest);
+
+		if (transferRateRequest != null) {
+			userDataProp.setProperty(thisPrefix + ATTR_MAX_UPLOAD_RATE,
+					transferRateRequest.getMaxUploadRate());
+			userDataProp.setProperty(thisPrefix + ATTR_MAX_DOWNLOAD_RATE,
+					transferRateRequest.getMaxDownloadRate());
+		} else {
+			userDataProp.remove(thisPrefix + ATTR_MAX_UPLOAD_RATE);
+			userDataProp.remove(thisPrefix + ATTR_MAX_DOWNLOAD_RATE);
+		}
+
+		// request that always will succeed
+		ConcurrentLoginRequest concurrentLoginRequest = new ConcurrentLoginRequest(
+				0, 0);
+		concurrentLoginRequest = (ConcurrentLoginRequest) usr
+				.authorize(concurrentLoginRequest);
+
+		if (concurrentLoginRequest != null) {
+			userDataProp.setProperty(thisPrefix + ATTR_MAX_LOGIN_NUMBER,
+					concurrentLoginRequest.getMaxConcurrentLogins());
+			userDataProp.setProperty(thisPrefix + ATTR_MAX_LOGIN_PER_IP,
+					concurrentLoginRequest.getMaxConcurrentLoginsPerIP());
+		} else {
+			userDataProp.remove(thisPrefix + ATTR_MAX_LOGIN_NUMBER);
+			userDataProp.remove(thisPrefix + ATTR_MAX_LOGIN_PER_IP);
+		}
+
+		saveUserData();
+	}
+
+	/**
+	 * @throws FtpException
+	 */
+	private void saveUserData() throws FtpException {
+		File dir = userDataFile.getAbsoluteFile().getParentFile();
+		if (dir != null && !dir.exists() && !dir.mkdirs()) {
+			String dirName = dir.getAbsolutePath();
+			throw new FtpServerConfigurationException(
+					"Cannot create directory for user data file : " + dirName);
+		}
+
+		// save user data
+		FileOutputStream fos = null;
+		try {
+			fos = new FileOutputStream(userDataFile);
+			userDataProp.store(fos, "Generated file - don't edit (please)");
+		} catch (IOException ex) {
+			LOG.error("Failed saving user data", ex);
+			throw new FtpException("Failed saving user data", ex);
+		} finally {
+			IoUtils.close(fos);
+		}
+	}
+
+
+	public synchronized void list() throws FtpException {
+		lazyInit();
+
+		Map dataMap = new HashMap();
+		Enumeration<String> propNames = (Enumeration<String>) userDataProp.propertyNames();
+		ArrayList<String> a = Collections.list(propNames);
+		a.remove("i18nMap");//去除i18nMap
+		for(String attrName : a){
+//			dataMap.put(attrName, propNames.);
+		}
+
+	}
+
+	/**
+	 * Delete an user. Removes all this user entries from the properties. After
+	 * removing the corresponding from the properties, save the data.
+	 */
+	public synchronized void delete(String usrName) throws FtpException {
+		lazyInit();
+
+		// remove entries from properties
+		String thisPrefix = PREFIX + usrName + '.';
+		Enumeration<?> propNames = userDataProp.propertyNames();
+		ArrayList<String> remKeys = new ArrayList<String>();
+		while (propNames.hasMoreElements()) {
+			String thisKey = propNames.nextElement().toString();
+			if (thisKey.startsWith(thisPrefix)) {
+				remKeys.add(thisKey);
+			}
+		}
+		Iterator<String> remKeysIt = remKeys.iterator();
+		while (remKeysIt.hasNext()) {
+			userDataProp.remove(remKeysIt.next());
+		}
+
+		saveUserData();
+	}
+
+	/**
+	 * Get user password. Returns the encrypted value.
+	 * <p/>
+	 * <pre>
+	 * If the password value is not null
+	 *    password = new password
+	 * else
+	 *   if user does exist
+	 *     password = old password
+	 *   else
+	 *     password = &quot;&quot;
+	 * </pre>
+	 */
+	private String getPassword(User usr) {
+		String name = usr.getName();
+		String password = usr.getPassword();
+
+		if (password != null) {
+			password = passwordEncryptor.encrypt(password);
+		} else {
+			String blankPassword = passwordEncryptor.encrypt("");
+
+			if (doesExist(name)) {
+				String key = PREFIX + name + '.' + ATTR_PASSWORD;
+				password = userDataProp.getProperty(key, blankPassword);
+			} else {
+				password = blankPassword;
+			}
+		}
+		return password;
+	}
+
+	/**
+	 * Get all user names.
+	 */
+	public synchronized String[] getAllUserNames() {
+		lazyInit();
+
+		// get all user names
+		String suffix = '.' + ATTR_HOME;
+		ArrayList<String> ulst = new ArrayList<String>();
+		Enumeration<?> allKeys = userDataProp.propertyNames();
+		int prefixlen = PREFIX.length();
+		int suffixlen = suffix.length();
+		while (allKeys.hasMoreElements()) {
+			String key = (String) allKeys.nextElement();
+			if (key.endsWith(suffix)) {
+				String name = key.substring(prefixlen);
+				int endIndex = name.length() - suffixlen;
+				name = name.substring(0, endIndex);
+				ulst.add(name);
+			}
+		}
+
+		Collections.sort(ulst);
+		return ulst.toArray(new String[0]);
+	}
+
+	private ArrayList<String> parseGroups(String groupsLine) {
+		String groupsArray[] = groupsLine.split(",");
+		return new ArrayList(Arrays.asList(groupsArray));
+	}
+
+	public static synchronized boolean getUserRenamePush(String userName) {
+		return userDataProp.getBoolean(PREFIX + userName + ".rename.push", false);
+	}
+
+	/**
+	 * Load user data.
+	 */
+	public synchronized User getUserByName(String userName) {
+		lazyInit();
+
+		if (!doesExist(userName)) {
+			return null;
+		}
+
+		String baseKey = PREFIX + userName + '.';
+		HdfsUser user = new HdfsUser();
+		user.setName(userName);
+		user.setEnabled(userDataProp.getBoolean(baseKey + ATTR_ENABLE, true));
+		user.setHomeDirectory(userDataProp
+				.getProperty(baseKey + ATTR_HOME, "/"));
+
+//		user.setGroups(parseGroups(userDataProp
+//				.getProperty(baseKey + "groups")));
+
+		List<Authority> authorities = new ArrayList<Authority>();
+
+		if (userDataProp.getBoolean(baseKey + ATTR_WRITE_PERM, false)) {
+			authorities.add(new WritePermission());
+		}
+
+		int maxLogin = userDataProp.getInteger(baseKey + ATTR_MAX_LOGIN_NUMBER,
+				0);
+		int maxLoginPerIP = userDataProp.getInteger(baseKey
+				+ ATTR_MAX_LOGIN_PER_IP, 0);
+
+		authorities.add(new ConcurrentLoginPermission(maxLogin, maxLoginPerIP));
+
+		int uploadRate = userDataProp.getInteger(
+				baseKey + ATTR_MAX_UPLOAD_RATE, 0);
+		int downloadRate = userDataProp.getInteger(baseKey
+				+ ATTR_MAX_DOWNLOAD_RATE, 0);
+
+		authorities.add(new TransferRatePermission(downloadRate, uploadRate));
+
+		user.setAuthorities(authorities);
+
+		user.setMaxIdleTime(userDataProp.getInteger(baseKey
+				+ ATTR_MAX_IDLE_TIME, 0));
+
+		return user;
+	}
+
+	/**
+	 * User existance check
+	 */
+	public synchronized boolean doesExist(String name) {
+		lazyInit();
+
+		String key = PREFIX + name + '.' + ATTR_HOME;
+		return userDataProp.containsKey(key);
+	}
+
+	/**
+	 * User authenticate method
+	 */
+	public synchronized User authenticate(Authentication authentication)
+			throws AuthenticationFailedException {
+		lazyInit();
+
+		if (authentication instanceof UsernamePasswordAuthentication) {
+			UsernamePasswordAuthentication upauth = (UsernamePasswordAuthentication) authentication;
+
+			String user = upauth.getUsername();
+			String password = upauth.getPassword();
+
+			if (user == null) {
+				throw new AuthenticationFailedException("Authentication failed");
+			}
+
+			if (password == null) {
+				password = "";
+			}
+
+			String storedPassword = userDataProp.getProperty(PREFIX + user + '.'
+					+ ATTR_PASSWORD);
+
+			if (storedPassword == null) {
+				// user does not exist
+				throw new AuthenticationFailedException("Authentication failed");
+			}
+
+			if (passwordEncryptor.matches(password, storedPassword)) {
+				return getUserByName(user);
+			} else {
+				throw new AuthenticationFailedException("Authentication failed");
+			}
+
+		} else if (authentication instanceof AnonymousAuthentication) {
+			if (doesExist("anonymous")) {
+				return getUserByName("anonymous");
+			} else {
+				throw new AuthenticationFailedException("Authentication failed");
+			}
+		} else {
+			throw new IllegalArgumentException(
+					"Authentication not supported by this user manager");
+		}
+	}
+
+	/**
+	 * Close the user manager - remove existing entries.
+	 */
+	public synchronized void dispose() {
+		if (userDataProp != null) {
+			userDataProp.clear();
+			userDataProp = null;
+		}
+	}
+}
@@ -0,0 +1,15 @@
+server:
+  port: 8080
+
+ftp:
+  port: 2222
+  passive-address: localhost
+  passive-ports: 30000-30999
+
+hdfs:
+  uri: seaweedfs://localhost:8888
+
+seaweedFs:
+  enable: true
+  access: direct # direct/filerProxy/publicUrl
+  replication: "000"
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
+
+    <id>package</id>
+    <formats>
+        <!-- 指定打包格式，支持的打包格式有zip、tar、tar.gz (or tgz)、tar.bz2 (or tbz2)、jar、dir、war，可以同时指定多个打包格式 -->
+        <format>tar.gz</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+
+    <fileSets>
+        <fileSet>
+            <directory>src/main/resources</directory>
+            <outputDirectory>/</outputDirectory>
+            <includes>
+                <include>application.yml</include>
+                <include>logback-spring.xml</include>
+                <include>users.properties</include>
+                <include>kafka-producer.properties</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>${project.build.directory}</directory>
+            <outputDirectory>/</outputDirectory>
+            <includes>
+                <include>*.jar</include>
+            </includes>
+        </fileSet>
+    </fileSets>
+    <dependencySets>
+        <dependencySet>
+            <useProjectArtifact>false</useProjectArtifact>
+            <outputDirectory>lib</outputDirectory>
+            <scope>runtime</scope>
+            <unpack>false</unpack>
+        </dependencySet>
+    </dependencySets>
+</assembly>
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<configuration>
+    <!--定义日志文件的存储地址 勿在 LogBack 的配置中使用相对路径-->
+    <property name="LOG_HOME" value="${user.dir}/logs/" />
+
+    <!-- 控制台输出 -->
+    <appender name="Stdout" class="ch.qos.logback.core.ConsoleAppender">
+        <!-- 日志输出编码 -->
+        <layout class="ch.qos.logback.classic.PatternLayout">
+            <!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度%msg：日志消息，%n是换行符-->
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n
+            </pattern>
+        </layout>
+    </appender>
+
+    <!-- 按照每天生成日志文件 -->
+    <appender name="RollingFile"
+              class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <File>${LOG_HOME}/fileLog.log</File>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>${LOG_HOME}/fileLog.log.%d.%i</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100 MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+        </rollingPolicy>
+        <encoder>
+            <pattern>
+                %d %p (%file:%line\)- %m%n
+            </pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
+
+    <!-- 日志输出级别 -->
+    <root level="info">
+        <appender-ref ref="Stdout" />
+        <appender-ref ref="RollingFile" />
+    </root>
+
+</configuration>
@@ -0,0 +1,12 @@
+#Generated file - don't edit (please)
+#Thu Mar 11 19:11:12 CST 2021
+ftpserver.user.test.idletime=0
+ftpserver.user.test.maxloginperip=0
+ftpserver.user.test.userpassword=44664D4D827C740293D2AA244FB60445
+ftpserver.user.test.enableflag=true
+ftpserver.user.test.maxloginnumber=0
+ftpserver.user.test.rename.push=true
+ftpserver.user.test.homedirectory=/buckets/test/
+ftpserver.user.test.downloadrate=0
+ftpserver.user.test.writepermission=true
+ftpserver.user.test.uploadrate=0
@@ -301,7 +301,7 @@
    </snapshotRepository>
  </distributionManagement>
  <properties>
-    <seaweedfs.client.version>1.6.2</seaweedfs.client.version>
+    <seaweedfs.client.version>1.6.4</seaweedfs.client.version>
    <hadoop.version>2.9.2</hadoop.version>
  </properties>
 </project>
@@ -5,7 +5,7 @@
    <modelVersion>4.0.0</modelVersion>

    <properties>
-        <seaweedfs.client.version>1.6.2</seaweedfs.client.version>
+        <seaweedfs.client.version>1.6.4</seaweedfs.client.version>
        <hadoop.version>2.9.2</hadoop.version>
    </properties>

@@ -309,7 +309,7 @@
    </snapshotRepository>
  </distributionManagement>
  <properties>
-    <seaweedfs.client.version>1.6.2</seaweedfs.client.version>
+    <seaweedfs.client.version>1.6.4</seaweedfs.client.version>
    <hadoop.version>3.1.1</hadoop.version>
  </properties>
 </project>
@@ -5,7 +5,7 @@
    <modelVersion>4.0.0</modelVersion>

    <properties>
-        <seaweedfs.client.version>1.6.2</seaweedfs.client.version>
+        <seaweedfs.client.version>1.6.4</seaweedfs.client.version>
        <hadoop.version>3.1.1</hadoop.version>
    </properties>

@@ -8,7 +8,7 @@
    <version>1.0-SNAPSHOT</version>

    <properties>
-        <guava.version>28.0-jre</guava.version>
+        <guava.version>30.0-jre</guava.version>
    </properties>

    <dependencies>
@@ -305,7 +305,7 @@ func readFiles(fileIdLineChan chan string, s *stat) {
 		}
 		var bytes []byte
 		for _, url := range urls {
-			bytes, _, err = util.FastGet(url)
+			bytes, _, err = util.Get(url)
 			if err == nil {
 				break
 			}
@@ -22,9 +22,11 @@ var Commands = []*Command{
 	cmdFilerReplicate,
 	cmdFilerSynchronize,
 	cmdFix,
+	cmdGateway,
 	cmdMaster,
 	cmdMount,
 	cmdS3,
+	cmdIam,
 	cmdMsgBroker,
 	cmdScaffold,
 	cmdServer,
@@ -25,6 +25,8 @@ var (
 	filerS3Options     S3Options
 	filerStartWebDav   *bool
 	filerWebDavOptions WebDavOption
+	filerStartIam      *bool
+	filerIamOptions    IamOptions
 )

 type FilerOptions struct {
@@ -47,6 +49,7 @@ type FilerOptions struct {
 	metricsHttpPort         *int
 	saveToFilerLimit        *int
 	defaultLevelDbDirectory *string
+	concurrentUploadLimitMB *int
 }

 func init() {
@@ -59,7 +62,7 @@ func init() {
 	f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
 	f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
 	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
-	f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit")
+	f.maxMB = cmdFiler.Flag.Int("maxMB", 4, "split files larger than the limit")
 	f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
 	f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
 	f.rack = cmdFiler.Flag.String("rack", "", "prefer to write to volumes in this rack")
@@ -69,6 +72,7 @@ func init() {
 	f.metricsHttpPort = cmdFiler.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
 	f.saveToFilerLimit = cmdFiler.Flag.Int("saveToFilerLimit", 0, "files smaller than this limit will be saved in filer store")
 	f.defaultLevelDbDirectory = cmdFiler.Flag.String("defaultStoreDir", ".", "if filer.toml is empty, use an embedded filer store in the directory")
+	f.concurrentUploadLimitMB = cmdFiler.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size")

 	// start s3 on filer
 	filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
@@ -89,6 +93,10 @@ func init() {
 	filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
 	filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
 	filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 1000, "local cache capacity in MB")
+
+	// start iam on filer
+	filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
+	filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port")
 }

 var cmdFiler = &Command{
@@ -106,7 +114,7 @@ var cmdFiler = &Command{
 	GET /path/to/

 	The configuration file "filer.toml" is read from ".", "$HOME/.seaweedfs/", "/usr/local/etc/seaweedfs/", or "/etc/seaweedfs/", in that order.
-	If the "filer.toml" is not found, an embedded filer store will be craeted under "-defaultStoreDir".
+	If the "filer.toml" is not found, an embedded filer store will be created under "-defaultStoreDir".

 	The example filer.toml configuration file can be generated by "weed scaffold -config=filer"

@@ -119,22 +127,33 @@ func runFiler(cmd *Command, args []string) bool {

 	go stats_collect.StartMetricsServer(*f.metricsHttpPort)

+	filerAddress := fmt.Sprintf("%s:%d", *f.ip, *f.port)
+	startDelay := time.Duration(2)
 	if *filerStartS3 {
-		filerAddress := fmt.Sprintf("%s:%d", *f.ip, *f.port)
 		filerS3Options.filer = &filerAddress
 		go func() {
-			time.Sleep(2 * time.Second)
+			time.Sleep(startDelay * time.Second)
 			filerS3Options.startS3Server()
 		}()
+		startDelay++
 	}

 	if *filerStartWebDav {
-		filerAddress := fmt.Sprintf("%s:%d", *f.ip, *f.port)
 		filerWebDavOptions.filer = &filerAddress
 		go func() {
-			time.Sleep(2 * time.Second)
+			time.Sleep(startDelay * time.Second)
 			filerWebDavOptions.startWebDav()
 		}()
+		startDelay++
+	}
+
+	if *filerStartIam {
+		filerIamOptions.filer = &filerAddress
+		filerIamOptions.masters = f.masters
+		go func() {
+			time.Sleep(startDelay * time.Second)
+			filerIamOptions.startIamServer()
+		}()
 	}

 	f.startFiler()
@@ -159,21 +178,22 @@ func (fo *FilerOptions) startFiler() {
 	}

 	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
-		Masters:            strings.Split(*fo.masters, ","),
-		Collection:         *fo.collection,
-		DefaultReplication: *fo.defaultReplicaPlacement,
-		DisableDirListing:  *fo.disableDirListing,
-		MaxMB:              *fo.maxMB,
-		DirListingLimit:    *fo.dirListingLimit,
-		DataCenter:         *fo.dataCenter,
-		Rack:               *fo.rack,
-		DefaultLevelDbDir:  defaultLevelDbDirectory,
-		DisableHttp:        *fo.disableHttp,
-		Host:               *fo.ip,
-		Port:               uint32(*fo.port),
-		Cipher:             *fo.cipher,
-		SaveToFilerLimit:   *fo.saveToFilerLimit,
-		Filers:             peers,
+		Masters:               strings.Split(*fo.masters, ","),
+		Collection:            *fo.collection,
+		DefaultReplication:    *fo.defaultReplicaPlacement,
+		DisableDirListing:     *fo.disableDirListing,
+		MaxMB:                 *fo.maxMB,
+		DirListingLimit:       *fo.dirListingLimit,
+		DataCenter:            *fo.dataCenter,
+		Rack:                  *fo.rack,
+		DefaultLevelDbDir:     defaultLevelDbDirectory,
+		DisableHttp:           *fo.disableHttp,
+		Host:                  *fo.ip,
+		Port:                  uint32(*fo.port),
+		Cipher:                *fo.cipher,
+		SaveToFilerLimit:      int64(*fo.saveToFilerLimit),
+		Filers:                peers,
+		ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
 	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)
@@ -110,7 +110,7 @@ func runFilerCat(cmd *Command, args []string) bool {

 		filerCat.filerClient = client

-		return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
+		return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false)

 	})

@@ -56,7 +56,7 @@ func init() {
 	copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name")
 	copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
 	copy.diskType = cmdCopy.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
-	copy.maxMB = cmdCopy.Flag.Int("maxMB", 32, "split files larger than the limit")
+	copy.maxMB = cmdCopy.Flag.Int("maxMB", 4, "split files larger than the limit")
 	copy.concurrenctFiles = cmdCopy.Flag.Int("c", 8, "concurrent file copy goroutines")
 	copy.concurrenctChunks = cmdCopy.Flag.Int("concurrentChunks", 8, "concurrent chunk copy goroutines for each file")
 }
@@ -0,0 +1,93 @@
+package command
+
+import (
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+var (
+	gatewayOptions GatewayOptions
+)
+
+type GatewayOptions struct {
+	masters *string
+	filers  *string
+	bindIp  *string
+	port    *int
+	maxMB   *int
+}
+
+func init() {
+	cmdGateway.Run = runGateway // break init cycle
+	gatewayOptions.masters = cmdGateway.Flag.String("master", "localhost:9333", "comma-separated master servers")
+	gatewayOptions.filers = cmdGateway.Flag.String("filer", "localhost:8888", "comma-separated filer servers")
+	gatewayOptions.bindIp = cmdGateway.Flag.String("ip.bind", "localhost", "ip address to bind to")
+	gatewayOptions.port = cmdGateway.Flag.Int("port", 5647, "gateway http listen port")
+	gatewayOptions.maxMB = cmdGateway.Flag.Int("maxMB", 4, "split files larger than the limit")
+}
+
+var cmdGateway = &Command{
+	UsageLine: "gateway -port=8888 -master=<ip:port>[,<ip:port>]* -filer=<ip:port>[,<ip:port>]*",
+	Short:     "start a gateway server that points to a list of master servers or a list of filers",
+	Long: `start a gateway server which accepts REST operation to write any blobs, files, or topic messages.
+
+	POST /blobs/
+		upload the blob and return a chunk id
+	DELETE /blobs/<chunk_id>
+		delete a chunk id
+
+	/*
+	POST /files/path/to/a/file
+		save /path/to/a/file on filer 
+	DELETE /files/path/to/a/file
+		delete /path/to/a/file on filer 
+
+	POST /topics/topicName
+		save on filer to /topics/topicName/<ds>/ts.json
+	*/
+`,
+}
+
+func runGateway(cmd *Command, args []string) bool {
+
+	util.LoadConfiguration("security", false)
+
+	gatewayOptions.startGateway()
+
+	return true
+}
+
+func (gw *GatewayOptions) startGateway() {
+
+	defaultMux := http.NewServeMux()
+
+	_, gws_err := weed_server.NewGatewayServer(defaultMux, &weed_server.GatewayOption{
+		Masters: strings.Split(*gw.masters, ","),
+		Filers:  strings.Split(*gw.filers, ","),
+		MaxMB:   *gw.maxMB,
+	})
+	if gws_err != nil {
+		glog.Fatalf("Gateway startup error: %v", gws_err)
+	}
+
+	glog.V(0).Infof("Start Seaweed Gateway %s at %s:%d", util.Version(), *gw.bindIp, *gw.port)
+	gatewayListener, e := util.NewListener(
+		*gw.bindIp+":"+strconv.Itoa(*gw.port),
+		time.Duration(10)*time.Second,
+	)
+	if e != nil {
+		glog.Fatalf("Filer listener error: %v", e)
+	}
+
+	httpS := &http.Server{Handler: defaultMux}
+	if err := httpS.Serve(gatewayListener); err != nil {
+		glog.Fatalf("Gateway Fail to serve: %v", e)
+	}
+
+}
@@ -0,0 +1,97 @@
+package command
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/iamapi"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/gorilla/mux"
+	"time"
+)
+
+var (
+	iamStandaloneOptions IamOptions
+)
+
+type IamOptions struct {
+	filer   *string
+	masters *string
+	port    *int
+}
+
+func init() {
+	cmdIam.Run = runIam // break init cycle
+	iamStandaloneOptions.filer = cmdIam.Flag.String("filer", "localhost:8888", "filer server address")
+	iamStandaloneOptions.masters = cmdIam.Flag.String("master", "localhost:9333", "comma-separated master servers")
+	iamStandaloneOptions.port = cmdIam.Flag.Int("port", 8111, "iam server http listen port")
+}
+
+var cmdIam = &Command{
+	UsageLine: "iam [-port=8111] [-filer=<ip:port>] [-masters=<ip:port>,<ip:port>]",
+	Short:     "start a iam API compatible server",
+	Long:      "start a iam API compatible server.",
+}
+
+func runIam(cmd *Command, args []string) bool {
+	return iamStandaloneOptions.startIamServer()
+}
+
+func (iamopt *IamOptions) startIamServer() bool {
+	filerGrpcAddress, err := pb.ParseServerToGrpcAddress(*iamopt.filer)
+	if err != nil {
+		glog.Fatal(err)
+		return false
+	}
+
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+	for {
+		err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+			if err != nil {
+				return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
+			}
+			glog.V(0).Infof("IAM read filer configuration: %s", resp)
+			return nil
+		})
+		if err != nil {
+			glog.V(0).Infof("wait to connect to filer %s grpc address %s", *iamopt.filer, filerGrpcAddress)
+			time.Sleep(time.Second)
+		} else {
+			glog.V(0).Infof("connected to filer %s grpc address %s", *iamopt.filer, filerGrpcAddress)
+			break
+		}
+	}
+
+	router := mux.NewRouter().SkipClean(true)
+	_, iamApiServer_err := iamapi.NewIamApiServer(router, &iamapi.IamServerOption{
+		Filer:            *iamopt.filer,
+		Port:             *iamopt.port,
+		FilerGrpcAddress: filerGrpcAddress,
+		GrpcDialOption:   grpcDialOption,
+	})
+	glog.V(0).Info("NewIamApiServer created")
+	if iamApiServer_err != nil {
+		glog.Fatalf("IAM API Server startup error: %v", iamApiServer_err)
+	}
+
+	httpS := &http.Server{Handler: router}
+
+	listenAddress := fmt.Sprintf(":%d", *iamopt.port)
+	iamApiListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
+	if err != nil {
+		glog.Fatalf("IAM API Server listener on %s error: %v", listenAddress, err)
+	}
+
+	glog.V(0).Infof("Start Seaweed IAM API Server %s at http port %d", util.Version(), *iamopt.port)
+	if err = httpS.Serve(iamApiListener); err != nil {
+		glog.Fatalf("IAM API Server Fail to serve: %v", err)
+	}
+
+	return true
+}
@@ -6,7 +6,6 @@ import (
 	"google.golang.org/grpc/reflection"
 	"net/http"
 	"os"
-	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -48,7 +47,7 @@ type MasterOptions struct {
 func init() {
 	cmdMaster.Run = runMaster // break init cycle
 	m.port = cmdMaster.Flag.Int("port", 9333, "http listen port")
-	m.ip = cmdMaster.Flag.String("ip", util.DetectedHostAddress(), "master <ip>|<server> address")
+	m.ip = cmdMaster.Flag.String("ip", util.DetectedHostAddress(), "master <ip>|<server> address, also used as identifier")
 	m.ipBind = cmdMaster.Flag.String("ip.bind", "", "ip address to bind to")
 	m.metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
 	m.peers = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095")
@@ -86,7 +85,6 @@ func runMaster(cmd *Command, args []string) bool {
 	util.LoadConfiguration("security", false)
 	util.LoadConfiguration("master", false)

-	runtime.GOMAXPROCS(runtime.NumCPU())
 	grace.SetupProfiling(*masterCpuProfile, *masterMemProfile)

 	parent, _ := util.FullPath(*m.metaFolder).DirAndName()
@@ -25,6 +25,7 @@ type MountOptions struct {
 	volumeServerAccess *string
 	uidMap             *string
 	gidMap             *string
+	readOnly           *bool
 }

 var (
@@ -45,7 +46,7 @@ func init() {
 	mountOptions.diskType = cmdMount.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
 	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files")
-	mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 128, "limit concurrent goroutine writers if not 0")
+	mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers if not 0")
 	mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data")
 	mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 1000, "local file chunk cache capacity in MB (0 will disable cache)")
 	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
@@ -55,6 +56,7 @@ func init() {
 	mountOptions.volumeServerAccess = cmdMount.Flag.String("volumeServerAccess", "direct", "access volume servers by [direct|publicUrl|filerProxy]")
 	mountOptions.uidMap = cmdMount.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated <local_uid>:<filer_uid>")
 	mountOptions.gidMap = cmdMount.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated <local_gid>:<filer_gid>")
+	mountOptions.readOnly = cmdMount.Flag.Bool("readOnly", false, "read only")

 	mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
 	mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
@@ -149,8 +149,6 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		fuse.Subtype("seaweedfs"),
 		// fuse.NoAppleDouble(), // include .DS_Store, otherwise can not delete non-empty folders
 		fuse.NoAppleXattr(),
-		fuse.NoBrowse(),
-		fuse.AutoXattr(),
 		fuse.ExclCreate(),
 		fuse.DaemonTimeout("3600"),
 		fuse.AllowSUID(),
@@ -203,6 +201,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		VolumeServerAccess: *mountOptions.volumeServerAccess,
 		Cipher:             cipher,
 		UidGidMapper:       uidGidMapper,
+		ReadOnly:           *option.readOnly,
 	})

 	// mount
@@ -103,9 +103,9 @@ dir = "./filerrdb"					# directory to store rocksdb files

 [mysql]  # or memsql, tidb
 # CREATE TABLE IF NOT EXISTS filemeta (
-#   dirhash     BIGINT         COMMENT 'first 64 bits of MD5 hash value of directory field',
-#   name        VARCHAR(1000)  COMMENT 'directory or file name',
-#   directory   TEXT           COMMENT 'full path to parent directory',
+#   dirhash     BIGINT               COMMENT 'first 64 bits of MD5 hash value of directory field',
+#   name        VARCHAR(1000) BINARY COMMENT 'directory or file name',
+#   directory   TEXT                 COMMENT 'full path to parent directory',
 #   meta        LONGBLOB,
 #   PRIMARY KEY (dirhash, name)
 # ) DEFAULT CHARSET=utf8;
@@ -120,13 +120,16 @@ connection_max_idle = 2
 connection_max_open = 100
 connection_max_lifetime_seconds = 0
 interpolateParams = false
+# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
+enableUpsert = true
+upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""

 [mysql2]  # or memsql, tidb
 enabled = false
 createTable = """
  CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` (
    dirhash BIGINT,
-    name VARCHAR(1000),
+    name VARCHAR(1000) BINARY,
    directory TEXT,
    meta LONGBLOB,
    PRIMARY KEY (dirhash, name)
@@ -141,6 +144,9 @@ connection_max_idle = 2
 connection_max_open = 100
 connection_max_lifetime_seconds = 0
 interpolateParams = false
+# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
+enableUpsert = true
+upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""

 [postgres] # or cockroachdb, YugabyteDB
 # CREATE TABLE IF NOT EXISTS filemeta (
@@ -161,6 +167,9 @@ sslmode = "disable"
 connection_max_idle = 100
 connection_max_open = 100
 connection_max_lifetime_seconds = 0
+# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
+enableUpsert = true
+upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""

 [postgres2]
 enabled = false
@@ -183,6 +192,9 @@ sslmode = "disable"
 connection_max_idle = 100
 connection_max_open = 100
 connection_max_lifetime_seconds = 0
+# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
+enableUpsert = true
+upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""

 [cassandra]
 # CREATE TABLE filemeta (
@@ -506,7 +518,7 @@ default = "localhost:8888"    # used by maintenance scripts if the scripts needs


 [master.sequencer]
-type = "raft"     # Choose [raft|etcd] type for storing the file id sequence
+type = "raft"     # Choose [raft|etcd|snowflake] type for storing the file id sequence
 # when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence
 # example : http://127.0.0.1:2379,http://127.0.0.1:2389
 sequencer_etcd_urls = "http://127.0.0.1:2379"
@@ -2,9 +2,8 @@ package command

 import (
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/util/grace"
 	"os"
-	"runtime"
-	"runtime/pprof"
 	"strings"
 	"time"

@@ -16,6 +15,7 @@ import (

 type ServerOptions struct {
 	cpuprofile *string
+	memprofile *string
 	v          VolumeServerOptions
 }

@@ -49,7 +49,7 @@ var cmdServer = &Command{
 }

 var (
-	serverIp                  = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
+	serverIp                  = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
 	serverBindIp              = cmdServer.Flag.String("ip.bind", "", "ip address to bind to")
 	serverTimeout             = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
 	serverDataCenter          = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
@@ -76,6 +76,7 @@ var (

 func init() {
 	serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")
+	serverOptions.memprofile = cmdServer.Flag.String("memprofile", "", "memory profile output file")

 	masterOptions.port = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
 	masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
@@ -93,11 +94,12 @@ func init() {
 	filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
 	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
 	filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
-	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit")
+	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 4, "split files larger than the limit")
 	filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
 	filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers")
 	filerOptions.peers = cmdServer.Flag.String("filer.peers", "", "all filers sharing the same filer store in comma separated ip:port list")
 	filerOptions.saveToFilerLimit = cmdServer.Flag.Int("filer.saveToFilerLimit", 0, "Small files smaller than this limit can be cached in filer store.")
+	filerOptions.concurrentUploadLimitMB = cmdServer.Flag.Int("filer.concurrentUploadLimitMB", 64, "limit total concurrent upload size")

 	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
 	serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
@@ -107,6 +109,7 @@ func init() {
 	serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
 	serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
 	serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")
+	serverOptions.v.concurrentUploadLimitMB = cmdServer.Flag.Int("volume.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
 	serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
 	serverOptions.v.preStopSeconds = cmdServer.Flag.Int("volume.preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
 	serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
@@ -138,14 +141,7 @@ func runServer(cmd *Command, args []string) bool {
 	util.LoadConfiguration("security", false)
 	util.LoadConfiguration("master", false)

-	if *serverOptions.cpuprofile != "" {
-		f, err := os.Create(*serverOptions.cpuprofile)
-		if err != nil {
-			glog.Fatal(err)
-		}
-		pprof.StartCPUProfile(f)
-		defer pprof.StopCPUProfile()
-	}
+	grace.SetupProfiling(*serverOptions.cpuprofile, *serverOptions.memprofile)

 	if *isStartingS3 {
 		*isStartingFiler = true
@@ -192,7 +188,6 @@ func runServer(cmd *Command, args []string) bool {
 	webdavOptions.filer = &filerAddress
 	msgBrokerOptions.filer = &filerAddress

-	runtime.GOMAXPROCS(runtime.NumCPU())
 	go stats_collect.StartMetricsServer(*serverMetricsHttpPort)

 	folders := strings.Split(*volumeDataFolders, ",")
@@ -43,7 +43,7 @@ func init() {
 	upload.dataCenter = cmdUpload.Flag.String("dataCenter", "", "optional data center name")
 	upload.diskType = cmdUpload.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 	upload.ttl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
-	upload.maxMB = cmdUpload.Flag.Int("maxMB", 32, "split files larger than the limit")
+	upload.maxMB = cmdUpload.Flag.Int("maxMB", 4, "split files larger than the limit")
 	upload.usePublicUrl = cmdUpload.Flag.Bool("usePublicUrl", false, "upload to public url from volume server")
 }

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	httppprof "net/http/pprof"
 	"os"
-	"runtime"
 	"runtime/pprof"
 	"strconv"
 	"strings"
@@ -36,31 +35,32 @@ var (
 )

 type VolumeServerOptions struct {
-	port                  *int
-	publicPort            *int
-	folders               []string
-	folderMaxLimits       []int
-	idxFolder             *string
-	ip                    *string
-	publicUrl             *string
-	bindIp                *string
-	masters               *string
-	idleConnectionTimeout *int
-	dataCenter            *string
-	rack                  *string
-	whiteList             []string
-	indexType             *string
-	diskType              *string
-	fixJpgOrientation     *bool
-	readRedirect          *bool
-	cpuProfile            *string
-	memProfile            *string
-	compactionMBPerSecond *int
-	fileSizeLimitMB       *int
-	minFreeSpacePercents  []float32
-	pprof                 *bool
-	preStopSeconds        *int
-	metricsHttpPort       *int
+	port                    *int
+	publicPort              *int
+	folders                 []string
+	folderMaxLimits         []int
+	idxFolder               *string
+	ip                      *string
+	publicUrl               *string
+	bindIp                  *string
+	masters                 *string
+	idleConnectionTimeout   *int
+	dataCenter              *string
+	rack                    *string
+	whiteList               []string
+	indexType               *string
+	diskType                *string
+	fixJpgOrientation       *bool
+	readRedirect            *bool
+	cpuProfile              *string
+	memProfile              *string
+	compactionMBPerSecond   *int
+	fileSizeLimitMB         *int
+	concurrentUploadLimitMB *int
+	minFreeSpacePercents    []float32
+	pprof                   *bool
+	preStopSeconds          *int
+	metricsHttpPort         *int
 	// pulseSeconds          *int
 	enableTcp *bool
 }
@@ -69,7 +69,7 @@ func init() {
 	cmdVolume.Run = runVolume // break init cycle
 	v.port = cmdVolume.Flag.Int("port", 8080, "http listen port")
 	v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public")
-	v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
+	v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
 	v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
 	v.bindIp = cmdVolume.Flag.String("ip.bind", "", "ip address to bind to")
 	v.masters = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
@@ -86,6 +86,7 @@ func init() {
 	v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
 	v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
 	v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
+	v.concurrentUploadLimitMB = cmdVolume.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size")
 	v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
 	v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
 	v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files")
@@ -111,8 +112,6 @@ func runVolume(cmd *Command, args []string) bool {

 	util.LoadConfiguration("security", false)

-	runtime.GOMAXPROCS(runtime.NumCPU())
-
 	// If --pprof is set we assume the caller wants to be able to collect
 	// cpu and memory profiles via go tool pprof
 	if !*v.pprof {
@@ -240,6 +239,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 		*v.fixJpgOrientation, *v.readRedirect,
 		*v.compactionMBPerSecond,
 		*v.fileSizeLimitMB,
+		int64(*v.concurrentUploadLimitMB)*1024*1024,
 	)
 	// starting grpc server
 	grpcS := v.startGrpcService(volumeServer)
@@ -13,15 +13,15 @@ import (
 )

 type SqlGenerator interface {
-	GetSqlInsert(bucket string) string
-	GetSqlUpdate(bucket string) string
-	GetSqlFind(bucket string) string
-	GetSqlDelete(bucket string) string
-	GetSqlDeleteFolderChildren(bucket string) string
-	GetSqlListExclusive(bucket string) string
-	GetSqlListInclusive(bucket string) string
-	GetSqlCreateTable(bucket string) string
-	GetSqlDropTable(bucket string) string
+	GetSqlInsert(tableName string) string
+	GetSqlUpdate(tableName string) string
+	GetSqlFind(tableName string) string
+	GetSqlDelete(tableName string) string
+	GetSqlDeleteFolderChildren(tableName string) string
+	GetSqlListExclusive(tableName string) string
+	GetSqlListInclusive(tableName string) string
+	GetSqlCreateTable(tableName string) string
+	GetSqlDropTable(tableName string) string
 }

 type AbstractSqlStore struct {
@@ -36,6 +36,8 @@ func (store *AbstractSqlStore) OnBucketCreation(bucket string) {
 	store.dbsLock.Lock()
 	defer store.dbsLock.Unlock()

+	store.CreateTable(context.Background(), bucket)
+
 	if store.dbs == nil {
 		return
 	}
@@ -45,6 +47,8 @@ func (store *AbstractSqlStore) OnBucketDeletion(bucket string) {
 	store.dbsLock.Lock()
 	defer store.dbsLock.Unlock()

+	store.deleteTable(context.Background(), bucket)
+
 	if store.dbs == nil {
 		return
 	}
@@ -97,20 +97,20 @@ func fetchChunk(lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string,
 func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) ([]byte, error) {

 	var err error
-	var buffer bytes.Buffer
 	var shouldRetry bool
+	receivedData := make([]byte, 0, size)

 	for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
 		for _, urlString := range urlStrings {
-			shouldRetry, err = util.FastReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
-				buffer.Write(data)
+			receivedData = receivedData[:0]
+			shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
+				receivedData = append(receivedData, data...)
 			})
 			if !shouldRetry {
 				break
 			}
 			if err != nil {
 				glog.V(0).Infof("read %s failed, err: %v", urlString, err)
-				buffer.Reset()
 			} else {
 				break
 			}
@@ -123,7 +123,8 @@ func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool
 		}
 	}

-	return buffer.Bytes(), err
+	return receivedData, err
+
 }

 func MaybeManifestize(saveFunc SaveDataAsChunkFunctionType, inputChunks []*filer_pb.FileChunk) (chunks []*filer_pb.FileChunk, err error) {
@@ -23,10 +23,6 @@ func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
 	return
 }

-func FileSize2(entry *Entry) (size uint64) {
-	return maxUint64(TotalSize(entry.Chunks), entry.Attr.FileSize)
-}
-
 func FileSize(entry *filer_pb.Entry) (size uint64) {
 	return maxUint64(TotalSize(entry.Chunks), entry.Attributes.FileSize)
 }
@@ -241,12 +241,12 @@ func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err er
 	if oldEntry != nil {
 		entry.Attr.Crtime = oldEntry.Attr.Crtime
 		if oldEntry.IsDirectory() && !entry.IsDirectory() {
-			glog.Errorf("existing %s is a directory", entry.FullPath)
-			return fmt.Errorf("existing %s is a directory", entry.FullPath)
+			glog.Errorf("existing %s is a directory", oldEntry.FullPath)
+			return fmt.Errorf("existing %s is a directory", oldEntry.FullPath)
 		}
 		if !oldEntry.IsDirectory() && entry.IsDirectory() {
-			glog.Errorf("existing %s is a file", entry.FullPath)
-			return fmt.Errorf("existing %s is a file", entry.FullPath)
+			glog.Errorf("existing %s is a file", oldEntry.FullPath)
+			return fmt.Errorf("existing %s is a file", oldEntry.FullPath)
 		}
 	}
 	return f.Store.UpdateEntry(ctx, entry)
@@ -52,7 +52,7 @@ func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataR

 func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) {
 	var buf bytes.Buffer
-	err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64)
+	err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64, false)
 	if err != nil {
 		return nil, err
 	}
@@ -2,6 +2,7 @@ package mysql

 import (
 	"fmt"
+
 	"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
 	_ "github.com/go-sql-driver/mysql"
 )
@@ -9,44 +10,49 @@ import (
 type SqlGenMysql struct {
 	CreateTableSqlTemplate string
 	DropTableSqlTemplate   string
+	UpsertQueryTemplate    string
 }

 var (
 	_ = abstract_sql.SqlGenerator(&SqlGenMysql{})
 )

-func (gen *SqlGenMysql) GetSqlInsert(bucket string) string {
-	return fmt.Sprintf("INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?)", bucket)
+func (gen *SqlGenMysql) GetSqlInsert(tableName string) string {
+	if gen.UpsertQueryTemplate != "" {
+		return fmt.Sprintf(gen.UpsertQueryTemplate, tableName)
+	} else {
+		return fmt.Sprintf("INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?)", tableName)
+	}
 }

-func (gen *SqlGenMysql) GetSqlUpdate(bucket string) string {
-	return fmt.Sprintf("UPDATE `%s` SET meta=? WHERE dirhash=? AND name=? AND directory=?", bucket)
+func (gen *SqlGenMysql) GetSqlUpdate(tableName string) string {
+	return fmt.Sprintf("UPDATE `%s` SET meta=? WHERE dirhash=? AND name=? AND directory=?", tableName)
 }

-func (gen *SqlGenMysql) GetSqlFind(bucket string) string {
-	return fmt.Sprintf("SELECT meta FROM `%s` WHERE dirhash=? AND name=? AND directory=?", bucket)
+func (gen *SqlGenMysql) GetSqlFind(tableName string) string {
+	return fmt.Sprintf("SELECT meta FROM `%s` WHERE dirhash=? AND name=? AND directory=?", tableName)
 }

-func (gen *SqlGenMysql) GetSqlDelete(bucket string) string {
-	return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND name=? AND directory=?", bucket)
+func (gen *SqlGenMysql) GetSqlDelete(tableName string) string {
+	return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND name=? AND directory=?", tableName)
 }

-func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(bucket string) string {
-	return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND directory=?", bucket)
+func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(tableName string) string {
+	return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND directory=?", tableName)
 }

-func (gen *SqlGenMysql) GetSqlListExclusive(bucket string) string {
-	return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", bucket)
+func (gen *SqlGenMysql) GetSqlListExclusive(tableName string) string {
+	return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", tableName)
 }

-func (gen *SqlGenMysql) GetSqlListInclusive(bucket string) string {
-	return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", bucket)
+func (gen *SqlGenMysql) GetSqlListInclusive(tableName string) string {
+	return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", tableName)
 }

-func (gen *SqlGenMysql) GetSqlCreateTable(bucket string) string {
-	return fmt.Sprintf(gen.CreateTableSqlTemplate, bucket)
+func (gen *SqlGenMysql) GetSqlCreateTable(tableName string) string {
+	return fmt.Sprintf(gen.CreateTableSqlTemplate, tableName)
 }

-func (gen *SqlGenMysql) GetSqlDropTable(bucket string) string {
-	return fmt.Sprintf(gen.DropTableSqlTemplate, bucket)
+func (gen *SqlGenMysql) GetSqlDropTable(tableName string) string {
+	return fmt.Sprintf(gen.DropTableSqlTemplate, tableName)
 }
@@ -3,9 +3,10 @@ package mysql
 import (
 	"database/sql"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/filer"
 	"time"

+	"github.com/chrislusf/seaweedfs/weed/filer"
+
 	"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	_ "github.com/go-sql-driver/mysql"
@@ -29,6 +30,8 @@ func (store *MysqlStore) GetName() string {

 func (store *MysqlStore) Initialize(configuration util.Configuration, prefix string) (err error) {
 	return store.initialize(
+		configuration.GetString(prefix+"upsertQuery"),
+		configuration.GetBool(prefix+"enableUpsert"),
 		configuration.GetString(prefix+"username"),
 		configuration.GetString(prefix+"password"),
 		configuration.GetString(prefix+"hostname"),
@@ -41,13 +44,17 @@ func (store *MysqlStore) Initialize(configuration util.Configuration, prefix str
 	)
 }

-func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen,
+func (store *MysqlStore) initialize(upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database string, maxIdle, maxOpen,
 	maxLifetimeSeconds int, interpolateParams bool) (err error) {

 	store.SupportBucketTable = false
+	if !enableUpsert {
+		upsertQuery = ""
+	}
 	store.SqlGenerator = &SqlGenMysql{
 		CreateTableSqlTemplate: "",
 		DropTableSqlTemplate:   "drop table `%s`",
+		UpsertQueryTemplate:    upsertQuery,
 	}

 	sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)
@@ -32,6 +32,8 @@ func (store *MysqlStore2) GetName() string {
 func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
 	return store.initialize(
 		configuration.GetString(prefix+"createTable"),
+		configuration.GetString(prefix+"upsertQuery"),
+		configuration.GetBool(prefix+"enableUpsert"),
 		configuration.GetString(prefix+"username"),
 		configuration.GetString(prefix+"password"),
 		configuration.GetString(prefix+"hostname"),
@@ -44,13 +46,17 @@ func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix st
 	)
 }

-func (store *MysqlStore2) initialize(createTable, user, password, hostname string, port int, database string, maxIdle, maxOpen,
+func (store *MysqlStore2) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database string, maxIdle, maxOpen,
 	maxLifetimeSeconds int, interpolateParams bool) (err error) {

 	store.SupportBucketTable = true
+	if !enableUpsert {
+		upsertQuery = ""
+	}
 	store.SqlGenerator = &mysql.SqlGenMysql{
 		CreateTableSqlTemplate: createTable,
 		DropTableSqlTemplate:   "drop table `%s`",
+		UpsertQueryTemplate:    upsertQuery,
 	}

 	sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)
@@ -10,44 +10,49 @@ import (
 type SqlGenPostgres struct {
 	CreateTableSqlTemplate string
 	DropTableSqlTemplate   string
+	UpsertQueryTemplate    string
 }

 var (
 	_ = abstract_sql.SqlGenerator(&SqlGenPostgres{})
 )

-func (gen *SqlGenPostgres) GetSqlInsert(bucket string) string {
-	return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, bucket)
+func (gen *SqlGenPostgres) GetSqlInsert(tableName string) string {
+	if gen.UpsertQueryTemplate != "" {
+		return fmt.Sprintf(gen.UpsertQueryTemplate, tableName)
+	} else {
+		return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, tableName)
+	}
 }

-func (gen *SqlGenPostgres) GetSqlUpdate(bucket string) string {
-	return fmt.Sprintf(`UPDATE "%s" SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4`, bucket)
+func (gen *SqlGenPostgres) GetSqlUpdate(tableName string) string {
+	return fmt.Sprintf(`UPDATE "%s" SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4`, tableName)
 }

-func (gen *SqlGenPostgres) GetSqlFind(bucket string) string {
-	return fmt.Sprintf(`SELECT meta FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, bucket)
+func (gen *SqlGenPostgres) GetSqlFind(tableName string) string {
+	return fmt.Sprintf(`SELECT meta FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, tableName)
 }

-func (gen *SqlGenPostgres) GetSqlDelete(bucket string) string {
-	return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, bucket)
+func (gen *SqlGenPostgres) GetSqlDelete(tableName string) string {
+	return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, tableName)
 }

-func (gen *SqlGenPostgres) GetSqlDeleteFolderChildren(bucket string) string {
-	return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND directory=$2`, bucket)
+func (gen *SqlGenPostgres) GetSqlDeleteFolderChildren(tableName string) string {
+	return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND directory=$2`, tableName)
 }

-func (gen *SqlGenPostgres) GetSqlListExclusive(bucket string) string {
-	return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, bucket)
+func (gen *SqlGenPostgres) GetSqlListExclusive(tableName string) string {
+	return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, tableName)
 }

-func (gen *SqlGenPostgres) GetSqlListInclusive(bucket string) string {
-	return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>=$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, bucket)
+func (gen *SqlGenPostgres) GetSqlListInclusive(tableName string) string {
+	return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>=$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, tableName)
 }

-func (gen *SqlGenPostgres) GetSqlCreateTable(bucket string) string {
-	return fmt.Sprintf(gen.CreateTableSqlTemplate, bucket)
+func (gen *SqlGenPostgres) GetSqlCreateTable(tableName string) string {
+	return fmt.Sprintf(gen.CreateTableSqlTemplate, tableName)
 }

-func (gen *SqlGenPostgres) GetSqlDropTable(bucket string) string {
-	return fmt.Sprintf(gen.DropTableSqlTemplate, bucket)
+func (gen *SqlGenPostgres) GetSqlDropTable(tableName string) string {
+	return fmt.Sprintf(gen.DropTableSqlTemplate, tableName)
 }
@@ -29,6 +29,8 @@ func (store *PostgresStore) GetName() string {

 func (store *PostgresStore) Initialize(configuration util.Configuration, prefix string) (err error) {
 	return store.initialize(
+		configuration.GetString(prefix+"upsertQuery"),
+		configuration.GetBool(prefix+"enableUpsert"),
 		configuration.GetString(prefix+"username"),
 		configuration.GetString(prefix+"password"),
 		configuration.GetString(prefix+"hostname"),
@@ -42,12 +44,16 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix
 	)
 }

-func (store *PostgresStore) initialize(user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
+func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {

 	store.SupportBucketTable = false
+	if !enableUpsert {
+		upsertQuery = ""
+	}
 	store.SqlGenerator = &SqlGenPostgres{
 		CreateTableSqlTemplate: "",
 		DropTableSqlTemplate:   `drop table "%s"`,
+		UpsertQueryTemplate:    upsertQuery,
 	}

 	sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode)
@@ -32,6 +32,8 @@ func (store *PostgresStore2) GetName() string {
 func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
 	return store.initialize(
 		configuration.GetString(prefix+"createTable"),
+		configuration.GetString(prefix+"upsertQuery"),
+		configuration.GetBool(prefix+"enableUpsert"),
 		configuration.GetString(prefix+"username"),
 		configuration.GetString(prefix+"password"),
 		configuration.GetString(prefix+"hostname"),
@@ -45,12 +47,16 @@ func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix
 	)
 }

-func (store *PostgresStore2) initialize(createTable, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
+func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {

 	store.SupportBucketTable = true
+	if !enableUpsert {
+		upsertQuery = ""
+	}
 	store.SqlGenerator = &postgres.SqlGenPostgres{
 		CreateTableSqlTemplate: createTable,
 		DropTableSqlTemplate:   `drop table "%s"`,
+		UpsertQueryTemplate:    upsertQuery,
 	}

 	sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode)
@@ -27,7 +27,7 @@ func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.Seaweed
 		return err
 	}

-	return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
+	return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false)

 }

@@ -35,14 +35,18 @@ func ReadContent(filerAddress string, dir, name string) ([]byte, error) {

 	target := fmt.Sprintf("http://%s%s/%s", filerAddress, dir, name)

-	data, _, err := util.FastGet(target)
+	data, _, err := util.Get(target)

 	return data, err
 }

 func SaveAs(host string, port int, dir, name string, contentType string, byteBuffer *bytes.Buffer) error {
-
-	target := fmt.Sprintf("http://%s:%d%s/%s", host, port, dir, name)
+	var target string
+	if port == 0 {
+		target = fmt.Sprintf("http://%s%s/%s", host, dir, name)
+	} else {
+		target = fmt.Sprintf("http://%s:%d%s/%s", host, port, dir, name)
+	}

 	// set the HTTP method, url, and request body
 	req, err := http.NewRequest(http.MethodPut, target, byteBuffer)
@@ -3,6 +3,7 @@ package filer
 import (
 	"bytes"
 	"fmt"
+	"golang.org/x/sync/errgroup"
 	"io"
 	"math"
 	"strings"
@@ -13,7 +14,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 )

-func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {
+func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64, isCheck bool) error {

 	glog.V(9).Infof("start to stream content for chunks: %+v\n", chunks)
 	chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size)
@@ -33,10 +34,20 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, c
 		fileId2Url[chunkView.FileId] = urlStrings
 	}

+	if isCheck {
+		// Pre-check all chunkViews urls
+		gErr := new(errgroup.Group)
+		CheckAllChunkViews(chunkViews, &fileId2Url, gErr)
+		if err := gErr.Wait(); err != nil {
+			glog.Errorf("check all chunks: %v", err)
+			return fmt.Errorf("check all chunks: %v", err)
+		}
+		return nil
+	}
+
 	for _, chunkView := range chunkViews {

 		urlStrings := fileId2Url[chunkView.FileId]
-
 		data, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
 		if err != nil {
 			glog.Errorf("read chunk: %v", err)
@@ -54,6 +65,17 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, c

 }

+func CheckAllChunkViews(chunkViews []*ChunkView, fileId2Url *map[string][]string, gErr *errgroup.Group) {
+	for _, chunkView := range chunkViews {
+		urlStrings := (*fileId2Url)[chunkView.FileId]
+		glog.V(9).Infof("Check chunk: %+v\n url: %v", chunkView, urlStrings)
+		gErr.Go(func() error {
+			_, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
+			return err
+		})
+	}
+}
+
 // ----------------  ReadAllReader ----------------------------------

 func ReadAll(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) ([]byte, error) {
@@ -185,7 +207,7 @@ func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
 	var buffer bytes.Buffer
 	var shouldRetry bool
 	for _, urlString := range urlStrings {
-		shouldRetry, err = util.FastReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
+		shouldRetry, err = util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
 			buffer.Write(data)
 		})
 		if !shouldRetry {
@@ -1,6 +1,7 @@
 package filesys

 import (
+	"bytes"
 	"context"
 	"math"
 	"os"
@@ -21,7 +22,7 @@ import (
 type Dir struct {
 	name   string
 	wfs    *WFS
-	entry  *filer.Entry
+	entry  *filer_pb.Entry
 	parent *Dir
 }

@@ -58,11 +59,11 @@ func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error {
 	}

 	// attr.Inode = util.FullPath(dir.FullPath()).AsInode()
-	attr.Mode = dir.entry.Attr.Mode | os.ModeDir
-	attr.Mtime = dir.entry.Attr.Mtime
-	attr.Crtime = dir.entry.Attr.Crtime
-	attr.Gid = dir.entry.Attr.Gid
-	attr.Uid = dir.entry.Attr.Uid
+	attr.Mode = os.FileMode(dir.entry.Attributes.FileMode) | os.ModeDir
+	attr.Mtime = time.Unix(dir.entry.Attributes.Mtime, 0)
+	attr.Crtime = time.Unix(dir.entry.Attributes.Crtime, 0)
+	attr.Gid = dir.entry.Attributes.Gid
+	attr.Uid = dir.entry.Attributes.Uid

 	glog.V(4).Infof("dir Attr %s, attr: %+v", dir.FullPath(), attr)

@@ -102,13 +103,12 @@ func (dir *Dir) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {
 }

 func (dir *Dir) newFile(name string, entry *filer_pb.Entry) fs.Node {
-	dirPath := dir.FullPath()
-	f := dir.wfs.fsNodeCache.EnsureFsNode(util.NewFullPath(dirPath, name), func() fs.Node {
+	f := dir.wfs.fsNodeCache.EnsureFsNode(util.NewFullPath(dir.FullPath(), name), func() fs.Node {
 		return &File{
 			Name:           name,
 			dir:            dir,
 			wfs:            dir.wfs,
-			entry:          filer.FromPbEntry(dirPath, entry),
+			entry:          entry,
 			entryViewCache: nil,
 		}
 	})
@@ -119,7 +119,7 @@ func (dir *Dir) newFile(name string, entry *filer_pb.Entry) fs.Node {
 func (dir *Dir) newDirectory(fullpath util.FullPath, entry *filer_pb.Entry) fs.Node {

 	d := dir.wfs.fsNodeCache.EnsureFsNode(fullpath, func() fs.Node {
-		return &Dir{name: entry.Name, wfs: dir.wfs, entry: filer.FromPbEntry(dir.FullPath(), entry), parent: dir}
+		return &Dir{name: entry.Name, wfs: dir.wfs, entry: entry, parent: dir}
 	})
 	d.(*Dir).parent = dir // in case dir node was created later
 	return d
@@ -128,6 +128,10 @@ func (dir *Dir) newDirectory(fullpath util.FullPath, entry *filer_pb.Entry) fs.N
 func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
 	resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {

+	if dir.wfs.option.ReadOnly {
+		return nil, nil, fuse.EPERM
+	}
+
 	request, err := dir.doCreateEntry(req.Name, req.Mode, req.Uid, req.Gid, req.Flags&fuse.OpenExclusive != 0)

 	if err != nil {
@@ -148,6 +152,10 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,

 func (dir *Dir) Mknod(ctx context.Context, req *fuse.MknodRequest) (fs.Node, error) {

+	if dir.wfs.option.ReadOnly {
+		return nil, fuse.EPERM
+	}
+
 	request, err := dir.doCreateEntry(req.Name, req.Mode, req.Uid, req.Gid, false)

 	if err != nil {
@@ -202,6 +210,10 @@ func (dir *Dir) doCreateEntry(name string, mode os.FileMode, uid, gid uint32, ex

 func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {

+	if dir.wfs.option.ReadOnly {
+		return nil, fuse.EPERM
+	}
+
 	glog.V(4).Infof("mkdir %s: %s", dir.FullPath(), req.Name)

 	newEntry := &filer_pb.Entry{
@@ -356,6 +368,10 @@ func findFileType(mode uint16) fuse.DirentType {

 func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {

+	if dir.wfs.option.ReadOnly {
+		return fuse.EPERM
+	}
+
 	if !req.Dir {
 		return dir.removeOneFile(req)
 	}
@@ -429,6 +445,10 @@ func (dir *Dir) removeFolder(req *fuse.RemoveRequest) error {

 func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {

+	if dir.wfs.option.ReadOnly {
+		return fuse.EPERM
+	}
+
 	glog.V(4).Infof("%v dir setattr %+v", dir.FullPath(), req)

 	if err := dir.maybeLoadEntry(); err != nil {
@@ -436,19 +456,19 @@ func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fus
 	}

 	if req.Valid.Mode() {
-		dir.entry.Attr.Mode = req.Mode
+		dir.entry.Attributes.FileMode = uint32(req.Mode)
 	}

 	if req.Valid.Uid() {
-		dir.entry.Attr.Uid = req.Uid
+		dir.entry.Attributes.Uid = req.Uid
 	}

 	if req.Valid.Gid() {
-		dir.entry.Attr.Gid = req.Gid
+		dir.entry.Attributes.Gid = req.Gid
 	}

 	if req.Valid.Mtime() {
-		dir.entry.Attr.Mtime = req.Mtime
+		dir.entry.Attributes.Mtime = req.Mtime.Unix()
 	}

 	return dir.saveEntry()
@@ -457,6 +477,10 @@ func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fus

 func (dir *Dir) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {

+	if dir.wfs.option.ReadOnly {
+		return fuse.EPERM
+	}
+
 	glog.V(4).Infof("dir Setxattr %s: %s", dir.FullPath(), req.Name)

 	if err := dir.maybeLoadEntry(); err != nil {
@@ -473,6 +497,10 @@ func (dir *Dir) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {

 func (dir *Dir) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error {

+	if dir.wfs.option.ReadOnly {
+		return fuse.EPERM
+	}
+
 	glog.V(4).Infof("dir Removexattr %s: %s", dir.FullPath(), req.Name)

 	if err := dir.maybeLoadEntry(); err != nil {
@@ -527,14 +555,12 @@ func (dir *Dir) saveEntry() error {

 	return dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

-		pbEntry := dir.entry.ToProtoEntry()
-
-		dir.wfs.mapPbIdFromLocalToFiler(pbEntry)
-		defer dir.wfs.mapPbIdFromFilerToLocal(pbEntry)
+		dir.wfs.mapPbIdFromLocalToFiler(dir.entry)
+		defer dir.wfs.mapPbIdFromFilerToLocal(dir.entry)

 		request := &filer_pb.UpdateEntryRequest{
 			Directory:  parentDir,
-			Entry:      pbEntry,
+			Entry:      dir.entry,
 			Signatures: []int32{dir.wfs.signature},
 		}

@@ -552,5 +578,25 @@ func (dir *Dir) saveEntry() error {
 }

 func (dir *Dir) FullPath() string {
-	return string(dir.entry.FullPath)
+	var parts []string
+	for p := dir; p != nil; p = p.parent {
+		if strings.HasPrefix(p.name, "/") {
+			if len(p.name) > 1 {
+				parts = append(parts, p.name[1:])
+			}
+		} else {
+			parts = append(parts, p.name)
+		}
+	}
+
+	if len(parts) == 0 {
+		return "/"
+	}
+
+	var buf bytes.Buffer
+	for i := len(parts) - 1; i >= 0; i-- {
+		buf.WriteString("/")
+		buf.WriteString(parts[i])
+	}
+	return buf.String()
 }
@@ -24,6 +24,10 @@ const (

 func (dir *Dir) Link(ctx context.Context, req *fuse.LinkRequest, old fs.Node) (fs.Node, error) {

+	if dir.wfs.option.ReadOnly {
+		return nil, fuse.EPERM
+	}
+
 	oldFile, ok := old.(*File)
 	if !ok {
 		glog.Errorf("old node is not a file: %+v", old)
@@ -48,7 +52,7 @@ func (dir *Dir) Link(ctx context.Context, req *fuse.LinkRequest, old fs.Node) (f
 	oldEntry.HardLinkCounter++
 	updateOldEntryRequest := &filer_pb.UpdateEntryRequest{
 		Directory:  oldFile.dir.FullPath(),
-		Entry:      oldEntry.ToProtoEntry(),
+		Entry:      oldEntry,
 		Signatures: []int32{dir.wfs.signature},
 	}

@@ -58,7 +62,7 @@ func (dir *Dir) Link(ctx context.Context, req *fuse.LinkRequest, old fs.Node) (f
 		Entry: &filer_pb.Entry{
 			Name:            req.NewName,
 			IsDirectory:     false,
-			Attributes:      filer.EntryAttributeToPb(oldEntry),
+			Attributes:      oldEntry.Attributes,
 			Chunks:          oldEntry.Chunks,
 			Extended:        oldEntry.Extended,
 			HardLinkId:      oldEntry.HardLinkId,
@@ -105,6 +109,10 @@ func (dir *Dir) Link(ctx context.Context, req *fuse.LinkRequest, old fs.Node) (f

 func (dir *Dir) Symlink(ctx context.Context, req *fuse.SymlinkRequest) (fs.Node, error) {

+	if dir.wfs.option.ReadOnly {
+		return nil, fuse.EPERM
+	}
+
 	glog.V(4).Infof("Symlink: %v/%v to %v", dir.FullPath(), req.NewName, req.Target)

 	request := &filer_pb.CreateEntryRequest{
@@ -152,12 +160,12 @@ func (file *File) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (stri
 		return "", err
 	}

-	if entry.Attr.Mode&os.ModeSymlink == 0 {
+	if os.FileMode(entry.Attributes.FileMode)&os.ModeSymlink == 0 {
 		return "", fuse.Errno(syscall.EINVAL)
 	}

-	glog.V(4).Infof("Readlink: %v/%v => %v", file.dir.FullPath(), file.Name, entry.Attr.SymlinkTarget)
+	glog.V(4).Infof("Readlink: %v/%v => %v", file.dir.FullPath(), file.Name, entry.Attributes.SymlinkTarget)

-	return entry.Attr.SymlinkTarget, nil
+	return entry.Attributes.SymlinkTarget, nil

 }
@@ -13,6 +13,10 @@ import (

 func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirectory fs.Node) error {

+	if dir.wfs.option.ReadOnly {
+		return fuse.EPERM
+	}
+
 	newDir := newDirectory.(*Dir)

 	newPath := util.NewFullPath(newDir.FullPath(), req.NewName)
@@ -74,7 +74,7 @@ func (pages *ContinuousDirtyPages) saveExistingLargestPageToStorage() (hasSavedD
 		return false
 	}

-	fileSize := int64(entry.Attr.FileSize)
+	fileSize := int64(entry.Attributes.FileSize)

 	chunkSize := min(maxList.Size(), fileSize-maxList.Offset())
 	if chunkSize == 0 {
@@ -33,7 +33,7 @@ type File struct {
 	Name           string
 	dir            *Dir
 	wfs            *WFS
-	entry          *filer.Entry
+	entry          *filer_pb.Entry
 	entryLock      sync.RWMutex
 	entryViewCache []filer.VisibleInterval
 	isOpen         int
@@ -56,18 +56,22 @@ func (file *File) Attr(ctx context.Context, attr *fuse.Attr) (err error) {
 		}
 	}

+	if entry == nil {
+		return fuse.ENOENT
+	}
+
 	// attr.Inode = file.fullpath().AsInode()
 	attr.Valid = time.Second
-	attr.Mode = os.FileMode(entry.Attr.Mode)
-	attr.Size = filer.FileSize2(entry)
+	attr.Mode = os.FileMode(entry.Attributes.FileMode)
+	attr.Size = filer.FileSize(entry)
 	if file.isOpen > 0 {
-		attr.Size = entry.Attr.FileSize
+		attr.Size = entry.Attributes.FileSize
 		glog.V(4).Infof("file Attr %s, open:%v, size: %d", file.fullpath(), file.isOpen, attr.Size)
 	}
-	attr.Crtime = entry.Attr.Crtime
-	attr.Mtime = entry.Attr.Mtime
-	attr.Gid = entry.Attr.Gid
-	attr.Uid = entry.Attr.Uid
+	attr.Crtime = time.Unix(entry.Attributes.Crtime, 0)
+	attr.Mtime = time.Unix(entry.Attributes.Mtime, 0)
+	attr.Gid = entry.Attributes.Gid
+	attr.Uid = entry.Attributes.Uid
 	attr.Blocks = attr.Size/blockSize + 1
 	attr.BlockSize = uint32(file.wfs.option.ChunkSizeLimit)
 	if entry.HardLinkCounter > 0 {
@@ -106,6 +110,10 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op

 func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {

+	if file.wfs.option.ReadOnly {
+		return fuse.EPERM
+	}
+
 	glog.V(4).Infof("%v file setattr %+v", file.fullpath(), req)

 	entry, err := file.maybeLoadEntry(ctx)
@@ -126,7 +134,7 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
 	if req.Valid.Size() {

 		glog.V(4).Infof("%v file setattr set size=%v chunks=%d", file.fullpath(), req.Size, len(entry.Chunks))
-		if req.Size < filer.FileSize2(entry) {
+		if req.Size < filer.FileSize(entry) {
 			// fmt.Printf("truncate %v \n", fullPath)
 			var chunks []*filer_pb.FileChunk
 			var truncatedChunks []*filer_pb.FileChunk
@@ -147,34 +155,34 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
 			}
 			entry.Chunks = chunks
 			file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(file.wfs.LookupFn(), chunks)
-			file.reader = nil
+			file.setReader(nil)
 		}
-		entry.Attr.FileSize = req.Size
+		entry.Attributes.FileSize = req.Size
 		file.dirtyMetadata = true
 	}

 	if req.Valid.Mode() {
-		entry.Attr.Mode = req.Mode
+		entry.Attributes.FileMode = uint32(req.Mode)
 		file.dirtyMetadata = true
 	}

 	if req.Valid.Uid() {
-		entry.Attr.Uid = req.Uid
+		entry.Attributes.Uid = req.Uid
 		file.dirtyMetadata = true
 	}

 	if req.Valid.Gid() {
-		entry.Attr.Gid = req.Gid
+		entry.Attributes.Gid = req.Gid
 		file.dirtyMetadata = true
 	}

 	if req.Valid.Crtime() {
-		entry.Attr.Crtime = req.Crtime
+		entry.Attributes.Crtime = req.Crtime.Unix()
 		file.dirtyMetadata = true
 	}

 	if req.Valid.Mtime() {
-		entry.Attr.Mtime = req.Mtime
+		entry.Attributes.Mtime = req.Mtime.Unix()
 		file.dirtyMetadata = true
 	}

@@ -196,6 +204,10 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f

 func (file *File) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {

+	if file.wfs.option.ReadOnly {
+		return fuse.EPERM
+	}
+
 	glog.V(4).Infof("file Setxattr %s: %s", file.fullpath(), req.Name)

 	entry, err := file.maybeLoadEntry(ctx)
@@ -213,6 +225,10 @@ func (file *File) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error

 func (file *File) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error {

+	if file.wfs.option.ReadOnly {
+		return fuse.EPERM
+	}
+
 	glog.V(4).Infof("file Removexattr %s: %s", file.fullpath(), req.Name)

 	entry, err := file.maybeLoadEntry(ctx)
@@ -257,9 +273,11 @@ func (file *File) Forget() {
 	t := util.NewFullPath(file.dir.FullPath(), file.Name)
 	glog.V(4).Infof("Forget file %s", t)
 	file.wfs.fsNodeCache.DeleteFsNode(t)
+	file.wfs.ReleaseHandle(t, 0)
+	file.setReader(nil)
 }

-func (file *File) maybeLoadEntry(ctx context.Context) (entry *filer.Entry, err error) {
+func (file *File) maybeLoadEntry(ctx context.Context) (entry *filer_pb.Entry, err error) {
 	entry = file.getEntry()
 	if file.isOpen > 0 {
 		return entry, nil
@@ -323,19 +341,29 @@ func (file *File) addChunks(chunks []*filer_pb.FileChunk) {
 		file.entryViewCache = filer.MergeIntoVisibles(file.entryViewCache, chunk)
 	}

-	file.reader = nil
+	file.setReader(nil)

 	glog.V(4).Infof("%s existing %d chunks adds %d more", file.fullpath(), len(entry.Chunks), len(chunks))

 	entry.Chunks = append(entry.Chunks, newChunks...)
 }

-func (file *File) setEntry(entry *filer.Entry) {
+func (file *File) setReader(reader io.ReaderAt) {
+	r := file.reader
+	if r != nil {
+		if closer, ok := r.(io.Closer); ok {
+			closer.Close()
+		}
+	}
+	file.reader = reader
+}
+
+func (file *File) setEntry(entry *filer_pb.Entry) {
 	file.entryLock.Lock()
 	defer file.entryLock.Unlock()
 	file.entry = entry
 	file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(file.wfs.LookupFn(), entry.Chunks)
-	file.reader = nil
+	file.setReader(nil)
 }

 func (file *File) clearEntry() {
@@ -343,20 +371,18 @@ func (file *File) clearEntry() {
 	defer file.entryLock.Unlock()
 	file.entry = nil
 	file.entryViewCache = nil
-	file.reader = nil
+	file.setReader(nil)
 }

-func (file *File) saveEntry(entry *filer.Entry) error {
+func (file *File) saveEntry(entry *filer_pb.Entry) error {
 	return file.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

-		pbEntry := entry.ToProtoEntry()
-
-		file.wfs.mapPbIdFromLocalToFiler(pbEntry)
-		defer file.wfs.mapPbIdFromFilerToLocal(pbEntry)
+		file.wfs.mapPbIdFromLocalToFiler(entry)
+		defer file.wfs.mapPbIdFromFilerToLocal(entry)

 		request := &filer_pb.UpdateEntryRequest{
 			Directory:  file.dir.FullPath(),
-			Entry:      pbEntry,
+			Entry:      entry,
 			Signatures: []int32{file.wfs.signature},
 		}

@@ -373,7 +399,7 @@ func (file *File) saveEntry(entry *filer.Entry) error {
 	})
 }

-func (file *File) getEntry() *filer.Entry {
+func (file *File) getEntry() *filer_pb.Entry {
 	file.entryLock.RLock()
 	defer file.entryLock.RUnlock()
 	return file.entry
@@ -6,6 +6,7 @@ import (
 	"io"
 	"math"
 	"net/http"
+	"os"
 	"sync"
 	"time"

@@ -22,7 +23,7 @@ type FileHandle struct {
 	dirtyPages  *ContinuousDirtyPages
 	contentType string
 	handle      uint64
-	sync.RWMutex
+	sync.Mutex

 	f         *File
 	RequestId fuse.RequestID // unique ID for request
@@ -41,7 +42,7 @@ func newFileHandle(file *File, uid, gid uint32) *FileHandle {
 	}
 	entry := fh.f.getEntry()
 	if entry != nil {
-		entry.Attr.FileSize = filer.FileSize2(entry)
+		entry.Attributes.FileSize = filer.FileSize(entry)
 	}

 	return fh
@@ -58,8 +59,8 @@ var _ = fs.HandleReleaser(&FileHandle{})
 func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {

 	glog.V(4).Infof("%s read fh %d: [%d,%d) size %d resp.Data cap=%d", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size), req.Size, cap(resp.Data))
-	fh.RLock()
-	defer fh.RUnlock()
+	fh.Lock()
+	defer fh.Unlock()

 	if req.Size <= 0 {
 		return nil
@@ -109,7 +110,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {
 		return 0, io.EOF
 	}

-	fileSize := int64(filer.FileSize2(entry))
+	fileSize := int64(filer.FileSize(entry))
 	fileFullPath := fh.f.fullpath()

 	if fileSize == 0 {
@@ -129,7 +130,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {
 		if chunkResolveErr != nil {
 			return 0, fmt.Errorf("fail to resolve chunk manifest: %v", chunkResolveErr)
 		}
-		fh.f.reader = nil
+		fh.f.setReader(nil)
 	}

 	reader := fh.f.reader
@@ -137,7 +138,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {
 		chunkViews := filer.ViewFromVisibleIntervals(fh.f.entryViewCache, 0, math.MaxInt64)
 		reader = filer.NewChunkReaderAtFromClient(fh.f.wfs.LookupFn(), chunkViews, fh.f.wfs.chunkCache, fileSize)
 	}
-	fh.f.reader = reader
+	fh.f.setReader(reader)

 	totalRead, err := reader.ReadAt(buff, offset)

@@ -153,6 +154,10 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {
 // Write to the file handle
 func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error {

+	if fh.f.wfs.option.ReadOnly {
+		return fuse.EPERM
+	}
+
 	fh.Lock()
 	defer fh.Unlock()

@@ -170,7 +175,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f
 	}

 	entry.Content = nil
-	entry.Attr.FileSize = uint64(max(req.Offset+int64(len(data)), int64(entry.Attr.FileSize)))
+	entry.Attributes.FileSize = uint64(max(req.Offset+int64(len(data)), int64(entry.Attributes.FileSize)))
 	glog.V(4).Infof("%v write [%d,%d) %d", fh.f.fullpath(), req.Offset, req.Offset+int64(len(req.Data)), len(req.Data))

 	fh.dirtyPages.AddPage(req.Offset, data)
@@ -195,6 +200,8 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err
 	fh.Lock()
 	defer fh.Unlock()

+	fh.f.entryViewCache = nil
+
 	if fh.f.isOpen <= 0 {
 		glog.V(0).Infof("Release reset %s open count %d => %d", fh.f.Name, fh.f.isOpen, 0)
 		fh.f.isOpen = 0
@@ -206,12 +213,7 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err
 		fh.f.isOpen--

 		fh.f.wfs.ReleaseHandle(fh.f.fullpath(), fuse.HandleID(fh.handle))
-		if closer, ok := fh.f.reader.(io.Closer); ok {
-			if closer != nil {
-				closer.Close()
-			}
-		}
-		fh.f.reader = nil
+		fh.f.setReader(nil)
 	}

 	return nil
@@ -258,24 +260,26 @@ func (fh *FileHandle) doFlush(ctx context.Context, header fuse.Header) error {
 			return nil
 		}

-		entry.Attr.Mime = fh.contentType
-		if entry.Attr.Uid == 0 {
-			entry.Attr.Uid = header.Uid
+		if entry.Attributes != nil {
+			entry.Attributes.Mime = fh.contentType
+			if entry.Attributes.Uid == 0 {
+				entry.Attributes.Uid = header.Uid
+			}
+			if entry.Attributes.Gid == 0 {
+				entry.Attributes.Gid = header.Gid
+			}
+			if entry.Attributes.Crtime == 0 {
+				entry.Attributes.Crtime = time.Now().Unix()
+			}
+			entry.Attributes.Mtime = time.Now().Unix()
+			entry.Attributes.FileMode = uint32(os.FileMode(entry.Attributes.FileMode) &^ fh.f.wfs.option.Umask)
+			entry.Attributes.Collection = fh.dirtyPages.collection
+			entry.Attributes.Replication = fh.dirtyPages.replication
 		}
-		if entry.Attr.Gid == 0 {
-			entry.Attr.Gid = header.Gid
-		}
-		if entry.Attr.Crtime.IsZero() {
-			entry.Attr.Crtime = time.Now()
-		}
-		entry.Attr.Mtime = time.Now()
-		entry.Attr.Mode = entry.Attr.Mode &^ fh.f.wfs.option.Umask
-		entry.Attr.Collection = fh.dirtyPages.collection
-		entry.Attr.Replication = fh.dirtyPages.replication

 		request := &filer_pb.CreateEntryRequest{
 			Directory:  fh.f.dir.FullPath(),
-			Entry:      entry.ToProtoEntry(),
+			Entry:      entry,
 			Signatures: []int32{fh.f.wfs.signature},
 		}

@@ -126,7 +126,7 @@ func (c *FsCache) Move(oldPath util.FullPath, newPath util.FullPath) *FsNode {
 		f.Name = target.name
 		entry := f.getEntry()
 		if entry != nil {
-			entry.FullPath = newPath
+			entry.Name = f.Name
 		}
 	}
 	parent.disconnectChild(target)
@@ -43,6 +43,7 @@ type Option struct {
 	DataCenter         string
 	EntryCacheTtl      time.Duration
 	Umask              os.FileMode
+	ReadOnly           bool

 	MountUid   uint32
 	MountGid   uint32
@@ -131,7 +132,7 @@ func NewSeaweedFileSystem(option *Option) *WFS {
 	})

 	entry, _ := filer_pb.GetEntry(wfs, util.FullPath(wfs.option.FilerMountRootPath))
-	wfs.root = &Dir{name: wfs.option.FilerMountRootPath, wfs: wfs, entry: filer.FromPbEntry(wfs.option.FilerMountRootPath, entry)}
+	wfs.root = &Dir{name: wfs.option.FilerMountRootPath, wfs: wfs, entry: entry}
 	wfs.fsNodeCache = newFsCache(wfs.root)

 	if wfs.option.ConcurrentWriters > 0 {
@@ -2,7 +2,6 @@ package filesys

 import (
 	"context"
-	"github.com/chrislusf/seaweedfs/weed/filer"

 	"github.com/seaweedfs/fuse"

@@ -11,7 +10,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 )

-func getxattr(entry *filer.Entry, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
+func getxattr(entry *filer_pb.Entry, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {

 	if entry == nil {
 		return fuse.ErrNoXattr
@@ -39,7 +38,7 @@ func getxattr(entry *filer.Entry, req *fuse.GetxattrRequest, resp *fuse.Getxattr

 }

-func setxattr(entry *filer.Entry, req *fuse.SetxattrRequest) error {
+func setxattr(entry *filer_pb.Entry, req *fuse.SetxattrRequest) error {

 	if entry == nil {
 		return fuse.EIO
@@ -62,7 +61,7 @@ func setxattr(entry *filer.Entry, req *fuse.SetxattrRequest) error {

 }

-func removexattr(entry *filer.Entry, req *fuse.RemovexattrRequest) error {
+func removexattr(entry *filer_pb.Entry, req *fuse.RemovexattrRequest) error {

 	if entry == nil {
 		return fuse.ErrNoXattr
@@ -84,7 +83,7 @@ func removexattr(entry *filer.Entry, req *fuse.RemovexattrRequest) error {

 }

-func listxattr(entry *filer.Entry, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {
+func listxattr(entry *filer_pb.Entry, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {

 	if entry == nil {
 		return fuse.EIO
@@ -109,7 +108,7 @@ func listxattr(entry *filer.Entry, req *fuse.ListxattrRequest, resp *fuse.Listxa

 }

-func (wfs *WFS) maybeLoadEntry(dir, name string) (entry *filer.Entry, err error) {
+func (wfs *WFS) maybeLoadEntry(dir, name string) (entry *filer_pb.Entry, err error) {

 	fullpath := util.NewFullPath(dir, name)
 	// glog.V(3).Infof("read entry cache miss %s", fullpath)
@@ -120,5 +119,5 @@ func (wfs *WFS) maybeLoadEntry(dir, name string) (entry *filer.Entry, err error)
 	if cacheErr == filer_pb.ErrNotFound {
 		return nil, fuse.ENOENT
 	}
-	return cachedEntry, cacheErr
+	return cachedEntry.ToProtoEntry(), cacheErr
 }
@@ -0,0 +1,100 @@
+package iamapi
+
+import (
+	"bytes"
+	"encoding/xml"
+	"fmt"
+	"strconv"
+
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
+
+	"github.com/aws/aws-sdk-go/service/iam"
+)
+
+type mimeType string
+
+const (
+	mimeNone mimeType = ""
+	mimeXML  mimeType = "application/xml"
+)
+
+func setCommonHeaders(w http.ResponseWriter) {
+	w.Header().Set("x-amz-request-id", fmt.Sprintf("%d", time.Now().UnixNano()))
+	w.Header().Set("Accept-Ranges", "bytes")
+}
+
+// Encodes the response headers into XML format.
+func encodeResponse(response interface{}) []byte {
+	var bytesBuffer bytes.Buffer
+	bytesBuffer.WriteString(xml.Header)
+	e := xml.NewEncoder(&bytesBuffer)
+	e.Encode(response)
+	return bytesBuffer.Bytes()
+}
+
+// If none of the http routes match respond with MethodNotAllowed
+func notFoundHandler(w http.ResponseWriter, r *http.Request) {
+	glog.V(0).Infof("unsupported %s %s", r.Method, r.RequestURI)
+	writeErrorResponse(w, s3err.ErrMethodNotAllowed, r.URL)
+}
+
+func writeErrorResponse(w http.ResponseWriter, errorCode s3err.ErrorCode, reqURL *url.URL) {
+	apiError := s3err.GetAPIError(errorCode)
+	errorResponse := getRESTErrorResponse(apiError, reqURL.Path)
+	encodedErrorResponse := encodeResponse(errorResponse)
+	writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeXML)
+}
+
+func writeIamErrorResponse(w http.ResponseWriter, err error, object string, value string) {
+	errCode := err.Error()
+	errorResp := ErrorResponse{}
+	errorResp.Error.Type = "Sender"
+	errorResp.Error.Code = &errCode
+	glog.Errorf("Response %+v", err)
+	switch errCode {
+	case iam.ErrCodeNoSuchEntityException:
+		msg := fmt.Sprintf("The %s with name %s cannot be found.", object, value)
+		errorResp.Error.Message = &msg
+		writeResponse(w, http.StatusNotFound, encodeResponse(errorResp), mimeXML)
+	default:
+		writeResponse(w, http.StatusInternalServerError, encodeResponse(errorResp), mimeXML)
+
+	}
+}
+
+func getRESTErrorResponse(err s3err.APIError, resource string) s3err.RESTErrorResponse {
+	return s3err.RESTErrorResponse{
+		Code:      err.Code,
+		Message:   err.Description,
+		Resource:  resource,
+		RequestID: fmt.Sprintf("%d", time.Now().UnixNano()),
+	}
+}
+
+func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
+	setCommonHeaders(w)
+	if response != nil {
+		w.Header().Set("Content-Length", strconv.Itoa(len(response)))
+	}
+	if mType != mimeNone {
+		w.Header().Set("Content-Type", string(mType))
+	}
+	w.WriteHeader(statusCode)
+	if response != nil {
+		glog.V(4).Infof("status %d %s: %s", statusCode, mType, string(response))
+		_, err := w.Write(response)
+		if err != nil {
+			glog.V(0).Infof("write err: %v", err)
+		}
+		w.(http.Flusher).Flush()
+	}
+}
+
+func writeSuccessResponseXML(w http.ResponseWriter, response []byte) {
+	writeResponse(w, http.StatusOK, response, mimeXML)
+}
@@ -0,0 +1,322 @@
+package iamapi
+
+import (
+	"crypto/sha1"
+	"encoding/json"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
+	"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
+	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
+	"math/rand"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/iam"
+)
+
+const (
+	charsetUpper = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+	charset      = charsetUpper + "abcdefghijklmnopqrstuvwxyz/"
+)
+
+var (
+	seededRand *rand.Rand = rand.New(
+		rand.NewSource(time.Now().UnixNano()))
+	policyDocuments = map[string]*PolicyDocument{}
+)
+
+type Statement struct {
+	Effect   string   `json:"Effect"`
+	Action   []string `json:"Action"`
+	Resource []string `json:"Resource"`
+}
+
+type PolicyDocument struct {
+	Version   string       `json:"Version"`
+	Statement []*Statement `json:"Statement"`
+}
+
+func Hash(s *string) string {
+	h := sha1.New()
+	h.Write([]byte(*s))
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+func StringWithCharset(length int, charset string) string {
+	b := make([]byte, length)
+	for i := range b {
+		b[i] = charset[seededRand.Intn(len(charset))]
+	}
+	return string(b)
+}
+
+func (iama *IamApiServer) ListUsers(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp ListUsersResponse) {
+	for _, ident := range s3cfg.Identities {
+		resp.ListUsersResult.Users = append(resp.ListUsersResult.Users, &iam.User{UserName: &ident.Name})
+	}
+	return resp
+}
+
+func (iama *IamApiServer) ListAccessKeys(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp ListAccessKeysResponse) {
+	status := iam.StatusTypeActive
+	for _, ident := range s3cfg.Identities {
+		for _, cred := range ident.Credentials {
+			resp.ListAccessKeysResult.AccessKeyMetadata = append(resp.ListAccessKeysResult.AccessKeyMetadata,
+				&iam.AccessKeyMetadata{UserName: &ident.Name, AccessKeyId: &cred.AccessKey, Status: &status},
+			)
+		}
+	}
+	return resp
+}
+
+func (iama *IamApiServer) CreateUser(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp CreateUserResponse) {
+	userName := values.Get("UserName")
+	resp.CreateUserResult.User.UserName = &userName
+	s3cfg.Identities = append(s3cfg.Identities, &iam_pb.Identity{Name: userName})
+	return resp
+}
+
+func (iama *IamApiServer) DeleteUser(s3cfg *iam_pb.S3ApiConfiguration, userName string) (resp DeleteUserResponse, err error) {
+	for i, ident := range s3cfg.Identities {
+		if userName == ident.Name {
+			ident.Credentials = append(ident.Credentials[:i], ident.Credentials[i+1:]...)
+			return resp, nil
+		}
+	}
+	return resp, fmt.Errorf(iam.ErrCodeNoSuchEntityException)
+}
+
+func (iama *IamApiServer) GetUser(s3cfg *iam_pb.S3ApiConfiguration, userName string) (resp GetUserResponse, err error) {
+	for _, ident := range s3cfg.Identities {
+		if userName == ident.Name {
+			resp.GetUserResult.User = iam.User{UserName: &ident.Name}
+			return resp, nil
+		}
+	}
+	return resp, fmt.Errorf(iam.ErrCodeNoSuchEntityException)
+}
+
+func GetPolicyDocument(policy *string) (policyDocument PolicyDocument, err error) {
+	if err = json.Unmarshal([]byte(*policy), &policyDocument); err != nil {
+		return PolicyDocument{}, err
+	}
+	return policyDocument, err
+}
+
+func (iama *IamApiServer) CreatePolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp CreatePolicyResponse, err error) {
+	policyName := values.Get("PolicyName")
+	policyDocumentString := values.Get("PolicyDocument")
+	policyDocument, err := GetPolicyDocument(&policyDocumentString)
+	if err != nil {
+		return CreatePolicyResponse{}, err
+	}
+	policyId := Hash(&policyDocumentString)
+	arn := fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)
+	resp.CreatePolicyResult.Policy.PolicyName = &policyName
+	resp.CreatePolicyResult.Policy.Arn = &arn
+	resp.CreatePolicyResult.Policy.PolicyId = &policyId
+	policyDocuments[policyName] = &policyDocument
+	return resp, nil
+}
+
+func (iama *IamApiServer) PutUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp PutUserPolicyResponse, err error) {
+	userName := values.Get("UserName")
+	policyName := values.Get("PolicyName")
+	policyDocumentString := values.Get("PolicyDocument")
+	policyDocument, err := GetPolicyDocument(&policyDocumentString)
+	if err != nil {
+		return PutUserPolicyResponse{}, err
+	}
+	policyDocuments[policyName] = &policyDocument
+	actions := GetActions(&policyDocument)
+	for _, ident := range s3cfg.Identities {
+		if userName == ident.Name {
+			for _, action := range actions {
+				ident.Actions = append(ident.Actions, action)
+			}
+			break
+		}
+	}
+	return resp, nil
+}
+
+func MapAction(action string) string {
+	switch action {
+	case "*":
+		return s3_constants.ACTION_ADMIN
+	case "Put*":
+		return s3_constants.ACTION_WRITE
+	case "Get*":
+		return s3_constants.ACTION_READ
+	case "List*":
+		return s3_constants.ACTION_LIST
+	default:
+		return s3_constants.ACTION_TAGGING
+	}
+}
+
+func GetActions(policy *PolicyDocument) (actions []string) {
+	for _, statement := range policy.Statement {
+		if statement.Effect != "Allow" {
+			continue
+		}
+		for _, resource := range statement.Resource {
+			// Parse "arn:aws:s3:::my-bucket/shared/*"
+			res := strings.Split(resource, ":")
+			if len(res) != 6 || res[0] != "arn" || res[1] != "aws" || res[2] != "s3" {
+				glog.Infof("not match resource: %s", res)
+				continue
+			}
+			for _, action := range statement.Action {
+				// Parse "s3:Get*"
+				act := strings.Split(action, ":")
+				if len(act) != 2 || act[0] != "s3" {
+					glog.Infof("not match action: %s", act)
+					continue
+				}
+				if res[5] == "*" {
+					actions = append(actions, MapAction(act[1]))
+					continue
+				}
+				// Parse my-bucket/shared/*
+				path := strings.Split(res[5], "/")
+				if len(path) != 2 || path[1] != "*" {
+					glog.Infof("not match bucket: %s", path)
+					continue
+				}
+				actions = append(actions, fmt.Sprintf("%s:%s", MapAction(act[1]), path[0]))
+			}
+		}
+	}
+	return actions
+}
+
+func (iama *IamApiServer) CreateAccessKey(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp CreateAccessKeyResponse) {
+	userName := values.Get("UserName")
+	status := iam.StatusTypeActive
+	accessKeyId := StringWithCharset(21, charsetUpper)
+	secretAccessKey := StringWithCharset(42, charset)
+	resp.CreateAccessKeyResult.AccessKey.AccessKeyId = &accessKeyId
+	resp.CreateAccessKeyResult.AccessKey.SecretAccessKey = &secretAccessKey
+	resp.CreateAccessKeyResult.AccessKey.UserName = &userName
+	resp.CreateAccessKeyResult.AccessKey.Status = &status
+	changed := false
+	for _, ident := range s3cfg.Identities {
+		if userName == ident.Name {
+			ident.Credentials = append(ident.Credentials,
+				&iam_pb.Credential{AccessKey: accessKeyId, SecretKey: secretAccessKey})
+			changed = true
+			break
+		}
+	}
+	if !changed {
+		s3cfg.Identities = append(s3cfg.Identities,
+			&iam_pb.Identity{Name: userName,
+				Credentials: []*iam_pb.Credential{
+					{
+						AccessKey: accessKeyId,
+						SecretKey: secretAccessKey,
+					},
+				},
+			},
+		)
+	}
+	return resp
+}
+
+func (iama *IamApiServer) DeleteAccessKey(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp DeleteAccessKeyResponse) {
+	userName := values.Get("UserName")
+	accessKeyId := values.Get("AccessKeyId")
+	for _, ident := range s3cfg.Identities {
+		if userName == ident.Name {
+			for i, cred := range ident.Credentials {
+				if cred.AccessKey == accessKeyId {
+					ident.Credentials = append(ident.Credentials[:i], ident.Credentials[i+1:]...)
+					break
+				}
+			}
+			break
+		}
+	}
+	return resp
+}
+
+func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
+	if err := r.ParseForm(); err != nil {
+		writeErrorResponse(w, s3err.ErrInvalidRequest, r.URL)
+		return
+	}
+	values := r.PostForm
+	var s3cfgLock sync.RWMutex
+	s3cfgLock.RLock()
+	s3cfg := &iam_pb.S3ApiConfiguration{}
+	if err := iama.s3ApiConfig.GetS3ApiConfiguration(s3cfg); err != nil {
+		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
+		return
+	}
+	s3cfgLock.RUnlock()
+
+	glog.V(4).Infof("DoActions: %+v", values)
+	var response interface{}
+	var err error
+	changed := true
+	switch r.Form.Get("Action") {
+	case "ListUsers":
+		response = iama.ListUsers(s3cfg, values)
+		changed = false
+	case "ListAccessKeys":
+		response = iama.ListAccessKeys(s3cfg, values)
+		changed = false
+	case "CreateUser":
+		response = iama.CreateUser(s3cfg, values)
+	case "GetUser":
+		userName := values.Get("UserName")
+		response, err = iama.GetUser(s3cfg, userName)
+		if err != nil {
+			writeIamErrorResponse(w, err, "user", userName)
+			return
+		}
+	case "DeleteUser":
+		userName := values.Get("UserName")
+		response, err = iama.DeleteUser(s3cfg, userName)
+		if err != nil {
+			writeIamErrorResponse(w, err, "user", userName)
+			return
+		}
+	case "CreateAccessKey":
+		response = iama.CreateAccessKey(s3cfg, values)
+	case "DeleteAccessKey":
+		response = iama.DeleteAccessKey(s3cfg, values)
+	case "CreatePolicy":
+		response, err = iama.CreatePolicy(s3cfg, values)
+		if err != nil {
+			glog.Errorf("CreatePolicy:  %+v", err)
+			writeErrorResponse(w, s3err.ErrInvalidRequest, r.URL)
+			return
+		}
+	case "PutUserPolicy":
+		response, err = iama.PutUserPolicy(s3cfg, values)
+		if err != nil {
+			glog.Errorf("PutUserPolicy:  %+v", err)
+			writeErrorResponse(w, s3err.ErrInvalidRequest, r.URL)
+			return
+		}
+	default:
+		writeErrorResponse(w, s3err.ErrNotImplemented, r.URL)
+		return
+	}
+	if changed {
+		s3cfgLock.Lock()
+		err := iama.s3ApiConfig.PutS3ApiConfiguration(s3cfg)
+		s3cfgLock.Unlock()
+		if err != nil {
+			writeErrorResponse(w, s3err.ErrInternalError, r.URL)
+			return
+		}
+	}
+	writeSuccessResponseXML(w, encodeResponse(response))
+}
@@ -0,0 +1,93 @@
+package iamapi
+
+import (
+	"encoding/xml"
+	"fmt"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/iam"
+)
+
+type CommonResponse struct {
+	ResponseMetadata struct {
+		RequestId string `xml:"RequestId"`
+	} `xml:"ResponseMetadata"`
+}
+
+type ListUsersResponse struct {
+	CommonResponse
+	XMLName         xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListUsersResponse"`
+	ListUsersResult struct {
+		Users       []*iam.User `xml:"Users>member"`
+		IsTruncated bool        `xml:"IsTruncated"`
+	} `xml:"ListUsersResult"`
+}
+
+type ListAccessKeysResponse struct {
+	CommonResponse
+	XMLName              xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListAccessKeysResponse"`
+	ListAccessKeysResult struct {
+		AccessKeyMetadata []*iam.AccessKeyMetadata `xml:"AccessKeyMetadata>member"`
+		IsTruncated       bool                     `xml:"IsTruncated"`
+	} `xml:"ListAccessKeysResult"`
+}
+
+type DeleteAccessKeyResponse struct {
+	CommonResponse
+	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteAccessKeyResponse"`
+}
+
+type CreatePolicyResponse struct {
+	CommonResponse
+	XMLName            xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreatePolicyResponse"`
+	CreatePolicyResult struct {
+		Policy iam.Policy `xml:"Policy"`
+	} `xml:"CreatePolicyResult"`
+}
+
+type CreateUserResponse struct {
+	CommonResponse
+	XMLName          xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateUserResponse"`
+	CreateUserResult struct {
+		User iam.User `xml:"User"`
+	} `xml:"CreateUserResult"`
+}
+
+type DeleteUserResponse struct {
+	CommonResponse
+	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteUserResponse"`
+}
+
+type GetUserResponse struct {
+	CommonResponse
+	XMLName       xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ GetUserResponse"`
+	GetUserResult struct {
+		User iam.User `xml:"User"`
+	} `xml:"GetUserResult"`
+}
+
+type CreateAccessKeyResponse struct {
+	CommonResponse
+	XMLName               xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateAccessKeyResponse"`
+	CreateAccessKeyResult struct {
+		AccessKey iam.AccessKey `xml:"AccessKey"`
+	} `xml:"CreateAccessKeyResult"`
+}
+
+type PutUserPolicyResponse struct {
+	CommonResponse
+	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ PutUserPolicyResponse"`
+}
+
+type ErrorResponse struct {
+	CommonResponse
+	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ErrorResponse"`
+	Error   struct {
+		iam.ErrorDetails
+		Type string `xml:"Type"`
+	} `xml:"Error"`
+}
+
+func (r *CommonResponse) SetRequestId() {
+	r.ResponseMetadata.RequestId = fmt.Sprintf("%d", time.Now().UnixNano())
+}
@@ -0,0 +1,104 @@
+package iamapi
+
+// https://docs.aws.amazon.com/cli/latest/reference/iam/list-roles.html
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
+	"github.com/chrislusf/seaweedfs/weed/wdclient"
+	"github.com/gorilla/mux"
+	"google.golang.org/grpc"
+	"net/http"
+	"strings"
+)
+
+type IamS3ApiConfig interface {
+	GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error)
+	PutS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error)
+}
+
+type IamS3ApiConfigure struct {
+	option       *IamServerOption
+	masterClient *wdclient.MasterClient
+}
+
+type IamServerOption struct {
+	Masters          string
+	Filer            string
+	Port             int
+	FilerGrpcAddress string
+	GrpcDialOption   grpc.DialOption
+}
+
+type IamApiServer struct {
+	s3ApiConfig IamS3ApiConfig
+	filerclient *filer_pb.SeaweedFilerClient
+}
+
+var s3ApiConfigure IamS3ApiConfig
+
+func NewIamApiServer(router *mux.Router, option *IamServerOption) (iamApiServer *IamApiServer, err error) {
+	s3ApiConfigure = IamS3ApiConfigure{
+		option:       option,
+		masterClient: wdclient.NewMasterClient(option.GrpcDialOption, pb.AdminShellClient, "", 0, "", strings.Split(option.Masters, ",")),
+	}
+
+	iamApiServer = &IamApiServer{
+		s3ApiConfig: s3ApiConfigure,
+	}
+
+	iamApiServer.registerRouter(router)
+
+	return iamApiServer, nil
+}
+
+func (iama *IamApiServer) registerRouter(router *mux.Router) {
+	// API Router
+	apiRouter := router.PathPrefix("/").Subrouter()
+	// ListBuckets
+
+	// apiRouter.Methods("GET").Path("/").HandlerFunc(track(s3a.iam.Auth(s3a.ListBucketsHandler, ACTION_ADMIN), "LIST"))
+	apiRouter.Path("/").Methods("POST").HandlerFunc(iama.DoActions)
+	// NotFound
+	apiRouter.NotFoundHandler = http.HandlerFunc(notFoundHandler)
+}
+
+func (iam IamS3ApiConfigure) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error) {
+	var buf bytes.Buffer
+	err = pb.WithGrpcFilerClient(iam.option.FilerGrpcAddress, iam.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		if err = filer.ReadEntry(iam.masterClient, client, filer.IamConfigDirecotry, filer.IamIdentityFile, &buf); err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	if buf.Len() > 0 {
+		if err = filer.ParseS3ConfigurationFromBytes(buf.Bytes(), s3cfg); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (iam IamS3ApiConfigure) PutS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error) {
+	buf := bytes.Buffer{}
+	if err := filer.S3ConfigurationToText(&buf, s3cfg); err != nil {
+		return fmt.Errorf("S3ConfigurationToText: %s", err)
+	}
+	return pb.WithGrpcFilerClient(
+		iam.option.FilerGrpcAddress,
+		iam.option.GrpcDialOption,
+		func(client filer_pb.SeaweedFilerClient) error {
+			if err := filer.SaveInsideFiler(client, filer.IamConfigDirecotry, filer.IamIdentityFile, buf.Bytes()); err != nil {
+				return err
+			}
+			return nil
+		},
+	)
+}
@@ -0,0 +1,157 @@
+package iamapi
+
+import (
+	"encoding/xml"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
+	"github.com/gorilla/mux"
+	"github.com/stretchr/testify/assert"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+var S3config iam_pb.S3ApiConfiguration
+var GetS3ApiConfiguration func(s3cfg *iam_pb.S3ApiConfiguration) (err error)
+var PutS3ApiConfiguration func(s3cfg *iam_pb.S3ApiConfiguration) (err error)
+
+type iamS3ApiConfigureMock struct{}
+
+func (iam iamS3ApiConfigureMock) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error) {
+	s3cfg = &S3config
+	return nil
+}
+
+func (iam iamS3ApiConfigureMock) PutS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error) {
+	S3config = *s3cfg
+	return nil
+}
+
+var a = IamApiServer{}
+
+func TestCreateUser(t *testing.T) {
+	userName := aws.String("Test")
+	params := &iam.CreateUserInput{UserName: userName}
+	req, _ := iam.New(session.New()).CreateUserRequest(params)
+	_ = req.Build()
+	out := CreateUserResponse{}
+	response, err := executeRequest(req.HTTPRequest, out)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, http.StatusOK, response.Code)
+	//assert.Equal(t, out.XMLName, "lol")
+}
+
+func TestListUsers(t *testing.T) {
+	params := &iam.ListUsersInput{}
+	req, _ := iam.New(session.New()).ListUsersRequest(params)
+	_ = req.Build()
+	out := ListUsersResponse{}
+	response, err := executeRequest(req.HTTPRequest, out)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, http.StatusOK, response.Code)
+}
+
+func TestListAccessKeys(t *testing.T) {
+	svc := iam.New(session.New())
+	params := &iam.ListAccessKeysInput{}
+	req, _ := svc.ListAccessKeysRequest(params)
+	_ = req.Build()
+	out := ListAccessKeysResponse{}
+	response, err := executeRequest(req.HTTPRequest, out)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, http.StatusOK, response.Code)
+}
+
+func TestDeleteUser(t *testing.T) {
+	userName := aws.String("Test")
+	params := &iam.DeleteUserInput{UserName: userName}
+	req, _ := iam.New(session.New()).DeleteUserRequest(params)
+	_ = req.Build()
+	out := DeleteUserResponse{}
+	response, err := executeRequest(req.HTTPRequest, out)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, http.StatusNotFound, response.Code)
+}
+
+func TestGetUser(t *testing.T) {
+	userName := aws.String("Test")
+	params := &iam.GetUserInput{UserName: userName}
+	req, _ := iam.New(session.New()).GetUserRequest(params)
+	_ = req.Build()
+	out := GetUserResponse{}
+	response, err := executeRequest(req.HTTPRequest, out)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, http.StatusNotFound, response.Code)
+}
+
+// Todo flat statement
+func TestCreatePolicy(t *testing.T) {
+	params := &iam.CreatePolicyInput{
+		PolicyName: aws.String("S3-read-only-example-bucket"),
+		PolicyDocument: aws.String(`
+			{
+			  "Version": "2012-10-17",
+			  "Statement": [
+				{
+				  "Effect": "Allow",
+				  "Action": [
+					"s3:Get*",
+					"s3:List*"
+				  ],
+				  "Resource": [
+					"arn:aws:s3:::EXAMPLE-BUCKET",
+					"arn:aws:s3:::EXAMPLE-BUCKET/*"
+				  ]
+				}
+			  ]
+			}`),
+	}
+	req, _ := iam.New(session.New()).CreatePolicyRequest(params)
+	_ = req.Build()
+	out := CreatePolicyResponse{}
+	response, err := executeRequest(req.HTTPRequest, out)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, http.StatusOK, response.Code)
+}
+
+func TestPutUserPolicy(t *testing.T) {
+	userName := aws.String("Test")
+	params := &iam.PutUserPolicyInput{
+		UserName:   userName,
+		PolicyName: aws.String("S3-read-only-example-bucket"),
+		PolicyDocument: aws.String(
+			`{
+				  "Version": "2012-10-17",
+				  "Statement": [
+					{
+					  "Effect": "Allow",
+					  "Action": [
+						"s3:Get*",
+						"s3:List*"
+					  ],
+					  "Resource": [
+						"arn:aws:s3:::EXAMPLE-BUCKET",
+						"arn:aws:s3:::EXAMPLE-BUCKET/*"
+					  ]
+					}
+				  ]
+			}`),
+	}
+	req, _ := iam.New(session.New()).PutUserPolicyRequest(params)
+	_ = req.Build()
+	out := PutUserPolicyResponse{}
+	response, err := executeRequest(req.HTTPRequest, out)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, http.StatusOK, response.Code)
+}
+
+func executeRequest(req *http.Request, v interface{}) (*httptest.ResponseRecorder, error) {
+	rr := httptest.NewRecorder()
+	apiRouter := mux.NewRouter().SkipClean(true)
+	a.s3ApiConfig = iamS3ApiConfigureMock{}
+	apiRouter.Path("/").Methods("POST").HandlerFunc(a.DoActions)
+	apiRouter.ServeHTTP(rr, req)
+	return rr, xml.Unmarshal(rr.Body.Bytes(), &v)
+}
@@ -0,0 +1,24 @@
+package operation
+
+import (
+	"github.com/valyala/bytebufferpool"
+	"sync/atomic"
+)
+
+var bufferCounter int64
+
+func GetBuffer() *bytebufferpool.ByteBuffer {
+	defer func() {
+		atomic.AddInt64(&bufferCounter, 1)
+		// println("+", bufferCounter)
+	}()
+	return bytebufferpool.Get()
+}
+
+func PutBuffer(buf *bytebufferpool.ByteBuffer) {
+	defer func() {
+		atomic.AddInt64(&bufferCounter, -1)
+		// println("-", bufferCounter)
+	}()
+	bytebufferpool.Put(buf)
+}
@@ -19,7 +19,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"github.com/valyala/bytebufferpool"
 )

 type UploadResult struct {
@@ -190,8 +189,8 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
 }

 func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, originalDataSize int, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
-	buf := bytebufferpool.Get()
-	defer bytebufferpool.Put(buf)
+	buf := GetBuffer()
+	defer PutBuffer(buf)
 	body_writer := multipart.NewWriter(buf)
 	h := make(textproto.MIMEHeader)
 	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, fileNameEscaper.Replace(filename)))
@@ -236,8 +235,10 @@ func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error
 	// print("+")
 	resp, post_err := HttpClient.Do(req)
 	if post_err != nil {
-		glog.Errorf("upload %s %d bytes to %v: %v", filename, originalDataSize, uploadUrl, post_err)
-		debug.PrintStack()
+		if !strings.Contains(post_err.Error(), "connection reset by peer") {
+			glog.Errorf("upload %s %d bytes to %v: %v", filename, originalDataSize, uploadUrl, post_err)
+			debug.PrintStack()
+		}
 		return nil, fmt.Errorf("upload %s %d bytes to %v: %v", filename, originalDataSize, uploadUrl, post_err)
 	}
 	// print("-")
@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.24.0
+// 	protoc-gen-go v1.25.0
 // 	protoc        v3.12.3
 // source: filer.proto

@@ -182,6 +182,26 @@ func Exists(filerClient FilerClient, parentDirectoryPath string, entryName strin
 	return
 }

+func Touch(filerClient FilerClient, parentDirectoryPath string, entryName string, entry *Entry) (err error) {
+
+	return filerClient.WithFilerClient(func(client SeaweedFilerClient) error {
+
+		request := &UpdateEntryRequest{
+			Directory: parentDirectoryPath,
+			Entry:     entry,
+		}
+
+		glog.V(4).Infof("touch entry %v/%v: %v", parentDirectoryPath, entryName, request)
+		if err := UpdateEntry(client, request); err != nil {
+			glog.V(0).Infof("touch exists entry %v: %v", request, err)
+			return fmt.Errorf("touch exists entry %s/%s: %v", parentDirectoryPath, entryName, err)
+		}
+
+		return nil
+	})
+
+}
+
 func Mkdir(filerClient FilerClient, parentDirectoryPath string, dirName string, fn func(entry *Entry)) error {
 	return filerClient.WithFilerClient(func(client SeaweedFilerClient) error {

@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.24.0
+// 	protoc-gen-go v1.25.0
 // 	protoc        v3.12.3
 // source: iam.proto

@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.24.0
+// 	protoc-gen-go v1.25.0
 // 	protoc        v3.12.3
 // source: master.proto

@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.24.0
+// 	protoc-gen-go v1.25.0
 // 	protoc        v3.12.3
 // source: messaging.proto

@@ -52,6 +52,11 @@ service VolumeServer {
    rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
    }

+    rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) {
+    }
+    rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) {
+    }
+
    rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
    }
    rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
@@ -253,6 +258,25 @@ message CopyFileResponse {
    bytes file_content = 1;
 }

+message ReadNeedleBlobRequest {
+    uint32 volume_id = 1;
+    uint64 needle_id = 2;
+    int64 offset = 3; // actual offset
+    int32 size = 4;
+}
+message ReadNeedleBlobResponse {
+    bytes needle_blob = 1;
+}
+
+message WriteNeedleBlobRequest {
+    uint32 volume_id = 1;
+    uint64 needle_id = 2;
+    int32 size = 3;
+    bytes needle_blob = 4;
+}
+message WriteNeedleBlobResponse {
+}
+
 message VolumeTailSenderRequest {
    uint32 volume_id = 1;
    uint64 since_ns = 2;
@@ -20,7 +20,7 @@ func CopyFromChunkViews(chunkViews []*filer.ChunkView, filerSource *source.Filer
 		var shouldRetry bool

 		for _, fileUrl := range fileUrls {
-			shouldRetry, err = util.FastReadUrlAsStream(fileUrl, nil, false, chunk.IsFullChunk(), chunk.Offset, int(chunk.Size), func(data []byte) {
+			shouldRetry, err = util.ReadUrlAsStream(fileUrl, nil, false, chunk.IsFullChunk(), chunk.Offset, int(chunk.Size), func(data []byte) {
 				writeErr = writeFunc(data)
 			})
 			if err != nil {
@@ -79,6 +79,12 @@ func (s3a *S3ApiServer) exists(parentDirectoryPath string, entryName string, isD

 }

+func (s3a *S3ApiServer) touch(parentDirectoryPath string, entryName string, entry *filer_pb.Entry) (err error) {
+
+	return filer_pb.Touch(s3a, parentDirectoryPath, entryName, entry)
+
+}
+
 func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry *filer_pb.Entry, err error) {
 	fullPath := util.NewFullPath(parentDirectoryPath, entryName)
 	return filer_pb.GetEntry(s3a, fullPath)
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
+	weed_server "github.com/chrislusf/seaweedfs/weed/server"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -25,6 +26,26 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 	}

 	srcBucket, srcObject := pathToBucketAndObject(cpSrcPath)
+
+	if (srcBucket == dstBucket && srcObject == dstObject || cpSrcPath == "") && isReplace(r) {
+		fullPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject))
+		dir, name := fullPath.DirAndName()
+		entry, err := s3a.getEntry(dir, name)
+		if err != nil {
+			writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
+		}
+		entry.Extended = weed_server.SaveAmzMetaData(r, entry.Extended, isReplace(r))
+		err = s3a.touch(dir, name, entry)
+		if err != nil {
+			writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
+		}
+		writeSuccessResponseXML(w, encodeResponse(CopyObjectResult{
+			ETag:         fmt.Sprintf("%x", entry.Attributes.Md5),
+			LastModified: time.Now().UTC(),
+		}))
+		return
+	}
+
 	// If source object is empty or bucket is empty, reply back invalid copy source.
 	if srcObject == "" || srcBucket == "" {
 		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
@@ -32,7 +53,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 	}

 	if srcBucket == dstBucket && srcObject == dstObject {
-		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
+		writeErrorResponse(w, s3err.ErrInvalidCopyDest, r.URL)
 		return
 	}

@@ -147,3 +168,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseXML(w, encodeResponse(response))

 }
+
+func isReplace(r *http.Request) bool {
+	return r.Header.Get("X-Amz-Metadata-Directive") == "REPLACE"
+}
@@ -311,7 +311,7 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
 	}
 	defer util.CloseResponse(resp)

-	if (resp.ContentLength == -1 || resp.StatusCode == 404) && !strings.HasSuffix(destUrl, "/") {
+	if resp.ContentLength == -1 || resp.StatusCode == 404 {
 		if r.Method != "DELETE" {
 			writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL)
 			return
@@ -326,7 +326,11 @@ func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) {
 	for k, v := range proxyResponse.Header {
 		w.Header()[k] = v
 	}
-	w.WriteHeader(proxyResponse.StatusCode)
+	if proxyResponse.Header.Get("Content-Range") != "" && proxyResponse.StatusCode == 200 {
+		w.WriteHeader(http.StatusPartialContent)
+	} else {
+		w.WriteHeader(proxyResponse.StatusCode)
+	}
 	io.Copy(w, proxyResponse.Body)
 }

@@ -206,7 +206,6 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
 		isTruncated = isTruncated || subIsTruncated
 		maxKeys -= subCounter
 		nextMarker = subDir + "/" + subNextMarker
-		counter += subCounter
 		// finished processing this sub directory
 		marker = subDir
 	}
@@ -0,0 +1,46 @@
+package sequence
+
+import (
+	"fmt"
+	"hash/fnv"
+
+	"github.com/bwmarrin/snowflake"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+)
+
+// a simple snowflake Sequencer
+type SnowflakeSequencer struct {
+	node *snowflake.Node
+}
+
+func NewSnowflakeSequencer(nodeid string) (*SnowflakeSequencer, error) {
+	nodeid_hash := hash(nodeid) & 0x3ff
+	glog.V(0).Infof("use snowflake seq id generator, nodeid:%s hex_of_nodeid: %x", nodeid, nodeid_hash)
+	node, err := snowflake.NewNode(int64(nodeid_hash))
+	if err != nil {
+		fmt.Println(err)
+		return nil, err
+	}
+
+	sequencer := &SnowflakeSequencer{node: node}
+	return sequencer, nil
+}
+
+func hash(s string) uint32 {
+	h := fnv.New32a()
+	h.Write([]byte(s))
+	return h.Sum32()
+}
+
+func (m *SnowflakeSequencer) NextFileId(count uint64) uint64 {
+	return uint64(m.node.Generate().Int64())
+}
+
+// ignore setmax as we are snowflake
+func (m *SnowflakeSequencer) SetMax(seenValue uint64) {
+}
+
+// return a new id as no Peek is stored
+func (m *SnowflakeSequencer) Peek() uint64 {
+	return uint64(m.node.Generate().Int64())
+}
@@ -234,12 +234,12 @@ func adjustHeaderContentDisposition(w http.ResponseWriter, r *http.Request, file
 	}
 }

-func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64, httpStatusCode int) error) {
+func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64) error) {
 	rangeReq := r.Header.Get("Range")

 	if rangeReq == "" {
 		w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
-		if err := writeFn(w, 0, totalSize, 0); err != nil {
+		if err := writeFn(w, 0, totalSize); err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
@@ -279,7 +279,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
 		w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10))
 		w.Header().Set("Content-Range", ra.contentRange(totalSize))

-		err = writeFn(w, ra.start, ra.length, http.StatusPartialContent)
+		err = writeFn(w, ra.start, ra.length)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
@@ -307,7 +307,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
 				pw.CloseWithError(e)
 				return
 			}
-			if e = writeFn(part, ra.start, ra.length, 0); e != nil {
+			if e = writeFn(part, ra.start, ra.length); e != nil {
 				pw.CloseWithError(e)
 				return
 			}
@@ -33,8 +33,7 @@ func (fs *FilerServer) AtomicRenameEntry(ctx context.Context, req *filer_pb.Atom
 		return nil, fmt.Errorf("%s/%s not found: %v", req.OldDirectory, req.OldName, err)
 	}

-	var events MoveEvents
-	moveErr := fs.moveEntry(ctx, oldParent, oldEntry, newParent, req.NewName, &events)
+	moveErr := fs.moveEntry(ctx, oldParent, oldEntry, newParent, req.NewName)
 	if moveErr != nil {
 		fs.filer.RollbackTransaction(ctx)
 		return nil, fmt.Errorf("%s/%s move error: %v", req.OldDirectory, req.OldName, moveErr)
@@ -48,11 +47,11 @@ func (fs *FilerServer) AtomicRenameEntry(ctx context.Context, req *filer_pb.Atom
 	return &filer_pb.AtomicRenameEntryResponse{}, nil
 }

-func (fs *FilerServer) moveEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, events *MoveEvents) error {
+func (fs *FilerServer) moveEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string) error {

-	if err := fs.moveSelfEntry(ctx, oldParent, entry, newParent, newName, events, func() error {
+	if err := fs.moveSelfEntry(ctx, oldParent, entry, newParent, newName, func() error {
 		if entry.IsDirectory() {
-			if err := fs.moveFolderSubEntries(ctx, oldParent, entry, newParent, newName, events); err != nil {
+			if err := fs.moveFolderSubEntries(ctx, oldParent, entry, newParent, newName); err != nil {
 				return err
 			}
 		}
@@ -64,7 +63,7 @@ func (fs *FilerServer) moveEntry(ctx context.Context, oldParent util.FullPath, e
 	return nil
 }

-func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, events *MoveEvents) error {
+func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string) error {

 	currentDirPath := oldParent.Child(entry.Name())
 	newDirPath := newParent.Child(newName)
@@ -85,7 +84,7 @@ func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.
 		for _, item := range entries {
 			lastFileName = item.Name()
 			// println("processing", lastFileName)
-			err := fs.moveEntry(ctx, currentDirPath, item, newDirPath, item.Name(), events)
+			err := fs.moveEntry(ctx, currentDirPath, item, newDirPath, item.Name())
 			if err != nil {
 				return err
 			}
@@ -97,8 +96,7 @@ func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.
 	return nil
 }

-func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, events *MoveEvents,
-	moveFolderSubEntries func() error) error {
+func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, moveFolderSubEntries func() error) error {

 	oldPath, newPath := oldParent.Child(entry.Name()), newParent.Child(newName)

@@ -122,8 +120,6 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPat
 		return createErr
 	}

-	events.newEntries = append(events.newEntries, newEntry)
-
 	if moveFolderSubEntries != nil {
 		if moveChildrenErr := moveFolderSubEntries(); moveChildrenErr != nil {
 			return moveChildrenErr
@@ -136,13 +132,6 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPat
 		return deleteErr
 	}

-	events.oldEntries = append(events.oldEntries, entry)
-
 	return nil

 }
-
-type MoveEvents struct {
-	oldEntries []*filer.Entry
-	newEntries []*filer.Entry
-}
@@ -45,22 +45,23 @@ import (
 )

 type FilerOption struct {
-	Masters            []string
-	Collection         string
-	DefaultReplication string
-	DisableDirListing  bool
-	MaxMB              int
-	DirListingLimit    int
-	DataCenter         string
-	Rack               string
-	DefaultLevelDbDir  string
-	DisableHttp        bool
-	Host               string
-	Port               uint32
-	recursiveDelete    bool
-	Cipher             bool
-	SaveToFilerLimit   int
-	Filers             []string
+	Masters               []string
+	Collection            string
+	DefaultReplication    string
+	DisableDirListing     bool
+	MaxMB                 int
+	DirListingLimit       int
+	DataCenter            string
+	Rack                  string
+	DefaultLevelDbDir     string
+	DisableHttp           bool
+	Host                  string
+	Port                  uint32
+	recursiveDelete       bool
+	Cipher                bool
+	SaveToFilerLimit      int64
+	Filers                []string
+	ConcurrentUploadLimit int64
 }

 type FilerServer struct {
@@ -79,14 +80,18 @@ type FilerServer struct {

 	brokers     map[string]map[string]bool
 	brokersLock sync.Mutex
+
+	inFlightDataSize      int64
+	inFlightDataLimitCond *sync.Cond
 }

 func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) (fs *FilerServer, err error) {

 	fs = &FilerServer{
-		option:         option,
-		grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.filer"),
-		brokers:        make(map[string]map[string]bool),
+		option:                option,
+		grpcDialOption:        security.LoadClientTLS(util.GetViper(), "grpc.filer"),
+		brokers:               make(map[string]map[string]bool),
+		inFlightDataLimitCond: sync.NewCond(new(sync.Mutex)),
 	}
 	fs.listenersCond = sync.NewCond(&fs.listenersLock)

@@ -4,6 +4,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"net/http"
 	"strings"
+	"sync/atomic"
 	"time"

 	"github.com/chrislusf/seaweedfs/weed/stats"
@@ -47,18 +48,34 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
 			fs.DeleteHandler(w, r)
 		}
 		stats.FilerRequestHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds())
-	case "PUT":
-		stats.FilerRequestCounter.WithLabelValues("put").Inc()
-		if _, ok := r.URL.Query()["tagging"]; ok {
-			fs.PutTaggingHandler(w, r)
-		} else {
-			fs.PostHandler(w, r)
+	case "POST", "PUT":
+
+		// wait until in flight data is less than the limit
+		contentLength := getContentLength(r)
+		fs.inFlightDataLimitCond.L.Lock()
+		for atomic.LoadInt64(&fs.inFlightDataSize) > fs.option.ConcurrentUploadLimit {
+			fs.inFlightDataLimitCond.Wait()
+		}
+		atomic.AddInt64(&fs.inFlightDataSize, contentLength)
+		fs.inFlightDataLimitCond.L.Unlock()
+		defer func() {
+			atomic.AddInt64(&fs.inFlightDataSize, -contentLength)
+			fs.inFlightDataLimitCond.Signal()
+		}()
+
+		if r.Method == "PUT" {
+			stats.FilerRequestCounter.WithLabelValues("put").Inc()
+			if _, ok := r.URL.Query()["tagging"]; ok {
+				fs.PutTaggingHandler(w, r)
+			} else {
+				fs.PostHandler(w, r, contentLength)
+			}
+			stats.FilerRequestHistogram.WithLabelValues("put").Observe(time.Since(start).Seconds())
+		} else { // method == "POST"
+			stats.FilerRequestCounter.WithLabelValues("post").Inc()
+			fs.PostHandler(w, r, contentLength)
+			stats.FilerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds())
 		}
-		stats.FilerRequestHistogram.WithLabelValues("put").Observe(time.Since(start).Seconds())
-	case "POST":
-		stats.FilerRequestCounter.WithLabelValues("post").Inc()
-		fs.PostHandler(w, r)
-		stats.FilerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds())
 	case "OPTIONS":
 		stats.FilerRequestCounter.WithLabelValues("options").Inc()
 		OptionsHandler(w, r, false)
@@ -131,6 +131,9 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,

 	if r.Method == "HEAD" {
 		w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
+		processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error {
+			return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size, true)
+		})
 		return
 	}

@@ -150,10 +153,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 		}
 	}

-	processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64, httpStatusCode int) error {
-		if httpStatusCode != 0 {
-			w.WriteHeader(httpStatusCode)
-		}
+	processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error {
 		if offset+size <= int64(len(entry.Content)) {
 			_, err := writer.Write(entry.Content[offset : offset+size])
 			if err != nil {
@@ -161,7 +161,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 			}
 			return err
 		}
-		return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size)
+		return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size, false)
 	})

 }
@@ -52,7 +52,7 @@ func (fs *FilerServer) assignNewFileInfo(so *operation.StorageOption) (fileId, u
 	return
 }

-func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
+func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request, contentLength int64) {

 	ctx := context.Background()

@@ -66,7 +66,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 		query.Get("rack"),
 	)

-	fs.autoChunk(ctx, w, r, so)
+	fs.autoChunk(ctx, w, r, contentLength, so)
 	util.CloseRequest(r)

 }
@@ -2,11 +2,8 @@ package weed_server

 import (
 	"context"
-	"crypto/md5"
 	"fmt"
-	"hash"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"path"
@@ -19,13 +16,12 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
-	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/stats"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )

-func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, so *operation.StorageOption) {
+func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, contentLength int64, so *operation.StorageOption) {

 	// autoChunking can be set at the command-line level or as a query param. Query param overrides command-line
 	query := r.URL.Query()
@@ -51,14 +47,16 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
 		if r.Header.Get("Content-Type") == "" && strings.HasSuffix(r.URL.Path, "/") {
 			reply, err = fs.mkdir(ctx, w, r)
 		} else {
-			reply, md5bytes, err = fs.doPostAutoChunk(ctx, w, r, chunkSize, so)
+			reply, md5bytes, err = fs.doPostAutoChunk(ctx, w, r, chunkSize, contentLength, so)
 		}
 	} else {
-		reply, md5bytes, err = fs.doPutAutoChunk(ctx, w, r, chunkSize, so)
+		reply, md5bytes, err = fs.doPutAutoChunk(ctx, w, r, chunkSize, contentLength, so)
 	}
 	if err != nil {
 		if strings.HasPrefix(err.Error(), "read input:") {
 			writeJsonError(w, r, 499, err)
+		} else if strings.HasSuffix(err.Error(), "is a file") {
+			writeJsonError(w, r, http.StatusConflict, err)
 		} else {
 			writeJsonError(w, r, http.StatusInternalServerError, err)
 		}
@@ -70,7 +68,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
 	}
 }

-func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
+func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, contentLength int64, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {

 	multipartReader, multipartReaderErr := r.MultipartReader()
 	if multipartReaderErr != nil {
@@ -91,7 +89,7 @@ func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWrite
 		contentType = ""
 	}

-	fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, part1, chunkSize, fileName, contentType, so)
+	fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, part1, chunkSize, fileName, contentType, contentLength, so)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -102,7 +100,7 @@ func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWrite
 	return
 }

-func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
+func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, contentLength int64, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {

 	fileName := path.Base(r.URL.Path)
 	contentType := r.Header.Get("Content-Type")
@@ -110,7 +108,7 @@ func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter
 		contentType = ""
 	}

-	fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, r.Body, chunkSize, fileName, contentType, so)
+	fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, r.Body, chunkSize, fileName, contentType, contentLength, so)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -212,7 +210,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
 		entry.Extended = make(map[string][]byte)
 	}

-	fs.saveAmzMetaData(r, entry)
+	SaveAmzMetaData(r, entry.Extended, false)

 	for k, v := range r.Header {
 		if len(v) > 0 && strings.HasPrefix(k, needle.PairNamePrefix) {
@@ -229,92 +227,6 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
 	return filerResult, replyerr
 }

-func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Request, reader io.Reader, chunkSize int32, fileName, contentType string, so *operation.StorageOption) ([]*filer_pb.FileChunk, hash.Hash, int64, error, []byte) {
-	var fileChunks []*filer_pb.FileChunk
-
-	md5Hash := md5.New()
-	var partReader = ioutil.NopCloser(io.TeeReader(reader, md5Hash))
-
-	chunkOffset := int64(0)
-	var smallContent []byte
-
-	for {
-		limitedReader := io.LimitReader(partReader, int64(chunkSize))
-
-		data, err := ioutil.ReadAll(limitedReader)
-		if err != nil {
-			return nil, nil, 0, err, nil
-		}
-		if chunkOffset == 0 && !isAppend(r) {
-			if len(data) < fs.option.SaveToFilerLimit || strings.HasPrefix(r.URL.Path, filer.DirectoryEtcRoot) && len(data) < 4*1024 {
-				smallContent = data
-				chunkOffset += int64(len(data))
-				break
-			}
-		}
-		dataReader := util.NewBytesReader(data)
-
-		// retry to assign a different file id
-		var fileId, urlLocation string
-		var auth security.EncodedJwt
-		var assignErr, uploadErr error
-		var uploadResult *operation.UploadResult
-		for i := 0; i < 3; i++ {
-			// assign one file id for one chunk
-			fileId, urlLocation, auth, assignErr = fs.assignNewFileInfo(so)
-			if assignErr != nil {
-				return nil, nil, 0, assignErr, nil
-			}
-
-			// upload the chunk to the volume server
-			uploadResult, uploadErr, _ = fs.doUpload(urlLocation, w, r, dataReader, fileName, contentType, nil, auth)
-			if uploadErr != nil {
-				time.Sleep(251 * time.Millisecond)
-				continue
-			}
-			break
-		}
-		if uploadErr != nil {
-			return nil, nil, 0, uploadErr, nil
-		}
-
-		// if last chunk exhausted the reader exactly at the border
-		if uploadResult.Size == 0 {
-			break
-		}
-
-		// Save to chunk manifest structure
-		fileChunks = append(fileChunks, uploadResult.ToPbFileChunk(fileId, chunkOffset))
-
-		glog.V(4).Infof("uploaded %s chunk %d to %s [%d,%d)", fileName, len(fileChunks), fileId, chunkOffset, chunkOffset+int64(uploadResult.Size))
-
-		// reset variables for the next chunk
-		chunkOffset = chunkOffset + int64(uploadResult.Size)
-
-		// if last chunk was not at full chunk size, but already exhausted the reader
-		if int64(uploadResult.Size) < int64(chunkSize) {
-			break
-		}
-	}
-
-	return fileChunks, md5Hash, chunkOffset, nil, smallContent
-}
-
-func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request, limitedReader io.Reader, fileName string, contentType string, pairMap map[string]string, auth security.EncodedJwt) (*operation.UploadResult, error, []byte) {
-
-	stats.FilerRequestCounter.WithLabelValues("chunkUpload").Inc()
-	start := time.Now()
-	defer func() {
-		stats.FilerRequestHistogram.WithLabelValues("chunkUpload").Observe(time.Since(start).Seconds())
-	}()
-
-	uploadResult, err, data := operation.Upload(urlLocation, fileName, fs.option.Cipher, limitedReader, false, contentType, pairMap, auth)
-	if uploadResult != nil && uploadResult.RetryCount > 0 {
-		stats.FilerRequestCounter.WithLabelValues("chunkUploadRetry").Add(float64(uploadResult.RetryCount))
-	}
-	return uploadResult, err, data
-}
-
 func (fs *FilerServer) saveAsChunk(so *operation.StorageOption) filer.SaveDataAsChunkFunctionType {

 	return func(reader io.Reader, name string, offset int64) (*filer_pb.FileChunk, string, string, error) {
@@ -383,17 +295,24 @@ func (fs *FilerServer) mkdir(ctx context.Context, w http.ResponseWriter, r *http
 	return filerResult, replyerr
 }

-func (fs *FilerServer) saveAmzMetaData(r *http.Request, entry *filer.Entry) {
+func SaveAmzMetaData(r *http.Request, existing map[string][]byte, isReplace bool) (metadata map[string][]byte) {
+
+	metadata = make(map[string][]byte)
+	if !isReplace {
+		for k, v := range existing {
+			metadata[k] = v
+		}
+	}

 	if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" {
-		entry.Extended[xhttp.AmzStorageClass] = []byte(sc)
+		metadata[xhttp.AmzStorageClass] = []byte(sc)
 	}

 	if tags := r.Header.Get(xhttp.AmzObjectTagging); tags != "" {
 		for _, v := range strings.Split(tags, "&") {
 			tag := strings.Split(v, "=")
 			if len(tag) == 2 {
-				entry.Extended[xhttp.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1])
+				metadata[xhttp.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1])
 			}
 		}
 	}
@@ -401,8 +320,11 @@ func (fs *FilerServer) saveAmzMetaData(r *http.Request, entry *filer.Entry) {
 	for header, values := range r.Header {
 		if strings.HasPrefix(header, xhttp.AmzUserMetaPrefix) {
 			for _, value := range values {
-				entry.Extended[header] = []byte(value)
+				metadata[header] = []byte(value)
 			}
 		}
 	}
+
+	return
+
 }
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Chris Lu	742ab1ec81	2.39	2021-04-11 19:47:11 -07:00
Chris Lu	0df5b53ad8	adjust help message	2021-04-11 00:26:28 -07:00
Chris Lu	f62c153274	go fmt	2021-04-10 23:48:18 -07:00
Chris Lu	af313dff58	add gateway for easier POST and DELETE blobs	2021-04-10 23:47:47 -07:00
Chris Lu	98c08a3dcd	raft: fix possible nil panic: runtime error: invalid memory address or nil pointer dereference [signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x1962c8a] goroutine 68239935 [running]: github.com/chrislusf/raft.(LogEntry).Term(...) /home/travis/gopath/pkg/mod/github.com/chrislusf/raft@v1.0.4/log_entry.go:59 github.com/chrislusf/raft.(server).TakeSnapshot(0xc00014f320, 0xc028b5a600, 0xc03995bc00) /home/travis/gopath/pkg/mod/github.com/chrislusf/raft@v1.0.4/server.go:1276 +0x50a github.com/chrislusf/raft.(server).maybeTakeSnapshot.func1(0xc00014f320) /home/travis/gopath/pkg/mod/github.com/chrislusf/raft@v1.0.4/server.go:1221 +0x5b created by github.com/chrislusf/raft.(server).maybeTakeSnapshot /home/travis/gopath/pkg/mod/github.com/chrislusf/raft@v1.0.4/server.go:1219 +0x98	2021-04-10 02:36:53 -07:00
Chris Lu	0b82edc0d2	filer: avoid stuck uploader fix https://github.com/chrislusf/seaweedfs/issues/1980 reverting the file upload batch executor	2021-04-09 13:05:15 -07:00
Chris Lu	93f4146ffa	properly release the view cache	2021-04-09 12:36:39 -07:00
Chris Lu	6deb647a8f	mount: fix possible memory leak if many files are read repeatedly, their metadata are accumulated in memory. This fix cleared the metadata after the file is read.	2021-04-08 19:47:31 -07:00
Chris Lu	b06c5b9d99	upgrade raft to v1.0.5 fix https://github.com/chrislusf/seaweedfs/issues/1974	2021-04-07 20:59:48 -07:00
Chris LuandGitHub	0f02bd210d	Merge pull request #1973 from philpep/patch-1	2021-04-07 13:48:57 -07:00
Philippe PepiotandGitHub	42a761ee20	Fix typo in weed filer long help	2021-04-07 22:47:23 +02:00
Chris Lu	3be061994f	skip connection reset error fix https://github.com/chrislusf/seaweedfs/issues/1971 this is because the connections are pooled but the volume server has reset the connection	2021-04-07 00:54:13 -07:00
Chris Lu	c5b08bac1b	remove mac specific mount options	2021-04-07 00:54:13 -07:00
Chris LuandGitHub	e1fc78ed46	Merge pull request #1969 from kmlebedev/iamapipr Iam Api Server	2021-04-06 01:58:30 -07:00
Konstantin Lebedev	f5f8eec8e2	fix get filerGrpcAddress	2021-04-06 13:53:56 +05:00
Konstantin Lebedev	011e6e90ee	Merge branch 'upstreamMaster' into iamapipr	2021-04-06 13:50:33 +05:00
Konstantin Lebedev	ed79baa30f	add tests	2021-04-06 13:43:08 +05:00
Chris LuandGitHub	100ed77387	Merge pull request #1968 from hilimd/master HCFS over ftp	2021-04-06 00:50:31 -07:00
Chris Lu	2327c0756b	fix to avoid loop	2021-04-05 23:24:26 -07:00
limd	6b8892c5ac	add HCFS over ftp	2021-04-06 13:42:51 +08:00
hilimdandGitHub	8693cdacae	Merge pull request #75 from chrislusf/master sync	2021-04-06 13:39:26 +08:00
Chris Lu	a37eca78cd	2.38	2021-04-05 19:41:54 -07:00
Chris Lu	c131764c34	ensure error is sent back in channel fix https://github.com/chrislusf/seaweedfs/issues/1966 avoid shared readErr variable	2021-04-05 19:40:12 -07:00
Chris LuandGitHub	0a9d76c9eb	Update README.md	2021-04-05 00:44:50 -07:00
Chris Lu	0f64f5b9c8	mount: add readOnly option fix https://github.com/chrislusf/seaweedfs/issues/1961	2021-04-04 21:40:58 -07:00
Chris Lu	6eee200c13	2.37	2021-04-04 18:45:48 -07:00
Chris Lu	fbb82a5c9c	skip limiting if limit is zero	2021-04-04 18:38:33 -07:00
Chris Lu	8251d1140e	refactor	2021-04-04 18:38:33 -07:00
Chris LuandGitHub	7cf7e05cbb	Merge pull request #1960 from xdevs23/patch-1	2021-04-04 12:24:03 -07:00
Simão Gomes VianaandGitHub	404cf9db47	README: fix typo thoudsands → thousands	2021-04-04 20:38:38 +02:00
Chris Lu	5dca98fe71	fix typo	2021-04-03 11:37:13 -07:00
Chris Lu	5e64f65632	ensure tables are created fix https://github.com/chrislusf/seaweedfs/issues/1957	2021-04-02 12:02:26 -07:00
Chris Lu	319743d330	go fmt	2021-04-02 02:22:59 -07:00
Chris Lu	8e404a1433	go fmt	2021-04-02 02:22:26 -07:00
Chris Lu	bdf2ddddfd	revert to same implementation as before This reverts commit `7e8edc3c4a`.	2021-04-02 02:21:38 -07:00
Chris Lu	67e019d54b	add missing changes	2021-04-02 01:11:44 -07:00
Chris Lu	7e8edc3c4a	refactoring	2021-04-02 01:10:24 -07:00
Chris Lu	cc0df36a9e	smaller file chunks are faster	2021-04-01 02:21:59 -07:00
Chris Lu	cefe66f159	dedicated upload processor avoid thundering effect of overloading volume servers	2021-04-01 02:21:40 -07:00
Chris Lu	1f984d2645	refactor buffer pool	2021-04-01 02:20:00 -07:00
Chris Lu	b5880334fc	refactor	2021-03-30 21:07:34 -07:00
Chris Lu	e79e2ddeed	remove unused MoveEvents	2021-03-30 20:42:44 -07:00
Chris Lu	e06807a55b	adjust logs	2021-03-30 20:36:06 -07:00
Chris Lu	a4db1fe32f	patch https://github.com/advisories/GHSA-5mg8-w23w-74h3 https://github.com/advisories/GHSA-5mg8-w23w-74h3	2021-03-30 02:30:34 -07:00
Chris Lu	ac875976c0	filer, volume: add concurrent upload size limit to avoid OOM add some back pressure when writes are slow	2021-03-30 02:10:53 -07:00
Chris Lu	a1e18a1384	server add memory profiling	2021-03-30 02:10:53 -07:00
Chris LuandGitHub	48188abc7b	Merge pull request #1952 from LazyDBA247-Anyvision/master fix postgres2 upsertQuery syntax also	2021-03-30 01:01:59 -07:00
Chris LuandGitHub	e5164a3c34	Merge pull request #1951 from chrislusf/dependabot/maven/other/java/client/com.google.guava-guava-29.0-jre Bump guava from 28.0-jre to 29.0-jre in /other/java/client	2021-03-30 01:01:14 -07:00
Chris LuandGitHub	3472c87c21	Merge pull request #1950 from chrislusf/dependabot/maven/test/random_access/com.google.guava-guava-29.0-jre Bump guava from 28.0-jre to 29.0-jre in /test/random_access	2021-03-30 01:00:54 -07:00
LazyDBA247-Anyvision	9385e2224d	fix postgres2 upsertQuery syntax also	2021-03-30 10:51:52 +03:00
dependabot[bot]andGitHub	9401890f18	Bump guava from 28.0-jre to 29.0-jre in /other/java/client Bumps [guava](https://github.com/google/guava) from 28.0-jre to 29.0-jre. - [Release notes](https://github.com/google/guava/releases) - [Commits](https://github.com/google/guava/commits) Signed-off-by: dependabot[bot] <support@github.com>	2021-03-30 07:46:34 +00:00
dependabot[bot]andGitHub	fb04fda28b	Bump guava from 28.0-jre to 29.0-jre in /test/random_access Bumps [guava](https://github.com/google/guava) from 28.0-jre to 29.0-jre. - [Release notes](https://github.com/google/guava/releases) - [Commits](https://github.com/google/guava/commits) Signed-off-by: dependabot[bot] <support@github.com>	2021-03-30 07:46:24 +00:00
Chris Lu	a5662c9e5e	adjust ordering	2021-03-30 00:26:57 -07:00
Chris LuandGitHub	bec74c3e6a	Merge pull request #1948 from LazyDBA247-Anyvision/master add enableUpsert=true	2021-03-30 00:25:01 -07:00
LazyDBA247-Anyvision	9f1cab179c	use a compatible pg syntax for upsert	2021-03-30 10:21:27 +03:00
Chris Lu	a95929e53c	reduce default concurrentWriters to 32	2021-03-30 00:17:52 -07:00
Chris Lu	be25bc6766	Java client 1.6.4	2021-03-30 00:17:35 -07:00
Chris Lu	e72e3453e8	Java client: ByteBuffer.clear supports higher JDK versions	2021-03-29 21:42:28 -07:00
LazyDBA247-Anyvision	7f44d953b5	fix GetBool	2021-03-30 01:36:02 +03:00
LazyDBA247-Anyvision	4c51e6a660	add enableUpsert=true and rename config to upsertQuery	2021-03-30 00:32:03 +03:00
Chris LuandGitHub	3cf84b5fae	Merge pull request #1947 from kmlebedev/minio-gateway-compose minio gateway s3 docker compose file	2021-03-29 09:42:07 -07:00
Konstantin Lebedev	6b3cc94ac4	minio gateway s3 docker compose file	2021-03-29 14:59:59 +05:00
Chris Lu	b61e9f6a1a	adjust ordering	2021-03-29 00:49:50 -07:00
Chris LuandGitHub	1ebdbad222	Merge pull request #1946 from LazyDBA247-Anyvision/master Adding Customisation - insertQuery for postgres/mysql	2021-03-29 00:42:48 -07:00
Konstantin Lebedev	8a95f9c10c	iam GetUser	2021-03-29 12:01:44 +05:00
LazyDBA247-Anyvision	96c62bd34d	scaffold - insertQuery Example for upsert query for mysql & postgres variants	2021-03-29 10:01:12 +03:00
LazyDBA247-Anyvision	4a02389eb0	Adding custom insertQuery support for postgres/2 mysql/2	2021-03-29 09:58:13 +03:00
Chris Lu	6b7aa9633f	2.36	2021-03-28 19:09:06 -07:00
Chris Lu	bd7471d877	refactor	2021-03-25 12:05:59 -07:00
Chris LuandGitHub	b8d3f11e92	Merge pull request #1945 from bash99/master Add a snowflake sequencer as more robust fid generator, but less compressable than small auto-inc id	2021-03-25 10:29:07 -07:00
李海	06be5dc6c3	log snowflake sequencer nodeid's hex when start, it'll be in part of new assigned fid	2021-03-25 18:59:40 +08:00
李海	69b2dab9c6	add a snowflake sequencer as more robust fid generator, but less compressable than small auto-inc id	2021-03-25 18:49:26 +08:00
Konstantin Lebedev	40938d6a47	SaveInsideFiler S3 Configuration	2021-03-25 12:34:11 +05:00
Konstantin Lebedev	9f26f2815c	SaveAs S3 Configuration	2021-03-25 12:34:11 +05:00
Konstantin Lebedev	82b0463fac	handler PutUserPolicy https://docs.aws.amazon.com/IAM/latest/APIReference/API_PutUserPolicy.html	2021-03-25 12:34:11 +05:00
Konstantin Lebedev	d7719d0542	base handlers	2021-03-25 12:34:11 +05:00
Konstantin Lebedev	03c7953254	init Iam Api Server	2021-03-25 12:34:11 +05:00
Chris LuandGitHub	82dfe06066	Merge pull request #1933 from kmlebedev/missingKeysThreshold add missingKeysThreshold for fsck	2021-03-24 10:56:25 -07:00
Konstantin Lebedev	020a5d40c3	avoid counter is null	2021-03-24 22:51:22 +05:00
Konstantin Lebedev	df6cf0a2fa	nonRepairThreshold	2021-03-24 22:07:13 +05:00
Chris LuandGitHub	26657f1d72	Merge pull request #1943 from kmlebedev/maxCPUs GOMAXPROCS use defaults to the value of runtime.NumCPU	2021-03-24 09:29:38 -07:00
Konstantin Lebedev	9ce82a9cd2	GOMAXPROCS use defaults to the value of runtime.NumCPU https://github.com/chrislusf/seaweedfs/issues/1942	2021-03-24 18:45:58 +05:00
Konstantin Lebedev	60972f1c97	rename option repairThreshold	2021-03-24 13:24:49 +05:00
Chris Lu	a801332b0d	filer: return 409 if file conflicts with a directory fix https://github.com/chrislusf/seaweedfs/issues/1938	2021-03-24 01:03:11 -07:00
Chris Lu	c42b95c596	more help message fix https://github.com/chrislusf/seaweedfs/issues/1937	2021-03-23 17:27:57 -07:00
Konstantin Lebedev	40a9e88c07	rename var	2021-03-23 21:42:35 +05:00
Konstantin Lebedev	d21a5bf5d0	add missingKeysThreshold for fsck	2021-03-23 15:04:07 +05:00
Chris Lu	85cff10787	mysql: ensure name column is case sensitive fix https://github.com/chrislusf/seaweedfs/issues/1931	2021-03-23 00:46:50 -07:00
Chris Lu	5f0e68c21e	revert wrong change	2021-03-22 23:28:40 -07:00
Chris Lu	b5e10bf511	Java client 1.6.3	2021-03-22 23:25:47 -07:00
Chris Lu	5a78c6ca99	Java client: support windows	2021-03-22 23:20:41 -07:00
Chris Lu	4abb511db3	make a local copy of the in memory cached data	2021-03-22 22:33:07 -07:00
Chris Lu	3cbc40fa48	avoid creating multiple reader	2021-03-22 22:32:47 -07:00
Chris Lu	2f7c7afdec	refactor: remove unused parameter	2021-03-22 22:13:19 -07:00
Chris Lu	d48dd0c738	use []byte directly instead of bytes.Buffer	2021-03-22 22:12:57 -07:00
Chris Lu	8eb14967ef	shell: volume.check.disk adds optional slow mode fix https://github.com/chrislusf/seaweedfs/issues/1930	2021-03-22 21:01:43 -07:00
Chris Lu	288369cfc7	mount: release resources when Forget() is called address https://github.com/chrislusf/seaweedfs/issues/1929	2021-03-22 16:22:19 -07:00
Chris Lu	5d931eff27	avoid possible nil fix https://github.com/chrislusf/seaweedfs/issues/1928 The nil was because of `dn.Parent().UnlinkChildNode(dn.Id())` in topo.UnRegisterDataNode() function, when the dn leaves the cluster.	2021-03-22 13:24:07 -07:00
Chris Lu	f315eb2bb8	fix build error	2021-03-22 00:12:53 -07:00
Chris Lu	1dd5bc134c	2.35	2021-03-22 00:05:09 -07:00
Chris Lu	b465095db1	shell: add volume.check.disk to fix inconsistency for replicated volumes fix https://github.com/chrislusf/seaweedfs/issues/1923	2021-03-22 00:03:16 -07:00
Chris Lu	df461402cc	ensure entry attribute is not nill fix https://github.com/chrislusf/seaweedfs/issues/1926	2021-03-21 22:27:30 -07:00
Chris Lu	102a951377	refactor, split into 2 files	2021-03-21 13:05:53 -07:00
Chris Lu	f6e2566ee5	better logs for cookie mismatch	2021-03-20 12:02:57 -07:00
Chris Lu	27e24162f2	minor	2021-03-20 06:37:03 -07:00
Chris Lu	41cc6e661b	fs.mv ensures there are 2 arguments	2021-03-20 06:34:13 -07:00
Chris Lu	b2c1c209a5	refactor	2021-03-20 06:34:13 -07:00
Chris LuandGitHub	0a51df02cb	Update README.md	2021-03-19 12:58:24 -07:00
Chris Lu	e9b1853d63	go fmt	2021-03-19 01:31:56 -07:00
Chris Lu	b1a86cf808	s3: copy object to itself fix https://github.com/chrislusf/seaweedfs/issues/1922	2021-03-19 01:31:49 -07:00
Chris LuandGitHub	aa5179ce3c	Merge pull request #1918 from kmlebedev/dir_slash_not_exist dir + slash key does not exist	2021-03-17 11:56:56 -07:00
Konstantin Lebedev	bf94149920	add line	2021-03-17 23:41:34 +05:00
Konstantin Lebedev	c5705e7a35	dir + slash key does not exist https://github.com/chrislusf/seaweedfs/issues/1917	2021-03-17 23:40:42 +05:00
Chris LuandGitHub	04604b89e1	Merge pull request #1913 from kmlebedev/check_chunkviews_mr Check chunkviews mr	2021-03-16 03:36:34 -07:00
Konstantin Lebedev	eb54993a4e	Merge branch 'upstreamMaster' into check_chunkviews_mr # Conflicts: # weed/filer/filechunk_manifest.go # weed/filer/stream.go # weed/replication/repl_util/replication_util.go # weed/util/fasthttp_util.go	2021-03-16 15:29:49 +05:00
Chris Lu	9672f9e1b2	2.34	2021-03-16 03:01:15 -07:00
Chris Lu	69694a17be	reverting `7d57664c2d`	2021-03-16 02:59:26 -07:00
Chris Lu	c00dd5e62e	report error if entry not found	2021-03-16 02:55:49 -07:00
Konstantin Lebedev	06da02739d	CheckAllChunkViews() for HEAD requests only	2021-03-16 14:15:17 +05:00
Chris Lu	91a3ac9731	2.33	2021-03-16 00:36:06 -07:00
Chris Lu	4b1ed227d1	revert fasthttp changes related to https://github.com/chrislusf/seaweedfs/issues/1907	2021-03-16 00:33:14 -07:00
Chris LuandGitHub	10164d0386	Merge pull request #1905 from wuh-fnst/s3ListCounterErr s3: count correction for the number of files in the dir	2021-03-15 20:54:58 -07:00
Chris Lu	22a07a12d2	[mount] [regression] `filer.path` option results in empty mount in 2.32 fix https://github.com/chrislusf/seaweedfs/issues/1906	2021-03-15 20:46:25 -07:00
wuh-fnst	050a7f9599	s3: count correction for the number of files in the dir	2021-03-16 09:22:03 +08:00
Konstantin Lebedev	3a3699867b	Status PartialContent for Content-Range response	2021-03-15 23:30:22 +05:00
Konstantin Lebedev	90510e3137	If WriteHeader is not called explicitly, the first call to Write will trigger an implicit WriteHeader(http.StatusOK). WriteHeader: Only one header may be written. Go does not currently!	2021-03-15 21:22:59 +05:00
Konstantin Lebedev	46b9f5cff4	add debug logging	2021-03-15 19:33:33 +05:00
Konstantin Lebedev	7194a5e7bf	avoid http error: superfluous response.WriteHeader https://github.com/chrislusf/seaweedfs/issues/1838	2021-03-15 18:52:59 +05:00
hilimdandGitHub	17d02264f3	Merge pull request #74 from chrislusf/master sync	2021-03-12 15:05:31 +08:00
hilimdandGitHub	620b91f23e	Merge pull request #73 from chrislusf/master sync	2021-02-23 13:41:30 +08:00
hilimdandGitHub	690d7c10b8	Merge pull request #72 from chrislusf/master Java: fix filerProxy mode	2021-02-06 14:50:31 +08:00
hilimdandGitHub	25b48a3e11	Merge pull request #71 from chrislusf/master sync	2021-02-06 14:18:35 +08:00