3.30

add ownership rest apis (#3765 )
fix delete key panic of remote sync dir (#3770 )
2026-10-04 05:33:05 +00:00 · 2022-10-02 18:13:44 -07:00 · 2022-10-01 19:18:00 -07:00 · 2022-10-01 02:33:47 -07:00 · 2022-09-30 08:49:38 -07:00 · 2022-09-30 08:44:41 -07:00
43 changed files with 1231 additions and 104 deletions
@@ -11,4 +11,4 @@ jobs:
      - name: 'Checkout Repository'
        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
      - name: 'Dependency Review'
-        uses: actions/dependency-review-action@23d1ffffb6fa5401173051ec21eba8c35242733f
+        uses: actions/dependency-review-action@375c5370086bfff256c37f8beec0f437e2e72ae1
@@ -5,11 +5,11 @@ go 1.19
 require (
 	cloud.google.com/go v0.104.0 // indirect
 	cloud.google.com/go/pubsub v1.25.1
-	cloud.google.com/go/storage v1.26.0
+	cloud.google.com/go/storage v1.27.0
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/Shopify/sarama v1.36.0
-	github.com/aws/aws-sdk-go v1.44.100
+	github.com/aws/aws-sdk-go v1.44.105
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
@@ -27,7 +27,7 @@ require (
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
-	github.com/fclairamb/ftpserverlib v0.19.1
+	github.com/fclairamb/ftpserverlib v0.20.0
 	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/go-errors/errors v1.1.1 // indirect
 	github.com/go-redis/redis/v8 v8.11.5
@@ -43,7 +43,7 @@ require (
 	github.com/google/go-cmp v0.5.8 // indirect
 	github.com/google/uuid v1.3.0
 	github.com/google/wire v0.5.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.4.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.5.1 // indirect
 	github.com/gorilla/mux v1.8.0
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
@@ -85,7 +85,7 @@ require (
 	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
 	github.com/seaweedfs/goexif v2.0.0+incompatible
 	github.com/seaweedfs/raft v1.1.0
-	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/spf13/afero v1.9.2 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
@@ -116,14 +116,14 @@ require (
 	golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd
 	golang.org/x/image v0.0.0-20200119044424-58c23975cae1
 	golang.org/x/net v0.0.0-20220909164309-bea034e7d591
-	golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 // indirect
-	golang.org/x/sys v0.0.0-20220818161305-2296e01440c6
+	golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 // indirect
+	golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/tools v0.1.12
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
-	google.golang.org/api v0.96.0
+	google.golang.org/api v0.97.0
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect
+	google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006 // indirect
 	google.golang.org/grpc v1.49.0
 	google.golang.org/protobuf v1.28.1
 	gopkg.in/inf.v0 v0.9.1 // indirect
@@ -150,7 +150,7 @@ require (
 	github.com/rabbitmq/amqp091-go v1.5.0
 	github.com/tikv/client-go/v2 v2.0.1
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.38.0
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.38.1
 	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
 	google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
 )
@@ -179,7 +179,7 @@ require (
 	github.com/d4l3k/messagediff v1.2.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
 	github.com/fatih/color v1.13.0 // indirect
-	github.com/fclairamb/go-log v0.3.0 // indirect
+	github.com/fclairamb/go-log v0.4.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
@@ -224,7 +224,7 @@ require (
 	go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
-	go.uber.org/zap v1.21.0 // indirect
+	go.uber.org/zap v1.22.0 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
@@ -77,8 +77,8 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 cloud.google.com/go/storage v1.21.0/go.mod h1:XmRlxkgPjlBONznT2dDUU/5XlpU2OjMnKuqnZI01LAA=
 cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y=
-cloud.google.com/go/storage v1.26.0 h1:lYAGjknyDJirSzfwUlkv4Nsnj7od7foxQNH/fqZqles=
-cloud.google.com/go/storage v1.26.0/go.mod h1:mk/N7YwIKEWyTvXAWQCIeiCTdLoRH6Pd5xmSnolQLTI=
+cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ=
+cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s=
 cloud.google.com/go/trace v1.0.0/go.mod h1:4iErSByzxkyHWzzlAj63/Gmjz0NH1ASqhJguHpGcr6A=
 cloud.google.com/go/trace v1.2.0/go.mod h1:Wc8y/uYyOhPy12KEnXG9XGrvfMz5F5SrYecQlbW1rwM=
 contrib.go.opencensus.io/exporter/aws v0.0.0-20200617204711-c478e41e60e9/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA=
@@ -154,8 +154,8 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb
 github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
-github.com/aws/aws-sdk-go v1.44.100 h1:7I86bWNQB+HGDT5z/dJy61J7qgbgLoZ7O51C9eL6hrA=
-github.com/aws/aws-sdk-go v1.44.100/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/aws/aws-sdk-go v1.44.105 h1:UUwoD1PRKIj3ltrDUYTDQj5fOTK3XsnqolLpRTMmSEM=
+github.com/aws/aws-sdk-go v1.44.105/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/aws/aws-sdk-go-v2 v1.16.2 h1:fqlCk6Iy3bnCumtrLz9r3mJ/2gUT0pJ0wLFVIdWh+JA=
 github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1/go.mod h1:n8Bs1ElDD2wJ9kCRTczA83gYbBmjSwZp3umc6zF4EeM=
@@ -288,10 +288,10 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
-github.com/fclairamb/ftpserverlib v0.19.1 h1:OIqW+AdcsUEq4apudrluDD1c4iCRidLAoQzJRBUJnbg=
-github.com/fclairamb/ftpserverlib v0.19.1/go.mod h1:cVeFR3wvEjgtK99686UXJaTvqZk8jbjHFnhaC23LGpc=
-github.com/fclairamb/go-log v0.3.0 h1:oSC7Zjt0FZIYC5xXahUUycKGkypSdr2srFPLsp7CLd0=
-github.com/fclairamb/go-log v0.3.0/go.mod h1:XG61EiPlAXnPDN8SA4N3zeA+GyBJmVOCCo12WORx/gA=
+github.com/fclairamb/ftpserverlib v0.20.0 h1:9NjQPxo3T0XQCyzSXZJ0XBH4mk4Qv4Lpw3GNVabvnpU=
+github.com/fclairamb/ftpserverlib v0.20.0/go.mod h1:7pR5Ckeygw3T006z1ND6HYSbJz+fTvkFAXlF6snW4yI=
+github.com/fclairamb/go-log v0.4.1 h1:rLtdSG9x2pK41AIAnE8WYpl05xBJfw1ZyYxZaXFcBsM=
+github.com/fclairamb/go-log v0.4.1/go.mod h1:sw1KvnkZ4wKCYkvy4SL3qVZcJSWFP8Ure4pM3z+KNn4=
 github.com/fluent/fluent-logger-golang v1.9.0 h1:zUdY44CHX2oIUc7VTNZc+4m+ORuO/mldQDA7czhWXEg=
 github.com/fluent/fluent-logger-golang v1.9.0/go.mod h1:2/HCT/jTy78yGyeNGQLGQsjF3zzzAuy6Xlk6FCMV5eU=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
@@ -476,8 +476,9 @@ github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pf
 github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
 github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM=
 github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM=
-github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk=
 github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c=
+github.com/googleapis/gax-go/v2 v2.5.1 h1:kBRZU0PSuI7PspsSb/ChWoVResUcwNVIdpB049pKTiw=
+github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo=
 github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
 github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
@@ -848,8 +849,9 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
-github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw=
 github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y=
@@ -928,8 +930,8 @@ github.com/ydb-platform/ydb-go-genproto v0.0.0-20220916101045-1c4d1accabb6/go.mo
 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 h1:EYSI1kulnHb0H0zt3yOw4cRj4ABMSMGwNe43D+fX7e4=
 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2/go.mod h1:Xfjce+VMU9yJVr1lj60yK2fFPWjB4jr/4cp3K7cjzi4=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3/go.mod h1:PFizF/vJsdAgEwjK3DVSBD52kdmRkWfSIS2q2pA+e88=
-github.com/ydb-platform/ydb-go-sdk/v3 v3.38.0 h1:MOcvHBWdAxUwNQgUZkCZy+tSfejIsteeFme+EpmkRL4=
-github.com/ydb-platform/ydb-go-sdk/v3 v3.38.0/go.mod h1:nc2x9ukuggHwRkIpEBAvDuUuITlZWY7J9jFKoetK6eE=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.38.1 h1:1PPvpBw6P6FVzFqEnfMHlCaamVWJln3f+cIccYW0tLQ=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.38.1/go.mod h1:nc2x9ukuggHwRkIpEBAvDuUuITlZWY7J9jFKoetK6eE=
 github.com/ydb-platform/ydb-go-yc v0.8.3 h1:92UUUMsfvtMl6mho8eQ9lbkiPrF3a9CT+RrVRAKNRwo=
 github.com/ydb-platform/ydb-go-yc v0.8.3/go.mod h1:zUolAFGzJ5XG8uwiseTLr9Lapm7L7hdVdZgLSuv9FXE=
 github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 h1:nMtixUijP0Z7iHJNT9fOL+dbmEzZxqU6Xk87ll7hqXg=
@@ -994,8 +996,9 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
 go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
-go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8=
 go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
+go.uber.org/zap v1.22.0 h1:Zcye5DUgBloQ9BaT4qc9BnjOFog5TvBSAGkJ3Nf70c0=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
 gocloud.dev v0.26.0 h1:4rM/SVL0lLs+rhC0Gmc+gt/82DBpb7nbpIZKXXnfMXg=
 gocloud.dev v0.26.0/go.mod h1:mkUgejbnbLotorqDyvedJO20XcZNTynmSeVSQS9btVg=
 gocloud.dev/pubsub/natspubsub v0.26.0 h1:f1hynU3D37dREFD9JzZLpCMoFJB5MyrHsujf9c6q+lE=
@@ -1157,8 +1160,9 @@ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j
 golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
-golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 h1:2o1E+E8TpNLklK9nHiPiK1uzIYrIHt+cQx3ynCwq9V8=
 golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
+golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 h1:lxqLZaMad/dJHMFZH0NiNpiEZI/nhgWhe4wgzpE+MuA=
+golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1273,11 +1277,12 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220818161305-2296e01440c6 h1:Sx/u41w+OwrInGdEckYmEuU5gHoGSL4QbDz3S9s6j4U=
-golang.org/x/sys v0.0.0-20220818161305-2296e01440c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc=
+golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -1426,8 +1431,8 @@ google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69
 google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
 google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg=
 google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o=
-google.golang.org/api v0.96.0 h1:F60cuQPJq7K7FzsxMYHAUJSiXh2oKctHxBMbDygxhfM=
-google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s=
+google.golang.org/api v0.97.0 h1:x/vEL1XDF/2V4xzdNgFPaKHluRESo2aTsL7QzHnBtGQ=
+google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1534,8 +1539,8 @@ google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP
 google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
 google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
 google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
-google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc h1:Nf+EdcTLHR8qDNN/KfkQL0u0ssxt9OhbaWCl5C0ucEI=
-google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk=
+google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006 h1:mmbq5q8M1t7dhkLw320YK4PsOXm6jdnUAkErImaIqOg=
+google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "3.29"
-version: "3.29"
+appVersion: "3.30"
+version: "3.30"
@@ -14,7 +14,9 @@ import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.security.SecureRandom;
+import java.security.MessageDigest;
 import java.util.List;
+import java.util.Base64;

 public class SeaweedWrite {

@@ -123,13 +125,20 @@ public class SeaweedWrite {
                                          final byte[] bytes,
                                          final long bytesOffset, final long bytesLength,
                                          byte[] cipherKey) throws IOException {
+        MessageDigest md = null;
+        try {
+            md = MessageDigest.getInstance("MD5");
+        } catch (java.security.NoSuchAlgorithmException e) {
+        }

        InputStream inputStream = null;
        if (cipherKey == null || cipherKey.length == 0) {
+            md.update(bytes, (int) bytesOffset, (int) bytesLength);
            inputStream = new ByteArrayInputStream(bytes, (int) bytesOffset, (int) bytesLength);
        } else {
            try {
                byte[] encryptedBytes = SeaweedCipher.encrypt(bytes, (int) bytesOffset, (int) bytesLength, cipherKey);
+                md.update(encryptedBytes);
                inputStream = new ByteArrayInputStream(encryptedBytes, 0, encryptedBytes.length);
            } catch (Exception e) {
                throw new IOException("fail to encrypt data", e);
@@ -140,6 +149,7 @@ public class SeaweedWrite {
        if (auth != null && auth.length() != 0) {
            post.addHeader("Authorization", "BEARER " + auth);
        }
+        post.addHeader("Content-MD5", Base64.getEncoder().encodeToString(md.digest()));

        post.setEntity(MultipartEntityBuilder.create()
                .setMode(HttpMultipartMode.BROWSER_COMPATIBLE)
@@ -149,6 +159,13 @@ public class SeaweedWrite {
        CloseableHttpResponse response = SeaweedUtil.getClosableHttpClient().execute(post);

        try {
+            if (response.getStatusLine().getStatusCode() / 100 != 2) {
+                if (response.getEntity().getContentType() != null && response.getEntity().getContentType().getValue().equals("application/json")) {
+                    throw new IOException(EntityUtils.toString(response.getEntity(), "UTF-8"));
+                } else {
+                    throw new IOException(response.getStatusLine().getReasonPhrase());
+                }
+            }

            String etag = response.getLastHeader("ETag").getValue();

@@ -37,11 +37,16 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour

 	var lastLogTsNs = time.Now().UnixNano()
 	processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error {
-		if *option.storageClass == "" {
-			delete(resp.EventNotification.NewEntry.Extended, s3_constants.AmzStorageClass)
-		} else {
-			resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass] = []byte(*option.storageClass)
+		if resp.EventNotification.NewEntry != nil {
+			if *option.storageClass == "" {
+				if _, ok := resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass]; ok {
+					delete(resp.EventNotification.NewEntry.Extended, s3_constants.AmzStorageClass)
+				}
+			} else {
+				resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass] = []byte(*option.storageClass)
+			}
 		}
+
 		processor.AddSyncJob(resp)
 		return nil
 	}, 3*time.Second, func(counter int64, lastTsNs int64) error {
@@ -190,6 +190,12 @@ func runFuse(cmd *Command, args []string) bool {
 			}
 		case "fusermount.path":
 			fusermountPath = parameter.value
+		default:
+			t := parameter.name
+			if parameter.value != "true" {
+				t = fmt.Sprintf("%s=%s", parameter.name, parameter.value)
+			}
+			mountOptions.extraOptions = append(mountOptions.extraOptions, t)
 		}
 	}

@@ -31,6 +31,7 @@ type MountOptions struct {
 	debugPort          *int
 	localSocket        *string
 	disableXAttr       *bool
+	extraOptions       []string
 }

 var (
@@ -166,7 +166,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 	// mount fuse
 	fuseMountOptions := &fuse.MountOptions{
 		AllowOther:               *option.allowOthers,
-		Options:                  nil,
+		Options:                  option.extraOptions,
 		MaxBackground:            128,
 		MaxWrite:                 1024 * 1024 * 2,
 		MaxReadAhead:             1024 * 1024 * 2,
@@ -92,10 +92,11 @@ func init() {
 	masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
 	masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
 	masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "", "Default replication type if not specified.")
-	masterOptions.garbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
-	masterOptions.metricsAddress = cmdServer.Flag.String("metrics.address", "", "Prometheus gateway address")
-	masterOptions.metricsIntervalSec = cmdServer.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
-	masterOptions.raftResumeState = cmdServer.Flag.Bool("resumeState", false, "resume previous state on start master server")
+	masterOptions.garbageThreshold = cmdServer.Flag.Float64("master.garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
+	masterOptions.metricsAddress = cmdServer.Flag.String("master.metrics.address", "", "Prometheus gateway address")
+	masterOptions.metricsIntervalSec = cmdServer.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
+	masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", false, "resume previous state on start master server")
+	masterOptions.raftHashicorp = cmdServer.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")
 	masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
 	masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")

@@ -265,6 +265,8 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 	// starting the cluster http server
 	clusterHttpServer := v.startClusterHttpService(volumeMux)

+	grace.OnReload(volumeServer.LoadNewVolumes)
+
 	stopChan := make(chan bool)
 	grace.OnInterrupt(func() {
 		fmt.Println("volume server has been killed")
@@ -10,7 +10,11 @@ import (
 )

 func ParseS3ConfigurationFromBytes[T proto.Message](content []byte, config T) error {
-	if err := jsonpb.Unmarshal(content, config); err != nil {
+	options := &jsonpb.UnmarshalOptions{
+		DiscardUnknown: true,
+		AllowPartial:   true,
+	}
+	if err := options.Unmarshal(content, config); err != nil {
 		return err
 	}
 	return nil
@@ -22,6 +22,7 @@ message Identity {
    string name = 1;
    repeated Credential credentials = 2;
    repeated string actions = 3;
+    string accountId = 4;
 }

 message Credential {
@@ -75,6 +75,7 @@ type Identity struct {
 	Name        string        `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
 	Credentials []*Credential `protobuf:"bytes,2,rep,name=credentials,proto3" json:"credentials,omitempty"`
 	Actions     []string      `protobuf:"bytes,3,rep,name=actions,proto3" json:"actions,omitempty"`
+	AccountId   string        `protobuf:"bytes,4,opt,name=accountId,proto3" json:"accountId,omitempty"`
 }

 func (x *Identity) Reset() {
@@ -130,6 +131,13 @@ func (x *Identity) GetActions() []string {
 	return nil
 }

+func (x *Identity) GetAccountId() string {
+	if x != nil {
+		return x.AccountId
+	}
+	return ""
+}
+
 type Credential struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -193,26 +201,28 @@ var file_iam_proto_rawDesc = []byte{
 	0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x0a, 0x69, 0x64, 0x65,
 	0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e,
 	0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52,
-	0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0x6e, 0x0a, 0x08, 0x49,
-	0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18,
-	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x0b, 0x63,
-	0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b,
-	0x32, 0x12, 0x2e, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e,
-	0x74, 0x69, 0x61, 0x6c, 0x52, 0x0b, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
-	0x73, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03,
-	0x28, 0x09, 0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x4a, 0x0a, 0x0a, 0x43,
-	0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63,
-	0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61,
-	0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x63, 0x72,
-	0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65,
-	0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x32, 0x21, 0x0a, 0x1f, 0x53, 0x65, 0x61, 0x77, 0x65,
-	0x65, 0x64, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73,
-	0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x4b, 0x0a, 0x10, 0x73, 0x65,
-	0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x08,
-	0x49, 0x61, 0x6d, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
-	0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73,
-	0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62,
-	0x2f, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0x8c, 0x01, 0x0a, 0x08,
+	0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x0b,
+	0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28,
+	0x0b, 0x32, 0x12, 0x2e, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x64, 0x65,
+	0x6e, 0x74, 0x69, 0x61, 0x6c, 0x52, 0x0b, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
+	0x6c, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20,
+	0x03, 0x28, 0x09, 0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09,
+	0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x22, 0x4a, 0x0a, 0x0a, 0x43, 0x72,
+	0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65,
+	0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63,
+	0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x63, 0x72, 0x65,
+	0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x63,
+	0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x32, 0x21, 0x0a, 0x1f, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65,
+	0x64, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4d,
+	0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x4b, 0x0a, 0x10, 0x73, 0x65, 0x61,
+	0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x08, 0x49,
+	0x61, 0x6d, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
+	0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65,
+	0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f,
+	0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }

 var (
@@ -16,6 +16,8 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 )

+var IdentityAnonymous *Identity
+
 type Action string

 type Iam interface {
@@ -32,10 +34,15 @@ type IdentityAccessManagement struct {

 type Identity struct {
 	Name        string
+	AccountId   string
 	Credentials []*Credential
 	Actions     []Action
 }

+func (i *Identity) isAnonymous() bool {
+	return i.Name == AccountAnonymous.Name
+}
+
 type Credential struct {
 	AccessKey string
 	SecretKey string
@@ -125,9 +132,23 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api
 	for _, ident := range config.Identities {
 		t := &Identity{
 			Name:        ident.Name,
+			AccountId:   AccountAdmin.Id,
 			Credentials: nil,
 			Actions:     nil,
 		}
+
+		if ident.Name == AccountAnonymous.Name {
+			if ident.AccountId != "" && ident.AccountId != AccountAnonymous.Id {
+				glog.Warningf("anonymous identity is associated with a non-anonymous account ID, the association is invalid")
+			}
+			t.AccountId = AccountAnonymous.Id
+			IdentityAnonymous = t
+		} else {
+			if len(ident.AccountId) > 0 {
+				t.AccountId = ident.AccountId
+			}
+		}
+
 		for _, action := range ident.Actions {
 			t.Actions = append(t.Actions, Action(action))
 		}
@@ -139,6 +160,13 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api
 		}
 		identities = append(identities, t)
 	}
+
+	if IdentityAnonymous == nil {
+		IdentityAnonymous = &Identity{
+			Name:      AccountAnonymous.Name,
+			AccountId: AccountAnonymous.Id,
+		}
+	}
 	iam.m.Lock()
 	// atomically switch
 	iam.identities = identities
@@ -173,7 +201,7 @@ func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, foun
 	iam.m.RLock()
 	defer iam.m.RUnlock()
 	for _, ident := range iam.identities {
-		if ident.Name == "anonymous" {
+		if ident.isAnonymous() {
 			return ident, true
 		}
 	}
@@ -259,6 +287,9 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action)
 		return identity, s3err.ErrAccessDenied
 	}

+	if !identity.isAnonymous() {
+		r.Header.Set(s3_constants.AmzAccountId, identity.AccountId)
+	}
 	return identity, s3err.ErrNone

 }
@@ -9,7 +9,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/util"
 )

-func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, lastTsNs int64) {
+func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, lastTsNs int64, prefix string, directoriesToWatch []string) {

 	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {

@@ -28,6 +28,7 @@ func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, la

 		_ = s3a.onIamConfigUpdate(dir, fileName, content)
 		_ = s3a.onCircuitBreakerConfigUpdate(dir, fileName, content)
+		_ = s3a.onBucketMetadataChange(dir, message.OldEntry, message.NewEntry)

 		return nil
 	}
@@ -35,7 +36,7 @@ func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, la
 	var clientEpoch int32
 	util.RetryForever("followIamChanges", func() error {
 		clientEpoch++
-		return pb.WithFilerClientFollowMetadata(s3a, clientName, s3a.randomClientId, clientEpoch, prefix, nil, &lastTsNs, 0, 0, processEventFn, pb.FatalOnError)
+		return pb.WithFilerClientFollowMetadata(s3a, clientName, s3a.randomClientId, clientEpoch, prefix, directoriesToWatch, &lastTsNs, 0, 0, processEventFn, pb.FatalOnError)
 	}, func(err error) bool {
 		glog.V(0).Infof("iam follow metadata changes: %v", err)
 		return true
@@ -63,3 +64,17 @@ func (s3a *S3ApiServer) onCircuitBreakerConfigUpdate(dir, filename string, conte
 	}
 	return nil
 }
+
+//reload bucket metadata
+func (s3a *S3ApiServer) onBucketMetadataChange(dir string, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) error {
+	if dir == s3a.option.BucketsPath {
+		if newEntry != nil {
+			s3a.bucketRegistry.LoadBucketMetadata(newEntry)
+			glog.V(0).Infof("updated bucketMetadata %s/%s", dir, newEntry)
+		} else {
+			s3a.bucketRegistry.RemoveBucketMetadata(oldEntry)
+			glog.V(0).Infof("remove bucketMetadata  %s/%s", dir, newEntry)
+		}
+	}
+	return nil
+}
@@ -3,6 +3,7 @@ package s3api
 import (
 	. "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/stretchr/testify/assert"
+	"reflect"
 	"testing"

 	jsonpb "google.golang.org/protobuf/encoding/protojson"
@@ -124,5 +125,98 @@ func TestCanDo(t *testing.T) {
 	}
 	assert.Equal(t, true, ident5.canDo(ACTION_READ, "special_bucket", "/a/b/c/d.txt"))
 	assert.Equal(t, true, ident5.canDo(ACTION_WRITE, "special_bucket", "/a/b/c/d.txt"))
+}

+type LoadS3ApiConfigurationTestCase struct {
+	pbIdent     *iam_pb.Identity
+	expectIdent *Identity
+}
+
+func TestLoadS3ApiConfiguration(t *testing.T) {
+	testCases := map[string]*LoadS3ApiConfigurationTestCase{
+		"notSpecifyAccountId": {
+			pbIdent: &iam_pb.Identity{
+				Name: "notSpecifyAccountId",
+				Actions: []string{
+					"Read",
+					"Write",
+				},
+				Credentials: []*iam_pb.Credential{
+					{
+						AccessKey: "some_access_key1",
+						SecretKey: "some_secret_key2",
+					},
+				},
+			},
+			expectIdent: &Identity{
+				Name:      "notSpecifyAccountId",
+				AccountId: AccountAdmin.Id,
+				Actions: []Action{
+					"Read",
+					"Write",
+				},
+				Credentials: []*Credential{
+					{
+						AccessKey: "some_access_key1",
+						SecretKey: "some_secret_key2",
+					},
+				},
+			},
+		},
+		"specifiedAccountID": {
+			pbIdent: &iam_pb.Identity{
+				Name:      "specifiedAccountID",
+				AccountId: "specifiedAccountID",
+				Actions: []string{
+					"Read",
+					"Write",
+				},
+			},
+			expectIdent: &Identity{
+				Name:      "specifiedAccountID",
+				AccountId: "specifiedAccountID",
+				Actions: []Action{
+					"Read",
+					"Write",
+				},
+			},
+		},
+		"anonymous": {
+			pbIdent: &iam_pb.Identity{
+				Name: "anonymous",
+				Actions: []string{
+					"Read",
+					"Write",
+				},
+			},
+			expectIdent: &Identity{
+				Name:      "anonymous",
+				AccountId: "anonymous",
+				Actions: []Action{
+					"Read",
+					"Write",
+				},
+			},
+		},
+	}
+
+	config := &iam_pb.S3ApiConfiguration{
+		Identities: make([]*iam_pb.Identity, 0),
+	}
+	for _, v := range testCases {
+		config.Identities = append(config.Identities, v.pbIdent)
+	}
+
+	iam := IdentityAccessManagement{}
+	err := iam.loadS3ApiConfiguration(config)
+	if err != nil {
+		return
+	}
+
+	for _, ident := range iam.identities {
+		tc := testCases[ident.Name]
+		if !reflect.DeepEqual(ident, tc.expectIdent) {
+			t.Error("not expect")
+		}
+	}
 }
@@ -0,0 +1,213 @@
+package s3api
+
+import (
+	"bytes"
+	"encoding/json"
+	"github.com/aws/aws-sdk-go/private/protocol/json/jsonutil"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+
+	//"github.com/seaweedfs/seaweedfs/weed/s3api"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"math"
+	"sync"
+)
+
+var loadBucketMetadataFromFiler = func(r *BucketRegistry, bucketName string) (*BucketMetaData, error) {
+	entry, err := filer_pb.GetEntry(r.s3a, util.NewFullPath(r.s3a.option.BucketsPath, bucketName))
+	if err != nil {
+		return nil, err
+	}
+
+	return buildBucketMetadata(entry), nil
+}
+
+type BucketMetaData struct {
+	_ struct{} `type:"structure"`
+
+	Name string
+
+	//By default, when another AWS account uploads an object to S3 bucket,
+	//that account (the object writer) owns the object, has access to it, and
+	//can grant other users access to it through ACLs. You can use Object Ownership
+	//to change this default behavior so that ACLs are disabled and you, as the
+	//bucket owner, automatically own every object in your bucket.
+	ObjectOwnership string
+
+	// Container for the bucket owner's display name and ID.
+	Owner *s3.Owner `type:"structure"`
+
+	// A list of grants for access controls.
+	Acl []*s3.Grant `locationName:"AccessControlList" locationNameList:"Grant" type:"list"`
+}
+
+type BucketRegistry struct {
+	metadataCache     map[string]*BucketMetaData
+	metadataCacheLock sync.RWMutex
+
+	notFound     map[string]struct{}
+	notFoundLock sync.RWMutex
+	s3a          *S3ApiServer
+}
+
+func NewBucketRegistry(s3a *S3ApiServer) *BucketRegistry {
+	br := &BucketRegistry{
+		metadataCache: make(map[string]*BucketMetaData),
+		notFound:      make(map[string]struct{}),
+		s3a:           s3a,
+	}
+	err := br.init()
+	if err != nil {
+		glog.Fatal("init bucket registry failed", err)
+		return nil
+	}
+	return br
+}
+
+func (r *BucketRegistry) init() error {
+	err := filer_pb.List(r.s3a, r.s3a.option.BucketsPath, "", func(entry *filer_pb.Entry, isLast bool) error {
+		r.LoadBucketMetadata(entry)
+		return nil
+	}, "", false, math.MaxUint32)
+	return err
+}
+
+func (r *BucketRegistry) LoadBucketMetadata(entry *filer_pb.Entry) {
+	bucketMetadata := buildBucketMetadata(entry)
+	r.metadataCacheLock.Lock()
+	defer r.metadataCacheLock.Unlock()
+	r.metadataCache[entry.Name] = bucketMetadata
+}
+
+func buildBucketMetadata(entry *filer_pb.Entry) *BucketMetaData {
+	entryJson, _ := json.Marshal(entry)
+	glog.V(3).Infof("build bucket metadata,entry=%s", entryJson)
+	bucketMetadata := &BucketMetaData{
+		Name: entry.Name,
+
+		//Default ownership: OwnershipBucketOwnerEnforced, which means Acl is disabled
+		ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced,
+
+		// Default owner: `AccountAdmin`
+		Owner: &s3.Owner{
+			ID:          &AccountAdmin.Id,
+			DisplayName: &AccountAdmin.Name,
+		},
+	}
+	if entry.Extended != nil {
+		//ownership control
+		ownership, ok := entry.Extended[s3_constants.ExtOwnershipKey]
+		if ok {
+			ownership := string(ownership)
+			valid := s3_constants.ValidateOwnership(ownership)
+			if valid {
+				bucketMetadata.ObjectOwnership = ownership
+			} else {
+				glog.Warningf("Invalid ownership: %s, bucket: %s", ownership, bucketMetadata.Name)
+			}
+		}
+
+		//access control policy
+		acpBytes, ok := entry.Extended[s3_constants.ExtAcpKey]
+		if ok {
+			var acp s3.AccessControlPolicy
+			err := jsonutil.UnmarshalJSON(&acp, bytes.NewReader(acpBytes))
+			if err == nil {
+				//validate owner
+				if acp.Owner != nil && acp.Owner.ID != nil {
+					bucketMetadata.Owner = acp.Owner
+				} else {
+					glog.Warningf("bucket ownerId is empty! bucket: %s", bucketMetadata.Name)
+				}
+
+				//acl
+				bucketMetadata.Acl = acp.Grants
+			} else {
+				glog.Warningf("Unmarshal ACP: %s(%v), bucket: %s", string(acpBytes), err, bucketMetadata.Name)
+			}
+		}
+	}
+	return bucketMetadata
+}
+
+func (r *BucketRegistry) RemoveBucketMetadata(entry *filer_pb.Entry) {
+	r.removeMetadataCache(entry.Name)
+	r.unMarkNotFound(entry.Name)
+}
+
+func (r *BucketRegistry) GetBucketMetadata(bucketName string) (*BucketMetaData, s3err.ErrorCode) {
+	r.metadataCacheLock.RLock()
+	bucketMetadata, ok := r.metadataCache[bucketName]
+	r.metadataCacheLock.RUnlock()
+	if ok {
+		return bucketMetadata, s3err.ErrNone
+	}
+
+	r.notFoundLock.RLock()
+	_, ok = r.notFound[bucketName]
+	r.notFoundLock.RUnlock()
+	if ok {
+		return nil, s3err.ErrNoSuchBucket
+	}
+
+	bucketMetadata, errCode := r.LoadBucketMetadataFromFiler(bucketName)
+	if errCode != s3err.ErrNone {
+		return nil, errCode
+	}
+
+	r.setMetadataCache(bucketMetadata)
+	r.unMarkNotFound(bucketName)
+	return bucketMetadata, s3err.ErrNone
+}
+
+func (r *BucketRegistry) LoadBucketMetadataFromFiler(bucketName string) (*BucketMetaData, s3err.ErrorCode) {
+	r.notFoundLock.Lock()
+	defer r.notFoundLock.Unlock()
+
+	//check if already exists
+	r.metadataCacheLock.RLock()
+	bucketMetaData, ok := r.metadataCache[bucketName]
+	r.metadataCacheLock.RUnlock()
+	if ok {
+		return bucketMetaData, s3err.ErrNone
+	}
+
+	//if not exists, load from filer
+	bucketMetadata, err := loadBucketMetadataFromFiler(r, bucketName)
+	if err != nil {
+		if err == filer_pb.ErrNotFound {
+			// The bucket doesn't actually exist and should no longer loaded from the filer
+			r.notFound[bucketName] = struct{}{}
+			return nil, s3err.ErrNoSuchBucket
+		}
+		return nil, s3err.ErrInternalError
+	}
+	return bucketMetadata, s3err.ErrNone
+}
+
+func (r *BucketRegistry) setMetadataCache(metadata *BucketMetaData) {
+	r.metadataCacheLock.Lock()
+	defer r.metadataCacheLock.Unlock()
+	r.metadataCache[metadata.Name] = metadata
+}
+
+func (r *BucketRegistry) removeMetadataCache(bucket string) {
+	r.metadataCacheLock.Lock()
+	defer r.metadataCacheLock.Unlock()
+	delete(r.metadataCache, bucket)
+}
+
+func (r *BucketRegistry) markNotFound(bucket string) {
+	r.notFoundLock.Lock()
+	defer r.notFoundLock.Unlock()
+	r.notFound[bucket] = struct{}{}
+}
+
+func (r *BucketRegistry) unMarkNotFound(bucket string) {
+	r.notFoundLock.Lock()
+	defer r.notFoundLock.Unlock()
+	delete(r.notFound, bucket)
+}
@@ -0,0 +1,236 @@
+package s3api
+
+import (
+	"fmt"
+	"github.com/aws/aws-sdk-go/private/protocol/json/jsonutil"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+	"reflect"
+	"sync"
+	"testing"
+	"time"
+)
+
+type BucketMetadataTestCase struct {
+	filerEntry           *filer_pb.Entry
+	expectBucketMetadata *BucketMetaData
+}
+
+var (
+	//bad entry
+	badEntry = &filer_pb.Entry{
+		Name: "badEntry",
+	}
+
+	//good entry
+	goodEntryAcp, _ = jsonutil.BuildJSON(&s3.AccessControlPolicy{
+		Owner: &s3.Owner{
+			DisplayName: &AccountAdmin.Name,
+			ID:          &AccountAdmin.Id,
+		},
+		Grants: s3_constants.PublicRead,
+	})
+	goodEntry = &filer_pb.Entry{
+		Name: "entryWithValidAcp",
+		Extended: map[string][]byte{
+			s3_constants.ExtOwnershipKey: []byte(s3_constants.OwnershipBucketOwnerEnforced),
+			s3_constants.ExtAcpKey:       goodEntryAcp,
+		},
+	}
+
+	//ownership is ""
+	ownershipEmptyStr = &filer_pb.Entry{
+		Name: "ownershipEmptyStr",
+		Extended: map[string][]byte{
+			s3_constants.ExtOwnershipKey: []byte(""),
+		},
+	}
+
+	//ownership valid
+	ownershipValid = &filer_pb.Entry{
+		Name: "ownershipValid",
+		Extended: map[string][]byte{
+			s3_constants.ExtOwnershipKey: []byte(s3_constants.OwnershipBucketOwnerEnforced),
+		},
+	}
+
+	//acp is ""
+	acpEmptyStr = &filer_pb.Entry{
+		Name: "acpEmptyStr",
+		Extended: map[string][]byte{
+			s3_constants.ExtAcpKey: []byte(""),
+		},
+	}
+
+	//acp is empty object
+	acpEmptyObjectAcp, _ = jsonutil.BuildJSON(&s3.AccessControlPolicy{
+		Owner:  nil,
+		Grants: nil,
+	})
+	acpEmptyObject = &filer_pb.Entry{
+		Name: "acpEmptyObject",
+		Extended: map[string][]byte{
+			s3_constants.ExtAcpKey: acpEmptyObjectAcp,
+		},
+	}
+
+	//acp owner is nil
+	acpOwnerNilAcp, _ = jsonutil.BuildJSON(&s3.AccessControlPolicy{
+		Owner:  nil,
+		Grants: make([]*s3.Grant, 1),
+	})
+	acpOwnerNil = &filer_pb.Entry{
+		Name: "acpOwnerNil",
+		Extended: map[string][]byte{
+			s3_constants.ExtAcpKey: acpOwnerNilAcp,
+		},
+	}
+
+	//load filer is
+	loadFilerBucket = make(map[string]int, 1)
+	//override `loadBucketMetadataFromFiler` to avoid really load from filer
+)
+
+var tcs = []*BucketMetadataTestCase{
+	{
+		badEntry, &BucketMetaData{
+			Name:            badEntry.Name,
+			ObjectOwnership: s3_constants.DefaultOwnershipForExists,
+			Owner: &s3.Owner{
+				DisplayName: &AccountAdmin.Name,
+				ID:          &AccountAdmin.Id,
+			},
+			Acl: nil,
+		},
+	},
+	{
+		goodEntry, &BucketMetaData{
+			Name:            goodEntry.Name,
+			ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced,
+			Owner: &s3.Owner{
+				DisplayName: &AccountAdmin.Name,
+				ID:          &AccountAdmin.Id,
+			},
+			Acl: s3_constants.PublicRead,
+		},
+	},
+	{
+		ownershipEmptyStr, &BucketMetaData{
+			Name:            ownershipEmptyStr.Name,
+			ObjectOwnership: s3_constants.DefaultOwnershipForExists,
+			Owner: &s3.Owner{
+				DisplayName: &AccountAdmin.Name,
+				ID:          &AccountAdmin.Id,
+			},
+			Acl: nil,
+		},
+	},
+	{
+		ownershipValid, &BucketMetaData{
+			Name:            ownershipValid.Name,
+			ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced,
+			Owner: &s3.Owner{
+				DisplayName: &AccountAdmin.Name,
+				ID:          &AccountAdmin.Id,
+			},
+			Acl: nil,
+		},
+	},
+	{
+		acpEmptyStr, &BucketMetaData{
+			Name:            acpEmptyStr.Name,
+			ObjectOwnership: s3_constants.DefaultOwnershipForExists,
+			Owner: &s3.Owner{
+				DisplayName: &AccountAdmin.Name,
+				ID:          &AccountAdmin.Id,
+			},
+			Acl: nil,
+		},
+	},
+	{
+		acpEmptyObject, &BucketMetaData{
+			Name:            acpEmptyObject.Name,
+			ObjectOwnership: s3_constants.DefaultOwnershipForExists,
+			Owner: &s3.Owner{
+				DisplayName: &AccountAdmin.Name,
+				ID:          &AccountAdmin.Id,
+			},
+			Acl: nil,
+		},
+	},
+	{
+		acpOwnerNil, &BucketMetaData{
+			Name:            acpOwnerNil.Name,
+			ObjectOwnership: s3_constants.DefaultOwnershipForExists,
+			Owner: &s3.Owner{
+				DisplayName: &AccountAdmin.Name,
+				ID:          &AccountAdmin.Id,
+			},
+			Acl: make([]*s3.Grant, 0),
+		},
+	},
+}
+
+func TestBuildBucketMetadata(t *testing.T) {
+	for _, tc := range tcs {
+		resultBucketMetadata := buildBucketMetadata(tc.filerEntry)
+		if !reflect.DeepEqual(resultBucketMetadata, tc.expectBucketMetadata) {
+			t.Fatalf("result is unexpect: \nresult: %v, \nexpect: %v", resultBucketMetadata, tc.expectBucketMetadata)
+		}
+	}
+}
+
+func TestGetBucketMetadata(t *testing.T) {
+	loadBucketMetadataFromFiler = func(r *BucketRegistry, bucketName string) (*BucketMetaData, error) {
+		time.Sleep(time.Second)
+		loadFilerBucket[bucketName] = loadFilerBucket[bucketName] + 1
+		return &BucketMetaData{
+			Name: bucketName,
+		}, nil
+	}
+
+	br := &BucketRegistry{
+		metadataCache: make(map[string]*BucketMetaData),
+		notFound:      make(map[string]struct{}),
+		s3a:           nil,
+	}
+
+	//start 40 goroutine for
+	var wg sync.WaitGroup
+	closeCh := make(chan struct{})
+	for i := 0; i < 40; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+		outLoop:
+			for {
+				for j := 0; j < 5; j++ {
+					select {
+					case <-closeCh:
+						break outLoop
+					default:
+						reqBucket := fmt.Sprintf("%c", 67+j)
+						_, errCode := br.GetBucketMetadata(reqBucket)
+						if errCode != s3err.ErrNone {
+							close(closeCh)
+							t.Error("not expect")
+						}
+					}
+				}
+				time.Sleep(10 * time.Microsecond)
+			}
+		}()
+	}
+	time.Sleep(time.Second)
+	close(closeCh)
+	wg.Wait()
+
+	//Each bucket is loaded from the filer only once
+	for bucketName, loadCount := range loadFilerBucket {
+		if loadCount != 1 {
+			t.Fatalf("lock is uneffict: %s, %d", bucketName, loadCount)
+		}
+	}
+}
@@ -91,6 +91,22 @@ func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry *
 	return filer_pb.GetEntry(s3a, fullPath)
 }

+func (s3a *S3ApiServer) updateEntry(parentDirectoryPath string, newEntry *filer_pb.Entry) error {
+	updateEntryRequest := &filer_pb.UpdateEntryRequest{
+		Directory: parentDirectoryPath,
+		Entry:     newEntry,
+	}
+
+	err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		err := filer_pb.UpdateEntry(client, updateEntryRequest)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	return err
+}
+
 func objectKey(key *string) *string {
 	if strings.HasPrefix(*key, "/") {
 		t := (*key)[1:]
@@ -0,0 +1,65 @@
+package s3_constants
+
+import (
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+const (
+	CannedAclPrivate                = "private"
+	CannedAclPublicRead             = "public-read"
+	CannedAclPublicReadWrite        = "public-read-write"
+	CannedAclAuthenticatedRead      = "authenticated-read"
+	CannedAclLogDeliveryWrite       = "log-delivery-write"
+	CannedAclBucketOwnerRead        = "bucket-owner-read"
+	CannedAclBucketOwnerFullControl = "bucket-owner-full-control"
+	CannedAclAwsExecRead            = "aws-exec-read"
+)
+
+var (
+	PublicRead = []*s3.Grant{
+		{
+			Grantee: &s3.Grantee{
+				Type: &GrantTypeGroup,
+				URI:  &GranteeGroupAllUsers,
+			},
+			Permission: &PermissionRead,
+		},
+	}
+
+	PublicReadWrite = []*s3.Grant{
+		{
+			Grantee: &s3.Grantee{
+				Type: &GrantTypeGroup,
+				URI:  &GranteeGroupAllUsers,
+			},
+			Permission: &PermissionRead,
+		},
+		{
+			Grantee: &s3.Grantee{
+				Type: &GrantTypeGroup,
+				URI:  &GranteeGroupAllUsers,
+			},
+			Permission: &PermissionWrite,
+		},
+	}
+
+	AuthenticatedRead = []*s3.Grant{
+		{
+			Grantee: &s3.Grantee{
+				Type: &GrantTypeGroup,
+				URI:  &GranteeGroupAuthenticatedUsers,
+			},
+			Permission: &PermissionRead,
+		},
+	}
+
+	LogDeliveryWrite = []*s3.Grant{
+		{
+			Grantee: &s3.Grantee{
+				Type: &GrantTypeGroup,
+				URI:  &GranteeGroupLogDelivery,
+			},
+			Permission: &PermissionWrite,
+		},
+	}
+)
@@ -0,0 +1,8 @@
+package s3_constants
+
+//Amazon S3 predefined groups
+var (
+	GranteeGroupAllUsers           = "http://acs.amazonaws.com/groups/global/AllUsers"
+	GranteeGroupAuthenticatedUsers = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
+	GranteeGroupLogDelivery        = "http://acs.amazonaws.com/groups/s3/LogDelivery"
+)
@@ -0,0 +1,7 @@
+package s3_constants
+
+var (
+	GrantTypeCanonicalUser         = "CanonicalUser"
+	GrantTypeAmazonCustomerByEmail = "AmazonCustomerByEmail"
+	GrantTypeGroup                 = "Group"
+)
@@ -0,0 +1,18 @@
+package s3_constants
+
+var (
+	OwnershipBucketOwnerPreferred = "BucketOwnerPreferred"
+	OwnershipObjectWriter         = "ObjectWriter"
+	OwnershipBucketOwnerEnforced  = "BucketOwnerEnforced"
+
+	DefaultOwnershipForCreate = OwnershipObjectWriter
+	DefaultOwnershipForExists = OwnershipBucketOwnerEnforced
+)
+
+func ValidateOwnership(ownership string) bool {
+	if ownership == "" || (ownership != OwnershipBucketOwnerPreferred && ownership != OwnershipObjectWriter && ownership != OwnershipBucketOwnerEnforced) {
+		return false
+	} else {
+		return true
+	}
+}
@@ -0,0 +1,9 @@
+package s3_constants
+
+var (
+	PermissionFullControl = "FULL_CONTROL"
+	PermissionRead        = "READ"
+	PermissionWrite       = "WRITE"
+	PermissionReadAcp     = "READ_ACP"
+	PermissionWriteAcp    = "WRITE_ACP"
+)
@@ -0,0 +1,6 @@
+package s3_constants
+
+const (
+	ExtAcpKey       = "Seaweed-X-Amz-Acp"
+	ExtOwnershipKey = "Seaweed-X-Amz-Ownership"
+)
@@ -43,6 +43,7 @@ const (
 // Non-Standard S3 HTTP request constants
 const (
 	AmzIdentityId = "s3-identity-id"
+	AmzAccountId  = "s3-account-id"
 	AmzAuthType   = "s3-auth-type"
 	AmzIsAdmin    = "s3-is-admin" // only set to http request header as a context
 )
@@ -0,0 +1,69 @@
+package s3api
+
+import (
+	"sync"
+)
+
+//Predefined Accounts
+var (
+	// AccountAdmin is used as the default account for IAM-Credentials access without Account configured
+	AccountAdmin = Account{
+		Name:         "admin",
+		EmailAddress: "admin@example.com",
+		Id:           "admin",
+	}
+
+	// AccountAnonymous is used to represent the account for anonymous access
+	AccountAnonymous = Account{
+		Name:         "anonymous",
+		EmailAddress: "anonymous@example.com",
+		Id:           "anonymous",
+	}
+)
+
+//Account represents a system user, a system user can
+//configure multiple IAM-Users, IAM-Users can configure
+//permissions respectively, and each IAM-User can
+//configure multiple security credentials
+type Account struct {
+	//Name is also used to display the "DisplayName" as the owner of the bucket or object
+	Name         string
+	EmailAddress string
+
+	//Id is used to identify an Account when granting cross-account access(ACLs) to buckets and objects
+	Id string
+}
+
+type AccountManager struct {
+	sync.Mutex
+	s3a *S3ApiServer
+
+	IdNameMapping  map[string]string
+	EmailIdMapping map[string]string
+}
+
+func NewAccountManager(s3a *S3ApiServer) *AccountManager {
+	am := &AccountManager{
+		s3a:            s3a,
+		IdNameMapping:  make(map[string]string),
+		EmailIdMapping: make(map[string]string),
+	}
+	am.initialize()
+	return am
+}
+
+func (am *AccountManager) GetAccountNameById(canonicalId string) string {
+	return am.IdNameMapping[canonicalId]
+}
+
+func (am *AccountManager) GetAccountIdByEmail(email string) string {
+	return am.EmailIdMapping[email]
+}
+
+func (am *AccountManager) initialize() {
+	// load predefined Accounts
+	for _, account := range []Account{AccountAdmin, AccountAnonymous} {
+		am.IdNameMapping[account.Id] = account.Name
+		am.EmailIdMapping[account.EmailAddress] = account.Id
+	}
+}
@@ -0,0 +1,28 @@
+package s3api
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+	"net/http"
+)
+
+func getAccountId(r *http.Request) string {
+	id := r.Header.Get(s3_constants.AmzAccountId)
+	if len(id) == 0 {
+		return AccountAnonymous.Id
+	} else {
+		return id
+	}
+}
+
+func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s3err.ErrorCode {
+	metadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket)
+	if errCode != s3err.ErrNone {
+		return errCode
+	}
+	accountId := getAccountId(r)
+	if accountId == AccountAdmin.Id || accountId == *metadata.Owner.ID {
+		return s3err.ErrNone
+	}
+	return s3err.ErrAccessDenied
+}
@@ -5,6 +5,8 @@ import (
 	"encoding/xml"
 	"errors"
 	"fmt"
+	"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 	"math"
 	"net/http"
 	"time"
@@ -343,3 +345,159 @@ func (s3a *S3ApiServer) GetBucketLocationHandler(w http.ResponseWriter, r *http.
 func (s3a *S3ApiServer) GetBucketRequestPaymentHandler(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponseXML(w, r, RequestPaymentConfiguration{Payer: "BucketOwner"})
 }
+
+// PutBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketOwnershipControls.html
+func (s3a *S3ApiServer) PutBucketOwnershipControls(w http.ResponseWriter, r *http.Request) {
+	bucket, _ := s3_constants.GetBucketAndObject(r)
+	glog.V(3).Infof("PutBucketOwnershipControls %s", bucket)
+
+	errCode := s3a.checkAccessByOwnership(r, bucket)
+	if errCode != s3err.ErrNone {
+		s3err.WriteErrorResponse(w, r, errCode)
+		return
+	}
+
+	if r.Body == nil || r.Body == http.NoBody {
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
+		return
+	}
+
+	var v s3.OwnershipControls
+	defer util.CloseRequest(r)
+
+	err := xmlutil.UnmarshalXML(&v, xml.NewDecoder(r.Body), "")
+	if err != nil {
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
+		return
+	}
+
+	if len(v.Rules) != 1 {
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
+		return
+	}
+
+	printOwnership := true
+	ownership := *v.Rules[0].ObjectOwnership
+	switch ownership {
+	case s3_constants.OwnershipObjectWriter:
+	case s3_constants.OwnershipBucketOwnerPreferred:
+	case s3_constants.OwnershipBucketOwnerEnforced:
+		printOwnership = false
+	default:
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
+		return
+	}
+
+	bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
+	if err != nil {
+		if err == filer_pb.ErrNotFound {
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
+			return
+		}
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		return
+	}
+
+	oldOwnership, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey]
+	if !ok || string(oldOwnership) != ownership {
+		if bucketEntry.Extended == nil {
+			bucketEntry.Extended = make(map[string][]byte)
+		}
+		bucketEntry.Extended[s3_constants.ExtOwnershipKey] = []byte(ownership)
+		err = s3a.updateEntry(s3a.option.BucketsPath, bucketEntry)
+		if err != nil {
+			s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+			return
+		}
+	}
+
+	if printOwnership {
+		result := &s3.PutBucketOwnershipControlsInput{
+			OwnershipControls: &v,
+		}
+		s3err.WriteAwsXMLResponse(w, r, http.StatusOK, result)
+	} else {
+		writeSuccessResponseEmpty(w, r)
+	}
+}
+
+// GetBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketOwnershipControls.html
+func (s3a *S3ApiServer) GetBucketOwnershipControls(w http.ResponseWriter, r *http.Request) {
+	bucket, _ := s3_constants.GetBucketAndObject(r)
+	glog.V(3).Infof("GetBucketOwnershipControls %s", bucket)
+
+	errCode := s3a.checkAccessByOwnership(r, bucket)
+	if errCode != s3err.ErrNone {
+		s3err.WriteErrorResponse(w, r, errCode)
+		return
+	}
+
+	bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
+	if err != nil {
+		if err == filer_pb.ErrNotFound {
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
+			return
+		}
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		return
+	}
+
+	v, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey]
+	if !ok {
+		s3err.WriteErrorResponse(w, r, s3err.OwnershipControlsNotFoundError)
+		return
+	}
+	ownership := string(v)
+
+	result := &s3.PutBucketOwnershipControlsInput{
+		OwnershipControls: &s3.OwnershipControls{
+			Rules: []*s3.OwnershipControlsRule{
+				{
+					ObjectOwnership: &ownership,
+				},
+			},
+		},
+	}
+
+	s3err.WriteAwsXMLResponse(w, r, http.StatusOK, result)
+}
+
+// DeleteBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketOwnershipControls.html
+func (s3a *S3ApiServer) DeleteBucketOwnershipControls(w http.ResponseWriter, r *http.Request) {
+	bucket, _ := s3_constants.GetBucketAndObject(r)
+	glog.V(3).Infof("PutBucketOwnershipControls %s", bucket)
+
+	errCode := s3a.checkAccessByOwnership(r, bucket)
+	if errCode != s3err.ErrNone {
+		s3err.WriteErrorResponse(w, r, errCode)
+		return
+	}
+
+	bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
+	if err != nil {
+		if err == filer_pb.ErrNotFound {
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
+			return
+		}
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		return
+	}
+
+	_, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey]
+	if !ok {
+		s3err.WriteErrorResponse(w, r, s3err.OwnershipControlsNotFoundError)
+		return
+	}
+
+	delete(bucketEntry.Extended, s3_constants.ExtOwnershipKey)
+	err = s3a.updateEntry(s3a.option.BucketsPath, bucketEntry)
+	if err != nil {
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		return
+	}
+
+	emptyOwnershipControls := &s3.OwnershipControls{
+		Rules: []*s3.OwnershipControlsRule{},
+	}
+	s3err.WriteAwsXMLResponse(w, r, http.StatusOK, emptyOwnershipControls)
+}
@@ -40,6 +40,8 @@ type S3ApiServer struct {
 	randomClientId int32
 	filerGuard     *security.Guard
 	client         *http.Client
+	accountManager *AccountManager
+	bucketRegistry *BucketRegistry
 }

 func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer *S3ApiServer, err error) {
@@ -59,6 +61,8 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer
 		filerGuard:     security.NewGuard([]string{}, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec),
 		cb:             NewCircuitBreaker(option),
 	}
+	s3ApiServer.accountManager = NewAccountManager(s3ApiServer)
+	s3ApiServer.bucketRegistry = NewBucketRegistry(s3ApiServer)
 	if option.LocalFilerSocket == "" {
 		s3ApiServer.client = &http.Client{Transport: &http.Transport{
 			MaxIdleConns:        1024,
@@ -76,7 +80,7 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer

 	s3ApiServer.registerRouter(router)

-	go s3ApiServer.subscribeMetaEvents("s3", filer.DirectoryEtcRoot, time.Now().UnixNano())
+	go s3ApiServer.subscribeMetaEvents("s3", time.Now().UnixNano(), filer.DirectoryEtcRoot, []string{option.BucketsPath})
 	return s3ApiServer, nil
 }

@@ -212,6 +216,14 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
 		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV2Handler, ACTION_LIST)), "LIST")).Queries("list-type", "2")

 		// buckets with query
+		// PutBucketOwnershipControls
+		bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketOwnershipControls, ACTION_ADMIN), "PUT")).Queries("ownershipControls", "")
+
+		//GetBucketOwnershipControls
+		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.GetBucketOwnershipControls, ACTION_READ), "GET")).Queries("ownershipControls", "")
+
+		//DeleteBucketOwnershipControls
+		bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketOwnershipControls, ACTION_ADMIN), "DELETE")).Queries("ownershipControls", "")

 		// raw buckets

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/xml"
 	"fmt"
+	"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
 	"github.com/gorilla/mux"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"net/http"
@@ -19,6 +20,16 @@ const (
 	MimeXML  mimeType = "application/xml"
 )

+func WriteAwsXMLResponse(w http.ResponseWriter, r *http.Request, statusCode int, result interface{}) {
+	var bytesBuffer bytes.Buffer
+	err := xmlutil.BuildXML(result, xml.NewEncoder(&bytesBuffer))
+	if err != nil {
+		WriteErrorResponse(w, r, ErrInternalError)
+		return
+	}
+	WriteResponse(w, r, statusCode, bytesBuffer.Bytes(), MimeXML)
+}
+
 func WriteXMLResponse(w http.ResponseWriter, r *http.Request, statusCode int, response interface{}) {
 	WriteResponse(w, r, statusCode, EncodeXMLResponse(response), MimeXML)
 }
@@ -107,6 +107,8 @@ const (

 	ErrTooManyRequest
 	ErrRequestBytesExceed
+
+	OwnershipControlsNotFoundError
 )

 // error code to APIError structure, these fields carry respective
@@ -414,6 +416,12 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Simultaneous request bytes exceed limitations",
 		HTTPStatusCode: http.StatusTooManyRequests,
 	},
+
+	OwnershipControlsNotFoundError: {
+		Code:           "OwnershipControlsNotFoundError",
+		Description:    "The bucket ownership controls were not found",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 }

 // GetAPIError provides API Error for input API error code.
@@ -16,6 +16,15 @@ import (

 func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
 	start := time.Now()
+
+	if r.Header.Get("Origin") != "" {
+		w.Header().Set("Access-Control-Allow-Origin", "*")
+		w.Header().Set("Access-Control-Expose-Headers", "*")
+		w.Header().Set("Access-Control-Allow-Headers", "*")
+		w.Header().Set("Access-Control-Allow-Credentials", "true")
+		w.Header().Set("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS")
+	}
+
 	if r.Method == "OPTIONS" {
 		OptionsHandler(w, r, false)
 		return
@@ -45,10 +54,7 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
 	}()

 	w.Header().Set("Server", "SeaweedFS Filer "+util.VERSION)
-	if r.Header.Get("Origin") != "" {
-		w.Header().Set("Access-Control-Allow-Origin", "*")
-		w.Header().Set("Access-Control-Allow-Credentials", "true")
-	}
+
 	switch r.Method {
 	case "GET":
 		fs.GetOrHeadHandler(w, r)
@@ -92,6 +98,13 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
 func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Request) {

 	start := time.Now()
+
+	if r.Header.Get("Origin") != "" {
+		w.Header().Set("Access-Control-Allow-Origin", "*")
+		w.Header().Set("Access-Control-Allow-Headers", "*")
+		w.Header().Set("Access-Control-Allow-Credentials", "true")
+	}
+
 	stats.FilerRequestCounter.WithLabelValues(r.Method).Inc()
 	defer func() {
 		stats.FilerRequestHistogram.WithLabelValues(r.Method).Observe(time.Since(start).Seconds())
@@ -108,10 +121,7 @@ func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Reque
 	}

 	w.Header().Set("Server", "SeaweedFS Filer "+util.VERSION)
-	if r.Header.Get("Origin") != "" {
-		w.Header().Set("Access-Control-Allow-Origin", "*")
-		w.Header().Set("Access-Control-Allow-Credentials", "true")
-	}
+
 	switch r.Method {
 	case "GET":
 		fs.GetOrHeadHandler(w, r)
@@ -122,11 +132,14 @@ func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Reque

 func OptionsHandler(w http.ResponseWriter, r *http.Request, isReadOnly bool) {
 	if isReadOnly {
-		w.Header().Add("Access-Control-Allow-Methods", "GET, OPTIONS")
+		w.Header().Set("Access-Control-Allow-Methods", "GET, OPTIONS")
 	} else {
-		w.Header().Add("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS")
+		w.Header().Set("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS")
+		w.Header().Set("Access-Control-Expose-Headers", "*")
 	}
-	w.Header().Add("Access-Control-Allow-Headers", "*")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	w.Header().Set("Access-Control-Allow-Headers", "*")
+	w.Header().Set("Access-Control-Allow-Credentials", "true")
 }

 // maybeCheckJwtAuthorization returns true if access should be granted, false if it should be denied
@@ -3,6 +3,7 @@ package weed_server
 import (
 	"context"
 	"fmt"
+	//"github.com/seaweedfs/seaweedfs/weed/s3api"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"io"
 	"net/http"
@@ -374,6 +375,12 @@ func SaveAmzMetaData(r *http.Request, existing map[string][]byte, isReplace bool
 		}
 	}

+	//acp
+	acp := r.Header.Get(s3_constants.ExtAcpKey)
+	if len(acp) > 0 {
+		metadata[s3_constants.ExtAcpKey] = []byte(acp)
+	}
+
 	return

 }
@@ -137,6 +137,11 @@ func (vs *VolumeServer) SetStopping() {
 	vs.store.SetStopping()
 }

+func (vs *VolumeServer) LoadNewVolumes() {
+	glog.V(0).Infoln(" Loading new volume ids ...")
+	vs.store.LoadNewVolumes()
+}
+
 func (vs *VolumeServer) Shutdown() {
 	glog.V(0).Infoln("Shutting down volume server...")
 	vs.store.Close()
@@ -19,6 +19,7 @@ func init() {
 }

 type commandFsMetaLoad struct {
+	dirPrefix *string
 }

 func (c *commandFsMetaLoad) Name() string {
@@ -30,6 +31,7 @@ func (c *commandFsMetaLoad) Help() string {

 	fs.meta.load <filer_host>-<port>-<time>.meta
 	fs.meta.load -v=false <filer_host>-<port>-<time>.meta // skip printing out the verbose output
+	fs.meta.load -dirPrefix=/buckets/important* <filer_host>.meta // load any dirs with prefix "important"

 `
 }
@@ -44,6 +46,7 @@ func (c *commandFsMetaLoad) Do(args []string, commandEnv *CommandEnv, writer io.
 	fileName := args[len(args)-1]

 	metaLoadCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	c.dirPrefix = metaLoadCommand.String("dirPrefix", "", "load entries only with directories matching prefix")
 	verbose := metaLoadCommand.Bool("v", true, "verbose mode")
 	if err = metaLoadCommand.Parse(args[0 : len(args)-1]); err != nil {
 		return nil
@@ -83,11 +86,22 @@ func (c *commandFsMetaLoad) Do(args []string, commandEnv *CommandEnv, writer io.
 				return err
 			}

+			// check collection name pattern
+			entryFullName := string(util.FullPath(fullEntry.Dir).Child(fullEntry.Entry.Name))
+			if *c.dirPrefix != "" {
+				if !strings.HasPrefix(fullEntry.Dir, *c.dirPrefix) {
+					if *verbose {
+						fmt.Fprintf(writer, "not match dir prefix %s\n", entryFullName)
+					}
+					continue
+				}
+			}
+
 			if *verbose || lastLogTime.Add(time.Second).Before(time.Now()) {
 				if !*verbose {
 					lastLogTime = time.Now()
 				}
-				fmt.Fprintf(writer, "load %s\n", util.FullPath(fullEntry.Dir).Child(fullEntry.Entry.Name))
+				fmt.Fprintf(writer, "load %s\n", entryFullName)
 			}

 			fullEntry.Entry.Name = strings.ReplaceAll(fullEntry.Entry.Name, "/", "x")
@@ -393,8 +393,9 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, dataNodeI
 			}
 			buf.Write(resp.FileContent)
 		}
+		fileredBuf := filterDeletedNeedleFromIdx(buf.Bytes())
 		if vinfo.isReadOnly == false {
-			index, err := idx.FirstInvalidIndex(buf.Bytes(), func(key types.NeedleId, offset types.Offset, size types.Size) (bool, error) {
+			index, err := idx.FirstInvalidIndex(fileredBuf.Bytes(), func(key types.NeedleId, offset types.Offset, size types.Size) (bool, error) {
 				resp, err := volumeServerClient.ReadNeedleMeta(context.Background(), &volume_server_pb.ReadNeedleMetaRequest{
 					VolumeId: volumeId,
 					NeedleId: uint64(key),
@@ -407,12 +408,13 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, dataNodeI
 				return resp.LastModified <= cutoffFrom, nil
 			})
 			if err != nil {
-				fmt.Fprintf(writer, "Failed to search for last vilad index on volume %d with error %v", volumeId, err)
+				fmt.Fprintf(writer, "Failed to search for last valid index on volume %d with error %v", volumeId, err)
+			} else {
+				fileredBuf.Truncate(index * types.NeedleMapEntrySize)
 			}
-			buf.Truncate(index * types.NeedleMapEntrySize)
 		}
 		idxFilename := getVolumeFileIdFile(tempFolder, dataNodeId, volumeId)
-		err = writeToFile(buf.Bytes(), idxFilename)
+		err = writeToFile(fileredBuf.Bytes(), idxFilename)
 		if err != nil {
 			return fmt.Errorf("failed to copy %d%s from %s: %v", volumeId, ext, vinfo.server, err)
 		}
@@ -718,3 +720,14 @@ func writeToFile(bytes []byte, fileName string) error {
 	dst.Write(bytes)
 	return nil
 }
+
+func filterDeletedNeedleFromIdx(arr []byte) bytes.Buffer {
+	var filteredBuf bytes.Buffer
+	for i := 0; i < len(arr); i += types.NeedleMapEntrySize {
+		size := types.BytesToSize(arr[i+types.NeedleIdSize+types.OffsetSize : i+types.NeedleIdSize+types.OffsetSize+types.SizeSize])
+		if size > 0 {
+			filteredBuf.Write(arr[i : i+types.NeedleIdSize+types.OffsetSize+types.SizeSize])
+		}
+	}
+	return filteredBuf
+}
@@ -361,6 +361,12 @@ func (s *Store) SetStopping() {
 	}
 }

+func (s *Store) LoadNewVolumes() {
+	for _, location := range s.Locations {
+		location.loadExistingVolumes(s.NeedleMapKind)
+	}
+}
+
 func (s *Store) Close() {
 	for _, location := range s.Locations {
 		location.Close()
@@ -84,10 +84,6 @@ func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption, onReadSize
 func (v *Volume) readNeedleMetaAt(n *needle.Needle, offset int64, size int32) (err error) {
 	v.dataFileAccessLock.RLock()
 	defer v.dataFileAccessLock.RUnlock()
-	// read deleted meta data
-	if size < 0 {
-		size = -size
-	}
 	err = n.ReadNeedleMeta(v.DataBackend, offset, Size(size), v.Version())
 	if err == needle.ErrorSizeMismatch && OffsetSize == 4 {
 		err = n.ReadNeedleMeta(v.DataBackend, offset+int64(MaxPossibleVolumeSize), Size(size), v.Version())
@@ -5,7 +5,7 @@ import (
 )

 var (
-	VERSION_NUMBER = fmt.Sprintf("%.02f", 3.29)
+	VERSION_NUMBER = fmt.Sprintf("%.02f", 3.30)
 	VERSION        = sizeLimit + " " + VERSION_NUMBER
 	COMMIT         = ""
 )
@@ -11,39 +11,55 @@ import (
 )

 var signalChan chan os.Signal
-var hooks = make([]func(), 0)
-var hookLock sync.RWMutex
+var interruptHooks = make([]func(), 0)
+var interruptHookLock sync.RWMutex
+var reloadHooks = make([]func(), 0)
+var reloadHookLock sync.RWMutex

 func init() {
 	signalChan = make(chan os.Signal, 1)
-	signal.Ignore(syscall.SIGHUP)
 	signal.Notify(signalChan,
 		os.Interrupt,
 		os.Kill,
 		syscall.SIGALRM,
-		// syscall.SIGHUP,
+		syscall.SIGHUP,
 		syscall.SIGINT,
 		syscall.SIGTERM,
 		// syscall.SIGQUIT,
 	)
 	go func() {
-		for range signalChan {
-			hookLock.RLock()
-			for _, hook := range hooks {
-				hook()
+		for s := range signalChan {
+			if s.String() == syscall.SIGHUP.String() {
+				reloadHookLock.RLock()
+				for _, hook := range reloadHooks {
+					hook()
+				}
+				reloadHookLock.RUnlock()
+			} else {
+				interruptHookLock.RLock()
+				for _, hook := range interruptHooks {
+					hook()
+				}
+				interruptHookLock.RUnlock()
+				os.Exit(0)
 			}
-			hookLock.RUnlock()
-			os.Exit(0)
 		}
 	}()
 }

+func OnReload(fn func()) {
+	// prevent reentry
+	reloadHookLock.Lock()
+	defer reloadHookLock.Unlock()
+	reloadHooks = append(reloadHooks, fn)
+}
+
 func OnInterrupt(fn func()) {
 	// prevent reentry
-	hookLock.Lock()
-	defer hookLock.Unlock()
+	interruptHookLock.Lock()
+	defer interruptHookLock.Unlock()

 	// deal with control+c,etc
 	// controlling terminal close, daemon not exit
-	hooks = append(hooks, fn)
+	interruptHooks = append(interruptHooks, fn)
 }
@@ -89,7 +89,7 @@ func TestInsertAndFind(t *testing.T) {
 		}
 	}
 	// println("print list")
-	list.println()
+	// list.println()

 }
Author	SHA1	Message	Date
chrislu	29632d5a34	3.30	2022-10-02 18:13:44 -07:00
LHHDZandGitHub	e9584d9661	add ownership rest apis (#3765 )	2022-10-01 19:18:00 -07:00
Konstantin LebedevandGitHub	6fa3d0cc46	fix delete key panic of remote sync dir (#3770 )	2022-10-01 02:33:47 -07:00
chrislu	c3487d3da7	better fix for option format	2022-09-30 08:49:38 -07:00
chrislu	ba0db28dbb	mount: fix extra option format	2022-09-30 08:44:41 -07:00
chrislu	8e81619d02	mount: accept all extra mount options fix https://github.com/seaweedfs/seaweedfs/issues/3767	2022-09-30 08:40:37 -07:00
kvtbandGitHub	620be2be16	client-java: provide Content-MD5 when uploading chunks (#3761 ) * client-java: provide Content-MD5 when uploading chunks * Update SeaweedWrite.java	2022-09-29 22:56:12 -07:00
Eric Yang GitHub root	56c94cc08e	ADHOC: filter deleted files from idx file binary search (#3763 ) * ADHOC: filter deleted files from idx file binary search * remove unwanted check Co-authored-by: root <root@HQ-10MSTD3EY.roblox.local>	2022-09-29 12:48:36 -07:00
LHHDZandGitHub	3de1e19780	s3: sync bucket info from filer (#3759 )	2022-09-29 12:29:01 -07:00
kvtbandGitHub	5e9039d728	client-java: handle server HTTP errors (#3762 ) * client-java: handle server HTTP errors * Update SeaweedWrite.java	2022-09-29 12:20:14 -07:00
LHHDZandGitHub	e00a12b099	associate `Account` and `Identity` by accountId (#3754 )	2022-09-28 13:25:59 -07:00
LHHDZandGitHub	aacdcc4cad	s3: add account (#3753 ) associate `Account` and `Identity` by accountId	2022-09-28 12:45:18 -07:00
Konstantin LebedevandGitHub	301b678147	[volume] Add new volumes to HUP(reload) signal (#3755 ) Add new volumes to HUP(reload) signal	2022-09-28 12:44:13 -07:00
chrislu	47fd5d3fa1	skip debug message	2022-09-27 13:12:31 -07:00
Konstantin Lebedev GitHub Chris Lu	7ae51d1ec6	fix set master options for single server (#3708 ) Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>	2022-09-27 12:06:35 -07:00
jerebear12andGitHub	190afcc3fd	Allow CORS on Filer (#3750 ) Adjusted filer response headers	2022-09-27 11:47:52 -07:00
Konstantin LebedevandGitHub	faa6167b6b	fs.meta.load load any dirs with prefix "important" (#3747 ) * fs.meta.load load any dirs with prefix "important" * replace dirPattern to dirPrefix * help dirPrefix	2022-09-27 09:58:46 -07:00
chrislu	b6d7556dda	skip truncation on error fix https://github.com/seaweedfs/seaweedfs/issues/3746	2022-09-27 09:48:23 -07:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	ea15476b06	build(deps): bump github.com/fclairamb/ftpserverlib from 0.19.1 to 0.20.0 (#3742 ) build(deps): bump github.com/fclairamb/ftpserverlib Bumps [github.com/fclairamb/ftpserverlib](https://github.com/fclairamb/ftpserverlib) from 0.19.1 to 0.20.0. - [Release notes](https://github.com/fclairamb/ftpserverlib/releases) - [Commits](https://github.com/fclairamb/ftpserverlib/compare/v0.19.1...v0.20.0) --- updated-dependencies: - dependency-name: github.com/fclairamb/ftpserverlib dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2022-09-26 09:04:12 -07:00
dependabot[bot]andGitHub	53c7161f27	build(deps): bump github.com/aws/aws-sdk-go from 1.44.100 to 1.44.105 (#3744 )	2022-09-26 08:43:27 -07:00
dependabot[bot]andGitHub	30200cb6af	build(deps): bump google.golang.org/api from 0.96.0 to 0.97.0 (#3743 )	2022-09-26 08:42:06 -07:00
dependabot[bot]andGitHub	b5d99110d1	build(deps): bump github.com/ydb-platform/ydb-go-sdk/v3 from 3.38.0 to 3.38.1 (#3741 )	2022-09-26 08:41:19 -07:00
dependabot[bot]andGitHub	6b93148f9c	build(deps): bump cloud.google.com/go/storage from 1.26.0 to 1.27.0 (#3740 )	2022-09-26 08:40:47 -07:00
dependabot[bot]andGitHub	d1357c1a3a	build(deps): bump actions/dependency-review-action from 2.1.0 to 2.4.0 (#3739 )	2022-09-26 08:35:23 -07:00