3.31

fix https://github.com/seaweedfs/seaweedfs/discussions/3811

.github/workflows/depsreview.yml

@@ -11,4 +11,4 @@ jobs:
       - name: 'Checkout Repository'
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@23d1ffffb6fa5401173051ec21eba8c35242733f
+        uses: actions/dependency-review-action@375c5370086bfff256c37f8beec0f437e2e72ae1

docker/compose/local-hashicorp-raft-compose.yml

@@ -6,7 +6,7 @@ services:
     ports:
       - 9333:9333
       - 19333:19333
-    command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master0 -port=9333 -peers=master1:9334,master2:9335 -mdir=/data"
+    command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master0 -port=9333 -peers=master1:9334,master2:9335 -mdir=/data"
     volumes:
       - ./master/0:/data
     environment:
@@ -18,7 +18,7 @@ services:
     ports:
       - 9334:9334
       - 19334:19334
-    command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master1 -port=9334 -peers=master0:9333,master2:9335 -mdir=/data"
+    command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master1 -port=9334 -peers=master0:9333,master2:9335 -mdir=/data"
     volumes:
       - ./master/1:/data
     environment:
@@ -30,7 +30,7 @@ services:
     ports:
       - 9335:9335
       - 19335:19335
-    command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master2 -port=9335 -peers=master0:9333,master1:9334 -mdir=/data"
+    command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master2 -port=9335 -peers=master0:9333,master1:9334 -mdir=/data"
     volumes:
       - ./master/2:/data
     environment:

go.mod

@@ -5,16 +5,16 @@ go 1.19
 require (
 	cloud.google.com/go v0.104.0 // indirect
 	cloud.google.com/go/pubsub v1.25.1
-	cloud.google.com/go/storage v1.26.0
+	cloud.google.com/go/storage v1.27.0
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.15.0
-	github.com/Shopify/sarama v1.36.0
-	github.com/aws/aws-sdk-go v1.44.100
+	github.com/Shopify/sarama v1.37.0
+	github.com/aws/aws-sdk-go v1.44.110
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
-	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
+	github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/disintegration/imaging v1.6.2
@@ -27,7 +27,7 @@ require (
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
-	github.com/fclairamb/ftpserverlib v0.19.1
+	github.com/fclairamb/ftpserverlib v0.20.0
 	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/go-errors/errors v1.1.1 // indirect
 	github.com/go-redis/redis/v8 v8.11.5
@@ -43,10 +43,10 @@ require (
 	github.com/google/go-cmp v0.5.8 // indirect
 	github.com/google/uuid v1.3.0
 	github.com/google/wire v0.5.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.4.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.5.1 // indirect
 	github.com/gorilla/mux v1.8.0
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
-	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
@@ -57,7 +57,7 @@ require (
 	github.com/json-iterator/go v1.1.12
 	github.com/karlseguin/ccache/v2 v2.0.8
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.15.9 // indirect
+	github.com/klauspost/compress v1.15.11 // indirect
 	github.com/klauspost/reedsolomon v1.11.0
 	github.com/kurin/blazer v0.5.3
 	github.com/lib/pq v1.10.7
@@ -66,7 +66,7 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-ieproxy v0.0.9 // indirect
 	github.com/mattn/go-isatty v0.0.16 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -85,7 +85,7 @@ require (
 	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
 	github.com/seaweedfs/goexif v2.0.0+incompatible
 	github.com/seaweedfs/raft v1.1.0
-	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/spf13/afero v1.9.2 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
@@ -109,21 +109,21 @@ require (
 	go.etcd.io/etcd/client/v3 v3.5.5
 	go.mongodb.org/mongo-driver v1.10.2
 	go.opencensus.io v0.23.0 // indirect
-	gocloud.dev v0.26.0
-	gocloud.dev/pubsub/natspubsub v0.26.0
-	gocloud.dev/pubsub/rabbitpubsub v0.26.0
+	gocloud.dev v0.27.0
+	gocloud.dev/pubsub/natspubsub v0.27.0
+	gocloud.dev/pubsub/rabbitpubsub v0.27.0
 	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
 	golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd
 	golang.org/x/image v0.0.0-20200119044424-58c23975cae1
-	golang.org/x/net v0.0.0-20220909164309-bea034e7d591
-	golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 // indirect
-	golang.org/x/sys v0.0.0-20220818161305-2296e01440c6
+	golang.org/x/net v0.0.0-20220927171203-f486391704dc
+	golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 // indirect
+	golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/tools v0.1.12
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
-	google.golang.org/api v0.96.0
+	google.golang.org/api v0.98.0
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect
+	google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006 // indirect
 	google.golang.org/grpc v1.49.0
 	google.golang.org/protobuf v1.28.1
 	gopkg.in/inf.v0 v0.9.1 // indirect
@@ -143,15 +143,15 @@ require (
 	github.com/Jille/raft-grpc-transport v1.2.0
 	github.com/arangodb/go-driver v1.3.3
 	github.com/fluent/fluent-logger-golang v1.9.0
-	github.com/google/flatbuffers v2.0.8+incompatible
+	github.com/google/flatbuffers v22.9.29+incompatible
 	github.com/hanwen/go-fuse/v2 v2.1.1-0.20220627082937-d01fda7edf17
 	github.com/hashicorp/raft v1.3.10
 	github.com/hashicorp/raft-boltdb/v2 v2.2.2
 	github.com/rabbitmq/amqp091-go v1.5.0
 	github.com/tikv/client-go/v2 v2.0.1
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.38.0
-	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.38.3
+	golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7
 	google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
 )
 
@@ -160,30 +160,30 @@ require (
 	cloud.google.com/go/iam v0.3.0 // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
 	github.com/armon/go-metrics v0.3.10 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.16.2 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.15.3 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.11.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sns v1.17.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sqs v1.18.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.11.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 // indirect
-	github.com/aws/smithy-go v1.11.2 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.16.8 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.15.15 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.12.10 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.15 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.16 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sns v1.17.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.19.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.11.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.16.10 // indirect
+	github.com/aws/smithy-go v1.12.0 // indirect
 	github.com/benbjohnson/clock v1.1.0 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/d4l3k/messagediff v1.2.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
 	github.com/fatih/color v1.13.0 // indirect
-	github.com/fclairamb/go-log v0.3.0 // indirect
+	github.com/fclairamb/go-log v0.4.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
+	github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
-	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/hashicorp/go-hclog v1.2.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-msgpack v1.1.5 // indirect
@@ -194,17 +194,17 @@ require (
 	github.com/jonboulle/clockwork v0.2.2 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.1.1 // indirect
-	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-runewidth v0.0.7 // indirect
 	github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
-	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
-	github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d // indirect
+	github.com/montanaflynn/stats v0.6.6 // indirect
+	github.com/nats-io/nats.go v1.16.0 // indirect
 	github.com/nats-io/nkeys v0.3.0 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
 	github.com/philhofer/fwd v1.1.1 // indirect
-	github.com/pierrec/lz4/v4 v4.1.15 // indirect
+	github.com/pierrec/lz4/v4 v4.1.17 // indirect
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
 	github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect
 	github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898 // indirect
@@ -216,7 +216,7 @@ require (
 	github.com/tinylib/msgp v1.1.6 // indirect
 	github.com/twmb/murmur3 v1.1.3 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
-	github.com/ydb-platform/ydb-go-genproto v0.0.0-20220916101045-1c4d1accabb6 // indirect
+	github.com/ydb-platform/ydb-go-genproto v0.0.0-20220922065549-66df47a830ba // indirect
 	github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect
 	github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
@@ -224,7 +224,7 @@ require (
 	go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
-	go.uber.org/zap v1.21.0 // indirect
+	go.uber.org/zap v1.22.0 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect

k8s/helm_charts2/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "3.29"
-version: "3.29"
+appVersion: "3.31"
+version: "3.31"

other/java/client/pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>3.13</version>
+    <version>3.30</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -14,7 +14,7 @@
     </parent>
 
     <properties>
-        <protobuf.version>3.16.1</protobuf.version>
+        <protobuf.version>3.16.3</protobuf.version>
         <!-- follow https://github.com/grpc/grpc-java -->
         <grpc.version>1.23.0</grpc.version>
         <guava.version>30.0-jre</guava.version>
@@ -159,7 +159,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-javadoc-plugin</artifactId>
-                <version>2.9.1</version>
+                <version>3.0.1</version>
                 <executions>
                     <execution>
                         <id>attach-javadocs</id>

other/java/client/pom.xml.deploy

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>3.13</version>
+    <version>3.30</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -154,7 +154,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-javadoc-plugin</artifactId>
-                <version>2.9.1</version>
+                <version>3.0.1</version>
                 <executions>
                     <execution>
                         <id>attach-javadocs</id>

other/java/client/pom_debug.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>3.13</version>
+    <version>3.30</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -128,7 +128,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-javadoc-plugin</artifactId>
-                <version>2.9.1</version>
+                <version>3.0.1</version>
                 <executions>
                     <execution>
                         <id>attach-javadocs</id>

other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java

@@ -14,7 +14,9 @@ import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.security.SecureRandom;
+import java.security.MessageDigest;
 import java.util.List;
+import java.util.Base64;
 
 public class SeaweedWrite {
 
@@ -123,13 +125,20 @@ public class SeaweedWrite {
                                           final byte[] bytes,
                                           final long bytesOffset, final long bytesLength,
                                           byte[] cipherKey) throws IOException {
+        MessageDigest md = null;
+        try {
+            md = MessageDigest.getInstance("MD5");
+        } catch (java.security.NoSuchAlgorithmException e) {
+        }
 
         InputStream inputStream = null;
         if (cipherKey == null || cipherKey.length == 0) {
+            md.update(bytes, (int) bytesOffset, (int) bytesLength);
             inputStream = new ByteArrayInputStream(bytes, (int) bytesOffset, (int) bytesLength);
         } else {
             try {
                 byte[] encryptedBytes = SeaweedCipher.encrypt(bytes, (int) bytesOffset, (int) bytesLength, cipherKey);
+                md.update(encryptedBytes);
                 inputStream = new ByteArrayInputStream(encryptedBytes, 0, encryptedBytes.length);
             } catch (Exception e) {
                 throw new IOException("fail to encrypt data", e);
@@ -140,6 +149,7 @@ public class SeaweedWrite {
         if (auth != null && auth.length() != 0) {
             post.addHeader("Authorization", "BEARER " + auth);
         }
+        post.addHeader("Content-MD5", Base64.getEncoder().encodeToString(md.digest()));
 
         post.setEntity(MultipartEntityBuilder.create()
                 .setMode(HttpMultipartMode.BROWSER_COMPATIBLE)
@@ -149,6 +159,13 @@ public class SeaweedWrite {
         CloseableHttpResponse response = SeaweedUtil.getClosableHttpClient().execute(post);
 
         try {
+            if (response.getStatusLine().getStatusCode() / 100 != 2) {
+                if (response.getEntity().getContentType() != null && response.getEntity().getContentType().getValue().equals("application/json")) {
+                    throw new IOException(EntityUtils.toString(response.getEntity(), "UTF-8"));
+                } else {
+                    throw new IOException(response.getStatusLine().getReasonPhrase());
+                }
+            }
 
             String etag = response.getLastHeader("ETag").getValue();
 

other/java/examples/pom.xml

@@ -11,13 +11,13 @@
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-client</artifactId>
-            <version>3.13</version>
+            <version>3.30</version>
             <scope>compile</scope>
         </dependency>
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-hadoop2-client</artifactId>
-            <version>3.13</version>
+            <version>3.30</version>
             <scope>compile</scope>
         </dependency>
         <dependency>

other/java/hdfs2/dependency-reduced-pom.xml

@@ -108,7 +108,7 @@
       </plugin>
       <plugin>
         <artifactId>maven-javadoc-plugin</artifactId>
-        <version>2.9.1</version>
+        <version>3.0.1</version>
         <executions>
           <execution>
             <id>attach-javadocs</id>
@@ -124,7 +124,7 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-client</artifactId>
-      <version>2.10.1</version>
+      <version>3.2.4</version>
       <scope>provided</scope>
       <exclusions>
         <exclusion>
@@ -132,11 +132,11 @@
           <groupId>org.apache.hadoop</groupId>
         </exclusion>
         <exclusion>
-          <artifactId>hadoop-mapreduce-client-app</artifactId>
+          <artifactId>hadoop-yarn-api</artifactId>
           <groupId>org.apache.hadoop</groupId>
         </exclusion>
         <exclusion>
-          <artifactId>hadoop-yarn-api</artifactId>
+          <artifactId>hadoop-yarn-client</artifactId>
           <groupId>org.apache.hadoop</groupId>
         </exclusion>
         <exclusion>
@@ -156,7 +156,7 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
-      <version>2.10.1</version>
+      <version>3.2.4</version>
       <scope>provided</scope>
       <exclusions>
         <exclusion>
@@ -167,10 +167,6 @@
           <artifactId>commons-math3</artifactId>
           <groupId>org.apache.commons</groupId>
         </exclusion>
-        <exclusion>
-          <artifactId>xmlenc</artifactId>
-          <groupId>xmlenc</groupId>
-        </exclusion>
         <exclusion>
           <artifactId>commons-io</artifactId>
           <groupId>commons-io</groupId>
@@ -184,20 +180,28 @@
           <groupId>commons-collections</groupId>
         </exclusion>
         <exclusion>
-          <artifactId>servlet-api</artifactId>
+          <artifactId>javax.servlet-api</artifactId>
           <groupId>javax.servlet</groupId>
         </exclusion>
         <exclusion>
-          <artifactId>jetty</artifactId>
-          <groupId>org.mortbay.jetty</groupId>
+          <artifactId>javax.activation-api</artifactId>
+          <groupId>javax.activation</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jetty-server</artifactId>
+          <groupId>org.eclipse.jetty</groupId>
         </exclusion>
         <exclusion>
           <artifactId>jetty-util</artifactId>
-          <groupId>org.mortbay.jetty</groupId>
+          <groupId>org.eclipse.jetty</groupId>
         </exclusion>
         <exclusion>
-          <artifactId>jetty-sslengine</artifactId>
-          <groupId>org.mortbay.jetty</groupId>
+          <artifactId>jetty-servlet</artifactId>
+          <groupId>org.eclipse.jetty</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jetty-webapp</artifactId>
+          <groupId>org.eclipse.jetty</groupId>
         </exclusion>
         <exclusion>
           <artifactId>jsp-api</artifactId>
@@ -207,6 +211,10 @@
           <artifactId>jersey-core</artifactId>
           <groupId>com.sun.jersey</groupId>
         </exclusion>
+        <exclusion>
+          <artifactId>jersey-servlet</artifactId>
+          <groupId>com.sun.jersey</groupId>
+        </exclusion>
         <exclusion>
           <artifactId>jersey-json</artifactId>
           <groupId>com.sun.jersey</groupId>
@@ -216,49 +224,37 @@
           <groupId>com.sun.jersey</groupId>
         </exclusion>
         <exclusion>
-          <artifactId>log4j</artifactId>
-          <groupId>log4j</groupId>
-        </exclusion>
-        <exclusion>
-          <artifactId>jets3t</artifactId>
-          <groupId>net.java.dev.jets3t</groupId>
-        </exclusion>
-        <exclusion>
-          <artifactId>commons-lang</artifactId>
-          <groupId>commons-lang</groupId>
-        </exclusion>
-        <exclusion>
-          <artifactId>commons-configuration</artifactId>
-          <groupId>commons-configuration</groupId>
-        </exclusion>
-        <exclusion>
-          <artifactId>commons-digester</artifactId>
-          <groupId>commons-digester</groupId>
+          <artifactId>reload4j</artifactId>
+          <groupId>ch.qos.reload4j</groupId>
         </exclusion>
         <exclusion>
           <artifactId>commons-beanutils</artifactId>
           <groupId>commons-beanutils</groupId>
         </exclusion>
+        <exclusion>
+          <artifactId>commons-configuration2</artifactId>
+          <groupId>org.apache.commons</groupId>
+        </exclusion>
         <exclusion>
           <artifactId>commons-lang3</artifactId>
           <groupId>org.apache.commons</groupId>
         </exclusion>
         <exclusion>
-          <artifactId>slf4j-log4j12</artifactId>
+          <artifactId>commons-text</artifactId>
+          <groupId>org.apache.commons</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>slf4j-reload4j</artifactId>
           <groupId>org.slf4j</groupId>
         </exclusion>
-        <exclusion>
-          <artifactId>jackson-core-asl</artifactId>
-          <groupId>org.codehaus.jackson</groupId>
-        </exclusion>
-        <exclusion>
-          <artifactId>jackson-mapper-asl</artifactId>
-          <groupId>org.codehaus.jackson</groupId>
-        </exclusion>
         <exclusion>
           <artifactId>avro</artifactId>
           <groupId>org.apache.avro</groupId>
         </exclusion>
+        <exclusion>
+          <artifactId>re2j</artifactId>
+          <groupId>com.google.re2j</groupId>
+        </exclusion>
         <exclusion>
           <artifactId>hadoop-auth</artifactId>
           <groupId>org.apache.hadoop</groupId>
@@ -287,6 +283,14 @@
           <artifactId>commons-compress</artifactId>
           <groupId>org.apache.commons</groupId>
         </exclusion>
+        <exclusion>
+          <artifactId>kerb-simplekdc</artifactId>
+          <groupId>org.apache.kerby</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jackson-databind</artifactId>
+          <groupId>com.fasterxml.jackson.core</groupId>
+        </exclusion>
         <exclusion>
           <artifactId>stax2-api</artifactId>
           <groupId>org.codehaus.woodstox</groupId>
@@ -295,6 +299,10 @@
           <artifactId>woodstox-core</artifactId>
           <groupId>com.fasterxml.woodstox</groupId>
         </exclusion>
+        <exclusion>
+          <artifactId>dnsjava</artifactId>
+          <groupId>dnsjava</groupId>
+        </exclusion>
         <exclusion>
           <artifactId>hadoop-annotations</artifactId>
           <groupId>org.apache.hadoop</groupId>
@@ -309,7 +317,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>3.13</seaweedfs.client.version>
-    <hadoop.version>2.10.1</hadoop.version>
+    <seaweedfs.client.version>3.30</seaweedfs.client.version>
+    <hadoop.version>3.2.4</hadoop.version>
   </properties>
 </project>

other/java/hdfs2/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>3.13</seaweedfs.client.version>
+        <seaweedfs.client.version>3.30</seaweedfs.client.version>
         <hadoop.version>3.2.4</hadoop.version>
     </properties>
 
@@ -129,7 +129,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-javadoc-plugin</artifactId>
-                <version>2.9.1</version>
+                <version>3.0.1</version>
                 <executions>
                     <execution>
                         <id>attach-javadocs</id>

other/java/hdfs3/dependency-reduced-pom.xml

@@ -108,7 +108,7 @@
       </plugin>
       <plugin>
         <artifactId>maven-javadoc-plugin</artifactId>
-        <version>2.9.1</version>
+        <version>3.0.1</version>
         <executions>
           <execution>
             <id>attach-javadocs</id>
@@ -124,7 +124,7 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-client</artifactId>
-      <version>3.2.3</version>
+      <version>3.2.4</version>
       <scope>provided</scope>
       <exclusions>
         <exclusion>
@@ -156,7 +156,7 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
-      <version>3.2.3</version>
+      <version>3.2.4</version>
       <scope>provided</scope>
       <exclusions>
         <exclusion>
@@ -224,8 +224,8 @@
           <groupId>com.sun.jersey</groupId>
         </exclusion>
         <exclusion>
-          <artifactId>log4j</artifactId>
-          <groupId>log4j</groupId>
+          <artifactId>reload4j</artifactId>
+          <groupId>ch.qos.reload4j</groupId>
         </exclusion>
         <exclusion>
           <artifactId>commons-beanutils</artifactId>
@@ -244,7 +244,7 @@
           <groupId>org.apache.commons</groupId>
         </exclusion>
         <exclusion>
-          <artifactId>slf4j-log4j12</artifactId>
+          <artifactId>slf4j-reload4j</artifactId>
           <groupId>org.slf4j</groupId>
         </exclusion>
         <exclusion>
@@ -317,7 +317,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>3.13</seaweedfs.client.version>
-    <hadoop.version>3.2.3</hadoop.version>
+    <seaweedfs.client.version>3.30</seaweedfs.client.version>
+    <hadoop.version>3.2.4</hadoop.version>
   </properties>
 </project>

other/java/hdfs3/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>3.13</seaweedfs.client.version>
+        <seaweedfs.client.version>3.30</seaweedfs.client.version>
         <hadoop.version>3.2.4</hadoop.version>
     </properties>
 
@@ -129,7 +129,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-javadoc-plugin</artifactId>
-                <version>2.9.1</version>
+                <version>3.0.1</version>
                 <executions>
                     <execution>
                         <id>attach-javadocs</id>

weed/command/filer.go

@@ -298,7 +298,7 @@ func (fo *FilerOptions) startFiler() {
 			localSocket = fmt.Sprintf("/tmp/seaweedfs-filer-%d.sock", *fo.port)
 		}
 		if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) {
-			glog.Fatalf("Failed to remove %s, error: %v", localSocket, err)
+			glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error())
 		}
 		go func() {
 			// start on local unix socket

weed/command/filer_remote_sync_dir.go

@@ -37,11 +37,16 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
 
 	var lastLogTsNs = time.Now().UnixNano()
 	processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error {
-		if *option.storageClass == "" {
-			delete(resp.EventNotification.NewEntry.Extended, s3_constants.AmzStorageClass)
-		} else {
-			resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass] = []byte(*option.storageClass)
+		if resp.EventNotification.NewEntry != nil {
+			if *option.storageClass == "" {
+				if _, ok := resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass]; ok {
+					delete(resp.EventNotification.NewEntry.Extended, s3_constants.AmzStorageClass)
+				}
+			} else {
+				resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass] = []byte(*option.storageClass)
+			}
 		}
+
 		processor.AddSyncJob(resp)
 		return nil
 	}, 3*time.Second, func(counter int64, lastTsNs int64) error {

weed/command/fuse.go

@@ -47,7 +47,10 @@ func runFuse(cmd *Command, args []string) bool {
 			for i++; i < rawArgsLen && rawArgs[i] != ' '; i++ {
 				option.WriteByte(rawArgs[i])
 			}
-			options = append(options, parameter{option.String(), "true"})
+			// ignore "-o"
+			if option.String() != "o" {
+				options = append(options, parameter{option.String(), "true"})
+			}
 			option.Reset()
 
 			// equal separator start option with pending value
@@ -190,6 +193,12 @@ func runFuse(cmd *Command, args []string) bool {
 			}
 		case "fusermount.path":
 			fusermountPath = parameter.value
+		default:
+			t := parameter.name
+			if parameter.value != "true" {
+				t = fmt.Sprintf("%s=%s", parameter.name, parameter.value)
+			}
+			mountOptions.extraOptions = append(mountOptions.extraOptions, t)
 		}
 	}
 

weed/command/master.go

@@ -2,6 +2,7 @@ package command
 
 import (
 	"fmt"
+	hashicorpRaft "github.com/hashicorp/raft"
 	"net/http"
 	"os"
 	"path"
@@ -253,6 +254,13 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 		go httpS.Serve(masterListener)
 	}
 
+	grace.OnInterrupt(ms.Shutdown)
+	grace.OnInterrupt(grpcS.GracefulStop)
+	grace.OnReload(func() {
+		if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.State() == hashicorpRaft.Leader {
+			ms.Topo.HashicorpRaft.LeadershipTransfer()
+		}
+	})
 	select {}
 }
 

weed/command/mount.go

@@ -31,6 +31,7 @@ type MountOptions struct {
 	debugPort          *int
 	localSocket        *string
 	disableXAttr       *bool
+	extraOptions       []string
 }
 
 var (

weed/command/mount_std.go

@@ -166,7 +166,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 	// mount fuse
 	fuseMountOptions := &fuse.MountOptions{
 		AllowOther:               *option.allowOthers,
-		Options:                  nil,
+		Options:                  option.extraOptions,
 		MaxBackground:            128,
 		MaxWrite:                 1024 * 1024 * 2,
 		MaxReadAhead:             1024 * 1024 * 2,

weed/command/server.go

@@ -92,10 +92,11 @@ func init() {
 	masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
 	masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
 	masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "", "Default replication type if not specified.")
-	masterOptions.garbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
-	masterOptions.metricsAddress = cmdServer.Flag.String("metrics.address", "", "Prometheus gateway address")
-	masterOptions.metricsIntervalSec = cmdServer.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
-	masterOptions.raftResumeState = cmdServer.Flag.Bool("resumeState", false, "resume previous state on start master server")
+	masterOptions.garbageThreshold = cmdServer.Flag.Float64("master.garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
+	masterOptions.metricsAddress = cmdServer.Flag.String("master.metrics.address", "", "Prometheus gateway address")
+	masterOptions.metricsIntervalSec = cmdServer.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
+	masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", false, "resume previous state on start master server")
+	masterOptions.raftHashicorp = cmdServer.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")
 	masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
 	masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")
 

weed/command/volume.go

@@ -265,6 +265,8 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 	// starting the cluster http server
 	clusterHttpServer := v.startClusterHttpService(volumeMux)
 
+	grace.OnReload(volumeServer.LoadNewVolumes)
+
 	stopChan := make(chan bool)
 	grace.OnInterrupt(func() {
 		fmt.Println("volume server has been killed")

weed/filer/s3iam_conf.go

@@ -10,7 +10,11 @@ import (
 )
 
 func ParseS3ConfigurationFromBytes[T proto.Message](content []byte, config T) error {
-	if err := jsonpb.Unmarshal(content, config); err != nil {
+	options := &jsonpb.UnmarshalOptions{
+		DiscardUnknown: true,
+		AllowPartial:   true,
+	}
+	if err := options.Unmarshal(content, config); err != nil {
 		return err
 	}
 	return nil

weed/mount/weedfs.go

@@ -190,7 +190,7 @@ func (wfs *WFS) getCurrentFiler() pb.ServerAddress {
 }
 
 func (option *Option) setupUniqueCacheDirectory() {
-	cacheUniqueId := util.Md5String([]byte(option.MountDirectory + string(option.FilerAddresses[0]) + option.FilerMountRootPath + util.Version()))[0:8]
+	cacheUniqueId := util.Md5String([]byte(string(option.FilerAddresses[0]) + option.FilerMountRootPath + util.Version()))[0:8]
 	option.uniqueCacheDir = path.Join(option.CacheDir, cacheUniqueId)
 	option.uniqueCacheTempPageDir = filepath.Join(option.uniqueCacheDir, "swap")
 	os.MkdirAll(option.uniqueCacheTempPageDir, os.FileMode(0777)&^option.Umask)

weed/mq/broker/brokder_grpc_pub.go

@@ -1,7 +1,6 @@
 package broker
 
 import (
-	"github.com/seaweedfs/seaweedfs/weed/pb/message_fbs"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 )
 
@@ -12,36 +11,6 @@ The messages is buffered in memory, and saved to filer under
 	/topics/<topic>/info/segment_<id>.meta
 */
 
-func (broker *MessageQueueBroker) PublishMessage(stream mq_pb.SeaweedMessaging_PublishMessageServer) error {
-	println("connected")
-	for {
-		request, recvErr := stream.Recv()
-		if recvErr != nil {
-			return recvErr
-		}
-
-		print(">")
-		if request.Control != nil {
-
-		}
-		if request.Data != nil {
-			if err := broker.processDataMessage(stream, request.Data); err != nil {
-				return err
-			}
-		}
-
-	}
-	return nil
-}
-
-func (broker *MessageQueueBroker) processDataMessage(stream mq_pb.SeaweedMessaging_PublishMessageServer, data *mq_pb.PublishRequest_DataMessage) error {
-	mb := message_fbs.GetRootAsMessageBatch(data.Message, 0)
-
-	println("message count:", mb.MessagesLength(), len(data.Message))
-	m := &message_fbs.Message{}
-	for i := 0; i < mb.MessagesLength(); i++ {
-		mb.Messages(m, i)
-		println(i, ">", string(m.Data()))
-	}
+func (broker *MessageQueueBroker) Publish(stream mq_pb.SeaweedMessaging_PublishServer) error {
 	return nil
 }

weed/mq/client/publish_stream_processor.go

@@ -1,179 +0,0 @@
-package client
-
-import (
-	"context"
-	flatbuffers "github.com/google/flatbuffers/go"
-	"github.com/seaweedfs/seaweedfs/weed/mq/messages"
-	"github.com/seaweedfs/seaweedfs/weed/mq/segment"
-	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
-	"github.com/seaweedfs/seaweedfs/weed/util"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/insecure"
-	"log"
-	"sync"
-	"sync/atomic"
-	"time"
-)
-
-const (
-	batchCountLimit = 3
-)
-
-type PublishStreamProcessor struct {
-	// attributes
-	ProducerId     int32
-	ProducerEpoch  int32
-	grpcDialOption grpc.DialOption
-
-	// input
-	sync.Mutex
-
-	timeout time.Duration
-
-	// convert into bytes
-	messagesChan           chan *messages.Message
-	builders               chan *flatbuffers.Builder
-	batchMessageCountLimit int
-
-	messagesSequence int64
-
-	// done channel
-	doneChan chan struct{}
-}
-
-type UploadProcess struct {
-	bufferBuilder *flatbuffers.Builder
-	batchBuilder  *segment.MessageBatchBuilder
-}
-
-func NewPublishStreamProcessor(batchMessageCountLimit int, timeout time.Duration) *PublishStreamProcessor {
-	t := &PublishStreamProcessor{
-		grpcDialOption:         grpc.WithTransportCredentials(insecure.NewCredentials()),
-		batchMessageCountLimit: batchMessageCountLimit,
-		builders:               make(chan *flatbuffers.Builder, batchCountLimit),
-		messagesChan:           make(chan *messages.Message, 1024),
-		doneChan:               make(chan struct{}),
-		timeout:                timeout,
-	}
-	for i := 0; i < batchCountLimit; i++ {
-		t.builders <- flatbuffers.NewBuilder(4 * 1024 * 1024)
-	}
-	go t.doLoopUpload()
-	return t
-}
-
-func (p *PublishStreamProcessor) AddMessage(m *messages.Message) error {
-	p.messagesChan <- m
-	return nil
-}
-
-func (p *PublishStreamProcessor) Shutdown() error {
-	p.doneChan <- struct{}{}
-	return nil
-}
-
-func (p *PublishStreamProcessor) doFlush(stream mq_pb.SeaweedMessaging_PublishMessageClient, messages []*messages.Message) error {
-
-	if len(messages) == 0 {
-		return nil
-	}
-
-	builder := <-p.builders
-	bb := segment.NewMessageBatchBuilder(builder)
-	for _, m := range messages {
-		bb.AddMessage(p.messagesSequence, m.Ts.UnixNano(), m.Properties, m.Key, m.Content)
-		p.messagesSequence++
-	}
-	bb.BuildMessageBatch(p.ProducerId, p.ProducerEpoch, 3, 4)
-	defer func() {
-		p.builders <- builder
-	}()
-
-	return stream.Send(&mq_pb.PublishRequest{
-		Data: &mq_pb.PublishRequest_DataMessage{
-			Message: bb.GetBytes(),
-		},
-	})
-
-}
-
-func (p *PublishStreamProcessor) doLoopUpload() {
-
-	brokerGrpcAddress := "localhost:17777"
-
-	// TOOD parallelize the uploading with separate uploader
-	messages := make([]*messages.Message, 0, p.batchMessageCountLimit)
-
-	util.RetryForever("publish message", func() error {
-		return pb.WithBrokerGrpcClient(false, brokerGrpcAddress, p.grpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
-
-			ctx, cancel := context.WithCancel(context.Background())
-			defer cancel()
-
-			stream, err := client.PublishMessage(ctx)
-			if err != nil {
-				log.Printf("grpc PublishMessage: %v", err)
-				return err
-			}
-
-			var atomicStatus int64
-			go func() {
-				resp, err := stream.Recv()
-				if err != nil {
-					log.Printf("response error: %v", err)
-				} else {
-					log.Printf("response: %v", resp.AckSequence)
-				}
-				if atomic.LoadInt64(&atomicStatus) < 0 {
-					return
-				}
-			}()
-
-			var flushErr error
-			// retry previously failed messages
-			if len(messages) >= p.batchMessageCountLimit {
-				flushErr = p.doFlush(stream, messages)
-				if flushErr != nil {
-					return flushErr
-				}
-				messages = messages[:0]
-			}
-
-			for {
-				select {
-				case m := <-p.messagesChan:
-					messages = append(messages, m)
-					if len(messages) >= p.batchMessageCountLimit {
-						if flushErr = p.doFlush(stream, messages); flushErr != nil {
-							return flushErr
-						}
-						messages = messages[:0]
-					}
-				case <-time.After(p.timeout):
-					if flushErr = p.doFlush(stream, messages); flushErr != nil {
-						return flushErr
-					}
-					messages = messages[:0]
-				case <-p.doneChan:
-					if flushErr = p.doFlush(stream, messages); flushErr != nil {
-						return flushErr
-					}
-					messages = messages[:0]
-					println("$ stopping ...")
-					break
-				}
-			}
-
-			// stop the response consuming goroutine
-			atomic.StoreInt64(&atomicStatus, -1)
-
-			return flushErr
-
-		})
-	}, func(err error) (shouldContinue bool) {
-		log.Printf("failed with grpc %s: %v", brokerGrpcAddress, err)
-		return true
-	})
-
-}

weed/mq/client/publisher.go

@@ -1,40 +0,0 @@
-package client
-
-import (
-	"github.com/seaweedfs/seaweedfs/weed/mq/messages"
-	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"time"
-)
-
-type PublishProcessor interface {
-	AddMessage(m *messages.Message) error
-	Shutdown() error
-}
-
-type PublisherOption struct {
-	Masters string
-	Topic   string
-}
-
-type Publisher struct {
-	option    *PublisherOption
-	masters   []pb.ServerAddress
-	processor *PublishStreamProcessor
-}
-
-func NewPublisher(option *PublisherOption) *Publisher {
-	p := &Publisher{
-		masters:   pb.ServerAddresses(option.Masters).ToAddresses(),
-		option:    option,
-		processor: NewPublishStreamProcessor(3, 887*time.Millisecond),
-	}
-	return p
-}
-
-func (p Publisher) Publish(m *messages.Message) error {
-	return p.processor.AddMessage(m)
-}
-
-func (p Publisher) Shutdown() error {
-	return p.processor.Shutdown()
-}

weed/mq/cmd/qsend/qsend.go

@@ -1,62 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"flag"
-	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/mq/client"
-	"github.com/seaweedfs/seaweedfs/weed/mq/messages"
-	"os"
-	"time"
-)
-
-var (
-	master = flag.String("master", "localhost:9333", "master csv list")
-	topic  = flag.String("topic", "", "topic name")
-)
-
-func main() {
-	flag.Parse()
-
-	publisher := client.NewPublisher(&client.PublisherOption{
-		Masters: *master,
-		Topic:   *topic,
-	})
-
-	err := eachLineStdin(func(line string) error {
-		if len(line) > 0 {
-			if err := publisher.Publish(&messages.Message{
-				Key:        nil,
-				Content:    []byte(line),
-				Properties: nil,
-				Ts:         time.Time{},
-			}); err != nil {
-				return err
-			}
-		}
-		return nil
-	})
-
-	publisher.Shutdown()
-
-	if err != nil {
-		fmt.Printf("error: %v\n", err)
-	}
-}
-
-func eachLineStdin(eachLineFn func(string) error) error {
-	scanner := bufio.NewScanner(os.Stdin)
-	for scanner.Scan() {
-		text := scanner.Text()
-		if err := eachLineFn(text); err != nil {
-			return err
-		}
-	}
-
-	// handle error
-	if scanner.Err() != nil {
-		return fmt.Errorf("scan stdin: %v", scanner.Err())
-	}
-
-	return nil
-}

weed/mq/messages/message_buffer.go

@@ -1,53 +0,0 @@
-package messages
-
-import (
-	flatbuffers "github.com/google/flatbuffers/go"
-	"github.com/seaweedfs/seaweedfs/weed/mq/segment"
-)
-
-type MessageBuffer struct {
-	fbsBuffer    *flatbuffers.Builder
-	sequenceBase int64
-	counter      int64
-	bb           *segment.MessageBatchBuilder
-	isSealed     bool
-}
-
-func NewMessageBuffer() *MessageBuffer {
-	t := &MessageBuffer{
-		fbsBuffer: flatbuffers.NewBuilder(4 * 1024 * 1024),
-	}
-	t.bb = segment.NewMessageBatchBuilder(t.fbsBuffer)
-	return t
-}
-
-func (mb *MessageBuffer) Reset(sequenceBase int64) {
-	mb.sequenceBase = sequenceBase
-	mb.counter = 0
-	mb.bb.Reset()
-}
-
-func (mb *MessageBuffer) AddMessage(message *Message) {
-	mb.bb.AddMessage(mb.sequenceBase, message.Ts.UnixMilli(), message.Properties, message.Key, message.Content)
-	mb.sequenceBase++
-	mb.counter++
-}
-
-func (mb *MessageBuffer) Len() int {
-	return int(mb.counter)
-}
-
-func (mb *MessageBuffer) Seal(producerId int32,
-	producerEpoch int32,
-	segmentId int32,
-	flags int32) {
-	mb.isSealed = true
-	mb.bb.BuildMessageBatch(producerId, producerEpoch, segmentId, flags)
-}
-
-func (mb *MessageBuffer) Bytes() []byte {
-	if !mb.isSealed {
-		return nil
-	}
-	return mb.bb.GetBytes()
-}

weed/mq/messages/message_buffer_mover.go

@@ -1,32 +0,0 @@
-package messages
-
-import "fmt"
-
-type MessageBufferMover interface {
-	Setup()
-	TearDown()
-	MoveBuffer(buffer *MessageBuffer) (MessageBufferReference, error) // should be thread-safe
-}
-type MessageBufferReference struct {
-	sequence int64
-	fileId   string
-}
-
-var _ = MessageBufferMover(&EmptyMover{})
-
-type EmptyMover struct {
-}
-
-func (e *EmptyMover) Setup() {
-}
-
-func (e *EmptyMover) TearDown() {
-}
-
-func (e *EmptyMover) MoveBuffer(buffer *MessageBuffer) (MessageBufferReference, error) {
-	println("moving", buffer.sequenceBase)
-	return MessageBufferReference{
-		sequence: buffer.sequenceBase,
-		fileId:   fmt.Sprintf("buffer %d", buffer.sequenceBase),
-	}, nil
-}

weed/mq/messages/message_pipeline.go

@@ -1,163 +0,0 @@
-package messages
-
-import (
-	"github.com/seaweedfs/seaweedfs/weed/util"
-	"google.golang.org/grpc"
-	"log"
-	"sync"
-	"sync/atomic"
-	"time"
-)
-
-type OnMessageFunc func(message *Message)
-
-type MessagePipeline struct {
-	// atomic status
-	atomicPipelineStatus int64 // -1: stop
-
-	// attributes
-	ProducerId     int32
-	ProducerEpoch  int32
-	grpcDialOption grpc.DialOption
-
-	emptyBuffersChan  chan *MessageBuffer
-	sealedBuffersChan chan *MessageBuffer
-	movedBuffersChan  chan MessageBufferReference
-	onMessageFn       OnMessageFunc
-	mover             MessageBufferMover
-	moverPool         *util.LimitedAsyncExecutor
-
-	// control pipeline
-	doneChan  chan struct{}
-	batchSize int
-	timeout   time.Duration
-
-	incomingMessageLock   sync.Mutex
-	incomingMessageBuffer *MessageBuffer
-
-	messageSequence int64
-}
-
-func NewMessagePipeline(producerId int32, workerCount int, batchSize int, timeout time.Duration, mover MessageBufferMover) *MessagePipeline {
-	t := &MessagePipeline{
-		ProducerId:        producerId,
-		emptyBuffersChan:  make(chan *MessageBuffer, workerCount),
-		sealedBuffersChan: make(chan *MessageBuffer, workerCount),
-		movedBuffersChan:  make(chan MessageBufferReference, workerCount),
-		doneChan:          make(chan struct{}),
-		batchSize:         batchSize,
-		timeout:           timeout,
-		moverPool:         util.NewLimitedAsyncExecutor(workerCount),
-		mover:             mover,
-	}
-	go t.doLoopUpload()
-	return t
-}
-
-func (mp *MessagePipeline) NextMessageBufferReference() MessageBufferReference {
-	return mp.moverPool.NextFuture().Await().(MessageBufferReference)
-}
-
-func (mp *MessagePipeline) AddMessage(message *Message) {
-	mp.incomingMessageLock.Lock()
-	defer mp.incomingMessageLock.Unlock()
-
-	// get existing message buffer or create a new one
-	if mp.incomingMessageBuffer == nil {
-		select {
-		case mp.incomingMessageBuffer = <-mp.emptyBuffersChan:
-		default:
-			mp.incomingMessageBuffer = NewMessageBuffer()
-		}
-		mp.incomingMessageBuffer.Reset(mp.messageSequence)
-	}
-
-	// add one message
-	mp.incomingMessageBuffer.AddMessage(message)
-	mp.messageSequence++
-
-	// seal the message buffer if full
-	if mp.incomingMessageBuffer.Len() >= mp.batchSize {
-		mp.incomingMessageBuffer.Seal(mp.ProducerId, mp.ProducerEpoch, 0, 0)
-		mp.sealedBuffersChan <- mp.incomingMessageBuffer
-		mp.incomingMessageBuffer = nil
-	}
-}
-
-func (mp *MessagePipeline) doLoopUpload() {
-
-	mp.mover.Setup()
-	defer mp.mover.TearDown()
-
-	ticker := time.NewTicker(mp.timeout)
-	for {
-		status := atomic.LoadInt64(&mp.atomicPipelineStatus)
-		if status == -100 {
-			return
-		} else if status == -1 {
-			// entering shutting down mode
-			atomic.StoreInt64(&mp.atomicPipelineStatus, -2)
-			mp.incomingMessageLock.Lock()
-			mp.doFlushIncomingMessages()
-			mp.incomingMessageLock.Unlock()
-		}
-
-		select {
-		case messageBuffer := <-mp.sealedBuffersChan:
-			ticker.Reset(mp.timeout)
-			mp.moverPool.Execute(func() any {
-				var output MessageBufferReference
-				util.RetryForever("message mover", func() error {
-					if messageReference, flushErr := mp.mover.MoveBuffer(messageBuffer); flushErr != nil {
-						return flushErr
-					} else {
-						output = messageReference
-					}
-					return nil
-				}, func(err error) (shouldContinue bool) {
-					log.Printf("failed: %v", err)
-					return true
-				})
-				return output
-			})
-		case <-ticker.C:
-			if atomic.LoadInt64(&mp.atomicPipelineStatus) == -2 {
-				atomic.StoreInt64(&mp.atomicPipelineStatus, -100)
-				return
-			}
-			mp.incomingMessageLock.Lock()
-			mp.doFlushIncomingMessages()
-			mp.incomingMessageLock.Unlock()
-		}
-	}
-
-	atomic.StoreInt64(&mp.atomicPipelineStatus, -100)
-	close(mp.movedBuffersChan)
-
-}
-
-func (mp *MessagePipeline) doFlushIncomingMessages() {
-	if mp.incomingMessageBuffer == nil || mp.incomingMessageBuffer.Len() == 0 {
-		return
-	}
-	mp.incomingMessageBuffer.Seal(mp.ProducerId, mp.ProducerEpoch, 0, 0)
-	mp.sealedBuffersChan <- mp.incomingMessageBuffer
-	mp.incomingMessageBuffer = nil
-}
-
-func (mp *MessagePipeline) ShutdownStart() {
-	if atomic.LoadInt64(&mp.atomicPipelineStatus) == 0 {
-		atomic.StoreInt64(&mp.atomicPipelineStatus, -1)
-	}
-}
-func (mp *MessagePipeline) ShutdownWait() {
-	for atomic.LoadInt64(&mp.atomicPipelineStatus) != -100 {
-		time.Sleep(331 * time.Millisecond)
-	}
-}
-
-func (mp *MessagePipeline) ShutdownImmediate() {
-	if atomic.LoadInt64(&mp.atomicPipelineStatus) == 0 {
-		atomic.StoreInt64(&mp.atomicPipelineStatus, -100)
-	}
-}

weed/mq/messages/message_pipeline_test.go

@@ -1,53 +0,0 @@
-package messages
-
-import (
-	"testing"
-	"time"
-)
-
-func TestAddMessage1(t *testing.T) {
-	mp := NewMessagePipeline(0, 3, 10, time.Second, &EmptyMover{})
-	go func() {
-		for i := 0; i < 100; i++ {
-			mr := mp.NextMessageBufferReference()
-			println(mr.sequence, mr.fileId)
-		}
-	}()
-
-	for i := 0; i < 100; i++ {
-		message := &Message{
-			Key:        []byte("key"),
-			Content:    []byte("data"),
-			Properties: nil,
-			Ts:         time.Now(),
-		}
-		mp.AddMessage(message)
-	}
-
-	mp.ShutdownStart()
-	mp.ShutdownWait()
-
-}
-
-func TestAddMessage2(t *testing.T) {
-	mp := NewMessagePipeline(0, 3, 10, time.Second, &EmptyMover{})
-
-	for i := 0; i < 100; i++ {
-		message := &Message{
-			Key:        []byte("key"),
-			Content:    []byte("data"),
-			Properties: nil,
-			Ts:         time.Now(),
-		}
-		mp.AddMessage(message)
-	}
-
-	mp.ShutdownStart()
-	mp.ShutdownWait()
-
-	for i := 0; i < 100; i++ {
-		mr := mp.NextMessageBufferReference()
-		println(mr.sequence, mr.fileId)
-	}
-
-}

weed/mq/messages/messages.go

@@ -1,10 +0,0 @@
-package messages
-
-import "time"
-
-type Message struct {
-	Key        []byte
-	Content    []byte
-	Properties map[string]string
-	Ts         time.Time
-}

weed/mq/segment/message_serde.go

@@ -7,6 +7,10 @@ import (
 
 type MessageBatchBuilder struct {
 	b              *flatbuffers.Builder
+	producerId     int32
+	producerEpoch  int32
+	segmentId      int32
+	flags          int32
 	messageOffsets []flatbuffers.UOffsetT
 	segmentSeqBase int64
 	segmentSeqLast int64
@@ -14,20 +18,24 @@ type MessageBatchBuilder struct {
 	tsMsLast       int64
 }
 
-func NewMessageBatchBuilder(b *flatbuffers.Builder) *MessageBatchBuilder {
+func NewMessageBatchBuilder(b *flatbuffers.Builder,
+	producerId int32,
+	producerEpoch int32,
+	segmentId int32,
+	flags int32) *MessageBatchBuilder {
 
 	b.Reset()
 
 	return &MessageBatchBuilder{
-		b: b,
+		b:             b,
+		producerId:    producerId,
+		producerEpoch: producerEpoch,
+		segmentId:     segmentId,
+		flags:         flags,
 	}
 }
 
-func (builder *MessageBatchBuilder) Reset() {
-	builder.b.Reset()
-}
-
-func (builder *MessageBatchBuilder) AddMessage(segmentSeq int64, tsMs int64, properties map[string]string, key []byte, value []byte) {
+func (builder *MessageBatchBuilder) AddMessage(segmentSeq int64, tsMs int64, properties map[string][]byte, key []byte, value []byte) {
 	if builder.segmentSeqBase == 0 {
 		builder.segmentSeqBase = segmentSeq
 	}
@@ -40,7 +48,7 @@ func (builder *MessageBatchBuilder) AddMessage(segmentSeq int64, tsMs int64, pro
 	var names, values, pairs []flatbuffers.UOffsetT
 	for k, v := range properties {
 		names = append(names, builder.b.CreateString(k))
-		values = append(values, builder.b.CreateString(v))
+		values = append(values, builder.b.CreateByteVector(v))
 	}
 	for i, _ := range names {
 		message_fbs.NameValueStart(builder.b)
@@ -72,10 +80,7 @@ func (builder *MessageBatchBuilder) AddMessage(segmentSeq int64, tsMs int64, pro
 
 }
 
-func (builder *MessageBatchBuilder) BuildMessageBatch(producerId int32,
-	producerEpoch int32,
-	segmentId int32,
-	flags int32) {
+func (builder *MessageBatchBuilder) BuildMessageBatch() {
 	message_fbs.MessageBatchStartMessagesVector(builder.b, len(builder.messageOffsets))
 	for i := len(builder.messageOffsets) - 1; i >= 0; i-- {
 		builder.b.PrependUOffsetT(builder.messageOffsets[i])
@@ -83,10 +88,10 @@ func (builder *MessageBatchBuilder) BuildMessageBatch(producerId int32,
 	messagesOffset := builder.b.EndVector(len(builder.messageOffsets))
 
 	message_fbs.MessageBatchStart(builder.b)
-	message_fbs.MessageBatchAddProducerId(builder.b, producerId)
-	message_fbs.MessageBatchAddProducerEpoch(builder.b, producerEpoch)
-	message_fbs.MessageBatchAddSegmentId(builder.b, segmentId)
-	message_fbs.MessageBatchAddFlags(builder.b, flags)
+	message_fbs.MessageBatchAddProducerId(builder.b, builder.producerId)
+	message_fbs.MessageBatchAddProducerEpoch(builder.b, builder.producerEpoch)
+	message_fbs.MessageBatchAddSegmentId(builder.b, builder.segmentId)
+	message_fbs.MessageBatchAddFlags(builder.b, builder.flags)
 	message_fbs.MessageBatchAddSegmentSeqBase(builder.b, builder.segmentSeqBase)
 	message_fbs.MessageBatchAddSegmentSeqMaxDelta(builder.b, int32(builder.segmentSeqLast-builder.segmentSeqBase))
 	message_fbs.MessageBatchAddTsMsBase(builder.b, builder.tsMsBase)

weed/mq/segment/message_serde_test.go

@@ -10,15 +10,15 @@ import (
 func TestMessageSerde(t *testing.T) {
 	b := flatbuffers.NewBuilder(1024)
 
-	prop := make(map[string]string)
-	prop["n1"] = "v1"
-	prop["n2"] = "v2"
+	prop := make(map[string][]byte)
+	prop["n1"] = []byte("v1")
+	prop["n2"] = []byte("v2")
 
-	bb := NewMessageBatchBuilder(b)
+	bb := NewMessageBatchBuilder(b, 1, 2, 3, 4)
 
 	bb.AddMessage(5, 6, prop, []byte("the primary key"), []byte("body is here"))
 
-	bb.BuildMessageBatch(1, 2, 3, 4)
+	bb.BuildMessageBatch()
 
 	buf := bb.GetBytes()
 

weed/pb/iam.proto

@@ -22,6 +22,7 @@ message Identity {
     string name = 1;
     repeated Credential credentials = 2;
     repeated string actions = 3;
+    string accountId = 4;
 }
 
 message Credential {

weed/pb/iam_pb/iam.pb.go

@@ -75,6 +75,7 @@ type Identity struct {
 	Name        string        `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
 	Credentials []*Credential `protobuf:"bytes,2,rep,name=credentials,proto3" json:"credentials,omitempty"`
 	Actions     []string      `protobuf:"bytes,3,rep,name=actions,proto3" json:"actions,omitempty"`
+	AccountId   string        `protobuf:"bytes,4,opt,name=accountId,proto3" json:"accountId,omitempty"`
 }
 
 func (x *Identity) Reset() {
@@ -130,6 +131,13 @@ func (x *Identity) GetActions() []string {
 	return nil
 }
 
+func (x *Identity) GetAccountId() string {
+	if x != nil {
+		return x.AccountId
+	}
+	return ""
+}
+
 type Credential struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -193,26 +201,28 @@ var file_iam_proto_rawDesc = []byte{
 	0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x0a, 0x69, 0x64, 0x65,
 	0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e,
 	0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52,
-	0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0x6e, 0x0a, 0x08, 0x49,
-	0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18,
-	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x0b, 0x63,
-	0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b,
-	0x32, 0x12, 0x2e, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e,
-	0x74, 0x69, 0x61, 0x6c, 0x52, 0x0b, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
-	0x73, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03,
-	0x28, 0x09, 0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x4a, 0x0a, 0x0a, 0x43,
-	0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63,
-	0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61,
-	0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x63, 0x72,
-	0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65,
-	0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x32, 0x21, 0x0a, 0x1f, 0x53, 0x65, 0x61, 0x77, 0x65,
-	0x65, 0x64, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73,
-	0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x4b, 0x0a, 0x10, 0x73, 0x65,
-	0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x08,
-	0x49, 0x61, 0x6d, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
-	0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73,
-	0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62,
-	0x2f, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0x8c, 0x01, 0x0a, 0x08,
+	0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x0b,
+	0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28,
+	0x0b, 0x32, 0x12, 0x2e, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x64, 0x65,
+	0x6e, 0x74, 0x69, 0x61, 0x6c, 0x52, 0x0b, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
+	0x6c, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20,
+	0x03, 0x28, 0x09, 0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09,
+	0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x22, 0x4a, 0x0a, 0x0a, 0x43, 0x72,
+	0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65,
+	0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63,
+	0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x63, 0x72, 0x65,
+	0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x63,
+	0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x32, 0x21, 0x0a, 0x1f, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65,
+	0x64, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4d,
+	0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x4b, 0x0a, 0x10, 0x73, 0x65, 0x61,
+	0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x08, 0x49,
+	0x61, 0x6d, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
+	0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65,
+	0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f,
+	0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

weed/pb/master.proto

@@ -400,7 +400,8 @@ message RaftListClusterServersResponse {
   message ClusterServers {
     string id = 1;
     string address = 2;
-    string suffrage = 3; //
+    string suffrage = 3;
+    bool   isLeader = 4;
   }
   repeated ClusterServers cluster_servers = 1;
 }

weed/pb/master_pb/master.pb.go

@@ -3817,7 +3817,8 @@ type RaftListClusterServersResponse_ClusterServers struct {
 
 	Id       string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
 	Address  string `protobuf:"bytes,2,opt,name=address,proto3" json:"address,omitempty"`
-	Suffrage string `protobuf:"bytes,3,opt,name=suffrage,proto3" json:"suffrage,omitempty"` //
+	Suffrage string `protobuf:"bytes,3,opt,name=suffrage,proto3" json:"suffrage,omitempty"`
+	IsLeader bool   `protobuf:"varint,4,opt,name=isLeader,proto3" json:"isLeader,omitempty"`
 }
 
 func (x *RaftListClusterServersResponse_ClusterServers) Reset() {
@@ -3873,6 +3874,13 @@ func (x *RaftListClusterServersResponse_ClusterServers) GetSuffrage() string {
 	return ""
 }
 
+func (x *RaftListClusterServersResponse_ClusterServers) GetIsLeader() bool {
+	if x != nil {
+		return x.IsLeader
+	}
+	return false
+}
+
 var File_master_proto protoreflect.FileDescriptor
 
 var file_master_proto_rawDesc = []byte{
@@ -4443,7 +4451,7 @@ var file_master_proto_rawDesc = []byte{
 	0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65,
 	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1f, 0x0a, 0x1d, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69,
 	0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xdb, 0x01, 0x0a, 0x1e, 0x52, 0x61, 0x66, 0x74,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xf7, 0x01, 0x0a, 0x1e, 0x52, 0x61, 0x66, 0x74,
 	0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65,
 	0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x61, 0x0a, 0x0f, 0x63, 0x6c,
 	0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20,
@@ -4451,122 +4459,124 @@ var file_master_proto_rawDesc = []byte{
 	0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53,
 	0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x43,
 	0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x0e, 0x63,
-	0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x1a, 0x56, 0x0a,
+	0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x1a, 0x72, 0x0a,
 	0x0e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12,
 	0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12,
 	0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
 	0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x75, 0x66,
 	0x66, 0x72, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x75, 0x66,
-	0x66, 0x72, 0x61, 0x67, 0x65, 0x32, 0x94, 0x0d, 0x0a, 0x07, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65,
-	0x64, 0x12, 0x49, 0x0a, 0x0d, 0x53, 0x65, 0x6e, 0x64, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65,
-	0x61, 0x74, 0x12, 0x14, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x48,
-	0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65,
-	0x72, 0x5f, 0x70, 0x62, 0x2e, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x52, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x58, 0x0a, 0x0d,
-	0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x1f, 0x2e,
-	0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f,
-	0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20,
-	0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43,
-	0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x51, 0x0a, 0x0c, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70,
-	0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f,
-	0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f,
-	0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x06, 0x41, 0x73, 0x73,
-	0x69, 0x67, 0x6e, 0x12, 0x18, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
-	0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e,
-	0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e,
-	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0a, 0x53, 0x74,
-	0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65,
-	0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f,
-	0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e, 0x43, 0x6f, 0x6c, 0x6c, 0x65,
-	0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74,
-	0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e,
-	0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x61,
-	0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69,
-	0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00,
-	0x12, 0x5d, 0x0a, 0x10, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65,
-	0x6c, 0x65, 0x74, 0x65, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
+	0x66, 0x72, 0x61, 0x67, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x73, 0x4c, 0x65, 0x61, 0x64, 0x65,
+	0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x73, 0x4c, 0x65, 0x61, 0x64, 0x65,
+	0x72, 0x32, 0x94, 0x0d, 0x0a, 0x07, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x12, 0x49, 0x0a,
+	0x0d, 0x53, 0x65, 0x6e, 0x64, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x12, 0x14,
+	0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x48, 0x65, 0x61, 0x72, 0x74,
+	0x62, 0x65, 0x61, 0x74, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
+	0x2e, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x58, 0x0a, 0x0d, 0x4b, 0x65, 0x65, 0x70,
+	0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74,
+	0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63,
+	0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x73,
+	0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
+	0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01,
+	0x30, 0x01, 0x12, 0x51, 0x0a, 0x0c, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75,
+	0x6d, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c,
+	0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c,
+	0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x06, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x12,
+	0x18, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69,
+	0x67, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6d, 0x61, 0x73, 0x74,
+	0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x52, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73,
+	0x74, 0x69, 0x63, 0x73, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
+	0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53,
+	0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f,
+	0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
+	0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72,
+	0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69,
+	0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x10,
+	0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65,
+	0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c,
+	0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
 	0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74,
-	0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65,
-	0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44,
-	0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12,
-	0x4b, 0x0a, 0x0a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1c, 0x2e,
-	0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65,
-	0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61,
-	0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69,
-	0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e,
-	0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x20,
-	0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75,
-	0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x1a, 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f,
-	0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f,
-	0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x51, 0x0a, 0x0c, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56,
-	0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
-	0x62, 0x2e, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
-	0x62, 0x2e, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x63, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75,
-	0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x12, 0x24,
-	0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d,
-	0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
-	0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f,
-	0x6e, 0x6c, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a,
-	0x16, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
-	0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72,
-	0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e,
-	0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65,
-	0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61,
-	0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d,
-	0x0a, 0x10, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64,
-	0x65, 0x73, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c,
+	0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0a, 0x56,
+	0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74,
+	0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72,
+	0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e, 0x4c, 0x6f, 0x6f, 0x6b,
+	0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x20, 0x2e, 0x6d, 0x61, 0x73,
+	0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56,
+	0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d,
+	0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45,
+	0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
+	0x00, 0x12, 0x51, 0x0a, 0x0c, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d,
+	0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61,
+	0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61,
+	0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x22, 0x00, 0x12, 0x63, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61,
+	0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x12, 0x24, 0x2e, 0x6d, 0x61, 0x73,
+	0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72,
+	0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x1a, 0x25, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c,
+	0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x16, 0x47, 0x65, 0x74,
+	0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74,
+	0x69, 0x6f, 0x6e, 0x12, 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
+	0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75,
+	0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e,
+	0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73,
+	0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x10, 0x4c, 0x69,
+	0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x22,
+	0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43,
+	0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c,
 	0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f,
-	0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f,
-	0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5a, 0x0a,
-	0x0f, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e,
-	0x12, 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x65, 0x61,
-	0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75,
-	0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
-	0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x11, 0x52, 0x65, 0x6c,
-	0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x23,
-	0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61,
-	0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75,
-	0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
-	0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65,
-	0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x04, 0x50,
-	0x69, 0x6e, 0x67, 0x12, 0x16, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
-	0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6d, 0x61,
-	0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70,
-	0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x16, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69,
-	0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73,
-	0x12, 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66,
-	0x74, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76,
-	0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x61, 0x73,
-	0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x43,
-	0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x0d, 0x52, 0x61, 0x66, 0x74, 0x41,
-	0x64, 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65,
-	0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65, 0x72, 0x76,
-	0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74,
-	0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65, 0x72,
-	0x76, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a,
-	0x10, 0x52, 0x61, 0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65,
-	0x72, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5a, 0x0a, 0x0f, 0x4c, 0x65, 0x61,
+	0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x21, 0x2e, 0x6d,
+	0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64,
+	0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
+	0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x65, 0x61, 0x73,
+	0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x11, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65,
+	0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x23, 0x2e, 0x6d, 0x61, 0x73,
+	0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64,
+	0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
+	0x24, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6c, 0x65,
+	0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12,
+	0x16, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x67,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72,
+	0x5f, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x22, 0x00, 0x12, 0x6f, 0x0a, 0x16, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c,
+	0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x28, 0x2e, 0x6d,
+	0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73,
+	0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f,
+	0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74,
+	0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x0d, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65,
+	0x72, 0x76, 0x65, 0x72, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
+	0x2e, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
+	0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x10, 0x52, 0x61, 0x66,
+	0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x22, 0x2e,
+	0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x52, 0x65,
+	0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61,
 	0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
-	0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76,
-	0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x32, 0x5a, 0x30,
-	0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65,
-	0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77,
-	0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
-	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x32, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68,
+	0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73,
+	0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f,
+	0x70, 0x62, 0x2f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

weed/pb/master_pb/master_grpc.pb.go

@@ -11,7 +11,6 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // SeaweedClient is the client API for Seaweed service.

weed/pb/mq.proto

@@ -21,32 +21,18 @@ service SeaweedMessaging {
     }
 
     // data plane
-    rpc PublishMessage (stream PublishRequest) returns (stream PublishResponse) {
+    rpc Publish (stream PublishRequest) returns (stream PublishResponse) {
     }
 }
 
 //////////////////////////////////////////////////
-message SegmentInfos {
-    repeated SegmentInfo segment_infos = 1;
-}
-
-message Topic {
-    string namespace = 1;
-    string topic = 2;
-}
-
 message SegmentInfo {
     Segment segment = 1;
     int64 start_ts_ns = 2;
     repeated string brokers = 3;
     int64 stop_ts_ns = 4;
-}
-
-message Segment {
-    int32 id = 1;
-    int32 ring_size = 2;
-    int32 range_start = 3;
-    int32 range_stop = 4;
+    repeated int32 previous_segments = 5;
+    repeated int32 next_segments = 6;
 }
 
 //////////////////////////////////////////////////
@@ -59,9 +45,21 @@ message FindBrokerLeaderResponse {
     string broker = 1;
 }
 
+message Partition {
+    int32 ring_size = 1;
+    int32 range_start = 2;
+    int32 range_stop = 3;
+}
+
+message Segment {
+    string namespace = 1;
+    string topic = 2;
+    int32 id = 3;
+    Partition partition = 4;
+}
+
 message AssignSegmentBrokersRequest {
-    Topic topic = 1;
-    Segment segment = 2;
+    Segment segment = 1;
 }
 
 message AssignSegmentBrokersResponse {
@@ -69,8 +67,7 @@ message AssignSegmentBrokersResponse {
 }
 
 message CheckSegmentStatusRequest {
-    Topic topic = 1;
-    Segment segment = 2;
+    Segment segment = 1;
 }
 
 message CheckSegmentStatusResponse {
@@ -88,14 +85,11 @@ message CheckBrokerLoadResponse {
 
 //////////////////////////////////////////////////
 message PublishRequest {
-    message DataMessage {
-        bytes message = 1;
-    }
-    DataMessage data = 1;
-    message ControlMessage {
+    message InitMessage {
         Segment segment = 1;
     }
-    ControlMessage control = 2;
+    InitMessage init = 1;
+    bytes message = 2;
 }
 message PublishResponse {
     int64 ack_sequence = 1;

weed/pb/mq_pb/mq.pb.go

@@ -623,8 +623,8 @@ type PublishRequest struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	Data    *PublishRequest_DataMessage    `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
-	Control *PublishRequest_ControlMessage `protobuf:"bytes,2,opt,name=control,proto3" json:"control,omitempty"`
+	Init    *PublishRequest_InitMessage `protobuf:"bytes,1,opt,name=init,proto3" json:"init,omitempty"`
+	Message []byte                      `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
 }
 
 func (x *PublishRequest) Reset() {
@@ -659,16 +659,16 @@ func (*PublishRequest) Descriptor() ([]byte, []int) {
 	return file_mq_proto_rawDescGZIP(), []int{11}
 }
 
-func (x *PublishRequest) GetData() *PublishRequest_DataMessage {
+func (x *PublishRequest) GetInit() *PublishRequest_InitMessage {
 	if x != nil {
-		return x.Data
+		return x.Init
 	}
 	return nil
 }
 
-func (x *PublishRequest) GetControl() *PublishRequest_ControlMessage {
+func (x *PublishRequest) GetMessage() []byte {
 	if x != nil {
-		return x.Control
+		return x.Message
 	}
 	return nil
 }
@@ -728,16 +728,16 @@ func (x *PublishResponse) GetIsClosed() bool {
 	return false
 }
 
-type PublishRequest_DataMessage struct {
+type PublishRequest_InitMessage struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	Message []byte `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"`
+	Segment *Segment `protobuf:"bytes,1,opt,name=segment,proto3" json:"segment,omitempty"`
 }
 
-func (x *PublishRequest_DataMessage) Reset() {
-	*x = PublishRequest_DataMessage{}
+func (x *PublishRequest_InitMessage) Reset() {
+	*x = PublishRequest_InitMessage{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_mq_proto_msgTypes[13]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -745,13 +745,13 @@ func (x *PublishRequest_DataMessage) Reset() {
 	}
 }
 
-func (x *PublishRequest_DataMessage) String() string {
+func (x *PublishRequest_InitMessage) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*PublishRequest_DataMessage) ProtoMessage() {}
+func (*PublishRequest_InitMessage) ProtoMessage() {}
 
-func (x *PublishRequest_DataMessage) ProtoReflect() protoreflect.Message {
+func (x *PublishRequest_InitMessage) ProtoReflect() protoreflect.Message {
 	mi := &file_mq_proto_msgTypes[13]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -763,59 +763,12 @@ func (x *PublishRequest_DataMessage) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use PublishRequest_DataMessage.ProtoReflect.Descriptor instead.
-func (*PublishRequest_DataMessage) Descriptor() ([]byte, []int) {
+// Deprecated: Use PublishRequest_InitMessage.ProtoReflect.Descriptor instead.
+func (*PublishRequest_InitMessage) Descriptor() ([]byte, []int) {
 	return file_mq_proto_rawDescGZIP(), []int{11, 0}
 }
 
-func (x *PublishRequest_DataMessage) GetMessage() []byte {
-	if x != nil {
-		return x.Message
-	}
-	return nil
-}
-
-type PublishRequest_ControlMessage struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Segment *Segment `protobuf:"bytes,1,opt,name=segment,proto3" json:"segment,omitempty"`
-}
-
-func (x *PublishRequest_ControlMessage) Reset() {
-	*x = PublishRequest_ControlMessage{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_mq_proto_msgTypes[14]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *PublishRequest_ControlMessage) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*PublishRequest_ControlMessage) ProtoMessage() {}
-
-func (x *PublishRequest_ControlMessage) ProtoReflect() protoreflect.Message {
-	mi := &file_mq_proto_msgTypes[14]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use PublishRequest_ControlMessage.ProtoReflect.Descriptor instead.
-func (*PublishRequest_ControlMessage) Descriptor() ([]byte, []int) {
-	return file_mq_proto_rawDescGZIP(), []int{11, 1}
-}
-
-func (x *PublishRequest_ControlMessage) GetSegment() *Segment {
+func (x *PublishRequest_InitMessage) GetSegment() *Segment {
 	if x != nil {
 		return x.Segment
 	}
@@ -888,57 +841,51 @@ var file_mq_proto_rawDesc = []byte{
 	0x01, 0x28, 0x03, 0x52, 0x0c, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x75, 0x6e,
 	0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74,
 	0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x62, 0x79, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x75,
-	0x6e, 0x74, 0x22, 0x81, 0x02, 0x0a, 0x0e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20,
+	0x6e, 0x74, 0x22, 0xa8, 0x01, 0x0a, 0x0e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x04, 0x69, 0x6e, 0x69, 0x74, 0x18, 0x01, 0x20,
 	0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f,
 	0x70, 0x62, 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x52, 0x04, 0x64,
-	0x61, 0x74, 0x61, 0x12, 0x45, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x18, 0x02,
-	0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67,
-	0x5f, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65,
-	0x73, 0x74, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67,
-	0x65, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x1a, 0x27, 0x0a, 0x0b, 0x44, 0x61,
-	0x74, 0x61, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73,
-	0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73,
-	0x61, 0x67, 0x65, 0x1a, 0x41, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x4d, 0x65,
-	0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x2f, 0x0a, 0x07, 0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69,
-	0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x07, 0x73,
-	0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x22, 0x51, 0x0a, 0x0f, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73,
-	0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x61, 0x63, 0x6b,
-	0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52,
-	0x0b, 0x61, 0x63, 0x6b, 0x53, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x1b, 0x0a, 0x09,
-	0x69, 0x73, 0x5f, 0x63, 0x6c, 0x6f, 0x73, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52,
-	0x08, 0x69, 0x73, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x64, 0x32, 0x8a, 0x04, 0x0a, 0x10, 0x53, 0x65,
-	0x61, 0x77, 0x65, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x12, 0x63,
-	0x0a, 0x10, 0x46, 0x69, 0x6e, 0x64, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x4c, 0x65, 0x61, 0x64,
-	0x65, 0x72, 0x12, 0x25, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70,
-	0x62, 0x2e, 0x46, 0x69, 0x6e, 0x64, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x4c, 0x65, 0x61, 0x64,
-	0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x6d, 0x65, 0x73, 0x73,
+	0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x52, 0x04, 0x69,
+	0x6e, 0x69, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x3e, 0x0a,
+	0x0b, 0x49, 0x6e, 0x69, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x2f, 0x0a, 0x07,
+	0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e,
+	0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x65, 0x67,
+	0x6d, 0x65, 0x6e, 0x74, 0x52, 0x07, 0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x22, 0x51, 0x0a,
+	0x0f, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x12, 0x21, 0x0a, 0x0c, 0x61, 0x63, 0x6b, 0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x61, 0x63, 0x6b, 0x53, 0x65, 0x71, 0x75, 0x65,
+	0x6e, 0x63, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x69, 0x73, 0x5f, 0x63, 0x6c, 0x6f, 0x73, 0x65, 0x64,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x73, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x64,
+	0x32, 0x83, 0x04, 0x0a, 0x10, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73,
+	0x61, 0x67, 0x69, 0x6e, 0x67, 0x12, 0x63, 0x0a, 0x10, 0x46, 0x69, 0x6e, 0x64, 0x42, 0x72, 0x6f,
+	0x6b, 0x65, 0x72, 0x4c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x25, 0x2e, 0x6d, 0x65, 0x73, 0x73,
 	0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x6e, 0x64, 0x42, 0x72, 0x6f,
-	0x6b, 0x65, 0x72, 0x4c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
-	0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x14, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x53, 0x65, 0x67,
-	0x6d, 0x65, 0x6e, 0x74, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x73, 0x12, 0x29, 0x2e, 0x6d, 0x65,
-	0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67,
-	0x6e, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x73, 0x52,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69,
-	0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x53, 0x65, 0x67, 0x6d,
-	0x65, 0x6e, 0x74, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
-	0x73, 0x65, 0x22, 0x00, 0x12, 0x69, 0x0a, 0x12, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x65, 0x67,
-	0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x27, 0x2e, 0x6d, 0x65, 0x73,
+	0x6b, 0x65, 0x72, 0x4c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x1a, 0x26, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e,
+	0x46, 0x69, 0x6e, 0x64, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x4c, 0x65, 0x61, 0x64, 0x65, 0x72,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x14, 0x41, 0x73,
+	0x73, 0x69, 0x67, 0x6e, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x72, 0x6f, 0x6b, 0x65,
+	0x72, 0x73, 0x12, 0x29, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70,
+	0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x42,
+	0x72, 0x6f, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e,
+	0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73,
+	0x69, 0x67, 0x6e, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72,
+	0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x69, 0x0a, 0x12, 0x43,
+	0x68, 0x65, 0x63, 0x6b, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75,
+	0x73, 0x12, 0x27, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62,
+	0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61,
+	0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x6d, 0x65, 0x73,
 	0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53,
-	0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75,
-	0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f,
-	0x70, 0x62, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x53,
-	0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12,
-	0x60, 0x0a, 0x0f, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x4c, 0x6f,
-	0x61, 0x64, 0x12, 0x24, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70,
-	0x62, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x4c, 0x6f, 0x61,
-	0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61,
-	0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x42, 0x72, 0x6f,
-	0x6b, 0x65, 0x72, 0x4c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
-	0x00, 0x12, 0x53, 0x0a, 0x0e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x4d, 0x65, 0x73, 0x73,
-	0x61, 0x67, 0x65, 0x12, 0x1c, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f,
+	0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x0f, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x42,
+	0x72, 0x6f, 0x6b, 0x65, 0x72, 0x4c, 0x6f, 0x61, 0x64, 0x12, 0x24, 0x2e, 0x6d, 0x65, 0x73, 0x73,
+	0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x42, 0x72,
+	0x6f, 0x6b, 0x65, 0x72, 0x4c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
+	0x25, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x43,
+	0x68, 0x65, 0x63, 0x6b, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x4c, 0x6f, 0x61, 0x64, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x07, 0x50, 0x75, 0x62, 0x6c,
+	0x69, 0x73, 0x68, 0x12, 0x1c, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f,
 	0x70, 0x62, 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
 	0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62,
 	0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
@@ -962,47 +909,45 @@ func file_mq_proto_rawDescGZIP() []byte {
 	return file_mq_proto_rawDescData
 }
 
-var file_mq_proto_msgTypes = make([]protoimpl.MessageInfo, 15)
+var file_mq_proto_msgTypes = make([]protoimpl.MessageInfo, 14)
 var file_mq_proto_goTypes = []interface{}{
-	(*SegmentInfo)(nil),                   // 0: messaging_pb.SegmentInfo
-	(*FindBrokerLeaderRequest)(nil),       // 1: messaging_pb.FindBrokerLeaderRequest
-	(*FindBrokerLeaderResponse)(nil),      // 2: messaging_pb.FindBrokerLeaderResponse
-	(*Partition)(nil),                     // 3: messaging_pb.Partition
-	(*Segment)(nil),                       // 4: messaging_pb.Segment
-	(*AssignSegmentBrokersRequest)(nil),   // 5: messaging_pb.AssignSegmentBrokersRequest
-	(*AssignSegmentBrokersResponse)(nil),  // 6: messaging_pb.AssignSegmentBrokersResponse
-	(*CheckSegmentStatusRequest)(nil),     // 7: messaging_pb.CheckSegmentStatusRequest
-	(*CheckSegmentStatusResponse)(nil),    // 8: messaging_pb.CheckSegmentStatusResponse
-	(*CheckBrokerLoadRequest)(nil),        // 9: messaging_pb.CheckBrokerLoadRequest
-	(*CheckBrokerLoadResponse)(nil),       // 10: messaging_pb.CheckBrokerLoadResponse
-	(*PublishRequest)(nil),                // 11: messaging_pb.PublishRequest
-	(*PublishResponse)(nil),               // 12: messaging_pb.PublishResponse
-	(*PublishRequest_DataMessage)(nil),    // 13: messaging_pb.PublishRequest.DataMessage
-	(*PublishRequest_ControlMessage)(nil), // 14: messaging_pb.PublishRequest.ControlMessage
+	(*SegmentInfo)(nil),                  // 0: messaging_pb.SegmentInfo
+	(*FindBrokerLeaderRequest)(nil),      // 1: messaging_pb.FindBrokerLeaderRequest
+	(*FindBrokerLeaderResponse)(nil),     // 2: messaging_pb.FindBrokerLeaderResponse
+	(*Partition)(nil),                    // 3: messaging_pb.Partition
+	(*Segment)(nil),                      // 4: messaging_pb.Segment
+	(*AssignSegmentBrokersRequest)(nil),  // 5: messaging_pb.AssignSegmentBrokersRequest
+	(*AssignSegmentBrokersResponse)(nil), // 6: messaging_pb.AssignSegmentBrokersResponse
+	(*CheckSegmentStatusRequest)(nil),    // 7: messaging_pb.CheckSegmentStatusRequest
+	(*CheckSegmentStatusResponse)(nil),   // 8: messaging_pb.CheckSegmentStatusResponse
+	(*CheckBrokerLoadRequest)(nil),       // 9: messaging_pb.CheckBrokerLoadRequest
+	(*CheckBrokerLoadResponse)(nil),      // 10: messaging_pb.CheckBrokerLoadResponse
+	(*PublishRequest)(nil),               // 11: messaging_pb.PublishRequest
+	(*PublishResponse)(nil),              // 12: messaging_pb.PublishResponse
+	(*PublishRequest_InitMessage)(nil),   // 13: messaging_pb.PublishRequest.InitMessage
 }
 var file_mq_proto_depIdxs = []int32{
 	4,  // 0: messaging_pb.SegmentInfo.segment:type_name -> messaging_pb.Segment
 	3,  // 1: messaging_pb.Segment.partition:type_name -> messaging_pb.Partition
 	4,  // 2: messaging_pb.AssignSegmentBrokersRequest.segment:type_name -> messaging_pb.Segment
 	4,  // 3: messaging_pb.CheckSegmentStatusRequest.segment:type_name -> messaging_pb.Segment
-	13, // 4: messaging_pb.PublishRequest.data:type_name -> messaging_pb.PublishRequest.DataMessage
-	14, // 5: messaging_pb.PublishRequest.control:type_name -> messaging_pb.PublishRequest.ControlMessage
-	4,  // 6: messaging_pb.PublishRequest.ControlMessage.segment:type_name -> messaging_pb.Segment
-	1,  // 7: messaging_pb.SeaweedMessaging.FindBrokerLeader:input_type -> messaging_pb.FindBrokerLeaderRequest
-	5,  // 8: messaging_pb.SeaweedMessaging.AssignSegmentBrokers:input_type -> messaging_pb.AssignSegmentBrokersRequest
-	7,  // 9: messaging_pb.SeaweedMessaging.CheckSegmentStatus:input_type -> messaging_pb.CheckSegmentStatusRequest
-	9,  // 10: messaging_pb.SeaweedMessaging.CheckBrokerLoad:input_type -> messaging_pb.CheckBrokerLoadRequest
-	11, // 11: messaging_pb.SeaweedMessaging.PublishMessage:input_type -> messaging_pb.PublishRequest
-	2,  // 12: messaging_pb.SeaweedMessaging.FindBrokerLeader:output_type -> messaging_pb.FindBrokerLeaderResponse
-	6,  // 13: messaging_pb.SeaweedMessaging.AssignSegmentBrokers:output_type -> messaging_pb.AssignSegmentBrokersResponse
-	8,  // 14: messaging_pb.SeaweedMessaging.CheckSegmentStatus:output_type -> messaging_pb.CheckSegmentStatusResponse
-	10, // 15: messaging_pb.SeaweedMessaging.CheckBrokerLoad:output_type -> messaging_pb.CheckBrokerLoadResponse
-	12, // 16: messaging_pb.SeaweedMessaging.PublishMessage:output_type -> messaging_pb.PublishResponse
-	12, // [12:17] is the sub-list for method output_type
-	7,  // [7:12] is the sub-list for method input_type
-	7,  // [7:7] is the sub-list for extension type_name
-	7,  // [7:7] is the sub-list for extension extendee
-	0,  // [0:7] is the sub-list for field type_name
+	13, // 4: messaging_pb.PublishRequest.init:type_name -> messaging_pb.PublishRequest.InitMessage
+	4,  // 5: messaging_pb.PublishRequest.InitMessage.segment:type_name -> messaging_pb.Segment
+	1,  // 6: messaging_pb.SeaweedMessaging.FindBrokerLeader:input_type -> messaging_pb.FindBrokerLeaderRequest
+	5,  // 7: messaging_pb.SeaweedMessaging.AssignSegmentBrokers:input_type -> messaging_pb.AssignSegmentBrokersRequest
+	7,  // 8: messaging_pb.SeaweedMessaging.CheckSegmentStatus:input_type -> messaging_pb.CheckSegmentStatusRequest
+	9,  // 9: messaging_pb.SeaweedMessaging.CheckBrokerLoad:input_type -> messaging_pb.CheckBrokerLoadRequest
+	11, // 10: messaging_pb.SeaweedMessaging.Publish:input_type -> messaging_pb.PublishRequest
+	2,  // 11: messaging_pb.SeaweedMessaging.FindBrokerLeader:output_type -> messaging_pb.FindBrokerLeaderResponse
+	6,  // 12: messaging_pb.SeaweedMessaging.AssignSegmentBrokers:output_type -> messaging_pb.AssignSegmentBrokersResponse
+	8,  // 13: messaging_pb.SeaweedMessaging.CheckSegmentStatus:output_type -> messaging_pb.CheckSegmentStatusResponse
+	10, // 14: messaging_pb.SeaweedMessaging.CheckBrokerLoad:output_type -> messaging_pb.CheckBrokerLoadResponse
+	12, // 15: messaging_pb.SeaweedMessaging.Publish:output_type -> messaging_pb.PublishResponse
+	11, // [11:16] is the sub-list for method output_type
+	6,  // [6:11] is the sub-list for method input_type
+	6,  // [6:6] is the sub-list for extension type_name
+	6,  // [6:6] is the sub-list for extension extendee
+	0,  // [0:6] is the sub-list for field type_name
 }
 
 func init() { file_mq_proto_init() }
@@ -1168,19 +1113,7 @@ func file_mq_proto_init() {
 			}
 		}
 		file_mq_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*PublishRequest_DataMessage); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_mq_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*PublishRequest_ControlMessage); i {
+			switch v := v.(*PublishRequest_InitMessage); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -1198,7 +1131,7 @@ func file_mq_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_mq_proto_rawDesc,
 			NumEnums:      0,
-			NumMessages:   15,
+			NumMessages:   14,
 			NumExtensions: 0,
 			NumServices:   1,
 		},

weed/pb/mq_pb/mq_grpc.pb.go

@@ -24,7 +24,7 @@ type SeaweedMessagingClient interface {
 	CheckSegmentStatus(ctx context.Context, in *CheckSegmentStatusRequest, opts ...grpc.CallOption) (*CheckSegmentStatusResponse, error)
 	CheckBrokerLoad(ctx context.Context, in *CheckBrokerLoadRequest, opts ...grpc.CallOption) (*CheckBrokerLoadResponse, error)
 	// data plane
-	PublishMessage(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_PublishMessageClient, error)
+	Publish(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_PublishClient, error)
 }
 
 type seaweedMessagingClient struct {
@@ -71,30 +71,30 @@ func (c *seaweedMessagingClient) CheckBrokerLoad(ctx context.Context, in *CheckB
 	return out, nil
 }
 
-func (c *seaweedMessagingClient) PublishMessage(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_PublishMessageClient, error) {
-	stream, err := c.cc.NewStream(ctx, &SeaweedMessaging_ServiceDesc.Streams[0], "/messaging_pb.SeaweedMessaging/PublishMessage", opts...)
+func (c *seaweedMessagingClient) Publish(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_PublishClient, error) {
+	stream, err := c.cc.NewStream(ctx, &SeaweedMessaging_ServiceDesc.Streams[0], "/messaging_pb.SeaweedMessaging/Publish", opts...)
 	if err != nil {
 		return nil, err
 	}
-	x := &seaweedMessagingPublishMessageClient{stream}
+	x := &seaweedMessagingPublishClient{stream}
 	return x, nil
 }
 
-type SeaweedMessaging_PublishMessageClient interface {
+type SeaweedMessaging_PublishClient interface {
 	Send(*PublishRequest) error
 	Recv() (*PublishResponse, error)
 	grpc.ClientStream
 }
 
-type seaweedMessagingPublishMessageClient struct {
+type seaweedMessagingPublishClient struct {
 	grpc.ClientStream
 }
 
-func (x *seaweedMessagingPublishMessageClient) Send(m *PublishRequest) error {
+func (x *seaweedMessagingPublishClient) Send(m *PublishRequest) error {
 	return x.ClientStream.SendMsg(m)
 }
 
-func (x *seaweedMessagingPublishMessageClient) Recv() (*PublishResponse, error) {
+func (x *seaweedMessagingPublishClient) Recv() (*PublishResponse, error) {
 	m := new(PublishResponse)
 	if err := x.ClientStream.RecvMsg(m); err != nil {
 		return nil, err
@@ -112,7 +112,7 @@ type SeaweedMessagingServer interface {
 	CheckSegmentStatus(context.Context, *CheckSegmentStatusRequest) (*CheckSegmentStatusResponse, error)
 	CheckBrokerLoad(context.Context, *CheckBrokerLoadRequest) (*CheckBrokerLoadResponse, error)
 	// data plane
-	PublishMessage(SeaweedMessaging_PublishMessageServer) error
+	Publish(SeaweedMessaging_PublishServer) error
 	mustEmbedUnimplementedSeaweedMessagingServer()
 }
 
@@ -132,8 +132,8 @@ func (UnimplementedSeaweedMessagingServer) CheckSegmentStatus(context.Context, *
 func (UnimplementedSeaweedMessagingServer) CheckBrokerLoad(context.Context, *CheckBrokerLoadRequest) (*CheckBrokerLoadResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method CheckBrokerLoad not implemented")
 }
-func (UnimplementedSeaweedMessagingServer) PublishMessage(SeaweedMessaging_PublishMessageServer) error {
-	return status.Errorf(codes.Unimplemented, "method PublishMessage not implemented")
+func (UnimplementedSeaweedMessagingServer) Publish(SeaweedMessaging_PublishServer) error {
+	return status.Errorf(codes.Unimplemented, "method Publish not implemented")
 }
 func (UnimplementedSeaweedMessagingServer) mustEmbedUnimplementedSeaweedMessagingServer() {}
 
@@ -220,25 +220,25 @@ func _SeaweedMessaging_CheckBrokerLoad_Handler(srv interface{}, ctx context.Cont
 	return interceptor(ctx, in, info, handler)
 }
 
-func _SeaweedMessaging_PublishMessage_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(SeaweedMessagingServer).PublishMessage(&seaweedMessagingPublishMessageServer{stream})
+func _SeaweedMessaging_Publish_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(SeaweedMessagingServer).Publish(&seaweedMessagingPublishServer{stream})
 }
 
-type SeaweedMessaging_PublishMessageServer interface {
+type SeaweedMessaging_PublishServer interface {
 	Send(*PublishResponse) error
 	Recv() (*PublishRequest, error)
 	grpc.ServerStream
 }
 
-type seaweedMessagingPublishMessageServer struct {
+type seaweedMessagingPublishServer struct {
 	grpc.ServerStream
 }
 
-func (x *seaweedMessagingPublishMessageServer) Send(m *PublishResponse) error {
+func (x *seaweedMessagingPublishServer) Send(m *PublishResponse) error {
 	return x.ServerStream.SendMsg(m)
 }
 
-func (x *seaweedMessagingPublishMessageServer) Recv() (*PublishRequest, error) {
+func (x *seaweedMessagingPublishServer) Recv() (*PublishRequest, error) {
 	m := new(PublishRequest)
 	if err := x.ServerStream.RecvMsg(m); err != nil {
 		return nil, err
@@ -272,8 +272,8 @@ var SeaweedMessaging_ServiceDesc = grpc.ServiceDesc{
 	},
 	Streams: []grpc.StreamDesc{
 		{
-			StreamName:    "PublishMessage",
-			Handler:       _SeaweedMessaging_PublishMessage_Handler,
+			StreamName:    "Publish",
+			Handler:       _SeaweedMessaging_Publish_Handler,
 			ServerStreams: true,
 			ClientStreams: true,
 		},

weed/replication/sink/filersink/fetch_write.go

@@ -24,15 +24,16 @@ func (fs *FilerSink) replicateChunks(sourceChunks []*filer_pb.FileChunk, path st
 	var wg sync.WaitGroup
 	for chunkIndex, sourceChunk := range sourceChunks {
 		wg.Add(1)
-		go func(chunk *filer_pb.FileChunk, index int) {
+		index, source := chunkIndex, sourceChunk
+		fs.executor.Execute(func() {
 			defer wg.Done()
-			replicatedChunk, e := fs.replicateOneChunk(chunk, path)
+			replicatedChunk, e := fs.replicateOneChunk(source, path)
 			if e != nil {
 				err = e
 				return
 			}
 			replicatedChunks[index] = replicatedChunk
-		}(sourceChunk, chunkIndex)
+		})
 	}
 	wg.Wait()
 

weed/replication/sink/filersink/filer_sink.go

@@ -32,6 +32,7 @@ type FilerSink struct {
 	address           string
 	writeChunkByFiler bool
 	isIncremental     bool
+	executor          *util.LimitedConcurrentExecutor
 }
 
 func init() {
@@ -53,6 +54,7 @@ func (fs *FilerSink) IsIncremental() bool {
 func (fs *FilerSink) Initialize(configuration util.Configuration, prefix string) error {
 	fs.isIncremental = configuration.GetBool(prefix + "is_incremental")
 	fs.dataCenter = configuration.GetString(prefix + "dataCenter")
+	fs.executor = util.NewLimitedConcurrentExecutor(32)
 	return fs.DoInitialize(
 		"",
 		configuration.GetString(prefix+"grpcAddress"),

weed/s3api/auth_credentials.go

@@ -16,6 +16,8 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 )
 
+var IdentityAnonymous *Identity
+
 type Action string
 
 type Iam interface {
@@ -32,10 +34,15 @@ type IdentityAccessManagement struct {
 
 type Identity struct {
 	Name        string
+	AccountId   string
 	Credentials []*Credential
 	Actions     []Action
 }
 
+func (i *Identity) isAnonymous() bool {
+	return i.Name == AccountAnonymous.Name
+}
+
 type Credential struct {
 	AccessKey string
 	SecretKey string
@@ -125,9 +132,23 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api
 	for _, ident := range config.Identities {
 		t := &Identity{
 			Name:        ident.Name,
+			AccountId:   AccountAdmin.Id,
 			Credentials: nil,
 			Actions:     nil,
 		}
+
+		if ident.Name == AccountAnonymous.Name {
+			if ident.AccountId != "" && ident.AccountId != AccountAnonymous.Id {
+				glog.Warningf("anonymous identity is associated with a non-anonymous account ID, the association is invalid")
+			}
+			t.AccountId = AccountAnonymous.Id
+			IdentityAnonymous = t
+		} else {
+			if len(ident.AccountId) > 0 {
+				t.AccountId = ident.AccountId
+			}
+		}
+
 		for _, action := range ident.Actions {
 			t.Actions = append(t.Actions, Action(action))
 		}
@@ -139,6 +160,13 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api
 		}
 		identities = append(identities, t)
 	}
+
+	if IdentityAnonymous == nil {
+		IdentityAnonymous = &Identity{
+			Name:      AccountAnonymous.Name,
+			AccountId: AccountAnonymous.Id,
+		}
+	}
 	iam.m.Lock()
 	// atomically switch
 	iam.identities = identities
@@ -173,7 +201,7 @@ func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, foun
 	iam.m.RLock()
 	defer iam.m.RUnlock()
 	for _, ident := range iam.identities {
-		if ident.Name == "anonymous" {
+		if ident.isAnonymous() {
 			return ident, true
 		}
 	}
@@ -259,6 +287,9 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action)
 		return identity, s3err.ErrAccessDenied
 	}
 
+	if !identity.isAnonymous() {
+		r.Header.Set(s3_constants.AmzAccountId, identity.AccountId)
+	}
 	return identity, s3err.ErrNone
 
 }

weed/s3api/auth_credentials_subscribe.go

@@ -9,7 +9,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
-func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, lastTsNs int64) {
+func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, lastTsNs int64, prefix string, directoriesToWatch []string) {
 
 	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
 
@@ -28,6 +28,7 @@ func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, la
 
 		_ = s3a.onIamConfigUpdate(dir, fileName, content)
 		_ = s3a.onCircuitBreakerConfigUpdate(dir, fileName, content)
+		_ = s3a.onBucketMetadataChange(dir, message.OldEntry, message.NewEntry)
 
 		return nil
 	}
@@ -35,7 +36,7 @@ func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, la
 	var clientEpoch int32
 	util.RetryForever("followIamChanges", func() error {
 		clientEpoch++
-		return pb.WithFilerClientFollowMetadata(s3a, clientName, s3a.randomClientId, clientEpoch, prefix, nil, &lastTsNs, 0, 0, processEventFn, pb.FatalOnError)
+		return pb.WithFilerClientFollowMetadata(s3a, clientName, s3a.randomClientId, clientEpoch, prefix, directoriesToWatch, &lastTsNs, 0, 0, processEventFn, pb.FatalOnError)
 	}, func(err error) bool {
 		glog.V(0).Infof("iam follow metadata changes: %v", err)
 		return true
@@ -63,3 +64,17 @@ func (s3a *S3ApiServer) onCircuitBreakerConfigUpdate(dir, filename string, conte
 	}
 	return nil
 }
+
+//reload bucket metadata
+func (s3a *S3ApiServer) onBucketMetadataChange(dir string, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) error {
+	if dir == s3a.option.BucketsPath {
+		if newEntry != nil {
+			s3a.bucketRegistry.LoadBucketMetadata(newEntry)
+			glog.V(0).Infof("updated bucketMetadata %s/%s", dir, newEntry)
+		} else {
+			s3a.bucketRegistry.RemoveBucketMetadata(oldEntry)
+			glog.V(0).Infof("remove bucketMetadata  %s/%s", dir, newEntry)
+		}
+	}
+	return nil
+}

weed/s3api/auth_credentials_test.go

@@ -3,6 +3,7 @@ package s3api
 import (
 	. "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/stretchr/testify/assert"
+	"reflect"
 	"testing"
 
 	jsonpb "google.golang.org/protobuf/encoding/protojson"
@@ -124,5 +125,98 @@ func TestCanDo(t *testing.T) {
 	}
 	assert.Equal(t, true, ident5.canDo(ACTION_READ, "special_bucket", "/a/b/c/d.txt"))
 	assert.Equal(t, true, ident5.canDo(ACTION_WRITE, "special_bucket", "/a/b/c/d.txt"))
+}
 
+type LoadS3ApiConfigurationTestCase struct {
+	pbIdent     *iam_pb.Identity
+	expectIdent *Identity
+}
+
+func TestLoadS3ApiConfiguration(t *testing.T) {
+	testCases := map[string]*LoadS3ApiConfigurationTestCase{
+		"notSpecifyAccountId": {
+			pbIdent: &iam_pb.Identity{
+				Name: "notSpecifyAccountId",
+				Actions: []string{
+					"Read",
+					"Write",
+				},
+				Credentials: []*iam_pb.Credential{
+					{
+						AccessKey: "some_access_key1",
+						SecretKey: "some_secret_key2",
+					},
+				},
+			},
+			expectIdent: &Identity{
+				Name:      "notSpecifyAccountId",
+				AccountId: AccountAdmin.Id,
+				Actions: []Action{
+					"Read",
+					"Write",
+				},
+				Credentials: []*Credential{
+					{
+						AccessKey: "some_access_key1",
+						SecretKey: "some_secret_key2",
+					},
+				},
+			},
+		},
+		"specifiedAccountID": {
+			pbIdent: &iam_pb.Identity{
+				Name:      "specifiedAccountID",
+				AccountId: "specifiedAccountID",
+				Actions: []string{
+					"Read",
+					"Write",
+				},
+			},
+			expectIdent: &Identity{
+				Name:      "specifiedAccountID",
+				AccountId: "specifiedAccountID",
+				Actions: []Action{
+					"Read",
+					"Write",
+				},
+			},
+		},
+		"anonymous": {
+			pbIdent: &iam_pb.Identity{
+				Name: "anonymous",
+				Actions: []string{
+					"Read",
+					"Write",
+				},
+			},
+			expectIdent: &Identity{
+				Name:      "anonymous",
+				AccountId: "anonymous",
+				Actions: []Action{
+					"Read",
+					"Write",
+				},
+			},
+		},
+	}
+
+	config := &iam_pb.S3ApiConfiguration{
+		Identities: make([]*iam_pb.Identity, 0),
+	}
+	for _, v := range testCases {
+		config.Identities = append(config.Identities, v.pbIdent)
+	}
+
+	iam := IdentityAccessManagement{}
+	err := iam.loadS3ApiConfiguration(config)
+	if err != nil {
+		return
+	}
+
+	for _, ident := range iam.identities {
+		tc := testCases[ident.Name]
+		if !reflect.DeepEqual(ident, tc.expectIdent) {
+			t.Error("not expect")
+		}
+	}
 }

weed/s3api/bucket_metadata.go

@@ -0,0 +1,213 @@
+package s3api
+
+import (
+	"bytes"
+	"encoding/json"
+	"github.com/aws/aws-sdk-go/private/protocol/json/jsonutil"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+
+	//"github.com/seaweedfs/seaweedfs/weed/s3api"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"math"
+	"sync"
+)
+
+var loadBucketMetadataFromFiler = func(r *BucketRegistry, bucketName string) (*BucketMetaData, error) {
+	entry, err := filer_pb.GetEntry(r.s3a, util.NewFullPath(r.s3a.option.BucketsPath, bucketName))
+	if err != nil {
+		return nil, err
+	}
+
+	return buildBucketMetadata(entry), nil
+}
+
+type BucketMetaData struct {
+	_ struct{} `type:"structure"`
+
+	Name string
+
+	//By default, when another AWS account uploads an object to S3 bucket,
+	//that account (the object writer) owns the object, has access to it, and
+	//can grant other users access to it through ACLs. You can use Object Ownership
+	//to change this default behavior so that ACLs are disabled and you, as the
+	//bucket owner, automatically own every object in your bucket.
+	ObjectOwnership string
+
+	// Container for the bucket owner's display name and ID.
+	Owner *s3.Owner `type:"structure"`
+
+	// A list of grants for access controls.
+	Acl []*s3.Grant `locationName:"AccessControlList" locationNameList:"Grant" type:"list"`
+}
+
+type BucketRegistry struct {
+	metadataCache     map[string]*BucketMetaData
+	metadataCacheLock sync.RWMutex
+
+	notFound     map[string]struct{}
+	notFoundLock sync.RWMutex
+	s3a          *S3ApiServer
+}
+
+func NewBucketRegistry(s3a *S3ApiServer) *BucketRegistry {
+	br := &BucketRegistry{
+		metadataCache: make(map[string]*BucketMetaData),
+		notFound:      make(map[string]struct{}),
+		s3a:           s3a,
+	}
+	err := br.init()
+	if err != nil {
+		glog.Fatal("init bucket registry failed", err)
+		return nil
+	}
+	return br
+}
+
+func (r *BucketRegistry) init() error {
+	err := filer_pb.List(r.s3a, r.s3a.option.BucketsPath, "", func(entry *filer_pb.Entry, isLast bool) error {
+		r.LoadBucketMetadata(entry)
+		return nil
+	}, "", false, math.MaxUint32)
+	return err
+}
+
+func (r *BucketRegistry) LoadBucketMetadata(entry *filer_pb.Entry) {
+	bucketMetadata := buildBucketMetadata(entry)
+	r.metadataCacheLock.Lock()
+	defer r.metadataCacheLock.Unlock()
+	r.metadataCache[entry.Name] = bucketMetadata
+}
+
+func buildBucketMetadata(entry *filer_pb.Entry) *BucketMetaData {
+	entryJson, _ := json.Marshal(entry)
+	glog.V(3).Infof("build bucket metadata,entry=%s", entryJson)
+	bucketMetadata := &BucketMetaData{
+		Name: entry.Name,
+
+		//Default ownership: OwnershipBucketOwnerEnforced, which means Acl is disabled
+		ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced,
+
+		// Default owner: `AccountAdmin`
+		Owner: &s3.Owner{
+			ID:          &AccountAdmin.Id,
+			DisplayName: &AccountAdmin.Name,
+		},
+	}
+	if entry.Extended != nil {
+		//ownership control
+		ownership, ok := entry.Extended[s3_constants.ExtOwnershipKey]
+		if ok {
+			ownership := string(ownership)
+			valid := s3_constants.ValidateOwnership(ownership)
+			if valid {
+				bucketMetadata.ObjectOwnership = ownership
+			} else {
+				glog.Warningf("Invalid ownership: %s, bucket: %s", ownership, bucketMetadata.Name)
+			}
+		}
+
+		//access control policy
+		acpBytes, ok := entry.Extended[s3_constants.ExtAcpKey]
+		if ok {
+			var acp s3.AccessControlPolicy
+			err := jsonutil.UnmarshalJSON(&acp, bytes.NewReader(acpBytes))
+			if err == nil {
+				//validate owner
+				if acp.Owner != nil && acp.Owner.ID != nil {
+					bucketMetadata.Owner = acp.Owner
+				} else {
+					glog.Warningf("bucket ownerId is empty! bucket: %s", bucketMetadata.Name)
+				}
+
+				//acl
+				bucketMetadata.Acl = acp.Grants
+			} else {
+				glog.Warningf("Unmarshal ACP: %s(%v), bucket: %s", string(acpBytes), err, bucketMetadata.Name)
+			}
+		}
+	}
+	return bucketMetadata
+}
+
+func (r *BucketRegistry) RemoveBucketMetadata(entry *filer_pb.Entry) {
+	r.removeMetadataCache(entry.Name)
+	r.unMarkNotFound(entry.Name)
+}
+
+func (r *BucketRegistry) GetBucketMetadata(bucketName string) (*BucketMetaData, s3err.ErrorCode) {
+	r.metadataCacheLock.RLock()
+	bucketMetadata, ok := r.metadataCache[bucketName]
+	r.metadataCacheLock.RUnlock()
+	if ok {
+		return bucketMetadata, s3err.ErrNone
+	}
+
+	r.notFoundLock.RLock()
+	_, ok = r.notFound[bucketName]
+	r.notFoundLock.RUnlock()
+	if ok {
+		return nil, s3err.ErrNoSuchBucket
+	}
+
+	bucketMetadata, errCode := r.LoadBucketMetadataFromFiler(bucketName)
+	if errCode != s3err.ErrNone {
+		return nil, errCode
+	}
+
+	r.setMetadataCache(bucketMetadata)
+	r.unMarkNotFound(bucketName)
+	return bucketMetadata, s3err.ErrNone
+}
+
+func (r *BucketRegistry) LoadBucketMetadataFromFiler(bucketName string) (*BucketMetaData, s3err.ErrorCode) {
+	r.notFoundLock.Lock()
+	defer r.notFoundLock.Unlock()
+
+	//check if already exists
+	r.metadataCacheLock.RLock()
+	bucketMetaData, ok := r.metadataCache[bucketName]
+	r.metadataCacheLock.RUnlock()
+	if ok {
+		return bucketMetaData, s3err.ErrNone
+	}
+
+	//if not exists, load from filer
+	bucketMetadata, err := loadBucketMetadataFromFiler(r, bucketName)
+	if err != nil {
+		if err == filer_pb.ErrNotFound {
+			// The bucket doesn't actually exist and should no longer loaded from the filer
+			r.notFound[bucketName] = struct{}{}
+			return nil, s3err.ErrNoSuchBucket
+		}
+		return nil, s3err.ErrInternalError
+	}
+	return bucketMetadata, s3err.ErrNone
+}
+
+func (r *BucketRegistry) setMetadataCache(metadata *BucketMetaData) {
+	r.metadataCacheLock.Lock()
+	defer r.metadataCacheLock.Unlock()
+	r.metadataCache[metadata.Name] = metadata
+}
+
+func (r *BucketRegistry) removeMetadataCache(bucket string) {
+	r.metadataCacheLock.Lock()
+	defer r.metadataCacheLock.Unlock()
+	delete(r.metadataCache, bucket)
+}
+
+func (r *BucketRegistry) markNotFound(bucket string) {
+	r.notFoundLock.Lock()
+	defer r.notFoundLock.Unlock()
+	r.notFound[bucket] = struct{}{}
+}
+
+func (r *BucketRegistry) unMarkNotFound(bucket string) {
+	r.notFoundLock.Lock()
+	defer r.notFoundLock.Unlock()
+	delete(r.notFound, bucket)
+}

weed/s3api/bucket_metadata_test.go

@@ -0,0 +1,236 @@
+package s3api
+
+import (
+	"fmt"
+	"github.com/aws/aws-sdk-go/private/protocol/json/jsonutil"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+	"reflect"
+	"sync"
+	"testing"
+	"time"
+)
+
+type BucketMetadataTestCase struct {
+	filerEntry           *filer_pb.Entry
+	expectBucketMetadata *BucketMetaData
+}
+
+var (
+	//bad entry
+	badEntry = &filer_pb.Entry{
+		Name: "badEntry",
+	}
+
+	//good entry
+	goodEntryAcp, _ = jsonutil.BuildJSON(&s3.AccessControlPolicy{
+		Owner: &s3.Owner{
+			DisplayName: &AccountAdmin.Name,
+			ID:          &AccountAdmin.Id,
+		},
+		Grants: s3_constants.PublicRead,
+	})
+	goodEntry = &filer_pb.Entry{
+		Name: "entryWithValidAcp",
+		Extended: map[string][]byte{
+			s3_constants.ExtOwnershipKey: []byte(s3_constants.OwnershipBucketOwnerEnforced),
+			s3_constants.ExtAcpKey:       goodEntryAcp,
+		},
+	}
+
+	//ownership is ""
+	ownershipEmptyStr = &filer_pb.Entry{
+		Name: "ownershipEmptyStr",
+		Extended: map[string][]byte{
+			s3_constants.ExtOwnershipKey: []byte(""),
+		},
+	}
+
+	//ownership valid
+	ownershipValid = &filer_pb.Entry{
+		Name: "ownershipValid",
+		Extended: map[string][]byte{
+			s3_constants.ExtOwnershipKey: []byte(s3_constants.OwnershipBucketOwnerEnforced),
+		},
+	}
+
+	//acp is ""
+	acpEmptyStr = &filer_pb.Entry{
+		Name: "acpEmptyStr",
+		Extended: map[string][]byte{
+			s3_constants.ExtAcpKey: []byte(""),
+		},
+	}
+
+	//acp is empty object
+	acpEmptyObjectAcp, _ = jsonutil.BuildJSON(&s3.AccessControlPolicy{
+		Owner:  nil,
+		Grants: nil,
+	})
+	acpEmptyObject = &filer_pb.Entry{
+		Name: "acpEmptyObject",
+		Extended: map[string][]byte{
+			s3_constants.ExtAcpKey: acpEmptyObjectAcp,
+		},
+	}
+
+	//acp owner is nil
+	acpOwnerNilAcp, _ = jsonutil.BuildJSON(&s3.AccessControlPolicy{
+		Owner:  nil,
+		Grants: make([]*s3.Grant, 1),
+	})
+	acpOwnerNil = &filer_pb.Entry{
+		Name: "acpOwnerNil",
+		Extended: map[string][]byte{
+			s3_constants.ExtAcpKey: acpOwnerNilAcp,
+		},
+	}
+
+	//load filer is
+	loadFilerBucket = make(map[string]int, 1)
+	//override `loadBucketMetadataFromFiler` to avoid really load from filer
+)
+
+var tcs = []*BucketMetadataTestCase{
+	{
+		badEntry, &BucketMetaData{
+			Name:            badEntry.Name,
+			ObjectOwnership: s3_constants.DefaultOwnershipForExists,
+			Owner: &s3.Owner{
+				DisplayName: &AccountAdmin.Name,
+				ID:          &AccountAdmin.Id,
+			},
+			Acl: nil,
+		},
+	},
+	{
+		goodEntry, &BucketMetaData{
+			Name:            goodEntry.Name,
+			ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced,
+			Owner: &s3.Owner{
+				DisplayName: &AccountAdmin.Name,
+				ID:          &AccountAdmin.Id,
+			},
+			Acl: s3_constants.PublicRead,
+		},
+	},
+	{
+		ownershipEmptyStr, &BucketMetaData{
+			Name:            ownershipEmptyStr.Name,
+			ObjectOwnership: s3_constants.DefaultOwnershipForExists,
+			Owner: &s3.Owner{
+				DisplayName: &AccountAdmin.Name,
+				ID:          &AccountAdmin.Id,
+			},
+			Acl: nil,
+		},
+	},
+	{
+		ownershipValid, &BucketMetaData{
+			Name:            ownershipValid.Name,
+			ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced,
+			Owner: &s3.Owner{
+				DisplayName: &AccountAdmin.Name,
+				ID:          &AccountAdmin.Id,
+			},
+			Acl: nil,
+		},
+	},
+	{
+		acpEmptyStr, &BucketMetaData{
+			Name:            acpEmptyStr.Name,
+			ObjectOwnership: s3_constants.DefaultOwnershipForExists,
+			Owner: &s3.Owner{
+				DisplayName: &AccountAdmin.Name,
+				ID:          &AccountAdmin.Id,
+			},
+			Acl: nil,
+		},
+	},
+	{
+		acpEmptyObject, &BucketMetaData{
+			Name:            acpEmptyObject.Name,
+			ObjectOwnership: s3_constants.DefaultOwnershipForExists,
+			Owner: &s3.Owner{
+				DisplayName: &AccountAdmin.Name,
+				ID:          &AccountAdmin.Id,
+			},
+			Acl: nil,
+		},
+	},
+	{
+		acpOwnerNil, &BucketMetaData{
+			Name:            acpOwnerNil.Name,
+			ObjectOwnership: s3_constants.DefaultOwnershipForExists,
+			Owner: &s3.Owner{
+				DisplayName: &AccountAdmin.Name,
+				ID:          &AccountAdmin.Id,
+			},
+			Acl: make([]*s3.Grant, 0),
+		},
+	},
+}
+
+func TestBuildBucketMetadata(t *testing.T) {
+	for _, tc := range tcs {
+		resultBucketMetadata := buildBucketMetadata(tc.filerEntry)
+		if !reflect.DeepEqual(resultBucketMetadata, tc.expectBucketMetadata) {
+			t.Fatalf("result is unexpect: \nresult: %v, \nexpect: %v", resultBucketMetadata, tc.expectBucketMetadata)
+		}
+	}
+}
+
+func TestGetBucketMetadata(t *testing.T) {
+	loadBucketMetadataFromFiler = func(r *BucketRegistry, bucketName string) (*BucketMetaData, error) {
+		time.Sleep(time.Second)
+		loadFilerBucket[bucketName] = loadFilerBucket[bucketName] + 1
+		return &BucketMetaData{
+			Name: bucketName,
+		}, nil
+	}
+
+	br := &BucketRegistry{
+		metadataCache: make(map[string]*BucketMetaData),
+		notFound:      make(map[string]struct{}),
+		s3a:           nil,
+	}
+
+	//start 40 goroutine for
+	var wg sync.WaitGroup
+	closeCh := make(chan struct{})
+	for i := 0; i < 40; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+		outLoop:
+			for {
+				for j := 0; j < 5; j++ {
+					select {
+					case <-closeCh:
+						break outLoop
+					default:
+						reqBucket := fmt.Sprintf("%c", 67+j)
+						_, errCode := br.GetBucketMetadata(reqBucket)
+						if errCode != s3err.ErrNone {
+							close(closeCh)
+							t.Error("not expect")
+						}
+					}
+				}
+				time.Sleep(10 * time.Microsecond)
+			}
+		}()
+	}
+	time.Sleep(time.Second)
+	close(closeCh)
+	wg.Wait()
+
+	//Each bucket is loaded from the filer only once
+	for bucketName, loadCount := range loadFilerBucket {
+		if loadCount != 1 {
+			t.Fatalf("lock is uneffict: %s, %d", bucketName, loadCount)
+		}
+	}
+}

weed/s3api/filer_util.go

@@ -91,6 +91,22 @@ func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry *
 	return filer_pb.GetEntry(s3a, fullPath)
 }
 
+func (s3a *S3ApiServer) updateEntry(parentDirectoryPath string, newEntry *filer_pb.Entry) error {
+	updateEntryRequest := &filer_pb.UpdateEntryRequest{
+		Directory: parentDirectoryPath,
+		Entry:     newEntry,
+	}
+
+	err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		err := filer_pb.UpdateEntry(client, updateEntryRequest)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	return err
+}
+
 func objectKey(key *string) *string {
 	if strings.HasPrefix(*key, "/") {
 		t := (*key)[1:]

weed/s3api/s3_constants/acp_canned_acl.go

@@ -0,0 +1,65 @@
+package s3_constants
+
+import (
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+const (
+	CannedAclPrivate                = "private"
+	CannedAclPublicRead             = "public-read"
+	CannedAclPublicReadWrite        = "public-read-write"
+	CannedAclAuthenticatedRead      = "authenticated-read"
+	CannedAclLogDeliveryWrite       = "log-delivery-write"
+	CannedAclBucketOwnerRead        = "bucket-owner-read"
+	CannedAclBucketOwnerFullControl = "bucket-owner-full-control"
+	CannedAclAwsExecRead            = "aws-exec-read"
+)
+
+var (
+	PublicRead = []*s3.Grant{
+		{
+			Grantee: &s3.Grantee{
+				Type: &GrantTypeGroup,
+				URI:  &GranteeGroupAllUsers,
+			},
+			Permission: &PermissionRead,
+		},
+	}
+
+	PublicReadWrite = []*s3.Grant{
+		{
+			Grantee: &s3.Grantee{
+				Type: &GrantTypeGroup,
+				URI:  &GranteeGroupAllUsers,
+			},
+			Permission: &PermissionRead,
+		},
+		{
+			Grantee: &s3.Grantee{
+				Type: &GrantTypeGroup,
+				URI:  &GranteeGroupAllUsers,
+			},
+			Permission: &PermissionWrite,
+		},
+	}
+
+	AuthenticatedRead = []*s3.Grant{
+		{
+			Grantee: &s3.Grantee{
+				Type: &GrantTypeGroup,
+				URI:  &GranteeGroupAuthenticatedUsers,
+			},
+			Permission: &PermissionRead,
+		},
+	}
+
+	LogDeliveryWrite = []*s3.Grant{
+		{
+			Grantee: &s3.Grantee{
+				Type: &GrantTypeGroup,
+				URI:  &GranteeGroupLogDelivery,
+			},
+			Permission: &PermissionWrite,
+		},
+	}
+)

weed/s3api/s3_constants/acp_grantee_group.go

@@ -0,0 +1,8 @@
+package s3_constants
+
+//Amazon S3 predefined groups
+var (
+	GranteeGroupAllUsers           = "http://acs.amazonaws.com/groups/global/AllUsers"
+	GranteeGroupAuthenticatedUsers = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
+	GranteeGroupLogDelivery        = "http://acs.amazonaws.com/groups/s3/LogDelivery"
+)

weed/s3api/s3_constants/acp_grantee_type.go

@@ -0,0 +1,7 @@
+package s3_constants
+
+var (
+	GrantTypeCanonicalUser         = "CanonicalUser"
+	GrantTypeAmazonCustomerByEmail = "AmazonCustomerByEmail"
+	GrantTypeGroup                 = "Group"
+)

weed/s3api/s3_constants/acp_ownership.go

@@ -0,0 +1,18 @@
+package s3_constants
+
+var (
+	OwnershipBucketOwnerPreferred = "BucketOwnerPreferred"
+	OwnershipObjectWriter         = "ObjectWriter"
+	OwnershipBucketOwnerEnforced  = "BucketOwnerEnforced"
+
+	DefaultOwnershipForCreate = OwnershipObjectWriter
+	DefaultOwnershipForExists = OwnershipBucketOwnerEnforced
+)
+
+func ValidateOwnership(ownership string) bool {
+	if ownership == "" || (ownership != OwnershipBucketOwnerPreferred && ownership != OwnershipObjectWriter && ownership != OwnershipBucketOwnerEnforced) {
+		return false
+	} else {
+		return true
+	}
+}

weed/s3api/s3_constants/acp_permisson.go

@@ -0,0 +1,9 @@
+package s3_constants
+
+var (
+	PermissionFullControl = "FULL_CONTROL"
+	PermissionRead        = "READ"
+	PermissionWrite       = "WRITE"
+	PermissionReadAcp     = "READ_ACP"
+	PermissionWriteAcp    = "WRITE_ACP"
+)

weed/s3api/s3_constants/extend_key.go

@@ -0,0 +1,6 @@
+package s3_constants
+
+const (
+	ExtAcpKey       = "Seaweed-X-Amz-Acp"
+	ExtOwnershipKey = "Seaweed-X-Amz-Ownership"
+)

weed/s3api/s3_constants/header.go

@@ -43,6 +43,7 @@ const (
 // Non-Standard S3 HTTP request constants
 const (
 	AmzIdentityId = "s3-identity-id"
+	AmzAccountId  = "s3-account-id"
 	AmzAuthType   = "s3-auth-type"
 	AmzIsAdmin    = "s3-is-admin" // only set to http request header as a context
 )

weed/s3api/s3api_account.go

@@ -0,0 +1,69 @@
+package s3api
+
+import (
+	"sync"
+)
+
+//Predefined Accounts
+var (
+	// AccountAdmin is used as the default account for IAM-Credentials access without Account configured
+	AccountAdmin = Account{
+		Name:         "admin",
+		EmailAddress: "admin@example.com",
+		Id:           "admin",
+	}
+
+	// AccountAnonymous is used to represent the account for anonymous access
+	AccountAnonymous = Account{
+		Name:         "anonymous",
+		EmailAddress: "anonymous@example.com",
+		Id:           "anonymous",
+	}
+)
+
+//Account represents a system user, a system user can
+//configure multiple IAM-Users, IAM-Users can configure
+//permissions respectively, and each IAM-User can
+//configure multiple security credentials
+type Account struct {
+	//Name is also used to display the "DisplayName" as the owner of the bucket or object
+	Name         string
+	EmailAddress string
+
+	//Id is used to identify an Account when granting cross-account access(ACLs) to buckets and objects
+	Id string
+}
+
+type AccountManager struct {
+	sync.Mutex
+	s3a *S3ApiServer
+
+	IdNameMapping  map[string]string
+	EmailIdMapping map[string]string
+}
+
+func NewAccountManager(s3a *S3ApiServer) *AccountManager {
+	am := &AccountManager{
+		s3a:            s3a,
+		IdNameMapping:  make(map[string]string),
+		EmailIdMapping: make(map[string]string),
+	}
+	am.initialize()
+	return am
+}
+
+func (am *AccountManager) GetAccountNameById(canonicalId string) string {
+	return am.IdNameMapping[canonicalId]
+}
+
+func (am *AccountManager) GetAccountIdByEmail(email string) string {
+	return am.EmailIdMapping[email]
+}
+
+func (am *AccountManager) initialize() {
+	// load predefined Accounts
+	for _, account := range []Account{AccountAdmin, AccountAnonymous} {
+		am.IdNameMapping[account.Id] = account.Name
+		am.EmailIdMapping[account.EmailAddress] = account.Id
+	}
+}

weed/s3api/s3api_acp.go

@@ -0,0 +1,28 @@
+package s3api
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+	"net/http"
+)
+
+func getAccountId(r *http.Request) string {
+	id := r.Header.Get(s3_constants.AmzAccountId)
+	if len(id) == 0 {
+		return AccountAnonymous.Id
+	} else {
+		return id
+	}
+}
+
+func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s3err.ErrorCode {
+	metadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket)
+	if errCode != s3err.ErrNone {
+		return errCode
+	}
+	accountId := getAccountId(r)
+	if accountId == AccountAdmin.Id || accountId == *metadata.Owner.ID {
+		return s3err.ErrNone
+	}
+	return s3err.ErrAccessDenied
+}

weed/s3api/s3api_bucket_handlers.go

@@ -5,6 +5,8 @@ import (
 	"encoding/xml"
 	"errors"
 	"fmt"
+	"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 	"math"
 	"net/http"
 	"time"
@@ -343,3 +345,159 @@ func (s3a *S3ApiServer) GetBucketLocationHandler(w http.ResponseWriter, r *http.
 func (s3a *S3ApiServer) GetBucketRequestPaymentHandler(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponseXML(w, r, RequestPaymentConfiguration{Payer: "BucketOwner"})
 }
+
+// PutBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketOwnershipControls.html
+func (s3a *S3ApiServer) PutBucketOwnershipControls(w http.ResponseWriter, r *http.Request) {
+	bucket, _ := s3_constants.GetBucketAndObject(r)
+	glog.V(3).Infof("PutBucketOwnershipControls %s", bucket)
+
+	errCode := s3a.checkAccessByOwnership(r, bucket)
+	if errCode != s3err.ErrNone {
+		s3err.WriteErrorResponse(w, r, errCode)
+		return
+	}
+
+	if r.Body == nil || r.Body == http.NoBody {
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
+		return
+	}
+
+	var v s3.OwnershipControls
+	defer util.CloseRequest(r)
+
+	err := xmlutil.UnmarshalXML(&v, xml.NewDecoder(r.Body), "")
+	if err != nil {
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
+		return
+	}
+
+	if len(v.Rules) != 1 {
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
+		return
+	}
+
+	printOwnership := true
+	ownership := *v.Rules[0].ObjectOwnership
+	switch ownership {
+	case s3_constants.OwnershipObjectWriter:
+	case s3_constants.OwnershipBucketOwnerPreferred:
+	case s3_constants.OwnershipBucketOwnerEnforced:
+		printOwnership = false
+	default:
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
+		return
+	}
+
+	bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
+	if err != nil {
+		if err == filer_pb.ErrNotFound {
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
+			return
+		}
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		return
+	}
+
+	oldOwnership, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey]
+	if !ok || string(oldOwnership) != ownership {
+		if bucketEntry.Extended == nil {
+			bucketEntry.Extended = make(map[string][]byte)
+		}
+		bucketEntry.Extended[s3_constants.ExtOwnershipKey] = []byte(ownership)
+		err = s3a.updateEntry(s3a.option.BucketsPath, bucketEntry)
+		if err != nil {
+			s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+			return
+		}
+	}
+
+	if printOwnership {
+		result := &s3.PutBucketOwnershipControlsInput{
+			OwnershipControls: &v,
+		}
+		s3err.WriteAwsXMLResponse(w, r, http.StatusOK, result)
+	} else {
+		writeSuccessResponseEmpty(w, r)
+	}
+}
+
+// GetBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketOwnershipControls.html
+func (s3a *S3ApiServer) GetBucketOwnershipControls(w http.ResponseWriter, r *http.Request) {
+	bucket, _ := s3_constants.GetBucketAndObject(r)
+	glog.V(3).Infof("GetBucketOwnershipControls %s", bucket)
+
+	errCode := s3a.checkAccessByOwnership(r, bucket)
+	if errCode != s3err.ErrNone {
+		s3err.WriteErrorResponse(w, r, errCode)
+		return
+	}
+
+	bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
+	if err != nil {
+		if err == filer_pb.ErrNotFound {
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
+			return
+		}
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		return
+	}
+
+	v, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey]
+	if !ok {
+		s3err.WriteErrorResponse(w, r, s3err.OwnershipControlsNotFoundError)
+		return
+	}
+	ownership := string(v)
+
+	result := &s3.PutBucketOwnershipControlsInput{
+		OwnershipControls: &s3.OwnershipControls{
+			Rules: []*s3.OwnershipControlsRule{
+				{
+					ObjectOwnership: &ownership,
+				},
+			},
+		},
+	}
+
+	s3err.WriteAwsXMLResponse(w, r, http.StatusOK, result)
+}
+
+// DeleteBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketOwnershipControls.html
+func (s3a *S3ApiServer) DeleteBucketOwnershipControls(w http.ResponseWriter, r *http.Request) {
+	bucket, _ := s3_constants.GetBucketAndObject(r)
+	glog.V(3).Infof("PutBucketOwnershipControls %s", bucket)
+
+	errCode := s3a.checkAccessByOwnership(r, bucket)
+	if errCode != s3err.ErrNone {
+		s3err.WriteErrorResponse(w, r, errCode)
+		return
+	}
+
+	bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
+	if err != nil {
+		if err == filer_pb.ErrNotFound {
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
+			return
+		}
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		return
+	}
+
+	_, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey]
+	if !ok {
+		s3err.WriteErrorResponse(w, r, s3err.OwnershipControlsNotFoundError)
+		return
+	}
+
+	delete(bucketEntry.Extended, s3_constants.ExtOwnershipKey)
+	err = s3a.updateEntry(s3a.option.BucketsPath, bucketEntry)
+	if err != nil {
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		return
+	}
+
+	emptyOwnershipControls := &s3.OwnershipControls{
+		Rules: []*s3.OwnershipControlsRule{},
+	}
+	s3err.WriteAwsXMLResponse(w, r, http.StatusOK, emptyOwnershipControls)
+}

weed/s3api/s3api_server.go

@@ -40,6 +40,8 @@ type S3ApiServer struct {
 	randomClientId int32
 	filerGuard     *security.Guard
 	client         *http.Client
+	accountManager *AccountManager
+	bucketRegistry *BucketRegistry
 }
 
 func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer *S3ApiServer, err error) {
@@ -59,6 +61,8 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer
 		filerGuard:     security.NewGuard([]string{}, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec),
 		cb:             NewCircuitBreaker(option),
 	}
+	s3ApiServer.accountManager = NewAccountManager(s3ApiServer)
+	s3ApiServer.bucketRegistry = NewBucketRegistry(s3ApiServer)
 	if option.LocalFilerSocket == "" {
 		s3ApiServer.client = &http.Client{Transport: &http.Transport{
 			MaxIdleConns:        1024,
@@ -76,7 +80,7 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer
 
 	s3ApiServer.registerRouter(router)
 
-	go s3ApiServer.subscribeMetaEvents("s3", filer.DirectoryEtcRoot, time.Now().UnixNano())
+	go s3ApiServer.subscribeMetaEvents("s3", time.Now().UnixNano(), filer.DirectoryEtcRoot, []string{option.BucketsPath})
 	return s3ApiServer, nil
 }
 
@@ -212,6 +216,14 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
 		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV2Handler, ACTION_LIST)), "LIST")).Queries("list-type", "2")
 
 		// buckets with query
+		// PutBucketOwnershipControls
+		bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketOwnershipControls, ACTION_ADMIN), "PUT")).Queries("ownershipControls", "")
+
+		//GetBucketOwnershipControls
+		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.GetBucketOwnershipControls, ACTION_READ), "GET")).Queries("ownershipControls", "")
+
+		//DeleteBucketOwnershipControls
+		bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketOwnershipControls, ACTION_ADMIN), "DELETE")).Queries("ownershipControls", "")
 
 		// raw buckets
 

weed/s3api/s3err/error_handler.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/xml"
 	"fmt"
+	"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
 	"github.com/gorilla/mux"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"net/http"
@@ -19,6 +20,16 @@ const (
 	MimeXML  mimeType = "application/xml"
 )
 
+func WriteAwsXMLResponse(w http.ResponseWriter, r *http.Request, statusCode int, result interface{}) {
+	var bytesBuffer bytes.Buffer
+	err := xmlutil.BuildXML(result, xml.NewEncoder(&bytesBuffer))
+	if err != nil {
+		WriteErrorResponse(w, r, ErrInternalError)
+		return
+	}
+	WriteResponse(w, r, statusCode, bytesBuffer.Bytes(), MimeXML)
+}
+
 func WriteXMLResponse(w http.ResponseWriter, r *http.Request, statusCode int, response interface{}) {
 	WriteResponse(w, r, statusCode, EncodeXMLResponse(response), MimeXML)
 }

weed/s3api/s3err/s3api_errors.go

@@ -107,6 +107,8 @@ const (
 
 	ErrTooManyRequest
 	ErrRequestBytesExceed
+
+	OwnershipControlsNotFoundError
 )
 
 // error code to APIError structure, these fields carry respective
@@ -414,6 +416,12 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Simultaneous request bytes exceed limitations",
 		HTTPStatusCode: http.StatusTooManyRequests,
 	},
+
+	OwnershipControlsNotFoundError: {
+		Code:           "OwnershipControlsNotFoundError",
+		Description:    "The bucket ownership controls were not found",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 }
 
 // GetAPIError provides API Error for input API error code.

weed/s3api/stats.go

@@ -33,6 +33,12 @@ func track(f http.HandlerFunc, action string) http.HandlerFunc {
 		recorder := NewStatusResponseWriter(w)
 		start := time.Now()
 		f(recorder, r)
+		if recorder.Status == http.StatusForbidden {
+			if m, _ := stats_collect.S3RequestCounter.GetMetricWithLabelValues(
+				action, strconv.Itoa(http.StatusOK), bucket); m == nil {
+				bucket = ""
+			}
+		}
 		stats_collect.S3RequestHistogram.WithLabelValues(action, bucket).Observe(time.Since(start).Seconds())
 		stats_collect.S3RequestCounter.WithLabelValues(action, strconv.Itoa(recorder.Status), bucket).Inc()
 	}

weed/server/filer_server_handlers.go

@@ -16,6 +16,15 @@ import (
 
 func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
 	start := time.Now()
+
+	if r.Header.Get("Origin") != "" {
+		w.Header().Set("Access-Control-Allow-Origin", "*")
+		w.Header().Set("Access-Control-Expose-Headers", "*")
+		w.Header().Set("Access-Control-Allow-Headers", "*")
+		w.Header().Set("Access-Control-Allow-Credentials", "true")
+		w.Header().Set("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS")
+	}
+
 	if r.Method == "OPTIONS" {
 		OptionsHandler(w, r, false)
 		return
@@ -45,10 +54,7 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
 	}()
 
 	w.Header().Set("Server", "SeaweedFS Filer "+util.VERSION)
-	if r.Header.Get("Origin") != "" {
-		w.Header().Set("Access-Control-Allow-Origin", "*")
-		w.Header().Set("Access-Control-Allow-Credentials", "true")
-	}
+
 	switch r.Method {
 	case "GET":
 		fs.GetOrHeadHandler(w, r)
@@ -92,6 +98,13 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
 func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Request) {
 
 	start := time.Now()
+
+	if r.Header.Get("Origin") != "" {
+		w.Header().Set("Access-Control-Allow-Origin", "*")
+		w.Header().Set("Access-Control-Allow-Headers", "*")
+		w.Header().Set("Access-Control-Allow-Credentials", "true")
+	}
+
 	stats.FilerRequestCounter.WithLabelValues(r.Method).Inc()
 	defer func() {
 		stats.FilerRequestHistogram.WithLabelValues(r.Method).Observe(time.Since(start).Seconds())
@@ -108,10 +121,7 @@ func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Reque
 	}
 
 	w.Header().Set("Server", "SeaweedFS Filer "+util.VERSION)
-	if r.Header.Get("Origin") != "" {
-		w.Header().Set("Access-Control-Allow-Origin", "*")
-		w.Header().Set("Access-Control-Allow-Credentials", "true")
-	}
+
 	switch r.Method {
 	case "GET":
 		fs.GetOrHeadHandler(w, r)
@@ -122,11 +132,14 @@ func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Reque
 
 func OptionsHandler(w http.ResponseWriter, r *http.Request, isReadOnly bool) {
 	if isReadOnly {
-		w.Header().Add("Access-Control-Allow-Methods", "GET, OPTIONS")
+		w.Header().Set("Access-Control-Allow-Methods", "GET, OPTIONS")
 	} else {
-		w.Header().Add("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS")
+		w.Header().Set("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS")
+		w.Header().Set("Access-Control-Expose-Headers", "*")
 	}
-	w.Header().Add("Access-Control-Allow-Headers", "*")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	w.Header().Set("Access-Control-Allow-Headers", "*")
+	w.Header().Set("Access-Control-Allow-Credentials", "true")
 }
 
 // maybeCheckJwtAuthorization returns true if access should be granted, false if it should be denied

weed/server/filer_server_handlers_write_autochunk.go

@@ -3,6 +3,7 @@ package weed_server
 import (
 	"context"
 	"fmt"
+	//"github.com/seaweedfs/seaweedfs/weed/s3api"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"io"
 	"net/http"
@@ -374,6 +375,12 @@ func SaveAmzMetaData(r *http.Request, existing map[string][]byte, isReplace bool
 		}
 	}
 
+	//acp
+	acp := r.Header.Get(s3_constants.ExtAcpKey)
+	if len(acp) > 0 {
+		metadata[s3_constants.ExtAcpKey] = []byte(acp)
+	}
+
 	return
 
 }

weed/server/master_grpc_server.go

@@ -307,6 +307,8 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ
 		case <-ticker.C:
 			if !ms.Topo.IsLeader() {
 				stats.MasterRaftIsleader.Set(0)
+				stats.MasterAdminLock.Reset()
+				stats.MasterReplicaPlacementMismatch.Reset()
 				return ms.informNewLeader(stream)
 			} else {
 				stats.MasterRaftIsleader.Set(1)

weed/server/master_grpc_server_admin.go

@@ -3,6 +3,7 @@ package weed_server
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
 	"math/rand"
 	"sync"
 	"time"
@@ -112,11 +113,13 @@ func (locks *AdminLocks) generateToken(lockName string, clientName string) (ts t
 		lastClient:     clientName,
 	}
 	locks.locks[lockName] = lock
+	stats.MasterAdminLock.WithLabelValues(clientName).Set(1)
 	return lock.accessLockTime, lock.accessSecret
 }
 
 func (locks *AdminLocks) deleteLock(lockName string) {
 	locks.Lock()
+	stats.MasterAdminLock.WithLabelValues(locks.locks[lockName].lastClient).Set(0)
 	defer locks.Unlock()
 	delete(locks.locks, lockName)
 }

weed/server/master_grpc_server_raft.go

@@ -20,6 +20,7 @@ func (ms *MasterServer) RaftListClusterServers(ctx context.Context, req *master_
 	}
 
 	servers := ms.Topo.HashicorpRaft.GetConfiguration().Configuration().Servers
+	_, leaderId := ms.Topo.HashicorpRaft.LeaderWithID()
 	ms.Topo.RaftServerAccessLock.RUnlock()
 
 	for _, server := range servers {
@@ -27,6 +28,7 @@ func (ms *MasterServer) RaftListClusterServers(ctx context.Context, req *master_
 			Id:       string(server.ID),
 			Address:  string(server.Address),
 			Suffrage: server.Suffrage.String(),
+			IsLeader: server.ID == leaderId,
 		})
 	}
 	return resp, nil

weed/server/master_server.go

@@ -174,39 +174,45 @@ func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) {
 			glog.V(0).Infof("leader change event: %+v => %+v", e.PrevValue(), e.Value())
 			stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", e.Value())).Inc()
 			if ms.Topo.RaftServer.Leader() != "" {
-				glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "becomes leader.")
+				glog.V(0).Infof("[%s] %s becomes leader.", ms.Topo.RaftServer.Name(), ms.Topo.RaftServer.Leader())
 			}
 		})
-		raftServerName = ms.Topo.RaftServer.Name()
+		raftServerName = fmt.Sprintf("[%s]", ms.Topo.RaftServer.Name())
 	} else if raftServer.RaftHashicorp != nil {
 		ms.Topo.HashicorpRaft = raftServer.RaftHashicorp
 		leaderCh := raftServer.RaftHashicorp.LeaderCh()
-		prevLeader := ms.Topo.HashicorpRaft.Leader()
+		prevLeader, _ := ms.Topo.HashicorpRaft.LeaderWithID()
+		raftServerName = ms.Topo.HashicorpRaft.String()
 		go func() {
 			for {
 				select {
 				case isLeader := <-leaderCh:
-					leader := ms.Topo.HashicorpRaft.Leader()
+					ms.Topo.RaftServerAccessLock.RLock()
+					leader, _ := ms.Topo.HashicorpRaft.LeaderWithID()
+					ms.Topo.RaftServerAccessLock.RUnlock()
 					glog.V(0).Infof("is leader %+v change event: %+v => %+v", isLeader, prevLeader, leader)
 					stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", leader)).Inc()
 					prevLeader = leader
 				}
 			}
 		}()
-		raftServerName = ms.Topo.HashicorpRaft.String()
 	}
 	ms.Topo.RaftServerAccessLock.Unlock()
 
 	if ms.Topo.IsLeader() {
-		glog.V(0).Infoln("[", raftServerName, "]", "I am the leader!")
+		glog.V(0).Infof("%s I am the leader!", raftServerName)
 	} else {
+		var raftServerLeader string
 		ms.Topo.RaftServerAccessLock.RLock()
-		if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" {
-			glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "is the leader.")
-		} else if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.Leader() != "" {
-			glog.V(0).Infoln("[", ms.Topo.HashicorpRaft.String(), "]", ms.Topo.HashicorpRaft.Leader(), "is the leader.")
+		if ms.Topo.RaftServer != nil {
+			raftServerLeader = ms.Topo.RaftServer.Leader()
+		} else if ms.Topo.HashicorpRaft != nil {
+			raftServerName = ms.Topo.HashicorpRaft.String()
+			raftServerLeaderAddr, _ := ms.Topo.HashicorpRaft.LeaderWithID()
+			raftServerLeader = string(raftServerLeaderAddr)
 		}
 		ms.Topo.RaftServerAccessLock.RUnlock()
+		glog.V(0).Infof("%s %s - is the leader.", raftServerName, raftServerLeader)
 	}
 }
 
@@ -388,8 +394,17 @@ func (ms *MasterServer) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, startF
 			} else {
 				glog.V(0).Infof("master %s successfully responded to ping", peerName)
 			}
-
 			return nil
 		})
 	}
 }
+
+func (ms *MasterServer) Shutdown() {
+	if ms.Topo == nil || ms.Topo.HashicorpRaft == nil {
+		return
+	}
+	if ms.Topo.HashicorpRaft.State() == hashicorpRaft.Leader {
+		ms.Topo.HashicorpRaft.LeadershipTransfer()
+	}
+	ms.Topo.HashicorpRaft.Shutdown()
+}

weed/server/raft_hashicorp.go

@@ -6,10 +6,13 @@ package weed_server
 import (
 	"fmt"
 	transport "github.com/Jille/raft-grpc-transport"
+	"github.com/armon/go-metrics"
+	"github.com/armon/go-metrics/prometheus"
 	"github.com/hashicorp/raft"
 	boltdb "github.com/hashicorp/raft-boltdb/v2"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
 	"google.golang.org/grpc"
 	"math/rand"
 	"os"
@@ -182,5 +185,18 @@ func NewHashicorpRaftServer(option *RaftServerOption) (*RaftServer, error) {
 		}()
 	}
 
+	// Configure a prometheus sink as the raft metrics sink
+	if sink, err := prometheus.NewPrometheusSinkFrom(prometheus.PrometheusOpts{
+		Registerer: stats.Gather,
+	}); err != nil {
+		return nil, fmt.Errorf("NewPrometheusSink: %v", err)
+	} else {
+		metricsConf := metrics.DefaultConfig(stats.Namespace)
+		metricsConf.EnableRuntimeMetrics = false
+		if _, err = metrics.NewGlobal(metricsConf, sink); err != nil {
+			return nil, fmt.Errorf("metrics.NewGlobal: %v", err)
+		}
+	}
+
 	return s, nil
 }

weed/server/volume_server.go

@@ -137,6 +137,11 @@ func (vs *VolumeServer) SetStopping() {
 	vs.store.SetStopping()
 }
 
+func (vs *VolumeServer) LoadNewVolumes() {
+	glog.V(0).Infoln(" Loading new volume ids ...")
+	vs.store.LoadNewVolumes()
+}
+
 func (vs *VolumeServer) Shutdown() {
 	glog.V(0).Infoln("Shutting down volume server...")
 	vs.store.Close()

weed/shell/command_cluster_raft_ps.go

@@ -40,7 +40,11 @@ func (c *commandRaftClusterPs) Do(args []string, commandEnv *CommandEnv, writer
 		}
 		fmt.Fprintf(writer, "the raft cluster has %d servers\n", len(resp.ClusterServers))
 		for _, server := range resp.ClusterServers {
-			fmt.Fprintf(writer, "  * %s %s (%s)\n", server.Id, server.Address, server.Suffrage)
+			suffrage := server.Suffrage
+			if server.IsLeader {
+				suffrage = "Leader"
+			}
+			fmt.Fprintf(writer, "  * %s %s (%s)\n", server.Id, server.Address, suffrage)
 		}
 
 		return nil

weed/shell/command_ec_common.go

@@ -282,10 +282,6 @@ func mountEcShards(grpcDialOption grpc.DialOption, collection string, volumeId n
 	})
 }
 
-func divide(total, n int) float64 {
-	return float64(total) / float64(n)
-}
-
 func ceilDivide(total, n int) int {
 	return int(math.Ceil(float64(total) / float64(n)))
 }

weed/shell/command_fs_meta_load.go

@@ -19,6 +19,7 @@ func init() {
 }
 
 type commandFsMetaLoad struct {
+	dirPrefix *string
 }
 
 func (c *commandFsMetaLoad) Name() string {
@@ -30,6 +31,7 @@ func (c *commandFsMetaLoad) Help() string {
 
 	fs.meta.load <filer_host>-<port>-<time>.meta
 	fs.meta.load -v=false <filer_host>-<port>-<time>.meta // skip printing out the verbose output
+	fs.meta.load -dirPrefix=/buckets/important* <filer_host>.meta // load any dirs with prefix "important"
 
 `
 }
@@ -44,6 +46,7 @@ func (c *commandFsMetaLoad) Do(args []string, commandEnv *CommandEnv, writer io.
 	fileName := args[len(args)-1]
 
 	metaLoadCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	c.dirPrefix = metaLoadCommand.String("dirPrefix", "", "load entries only with directories matching prefix")
 	verbose := metaLoadCommand.Bool("v", true, "verbose mode")
 	if err = metaLoadCommand.Parse(args[0 : len(args)-1]); err != nil {
 		return nil
@@ -83,11 +86,22 @@ func (c *commandFsMetaLoad) Do(args []string, commandEnv *CommandEnv, writer io.
 				return err
 			}
 
+			// check collection name pattern
+			entryFullName := string(util.FullPath(fullEntry.Dir).Child(fullEntry.Entry.Name))
+			if *c.dirPrefix != "" {
+				if !strings.HasPrefix(fullEntry.Dir, *c.dirPrefix) {
+					if *verbose {
+						fmt.Fprintf(writer, "not match dir prefix %s\n", entryFullName)
+					}
+					continue
+				}
+			}
+
 			if *verbose || lastLogTime.Add(time.Second).Before(time.Now()) {
 				if !*verbose {
 					lastLogTime = time.Now()
 				}
-				fmt.Fprintf(writer, "load %s\n", util.FullPath(fullEntry.Dir).Child(fullEntry.Entry.Name))
+				fmt.Fprintf(writer, "load %s\n", entryFullName)
 			}
 
 			fullEntry.Entry.Name = strings.ReplaceAll(fullEntry.Entry.Name, "/", "x")

weed/shell/command_volume_balance.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
 	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
 	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 	"golang.org/x/exp/slices"
@@ -132,7 +133,7 @@ func balanceVolumeServersByDiskType(commandEnv *CommandEnv, diskType types.DiskT
 			return v.DiskType == string(diskType)
 		})
 	}
-	if err := balanceSelectedVolume(commandEnv, diskType, volumeReplicas, nodes, capacityByMaxVolumeCount(diskType), sortWritableVolumes, applyBalancing); err != nil {
+	if err := balanceSelectedVolume(commandEnv, diskType, volumeReplicas, nodes, sortWritableVolumes, applyBalancing); err != nil {
 		return err
 	}
 
@@ -183,34 +184,38 @@ type Node struct {
 	rack            string
 }
 
-type CapacityFunc func(*master_pb.DataNodeInfo) int
+type CapacityFunc func(*master_pb.DataNodeInfo) float64
 
 func capacityByMaxVolumeCount(diskType types.DiskType) CapacityFunc {
-	return func(info *master_pb.DataNodeInfo) int {
+	return func(info *master_pb.DataNodeInfo) float64 {
 		diskInfo, found := info.DiskInfos[string(diskType)]
 		if !found {
 			return 0
 		}
-		return int(diskInfo.MaxVolumeCount)
+		return float64(diskInfo.MaxVolumeCount)
 	}
 }
 
 func capacityByFreeVolumeCount(diskType types.DiskType) CapacityFunc {
-	return func(info *master_pb.DataNodeInfo) int {
+	return func(info *master_pb.DataNodeInfo) float64 {
 		diskInfo, found := info.DiskInfos[string(diskType)]
 		if !found {
 			return 0
 		}
-		return int(diskInfo.MaxVolumeCount - diskInfo.VolumeCount)
+		var ecShardCount int
+		for _, ecShardInfo := range diskInfo.EcShardInfos {
+			ecShardCount += erasure_coding.ShardBits(ecShardInfo.EcIndexBits).ShardIdCount()
+		}
+		return float64(diskInfo.MaxVolumeCount-diskInfo.VolumeCount) - float64(ecShardCount)/erasure_coding.DataShardsCount
 	}
 }
 
 func (n *Node) localVolumeRatio(capacityFunc CapacityFunc) float64 {
-	return divide(len(n.selectedVolumes), capacityFunc(n.info))
+	return float64(len(n.selectedVolumes)) / capacityFunc(n.info)
 }
 
 func (n *Node) localVolumeNextRatio(capacityFunc CapacityFunc) float64 {
-	return divide(len(n.selectedVolumes)+1, capacityFunc(n.info))
+	return float64(len(n.selectedVolumes)+1) / capacityFunc(n.info)
 }
 
 func (n *Node) isOneVolumeOnly() bool {
@@ -242,9 +247,10 @@ func sortWritableVolumes(volumes []*master_pb.VolumeInformationMessage) {
 	})
 }
 
-func balanceSelectedVolume(commandEnv *CommandEnv, diskType types.DiskType, volumeReplicas map[uint32][]*VolumeReplica, nodes []*Node, capacityFunc CapacityFunc, sortCandidatesFn func(volumes []*master_pb.VolumeInformationMessage), applyBalancing bool) (err error) {
-	selectedVolumeCount, volumeMaxCount := 0, 0
+func balanceSelectedVolume(commandEnv *CommandEnv, diskType types.DiskType, volumeReplicas map[uint32][]*VolumeReplica, nodes []*Node, sortCandidatesFn func(volumes []*master_pb.VolumeInformationMessage), applyBalancing bool) (err error) {
+	selectedVolumeCount, volumeMaxCount := 0, float64(0)
 	var nodesWithCapacity []*Node
+	capacityFunc := capacityByMaxVolumeCount(diskType)
 	for _, dn := range nodes {
 		selectedVolumeCount += len(dn.selectedVolumes)
 		capacity := capacityFunc(dn.info)
@@ -254,7 +260,7 @@ func balanceSelectedVolume(commandEnv *CommandEnv, diskType types.DiskType, volu
 		volumeMaxCount += capacity
 	}
 
-	idealVolumeRatio := divide(selectedVolumeCount, volumeMaxCount)
+	idealVolumeRatio := float64(selectedVolumeCount) / volumeMaxCount
 
 	hasMoved := true
 

weed/shell/command_volume_fsck.go

@@ -393,8 +393,9 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, dataNodeI
 			}
 			buf.Write(resp.FileContent)
 		}
+		fileredBuf := filterDeletedNeedleFromIdx(buf.Bytes())
 		if vinfo.isReadOnly == false {
-			index, err := idx.FirstInvalidIndex(buf.Bytes(), func(key types.NeedleId, offset types.Offset, size types.Size) (bool, error) {
+			index, err := idx.FirstInvalidIndex(fileredBuf.Bytes(), func(key types.NeedleId, offset types.Offset, size types.Size) (bool, error) {
 				resp, err := volumeServerClient.ReadNeedleMeta(context.Background(), &volume_server_pb.ReadNeedleMetaRequest{
 					VolumeId: volumeId,
 					NeedleId: uint64(key),
@@ -407,12 +408,13 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, dataNodeI
 				return resp.LastModified <= cutoffFrom, nil
 			})
 			if err != nil {
-				fmt.Fprintf(writer, "Failed to search for last vilad index on volume %d with error %v", volumeId, err)
+				fmt.Fprintf(writer, "Failed to search for last valid index on volume %d with error %v", volumeId, err)
+			} else {
+				fileredBuf.Truncate(index * types.NeedleMapEntrySize)
 			}
-			buf.Truncate(index * types.NeedleMapEntrySize)
 		}
 		idxFilename := getVolumeFileIdFile(tempFolder, dataNodeId, volumeId)
-		err = writeToFile(buf.Bytes(), idxFilename)
+		err = writeToFile(fileredBuf.Bytes(), idxFilename)
 		if err != nil {
 			return fmt.Errorf("failed to copy %d%s from %s: %v", volumeId, ext, vinfo.server, err)
 		}
@@ -718,3 +720,14 @@ func writeToFile(bytes []byte, fileName string) error {
 	dst.Write(bytes)
 	return nil
 }
+
+func filterDeletedNeedleFromIdx(arr []byte) bytes.Buffer {
+	var filteredBuf bytes.Buffer
+	for i := 0; i < len(arr); i += types.NeedleMapEntrySize {
+		size := types.BytesToSize(arr[i+types.NeedleIdSize+types.OffsetSize : i+types.NeedleIdSize+types.OffsetSize+types.SizeSize])
+		if size > 0 {
+			filteredBuf.Write(arr[i : i+types.NeedleIdSize+types.OffsetSize+types.SizeSize])
+		}
+	}
+	return filteredBuf
+}

weed/shell/command_volume_list_test.go

@@ -2,6 +2,7 @@ package shell
 
 import (
 	_ "embed"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
 	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 	"github.com/stretchr/testify/assert"
 	//"google.golang.org/protobuf/proto"
@@ -18,6 +19,14 @@ func TestParsing(t *testing.T) {
 
 	assert.Equal(t, 5, len(topo.DataCenterInfos))
 
+	topo = parseOutput(topoData2)
+
+	dataNodes := topo.DataCenterInfos[0].RackInfos[0].DataNodeInfos
+	assert.Equal(t, 14, len(dataNodes))
+	diskInfo := dataNodes[0].DiskInfos[""]
+	assert.Equal(t, 1559, len(diskInfo.VolumeInfos))
+	assert.Equal(t, 6740, len(diskInfo.EcShardInfos))
+
 }
 
 func parseOutput(output string) *master_pb.TopologyInfo {
@@ -84,6 +93,29 @@ func parseOutput(output string) *master_pb.TopologyInfo {
 			volume := &master_pb.VolumeInformationMessage{}
 			proto.UnmarshalText(volumeLine, volume)
 			disk.VolumeInfos = append(disk.VolumeInfos, volume)
+		case "ec":
+			ecVolumeLine := line[len("ec volume "):]
+			ecShard := &master_pb.VolumeEcShardInformationMessage{}
+			for _, part := range strings.Split(ecVolumeLine, " ") {
+				if strings.HasPrefix(part, "id:") {
+					id, _ := strconv.ParseInt(part[len("id:"):], 10, 64)
+					ecShard.Id = uint32(id)
+				}
+				if strings.HasPrefix(part, "collection:") {
+					ecShard.Collection = part[len("collection:"):]
+				}
+				if strings.HasPrefix(part, "shards:") {
+					shards := part[len("shards:["):]
+					shards = strings.TrimRight(shards, "]")
+					shardBits := erasure_coding.ShardBits(0)
+					for _, shardId := range strings.Split(shards, ",") {
+						sid, _ := strconv.Atoi(shardId)
+						shardBits = shardBits.AddShardId(erasure_coding.ShardId(sid))
+					}
+					ecShard.EcIndexBits = uint32(shardBits)
+				}
+			}
+			disk.EcShardInfos = append(disk.EcShardInfos, ecShard)
 		}
 	}
 
@@ -92,3 +124,6 @@ func parseOutput(output string) *master_pb.TopologyInfo {
 
 //go:embed volume.list.txt
 var topoData string
+
+//go:embed volume.list2.txt
+var topoData2 string

weed/shell/command_volume_server_evacuate.go

@@ -208,17 +208,22 @@ func (c *commandVolumeServerEvacuate) moveAwayOneEcVolume(commandEnv *CommandEnv
 }
 
 func moveAwayOneNormalVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][]*VolumeReplica, vol *master_pb.VolumeInformationMessage, thisNode *Node, otherNodes []*Node, applyChange bool) (hasMoved bool, err error) {
-	fn := capacityByFreeVolumeCount(types.ToDiskType(vol.DiskType))
+	freeVolumeCountfn := capacityByFreeVolumeCount(types.ToDiskType(vol.DiskType))
+	maxVolumeCountFn := capacityByMaxVolumeCount(types.ToDiskType(vol.DiskType))
 	for _, n := range otherNodes {
 		n.selectVolumes(func(v *master_pb.VolumeInformationMessage) bool {
 			return v.DiskType == vol.DiskType
 		})
 	}
+	// most empty one is in the front
 	slices.SortFunc(otherNodes, func(a, b *Node) bool {
-		return a.localVolumeRatio(fn) < b.localVolumeRatio(fn)
+		return a.localVolumeRatio(maxVolumeCountFn) < b.localVolumeRatio(maxVolumeCountFn)
 	})
 	for i := 0; i < len(otherNodes); i++ {
 		emptyNode := otherNodes[i]
+		if freeVolumeCountfn(emptyNode.info) < 0 {
+			continue
+		}
 		hasMoved, err = maybeMoveOneVolume(commandEnv, volumeReplicas, thisNode, vol, emptyNode, applyChange)
 		if err != nil {
 			return

weed/stats/metrics.go

@@ -19,6 +19,7 @@ import (
 
 // Readonly volume types
 const (
+	Namespace        = "SeaweedFS"
 	IsReadOnly       = "IsReadOnly"
 	NoWriteOrDelete  = "noWriteOrDelete"
 	NoWriteCanDelete = "noWriteCanDelete"
@@ -32,7 +33,7 @@ var (
 
 	MasterClientConnectCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "wdclient",
 			Name:      "connect_updates",
 			Help:      "Counter of master client leader updates.",
@@ -40,15 +41,23 @@ var (
 
 	MasterRaftIsleader = prometheus.NewGauge(
 		prometheus.GaugeOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "master",
 			Name:      "is_leader",
 			Help:      "is leader",
 		})
 
+	MasterAdminLock = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: Namespace,
+			Subsystem: "master",
+			Name:      "admin_lock",
+			Help:      "admin lock",
+		}, []string{"client"})
+
 	MasterReceivedHeartbeatCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "master",
 			Name:      "received_heartbeats",
 			Help:      "Counter of master received heartbeat.",
@@ -56,7 +65,7 @@ var (
 
 	MasterReplicaPlacementMismatch = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "master",
 			Name:      "replica_placement_mismatch",
 			Help:      "replica placement mismatch",
@@ -64,7 +73,7 @@ var (
 
 	MasterLeaderChangeCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "master",
 			Name:      "leader_changes",
 			Help:      "Counter of master leader changes.",
@@ -72,7 +81,7 @@ var (
 
 	FilerRequestCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "filer",
 			Name:      "request_total",
 			Help:      "Counter of filer requests.",
@@ -80,7 +89,7 @@ var (
 
 	FilerRequestHistogram = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "filer",
 			Name:      "request_seconds",
 			Help:      "Bucketed histogram of filer request processing time.",
@@ -89,7 +98,7 @@ var (
 
 	FilerServerLastSendTsOfSubscribeGauge = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "filer",
 			Name:      "last_send_timestamp_of_subscribe",
 			Help:      "The last send timestamp of the filer subscription.",
@@ -97,7 +106,7 @@ var (
 
 	FilerStoreCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "filerStore",
 			Name:      "request_total",
 			Help:      "Counter of filer store requests.",
@@ -105,7 +114,7 @@ var (
 
 	FilerStoreHistogram = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "filerStore",
 			Name:      "request_seconds",
 			Help:      "Bucketed histogram of filer store request processing time.",
@@ -114,7 +123,7 @@ var (
 
 	FilerSyncOffsetGauge = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "filerSync",
 			Name:      "sync_offset",
 			Help:      "The offset of the filer synchronization service.",
@@ -122,7 +131,7 @@ var (
 
 	VolumeServerRequestCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "volumeServer",
 			Name:      "request_total",
 			Help:      "Counter of volume server requests.",
@@ -130,7 +139,7 @@ var (
 
 	VolumeServerRequestHistogram = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "volumeServer",
 			Name:      "request_seconds",
 			Help:      "Bucketed histogram of volume server request processing time.",
@@ -139,7 +148,7 @@ var (
 
 	VolumeServerVolumeCounter = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "volumeServer",
 			Name:      "volumes",
 			Help:      "Number of volumes or shards.",
@@ -147,7 +156,7 @@ var (
 
 	VolumeServerReadOnlyVolumeGauge = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "volumeServer",
 			Name:      "read_only_volumes",
 			Help:      "Number of read only volumes.",
@@ -155,7 +164,7 @@ var (
 
 	VolumeServerMaxVolumeCounter = prometheus.NewGauge(
 		prometheus.GaugeOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "volumeServer",
 			Name:      "max_volumes",
 			Help:      "Maximum number of volumes.",
@@ -163,7 +172,7 @@ var (
 
 	VolumeServerDiskSizeGauge = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "volumeServer",
 			Name:      "total_disk_size",
 			Help:      "Actual disk size used by volumes.",
@@ -171,7 +180,7 @@ var (
 
 	VolumeServerResourceGauge = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "volumeServer",
 			Name:      "resource",
 			Help:      "Resource usage",
@@ -179,7 +188,7 @@ var (
 
 	S3RequestCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "s3",
 			Name:      "request_total",
 			Help:      "Counter of s3 requests.",
@@ -187,7 +196,7 @@ var (
 
 	S3RequestHistogram = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Namespace: "SeaweedFS",
+			Namespace: Namespace,
 			Subsystem: "s3",
 			Name:      "request_seconds",
 			Help:      "Bucketed histogram of s3 request processing time.",
@@ -198,6 +207,7 @@ var (
 func init() {
 	Gather.MustRegister(MasterClientConnectCounter)
 	Gather.MustRegister(MasterRaftIsleader)
+	Gather.MustRegister(MasterAdminLock)
 	Gather.MustRegister(MasterReceivedHeartbeatCounter)
 	Gather.MustRegister(MasterLeaderChangeCounter)
 	Gather.MustRegister(MasterReplicaPlacementMismatch)

weed/storage/needle_map/compact_map.go

@@ -144,9 +144,13 @@ func (cs *CompactSection) deleteOverflowEntry(key SectionalNeedleId) {
 
 // return old entry size
 func (cs *CompactSection) Delete(key NeedleId) Size {
-	skey := SectionalNeedleId(key - cs.start)
 	cs.Lock()
+	defer cs.Unlock()
 	ret := Size(0)
+	if key > cs.end {
+		return ret
+	}
+	skey := SectionalNeedleId(key - cs.start)
 	if i := cs.binarySearchValues(skey); i >= 0 {
 		if cs.values[i].Size > 0 && cs.values[i].Size.IsValid() {
 			ret = cs.values[i].Size
@@ -157,23 +161,23 @@ func (cs *CompactSection) Delete(key NeedleId) Size {
 		cs.deleteOverflowEntry(skey)
 		ret = v.Size
 	}
-	cs.Unlock()
 	return ret
 }
 func (cs *CompactSection) Get(key NeedleId) (*NeedleValue, bool) {
 	cs.RLock()
+	defer cs.RUnlock()
+	if key > cs.end {
+		return nil, false
+	}
 	skey := SectionalNeedleId(key - cs.start)
 	if ve, v, ok := cs.findOverflowEntry(skey); ok {
-		cs.RUnlock()
 		nv := toNeedleValue(ve, v, cs)
 		return &nv, true
 	}
 	if i := cs.binarySearchValues(skey); i >= 0 {
-		cs.RUnlock()
 		nv := toNeedleValue(cs.valuesExtra[i], cs.values[i], cs)
 		return &nv, true
 	}
-	cs.RUnlock()
 	return nil, false
 }
 func (cs *CompactSection) binarySearchValues(key SectionalNeedleId) int {

weed/storage/needle_map/compact_map_perf_test.go

@@ -89,4 +89,4 @@ func PrintMemUsage(totalRowCount uint64) {
 }
 func bToMb(b uint64) uint64 {
 	return b / 1024 / 1024
-}
+}

weed/storage/needle_map/compact_map_test.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/sequence"
 	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"log"
+	"os"
 	"testing"
 )
 
@@ -179,3 +181,40 @@ func TestOverflow(t *testing.T) {
 	println()
 
 }
+
+func TestCompactSection_Get(t *testing.T) {
+	var maps []*CompactMap
+	totalRowCount := uint64(0)
+	indexFile, ie := os.OpenFile("../../../test/data/sample.idx",
+		os.O_RDWR|os.O_RDONLY, 0644)
+	defer indexFile.Close()
+	if ie != nil {
+		log.Fatalln(ie)
+	}
+
+	m, rowCount := loadNewNeedleMap(indexFile)
+	maps = append(maps, m)
+	totalRowCount += rowCount
+	m.Set(1574318345753513987, ToOffset(10002), 10002)
+	nv,ok := m.Get(1574318345753513987)
+	if ok {
+		t.Log(uint64(nv.Key))
+	}
+
+	nv1, ok := m.Get(1574318350048481283)
+	if ok {
+		t.Error(uint64(nv1.Key))
+	}
+
+	m.Set(1574318350048481283, ToOffset(10002), 10002)
+	nv2,ok1 := m.Get(1574318350048481283)
+	if ok1 {
+		t.Log(uint64(nv2.Key))
+	}
+
+	m.Delete(nv2.Key)
+	nv3,has := m.Get(nv2.Key)
+	if has && nv3.Size > 0 {
+		t.Error(uint64(nv3.Size))
+	}
+}

weed/storage/store.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"path/filepath"
 	"strings"
+	"sync"
 	"sync/atomic"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -82,12 +83,21 @@ func NewStore(grpcDialOption grpc.DialOption, ip string, port int, grpcPort int,
 	minFreeSpaces []util.MinFreeSpace, idxFolder string, needleMapKind NeedleMapKind, diskTypes []DiskType) (s *Store) {
 	s = &Store{grpcDialOption: grpcDialOption, Port: port, Ip: ip, GrpcPort: grpcPort, PublicUrl: publicUrl, NeedleMapKind: needleMapKind}
 	s.Locations = make([]*DiskLocation, 0)
+
+	var wg sync.WaitGroup
 	for i := 0; i < len(dirnames); i++ {
 		location := NewDiskLocation(dirnames[i], int32(maxVolumeCounts[i]), minFreeSpaces[i], idxFolder, diskTypes[i])
-		location.loadExistingVolumes(needleMapKind)
 		s.Locations = append(s.Locations, location)
 		stats.VolumeServerMaxVolumeCounter.Add(float64(maxVolumeCounts[i]))
+
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			location.loadExistingVolumes(needleMapKind)
+		}()
 	}
+	wg.Wait()
+
 	s.NewVolumesChan = make(chan master_pb.VolumeShortInformationMessage, 3)
 	s.DeletedVolumesChan = make(chan master_pb.VolumeShortInformationMessage, 3)
 
@@ -361,6 +371,12 @@ func (s *Store) SetStopping() {
 	}
 }
 
+func (s *Store) LoadNewVolumes() {
+	for _, location := range s.Locations {
+		location.loadExistingVolumes(s.NeedleMapKind)
+	}
+}
+
 func (s *Store) Close() {
 	for _, location := range s.Locations {
 		location.Close()

weed/storage/volume_read.go

@@ -84,10 +84,6 @@ func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption, onReadSize
 func (v *Volume) readNeedleMetaAt(n *needle.Needle, offset int64, size int32) (err error) {
 	v.dataFileAccessLock.RLock()
 	defer v.dataFileAccessLock.RUnlock()
-	// read deleted meta data
-	if size < 0 {
-		size = -size
-	}
 	err = n.ReadNeedleMeta(v.DataBackend, offset, Size(size), v.Version())
 	if err == needle.ErrorSizeMismatch && OffsetSize == 4 {
 		err = n.ReadNeedleMeta(v.DataBackend, offset+int64(MaxPossibleVolumeSize), Size(size), v.Version())

weed/util/constants.go

@@ -5,7 +5,7 @@ import (
 )
 
 var (
-	VERSION_NUMBER = fmt.Sprintf("%.02f", 3.29)
+	VERSION_NUMBER = fmt.Sprintf("%.02f", 3.31)
 	VERSION        = sizeLimit + " " + VERSION_NUMBER
 	COMMIT         = ""
 )

weed/util/grace/signal_handling.go

@@ -11,39 +11,55 @@ import (
 )
 
 var signalChan chan os.Signal
-var hooks = make([]func(), 0)
-var hookLock sync.RWMutex
+var interruptHooks = make([]func(), 0)
+var interruptHookLock sync.RWMutex
+var reloadHooks = make([]func(), 0)
+var reloadHookLock sync.RWMutex
 
 func init() {
 	signalChan = make(chan os.Signal, 1)
-	signal.Ignore(syscall.SIGHUP)
 	signal.Notify(signalChan,
 		os.Interrupt,
 		os.Kill,
 		syscall.SIGALRM,
-		// syscall.SIGHUP,
+		syscall.SIGHUP,
 		syscall.SIGINT,
 		syscall.SIGTERM,
 		// syscall.SIGQUIT,
 	)
 	go func() {
-		for range signalChan {
-			hookLock.RLock()
-			for _, hook := range hooks {
-				hook()
+		for s := range signalChan {
+			if s.String() == syscall.SIGHUP.String() {
+				reloadHookLock.RLock()
+				for _, hook := range reloadHooks {
+					hook()
+				}
+				reloadHookLock.RUnlock()
+			} else {
+				interruptHookLock.RLock()
+				for _, hook := range interruptHooks {
+					hook()
+				}
+				interruptHookLock.RUnlock()
+				os.Exit(0)
 			}
-			hookLock.RUnlock()
-			os.Exit(0)
 		}
 	}()
 }
 
+func OnReload(fn func()) {
+	// prevent reentry
+	reloadHookLock.Lock()
+	defer reloadHookLock.Unlock()
+	reloadHooks = append(reloadHooks, fn)
+}
+
 func OnInterrupt(fn func()) {
 	// prevent reentry
-	hookLock.Lock()
-	defer hookLock.Unlock()
+	interruptHookLock.Lock()
+	defer interruptHookLock.Unlock()
 
 	// deal with control+c,etc
 	// controlling terminal close, daemon not exit
-	hooks = append(hooks, fn)
+	interruptHooks = append(interruptHooks, fn)
 }

weed/util/skiplist/skiplist_test.go

@@ -89,7 +89,7 @@ func TestInsertAndFind(t *testing.T) {
 		}
 	}
 	// println("print list")
-	list.println()
+	// list.println()
 
 }