2.77

following https://github.com/wangyoucao577/go-release-action/issues/63#issuecomment-962417798

.github/workflows/binaries_dev.yml

@@ -20,16 +20,13 @@ jobs:
           assets: |
             weed-*
 
-  build_dev:
+  build_dev_linux:
     needs: cleanup
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        goos: [linux, windows, darwin, freebsd]
-        goarch: [amd64, arm64]
-        exclude:
-          - goarch: arm64
-            goos: windows
+        goos: [linux]
+        goarch: [amd64]
 
     steps:
 
@@ -40,7 +37,7 @@ jobs:
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@v1.20
+        uses: wangyoucao577/go-release-action@bugfix/upload-fail
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -56,7 +53,54 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.20
+        uses: wangyoucao577/go-release-action@bugfix/upload-fail
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          release_tag: dev
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed-normal-disk
+          asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
+
+  build_dev_darwin:
+    needs: build_dev_linux
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [darwin]
+        goarch: [amd64, arm64]
+
+    steps:
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set BUILD_TIME env
+        run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
+
+      - name: Go Release Binaries Large Disk
+        uses: wangyoucao577/go-release-action@bugfix/upload-fail
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          release_tag: dev
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed-large-disk
+          asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
+
+      - name: Go Release Binaries Normal Volume Size
+        uses: wangyoucao577/go-release-action@bugfix/upload-fail
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/container_latest.yml

@@ -2,7 +2,8 @@ name: "docker: build latest container"
 
 on:
   push:
-    branches: [ master ]
+    tags:
+      - '*'
   workflow_dispatch: []
 
 jobs:
@@ -23,7 +24,7 @@ jobs:
             chrislusf/seaweedfs
             ghcr.io/chrislusf/seaweedfs
           tags: |
-            type=raw,value=dev
+            type=raw,value=latest
           labels: |
             org.opencontainers.image.title=seaweedfs
             org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!

docker/Dockerfile.go_build

@@ -1,4 +1,4 @@
-FROM amd64/golang:1.17-alpine as builder
+FROM golang:1.17-alpine as builder
 RUN apk add git g++ fuse
 RUN mkdir -p /go/src/github.com/chrislusf/
 RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs

docker/Dockerfile.go_build_large

@@ -1,4 +1,4 @@
-FROM amd64/golang:1.17-alpine as builder
+FROM golang:1.17-alpine as builder
 RUN apk add git g++ fuse
 RUN mkdir -p /go/src/github.com/chrislusf/
 RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs

docker/compose/local-cluster-compose.yml

@@ -6,25 +6,37 @@ services:
     ports:
       - 9333:9333
       - 19333:19333
-    command: "master -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m1"
+    command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
   master1:
     image: chrislusf/seaweedfs:local
     ports:
       - 9334:9334
       - 19334:19334
-    command: "master -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m2"
+    command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
   master2:
     image: chrislusf/seaweedfs:local
     ports:
       - 9335:9335
       - 19335:19335
-    command: "master -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m3"
+    command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
   volume1:
     image: chrislusf/seaweedfs:local
     ports:
       - 8080:8080
       - 18080:18080
-    command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1 -disk=ssd1'
+    command: 'volume -dataCenter dc1 -rack v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
     depends_on:
       - master0
       - master1
@@ -34,7 +46,7 @@ services:
     ports:
       - 8082:8082
       - 18082:18082
-    command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1 -disk=ssd1'
+    command: 'volume -dataCenter dc2 -rack v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
     depends_on:
       - master0
       - master1
@@ -44,7 +56,7 @@ services:
     ports:
       - 8083:8083
       - 18083:18083
-    command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
+    command: 'volume -dataCenter dc3 -rack v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
     depends_on:
       - master0
       - master1
@@ -54,7 +66,8 @@ services:
     ports:
       - 8888:8888
       - 18888:18888
-    command: 'filer -master="master0:9333,master1:9334,master2:9335"'
+      - 8111:8111
+    command: 'filer -defaultReplicaPlacement 100 -iam -master="master0:9333,master1:9334,master2:9335"'
     depends_on:
       - master0
       - master1
@@ -65,11 +78,11 @@ services:
     image: chrislusf/seaweedfs:local
     ports:
       - 8333:8333
-    command: 's3 -filer="filer:8888"'
+    command: '-v=9 s3 -filer="filer:8888"'
     depends_on:
       - master0
       - master1
       - master2
       - volume1
       - volume2
-      - filer
+      - filer

go.mod

@@ -112,9 +112,9 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
-	github.com/tidwall/gjson v1.8.1
-	github.com/tidwall/match v1.0.3
-	github.com/tidwall/pretty v1.1.0 // indirect
+	github.com/tidwall/gjson v1.10.2
+	github.com/tidwall/match v1.1.1
+	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
 	github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43
 	github.com/valyala/bytebufferpool v1.0.0

go.sum

@@ -778,11 +778,17 @@ github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7
 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
 github.com/tidwall/gjson v1.8.1 h1:8j5EE9Hrh3l9Od1OIEDAb7IpezNA20UdRngNAj5N0WU=
 github.com/tidwall/gjson v1.8.1/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk=
+github.com/tidwall/gjson v1.10.2 h1:APbLGOM0rrEkd8WBw9C24nllro4ajFuJu0Sc9hRz8Bo=
+github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
 github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8=
 github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 h1:6iRwZdrFUzbcVYZwa8dXTIILGIxmmhjyUPJEcwzPGaU=

k8s/helm_charts2/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "2.71"
-version: "2.71"
+appVersion: "2.77"
+version: "2.77"

k8s/helm_charts2/templates/filer-statefulset.yaml

@@ -133,11 +133,6 @@ spec:
               -encryptVolumeData \
               {{- end }}
               -ip=${POD_IP} \
-              {{- if .Values.filer.enable_peers }}
-              {{- if gt (.Values.filer.replicas | int) 1 }}
-              -peers=$(echo -n "{{ range $index := until (.Values.filer.replicas | int) }}${SEAWEEDFS_FULLNAME}-filer-{{ $index }}.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}{{ if lt $index (sub ($.Values.filer.replicas | int) 1) }},{{ end }}{{ end }}" | sed "s/$HOSTNAME.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}//" | sed 's/,$//; 's/^,//'; s/,,/,/;' )  \
-              {{- end }}
-              {{- end }}
               {{- if .Values.filer.s3.enabled }}
               -s3 \
               -s3.port={{ .Values.filer.s3.port }} \

k8s/helm_charts2/values.yaml

@@ -246,8 +246,6 @@ filer:
   maxMB: null
   # encrypt data on volume servers
   encryptVolumeData: false
-  # enable peers sync metadata, for leveldb (localdb for filer but with sync across)
-  enable_peers: false
 
   # Whether proxy or redirect to volume server during file GET request
   redirectOnRead: false

other/java/client/pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.8</version>
+    <version>1.7.0</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -60,7 +60,7 @@
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpmime</artifactId>
-            <version>4.5.6</version>
+            <version>4.5.13</version>
         </dependency>
         <dependency>
             <groupId>junit</groupId>

other/java/client/pom.xml.deploy

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.8</version>
+    <version>1.7.0</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -60,7 +60,7 @@
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpmime</artifactId>
-            <version>4.5.6</version>
+            <version>4.5.13</version>
         </dependency>
         <dependency>
             <groupId>junit</groupId>

other/java/client/pom_debug.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.8</version>
+    <version>1.7.0</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -60,7 +60,7 @@
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpmime</artifactId>
-            <version>4.5.6</version>
+            <version>4.5.13</version>
         </dependency>
         <dependency>
             <groupId>junit</groupId>

other/java/client/src/main/java/seaweedfs/client/ReadChunks.java

@@ -0,0 +1,109 @@
+package seaweedfs.client;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+public class ReadChunks {
+
+    public static List<SeaweedRead.VisibleInterval> readResolvedChunks(List<FilerProto.FileChunk> chunkList) throws IOException {
+        List<Point> points = new ArrayList<>(chunkList.size() * 2);
+        for (FilerProto.FileChunk chunk : chunkList) {
+            points.add(new Point(chunk.getOffset(), chunk, true));
+            points.add(new Point(chunk.getOffset() + chunk.getSize(), chunk, false));
+        }
+        Collections.sort(points, new Comparator<Point>() {
+            @Override
+            public int compare(Point a, Point b) {
+                int x = (int) (a.x - b.x);
+                if (a.x != b.x) {
+                    return (int) (a.x - b.x);
+                }
+                if (a.ts != b.ts) {
+                    return (int) (a.ts - b.ts);
+                }
+                if (!a.isStart) {
+                    return -1;
+                }
+                return 1;
+            }
+        });
+
+        long prevX = 0;
+        List<SeaweedRead.VisibleInterval> visibles = new ArrayList<>();
+        ArrayList<Point> queue = new ArrayList<>();
+        for (Point point : points) {
+            if (point.isStart) {
+                if (queue.size() > 0) {
+                    int lastIndex = queue.size() - 1;
+                    Point lastPoint = queue.get(lastIndex);
+                    if (point.x != prevX && lastPoint.ts < point.ts) {
+                        addToVisibles(visibles, prevX, lastPoint, point);
+                        prevX = point.x;
+                    }
+                }
+                // insert into queue
+                for (int i = queue.size(); i >= 0; i--) {
+                    if (i == 0 || queue.get(i - 1).ts <= point.ts) {
+                        if (i == queue.size()) {
+                            prevX = point.x;
+                        }
+                        queue.add(i, point);
+                        break;
+                    }
+                }
+            } else {
+                int lastIndex = queue.size() - 1;
+                int index = lastIndex;
+                Point startPoint = null;
+                for (; index >= 0; index--) {
+                    startPoint = queue.get(index);
+                    if (startPoint.ts == point.ts) {
+                        queue.remove(index);
+                        break;
+                    }
+                }
+                if (index == lastIndex && startPoint != null) {
+                    addToVisibles(visibles, prevX, startPoint, point);
+                    prevX = point.x;
+                }
+            }
+        }
+
+        return visibles;
+
+    }
+
+    private static void addToVisibles(List<SeaweedRead.VisibleInterval> visibles, long prevX, Point startPoint, Point point) {
+        if (prevX < point.x) {
+            FilerProto.FileChunk chunk = startPoint.chunk;
+            visibles.add(new SeaweedRead.VisibleInterval(
+                    prevX,
+                    point.x,
+                    chunk.getFileId(),
+                    chunk.getMtime(),
+                    prevX - chunk.getOffset(),
+                    chunk.getOffset() == prevX && chunk.getSize() == prevX - startPoint.x,
+                    chunk.getCipherKey().toByteArray(),
+                    chunk.getIsCompressed()
+            ));
+        }
+    }
+
+    static class Point {
+        long x;
+        long ts;
+        FilerProto.FileChunk chunk;
+        boolean isStart;
+
+        public Point(long x, FilerProto.FileChunk chunk, boolean isStart) {
+            this.x = x;
+            this.ts = chunk.getMtime();
+            this.chunk = chunk;
+            this.isStart = isStart;
+        }
+    }
+
+}

other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java

@@ -14,10 +14,10 @@ public class RemoteUtil {
         String dir = SeaweedOutputStream.getParentDirectory(fullpath);
         String name = SeaweedOutputStream.getFileName(fullpath);
 
-        final FilerProto.DownloadToLocalResponse downloadToLocalResponse = filerClient.getBlockingStub()
-                .downloadToLocal(FilerProto.DownloadToLocalRequest.newBuilder()
+        final FilerProto.CacheRemoteObjectToLocalClusterResponse response = filerClient.getBlockingStub()
+                .cacheRemoteObjectToLocalCluster(FilerProto.CacheRemoteObjectToLocalClusterRequest.newBuilder()
                 .setDirectory(dir).setName(name).build());
 
-        return downloadToLocalResponse.getEntry();
+        return response.getEntry();
     }
 }

other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java

@@ -226,96 +226,8 @@ public class SeaweedRead {
 
         chunkList = FileChunkManifest.resolveChunkManifest(filerClient, chunkList);
 
-        FilerProto.FileChunk[] chunks = chunkList.toArray(new FilerProto.FileChunk[0]);
-        Arrays.sort(chunks, new Comparator<FilerProto.FileChunk>() {
-            @Override
-            public int compare(FilerProto.FileChunk a, FilerProto.FileChunk b) {
-                // if just a.getMtime() - b.getMtime(), it will overflow!
-                if (a.getMtime() < b.getMtime()) {
-                    return -1;
-                } else if (a.getMtime() > b.getMtime()) {
-                    return 1;
-                }
-                return 0;
-            }
-        });
+        return ReadChunks.readResolvedChunks(chunkList);
 
-        List<VisibleInterval> visibles = new ArrayList<>();
-        for (FilerProto.FileChunk chunk : chunks) {
-            List<VisibleInterval> newVisibles = new ArrayList<>();
-            visibles = mergeIntoVisibles(visibles, newVisibles, chunk);
-        }
-
-        return visibles;
-    }
-
-    private static List<VisibleInterval> mergeIntoVisibles(List<VisibleInterval> visibles,
-                                                           List<VisibleInterval> newVisibles,
-                                                           FilerProto.FileChunk chunk) {
-        VisibleInterval newV = new VisibleInterval(
-                chunk.getOffset(),
-                chunk.getOffset() + chunk.getSize(),
-                chunk.getFileId(),
-                chunk.getMtime(),
-                0,
-                true,
-                chunk.getCipherKey().toByteArray(),
-                chunk.getIsCompressed()
-        );
-
-        // easy cases to speed up
-        if (visibles.size() == 0) {
-            visibles.add(newV);
-            return visibles;
-        }
-        if (visibles.get(visibles.size() - 1).stop <= chunk.getOffset()) {
-            visibles.add(newV);
-            return visibles;
-        }
-
-        for (VisibleInterval v : visibles) {
-            if (v.start < chunk.getOffset() && chunk.getOffset() < v.stop) {
-                newVisibles.add(new VisibleInterval(
-                        v.start,
-                        chunk.getOffset(),
-                        v.fileId,
-                        v.modifiedTime,
-                        v.chunkOffset,
-                        false,
-                        v.cipherKey,
-                        v.isCompressed
-                ));
-            }
-            long chunkStop = chunk.getOffset() + chunk.getSize();
-            if (v.start < chunkStop && chunkStop < v.stop) {
-                newVisibles.add(new VisibleInterval(
-                        chunkStop,
-                        v.stop,
-                        v.fileId,
-                        v.modifiedTime,
-                        v.chunkOffset + (chunkStop - v.start),
-                        false,
-                        v.cipherKey,
-                        v.isCompressed
-                ));
-            }
-            if (chunkStop <= v.start || v.stop <= chunk.getOffset()) {
-                newVisibles.add(v);
-            }
-        }
-        newVisibles.add(newV);
-
-        // keep everything sorted
-        for (int i = newVisibles.size() - 1; i >= 0; i--) {
-            if (i > 0 && newV.start < newVisibles.get(i - 1).start) {
-                newVisibles.set(i, newVisibles.get(i - 1));
-            } else {
-                newVisibles.set(i, newV);
-                break;
-            }
-        }
-
-        return newVisibles;
     }
 
     public static String parseVolumeId(String fileId) {

other/java/client/src/main/proto/filer.proto

@@ -30,6 +30,8 @@ service SeaweedFiler {
 
     rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) {
     }
+    rpc StreamRenameEntry (StreamRenameEntryRequest) returns (stream StreamRenameEntryResponse) {
+    }
 
     rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
     }
@@ -67,7 +69,7 @@ service SeaweedFiler {
     rpc KvPut (KvPutRequest) returns (KvPutResponse) {
     }
 
-    rpc DownloadToLocal (DownloadToLocalRequest) returns (DownloadToLocalResponse) {
+    rpc CacheRemoteObjectToLocalCluster (CacheRemoteObjectToLocalClusterRequest) returns (CacheRemoteObjectToLocalClusterResponse) {
     }
 }
 
@@ -225,6 +227,18 @@ message AtomicRenameEntryRequest {
 message AtomicRenameEntryResponse {
 }
 
+message StreamRenameEntryRequest {
+    string old_directory = 1;
+    string old_name = 2;
+    string new_directory = 3;
+    string new_name = 4;
+    repeated int32 signatures = 5;
+}
+message StreamRenameEntryResponse {
+    string directory = 1;
+    EventNotification event_notification = 2;
+    int64 ts_ns = 3;
+}
 message AssignVolumeRequest {
     int32 count = 1;
     string collection = 2;
@@ -389,10 +403,10 @@ message FilerConf {
 /////////////////////////
 // Remote Storage related
 /////////////////////////
-message DownloadToLocalRequest {
+message CacheRemoteObjectToLocalClusterRequest {
     string directory = 1;
     string name = 2;
 }
-message DownloadToLocalResponse {
+message CacheRemoteObjectToLocalClusterResponse {
     Entry entry = 1;
 }

other/java/client/src/test/java/seaweedfs/client/SeaweedReadTest.java

@@ -6,6 +6,7 @@ import org.junit.Test;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Random;
 
 public class SeaweedReadTest {
 
@@ -13,17 +14,17 @@ public class SeaweedReadTest {
     public void testNonOverlappingVisibleIntervals() throws IOException {
         List<FilerProto.FileChunk> chunks = new ArrayList<>();
         chunks.add(FilerProto.FileChunk.newBuilder()
-            .setFileId("aaa")
-            .setOffset(0)
-            .setSize(100)
-            .setMtime(1000)
-            .build());
+                .setFileId("aaa")
+                .setOffset(0)
+                .setSize(100)
+                .setMtime(1000)
+                .build());
         chunks.add(FilerProto.FileChunk.newBuilder()
-            .setFileId("bbb")
-            .setOffset(100)
-            .setSize(133)
-            .setMtime(2000)
-            .build());
+                .setFileId("bbb")
+                .setOffset(100)
+                .setSize(133)
+                .setMtime(2000)
+                .build());
 
         List<SeaweedRead.VisibleInterval> visibleIntervals = SeaweedRead.nonOverlappingVisibleIntervals(null, chunks);
         for (SeaweedRead.VisibleInterval visibleInterval : visibleIntervals) {
@@ -61,4 +62,106 @@ public class SeaweedReadTest {
 
     }
 
+
+    @Test
+    public void testReadResolvedChunks() throws IOException {
+        List<FilerProto.FileChunk> chunks = new ArrayList<>();
+        chunks.add(FilerProto.FileChunk.newBuilder()
+                .setFileId("a")
+                .setOffset(0)
+                .setSize(100)
+                .setMtime(1)
+                .build());
+        chunks.add(FilerProto.FileChunk.newBuilder()
+                .setFileId("b")
+                .setOffset(50)
+                .setSize(100)
+                .setMtime(2)
+                .build());
+        chunks.add(FilerProto.FileChunk.newBuilder()
+                .setFileId("c")
+                .setOffset(200)
+                .setSize(50)
+                .setMtime(3)
+                .build());
+        chunks.add(FilerProto.FileChunk.newBuilder()
+                .setFileId("d")
+                .setOffset(250)
+                .setSize(50)
+                .setMtime(4)
+                .build());
+        chunks.add(FilerProto.FileChunk.newBuilder()
+                .setFileId("e")
+                .setOffset(175)
+                .setSize(100)
+                .setMtime(5)
+                .build());
+
+        List<SeaweedRead.VisibleInterval> visibleIntervals = ReadChunks.readResolvedChunks(chunks);
+        for (SeaweedRead.VisibleInterval visibleInterval : visibleIntervals) {
+            System.out.println("visible:" + visibleInterval);
+        }
+
+        Assert.assertEquals(4, visibleIntervals.size());
+
+        SeaweedRead.VisibleInterval visibleInterval = visibleIntervals.get(0);
+        Assert.assertEquals(visibleInterval.start, 0);
+        Assert.assertEquals(visibleInterval.stop, 50);
+        Assert.assertEquals(visibleInterval.modifiedTime, 1);
+        Assert.assertEquals(visibleInterval.fileId, "a");
+
+        visibleInterval = visibleIntervals.get(1);
+        Assert.assertEquals(visibleInterval.start, 50);
+        Assert.assertEquals(visibleInterval.stop, 150);
+        Assert.assertEquals(visibleInterval.modifiedTime, 2);
+        Assert.assertEquals(visibleInterval.fileId, "b");
+
+        visibleInterval = visibleIntervals.get(2);
+        Assert.assertEquals(visibleInterval.start, 175);
+        Assert.assertEquals(visibleInterval.stop, 275);
+        Assert.assertEquals(visibleInterval.modifiedTime, 5);
+        Assert.assertEquals(visibleInterval.fileId, "e");
+
+        visibleInterval = visibleIntervals.get(3);
+        Assert.assertEquals(visibleInterval.start, 275);
+        Assert.assertEquals(visibleInterval.stop, 300);
+        Assert.assertEquals(visibleInterval.modifiedTime, 4);
+        Assert.assertEquals(visibleInterval.fileId, "d");
+
+    }
+
+
+    @Test
+    public void testRandomizedReadResolvedChunks() throws IOException {
+        Random random = new Random();
+        int limit = 1024*1024;
+        long[] array = new long[limit];
+        List<FilerProto.FileChunk> chunks = new ArrayList<>();
+        for (long ts=0;ts<1024;ts++){
+            int x = random.nextInt(limit);
+            int y = random.nextInt(limit);
+            int size = Math.min(Math.abs(x-y), 1024);
+            chunks.add(randomWrite(array, Math.min(x,y), size, ts));
+        }
+
+        List<SeaweedRead.VisibleInterval> visibleIntervals = ReadChunks.readResolvedChunks(chunks);
+        for (SeaweedRead.VisibleInterval visibleInterval : visibleIntervals) {
+            System.out.println("visible:" + visibleInterval);
+            for (int i = (int) visibleInterval.start; i<visibleInterval.stop; i++) {
+                Assert.assertEquals(array[i], visibleInterval.modifiedTime);
+            }
+        }
+
+    }
+    private FilerProto.FileChunk randomWrite(long[] array, int start, int size, long ts) {
+        for (int i=start;i<start+size;i++) {
+            array[i] = ts;
+        }
+        return FilerProto.FileChunk.newBuilder()
+                .setFileId("")
+                .setOffset(start)
+                .setSize(size)
+                .setMtime(ts)
+                .build();
+    }
 }

other/java/examples/pom.xml

@@ -11,13 +11,13 @@
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-client</artifactId>
-            <version>1.6.8</version>
+            <version>1.7.0</version>
             <scope>compile</scope>
         </dependency>
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-hadoop2-client</artifactId>
-            <version>1.6.8</version>
+            <version>1.7.0</version>
             <scope>compile</scope>
         </dependency>
         <dependency>

other/java/hdfs2/dependency-reduced-pom.xml

@@ -301,7 +301,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.6.8</seaweedfs.client.version>
+    <seaweedfs.client.version>1.7.0</seaweedfs.client.version>
     <hadoop.version>2.9.2</hadoop.version>
   </properties>
 </project>

other/java/hdfs2/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.6.8</seaweedfs.client.version>
+        <seaweedfs.client.version>1.7.0</seaweedfs.client.version>
         <hadoop.version>2.9.2</hadoop.version>
     </properties>
 

other/java/hdfs3/dependency-reduced-pom.xml

@@ -309,7 +309,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.6.8</seaweedfs.client.version>
+    <seaweedfs.client.version>1.7.0</seaweedfs.client.version>
     <hadoop.version>3.1.1</hadoop.version>
   </properties>
 </project>

other/java/hdfs3/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.6.8</seaweedfs.client.version>
+        <seaweedfs.client.version>1.7.0</seaweedfs.client.version>
         <hadoop.version>3.1.1</hadoop.version>
     </properties>
 

test/s3/basic/basic_test.go

@@ -2,14 +2,15 @@ package basic
 
 import (
 	"fmt"
+	"io"
+	"os"
+	"strings"
+	"testing"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
-	"io/ioutil"
-	"os"
-	"strings"
-	"testing"
 )
 
 var (
@@ -108,8 +109,8 @@ func TestListBucket(t *testing.T) {
 func TestListObjectV2(t *testing.T) {
 
 	listObj, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{
-		Bucket: aws.String(Bucket),
-		Prefix: aws.String("foo"),
+		Bucket:    aws.String(Bucket),
+		Prefix:    aws.String("foo"),
 		Delimiter: aws.String("/"),
 	})
 	if err != nil {
@@ -169,7 +170,7 @@ func TestObjectOp(t *testing.T) {
 		exitErrorf("Unable to get copy object, %v", err)
 	}
 
-	data, err := ioutil.ReadAll(getObj.Body)
+	data, err := io.ReadAll(getObj.Body)
 	if err != nil {
 		exitErrorf("Unable to read object data, %v", err)
 	}

unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go

@@ -5,7 +5,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"math/rand"
 	"mime/multipart"
@@ -45,7 +44,7 @@ func main() {
 			defer wg.Done()
 
 			client := &http.Client{Transport: &http.Transport{
-				MaxIdleConns:     1024,
+				MaxIdleConns:        1024,
 				MaxIdleConnsPerHost: 1024,
 			}}
 			r := rand.New(rand.NewSource(time.Now().UnixNano() + int64(x)))
@@ -128,7 +127,7 @@ func uploadFileToFiler(client *http.Client, data []byte, filename, destination s
 		if err != nil {
 			return 0, fmt.Errorf("read http POST %s response: %v", uri, err)
 		}
-		io.Copy(ioutil.Discard, resp.Body)
+		io.Copy(io.Discard, resp.Body)
 		resp.Body.Close()
 	}
 

unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go

@@ -5,7 +5,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"math/rand"
 	"mime/multipart"
@@ -36,7 +35,7 @@ func main() {
 
 	var fileNames []string
 
-	files, err := ioutil.ReadDir(*dir)
+	files, err := os.ReadDir(*dir)
 	if err != nil {
 		log.Fatalf("fail to read dir %v: %v", *dir, err)
 	}
@@ -142,7 +141,7 @@ func uploadFileToFiler(client *http.Client, filename, destination string) (size
 		if err != nil {
 			return 0, fmt.Errorf("read http POST %s response: %v", uri, err)
 		}
-		io.Copy(ioutil.Discard, resp.Body)
+		io.Copy(io.Discard, resp.Body)
 		resp.Body.Close()
 	}
 

weed/command/backup.go

@@ -120,7 +120,7 @@ func runBackup(cmd *Command, args []string) bool {
 	}
 
 	if v.SuperBlock.CompactionRevision < uint16(stats.CompactRevision) {
-		if err = v.Compact2(30*1024*1024*1024, 0); err != nil {
+		if err = v.Compact2(30*1024*1024*1024, 0, nil); err != nil {
 			fmt.Printf("Compact Volume before synchronizing %v\n", err)
 			return true
 		}

weed/command/compact.go

@@ -50,7 +50,7 @@ func runCompact(cmd *Command, args []string) bool {
 			glog.Fatalf("Compact Volume [ERROR] %s\n", err)
 		}
 	} else {
-		if err = v.Compact2(preallocate, 0); err != nil {
+		if err = v.Compact2(preallocate, 0, nil); err != nil {
 			glog.Fatalf("Compact Volume [ERROR] %s\n", err)
 		}
 	}

weed/command/download.go

@@ -2,17 +2,17 @@ package command
 
 import (
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/pb"
-	"github.com/chrislusf/seaweedfs/weed/security"
-	"google.golang.org/grpc"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"path"
 	"strings"
 
+	"google.golang.org/grpc"
+
 	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -82,7 +82,7 @@ func downloadToFile(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpti
 	}
 	defer f.Close()
 	if isFileList {
-		content, err := ioutil.ReadAll(rc.Body)
+		content, err := io.ReadAll(rc.Body)
 		if err != nil {
 			return err
 		}
@@ -119,7 +119,7 @@ func fetchContent(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption
 		return "", nil, e
 	}
 	defer util.CloseResponse(rc)
-	content, e = ioutil.ReadAll(rc.Body)
+	content, e = io.ReadAll(rc.Body)
 	return
 }
 

weed/command/filer.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"net/http"
 	"os"
-	"strings"
 	"time"
 
 	"google.golang.org/grpc/reflection"
@@ -46,7 +45,6 @@ type FilerOptions struct {
 	enableNotification      *bool
 	disableHttp             *bool
 	cipher                  *bool
-	peers                   *string
 	metricsHttpPort         *int
 	saveToFilerLimit        *int
 	defaultLevelDbDirectory *string
@@ -72,7 +70,6 @@ func init() {
 	f.rack = cmdFiler.Flag.String("rack", "", "prefer to write to volumes in this rack")
 	f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed")
 	f.cipher = cmdFiler.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers")
-	f.peers = cmdFiler.Flag.String("peers", "", "all filers sharing the same filer store in comma separated ip:port list")
 	f.metricsHttpPort = cmdFiler.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
 	f.saveToFilerLimit = cmdFiler.Flag.Int("saveToFilerLimit", 0, "files smaller than this limit will be saved in filer store")
 	f.defaultLevelDbDirectory = cmdFiler.Flag.String("defaultStoreDir", ".", "if filer.toml is empty, use an embedded filer store in the directory")
@@ -186,11 +183,6 @@ func (fo *FilerOptions) startFiler() {
 
 	defaultLevelDbDirectory := util.ResolvePath(*fo.defaultLevelDbDirectory + "/filerldb2")
 
-	var peers []string
-	if *fo.peers != "" {
-		peers = strings.Split(*fo.peers, ",")
-	}
-
 	filerAddress := pb.NewServerAddress(*fo.ip, *fo.port, *fo.portGrpc)
 
 	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
@@ -207,7 +199,6 @@ func (fo *FilerOptions) startFiler() {
 		Host:                  filerAddress,
 		Cipher:                *fo.cipher,
 		SaveToFilerLimit:      int64(*fo.saveToFilerLimit),
-		Filers:                pb.FromAddressStrings(peers),
 		ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
 	})
 	if nfs_err != nil {

weed/command/filer_copy.go

@@ -3,9 +3,7 @@ package command
 import (
 	"context"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/filer"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -16,14 +14,14 @@ import (
 
 	"google.golang.org/grpc"
 
-	"github.com/chrislusf/seaweedfs/weed/util/grace"
-
+	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/util/grace"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 )
 
@@ -212,7 +210,7 @@ func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan Fi
 	}
 
 	if mode.IsDir() {
-		files, _ := ioutil.ReadDir(fileOrDir)
+		files, _ := os.ReadDir(fileOrDir)
 		for _, subFileOrDir := range files {
 			cleanedDestDirectory := filepath.Clean(destPath + fi.Name())
 			if err = genFileCopyTask(fileOrDir+"/"+subFileOrDir.Name(), cleanedDestDirectory+"/", fileCopyTaskChan); err != nil {
@@ -339,7 +337,7 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
 	if task.fileMode&os.ModeDir == 0 && task.fileSize > 0 {
 
 		mimeType = detectMimeType(f)
-		data, err := ioutil.ReadAll(f)
+		data, err := io.ReadAll(f)
 		if err != nil {
 			return err
 		}

weed/command/filer_meta_tail.go

@@ -30,6 +30,8 @@ var cmdFilerMetaTail = &Command{
 	weed filer.meta.tail -timeAgo=30h | jq .
 	weed filer.meta.tail -timeAgo=30h | jq .eventNotification.newEntry.name
 
+	weed filer.meta.tail -timeAgo=30h -es=http://<elasticSearchServerHost>:<port> -es.index=seaweedfs
+
   `,
 }
 

weed/command/scaffold.go

@@ -2,9 +2,10 @@ package command
 
 import (
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/command/scaffold"
-	"io/ioutil"
+	"os"
 	"path/filepath"
+
+	"github.com/chrislusf/seaweedfs/weed/command/scaffold"
 )
 
 func init() {
@@ -55,7 +56,7 @@ func runScaffold(cmd *Command, args []string) bool {
 	}
 
 	if *outputPath != "" {
-		ioutil.WriteFile(filepath.Join(*outputPath, *config+".toml"), []byte(content), 0644)
+		os.WriteFile(filepath.Join(*outputPath, *config+".toml"), []byte(content), 0644)
 	} else {
 		fmt.Println(content)
 	}

weed/command/server.go

@@ -106,7 +106,6 @@ func init() {
 	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 4, "split files larger than the limit")
 	filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
 	filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers")
-	filerOptions.peers = cmdServer.Flag.String("filer.peers", "", "all filers sharing the same filer store in comma separated ip:port list")
 	filerOptions.saveToFilerLimit = cmdServer.Flag.Int("filer.saveToFilerLimit", 0, "Small files smaller than this limit can be cached in filer store.")
 	filerOptions.concurrentUploadLimitMB = cmdServer.Flag.Int("filer.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
 

weed/command/shell.go

@@ -37,7 +37,7 @@ func runShell(command *Command, args []string) bool {
 	util.LoadConfiguration("security", false)
 	shellOptions.GrpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
 
-	if *shellOptions.Masters == "" && *shellInitialFiler == "" {
+	if *shellOptions.Masters == "" {
 		util.LoadConfiguration("shell", false)
 		v := util.GetViper()
 		cluster := v.GetString("cluster.default")
@@ -45,15 +45,14 @@ func runShell(command *Command, args []string) bool {
 			cluster = *shellCluster
 		}
 		if cluster == "" {
-			*shellOptions.Masters, *shellInitialFiler = "localhost:9333", "localhost:8888"
+			*shellOptions.Masters = "localhost:9333"
 		} else {
 			*shellOptions.Masters = v.GetString("cluster." + cluster + ".master")
 			*shellInitialFiler = v.GetString("cluster." + cluster + ".filer")
+			fmt.Printf("master: %s filer: %s\n", *shellOptions.Masters, *shellInitialFiler)
 		}
 	}
 
-	fmt.Printf("master: %s filer: %s\n", *shellOptions.Masters, *shellInitialFiler)
-
 	shellOptions.FilerAddress = pb.ServerAddress(*shellInitialFiler)
 	shellOptions.Directory = "/"
 

weed/election/cluster.go

@@ -0,0 +1,207 @@
+package election
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+	"math"
+	"sync"
+	"time"
+)
+
+type ClusterNode struct {
+	Address   pb.ServerAddress
+	Version   string
+	counter   int
+	createdTs time.Time
+}
+
+type Leaders struct {
+	leaders [3]pb.ServerAddress
+}
+
+type Cluster struct {
+	nodes     map[pb.ServerAddress]*ClusterNode
+	nodesLock sync.RWMutex
+	leaders   *Leaders
+}
+
+func NewCluster() *Cluster {
+	return &Cluster{
+		nodes:   make(map[pb.ServerAddress]*ClusterNode),
+		leaders: &Leaders{},
+	}
+}
+
+func (cluster *Cluster) AddClusterNode(nodeType string, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
+	switch nodeType {
+	case "filer":
+		cluster.nodesLock.Lock()
+		defer cluster.nodesLock.Unlock()
+		if existingNode, found := cluster.nodes[address]; found {
+			existingNode.counter++
+			return nil
+		}
+		cluster.nodes[address] = &ClusterNode{
+			Address:   address,
+			Version:   version,
+			counter:   1,
+			createdTs: time.Now(),
+		}
+		return cluster.ensureLeader(true, nodeType, address)
+	case "master":
+	}
+	return nil
+}
+
+func (cluster *Cluster) RemoveClusterNode(nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
+	switch nodeType {
+	case "filer":
+		cluster.nodesLock.Lock()
+		defer cluster.nodesLock.Unlock()
+		if existingNode, found := cluster.nodes[address]; !found {
+			return nil
+		} else {
+			existingNode.counter--
+			if existingNode.counter <= 0 {
+				delete(cluster.nodes, address)
+				return cluster.ensureLeader(false, nodeType, address)
+			}
+		}
+	case "master":
+	}
+	return nil
+}
+
+func (cluster *Cluster) ListClusterNode(nodeType string) (nodes []*ClusterNode) {
+	switch nodeType {
+	case "filer":
+		cluster.nodesLock.RLock()
+		defer cluster.nodesLock.RUnlock()
+		for _, node := range cluster.nodes {
+			nodes = append(nodes, node)
+		}
+	case "master":
+	}
+	return
+}
+
+func (cluster *Cluster) IsOneLeader(address pb.ServerAddress) bool {
+	return cluster.leaders.isOneLeader(address)
+}
+
+func (cluster *Cluster) ensureLeader(isAdd bool, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
+	if isAdd {
+		if cluster.leaders.addLeaderIfVacant(address) {
+			// has added the address as one leader
+			result = append(result, &master_pb.KeepConnectedResponse{
+				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+					NodeType: nodeType,
+					Address:  string(address),
+					IsLeader: true,
+					IsAdd:    true,
+				},
+			})
+		} else {
+			result = append(result, &master_pb.KeepConnectedResponse{
+				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+					NodeType: nodeType,
+					Address:  string(address),
+					IsLeader: false,
+					IsAdd:    true,
+				},
+			})
+		}
+	} else {
+		if cluster.leaders.removeLeaderIfExists(address) {
+
+			result = append(result, &master_pb.KeepConnectedResponse{
+				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+					NodeType: nodeType,
+					Address:  string(address),
+					IsLeader: true,
+					IsAdd:    false,
+				},
+			})
+
+			// pick the freshest one, since it is less likely to go away
+			var shortestDuration int64 = math.MaxInt64
+			now := time.Now()
+			var candidateAddress pb.ServerAddress
+			for _, node := range cluster.nodes {
+				if cluster.leaders.isOneLeader(node.Address) {
+					continue
+				}
+				duration := now.Sub(node.createdTs).Nanoseconds()
+				if duration < shortestDuration {
+					shortestDuration = duration
+					candidateAddress = node.Address
+				}
+			}
+			if candidateAddress != "" {
+				cluster.leaders.addLeaderIfVacant(candidateAddress)
+				// added a new leader
+				result = append(result, &master_pb.KeepConnectedResponse{
+					ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+						NodeType: nodeType,
+						Address:  string(candidateAddress),
+						IsLeader: true,
+						IsAdd:    true,
+					},
+				})
+			}
+		} else {
+			result = append(result, &master_pb.KeepConnectedResponse{
+				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+					NodeType: nodeType,
+					Address:  string(address),
+					IsLeader: false,
+					IsAdd:    false,
+				},
+			})
+		}
+	}
+	return
+}
+
+func (leaders *Leaders) addLeaderIfVacant(address pb.ServerAddress) (hasChanged bool) {
+	if leaders.isOneLeader(address) {
+		return
+	}
+	for i := 0; i < len(leaders.leaders); i++ {
+		if leaders.leaders[i] == "" {
+			leaders.leaders[i] = address
+			hasChanged = true
+			return
+		}
+	}
+	return
+}
+func (leaders *Leaders) removeLeaderIfExists(address pb.ServerAddress) (hasChanged bool) {
+	if !leaders.isOneLeader(address) {
+		return
+	}
+	for i := 0; i < len(leaders.leaders); i++ {
+		if leaders.leaders[i] == address {
+			leaders.leaders[i] = ""
+			hasChanged = true
+			return
+		}
+	}
+	return
+}
+func (leaders *Leaders) isOneLeader(address pb.ServerAddress) bool {
+	for i := 0; i < len(leaders.leaders); i++ {
+		if leaders.leaders[i] == address {
+			return true
+		}
+	}
+	return false
+}
+func (leaders *Leaders) GetLeaders() (addresses []pb.ServerAddress) {
+	for i := 0; i < len(leaders.leaders); i++ {
+		if leaders.leaders[i] != "" {
+			addresses = append(addresses, leaders.leaders[i])
+		}
+	}
+	return
+}

weed/election/cluster_test.go

@@ -0,0 +1,47 @@
+package election
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestClusterAddRemoveNodes(t *testing.T) {
+	c := NewCluster()
+
+	c.AddClusterNode("filer", pb.ServerAddress("111:1"), "23.45")
+	c.AddClusterNode("filer", pb.ServerAddress("111:2"), "23.45")
+	assert.Equal(t, []pb.ServerAddress{
+		pb.ServerAddress("111:1"),
+		pb.ServerAddress("111:2"),
+	}, c.leaders.GetLeaders())
+
+	c.AddClusterNode("filer", pb.ServerAddress("111:3"), "23.45")
+	c.AddClusterNode("filer", pb.ServerAddress("111:4"), "23.45")
+	assert.Equal(t, []pb.ServerAddress{
+		pb.ServerAddress("111:1"),
+		pb.ServerAddress("111:2"),
+		pb.ServerAddress("111:3"),
+	}, c.leaders.GetLeaders())
+
+	c.AddClusterNode("filer", pb.ServerAddress("111:5"), "23.45")
+	c.AddClusterNode("filer", pb.ServerAddress("111:6"), "23.45")
+	c.RemoveClusterNode("filer", pb.ServerAddress("111:4"))
+	assert.Equal(t, []pb.ServerAddress{
+		pb.ServerAddress("111:1"),
+		pb.ServerAddress("111:2"),
+		pb.ServerAddress("111:3"),
+	}, c.leaders.GetLeaders())
+
+	// remove oldest
+	c.RemoveClusterNode("filer", pb.ServerAddress("111:1"))
+	assert.Equal(t, []pb.ServerAddress{
+		pb.ServerAddress("111:6"),
+		pb.ServerAddress("111:2"),
+		pb.ServerAddress("111:3"),
+	}, c.leaders.GetLeaders())
+
+	// remove oldest
+	c.RemoveClusterNode("filer", pb.ServerAddress("111:1"))
+
+}

weed/filer.toml

@@ -1,5 +0,0 @@
-[redis3]
-enabled = true
-address = "localhost:6379"
-password = ""
-database = 0

weed/filer/filechunk_manifest.go

@@ -6,6 +6,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 	"io"
 	"math"
+	"math/rand"
 	"net/url"
 	"strings"
 	"time"
@@ -142,6 +143,9 @@ func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, cipherKe
 	var shouldRetry bool
 	var totalWritten int
 
+	rand.Shuffle(len(urlStrings), func(i, j int) {
+		urlStrings[i], urlStrings[j] = urlStrings[j], urlStrings[i]
+	})
 	for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
 		for _, urlString := range urlStrings {
 			var localProcesed int

weed/filer/filechunks.go

@@ -225,6 +225,12 @@ func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunction
 
 	chunks, _, err = ResolveChunkManifest(lookupFileIdFn, chunks, startOffset, stopOffset)
 
+	visibles2 := readResolvedChunks(chunks)
+
+	if true {
+		return visibles2, err
+	}
+
 	sort.Slice(chunks, func(i, j int) bool {
 		if chunks[i].Mtime == chunks[j].Mtime {
 			filer_pb.EnsureFid(chunks[i])
@@ -246,9 +252,26 @@ func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunction
 
 	}
 
+	if len(visibles) != len(visibles2) {
+		fmt.Printf("different visibles size %d : %d\n", len(visibles), len(visibles2))
+	} else {
+		for i := 0; i < len(visibles); i++ {
+			checkDifference(visibles[i], visibles2[i])
+		}
+	}
+
 	return
 }
 
+func checkDifference(x, y VisibleInterval) {
+	if x.start != y.start ||
+		x.stop != y.stop ||
+		x.fileId != y.fileId ||
+		x.modifiedTime != y.modifiedTime {
+		fmt.Printf("different visible %+v : %+v\n", x, y)
+	}
+}
+
 // find non-overlapping visible intervals
 // visible interval map to one file chunk
 

weed/filer/filechunks_read.go

@@ -0,0 +1,119 @@
+package filer
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"sort"
+)
+
+func readResolvedChunks(chunks []*filer_pb.FileChunk) (visibles []VisibleInterval) {
+
+	var points []*Point
+	for _, chunk := range chunks {
+		points = append(points, &Point{
+			x:       chunk.Offset,
+			ts:      chunk.Mtime,
+			chunk:   chunk,
+			isStart: true,
+		})
+		points = append(points, &Point{
+			x:       chunk.Offset + int64(chunk.Size),
+			ts:      chunk.Mtime,
+			chunk:   chunk,
+			isStart: false,
+		})
+	}
+	sort.Slice(points, func(i, j int) bool {
+		if points[i].x != points[j].x {
+			return points[i].x < points[j].x
+		}
+		if points[i].ts != points[j].ts {
+			return points[i].ts < points[j].ts
+		}
+		if !points[i].isStart {
+			return true
+		}
+		return false
+	})
+
+	var prevX int64
+	var queue []*Point
+	for _, point := range points {
+		if point.isStart {
+			if len(queue) > 0 {
+				lastIndex := len(queue) - 1
+				lastPoint := queue[lastIndex]
+				if point.x != prevX && lastPoint.ts < point.ts {
+					visibles = addToVisibles(visibles, prevX, lastPoint, point)
+					prevX = point.x
+				}
+			}
+			// insert into queue
+			for i := len(queue); i >= 0; i-- {
+				if i == 0 || queue[i-1].ts <= point.ts {
+					if i == len(queue) {
+						prevX = point.x
+					}
+					queue = addToQueue(queue, i, point)
+					break
+				}
+			}
+		} else {
+			lastIndex := len(queue) - 1
+			index := lastIndex
+			var startPoint *Point
+			for ; index >= 0; index-- {
+				startPoint = queue[index]
+				if startPoint.ts == point.ts {
+					queue = removeFromQueue(queue, index)
+					break
+				}
+			}
+			if index == lastIndex && startPoint != nil {
+				visibles = addToVisibles(visibles, prevX, startPoint, point)
+				prevX = point.x
+			}
+		}
+	}
+
+	return
+}
+
+func removeFromQueue(queue []*Point, index int) []*Point {
+	for i := index; i < len(queue)-1; i++ {
+		queue[i] = queue[i+1]
+	}
+	queue = queue[:len(queue)-1]
+	return queue
+}
+
+func addToQueue(queue []*Point, index int, point *Point) []*Point {
+	queue = append(queue, point)
+	for i := len(queue) - 1; i > index; i-- {
+		queue[i], queue[i-1] = queue[i-1], queue[i]
+	}
+	return queue
+}
+
+func addToVisibles(visibles []VisibleInterval, prevX int64, startPoint *Point, point *Point) []VisibleInterval {
+	if prevX < point.x {
+		chunk := startPoint.chunk
+		visibles = append(visibles, VisibleInterval{
+			start:        prevX,
+			stop:         point.x,
+			fileId:       chunk.GetFileIdString(),
+			modifiedTime: chunk.Mtime,
+			chunkOffset:  prevX - chunk.Offset,
+			chunkSize:    chunk.Size,
+			cipherKey:    chunk.CipherKey,
+			isGzipped:    chunk.IsCompressed,
+		})
+	}
+	return visibles
+}
+
+type Point struct {
+	x       int64
+	ts      int64
+	chunk   *filer_pb.FileChunk
+	isStart bool
+}

weed/filer/filechunks_read_test.go

@@ -0,0 +1,210 @@
+package filer
+
+import (
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"math/rand"
+	"testing"
+)
+
+func TestReadResolvedChunks(t *testing.T) {
+
+	chunks := []*filer_pb.FileChunk{
+		{
+			FileId: "a",
+			Offset: 0,
+			Size:   100,
+			Mtime:  1,
+		},
+		{
+			FileId: "b",
+			Offset: 50,
+			Size:   100,
+			Mtime:  2,
+		},
+		{
+			FileId: "c",
+			Offset: 200,
+			Size:   50,
+			Mtime:  3,
+		},
+		{
+			FileId: "d",
+			Offset: 250,
+			Size:   50,
+			Mtime:  4,
+		},
+		{
+			FileId: "e",
+			Offset: 175,
+			Size:   100,
+			Mtime:  5,
+		},
+	}
+
+	visibles := readResolvedChunks(chunks)
+
+	for _, visible := range visibles {
+		fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTime)
+	}
+
+}
+
+func TestRandomizedReadResolvedChunks(t *testing.T) {
+
+	var limit int64 = 1024 * 1024
+	array := make([]int64, limit)
+	var chunks []*filer_pb.FileChunk
+	for ts := int64(0); ts < 1024; ts++ {
+		x := rand.Int63n(limit)
+		y := rand.Int63n(limit)
+		size := x - y
+		if size < 0 {
+			size = -size
+		}
+		if size > 1024 {
+			size = 1024
+		}
+		start := x
+		if start > y {
+			start = y
+		}
+		chunks = append(chunks, randomWrite(array, start, size, ts))
+	}
+
+	visibles := readResolvedChunks(chunks)
+
+	for _, visible := range visibles {
+		for i := visible.start; i < visible.stop; i++ {
+			if array[i] != visible.modifiedTime {
+				t.Errorf("position %d expected ts %d actual ts %d", i, array[i], visible.modifiedTime)
+			}
+		}
+	}
+
+	// fmt.Printf("visibles %d", len(visibles))
+
+}
+
+func randomWrite(array []int64, start int64, size int64, ts int64) *filer_pb.FileChunk {
+	for i := start; i < start+size; i++ {
+		array[i] = ts
+	}
+	// fmt.Printf("write [%d,%d) %d\n", start, start+size, ts)
+	return &filer_pb.FileChunk{
+		FileId: "",
+		Offset: start,
+		Size:   uint64(size),
+		Mtime:  ts,
+	}
+}
+
+func TestSequentialReadResolvedChunks(t *testing.T) {
+
+	var chunkSize int64 = 1024 * 1024 * 2
+	var chunks []*filer_pb.FileChunk
+	for ts := int64(0); ts < 13; ts++ {
+		chunks = append(chunks, &filer_pb.FileChunk{
+			FileId: "",
+			Offset: chunkSize * ts,
+			Size:   uint64(chunkSize),
+			Mtime:  1,
+		})
+	}
+
+	visibles := readResolvedChunks(chunks)
+
+	fmt.Printf("visibles %d", len(visibles))
+
+}
+
+func TestActualReadResolvedChunks(t *testing.T) {
+
+	chunks := []*filer_pb.FileChunk{
+		{
+			FileId: "5,e7b96fef48",
+			Offset: 0,
+			Size:   2097152,
+			Mtime:  1634447487595823000,
+		},
+		{
+			FileId: "5,e5562640b9",
+			Offset: 2097152,
+			Size:   2097152,
+			Mtime:  1634447487595826000,
+		},
+		{
+			FileId: "5,df033e0fe4",
+			Offset: 4194304,
+			Size:   2097152,
+			Mtime:  1634447487595827000,
+		},
+		{
+			FileId: "7,eb08148a9b",
+			Offset: 6291456,
+			Size:   2097152,
+			Mtime:  1634447487595827000,
+		},
+		{
+			FileId: "7,e0f92d1604",
+			Offset: 8388608,
+			Size:   2097152,
+			Mtime:  1634447487595828000,
+		},
+		{
+			FileId: "7,e33cb63262",
+			Offset: 10485760,
+			Size:   2097152,
+			Mtime:  1634447487595828000,
+		},
+		{
+			FileId: "5,ea98e40e93",
+			Offset: 12582912,
+			Size:   2097152,
+			Mtime:  1634447487595829000,
+		},
+		{
+			FileId: "5,e165661172",
+			Offset: 14680064,
+			Size:   2097152,
+			Mtime:  1634447487595829000,
+		},
+		{
+			FileId: "3,e692097486",
+			Offset: 16777216,
+			Size:   2097152,
+			Mtime:  1634447487595830000,
+		},
+		{
+			FileId: "3,e28e2e3cbd",
+			Offset: 18874368,
+			Size:   2097152,
+			Mtime:  1634447487595830000,
+		},
+		{
+			FileId: "3,e443974d4e",
+			Offset: 20971520,
+			Size:   2097152,
+			Mtime:  1634447487595830000,
+		},
+		{
+			FileId: "2,e815bed597",
+			Offset: 23068672,
+			Size:   2097152,
+			Mtime:  1634447487595831000,
+		},
+		{
+			FileId: "5,e94715199e",
+			Offset: 25165824,
+			Size:   1974736,
+			Mtime:  1634447487595832000,
+		},
+	}
+
+	visibles := readResolvedChunks(chunks)
+
+	for _, visible := range visibles {
+		fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTime)
+	}
+
+}

weed/filer/filer.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"os"
 	"strings"
 	"time"
@@ -66,22 +67,38 @@ func NewFiler(masters []pb.ServerAddress, grpcDialOption grpc.DialOption,
 	return f
 }
 
-func (f *Filer) AggregateFromPeers(self pb.ServerAddress, filers []pb.ServerAddress) {
+func (f *Filer) AggregateFromPeers(self pb.ServerAddress) {
 
-	// set peers
-	found := false
-	for _, peer := range filers {
-		if peer == self {
-			found = true
+	f.MetaAggregator = NewMetaAggregator(f, self, f.GrpcDialOption)
+	f.MasterClient.OnPeerUpdate = f.MetaAggregator.OnPeerUpdate
+
+	for _, peerUpdate := range f.ListExistingPeerUpdates() {
+		f.MetaAggregator.OnPeerUpdate(peerUpdate)
+	}
+
+}
+
+func (f *Filer) ListExistingPeerUpdates() (existingNodes []*master_pb.ClusterNodeUpdate){
+
+	if grpcErr := pb.WithMasterClient(f.MasterClient.GetMaster(), f.GrpcDialOption, func(client master_pb.SeaweedClient) error {
+		resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
+			ClientType: "filer",
+		})
+
+		glog.V(0).Infof("the cluster has %d filers\n", len(resp.ClusterNodes))
+		for _, node := range resp.ClusterNodes {
+			existingNodes = append(existingNodes, &master_pb.ClusterNodeUpdate{
+				NodeType: "filer",
+				Address:  node.Address,
+				IsLeader: node.IsLeader,
+				IsAdd:    true,
+			})
 		}
+		return err
+	}); grpcErr != nil {
+		glog.V(0).Infof("connect to %s: %v", f.MasterClient.GetMaster(), grpcErr)
 	}
-	if !found {
-		filers = append(filers, self)
-	}
-
-	f.MetaAggregator = NewMetaAggregator(filers, f.GrpcDialOption)
-	f.MetaAggregator.StartLoopSubscribe(f, self)
-
+	return
 }
 
 func (f *Filer) SetStore(store FilerStore) {
@@ -117,7 +134,7 @@ func (fs *Filer) GetMaster() pb.ServerAddress {
 	return fs.MasterClient.GetMaster()
 }
 
-func (fs *Filer) KeepConnectedToMaster() {
+func (fs *Filer) KeepMasterClientConnected() {
 	fs.MasterClient.KeepConnectedToMaster()
 }
 

weed/filer/filer_conf.go

@@ -3,6 +3,10 @@ package filer
 import (
 	"bytes"
 	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/wdclient"
+	"google.golang.org/grpc"
 	"io"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -26,6 +30,29 @@ type FilerConf struct {
 	rules ptrie.Trie
 }
 
+func ReadFilerConf(filerGrpcAddress pb.ServerAddress, grpcDialOption grpc.DialOption, masterClient *wdclient.MasterClient) (*FilerConf, error) {
+	var buf bytes.Buffer
+	if err := pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		if masterClient != nil {
+			return ReadEntry(masterClient, client, DirectoryEtcSeaweedFS, FilerConfName, &buf)
+		} else {
+			content, err := ReadInsideFiler(client, DirectoryEtcSeaweedFS, FilerConfName)
+			buf = *bytes.NewBuffer(content)
+			return err
+		}
+	}); err != nil && err != filer_pb.ErrNotFound {
+		return nil, fmt.Errorf("read %s/%s: %v", DirectoryEtcSeaweedFS, FilerConfName, err)
+	}
+
+	fc := NewFilerConf()
+	if buf.Len() > 0 {
+		if err := fc.LoadFromBytes(buf.Bytes()); err != nil {
+			return nil, fmt.Errorf("parse %s/%s: %v", DirectoryEtcSeaweedFS, FilerConfName, err)
+		}
+	}
+	return fc, nil
+}
+
 func NewFilerConf() (fc *FilerConf) {
 	fc = &FilerConf{
 		rules: ptrie.New(),
@@ -115,6 +142,18 @@ func (fc *FilerConf) MatchStorageRule(path string) (pathConf *filer_pb.FilerConf
 	return pathConf
 }
 
+func (fc *FilerConf) GetCollectionTtls(collection string) (ttls map[string]string) {
+	ttls = make(map[string]string)
+	fc.rules.Walk(func(key []byte, value interface{}) bool {
+		t := value.(*filer_pb.FilerConf_PathConf)
+		if t.Collection == collection {
+			ttls[t.LocationPrefix] = t.GetTtl()
+		}
+		return true
+	})
+	return ttls
+}
+
 // merge if values in b is not empty, merge them into a
 func mergePathConf(a, b *filer_pb.FilerConf_PathConf) {
 	a.Collection = util.Nvl(b.Collection, a.Collection)

weed/filer/filer_delete_entry.go

@@ -15,6 +15,9 @@ const (
 	MsgFailDelNonEmptyFolder = "fail to delete non-empty folder"
 )
 
+type OnChunksFunc func([]*filer_pb.FileChunk) error
+type OnHardLinkIdsFunc func([]HardLinkId) error
+
 func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isFromOtherCluster bool, signatures []int32) (err error) {
 	if p == "/" {
 		return nil
@@ -27,20 +30,29 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 
 	isDeleteCollection := f.isBucket(entry)
 
-	var chunks []*filer_pb.FileChunk
-	var hardLinkIds []HardLinkId
-	chunks = append(chunks, entry.Chunks...)
 	if entry.IsDirectory() {
 		// delete the folder children, not including the folder itself
-		var dirChunks []*filer_pb.FileChunk
-		var dirHardLinkIds []HardLinkId
-		dirChunks, dirHardLinkIds, err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks && !isDeleteCollection, isDeleteCollection, isFromOtherCluster, signatures)
+		err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks && !isDeleteCollection, isDeleteCollection, isFromOtherCluster, signatures, func(chunks []*filer_pb.FileChunk) error {
+			if shouldDeleteChunks && !isDeleteCollection {
+				f.DirectDeleteChunks(chunks)
+			}
+			return nil
+		}, func(hardLinkIds []HardLinkId) error {
+			// A case not handled:
+			// what if the chunk is in a different collection?
+			if shouldDeleteChunks {
+				f.maybeDeleteHardLinks(hardLinkIds)
+			}
+			return nil
+		})
 		if err != nil {
 			glog.V(0).Infof("delete directory %s: %v", p, err)
 			return fmt.Errorf("delete directory %s: %v", p, err)
 		}
-		chunks = append(chunks, dirChunks...)
-		hardLinkIds = append(hardLinkIds, dirHardLinkIds...)
+	}
+
+	if shouldDeleteChunks && !isDeleteCollection {
+		f.DirectDeleteChunks(entry.Chunks)
 	}
 
 	// delete the file or folder
@@ -49,15 +61,6 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 		return fmt.Errorf("delete file %s: %v", p, err)
 	}
 
-	if shouldDeleteChunks && !isDeleteCollection {
-		f.DirectDeleteChunks(chunks)
-	}
-	// A case not handled:
-	// what if the chunk is in a different collection?
-	if shouldDeleteChunks {
-		f.maybeDeleteHardLinks(hardLinkIds)
-	}
-
 	if isDeleteCollection {
 		collectionName := entry.Name()
 		f.doDeleteCollection(collectionName)
@@ -67,7 +70,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 	return nil
 }
 
-func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isDeletingBucket, isFromOtherCluster bool, signatures []int32) (chunks []*filer_pb.FileChunk, hardlinkIds []HardLinkId, err error) {
+func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isDeletingBucket, isFromOtherCluster bool, signatures []int32, onChunksFn OnChunksFunc, onHardLinkIdsFn OnHardLinkIdsFunc) (err error) {
 
 	lastFileName := ""
 	includeLastFile := false
@@ -76,34 +79,30 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
 			entries, _, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "", "")
 			if err != nil {
 				glog.Errorf("list folder %s: %v", entry.FullPath, err)
-				return nil, nil, fmt.Errorf("list folder %s: %v", entry.FullPath, err)
+				return fmt.Errorf("list folder %s: %v", entry.FullPath, err)
 			}
 			if lastFileName == "" && !isRecursive && len(entries) > 0 {
 				// only for first iteration in the loop
-				glog.Errorf("deleting a folder %s has children: %+v ...", entry.FullPath, entries[0].Name())
-				return nil, nil, fmt.Errorf("%s: %s", MsgFailDelNonEmptyFolder, entry.FullPath)
+				glog.V(0).Infof("deleting a folder %s has children: %+v ...", entry.FullPath, entries[0].Name())
+				return fmt.Errorf("%s: %s", MsgFailDelNonEmptyFolder, entry.FullPath)
 			}
 
 			for _, sub := range entries {
 				lastFileName = sub.Name()
-				var dirChunks []*filer_pb.FileChunk
-				var dirHardLinkIds []HardLinkId
 				if sub.IsDirectory() {
 					subIsDeletingBucket := f.isBucket(sub)
-					dirChunks, dirHardLinkIds, err = f.doBatchDeleteFolderMetaAndData(ctx, sub, isRecursive, ignoreRecursiveError, shouldDeleteChunks, subIsDeletingBucket, false, nil)
-					chunks = append(chunks, dirChunks...)
-					hardlinkIds = append(hardlinkIds, dirHardLinkIds...)
+					err = f.doBatchDeleteFolderMetaAndData(ctx, sub, isRecursive, ignoreRecursiveError, shouldDeleteChunks, subIsDeletingBucket, false, nil, onChunksFn, onHardLinkIdsFn)
 				} else {
 					f.NotifyUpdateEvent(ctx, sub, nil, shouldDeleteChunks, isFromOtherCluster, nil)
 					if len(sub.HardLinkId) != 0 {
 						// hard link chunk data are deleted separately
-						hardlinkIds = append(hardlinkIds, sub.HardLinkId)
+						err = onHardLinkIdsFn([]HardLinkId{sub.HardLinkId})
 					} else {
-						chunks = append(chunks, sub.Chunks...)
+						err = onChunksFn(sub.Chunks)
 					}
 				}
 				if err != nil && !ignoreRecursiveError {
-					return nil, nil, err
+					return err
 				}
 			}
 
@@ -113,15 +112,15 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
 		}
 	}
 
-	glog.V(3).Infof("deleting directory %v delete %d chunks: %v", entry.FullPath, len(chunks), shouldDeleteChunks)
+	glog.V(3).Infof("deleting directory %v delete chunks: %v", entry.FullPath, shouldDeleteChunks)
 
 	if storeDeletionErr := f.Store.DeleteFolderChildren(ctx, entry.FullPath); storeDeletionErr != nil {
-		return nil, nil, fmt.Errorf("filer store delete: %v", storeDeletionErr)
+		return fmt.Errorf("filer store delete: %v", storeDeletionErr)
 	}
 
 	f.NotifyUpdateEvent(ctx, entry, nil, shouldDeleteChunks, isFromOtherCluster, signatures)
 
-	return chunks, hardlinkIds, nil
+	return nil
 }
 
 func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shouldDeleteChunks bool, isFromOtherCluster bool, signatures []int32) (err error) {

weed/filer/filerstore_wrapper.go

@@ -253,13 +253,14 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context,
 		limit = math.MaxInt32 - 1
 	}
 	glog.V(4).Infof("ListDirectoryPrefixedEntries %s from %s prefix %s limit %d", dirPath, startFileName, prefix, limit)
-	lastFileName, err = actualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, eachEntryFunc)
+	adjustedEntryFunc := func(entry *Entry) bool {
+		fsw.maybeReadHardLink(ctx, entry)
+		filer_pb.AfterEntryDeserialization(entry.Chunks)
+		return eachEntryFunc(entry)
+	}
+	lastFileName, err = actualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, adjustedEntryFunc)
 	if err == ErrUnsupportedListDirectoryPrefixed {
-		lastFileName, err = fsw.prefixFilterEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, func(entry *Entry) bool {
-			fsw.maybeReadHardLink(ctx, entry)
-			filer_pb.AfterEntryDeserialization(entry.Chunks)
-			return eachEntryFunc(entry)
-		})
+		lastFileName, err = fsw.prefixFilterEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, adjustedEntryFunc)
 	}
 	return lastFileName, err
 }
@@ -295,7 +296,7 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u
 		}
 		if count < limit {
 			notPrefixed = notPrefixed[:0]
-			_, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit, func(entry *Entry) bool {
+			lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit, func(entry *Entry) bool {
 				notPrefixed = append(notPrefixed, entry)
 				return true
 			})

weed/filer/leveldb/leveldb_store_test.go

@@ -3,7 +3,6 @@ package leveldb
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"testing"
 	"time"
@@ -14,7 +13,7 @@ import (
 
 func TestCreateAndFind(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test")
 	defer os.RemoveAll(dir)
 	store := &LevelDBStore{}
 	store.initialize(dir)
@@ -68,7 +67,7 @@ func TestCreateAndFind(t *testing.T) {
 
 func TestEmptyRoot(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test2")
 	defer os.RemoveAll(dir)
 	store := &LevelDBStore{}
 	store.initialize(dir)
@@ -91,7 +90,7 @@ func TestEmptyRoot(t *testing.T) {
 
 func BenchmarkInsertEntry(b *testing.B) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_bench")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_bench")
 	defer os.RemoveAll(dir)
 	store := &LevelDBStore{}
 	store.initialize(dir)

weed/filer/leveldb2/leveldb2_store_test.go

@@ -2,7 +2,6 @@ package leveldb
 
 import (
 	"context"
-	"io/ioutil"
 	"os"
 	"testing"
 
@@ -12,7 +11,7 @@ import (
 
 func TestCreateAndFind(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test")
 	defer os.RemoveAll(dir)
 	store := &LevelDB2Store{}
 	store.initialize(dir, 2)
@@ -66,7 +65,7 @@ func TestCreateAndFind(t *testing.T) {
 
 func TestEmptyRoot(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test2")
 	defer os.RemoveAll(dir)
 	store := &LevelDB2Store{}
 	store.initialize(dir, 2)

weed/filer/leveldb3/leveldb3_store_test.go

@@ -2,7 +2,6 @@ package leveldb
 
 import (
 	"context"
-	"io/ioutil"
 	"os"
 	"testing"
 
@@ -12,7 +11,7 @@ import (
 
 func TestCreateAndFind(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test")
 	defer os.RemoveAll(dir)
 	store := &LevelDB3Store{}
 	store.initialize(dir)
@@ -66,7 +65,7 @@ func TestCreateAndFind(t *testing.T) {
 
 func TestEmptyRoot(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test2")
 	defer os.RemoveAll(dir)
 	store := &LevelDB3Store{}
 	store.initialize(dir)

weed/filer/meta_aggregator.go

@@ -3,6 +3,7 @@ package filer
 import (
 	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"io"
 	"sync"
@@ -18,9 +19,13 @@ import (
 )
 
 type MetaAggregator struct {
-	filers         []pb.ServerAddress
-	grpcDialOption grpc.DialOption
-	MetaLogBuffer  *log_buffer.LogBuffer
+	filer           *Filer
+	self            pb.ServerAddress
+	isLeader        bool
+	grpcDialOption  grpc.DialOption
+	MetaLogBuffer   *log_buffer.LogBuffer
+	peerStatues     map[pb.ServerAddress]struct{}
+	peerStatuesLock sync.Mutex
 	// notifying clients
 	ListenersLock sync.Mutex
 	ListenersCond *sync.Cond
@@ -28,10 +33,12 @@ type MetaAggregator struct {
 
 // MetaAggregator only aggregates data "on the fly". The logs are not re-persisted to disk.
 // The old data comes from what each LocalMetadata persisted on disk.
-func NewMetaAggregator(filers []pb.ServerAddress, grpcDialOption grpc.DialOption) *MetaAggregator {
+func NewMetaAggregator(filer *Filer, self pb.ServerAddress, grpcDialOption grpc.DialOption) *MetaAggregator {
 	t := &MetaAggregator{
-		filers:         filers,
+		filer:          filer,
+		self:           self,
 		grpcDialOption: grpcDialOption,
+		peerStatues:    make(map[pb.ServerAddress]struct{}),
 	}
 	t.ListenersCond = sync.NewCond(&t.ListenersLock)
 	t.MetaLogBuffer = log_buffer.NewLogBuffer("aggr", LogFlushInterval, nil, func() {
@@ -40,10 +47,35 @@ func NewMetaAggregator(filers []pb.ServerAddress, grpcDialOption grpc.DialOption
 	return t
 }
 
-func (ma *MetaAggregator) StartLoopSubscribe(f *Filer, self pb.ServerAddress) {
-	for _, filer := range ma.filers {
-		go ma.subscribeToOneFiler(f, self, filer)
+func (ma *MetaAggregator) OnPeerUpdate(update *master_pb.ClusterNodeUpdate) {
+	if update.NodeType != "filer" {
+		return
 	}
+
+	address := pb.ServerAddress(update.Address)
+	if update.IsAdd {
+		// every filer should subscribe to a new filer
+		ma.setActive(address, true)
+		go ma.subscribeToOneFiler(ma.filer, ma.self, address)
+	} else {
+		ma.setActive(address, false)
+	}
+}
+
+func (ma *MetaAggregator) setActive(address pb.ServerAddress, isActive bool) {
+	ma.peerStatuesLock.Lock()
+	defer ma.peerStatuesLock.Unlock()
+	if isActive {
+		ma.peerStatues[address] = struct{}{}
+	} else {
+		delete(ma.peerStatues, address)
+	}
+}
+func (ma *MetaAggregator) isActive(address pb.ServerAddress)(isActive bool) {
+	ma.peerStatuesLock.Lock()
+	defer ma.peerStatuesLock.Unlock()
+	_, isActive = ma.peerStatues[address]
+	return
 }
 
 func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress) {
@@ -149,6 +181,10 @@ func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self pb.ServerAddress, p
 
 			}
 		})
+		if !ma.isActive(peer) {
+			glog.V(0).Infof("stop subscribing remote %s meta change", peer)
+			return
+		}
 		if err != nil {
 			glog.V(0).Infof("subscribing remote %s meta change: %v", peer, err)
 			time.Sleep(1733 * time.Millisecond)

weed/filer/read_remote.go

@@ -25,9 +25,9 @@ func MapRemoteStorageLocationPathToFullPath(localMountedDir util.FullPath, remot
 	return localMountedDir.Child(remoteLocationPath[len(remoteMountedLocation.Path):])
 }
 
-func DownloadToLocal(filerClient filer_pb.FilerClient, remoteConf *remote_pb.RemoteConf, remoteLocation *remote_pb.RemoteStorageLocation, parent util.FullPath, entry *filer_pb.Entry) error {
+func CacheRemoteObjectToLocalCluster(filerClient filer_pb.FilerClient, remoteConf *remote_pb.RemoteConf, remoteLocation *remote_pb.RemoteStorageLocation, parent util.FullPath, entry *filer_pb.Entry) error {
 	return filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-		_, err := client.DownloadToLocal(context.Background(), &filer_pb.DownloadToLocalRequest{
+		_, err := client.CacheRemoteObjectToLocalCluster(context.Background(), &filer_pb.CacheRemoteObjectToLocalClusterRequest{
 			Directory: string(parent),
 			Name:      entry.Name,
 		})

weed/filer/redis2/universal_redis_store.go

@@ -133,7 +133,10 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful
 		return nil
 	}
 
-	members, err := store.Client.ZRange(ctx, genDirectoryListKey(string(fullpath)), 0, -1).Result()
+	members, err := store.Client.ZRangeByLex(ctx, genDirectoryListKey(string(fullpath)), &redis.ZRangeBy{
+		Min: "-",
+		Max: "+",
+	}).Result()
 	if err != nil {
 		return fmt.Errorf("DeleteFolderChildren %s : %v", fullpath, err)
 	}
@@ -158,14 +161,22 @@ func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Cont
 func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
 
 	dirListKey := genDirectoryListKey(string(dirPath))
-	start := int64(0)
+
+	min := "-"
 	if startFileName != "" {
-		start, _ = store.Client.ZRank(ctx, dirListKey, startFileName).Result()
-		if !includeStartFile {
-			start++
+		if includeStartFile {
+			min = "[" + startFileName
+		} else {
+			min = "(" + startFileName
 		}
 	}
-	members, err := store.Client.ZRange(ctx, dirListKey, start, start+int64(limit)-1).Result()
+
+	members, err := store.Client.ZRangeByLex(ctx, dirListKey, &redis.ZRangeBy{
+		Min:    min,
+		Max:    "+",
+		Offset: 0,
+		Count:  limit,
+	}).Result()
 	if err != nil {
 		return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
 	}

weed/filer/rocksdb/rocksdb_store_test.go

@@ -1,3 +1,4 @@
+//go:build rocksdb
 // +build rocksdb
 
 package rocksdb
@@ -5,7 +6,6 @@ package rocksdb
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"testing"
 	"time"
@@ -16,7 +16,7 @@ import (
 
 func TestCreateAndFind(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test")
 	defer os.RemoveAll(dir)
 	store := &RocksDBStore{}
 	store.initialize(dir)
@@ -70,7 +70,7 @@ func TestCreateAndFind(t *testing.T) {
 
 func TestEmptyRoot(t *testing.T) {
 	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_test2")
 	defer os.RemoveAll(dir)
 	store := &RocksDBStore{}
 	store.initialize(dir)
@@ -93,7 +93,7 @@ func TestEmptyRoot(t *testing.T) {
 
 func BenchmarkInsertEntry(b *testing.B) {
 	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
-	dir, _ := ioutil.TempDir("", "seaweedfs_filer_bench")
+	dir, _ := os.MkdirTemp("", "seaweedfs_filer_bench")
 	defer os.RemoveAll(dir)
 	store := &RocksDBStore{}
 	store.initialize(dir)

weed/filesys/dir_rename.go

@@ -2,12 +2,10 @@ package filesys
 
 import (
 	"context"
-	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/filer"
-	"math"
-
 	"github.com/seaweedfs/fuse"
 	"github.com/seaweedfs/fuse/fs"
+	"io"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
@@ -23,19 +21,12 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
 
 	glog.V(4).Infof("dir Rename %s => %s", oldPath, newPath)
 
-	// find local old entry
-	oldEntry, err := dir.wfs.metaCache.FindEntry(context.Background(), oldPath)
-	if err != nil {
-		glog.Errorf("dir Rename can not find source %s : %v", oldPath, err)
-		return fuse.ENOENT
-	}
-
 	// update remote filer
-	err = dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 		ctx, cancel := context.WithCancel(context.Background())
 		defer cancel()
 
-		request := &filer_pb.AtomicRenameEntryRequest{
+		request := &filer_pb.StreamRenameEntryRequest{
 			OldDirectory: dir.FullPath(),
 			OldName:      req.OldName,
 			NewDirectory: newDir.FullPath(),
@@ -43,12 +34,28 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
 			Signatures:   []int32{dir.wfs.signature},
 		}
 
-		_, err := client.AtomicRenameEntry(ctx, request)
+		stream, err := client.StreamRenameEntry(ctx, request)
 		if err != nil {
 			glog.Errorf("dir AtomicRenameEntry %s => %s : %v", oldPath, newPath, err)
 			return fuse.EXDEV
 		}
 
+		for {
+			resp, recvErr := stream.Recv()
+			if recvErr != nil {
+				if recvErr == io.EOF {
+					break
+				} else {
+					return recvErr
+				}
+			}
+
+			if err = dir.handleRenameResponse(ctx, resp); err != nil {
+				return err
+			}
+
+		}
+
 		return nil
 
 	})
@@ -57,23 +64,24 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
 		return fuse.EIO
 	}
 
-	err = dir.moveEntry(context.Background(), util.FullPath(dir.FullPath()), oldEntry, util.FullPath(newDir.FullPath()), req.NewName)
-	if err != nil {
-		glog.V(0).Infof("dir local Rename %s => %s : %v", oldPath, newPath, err)
-		return fuse.EIO
-	}
-
 	return nil
 }
 
-func (dir *Dir) moveEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string) error {
+func (dir *Dir) handleRenameResponse(ctx context.Context, resp *filer_pb.StreamRenameEntryResponse) error {
+	// comes from filer StreamRenameEntry, can only be create or delete entry
 
-	oldName := entry.Name()
+	if resp.EventNotification.NewEntry != nil {
+		// with new entry, the old entry name also exists. This is the first step to create new entry
+		newEntry := filer.FromPbEntry(resp.EventNotification.NewParentPath, resp.EventNotification.NewEntry)
+		if err := dir.wfs.metaCache.AtomicUpdateEntryFromFiler(ctx, "", newEntry); err != nil {
+			return err
+		}
 
-	oldPath := oldParent.Child(oldName)
-	newPath := newParent.Child(newName)
-	if err := dir.moveSelfEntry(ctx, oldParent, entry, newParent, newName, func() error {
+		oldParent, newParent := util.FullPath(resp.Directory), util.FullPath(resp.EventNotification.NewParentPath)
+		oldName, newName := resp.EventNotification.OldEntry.Name, resp.EventNotification.NewEntry.Name
 
+		oldPath := oldParent.Child(oldName)
+		newPath := newParent.Child(newName)
 		oldFsNode := NodeWithId(oldPath.AsInode())
 		newFsNode := NodeWithId(newPath.AsInode())
 		newDirNode, found := dir.wfs.Server.FindInternalNode(NodeWithId(newParent.AsInode()))
@@ -110,65 +118,13 @@ func (dir *Dir) moveEntry(ctx context.Context, oldParent util.FullPath, entry *f
 		}
 		dir.wfs.handlesLock.Unlock()
 
-		if entry.IsDirectory() {
-			if err := dir.moveFolderSubEntries(ctx, oldParent, oldName, newParent, newName); err != nil {
-				return err
-			}
-		}
-		return nil
-	}); err != nil {
-		return fmt.Errorf("fail to move %s => %s: %v", oldPath, newPath, err)
-	}
-
-	return nil
-}
-
-func (dir *Dir) moveFolderSubEntries(ctx context.Context, oldParent util.FullPath, oldName string, newParent util.FullPath, newName string) error {
-
-	currentDirPath := oldParent.Child(oldName)
-	newDirPath := newParent.Child(newName)
-
-	glog.V(1).Infof("moving folder %s => %s", currentDirPath, newDirPath)
-
-	var moveErr error
-	listErr := dir.wfs.metaCache.ListDirectoryEntries(ctx, currentDirPath, "", false, int64(math.MaxInt32), func(item *filer.Entry) bool {
-		moveErr = dir.moveEntry(ctx, currentDirPath, item, newDirPath, item.Name())
-		if moveErr != nil {
-			return false
-		}
-		return true
-	})
-	if listErr != nil {
-		return listErr
-	}
-	if moveErr != nil {
-		return moveErr
-	}
-
-	return nil
-}
-
-func (dir *Dir) moveSelfEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, moveFolderSubEntries func() error) error {
-
-	newPath := newParent.Child(newName)
-	oldPath := oldParent.Child(entry.Name())
-
-	entry.FullPath = newPath
-	if err := dir.wfs.metaCache.InsertEntry(ctx, entry); err != nil {
-		glog.V(0).Infof("dir Rename insert local %s => %s : %v", oldPath, newPath, err)
-		return fuse.EIO
-	}
-
-	if moveFolderSubEntries != nil {
-		if moveChildrenErr := moveFolderSubEntries(); moveChildrenErr != nil {
-			return moveChildrenErr
+	} else if resp.EventNotification.OldEntry != nil {
+		// without new entry, only old entry name exists. This is the second step to delete old entry
+		if err := dir.wfs.metaCache.AtomicUpdateEntryFromFiler(ctx, util.NewFullPath(resp.Directory, resp.EventNotification.OldEntry.Name), nil); err != nil {
+			return err
 		}
 	}
 
-	if err := dir.wfs.metaCache.DeleteEntry(ctx, oldPath); err != nil {
-		glog.V(0).Infof("dir Rename delete local %s => %s : %v", oldPath, newPath, err)
-		return fuse.EIO
-	}
-
 	return nil
+
 }

weed/filesys/file.go

@@ -364,15 +364,15 @@ func (file *File) getEntry() *filer_pb.Entry {
 func (file *File) downloadRemoteEntry(entry *filer_pb.Entry) (*filer_pb.Entry, error) {
 	err := file.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
-		request := &filer_pb.DownloadToLocalRequest{
+		request := &filer_pb.CacheRemoteObjectToLocalClusterRequest{
 			Directory: file.dir.FullPath(),
 			Name:      entry.Name,
 		}
 
 		glog.V(4).Infof("download entry: %v", request)
-		resp, err := client.DownloadToLocal(context.Background(), request)
+		resp, err := client.CacheRemoteObjectToLocalCluster(context.Background(), request)
 		if err != nil {
-			glog.Errorf("DownloadToLocal file %s/%s: %v", file.dir.FullPath(), file.Name, err)
+			glog.Errorf("CacheRemoteObjectToLocalCluster file %s/%s: %v", file.dir.FullPath(), file.Name, err)
 			return fuse.EIO
 		}
 

weed/filesys/meta_cache/meta_cache.go

@@ -2,7 +2,6 @@ package meta_cache
 
 import (
 	"context"
-	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/filer/leveldb"
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -122,7 +121,8 @@ func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.Full
 	//defer mc.RUnlock()
 
 	if !mc.visitedBoundary.HasVisited(dirPath) {
-		return fmt.Errorf("unsynchronized dir: %v", dirPath)
+		// if this request comes after renaming, it should be fine
+		glog.Warningf("unsynchronized dir: %v", dirPath)
 	}
 
 	_, err := mc.localStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *filer.Entry) bool {

weed/filesys/meta_cache/meta_cache_subscribe.go

@@ -58,7 +58,7 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil
 	}
 
 	util.RetryForever("followMetaUpdates", func() error {
-		return pb.WithFilerClientFollowMetadata(client, "mount", dir, lastTsNs, selfSignature, processEventFn, true)
+		return pb.WithFilerClientFollowMetadata(client, "mount", dir, &lastTsNs, selfSignature, processEventFn, true)
 	}, func(err error) bool {
 		glog.Errorf("follow metadata updates: %v", err)
 		return true

weed/iamapi/iamapi_handlers.go

@@ -8,7 +8,7 @@ import (
 	"net/http"
 )
 
-func writeIamErrorResponse(w http.ResponseWriter, err error, object string, value string, msg error) {
+func writeIamErrorResponse(w http.ResponseWriter, r *http.Request, err error, object string, value string, msg error) {
 	errCode := err.Error()
 	errorResp := ErrorResponse{}
 	errorResp.Error.Type = "Sender"
@@ -22,10 +22,10 @@ func writeIamErrorResponse(w http.ResponseWriter, err error, object string, valu
 	case iam.ErrCodeNoSuchEntityException:
 		msg := fmt.Sprintf("The %s with name %s cannot be found.", object, value)
 		errorResp.Error.Message = &msg
-		s3err.WriteXMLResponse(w, http.StatusNotFound, errorResp)
+		s3err.WriteXMLResponse(w, r, http.StatusNotFound, errorResp)
 	case iam.ErrCodeServiceFailureException:
-		s3err.WriteXMLResponse(w, http.StatusInternalServerError, errorResp)
+		s3err.WriteXMLResponse(w, r, http.StatusInternalServerError, errorResp)
 	default:
-		s3err.WriteXMLResponse(w, http.StatusInternalServerError, errorResp)
+		s3err.WriteXMLResponse(w, r, http.StatusInternalServerError, errorResp)
 	}
 }

weed/iamapi/iamapi_management_handlers.go

@@ -362,7 +362,7 @@ func (iama *IamApiServer) DeleteAccessKey(s3cfg *iam_pb.S3ApiConfiguration, valu
 
 func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 	if err := r.ParseForm(); err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidRequest, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
 		return
 	}
 	values := r.PostForm
@@ -370,7 +370,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 	s3cfgLock.RLock()
 	s3cfg := &iam_pb.S3ApiConfiguration{}
 	if err := iama.s3ApiConfig.GetS3ApiConfiguration(s3cfg); err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}
 	s3cfgLock.RUnlock()
@@ -392,7 +392,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 		userName := values.Get("UserName")
 		response, err = iama.GetUser(s3cfg, userName)
 		if err != nil {
-			writeIamErrorResponse(w, err, "user", userName, nil)
+			writeIamErrorResponse(w, r, err, "user", userName, nil)
 			return
 		}
 		changed = false
@@ -400,7 +400,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 		userName := values.Get("UserName")
 		response, err = iama.DeleteUser(s3cfg, userName)
 		if err != nil {
-			writeIamErrorResponse(w, err, "user", userName, nil)
+			writeIamErrorResponse(w, r, err, "user", userName, nil)
 			return
 		}
 	case "CreateAccessKey":
@@ -411,33 +411,33 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 		response, err = iama.CreatePolicy(s3cfg, values)
 		if err != nil {
 			glog.Errorf("CreatePolicy:  %+v", err)
-			s3err.WriteErrorResponse(w, s3err.ErrInvalidRequest, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
 			return
 		}
 	case "PutUserPolicy":
 		response, err = iama.PutUserPolicy(s3cfg, values)
 		if err != nil {
 			glog.Errorf("PutUserPolicy:  %+v", err)
-			s3err.WriteErrorResponse(w, s3err.ErrInvalidRequest, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
 			return
 		}
 	case "GetUserPolicy":
 		response, err = iama.GetUserPolicy(s3cfg, values)
 		if err != nil {
-			writeIamErrorResponse(w, err, "user", values.Get("UserName"), nil)
+			writeIamErrorResponse(w, r, err, "user", values.Get("UserName"), nil)
 			return
 		}
 		changed = false
 	case "DeleteUserPolicy":
 		if response, err = iama.DeleteUserPolicy(s3cfg, values); err != nil {
-			writeIamErrorResponse(w, err, "user", values.Get("UserName"), nil)
+			writeIamErrorResponse(w, r, err, "user", values.Get("UserName"), nil)
 		}
 	default:
 		errNotImplemented := s3err.GetAPIError(s3err.ErrNotImplemented)
 		errorResponse := ErrorResponse{}
 		errorResponse.Error.Code = &errNotImplemented.Code
 		errorResponse.Error.Message = &errNotImplemented.Description
-		s3err.WriteXMLResponse(w, errNotImplemented.HTTPStatusCode, errorResponse)
+		s3err.WriteXMLResponse(w, r, errNotImplemented.HTTPStatusCode, errorResponse)
 		return
 	}
 	if changed {
@@ -445,9 +445,9 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 		err := iama.s3ApiConfig.PutS3ApiConfiguration(s3cfg)
 		s3cfgLock.Unlock()
 		if err != nil {
-			writeIamErrorResponse(w, fmt.Errorf(iam.ErrCodeServiceFailureException), "", "", err)
+			writeIamErrorResponse(w, r, fmt.Errorf(iam.ErrCodeServiceFailureException), "", "", err)
 			return
 		}
 	}
-	s3err.WriteXMLResponse(w, http.StatusOK, response)
+	s3err.WriteXMLResponse(w, r, http.StatusOK, response)
 }

weed/images/orientation_test.go

@@ -1,7 +1,6 @@
 package images
 
 import (
-	"io/ioutil"
 	"os"
 	"testing"
 )
@@ -9,11 +8,11 @@ import (
 func TestXYZ(t *testing.T) {
 	fname := "sample1.jpg"
 
-	dat, _ := ioutil.ReadFile(fname)
+	dat, _ := os.ReadFile(fname)
 
 	fixed_data := FixJpgOrientation(dat)
 
-	ioutil.WriteFile("fixed1.jpg", fixed_data, 0644)
+	os.WriteFile("fixed1.jpg", fixed_data, 0644)
 
 	os.Remove("fixed1.jpg")
 

weed/images/resizing_test.go

@@ -2,7 +2,6 @@ package images
 
 import (
 	"bytes"
-	"io/ioutil"
 	"os"
 	"testing"
 )
@@ -10,13 +9,13 @@ import (
 func TestResizing(t *testing.T) {
 	fname := "sample2.webp"
 
-	dat, _ := ioutil.ReadFile(fname)
+	dat, _ := os.ReadFile(fname)
 
 	resized, _, _ := Resized(".webp", bytes.NewReader(dat), 100, 30, "")
 	buf := new(bytes.Buffer)
 	buf.ReadFrom(resized)
 
-	ioutil.WriteFile("resized1.png", buf.Bytes(), 0644)
+	os.WriteFile("resized1.png", buf.Bytes(), 0644)
 
 	os.Remove("resized1.png")
 

weed/messaging/broker/broker_grpc_server_discovery.go

@@ -93,14 +93,16 @@ func (broker *MessageBroker) checkFilers() {
 	for !found {
 		for _, master := range masters {
 			err := broker.withMasterClient(master, func(client master_pb.SeaweedClient) error {
-				resp, err := client.ListMasterClients(context.Background(), &master_pb.ListMasterClientsRequest{
+				resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
 					ClientType: "filer",
 				})
 				if err != nil {
 					return err
 				}
 
-				filers = append(filers, pb.FromAddressStrings(resp.GrpcAddresses)...)
+				for _, clusterNode := range resp.ClusterNodes {
+					filers = append(filers, pb.ServerAddress(clusterNode.Address))
+				}
 
 				return nil
 			})

weed/operation/chunked_file.go

@@ -4,9 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/pb"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"sort"
 	"sync"
@@ -14,6 +12,7 @@ import (
 	"google.golang.org/grpc"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -108,7 +107,7 @@ func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (wri
 		return written, err
 	}
 	defer func() {
-		io.Copy(ioutil.Discard, resp.Body)
+		io.Copy(io.Discard, resp.Body)
 		resp.Body.Close()
 	}()
 

weed/operation/upload_content.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime"
 	"mime/multipart"
 	"net/http"
@@ -91,7 +90,7 @@ func doUpload(reader io.Reader, option *UploadOption) (uploadResult *UploadResul
 	if ok {
 		data = bytesReader.Bytes
 	} else {
-		data, err = ioutil.ReadAll(reader)
+		data, err = io.ReadAll(reader)
 		if err != nil {
 			err = fmt.Errorf("read input: %v", err)
 			return
@@ -278,7 +277,7 @@ func upload_content(fillBufferFunction func(w io.Writer) error, originalDataSize
 		return &ret, nil
 	}
 
-	resp_body, ra_err := ioutil.ReadAll(resp.Body)
+	resp_body, ra_err := io.ReadAll(resp.Body)
 	if ra_err != nil {
 		return nil, fmt.Errorf("read response body %v: %v", option.UploadUrl, ra_err)
 	}

weed/pb/filer.proto

@@ -30,6 +30,8 @@ service SeaweedFiler {
 
     rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) {
     }
+    rpc StreamRenameEntry (StreamRenameEntryRequest) returns (stream StreamRenameEntryResponse) {
+    }
 
     rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
     }
@@ -67,7 +69,7 @@ service SeaweedFiler {
     rpc KvPut (KvPutRequest) returns (KvPutResponse) {
     }
 
-    rpc DownloadToLocal (DownloadToLocalRequest) returns (DownloadToLocalResponse) {
+    rpc CacheRemoteObjectToLocalCluster (CacheRemoteObjectToLocalClusterRequest) returns (CacheRemoteObjectToLocalClusterResponse) {
     }
 }
 
@@ -225,6 +227,18 @@ message AtomicRenameEntryRequest {
 message AtomicRenameEntryResponse {
 }
 
+message StreamRenameEntryRequest {
+    string old_directory = 1;
+    string old_name = 2;
+    string new_directory = 3;
+    string new_name = 4;
+    repeated int32 signatures = 5;
+}
+message StreamRenameEntryResponse {
+    string directory = 1;
+    EventNotification event_notification = 2;
+    int64 ts_ns = 3;
+}
 message AssignVolumeRequest {
     int32 count = 1;
     string collection = 2;
@@ -389,10 +403,10 @@ message FilerConf {
 /////////////////////////
 // Remote Storage related
 /////////////////////////
-message DownloadToLocalRequest {
+message CacheRemoteObjectToLocalClusterRequest {
     string directory = 1;
     string name = 2;
 }
-message DownloadToLocalResponse {
+message CacheRemoteObjectToLocalClusterResponse {
     Entry entry = 1;
 }

weed/pb/filer_pb_tail.go

@@ -17,7 +17,7 @@ func FollowMetadata(filerAddress ServerAddress, grpcDialOption grpc.DialOption,
 	processEventFn ProcessMetadataFunc, fatalOnError bool) error {
 
 	err := WithFilerClient(filerAddress, grpcDialOption, makeFunc(clientName,
-		pathPrefix, additionalPathPrefixes, lastTsNs, selfSignature, processEventFn, fatalOnError))
+		pathPrefix, additionalPathPrefixes, &lastTsNs, selfSignature, processEventFn, fatalOnError))
 	if err != nil {
 		return fmt.Errorf("subscribing filer meta change: %v", err)
 	}
@@ -25,7 +25,7 @@ func FollowMetadata(filerAddress ServerAddress, grpcDialOption grpc.DialOption,
 }
 
 func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient,
-	clientName string, pathPrefix string, lastTsNs int64, selfSignature int32,
+	clientName string, pathPrefix string, lastTsNs *int64, selfSignature int32,
 	processEventFn ProcessMetadataFunc, fatalOnError bool) error {
 
 	err := filerClient.WithFilerClient(makeFunc(clientName,
@@ -37,7 +37,7 @@ func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient,
 	return nil
 }
 
-func makeFunc(clientName string, pathPrefix string, additionalPathPrefixes []string, lastTsNs int64, selfSignature int32,
+func makeFunc(clientName string, pathPrefix string, additionalPathPrefixes []string, lastTsNs *int64, selfSignature int32,
 	processEventFn ProcessMetadataFunc, fatalOnError bool) func(client filer_pb.SeaweedFilerClient) error {
 	return func(client filer_pb.SeaweedFilerClient) error {
 		ctx, cancel := context.WithCancel(context.Background())
@@ -46,7 +46,7 @@ func makeFunc(clientName string, pathPrefix string, additionalPathPrefixes []str
 			ClientName:   clientName,
 			PathPrefix:   pathPrefix,
 			PathPrefixes: additionalPathPrefixes,
-			SinceNs:      lastTsNs,
+			SinceNs:      *lastTsNs,
 			Signature:    selfSignature,
 		})
 		if err != nil {
@@ -69,7 +69,7 @@ func makeFunc(clientName string, pathPrefix string, additionalPathPrefixes []str
 					glog.Errorf("process %v: %v", resp, err)
 				}
 			}
-			lastTsNs = resp.TsNs
+			*lastTsNs = resp.TsNs
 		}
 	}
 }

weed/pb/master.proto

@@ -7,209 +7,222 @@ option go_package = "github.com/chrislusf/seaweedfs/weed/pb/master_pb";
 //////////////////////////////////////////////////
 
 service Seaweed {
-    rpc SendHeartbeat (stream Heartbeat) returns (stream HeartbeatResponse) {
-    }
-    rpc KeepConnected (stream KeepConnectedRequest) returns (stream VolumeLocation) {
-    }
-    rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) {
-    }
-    rpc Assign (AssignRequest) returns (AssignResponse) {
-    }
-    rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
-    }
-    rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) {
-    }
-    rpc CollectionDelete (CollectionDeleteRequest) returns (CollectionDeleteResponse) {
-    }
-    rpc VolumeList (VolumeListRequest) returns (VolumeListResponse) {
-    }
-    rpc LookupEcVolume (LookupEcVolumeRequest) returns (LookupEcVolumeResponse) {
-    }
-    rpc VacuumVolume (VacuumVolumeRequest) returns (VacuumVolumeResponse) {
-    }
-    rpc GetMasterConfiguration (GetMasterConfigurationRequest) returns (GetMasterConfigurationResponse) {
-    }
-    rpc ListMasterClients (ListMasterClientsRequest) returns (ListMasterClientsResponse) {
-    }
-    rpc LeaseAdminToken (LeaseAdminTokenRequest) returns (LeaseAdminTokenResponse) {
-    }
-    rpc ReleaseAdminToken (ReleaseAdminTokenRequest) returns (ReleaseAdminTokenResponse) {
-    }
+  rpc SendHeartbeat (stream Heartbeat) returns (stream HeartbeatResponse) {
+  }
+  rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) {
+  }
+  rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) {
+  }
+  rpc Assign (AssignRequest) returns (AssignResponse) {
+  }
+  rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
+  }
+  rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) {
+  }
+  rpc CollectionDelete (CollectionDeleteRequest) returns (CollectionDeleteResponse) {
+  }
+  rpc VolumeList (VolumeListRequest) returns (VolumeListResponse) {
+  }
+  rpc LookupEcVolume (LookupEcVolumeRequest) returns (LookupEcVolumeResponse) {
+  }
+  rpc VacuumVolume (VacuumVolumeRequest) returns (VacuumVolumeResponse) {
+  }
+  rpc GetMasterConfiguration (GetMasterConfigurationRequest) returns (GetMasterConfigurationResponse) {
+  }
+  rpc ListClusterNodes (ListClusterNodesRequest) returns (ListClusterNodesResponse) {
+  }
+  rpc LeaseAdminToken (LeaseAdminTokenRequest) returns (LeaseAdminTokenResponse) {
+  }
+  rpc ReleaseAdminToken (ReleaseAdminTokenRequest) returns (ReleaseAdminTokenResponse) {
+  }
 
 }
 
 //////////////////////////////////////////////////
 
 message Heartbeat {
-    string ip = 1;
-    uint32 port = 2;
-    string public_url = 3;
-    uint64 max_file_key = 5;
-    string data_center = 6;
-    string rack = 7;
-    uint32 admin_port = 8;
-    repeated VolumeInformationMessage volumes = 9;
-    // delta volumes
-    repeated VolumeShortInformationMessage new_volumes = 10;
-    repeated VolumeShortInformationMessage deleted_volumes = 11;
-    bool has_no_volumes = 12;
+  string ip = 1;
+  uint32 port = 2;
+  string public_url = 3;
+  uint64 max_file_key = 5;
+  string data_center = 6;
+  string rack = 7;
+  uint32 admin_port = 8;
+  repeated VolumeInformationMessage volumes = 9;
+  // delta volumes
+  repeated VolumeShortInformationMessage new_volumes = 10;
+  repeated VolumeShortInformationMessage deleted_volumes = 11;
+  bool has_no_volumes = 12;
 
-    // erasure coding
-    repeated VolumeEcShardInformationMessage ec_shards = 16;
-    // delta erasure coding shards
-    repeated VolumeEcShardInformationMessage new_ec_shards = 17;
-    repeated VolumeEcShardInformationMessage deleted_ec_shards = 18;
-    bool has_no_ec_shards = 19;
+  // erasure coding
+  repeated VolumeEcShardInformationMessage ec_shards = 16;
+  // delta erasure coding shards
+  repeated VolumeEcShardInformationMessage new_ec_shards = 17;
+  repeated VolumeEcShardInformationMessage deleted_ec_shards = 18;
+  bool has_no_ec_shards = 19;
 
-    map<string, uint32> max_volume_counts = 4;
-    uint32 grpc_port = 20;
+  map<string, uint32> max_volume_counts = 4;
+  uint32 grpc_port = 20;
 
 }
 
 message HeartbeatResponse {
-    uint64 volume_size_limit = 1;
-    string leader = 2;
-    string metrics_address = 3;
-    uint32 metrics_interval_seconds = 4;
-    repeated StorageBackend storage_backends = 5;
+  uint64 volume_size_limit = 1;
+  string leader = 2;
+  string metrics_address = 3;
+  uint32 metrics_interval_seconds = 4;
+  repeated StorageBackend storage_backends = 5;
 }
 
 message VolumeInformationMessage {
-    uint32 id = 1;
-    uint64 size = 2;
-    string collection = 3;
-    uint64 file_count = 4;
-    uint64 delete_count = 5;
-    uint64 deleted_byte_count = 6;
-    bool read_only = 7;
-    uint32 replica_placement = 8;
-    uint32 version = 9;
-    uint32 ttl = 10;
-    uint32 compact_revision = 11;
-    int64 modified_at_second = 12;
-    string remote_storage_name = 13;
-    string remote_storage_key = 14;
-    string disk_type = 15;
+  uint32 id = 1;
+  uint64 size = 2;
+  string collection = 3;
+  uint64 file_count = 4;
+  uint64 delete_count = 5;
+  uint64 deleted_byte_count = 6;
+  bool read_only = 7;
+  uint32 replica_placement = 8;
+  uint32 version = 9;
+  uint32 ttl = 10;
+  uint32 compact_revision = 11;
+  int64 modified_at_second = 12;
+  string remote_storage_name = 13;
+  string remote_storage_key = 14;
+  string disk_type = 15;
 }
 
 message VolumeShortInformationMessage {
-    uint32 id = 1;
-    string collection = 3;
-    uint32 replica_placement = 8;
-    uint32 version = 9;
-    uint32 ttl = 10;
-    string disk_type = 15;
+  uint32 id = 1;
+  string collection = 3;
+  uint32 replica_placement = 8;
+  uint32 version = 9;
+  uint32 ttl = 10;
+  string disk_type = 15;
 }
 
 message VolumeEcShardInformationMessage {
-    uint32 id = 1;
-    string collection = 2;
-    uint32 ec_index_bits = 3;
-    string disk_type = 4;
+  uint32 id = 1;
+  string collection = 2;
+  uint32 ec_index_bits = 3;
+  string disk_type = 4;
 }
 
 message StorageBackend {
-    string type = 1;
-    string id = 2;
-    map<string, string> properties = 3;
+  string type = 1;
+  string id = 2;
+  map<string, string> properties = 3;
 }
 
 message Empty {
 }
 
 message SuperBlockExtra {
-    message ErasureCoding {
-        uint32 data = 1;
-        uint32 parity = 2;
-        repeated uint32 volume_ids = 3;
-    }
-    ErasureCoding erasure_coding = 1;
+  message ErasureCoding {
+    uint32 data = 1;
+    uint32 parity = 2;
+    repeated uint32 volume_ids = 3;
+  }
+  ErasureCoding erasure_coding = 1;
 }
 
 message KeepConnectedRequest {
-    string name = 1;
-    string client_address = 3;
+  string client_type = 1;
+  string client_address = 3;
+  string version = 4;
 }
 
 message VolumeLocation {
-    string url = 1;
-    string public_url = 2;
-    repeated uint32 new_vids = 3;
-    repeated uint32 deleted_vids = 4;
-    string leader = 5; // optional when leader is not itself
-    string data_center = 6; // optional when DataCenter is in use
-    uint32 grpc_port = 7;
+  string url = 1;
+  string public_url = 2;
+  repeated uint32 new_vids = 3;
+  repeated uint32 deleted_vids = 4;
+  string leader = 5; // optional when leader is not itself
+  string data_center = 6; // optional when DataCenter is in use
+  uint32 grpc_port = 7;
+}
+
+message ClusterNodeUpdate {
+  string node_type = 1;
+  string address = 2;
+  bool is_leader = 3;
+  bool is_add = 4;
+}
+
+message KeepConnectedResponse {
+  VolumeLocation volume_location = 1;
+  ClusterNodeUpdate cluster_node_update = 2;
 }
 
 message LookupVolumeRequest {
-    repeated string volume_or_file_ids = 1;
-    string collection = 2; // optional, a bit faster if provided.
+  repeated string volume_or_file_ids = 1;
+  string collection = 2; // optional, a bit faster if provided.
 }
 message LookupVolumeResponse {
-    message VolumeIdLocation {
-        string volume_or_file_id = 1;
-        repeated Location locations = 2;
-        string error = 3;
-        string auth = 4;
-    }
-    repeated VolumeIdLocation volume_id_locations = 1;
+  message VolumeIdLocation {
+    string volume_or_file_id = 1;
+    repeated Location locations = 2;
+    string error = 3;
+    string auth = 4;
+  }
+  repeated VolumeIdLocation volume_id_locations = 1;
 }
 
 message Location {
-    string url = 1;
-    string public_url = 2;
-    uint32 grpc_port = 3;
+  string url = 1;
+  string public_url = 2;
+  uint32 grpc_port = 3;
 }
 
 message AssignRequest {
-    uint64 count = 1;
-    string replication = 2;
-    string collection = 3;
-    string ttl = 4;
-    string data_center = 5;
-    string rack = 6;
-    string data_node = 7;
-    uint32 memory_map_max_size_mb = 8;
-    uint32 Writable_volume_count = 9;
-    string disk_type = 10;
+  uint64 count = 1;
+  string replication = 2;
+  string collection = 3;
+  string ttl = 4;
+  string data_center = 5;
+  string rack = 6;
+  string data_node = 7;
+  uint32 memory_map_max_size_mb = 8;
+  uint32 Writable_volume_count = 9;
+  string disk_type = 10;
 }
 message AssignResponse {
-    string fid = 1;
-    uint64 count = 4;
-    string error = 5;
-    string auth = 6;
-    repeated Location replicas = 7;
-    Location location = 8;
+  string fid = 1;
+  uint64 count = 4;
+  string error = 5;
+  string auth = 6;
+  repeated Location replicas = 7;
+  Location location = 8;
 }
 
 message StatisticsRequest {
-    string replication = 1;
-    string collection = 2;
-    string ttl = 3;
-    string disk_type = 4;
+  string replication = 1;
+  string collection = 2;
+  string ttl = 3;
+  string disk_type = 4;
 }
 message StatisticsResponse {
-    uint64 total_size = 4;
-    uint64 used_size = 5;
-    uint64 file_count = 6;
+  uint64 total_size = 4;
+  uint64 used_size = 5;
+  uint64 file_count = 6;
 }
 
 //
 // collection related
 //
 message Collection {
-    string name = 1;
+  string name = 1;
 }
 message CollectionListRequest {
-    bool include_normal_volumes = 1;
-    bool include_ec_volumes = 2;
+  bool include_normal_volumes = 1;
+  bool include_ec_volumes = 2;
 }
 message CollectionListResponse {
-    repeated Collection collections = 1;
+  repeated Collection collections = 1;
 }
 
 message CollectionDeleteRequest {
-    string name = 1;
+  string name = 1;
 }
 message CollectionDeleteResponse {
 }
@@ -218,56 +231,56 @@ message CollectionDeleteResponse {
 // volume related
 //
 message DiskInfo {
-    string type = 1;
-    int64 volume_count = 2;
-    int64 max_volume_count = 3;
-    int64 free_volume_count = 4;
-    int64 active_volume_count = 5;
-    repeated VolumeInformationMessage volume_infos = 6;
-    repeated VolumeEcShardInformationMessage ec_shard_infos = 7;
-    int64 remote_volume_count = 8;
+  string type = 1;
+  int64 volume_count = 2;
+  int64 max_volume_count = 3;
+  int64 free_volume_count = 4;
+  int64 active_volume_count = 5;
+  repeated VolumeInformationMessage volume_infos = 6;
+  repeated VolumeEcShardInformationMessage ec_shard_infos = 7;
+  int64 remote_volume_count = 8;
 }
 message DataNodeInfo {
-    string id = 1;
-    map<string, DiskInfo> diskInfos = 2;
-    uint32 grpc_port = 3;
+  string id = 1;
+  map<string, DiskInfo> diskInfos = 2;
+  uint32 grpc_port = 3;
 }
 message RackInfo {
-    string id = 1;
-    repeated DataNodeInfo data_node_infos = 2;
-    map<string, DiskInfo> diskInfos = 3;
+  string id = 1;
+  repeated DataNodeInfo data_node_infos = 2;
+  map<string, DiskInfo> diskInfos = 3;
 }
 message DataCenterInfo {
-    string id = 1;
-    repeated RackInfo rack_infos = 2;
-    map<string, DiskInfo> diskInfos = 3;
+  string id = 1;
+  repeated RackInfo rack_infos = 2;
+  map<string, DiskInfo> diskInfos = 3;
 }
 message TopologyInfo {
-    string id = 1;
-    repeated DataCenterInfo data_center_infos = 2;
-    map<string, DiskInfo> diskInfos = 3;
+  string id = 1;
+  repeated DataCenterInfo data_center_infos = 2;
+  map<string, DiskInfo> diskInfos = 3;
 }
 message VolumeListRequest {
 }
 message VolumeListResponse {
-    TopologyInfo topology_info = 1;
-    uint64 volume_size_limit_mb = 2;
+  TopologyInfo topology_info = 1;
+  uint64 volume_size_limit_mb = 2;
 }
 
 message LookupEcVolumeRequest {
-    uint32 volume_id = 1;
+  uint32 volume_id = 1;
 }
 message LookupEcVolumeResponse {
-    uint32 volume_id = 1;
-    message EcShardIdLocation {
-        uint32 shard_id = 1;
-        repeated Location locations = 2;
-    }
-    repeated EcShardIdLocation shard_id_locations = 2;
+  uint32 volume_id = 1;
+  message EcShardIdLocation {
+    uint32 shard_id = 1;
+    repeated Location locations = 2;
+  }
+  repeated EcShardIdLocation shard_id_locations = 2;
 }
 
 message VacuumVolumeRequest {
-    float garbage_threshold = 1;
+  float garbage_threshold = 1;
 }
 message VacuumVolumeResponse {
 }
@@ -275,37 +288,42 @@ message VacuumVolumeResponse {
 message GetMasterConfigurationRequest {
 }
 message GetMasterConfigurationResponse {
-    string metrics_address = 1;
-    uint32 metrics_interval_seconds = 2;
-    repeated StorageBackend storage_backends = 3;
-    string default_replication = 4;
-    string leader = 5;
-    uint32 volume_size_limit_m_b = 6;
-    bool volume_preallocate = 7;
+  string metrics_address = 1;
+  uint32 metrics_interval_seconds = 2;
+  repeated StorageBackend storage_backends = 3;
+  string default_replication = 4;
+  string leader = 5;
+  uint32 volume_size_limit_m_b = 6;
+  bool volume_preallocate = 7;
 }
 
-message ListMasterClientsRequest {
-    string client_type = 1;
+message ListClusterNodesRequest {
+  string client_type = 1;
 }
-message ListMasterClientsResponse {
-    repeated string grpc_addresses = 1;
+message ListClusterNodesResponse {
+  message ClusterNode {
+    string address = 1;
+    string version = 2;
+    bool is_leader = 3;
+  }
+  repeated ClusterNode cluster_nodes = 1;
 }
 
 message LeaseAdminTokenRequest {
-    int64 previous_token = 1;
-    int64 previous_lock_time = 2;
-    string lock_name = 3;
-    string client_name = 4;
+  int64 previous_token = 1;
+  int64 previous_lock_time = 2;
+  string lock_name = 3;
+  string client_name = 4;
 }
 message LeaseAdminTokenResponse {
-    int64 token = 1;
-    int64 lock_ts_ns = 2;
+  int64 token = 1;
+  int64 lock_ts_ns = 2;
 }
 
 message ReleaseAdminTokenRequest {
-    int64 previous_token = 1;
-    int64 previous_lock_time = 2;
-    string lock_name = 3;
+  int64 previous_token = 1;
+  int64 previous_lock_time = 2;
+  string lock_name = 3;
 }
 message ReleaseAdminTokenResponse {
 }

weed/pb/volume_server.proto

@@ -14,7 +14,7 @@ service VolumeServer {
 
     rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) {
     }
-    rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (VacuumVolumeCompactResponse) {
+    rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (stream VacuumVolumeCompactResponse) {
     }
     rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) {
     }
@@ -47,7 +47,7 @@ service VolumeServer {
     }
 
     // copy the .idx .dat files, and mount this volume
-    rpc VolumeCopy (VolumeCopyRequest) returns (VolumeCopyResponse) {
+    rpc VolumeCopy (VolumeCopyRequest) returns (stream VolumeCopyResponse) {
     }
     rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
     }
@@ -142,6 +142,7 @@ message VacuumVolumeCompactRequest {
     int64 preallocate = 2;
 }
 message VacuumVolumeCompactResponse {
+    int64 processed_bytes = 1;
 }
 
 message VacuumVolumeCommitRequest {
@@ -251,6 +252,7 @@ message VolumeCopyRequest {
 }
 message VolumeCopyResponse {
     uint64 last_append_at_ns = 1;
+    int64 processed_bytes = 2;
 }
 
 message CopyFileRequest {

weed/remote_storage/azure/azure_storage_client.go

@@ -3,17 +3,17 @@ package azure
 import (
 	"context"
 	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"reflect"
+
 	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
 	"github.com/chrislusf/seaweedfs/weed/remote_storage"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"io"
-	"io/ioutil"
-	"net/url"
-	"os"
-	"reflect"
 )
 
 func init() {
@@ -115,7 +115,7 @@ func (az *azureRemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocatio
 	bodyStream := downloadResponse.Body(azblob.RetryReaderOptions{MaxRetryRequests: 20})
 	defer bodyStream.Close()
 
-	data, err = ioutil.ReadAll(bodyStream)
+	data, err = io.ReadAll(bodyStream)
 
 	if err != nil {
 		return nil, fmt.Errorf("failed to download file %s%s: %v", loc.Bucket, loc.Path, err)

weed/remote_storage/gcs/gcs_storage_client.go

@@ -1,9 +1,13 @@
 package gcs
 
 import (
-	"cloud.google.com/go/storage"
 	"context"
 	"fmt"
+	"io"
+	"os"
+	"reflect"
+
+	"cloud.google.com/go/storage"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
@@ -11,10 +15,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"google.golang.org/api/iterator"
 	"google.golang.org/api/option"
-	"io"
-	"io/ioutil"
-	"os"
-	"reflect"
 )
 
 func init() {
@@ -110,7 +110,7 @@ func (gcs *gcsRemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation
 	if readErr != nil {
 		return nil, readErr
 	}
-	data, err = ioutil.ReadAll(rangeReader)
+	data, err = io.ReadAll(rangeReader)
 
 	if err != nil {
 		return nil, fmt.Errorf("failed to download file %s%s: %v", loc.Bucket, loc.Path, err)

weed/remote_storage/s3/s3_storage_client.go

@@ -83,6 +83,7 @@ func (s *s3RemoteStorageClient) Traverse(remote *remote_pb.RemoteStorageLocation
 	}
 	isLastPage := false
 	for !isLastPage && err == nil {
+		var localErr error
 		listErr := s.conn.ListObjectsV2Pages(listInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
 			for _, content := range page.Contents {
 				key := *content.Key
@@ -94,6 +95,7 @@ func (s *s3RemoteStorageClient) Traverse(remote *remote_pb.RemoteStorageLocation
 					RemoteETag:  *content.ETag,
 					StorageName: s.conf.Name,
 				}); err != nil {
+					localErr = err
 					return false
 				}
 			}
@@ -104,6 +106,9 @@ func (s *s3RemoteStorageClient) Traverse(remote *remote_pb.RemoteStorageLocation
 		if listErr != nil {
 			err = fmt.Errorf("list %v: %v", remote, listErr)
 		}
+		if localErr != nil {
+			err = fmt.Errorf("process %v: %v", remote, localErr)
+		}
 	}
 	return
 }

weed/replication/sub/notification_kafka.go

@@ -3,7 +3,7 @@ package sub
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"sync"
 	"time"
 
@@ -119,7 +119,7 @@ type KafkaProgress struct {
 
 func loadProgress(offsetFile string) *KafkaProgress {
 	progress := &KafkaProgress{}
-	data, err := ioutil.ReadFile(offsetFile)
+	data, err := os.ReadFile(offsetFile)
 	if err != nil {
 		glog.Warningf("failed to read kafka progress file: %s", offsetFile)
 		return nil
@@ -137,7 +137,7 @@ func (progress *KafkaProgress) saveProgress() error {
 	if err != nil {
 		return fmt.Errorf("failed to marshal progress: %v", err)
 	}
-	err = ioutil.WriteFile(progress.offsetFile, data, 0640)
+	err = os.WriteFile(progress.offsetFile, data, 0640)
 	if err != nil {
 		return fmt.Errorf("failed to save progress to %s: %v", progress.offsetFile, err)
 	}

weed/s3api/auth_credentials.go

@@ -2,6 +2,10 @@ package s3api
 
 import (
 	"fmt"
+	"net/http"
+	"os"
+	"strings"
+
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb"
@@ -10,9 +14,6 @@ import (
 	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
 	"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
 	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
-	"io/ioutil"
-	"net/http"
-	"strings"
 )
 
 type Action string
@@ -37,6 +38,31 @@ type Credential struct {
 	SecretKey string
 }
 
+func (action Action) isAdmin() bool {
+	return strings.HasPrefix(string(action), s3_constants.ACTION_ADMIN)
+}
+
+func (action Action) isOwner(bucket string) bool {
+	return string(action) == s3_constants.ACTION_ADMIN+":"+bucket
+}
+
+func (action Action) overBucket(bucket string) bool {
+	return strings.HasSuffix(string(action), ":"+bucket) || strings.HasSuffix(string(action), ":*")
+}
+
+func (action Action) getPermission() Permission {
+	switch act := strings.Split(string(action), ":")[0]; act {
+	case s3_constants.ACTION_ADMIN:
+		return Permission("FULL_CONTROL")
+	case s3_constants.ACTION_WRITE:
+		return Permission("WRITE")
+	case s3_constants.ACTION_READ:
+		return Permission("READ")
+	default:
+		return Permission("")
+	}
+}
+
 func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManagement {
 	iam := &IdentityAccessManagement{
 		domain: option.DomainName,
@@ -66,7 +92,7 @@ func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFiler(option *S3A
 }
 
 func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFile(fileName string) error {
-	content, readErr := ioutil.ReadFile(fileName)
+	content, readErr := os.ReadFile(fileName)
 	if readErr != nil {
 		glog.Warningf("fail to read %s : %v", fileName, readErr)
 		return fmt.Errorf("fail to read %s : %v", fileName, readErr)
@@ -120,11 +146,13 @@ func (iam *IdentityAccessManagement) lookupByAccessKey(accessKey string) (identi
 
 	for _, ident := range iam.identities {
 		for _, cred := range ident.Credentials {
+			// println("checking", ident.Name, cred.AccessKey)
 			if cred.AccessKey == accessKey {
 				return ident, cred, true
 			}
 		}
 	}
+	glog.V(1).Infof("could not find accessKey %s", accessKey)
 	return nil, nil, false
 }
 
@@ -156,7 +184,7 @@ func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) htt
 			f(w, r)
 			return
 		}
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 	}
 }
 

weed/s3api/auth_credentials_subscribe.go

@@ -8,7 +8,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
-func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, lastTsNs int64) error {
+func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, lastTsNs int64) {
 
 	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
 
@@ -32,8 +32,11 @@ func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, la
 		return nil
 	}
 
-	return util.Retry("followIamChanges", func() error {
-		return pb.WithFilerClientFollowMetadata(s3a, clientName, prefix, lastTsNs, 0, processEventFn, true)
+	util.RetryForever("followIamChanges", func() error {
+		return pb.WithFilerClientFollowMetadata(s3a, clientName, prefix, &lastTsNs, 0, processEventFn, true)
+	}, func(err error) bool {
+		glog.V(0).Infof("iam follow metadata changes: %v", err)
+		return true
 	})
 
 }

weed/s3api/auth_signature_v4.go

@@ -23,8 +23,7 @@ import (
 	"crypto/sha256"
 	"crypto/subtle"
 	"encoding/hex"
-	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -33,6 +32,8 @@ import (
 	"strings"
 	"time"
 	"unicode/utf8"
+
+	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 )
 
 func (iam *IdentityAccessManagement) reqSignatureV4Verify(r *http.Request) (*Identity, s3err.ErrorCode) {
@@ -135,9 +136,9 @@ func (iam *IdentityAccessManagement) doesSignatureMatch(hashedPayload string, r
 
 	// Get hashed Payload
 	if signV4Values.Credential.scope.service != "s3" && hashedPayload == emptySHA256 && r.Body != nil {
-		buf, _ := ioutil.ReadAll(r.Body)
-		r.Body = ioutil.NopCloser(bytes.NewBuffer(buf))
-		b, _ := ioutil.ReadAll(bytes.NewBuffer(buf))
+		buf, _ := io.ReadAll(r.Body)
+		r.Body = io.NopCloser(bytes.NewBuffer(buf))
+		b, _ := io.ReadAll(bytes.NewBuffer(buf))
 		if len(b) != 0 {
 			bodyHash := sha256.Sum256(b)
 			hashedPayload = hex.EncodeToString(bodyHash[:])
@@ -433,7 +434,7 @@ func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(hashedPayload s
 		}
 	}
 
-	/// Verify finally if signature is same.
+	// / Verify finally if signature is same.
 
 	// Get canonical request.
 	presignedCanonicalReq := getCanonicalRequest(extractedSignedHeaders, hashedPayload, encodedQuery, req.URL.Path, req.Method)

weed/s3api/auto_signature_v4_test.go

@@ -8,9 +8,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"sort"
@@ -19,6 +17,8 @@ import (
 	"testing"
 	"time"
 	"unicode/utf8"
+
+	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 )
 
 // TestIsRequestPresignedSignatureV4 - Test validates the logic for presign signature verision v4 detection.
@@ -86,7 +86,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 	// Validates all testcases.
 	for i, testCase := range testCases {
 		if _, s3Error := iam.reqSignatureV4Verify(testCase.req); s3Error != testCase.s3Error {
-			ioutil.ReadAll(testCase.req.Body)
+			io.ReadAll(testCase.req.Body)
 			t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d", i, testCase.s3Error, s3Error)
 		}
 	}
@@ -167,7 +167,7 @@ func newTestRequest(method, urlStr string, contentLength int64, body io.ReadSeek
 	case body == nil:
 		hashedPayload = getSHA256Hash([]byte{})
 	default:
-		payloadBytes, err := ioutil.ReadAll(body)
+		payloadBytes, err := io.ReadAll(body)
 		if err != nil {
 			return nil, err
 		}

weed/s3api/filer_multipart.go

@@ -38,6 +38,9 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp
 		for k, v := range input.Metadata {
 			entry.Extended[k] = []byte(*v)
 		}
+		if input.ContentType != nil {
+			entry.Attributes.Mime = *input.ContentType
+		}
 	}); err != nil {
 		glog.Errorf("NewMultipartUpload error: %v", err)
 		return nil, s3err.ErrInternalError
@@ -65,7 +68,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
 
 	uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId
 
-	entries, _, err := s3a.list(uploadDirectory, "", "", false, 0)
+	entries, _, err := s3a.list(uploadDirectory, "", "", false, maxPartsList)
 	if err != nil || len(entries) == 0 {
 		glog.Errorf("completeMultipartUpload %s %s error: %v, entries:%d", *input.Bucket, *input.UploadId, err, len(entries))
 		return nil, s3err.ErrNoSuchUpload
@@ -79,9 +82,13 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
 
 	var finalParts []*filer_pb.FileChunk
 	var offset int64
+	var mime string
 
 	for _, entry := range entries {
 		if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory {
+			if entry.Name == "0001.part" && entry.Attributes.Mime != "" {
+				mime = entry.Attributes.Mime
+			}
 			for _, chunk := range entry.Chunks {
 				p := &filer_pb.FileChunk{
 					FileId:    chunk.GetFileIdString(),
@@ -121,6 +128,11 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
 				entry.Extended[k] = v
 			}
 		}
+		if pentry.Attributes.Mime != "" {
+			entry.Attributes.Mime = pentry.Attributes.Mime
+		} else if mime != "" {
+			entry.Attributes.Mime = mime
+		}
 	})
 
 	if err != nil {

weed/s3api/s3api_bucket_handlers.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"encoding/xml"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"math"
 	"net/http"
 	"time"
@@ -34,7 +36,7 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques
 	if s3a.iam.isEnabled() {
 		identity, s3Err = s3a.iam.authUser(r)
 		if s3Err != s3err.ErrNone {
-			s3err.WriteErrorResponse(w, s3Err, r)
+			s3err.WriteErrorResponse(w, r, s3Err)
 			return
 		}
 	}
@@ -44,7 +46,7 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques
 	entries, _, err := s3a.list(s3a.option.BucketsPath, "", "", false, math.MaxInt32)
 
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}
 
@@ -71,7 +73,7 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques
 		Buckets: buckets,
 	}
 
-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)
 }
 
 func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
@@ -98,17 +100,24 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
 		}
 		return nil
 	}); err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}
 	if exist, err := s3a.exists(s3a.option.BucketsPath, bucket, true); err == nil && exist {
 		errCode = s3err.ErrBucketAlreadyExists
 	}
 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}
 
+	if s3a.iam.isEnabled() {
+		if _, errCode = s3a.iam.authRequest(r, s3_constants.ACTION_ADMIN); errCode != s3err.ErrNone {
+			s3err.WriteErrorResponse(w, r, errCode)
+			return
+		}
+	}
+
 	fn := func(entry *filer_pb.Entry) {
 		if identityId := r.Header.Get(xhttp.AmzIdentityId); identityId != "" {
 			if entry.Extended == nil {
@@ -121,11 +130,11 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
 	// create the folder for bucket, but lazily create actual collection
 	if err := s3a.mkdir(s3a.option.BucketsPath, bucket, fn); err != nil {
 		glog.Errorf("PutBucketHandler mkdir: %v", err)
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}
 
-	writeSuccessResponseEmpty(w)
+	writeSuccessResponseEmpty(w, r)
 }
 
 func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
@@ -134,7 +143,7 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
 	glog.V(3).Infof("DeleteBucketHandler %s", bucket)
 
 	if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, err, r)
+		s3err.WriteErrorResponse(w, r, err)
 		return
 	}
 
@@ -156,11 +165,11 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
 	err = s3a.rm(s3a.option.BucketsPath, bucket, false, true)
 
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}
 
-	s3err.WriteEmptyResponse(w, http.StatusNoContent)
+	s3err.WriteEmptyResponse(w, r, http.StatusNoContent)
 }
 
 func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
@@ -169,11 +178,11 @@ func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request
 	glog.V(3).Infof("HeadBucketHandler %s", bucket)
 
 	if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, err, r)
+		s3err.WriteErrorResponse(w, r, err)
 		return
 	}
 
-	writeSuccessResponseEmpty(w)
+	writeSuccessResponseEmpty(w, r)
 }
 
 func (s3a *S3ApiServer) checkBucket(r *http.Request, bucket string) s3err.ErrorCode {
@@ -205,3 +214,99 @@ func (s3a *S3ApiServer) hasAccess(r *http.Request, entry *filer_pb.Entry) bool {
 	}
 	return true
 }
+
+// GetBucketAclHandler Get Bucket ACL
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketAcl.html
+func (s3a *S3ApiServer) GetBucketAclHandler(w http.ResponseWriter, r *http.Request) {
+	// collect parameters
+	bucket, _ := getBucketAndObject(r)
+	glog.V(3).Infof("GetBucketAclHandler %s", bucket)
+
+	if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
+		s3err.WriteErrorResponse(w, r, err)
+		return
+	}
+
+	response := AccessControlPolicy{}
+	for _, ident := range s3a.iam.identities {
+		if len(ident.Credentials) == 0 {
+			continue
+		}
+		for _, action := range ident.Actions {
+			if !action.overBucket(bucket) || action.getPermission() == "" {
+				continue
+			}
+			id := ident.Credentials[0].AccessKey
+			if response.Owner.DisplayName == "" && action.isOwner(bucket) && len(ident.Credentials) > 0 {
+				response.Owner.DisplayName = ident.Name
+				response.Owner.ID = id
+			}
+			response.AccessControlList.Grant = append(response.AccessControlList.Grant, Grant{
+				Grantee: Grantee{
+					ID:          id,
+					DisplayName: ident.Name,
+					Type:        "CanonicalUser",
+					XMLXSI:      "CanonicalUser",
+					XMLNS:       "http://www.w3.org/2001/XMLSchema-instance"},
+				Permission: action.getPermission(),
+			})
+		}
+	}
+	writeSuccessResponseXML(w, r, response)
+}
+
+// GetBucketLifecycleConfigurationHandler Get Bucket Lifecycle configuration
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLifecycleConfiguration.html
+func (s3a *S3ApiServer) GetBucketLifecycleConfigurationHandler(w http.ResponseWriter, r *http.Request) {
+	// collect parameters
+	bucket, _ := getBucketAndObject(r)
+	glog.V(3).Infof("GetBucketAclHandler %s", bucket)
+
+	if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
+		s3err.WriteErrorResponse(w, r, err)
+		return
+	}
+	fc, err := filer.ReadFilerConf(s3a.option.Filer, s3a.option.GrpcDialOption, nil)
+	if err != nil {
+		glog.Errorf("GetBucketLifecycleConfigurationHandler: %s", err)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		return
+	}
+	ttls := fc.GetCollectionTtls(bucket)
+	if len(ttls) == 0 {
+		s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchLifecycleConfiguration)
+		return
+	}
+	response := Lifecycle{}
+	for prefix, internalTtl := range ttls {
+		ttl, _ := needle.ReadTTL(internalTtl)
+		days := int(ttl.Minutes() / 60 / 24)
+		if days == 0 {
+			continue
+		}
+		response.Rules = append(response.Rules, Rule{
+			Status: Enabled, Filter: Filter{
+				Prefix: Prefix{string: prefix, set: true},
+				set:    true,
+			},
+			Expiration: Expiration{Days: days, set: true},
+		})
+	}
+	writeSuccessResponseXML(w, r, response)
+}
+
+// PutBucketLifecycleConfigurationHandler Put Bucket Lifecycle configuration
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html
+func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWriter, r *http.Request) {
+
+	s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
+
+}
+
+// DeleteBucketMetricsConfiguration Delete Bucket Lifecycle
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketLifecycle.html
+func (s3a *S3ApiServer) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
+
+	s3err.WriteEmptyResponse(w, r, http.StatusNoContent)
+
+}

weed/s3api/s3api_handlers.go

@@ -21,16 +21,17 @@ func (s3a *S3ApiServer) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) err
 	}, s3a.option.Filer.ToGrpcAddress(), s3a.option.GrpcDialOption)
 
 }
+
 func (s3a *S3ApiServer) AdjustedUrl(location *filer_pb.Location) string {
 	return location.Url
 }
 
-func writeSuccessResponseXML(w http.ResponseWriter, response interface{}) {
-	s3err.WriteXMLResponse(w, http.StatusOK, response)
+func writeSuccessResponseXML(w http.ResponseWriter, r *http.Request, response interface{}) {
+	s3err.WriteXMLResponse(w, r, http.StatusOK, response)
 }
 
-func writeSuccessResponseEmpty(w http.ResponseWriter) {
-	s3err.WriteEmptyResponse(w, http.StatusOK)
+func writeSuccessResponseEmpty(w http.ResponseWriter, r *http.Request) {
+	s3err.WriteEmptyResponse(w, r, http.StatusOK)
 }
 
 func validateContentMd5(h http.Header) ([]byte, error) {

weed/s3api/s3api_object_copy_handlers.go

@@ -34,16 +34,16 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 		dir, name := fullPath.DirAndName()
 		entry, err := s3a.getEntry(dir, name)
 		if err != nil {
-			s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
 			return
 		}
 		entry.Extended = weed_server.SaveAmzMetaData(r, entry.Extended, isReplace(r))
 		err = s3a.touch(dir, name, entry)
 		if err != nil {
-			s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
 			return
 		}
-		writeSuccessResponseXML(w, CopyObjectResult{
+		writeSuccessResponseXML(w, r, CopyObjectResult{
 			ETag:         fmt.Sprintf("%x", entry.Attributes.Md5),
 			LastModified: time.Now().UTC(),
 		})
@@ -52,19 +52,19 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 
 	// If source object is empty or bucket is empty, reply back invalid copy source.
 	if srcObject == "" || srcBucket == "" {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
 		return
 	}
 	srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
 	dir, name := srcPath.DirAndName()
 	_, err = s3a.getEntry(dir, name)
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
 		return
 	}
 
 	if srcBucket == dstBucket && srcObject == dstObject {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidCopyDest, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopyDest)
 		return
 	}
 
@@ -75,7 +75,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 
 	_, _, resp, err := util.DownloadFile(srcUrl, "")
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
 		return
 	}
 	defer util.CloseResponse(resp)
@@ -84,7 +84,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 	etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body)
 
 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}
 
@@ -95,7 +95,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 		LastModified: time.Now().UTC(),
 	}
 
-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)
 
 }
 
@@ -128,7 +128,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
 	srcBucket, srcObject := pathToBucketAndObject(cpSrcPath)
 	// If source object is empty or bucket is empty, reply back invalid copy source.
 	if srcObject == "" || srcBucket == "" {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
 		return
 	}
 
@@ -137,7 +137,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
 
 	partID, err := strconv.Atoi(partIDString)
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidPart, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidPart)
 		return
 	}
 
@@ -145,7 +145,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
 
 	// check partID with maximum part ID for multipart objects
 	if partID > globalMaxPartID {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxParts, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxParts)
 		return
 	}
 
@@ -158,7 +158,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
 
 	dataReader, err := util.ReadUrlAsReaderCloser(srcUrl, rangeHeader)
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
 		return
 	}
 	defer dataReader.Close()
@@ -167,7 +167,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
 	etag, errCode := s3a.putToFiler(r, dstUrl, dataReader)
 
 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}
 
@@ -178,7 +178,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
 		LastModified: time.Now().UTC(),
 	}
 
-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)
 
 }
 

weed/s3api/s3api_object_handlers.go

@@ -1,20 +1,21 @@
 package s3api
 
 import (
+	"bytes"
 	"crypto/md5"
 	"encoding/json"
 	"encoding/xml"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/filer"
-	"github.com/pquerna/cachecontrol/cacheobject"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"sort"
 	"strings"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/pquerna/cachecontrol/cacheobject"
+
 	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 
 	"github.com/gorilla/mux"
@@ -36,6 +37,16 @@ func init() {
 	}}
 }
 
+func mimeDetect(r *http.Request, dataReader io.Reader) io.ReadCloser {
+	mimeBuffer := make([]byte, 512)
+	size, _ := dataReader.Read(mimeBuffer)
+	if size > 0 {
+		r.Header.Set("Content-Type", http.DetectContentType(mimeBuffer[:size]))
+		return io.NopCloser(io.MultiReader(bytes.NewReader(mimeBuffer[:size]), dataReader))
+	}
+	return io.NopCloser(dataReader)
+}
+
 func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
 
 	// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html
@@ -45,20 +56,20 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
 
 	_, err := validateContentMd5(r.Header)
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidDigest, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidDigest)
 		return
 	}
 
 	if r.Header.Get("Cache-Control") != "" {
 		if _, err = cacheobject.ParseRequestCacheControl(r.Header.Get("Cache-Control")); err != nil {
-			s3err.WriteErrorResponse(w, s3err.ErrInvalidDigest, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidDigest)
 			return
 		}
 	}
 
 	if r.Header.Get("Expires") != "" {
 		if _, err = time.Parse(http.TimeFormat, r.Header.Get("Expires")); err != nil {
-			s3err.WriteErrorResponse(w, s3err.ErrInvalidDigest, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidDigest)
 			return
 		}
 	}
@@ -76,12 +87,12 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
 			_, s3ErrCode = s3a.iam.reqSignatureV4Verify(r)
 		}
 		if s3ErrCode != s3err.ErrNone {
-			s3err.WriteErrorResponse(w, s3ErrCode, r)
+			s3err.WriteErrorResponse(w, r, s3ErrCode)
 			return
 		}
 	} else {
 		if authTypeStreamingSigned == rAuthType {
-			s3err.WriteErrorResponse(w, s3err.ErrAuthNotSetup, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrAuthNotSetup)
 			return
 		}
 	}
@@ -89,23 +100,27 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
 
 	if strings.HasSuffix(object, "/") {
 		if err := s3a.mkdir(s3a.option.BucketsPath, bucket+object, nil); err != nil {
-			s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 			return
 		}
 	} else {
 		uploadUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, bucket, urlPathEscape(object))
 
+		if r.Header.Get("Content-Type") == "" {
+			dataReader = mimeDetect(r, dataReader)
+		}
+
 		etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader)
 
 		if errCode != s3err.ErrNone {
-			s3err.WriteErrorResponse(w, errCode, r)
+			s3err.WriteErrorResponse(w, r, errCode)
 			return
 		}
 
 		setEtag(w, etag)
 	}
 
-	writeSuccessResponseEmpty(w)
+	writeSuccessResponseEmpty(w, r)
 }
 
 func urlPathEscape(object string) string {
@@ -122,7 +137,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
 	glog.V(3).Infof("GetObjectHandler %s %s", bucket, object)
 
 	if strings.HasSuffix(r.URL.Path, "/") {
-		s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
 		return
 	}
 
@@ -198,15 +213,15 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
 	bucket, _ := getBucketAndObject(r)
 	glog.V(3).Infof("DeleteMultipleObjectsHandler %s", bucket)
 
-	deleteXMLBytes, err := ioutil.ReadAll(r.Body)
+	deleteXMLBytes, err := io.ReadAll(r.Body)
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}
 
 	deleteObjects := &DeleteObjectsRequest{}
 	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrMalformedXML, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML)
 		return
 	}
 
@@ -258,7 +273,7 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
 	}
 	deleteResp.Errors = deleteErrors
 
-	writeSuccessResponseXML(w, deleteResp)
+	writeSuccessResponseXML(w, r, deleteResp)
 
 }
 
@@ -302,7 +317,7 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
 
 	if err != nil {
 		glog.Errorf("NewRequest %s: %v", destUrl, err)
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}
 
@@ -331,19 +346,19 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
 
 	if postErr != nil {
 		glog.Errorf("post to filer: %v", postErr)
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}
 	defer util.CloseResponse(resp)
 
 	if resp.StatusCode == http.StatusPreconditionFailed {
-		s3err.WriteErrorResponse(w, s3err.ErrPreconditionFailed, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrPreconditionFailed)
 		return
 	}
 
 	if (resp.ContentLength == -1 || resp.StatusCode == 404) && resp.StatusCode != 304 {
 		if r.Method != "DELETE" {
-			s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
 			return
 		}
 	}
@@ -394,7 +409,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader
 
 	etag = fmt.Sprintf("%x", hash.Sum(nil))
 
-	resp_body, ra_err := ioutil.ReadAll(resp.Body)
+	resp_body, ra_err := io.ReadAll(resp.Body)
 	if ra_err != nil {
 		glog.Errorf("upload to filer response read %d: %v", resp.StatusCode, ra_err)
 		return etag, s3err.ErrInternalError

weed/s3api/s3api_object_handlers_postpolicy.go

@@ -5,17 +5,17 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/url"
+	"strings"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/s3api/policy"
 	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 	"github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
-	"io"
-	"io/ioutil"
-	"mime/multipart"
-	"net/http"
-	"net/url"
-	"strings"
 )
 
 func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
@@ -29,23 +29,23 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 
 	reader, err := r.MultipartReader()
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrMalformedPOSTRequest)
 		return
 	}
 	form, err := reader.ReadForm(int64(5 * humanize.MiByte))
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrMalformedPOSTRequest)
 		return
 	}
 	defer form.RemoveAll()
 
 	fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(form)
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrMalformedPOSTRequest)
 		return
 	}
 	if fileBody == nil {
-		s3err.WriteErrorResponse(w, s3err.ErrPOSTFileRequired, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrPOSTFileRequired)
 		return
 	}
 	defer fileBody.Close()
@@ -63,7 +63,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 	if successRedirect != "" {
 		redirectURL, err = url.Parse(successRedirect)
 		if err != nil {
-			s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrMalformedPOSTRequest)
 			return
 		}
 	}
@@ -71,13 +71,13 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 	// Verify policy signature.
 	errCode := s3a.iam.doesPolicySignatureMatch(formValues)
 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}
 
 	policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy"))
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrMalformedPOSTRequest)
 		return
 	}
 
@@ -86,7 +86,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 
 		postPolicyForm, err := policy.ParsePostPolicyForm(string(policyBytes))
 		if err != nil {
-			s3err.WriteErrorResponse(w, s3err.ErrPostPolicyConditionInvalidFormat, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrPostPolicyConditionInvalidFormat)
 			return
 		}
 
@@ -102,12 +102,12 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 		lengthRange := postPolicyForm.Conditions.ContentLengthRange
 		if lengthRange.Valid {
 			if fileSize < lengthRange.Min {
-				s3err.WriteErrorResponse(w, s3err.ErrEntityTooSmall, r)
+				s3err.WriteErrorResponse(w, r, s3err.ErrEntityTooSmall)
 				return
 			}
 
 			if fileSize > lengthRange.Max {
-				s3err.WriteErrorResponse(w, s3err.ErrEntityTooLarge, r)
+				s3err.WriteErrorResponse(w, r, s3err.ErrEntityTooLarge)
 				return
 			}
 		}
@@ -118,7 +118,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 	etag, errCode := s3a.putToFiler(r, uploadUrl, fileBody)
 
 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}
 
@@ -126,7 +126,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 		// Replace raw query params..
 		redirectURL.RawQuery = getRedirectPostRawQuery(bucket, object, etag)
 		w.Header().Set("Location", redirectURL.String())
-		s3err.WriteEmptyResponse(w, http.StatusSeeOther)
+		s3err.WriteEmptyResponse(w, r, http.StatusSeeOther)
 		return
 	}
 
@@ -141,18 +141,18 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 			ETag:     `"` + etag + `"`,
 			Location: w.Header().Get("Location"),
 		}
-		s3err.WriteXMLResponse(w, http.StatusCreated, resp)
+		s3err.WriteXMLResponse(w, r, http.StatusCreated, resp)
 	case "200":
-		s3err.WriteEmptyResponse(w, http.StatusOK)
+		s3err.WriteEmptyResponse(w, r, http.StatusOK)
 	default:
-		writeSuccessResponseEmpty(w)
+		writeSuccessResponseEmpty(w, r)
 	}
 
 }
 
 // Extract form fields and file data from a HTTP POST Policy
 func extractPostPolicyFormValues(form *multipart.Form) (filePart io.ReadCloser, fileName string, fileSize int64, formValues http.Header, err error) {
-	/// HTML Form values
+	// / HTML Form values
 	fileName = ""
 
 	// Canonicalize the form values into http.Header.
@@ -175,7 +175,7 @@ func extractPostPolicyFormValues(form *multipart.Form) (filePart io.ReadCloser,
 			b.WriteString(v)
 		}
 		fileSize = int64(b.Len())
-		filePart = ioutil.NopCloser(b)
+		filePart = io.NopCloser(b)
 		return filePart, fileName, fileSize, formValues, nil
 	}
 

weed/s3api/s3api_object_multipart_handlers.go

@@ -36,16 +36,20 @@ func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http
 		createMultipartUploadInput.Metadata[k] = aws.String(string(v))
 	}
 
+	contentType := r.Header.Get("Content-Type")
+	if contentType != "" {
+		createMultipartUploadInput.ContentType = &contentType
+	}
 	response, errCode := s3a.createMultipartUpload(createMultipartUploadInput)
 
 	glog.V(2).Info("NewMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)), errCode)
 
 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}
 
-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)
 
 }
 
@@ -65,11 +69,11 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r
 	glog.V(2).Info("CompleteMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)), errCode)
 
 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}
 
-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)
 
 }
 
@@ -87,13 +91,13 @@ func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *ht
 	})
 
 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}
 
 	glog.V(2).Info("AbortMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)))
 
-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)
 
 }
 
@@ -103,13 +107,13 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht
 
 	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType := getBucketMultipartResources(r.URL.Query())
 	if maxUploads < 0 {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxUploads, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxUploads)
 		return
 	}
 	if keyMarker != "" {
 		// Marker not common with prefix is not implemented.
 		if !strings.HasPrefix(keyMarker, prefix) {
-			s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
 			return
 		}
 	}
@@ -124,16 +128,16 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht
 		UploadIdMarker: aws.String(uploadIDMarker),
 	})
 
-	glog.V(2).Info("ListMultipartUploadsHandler", string(s3err.EncodeXMLResponse(response)), errCode)
+	glog.V(2).Infof("ListMultipartUploadsHandler %s errCode=%d", string(s3err.EncodeXMLResponse(response)), errCode)
 
 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}
 
 	// TODO handle encodingType
 
-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)
 }
 
 // ListObjectPartsHandler - Lists object parts in a multipart upload.
@@ -142,11 +146,11 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re
 
 	uploadID, partNumberMarker, maxParts, _ := getObjectResources(r.URL.Query())
 	if partNumberMarker < 0 {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidPartNumberMarker, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidPartNumberMarker)
 		return
 	}
 	if maxParts < 0 {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxParts, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxParts)
 		return
 	}
 
@@ -158,14 +162,14 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re
 		UploadId:         aws.String(uploadID),
 	})
 
-	glog.V(2).Info("ListObjectPartsHandler", string(s3err.EncodeXMLResponse(response)), errCode)
+	glog.V(2).Infof("ListObjectPartsHandler %s count=%d", string(s3err.EncodeXMLResponse(response)), len(response.Part))
 
 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}
 
-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)
 
 }
 
@@ -176,18 +180,18 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ
 	uploadID := r.URL.Query().Get("uploadId")
 	exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true)
 	if !exists {
-		s3err.WriteErrorResponse(w, s3err.ErrNoSuchUpload, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
 		return
 	}
 
 	partIDString := r.URL.Query().Get("partNumber")
 	partID, err := strconv.Atoi(partIDString)
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidPart, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidPart)
 		return
 	}
 	if partID > globalMaxPartID {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxParts, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxParts)
 		return
 	}
 
@@ -204,25 +208,30 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ
 			_, s3ErrCode = s3a.iam.reqSignatureV4Verify(r)
 		}
 		if s3ErrCode != s3err.ErrNone {
-			s3err.WriteErrorResponse(w, s3ErrCode, r)
+			s3err.WriteErrorResponse(w, r, s3ErrCode)
 			return
 		}
 	}
 	defer dataReader.Close()
 
+	glog.V(2).Infof("PutObjectPartHandler %s %s %04d", bucket, uploadID, partID)
+
 	uploadUrl := fmt.Sprintf("http://%s%s/%s/%04d.part?collection=%s",
 		s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(bucket), uploadID, partID, bucket)
 
-	etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader)
+	if partID == 1 && r.Header.Get("Content-Type") == "" {
+		dataReader = mimeDetect(r, dataReader)
+	}
 
+	etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader)
 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}
 
 	setEtag(w, etag)
 
-	writeSuccessResponseEmpty(w)
+	writeSuccessResponseEmpty(w, r)
 
 }
 

weed/s3api/s3api_object_skip_handlers.go

@@ -4,6 +4,14 @@ import (
 	"net/http"
 )
 
+// GetObjectAclHandler Put object ACL
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAcl.html
+func (s3a *S3ApiServer) GetObjectAclHandler(w http.ResponseWriter, r *http.Request) {
+
+	w.WriteHeader(http.StatusNoContent)
+
+}
+
 // PutObjectAclHandler Put object ACL
 // https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectAcl.html
 func (s3a *S3ApiServer) PutObjectAclHandler(w http.ResponseWriter, r *http.Request) {

weed/s3api/s3api_object_tagging_handlers.go

@@ -3,13 +3,13 @@ package s3api
 import (
 	"encoding/xml"
 	"fmt"
+	"io"
+	"net/http"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"io"
-	"io/ioutil"
-	"net/http"
 )
 
 // GetObjectTaggingHandler - GET object tagging
@@ -26,15 +26,15 @@ func (s3a *S3ApiServer) GetObjectTaggingHandler(w http.ResponseWriter, r *http.R
 	if err != nil {
 		if err == filer_pb.ErrNotFound {
 			glog.Errorf("GetObjectTaggingHandler %s: %v", r.URL, err)
-			s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
 		} else {
 			glog.Errorf("GetObjectTaggingHandler %s: %v", r.URL, err)
-			s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		}
 		return
 	}
 
-	writeSuccessResponseXML(w, FromTags(tags))
+	writeSuccessResponseXML(w, r, FromTags(tags))
 
 }
 
@@ -49,32 +49,32 @@ func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.R
 	dir, name := target.DirAndName()
 
 	tagging := &Tagging{}
-	input, err := ioutil.ReadAll(io.LimitReader(r.Body, r.ContentLength))
+	input, err := io.ReadAll(io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		glog.Errorf("PutObjectTaggingHandler read input %s: %v", r.URL, err)
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}
 	if err = xml.Unmarshal(input, tagging); err != nil {
 		glog.Errorf("PutObjectTaggingHandler Unmarshal %s: %v", r.URL, err)
-		s3err.WriteErrorResponse(w, s3err.ErrMalformedXML, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML)
 		return
 	}
 	tags := tagging.ToTags()
 	if len(tags) > 10 {
 		glog.Errorf("PutObjectTaggingHandler tags %s: %d tags more than 10", r.URL, len(tags))
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidTag, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag)
 		return
 	}
 	for k, v := range tags {
 		if len(k) > 128 {
 			glog.Errorf("PutObjectTaggingHandler tags %s: tag key %s longer than 128", r.URL, k)
-			s3err.WriteErrorResponse(w, s3err.ErrInvalidTag, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag)
 			return
 		}
 		if len(v) > 256 {
 			glog.Errorf("PutObjectTaggingHandler tags %s: tag value %s longer than 256", r.URL, v)
-			s3err.WriteErrorResponse(w, s3err.ErrInvalidTag, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag)
 			return
 		}
 	}
@@ -82,15 +82,15 @@ func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.R
 	if err = s3a.setTags(dir, name, tagging.ToTags()); err != nil {
 		if err == filer_pb.ErrNotFound {
 			glog.Errorf("PutObjectTaggingHandler setTags %s: %v", r.URL, err)
-			s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
 		} else {
 			glog.Errorf("PutObjectTaggingHandler setTags %s: %v", r.URL, err)
-			s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		}
 		return
 	}
 
-	w.WriteHeader(http.StatusNoContent)
+	w.WriteHeader(http.StatusOK)
 
 }
 
@@ -108,10 +108,10 @@ func (s3a *S3ApiServer) DeleteObjectTaggingHandler(w http.ResponseWriter, r *htt
 	if err != nil {
 		if err == filer_pb.ErrNotFound {
 			glog.Errorf("DeleteObjectTaggingHandler %s: %v", r.URL, err)
-			s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
 		} else {
 			glog.Errorf("DeleteObjectTaggingHandler %s: %v", r.URL, err)
-			s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		}
 		return
 	}

weed/s3api/s3api_objects_list_handlers.go

@@ -45,11 +45,11 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
 	originalPrefix, continuationToken, startAfter, delimiter, _, maxKeys := getListObjectsV2Args(r.URL.Query())
 
 	if maxKeys < 0 {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxKeys, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys)
 		return
 	}
 	if delimiter != "" && delimiter != "/" {
-		s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
 		return
 	}
 
@@ -61,13 +61,13 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
 	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)
 
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}
 
 	if len(response.Contents) == 0 {
 		if exists, existErr := s3a.exists(s3a.option.BucketsPath, bucket, true); existErr == nil && !exists {
-			s3err.WriteErrorResponse(w, s3err.ErrNoSuchBucket, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
 			return
 		}
 	}
@@ -87,7 +87,7 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
 		StartAfter:            startAfter,
 	}
 
-	writeSuccessResponseXML(w, responseV2)
+	writeSuccessResponseXML(w, r, responseV2)
 }
 
 func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
@@ -101,29 +101,29 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
 	originalPrefix, marker, delimiter, maxKeys := getListObjectsV1Args(r.URL.Query())
 
 	if maxKeys < 0 {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxKeys, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys)
 		return
 	}
 	if delimiter != "" && delimiter != "/" {
-		s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
 		return
 	}
 
 	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)
 
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}
 
 	if len(response.Contents) == 0 {
 		if exists, existErr := s3a.exists(s3a.option.BucketsPath, bucket, true); existErr == nil && !exists {
-			s3err.WriteErrorResponse(w, s3err.ErrNoSuchBucket, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
 			return
 		}
 	}
 
-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)
 }
 
 func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, maxKeys int, marker string, delimiter string) (response ListBucketResult, err error) {
@@ -220,12 +220,16 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
 			err = subErr
 			return
 		}
+		counter += subCounter
 		isTruncated = isTruncated || subIsTruncated
 		maxKeys -= subCounter
 		nextMarker = subDir + "/" + subNextMarker
 		// finished processing this sub directory
 		marker = subDir
 	}
+	if maxKeys <= 0 {
+		return
+	}
 
 	// now marker is also a direct child of dir
 	request := &filer_pb.ListEntriesRequest{

weed/s3api/s3api_policy.go

@@ -0,0 +1,147 @@
+package s3api
+
+import (
+	"encoding/xml"
+	"time"
+)
+
+// Status represents lifecycle configuration status
+type ruleStatus string
+
+// Supported status types
+const (
+	Enabled  ruleStatus = "Enabled"
+	Disabled ruleStatus = "Disabled"
+)
+
+// Lifecycle - Configuration for bucket lifecycle.
+type Lifecycle struct {
+	XMLName xml.Name `xml:"LifecycleConfiguration"`
+	Rules   []Rule   `xml:"Rule"`
+}
+
+// Rule - a rule for lifecycle configuration.
+type Rule struct {
+	XMLName    xml.Name   `xml:"Rule"`
+	ID         string     `xml:"ID,omitempty"`
+	Status     ruleStatus `xml:"Status"`
+	Filter     Filter     `xml:"Filter,omitempty"`
+	Prefix     Prefix     `xml:"Prefix,omitempty"`
+	Expiration Expiration `xml:"Expiration,omitempty"`
+	Transition Transition `xml:"Transition,omitempty"`
+}
+
+// Filter - a filter for a lifecycle configuration Rule.
+type Filter struct {
+	XMLName xml.Name `xml:"Filter"`
+	set     bool
+
+	Prefix Prefix
+
+	And    And
+	andSet bool
+
+	Tag    Tag
+	tagSet bool
+}
+
+// Prefix holds the prefix xml tag in <Rule> and <Filter>
+type Prefix struct {
+	string
+	set bool
+}
+
+// MarshalXML - decodes XML data.
+func (p Prefix) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
+	if !p.set {
+		return nil
+	}
+	return e.EncodeElement(p.string, startElement)
+}
+
+func (f Filter) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	if err := e.EncodeToken(start); err != nil {
+		return err
+	}
+	if err := e.EncodeElement(f.Prefix, xml.StartElement{Name: xml.Name{Local: "Prefix"}}); err != nil {
+		return err
+	}
+	return e.EncodeToken(xml.EndElement{Name: start.Name})
+}
+
+// And - a tag to combine a prefix and multiple tags for lifecycle configuration rule.
+type And struct {
+	XMLName xml.Name `xml:"And"`
+	Prefix  Prefix   `xml:"Prefix,omitempty"`
+	Tags    []Tag    `xml:"Tag,omitempty"`
+}
+
+// Expiration - expiration actions for a rule in lifecycle configuration.
+type Expiration struct {
+	XMLName      xml.Name           `xml:"Expiration"`
+	Days         int                `xml:"Days,omitempty"`
+	Date         ExpirationDate     `xml:"Date,omitempty"`
+	DeleteMarker ExpireDeleteMarker `xml:"ExpiredObjectDeleteMarker"`
+
+	set bool
+}
+
+// MarshalXML encodes expiration field into an XML form.
+func (e Expiration) MarshalXML(enc *xml.Encoder, startElement xml.StartElement) error {
+	if !e.set {
+		return nil
+	}
+	type expirationWrapper Expiration
+	return enc.EncodeElement(expirationWrapper(e), startElement)
+}
+
+// ExpireDeleteMarker represents value of ExpiredObjectDeleteMarker field in Expiration XML element.
+type ExpireDeleteMarker struct {
+	val bool
+	set bool
+}
+
+// MarshalXML encodes delete marker boolean into an XML form.
+func (b ExpireDeleteMarker) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
+	if !b.set {
+		return nil
+	}
+	return e.EncodeElement(b.val, startElement)
+}
+
+// ExpirationDate is a embedded type containing time.Time to unmarshal
+// Date in Expiration
+type ExpirationDate struct {
+	time.Time
+}
+
+// MarshalXML encodes expiration date if it is non-zero and encodes
+// empty string otherwise
+func (eDate ExpirationDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
+	if eDate.Time.IsZero() {
+		return nil
+	}
+	return e.EncodeElement(eDate.Format(time.RFC3339), startElement)
+}
+
+// Transition - transition actions for a rule in lifecycle configuration.
+type Transition struct {
+	XMLName      xml.Name  `xml:"Transition"`
+	Days         int       `xml:"Days,omitempty"`
+	Date         time.Time `xml:"Date,omitempty"`
+	StorageClass string    `xml:"StorageClass,omitempty"`
+
+	set bool
+}
+
+// MarshalXML encodes transition field into an XML form.
+func (t Transition) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
+	if !t.set {
+		return nil
+	}
+	type transitionWrapper Transition
+	return enc.EncodeElement(transitionWrapper(t), start)
+}
+
+// TransitionDays is a type alias to unmarshal Days in Transition
+type TransitionDays int

weed/s3api/s3api_server.go

@@ -66,7 +66,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
 		// HeadObject
 		bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.HeadObjectHandler, ACTION_READ), "GET"))
 		// HeadBucket
-		bucket.Methods("HEAD").HandlerFunc(track(s3a.iam.Auth(s3a.HeadBucketHandler, ACTION_ADMIN), "GET"))
+		bucket.Methods("HEAD").HandlerFunc(track(s3a.iam.Auth(s3a.HeadBucketHandler, ACTION_READ), "GET"))
 
 		// CopyObjectPart
 		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", `.*?(\/|%2F).*?`).HandlerFunc(track(s3a.iam.Auth(s3a.CopyObjectPartHandler, ACTION_WRITE), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
@@ -103,26 +103,45 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
 		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(track(s3a.iam.Auth(s3a.CopyObjectHandler, ACTION_WRITE), "COPY"))
 		// PutObject
 		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.PutObjectHandler, ACTION_WRITE), "PUT"))
-		// PutBucket
-		bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketHandler, ACTION_ADMIN), "PUT"))
 
 		// DeleteObject
 		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteObjectHandler, ACTION_WRITE), "DELETE"))
-		// DeleteBucket
-		bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketHandler, ACTION_WRITE), "DELETE"))
 
 		// ListObjectsV2
 		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.ListObjectsV2Handler, ACTION_LIST), "LIST")).Queries("list-type", "2")
 		// GetObject, but directory listing is not supported
 		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.GetObjectHandler, ACTION_READ), "GET"))
-		// ListObjectsV1 (Legacy)
-		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.ListObjectsV1Handler, ACTION_LIST), "LIST"))
 
 		// PostPolicy
 		bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(track(s3a.iam.Auth(s3a.PostPolicyBucketHandler, ACTION_WRITE), "POST"))
 
 		// DeleteMultipleObjects
 		bucket.Methods("POST").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteMultipleObjectsHandler, ACTION_WRITE), "DELETE")).Queries("delete", "")
+
+		// GetBucketACL
+		bucket.Methods("GET").HandlerFunc(s3a.iam.Auth(s3a.GetBucketAclHandler, ACTION_READ)).Queries("acl", "")
+
+		// GetObjectACL
+		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(s3a.iam.Auth(s3a.GetObjectAclHandler, ACTION_READ)).Queries("acl", "")
+
+		// GetBucketLifecycleConfiguration
+		bucket.Methods("GET").HandlerFunc(s3a.iam.Auth(s3a.GetBucketLifecycleConfigurationHandler, ACTION_READ)).Queries("lifecycle", "")
+
+		// PutBucketLifecycleConfiguration
+		bucket.Methods("PUT").HandlerFunc(s3a.iam.Auth(s3a.PutBucketLifecycleConfigurationHandler, ACTION_WRITE)).Queries("lifecycle", "")
+
+		// DeleteBucketLifecycleConfiguration
+		bucket.Methods("DELETE").HandlerFunc(s3a.iam.Auth(s3a.DeleteBucketLifecycleHandler, ACTION_WRITE)).Queries("lifecycle", "")
+
+		// ListObjectsV1 (Legacy)
+		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.ListObjectsV1Handler, ACTION_LIST), "LIST"))
+
+		// PutBucket
+		bucket.Methods("PUT").HandlerFunc(track(s3a.PutBucketHandler, "PUT"))
+
+		// DeleteBucket
+		bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketHandler, ACTION_WRITE), "DELETE"))
+
 		/*
 
 			// not implemented
@@ -132,8 +151,6 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
 			bucket.Methods("GET").HandlerFunc(s3a.GetBucketPolicyHandler).Queries("policy", "")
 			// GetObjectACL
 			bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(s3a.GetObjectACLHandler).Queries("acl", "")
-			// GetBucketACL
-			bucket.Methods("GET").HandlerFunc(s3a.GetBucketACLHandler).Queries("acl", "")
 			// PutBucketPolicy
 			bucket.Methods("PUT").HandlerFunc(s3a.PutBucketPolicyHandler).Queries("policy", "")
 			// DeleteBucketPolicy

weed/s3api/s3api_status_handlers.go

@@ -4,5 +4,5 @@ import "net/http"
 
 func (s3a *S3ApiServer) StatusHandler(w http.ResponseWriter, r *http.Request) {
 	// write out the response code and content type header
-	writeSuccessResponseEmpty(w)
+	writeSuccessResponseEmpty(w, r)
 }

weed/s3api/s3api_xsd_generated.go

@@ -8,12 +8,12 @@ import (
 )
 
 type AccessControlList struct {
-	Grant []Grant `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Grant,omitempty"`
+	Grant []Grant `xml:"Grant,omitempty"`
 }
 
 type AccessControlPolicy struct {
-	Owner             CanonicalUser     `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Owner"`
-	AccessControlList AccessControlList `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlList"`
+	Owner             CanonicalUser     `xml:"Owner"`
+	AccessControlList AccessControlList `xml:"AccessControlList"`
 }
 
 type AmazonCustomerByEmail struct {
@@ -467,11 +467,17 @@ func (t *GetObjectResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement) e
 }
 
 type Grant struct {
-	Grantee    Grantee    `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Grantee"`
-	Permission Permission `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Permission"`
+	Grantee    Grantee    `xml:"Grantee"`
+	Permission Permission `xml:"Permission"`
 }
 
 type Grantee struct {
+	XMLNS       string `xml:"xmlns:xsi,attr"`
+	XMLXSI      string `xml:"xsi:type,attr"`
+	Type        string `xml:"Type"`
+	ID          string `xml:"ID,omitempty"`
+	DisplayName string `xml:"DisplayName,omitempty"`
+	URI         string `xml:"URI,omitempty"`
 }
 
 type Group struct {

weed/s3api/s3err/error_handler.go

@@ -19,15 +19,15 @@ const (
 	MimeXML  mimeType = "application/xml"
 )
 
-func WriteXMLResponse(w http.ResponseWriter, statusCode int, response interface{}) {
-	WriteResponse(w, statusCode, EncodeXMLResponse(response), MimeXML)
+func WriteXMLResponse(w http.ResponseWriter, r *http.Request, statusCode int, response interface{}) {
+	WriteResponse(w, r, statusCode, EncodeXMLResponse(response), MimeXML)
 }
 
-func WriteEmptyResponse(w http.ResponseWriter, statusCode int) {
-	WriteResponse(w, statusCode, []byte{}, mimeNone)
+func WriteEmptyResponse(w http.ResponseWriter, r *http.Request, statusCode int) {
+	WriteResponse(w, r, statusCode, []byte{}, mimeNone)
 }
 
-func WriteErrorResponse(w http.ResponseWriter, errorCode ErrorCode, r *http.Request) {
+func WriteErrorResponse(w http.ResponseWriter, r *http.Request, errorCode ErrorCode) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object := vars["object"]
@@ -38,7 +38,7 @@ func WriteErrorResponse(w http.ResponseWriter, errorCode ErrorCode, r *http.Requ
 	apiError := GetAPIError(errorCode)
 	errorResponse := getRESTErrorResponse(apiError, r.URL.Path, bucket, object)
 	encodedErrorResponse := EncodeXMLResponse(errorResponse)
-	WriteResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, MimeXML)
+	WriteResponse(w, r, apiError.HTTPStatusCode, encodedErrorResponse, MimeXML)
 }
 
 func getRESTErrorResponse(err APIError, resource string, bucket, object string) RESTErrorResponse {
@@ -61,13 +61,17 @@ func EncodeXMLResponse(response interface{}) []byte {
 	return bytesBuffer.Bytes()
 }
 
-func setCommonHeaders(w http.ResponseWriter) {
+func setCommonHeaders(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("x-amz-request-id", fmt.Sprintf("%d", time.Now().UnixNano()))
 	w.Header().Set("Accept-Ranges", "bytes")
+	if r.Header.Get("Origin") != "" {
+		w.Header().Set("Access-Control-Allow-Origin", "*")
+		w.Header().Set("Access-Control-Allow-Credentials", "true")
+	}
 }
 
-func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
-	setCommonHeaders(w)
+func WriteResponse(w http.ResponseWriter, r *http.Request, statusCode int, response []byte, mType mimeType) {
+	setCommonHeaders(w, r)
 	if response != nil {
 		w.Header().Set("Content-Length", strconv.Itoa(len(response)))
 	}
@@ -88,5 +92,5 @@ func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType
 // If none of the http routes match respond with MethodNotAllowed
 func NotFoundHandler(w http.ResponseWriter, r *http.Request) {
 	glog.V(0).Infof("unsupported %s %s", r.Method, r.RequestURI)
-	WriteErrorResponse(w, ErrMethodNotAllowed, r)
+	WriteErrorResponse(w, r, ErrMethodNotAllowed)
 }

weed/s3api/s3err/s3api_errors.go

@@ -51,6 +51,7 @@ const (
 	ErrBucketAlreadyExists
 	ErrBucketAlreadyOwnedByYou
 	ErrNoSuchBucket
+	ErrNoSuchLifecycleConfiguration
 	ErrNoSuchKey
 	ErrNoSuchUpload
 	ErrInvalidBucketName
@@ -163,6 +164,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The specified bucket does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrNoSuchLifecycleConfiguration: {
+		Code:           "NoSuchLifecycleConfiguration",
+		Description:    "The lifecycle configuration does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrNoSuchKey: {
 		Code:           "NoSuchKey",
 		Description:    "The specified key does not exist.",
@@ -196,7 +202,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidTag: {
-		Code:           "InvalidArgument",
+		Code:           "InvalidTag",
 		Description:    "The Tag value you have provided is invalid",
 		HTTPStatusCode: http.StatusBadRequest,
 	},

weed/s3api/tags.go

@@ -14,8 +14,9 @@ type TagSet struct {
 }
 
 type Tagging struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"`
+	XMLName xml.Name `xml:"Tagging"`
 	TagSet  TagSet   `xml:"TagSet"`
+	Xmlns   string   `xml:"xmlns,attr"`
 }
 
 func (t *Tagging) ToTags() map[string]string {
@@ -27,7 +28,7 @@ func (t *Tagging) ToTags() map[string]string {
 }
 
 func FromTags(tags map[string]string) (t *Tagging) {
-	t = &Tagging{}
+	t = &Tagging{Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/"}
 	for k, v := range tags {
 		t.TagSet.Tag = append(t.TagSet.Tag, Tag{
 			Key:   k,

weed/s3api/tags_test.go

@@ -32,6 +32,7 @@ func TestXMLUnmarshall(t *testing.T) {
 
 func TestXMLMarshall(t *testing.T) {
 	tags := &Tagging{
+		Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/",
 		TagSet: TagSet{
 			[]Tag{
 				{

weed/security/tls.go

@@ -4,18 +4,18 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
-	"github.com/chrislusf/seaweedfs/weed/util"
-	grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/peer"
-	"google.golang.org/grpc/status"
-	"io/ioutil"
+	"os"
 	"strings"
 
+	grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/status"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 type Authenticator struct {
@@ -37,7 +37,7 @@ func LoadServerTLS(config *util.ViperProxy, component string) (grpc.ServerOption
 			err)
 		return nil, nil
 	}
-	caCert, err := ioutil.ReadFile(config.GetString("grpc.ca"))
+	caCert, err := os.ReadFile(config.GetString("grpc.ca"))
 	if err != nil {
 		glog.V(1).Infof("read ca cert file %s error: %v", config.GetString("grpc.ca"), err)
 		return nil, nil
@@ -82,7 +82,7 @@ func LoadClientTLS(config *util.ViperProxy, component string) grpc.DialOption {
 		glog.V(1).Infof("load cert/key error: %v", err)
 		return grpc.WithInsecure()
 	}
-	caCert, err := ioutil.ReadFile(caFileName)
+	caCert, err := os.ReadFile(caFileName)
 	if err != nil {
 		glog.V(1).Infof("read ca cert file error: %v", err)
 		return grpc.WithInsecure()

weed/server/common.go

@@ -9,6 +9,7 @@ import (
 	"io/fs"
 	"mime/multipart"
 	"net/http"
+	"net/url"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -250,7 +251,16 @@ func handleStaticResources2(r *mux.Router) {
 }
 
 func adjustHeaderContentDisposition(w http.ResponseWriter, r *http.Request, filename string) {
+	responseContentDisposition := r.FormValue("response-content-disposition")
+	if responseContentDisposition != "" {
+		w.Header().Set("Content-Disposition", responseContentDisposition)
+		return
+	}
+	if w.Header().Get("Content-Disposition") != "" {
+		return
+	}
 	if filename != "" {
+		filename = url.QueryEscape(filename)
 		contentDisposition := "inline"
 		if r.FormValue("dl") != "" {
 			if dl, _ := strconv.ParseBool(r.FormValue("dl")); dl {

weed/server/filer_grpc_server_remote.go

@@ -17,7 +17,7 @@ import (
 	"time"
 )
 
-func (fs *FilerServer) DownloadToLocal(ctx context.Context, req *filer_pb.DownloadToLocalRequest) (*filer_pb.DownloadToLocalResponse, error) {
+func (fs *FilerServer) CacheRemoteObjectToLocalCluster(ctx context.Context, req *filer_pb.CacheRemoteObjectToLocalClusterRequest) (*filer_pb.CacheRemoteObjectToLocalClusterResponse, error) {
 
 	// load all mappings
 	mappingEntry, err := fs.filer.FindEntry(ctx, util.JoinPath(filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE))
@@ -57,7 +57,7 @@ func (fs *FilerServer) DownloadToLocal(ctx context.Context, req *filer_pb.Downlo
 		return nil, err
 	}
 
-	resp := &filer_pb.DownloadToLocalResponse{}
+	resp := &filer_pb.CacheRemoteObjectToLocalClusterResponse{}
 	if entry.Remote == nil || entry.Remote.RemoteSize == 0 {
 		return resp, nil
 	}