mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-11 00:55:51 +00:00
Compare commits
31
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5f82bdbed0 | ||
|
|
852173f648 | ||
|
|
38e5871c3b | ||
|
|
dbb1e4b182 | ||
|
|
6079a0ae21 | ||
|
|
d93c90fdb3 | ||
|
|
b198a0e8d1 | ||
|
|
4e6e7b6ac5 | ||
|
|
50eba1ecf8 | ||
|
|
a3739c9c8c | ||
|
|
02a041b28d | ||
|
|
c6721bb18d | ||
|
|
d5f21fd8ba | ||
|
|
cea12ba3c4 | ||
|
|
9196696278 | ||
|
|
ff4855dcbe | ||
|
|
772459f93c | ||
|
|
086ab3e28c | ||
|
|
ca1ad9c4c2 | ||
|
|
1856eaca03 | ||
|
|
95f6a2bc13 | ||
|
|
51c630b5ff | ||
|
|
21ec3a51c1 | ||
|
|
4709dbf4f8 | ||
|
|
1a22bdbac1 | ||
|
|
a912371f2f | ||
|
|
52a7ed03bc | ||
|
|
0c8e8fb411 | ||
|
|
2f19409c32 | ||
|
|
887c9ee97a | ||
|
|
0a0eb21b86 |
@@ -38,13 +38,13 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -60,7 +60,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -87,13 +87,13 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -109,7 +109,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
|
||||
- name: Prepare Docker tag
|
||||
id: tag
|
||||
|
||||
@@ -9,6 +9,6 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@3c4e3dcb1aa7874d2c16be7d79418e9b7efd6261
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: ec-integration-test-logs
|
||||
path: test/erasure_coding
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
|
||||
@@ -12,7 +12,7 @@ jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
|
||||
- name: Publish Helm charts
|
||||
uses: stefanprodan/helm-gh-pages@v1.7.0
|
||||
with:
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
|
||||
@@ -35,10 +35,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go ${{ env.GO_VERSION }}
|
||||
uses: actions/setup-go@v5
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
|
||||
|
||||
@@ -87,7 +87,7 @@ Table of Contents
|
||||
* `export AWS_ACCESS_KEY_ID=admin ; export AWS_SECRET_ACCESS_KEY=key` as the admin credentials to access the object store.
|
||||
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
|
||||
|
||||
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
|
||||
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -master="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
|
||||
|
||||
## Quick Start SeaweedFS S3 on AWS ##
|
||||
* Setup fast production-ready [SeaweedFS S3 on AWS with cloudformation](https://aws.amazon.com/marketplace/pp/prodview-nzelz5gprlrjc)
|
||||
@@ -206,8 +206,8 @@ SeaweedFS uses HTTP REST operations to read, write, and delete. The responses ar
|
||||
### Start Volume Servers ###
|
||||
|
||||
```
|
||||
> weed volume -dir="/tmp/data1" -max=5 -mserver="localhost:9333" -port=8080 &
|
||||
> weed volume -dir="/tmp/data2" -max=10 -mserver="localhost:9333" -port=8081 &
|
||||
> weed volume -dir="/tmp/data1" -max=5 -master="localhost:9333" -port=8080 &
|
||||
> weed volume -dir="/tmp/data2" -max=10 -master="localhost:9333" -port=8081 &
|
||||
```
|
||||
|
||||
### Write File ###
|
||||
|
||||
@@ -25,7 +25,7 @@ services:
|
||||
ports:
|
||||
- "8080:8080"
|
||||
- "18080:18080"
|
||||
command: "volume -mserver=master:9333 -ip=volume1 -dir=/data -max=10"
|
||||
command: "volume -master=master:9333 -ip=volume1 -dir=/data -max=10"
|
||||
depends_on:
|
||||
- master
|
||||
volumes:
|
||||
@@ -38,7 +38,7 @@ services:
|
||||
ports:
|
||||
- "8081:8080"
|
||||
- "18081:18080"
|
||||
command: "volume -mserver=master:9333 -ip=volume2 -dir=/data -max=10"
|
||||
command: "volume -master=master:9333 -ip=volume2 -dir=/data -max=10"
|
||||
depends_on:
|
||||
- master
|
||||
volumes:
|
||||
@@ -51,7 +51,7 @@ services:
|
||||
ports:
|
||||
- "8082:8080"
|
||||
- "18082:18080"
|
||||
command: "volume -mserver=master:9333 -ip=volume3 -dir=/data -max=10"
|
||||
command: "volume -master=master:9333 -ip=volume3 -dir=/data -max=10"
|
||||
depends_on:
|
||||
- master
|
||||
volumes:
|
||||
@@ -64,7 +64,7 @@ services:
|
||||
ports:
|
||||
- "8083:8080"
|
||||
- "18083:18080"
|
||||
command: "volume -mserver=master:9333 -ip=volume4 -dir=/data -max=10"
|
||||
command: "volume -master=master:9333 -ip=volume4 -dir=/data -max=10"
|
||||
depends_on:
|
||||
- master
|
||||
volumes:
|
||||
@@ -77,7 +77,7 @@ services:
|
||||
ports:
|
||||
- "8084:8080"
|
||||
- "18084:18080"
|
||||
command: "volume -mserver=master:9333 -ip=volume5 -dir=/data -max=10"
|
||||
command: "volume -master=master:9333 -ip=volume5 -dir=/data -max=10"
|
||||
depends_on:
|
||||
- master
|
||||
volumes:
|
||||
@@ -90,7 +90,7 @@ services:
|
||||
ports:
|
||||
- "8085:8080"
|
||||
- "18085:18080"
|
||||
command: "volume -mserver=master:9333 -ip=volume6 -dir=/data -max=10"
|
||||
command: "volume -master=master:9333 -ip=volume6 -dir=/data -max=10"
|
||||
depends_on:
|
||||
- master
|
||||
volumes:
|
||||
|
||||
@@ -13,7 +13,7 @@ services:
|
||||
|
||||
volume:
|
||||
image: chrislusf/seaweedfs:e2e
|
||||
command: "-v=4 volume -mserver=master:9333 -ip=volume -ip.bind=0.0.0.0 -preStopSeconds=1"
|
||||
command: "-v=4 volume -master=master:9333 -ip=volume -ip.bind=0.0.0.0 -preStopSeconds=1"
|
||||
healthcheck:
|
||||
test: [ "CMD", "curl", "--fail", "-I", "http://localhost:8080/healthz" ]
|
||||
interval: 2s
|
||||
|
||||
@@ -36,7 +36,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: 'volume -dataCenter=dc1 -rack=v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc1 -rack=v1 -master="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
@@ -46,7 +46,7 @@ services:
|
||||
ports:
|
||||
- 8082:8082
|
||||
- 18082:18082
|
||||
command: 'volume -dataCenter=dc2 -rack=v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc2 -rack=v2 -master="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
@@ -56,7 +56,7 @@ services:
|
||||
ports:
|
||||
- 8083:8083
|
||||
- 18083:18083
|
||||
command: 'volume -dataCenter=dc3 -rack=v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc3 -rack=v3 -master="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
|
||||
@@ -36,7 +36,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: 'volume -dataCenter=dc1 -rack=v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc1 -rack=v1 -master="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
@@ -46,7 +46,7 @@ services:
|
||||
ports:
|
||||
- 8082:8082
|
||||
- 18082:18082
|
||||
command: 'volume -dataCenter=dc2 -rack=v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc2 -rack=v2 -master="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
@@ -56,7 +56,7 @@ services:
|
||||
ports:
|
||||
- 8083:8083
|
||||
- 18083:18083
|
||||
command: 'volume -dataCenter=dc3 -rack=v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc3 -rack=v3 -master="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
|
||||
@@ -16,7 +16,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: "-v=1 volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1 -max=10000"
|
||||
command: "-v=1 volume -master=master:9333 -port=8080 -ip=volume -preStopSeconds=1 -max=10000"
|
||||
depends_on:
|
||||
- master
|
||||
volumes:
|
||||
|
||||
@@ -42,7 +42,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: 'volume -dataCenter=dc1 -rack=v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc1 -rack=v1 -master="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
@@ -51,7 +51,7 @@ services:
|
||||
ports:
|
||||
- 8082:8082
|
||||
- 18082:18082
|
||||
command: 'volume -dataCenter=dc2 -rack=v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc2 -rack=v2 -master="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
@@ -60,7 +60,7 @@ services:
|
||||
ports:
|
||||
- 8083:8083
|
||||
- 18083:18083
|
||||
command: 'volume -dataCenter=dc3 -rack=v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc3 -rack=v3 -master="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
|
||||
@@ -12,7 +12,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: "volume -mserver=master:9333 -port=8080 -ip=volume"
|
||||
command: "volume -master=master:9333 -port=8080 -ip=volume"
|
||||
depends_on:
|
||||
- master
|
||||
mysql:
|
||||
|
||||
@@ -12,7 +12,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
|
||||
command: "volume -master=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
|
||||
depends_on:
|
||||
- master
|
||||
s3:
|
||||
|
||||
@@ -12,7 +12,7 @@ services:
|
||||
ports:
|
||||
- 7455:8080
|
||||
- 9325:9325
|
||||
command: 'volume -mserver="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455'
|
||||
command: 'volume -master="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455'
|
||||
depends_on:
|
||||
- master
|
||||
filer:
|
||||
|
||||
@@ -14,7 +14,7 @@ services:
|
||||
- 9325:9325
|
||||
volumes:
|
||||
- /Volumes/mobile_disk/99:/data
|
||||
command: 'volume -mserver="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455'
|
||||
command: 'volume -master="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455'
|
||||
depends_on:
|
||||
- master
|
||||
filer:
|
||||
|
||||
@@ -12,7 +12,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: "volume -mserver=master:9333 -port=8080 -ip=volume"
|
||||
command: "volume -master=master:9333 -port=8080 -ip=volume"
|
||||
depends_on:
|
||||
- master
|
||||
s3:
|
||||
|
||||
@@ -12,7 +12,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
|
||||
command: "volume -master=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
|
||||
depends_on:
|
||||
- master
|
||||
s3:
|
||||
|
||||
@@ -12,7 +12,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: "volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
|
||||
command: "volume -master=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
|
||||
depends_on:
|
||||
- master
|
||||
filer:
|
||||
|
||||
@@ -15,7 +15,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: "volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
|
||||
command: "volume -master=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
|
||||
depends_on:
|
||||
- master
|
||||
s3:
|
||||
|
||||
@@ -71,7 +71,7 @@ services:
|
||||
- net
|
||||
command:
|
||||
- 'volume'
|
||||
- '-mserver=master:9333'
|
||||
- '-master=master:9333'
|
||||
- '-port=8080'
|
||||
deploy:
|
||||
mode: global
|
||||
|
||||
@@ -17,7 +17,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
|
||||
command: "volume -master=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
|
||||
depends_on:
|
||||
- master
|
||||
s3:
|
||||
|
||||
@@ -14,7 +14,7 @@ services:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
- 9325:9325
|
||||
command: 'volume -ip=volume -mserver="master:9333" -ip.bind=0.0.0.0 -port=8080 -metricsPort=9325'
|
||||
command: 'volume -ip=volume -master="master:9333" -ip.bind=0.0.0.0 -port=8080 -metricsPort=9325'
|
||||
depends_on:
|
||||
- master
|
||||
filer:
|
||||
|
||||
@@ -12,7 +12,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: 'volume -ip=volume -mserver="master:9333" -ip.bind=0.0.0.0 -port=8080'
|
||||
command: 'volume -ip=volume -master="master:9333" -ip.bind=0.0.0.0 -port=8080'
|
||||
depends_on:
|
||||
- master
|
||||
filer:
|
||||
|
||||
@@ -51,7 +51,7 @@ require (
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/klauspost/compress v1.18.2
|
||||
github.com/klauspost/reedsolomon v1.12.5
|
||||
github.com/klauspost/reedsolomon v1.12.6
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/linxGnu/grocksdb v1.10.3
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
@@ -118,7 +118,7 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/kms v1.23.1
|
||||
cloud.google.com/go/kms v1.23.2
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
|
||||
github.com/Jille/raft-grpc-transport v1.6.1
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1
|
||||
@@ -126,10 +126,10 @@ require (
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20240515141816-262c6fe778ad
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.0
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.3
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.20
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.3
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0
|
||||
github.com/cognusion/imaging v1.0.2
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
github.com/getsentry/sentry-go v0.38.0
|
||||
@@ -162,7 +162,7 @@ require (
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.6
|
||||
go.uber.org/atomic v1.11.0
|
||||
golang.org/x/sync v0.18.0
|
||||
golang.org/x/sync v0.19.0
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated
|
||||
google.golang.org/grpc/security/advancedtls v1.0.0
|
||||
)
|
||||
@@ -260,23 +260,23 @@ require (
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 // indirect
|
||||
github.com/aws/smithy-go v1.23.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 // indirect
|
||||
github.com/aws/smithy-go v1.24.0 // indirect
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||
|
||||
@@ -290,8 +290,8 @@ cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxs
|
||||
cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
|
||||
cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0=
|
||||
cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w=
|
||||
cloud.google.com/go/kms v1.23.1 h1:Mesyv84WoP3tPjUC0O5LRqPWICO0ufdpWf9jtBCEz64=
|
||||
cloud.google.com/go/kms v1.23.1/go.mod h1:rZ5kK0I7Kn9W4erhYVoIRPtpizjunlrfU4fUkumUp8g=
|
||||
cloud.google.com/go/kms v1.23.2 h1:4IYDQL5hG4L+HzJBhzejUySoUOheh3Lk5YT4PCyyW6k=
|
||||
cloud.google.com/go/kms v1.23.2/go.mod h1:rZ5kK0I7Kn9W4erhYVoIRPtpizjunlrfU4fUkumUp8g=
|
||||
cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
|
||||
cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
|
||||
cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
|
||||
@@ -665,48 +665,48 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
|
||||
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc=
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2/go.mod h1:IusfVNTmiSN3t4rhxWFaBAqn+mcNdwKtPcV16eYdgko=
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.1 h1:difXb4maDZkRH0x//Qkwcfpdg1XQVXEAEs2DdXldFFc=
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.20 h1:KFndAnHd9NUuzikHjQ8D5CfFVO+bgELkmcGY8yAw98Q=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.20/go.mod h1:9mCi28a+fmBHSQ0UM79omkz6JtN+PEsvLrnG36uoUv0=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 h1:VO3FIM2TDbm0kqp6sFNR0PbioXJb/HzCDW6NtIZpIWE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12/go.mod h1:6C39gB8kg82tx3r72muZSrNhHia9rjGkX7ORaS2GKNE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.3 h1:01Ym72hK43hjwDeJUfi1l2oYLXBAOR8gNSZNmXmvuas=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.3/go.mod h1:55nWF/Sr9Zvls0bGnWkRxUdhzKqj9uRNlPvgV1vgxKc=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 h1:utxLraaifrSBkeyII9mIbVwXXWrZdlPO7FIKmyLCEcY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15/go.mod h1:hW6zjYUDQwfz3icf4g2O41PHi77u10oAzJ84iSzR/lo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 h1:p/9flfXdoAnwJnuW9xHEAFY22R3A6skYkW19JFF9F+8=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12/go.mod h1:ZTLHakoVCTtW8AaLGSwJ3LXqHD9uQKnOcv1TrpO6u2k=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 h1:2lTWFvRcnWFFLzHWmtddu5MTchc5Oj2OOey++99tPZ0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12/go.mod h1:hI92pK+ho8HVcWMHKHrK3Uml4pfG7wvL86FzO0LVtQQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 h1:Y5YXgygXwDI5P4RkteB5yF7v35neH7LfJKBG+hzIons=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15/go.mod h1:K+/1EpG42dFSY7CBj+Fruzm8PsCGWTXJ3jdeJ659oGQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 h1:AvltKnW9ewxX2hFmQS0FyJH93aSvJVUEFvXfU+HWtSE=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15/go.mod h1:3I4oCdZdmgrREhU74qS1dK9yZ62yumob+58AbFR4cQA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 h1:itu4KHu8JK/N6NcLIISlf3LL1LccMqruLUXZ9y7yBZw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12/go.mod h1:i+6vTU3xziikTY3vcox23X8pPGW5X3wVgd1VZ7ha+x8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 h1:NEe7FaViguRQEm8zl8Ay/kC/QRsMtWUiCGZajQIsLdc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3/go.mod h1:JLuCKu5VfiLBBBl/5IzZILU7rxS0koQpHzMOCzycOJU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 h1:MM8imH7NZ0ovIVX7D2RxfMDv7Jt9OiUXkcQ+GqywA7M=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12/go.mod h1:gf4OGwdNkbEsb7elw2Sy76odfhwNktWII3WgvQgQQ6w=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 h1:R3uW0iKl8rgNEXNjVGliW/oMEh9fO/LlUEV8RvIFr1I=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12/go.mod h1:XEttbEr5yqsw8ebi7vlDoGJJjMXRez4/s9pibpJyL5s=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 h1:Dq82AV+Qxpno/fG162eAhnD8d48t9S+GZCfz7yv1VeA=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1/go.mod h1:MbKLznDKpf7PnSonNRUVYZzfP0CeLkRIUexeblgKcU4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 h1:NLYTEyZmVZo0Qh183sC8nC+ydJXOOeIL/qI/sS3PdLY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15/go.mod h1:Z803iB3B0bc8oJV8zH2PERLRfQUJ2n2BXISpsA4+O1M=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 h1:P1MU/SuhadGvg2jtviDXPEejU3jBNhoeeAlRadHzvHI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6/go.mod h1:5KYaMG6wmVKMFBSfWoyG/zH8pWwzQFnKgpoSRlXHKdQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 h1:3/u/4yZOffg5jdNk1sDpOQ4Y+R6Xbh+GzpDrSZjuy3U=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15/go.mod h1:4Zkjq0FKjE78NKjabuM4tRXKFzUJWXgP0ItEZK8l7JU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 h1:wsSQ4SVz5YE1crz0Ap7VBZrV4nNqZt4CIBBT8mnwoNc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15/go.mod h1:I7sditnFGtYMIqPRU1QoHZAUrXkGp4SczmlLwrNPlD0=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 h1:IrbE3B8O9pm3lsg96AXIN5MXX4pECEuExh/A0Du3AuI=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0/go.mod h1:/sJLzHtiiZvs6C1RbxS/anSAFwZD6oC6M/kotQzOiLw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 h1:OBuZE9Wt8h2imuRktu+WfjiTGrnYdCIJg8IX92aalHE=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7/go.mod h1:4WYoZAhHt+dWYpoOQUgkUKfuQbE6Gg/hW4oXE0pKS9U=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 h1:80dpSqWMwx2dAm30Ib7J6ucz1ZHfiv5OCRwN/EnCOXQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8/go.mod h1:IzNt/udsXlETCdvBOL0nmyMe2t9cGmXmZgsdoZGYYhI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 h1:xHXvxst78wBpJFgDW07xllOx0IAzbryrSdM4nMVQ4Dw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0/go.mod h1:/e8m+AO6HNPPqMyfKRtzZ9+mBF5/x1Wk8QiDva4m07I=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 h1:tBw2Qhf0kj4ZwtsVpDiVRU3zKLvjvjgIjHMKirxXg8M=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4/go.mod h1:Deq4B7sRM6Awq/xyOBlxBdgW8/Z926KYNNaGMW2lrkA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 h1:C+BRMnasSYFcgDw8o9H5hzehKzXyAb9GY5v/8bP9DUY=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0/go.mod h1:4EjU+4mIx6+JqKQkruye+CaigV7alL3thVPfDd9VlMs=
|
||||
github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
|
||||
github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 h1:8sTTiw+9yuNXcfWeqKF2x01GqCF49CpP4Z9nKrrk/ts=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.6/go.mod h1:8WYg+Y40Sn3X2hioaaWAAIngndR8n1XFdRPPX+7QBaM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 h1:E+KqWoVsSrj1tJ6I/fjDIu5xoS2Zacuu1zT+H7KtiIk=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11/go.mod h1:qyWHz+4lvkXcr3+PoGlGHEI+3DLLiU6/GdrFfMaAhB0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 h1:tzMkjh0yTChUqJDgGkcDdxvZDSrJ/WB6R6ymI5ehqJI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.3/go.mod h1:T270C0R5sZNLbWUe8ueiAF42XSZxxPocTaGSgs5c/60=
|
||||
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
|
||||
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
|
||||
github.com/bazelbuild/rules_go v0.46.0/go.mod h1:Dhcz716Kqg1RHNWos+N6MlXNkjNP2EwZQ0LukRKJfMs=
|
||||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
@@ -1329,8 +1329,8 @@ github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxh
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/reedsolomon v1.12.5 h1:4cJuyH926If33BeDgiZpI5OU0pE+wUHZvMSyNGqN73Y=
|
||||
github.com/klauspost/reedsolomon v1.12.5/go.mod h1:LkXRjLYGM8K/iQfujYnaPeDmhZLqkrGUyG9p7zs5L68=
|
||||
github.com/klauspost/reedsolomon v1.12.6 h1:8pqE9aECQG/ZFitiUD1xK/E83zwosBAZtE3UbuZM8TQ=
|
||||
github.com/klauspost/reedsolomon v1.12.6/go.mod h1:ggJT9lc71Vu+cSOPBlxGvBN6TfAS77qB4fp8vJ05NSA=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
@@ -2110,8 +2110,8 @@ golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
|
||||
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
||||
@@ -220,7 +220,7 @@ spec:
|
||||
-s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
-master={{ if .Values.global.masterServer }}{{.Values.global.masterServer}}{{ else }}{{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}{{ end }} \
|
||||
-master={{ include "seaweedfs.masterServerArg" . }} \
|
||||
{{- range .Values.filer.extraArgs }}
|
||||
{{ . }} \
|
||||
{{- end }}
|
||||
|
||||
@@ -184,7 +184,7 @@ spec:
|
||||
-garbageThreshold={{ .Values.master.garbageThreshold }} \
|
||||
{{- end }}
|
||||
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master.{{ .Release.Namespace }} \
|
||||
-peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }} \
|
||||
-peers={{ include "seaweedfs.masterServers" . }} \
|
||||
{{- range .Values.master.extraArgs }}
|
||||
{{ . }} \
|
||||
{{- end }}
|
||||
|
||||
@@ -246,3 +246,28 @@ If allInOne is enabled, point to the all-in-one service; otherwise, point to the
|
||||
{{- end -}}
|
||||
{{- printf "%s%s.%s:%d" (include "seaweedfs.name" .) $serviceNameSuffix .Release.Namespace (int .Values.filer.port) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Generate comma-separated list of master server addresses.
|
||||
Usage: {{ include "seaweedfs.masterServers" . }}
|
||||
Output example: ${SEAWEEDFS_FULLNAME}-master-0.${SEAWEEDFS_FULLNAME}-master.namespace:9333,${SEAWEEDFS_FULLNAME}-master-1...
|
||||
*/}}
|
||||
{{- define "seaweedfs.masterServers" -}}
|
||||
{{- $fullname := include "seaweedfs.name" . -}}
|
||||
{{- range $index := until (.Values.master.replicas | int) -}}
|
||||
{{- if $index }},{{ end -}}
|
||||
${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Generate master server argument value, using global.masterServer if set, otherwise the generated list.
|
||||
Usage: {{ include "seaweedfs.masterServerArg" . }}
|
||||
*/}}
|
||||
{{- define "seaweedfs.masterServerArg" -}}
|
||||
{{- if .Values.global.masterServer -}}
|
||||
{{- .Values.global.masterServer -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.masterServers" . -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
@@ -196,7 +196,7 @@ spec:
|
||||
-minFreeSpacePercent={{ $volume.minFreeSpacePercent }} \
|
||||
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-{{ $volumeName }}.{{ $.Release.Namespace }} \
|
||||
-compactionMBps={{ $volume.compactionMBps }} \
|
||||
-mserver={{ if $.Values.global.masterServer }}{{ $.Values.global.masterServer}}{{ else }}{{ range $index := until ($.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}{{ end }}
|
||||
-master={{ include "seaweedfs.masterServerArg" $ }} \
|
||||
{{- range $volume.extraArgs }}
|
||||
{{ . }} \
|
||||
{{- end }}
|
||||
|
||||
@@ -32,7 +32,7 @@ These tests verify actual FileSystem operations against a running SeaweedFS inst
|
||||
weed master
|
||||
|
||||
# Terminal 2: Start volume server
|
||||
weed volume -mserver=localhost:9333
|
||||
weed volume -master=localhost:9333
|
||||
|
||||
# Terminal 3: Start filer
|
||||
weed filer -master=localhost:9333
|
||||
|
||||
@@ -32,7 +32,7 @@ services:
|
||||
- "18080:18080"
|
||||
command: >
|
||||
volume
|
||||
-mserver=seaweedfs-master:9333
|
||||
-master=seaweedfs-master:9333
|
||||
-port=8080
|
||||
-dir=/data
|
||||
-max=100
|
||||
|
||||
@@ -21,7 +21,7 @@ services:
|
||||
seaweedfs-volume:
|
||||
image: chrislusf/seaweedfs:latest
|
||||
container_name: seaweedfs-volume
|
||||
command: volume -mserver=seaweedfs-master:9333 -ip=seaweedfs-volume -port=8080 -dir=/data
|
||||
command: volume -master=seaweedfs-master:9333 -ip=seaweedfs-volume -port=8080 -dir=/data
|
||||
ports:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
|
||||
@@ -21,7 +21,7 @@ services:
|
||||
seaweedfs-volume:
|
||||
image: chrislusf/seaweedfs:latest
|
||||
container_name: seaweedfs-volume
|
||||
command: volume -mserver=seaweedfs-master:9333 -ip=seaweedfs-volume -port=8080 -dir=/data
|
||||
command: volume -master=seaweedfs-master:9333 -ip=seaweedfs-volume -port=8080 -dir=/data
|
||||
ports:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
|
||||
@@ -107,7 +107,7 @@ wait_for_service "Master" "http://localhost:9333/cluster/status"
|
||||
|
||||
# Start SeaweedFS Volume Server
|
||||
echo -e "${BLUE}💾 Starting SeaweedFS Volume Server...${NC}"
|
||||
./weed volume -mserver=localhost:9333 -port=8080 -dir=/tmp/seaweedfs-volume &
|
||||
./weed volume -master=localhost:9333 -port=8080 -dir=/tmp/seaweedfs-volume &
|
||||
VOLUME_PID=$!
|
||||
wait_for_service "Volume Server" "http://localhost:8080/status"
|
||||
|
||||
|
||||
@@ -431,7 +431,7 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
|
||||
"-port", port,
|
||||
"-dir", volumeDir,
|
||||
"-max", "10",
|
||||
"-mserver", "127.0.0.1:9333",
|
||||
"-master", "127.0.0.1:9333",
|
||||
"-ip", "127.0.0.1",
|
||||
"-dataCenter", "dc1",
|
||||
"-rack", rack,
|
||||
@@ -956,7 +956,7 @@ func startMultiDiskCluster(ctx context.Context, dataDir string) (*MultiDiskClust
|
||||
"-port", port,
|
||||
"-dir", strings.Join(diskDirs, ","),
|
||||
"-max", strings.Join(maxVolumes, ","),
|
||||
"-mserver", "127.0.0.1:9334",
|
||||
"-master", "127.0.0.1:9334",
|
||||
"-ip", "127.0.0.1",
|
||||
"-dataCenter", "dc1",
|
||||
"-rack", rack,
|
||||
|
||||
@@ -194,7 +194,7 @@ func (f *FuseTestFramework) startMaster(config *TestConfig) error {
|
||||
func (f *FuseTestFramework) startVolumeServers(config *TestConfig) error {
|
||||
args := []string{
|
||||
"volume",
|
||||
"-mserver=" + f.masterAddr,
|
||||
"-master=" + f.masterAddr,
|
||||
"-ip=127.0.0.1",
|
||||
"-port=18080",
|
||||
"-dir=" + filepath.Join(f.dataDir, "volume"),
|
||||
|
||||
@@ -27,7 +27,7 @@ services:
|
||||
ports:
|
||||
- "8080:8080"
|
||||
- "18080:18080"
|
||||
command: "volume -mserver=seaweedfs-master:9333 -ip=seaweedfs-volume -ip.bind=0.0.0.0 -port=8080 -port.grpc=18080 -publicUrl=seaweedfs-volume:8080 -max=100 -dir=/data -preStopSeconds=1"
|
||||
command: "volume -master=seaweedfs-master:9333 -ip=seaweedfs-volume -ip.bind=0.0.0.0 -port=8080 -port.grpc=18080 -publicUrl=seaweedfs-volume:8080 -max=100 -dir=/data -preStopSeconds=1"
|
||||
volumes:
|
||||
- seaweedfs-volume-data:/data
|
||||
depends_on:
|
||||
|
||||
@@ -111,7 +111,7 @@ services:
|
||||
- "18080:18080" # gRPC port
|
||||
command:
|
||||
- volume
|
||||
- -mserver=seaweedfs-master:9333
|
||||
- -master=seaweedfs-master:9333
|
||||
- -ip=seaweedfs-volume
|
||||
- -port=8080
|
||||
- -port.grpc=18080
|
||||
|
||||
+2
-2
@@ -141,7 +141,7 @@ require (
|
||||
github.com/karlseguin/ccache/v2 v2.0.8 // indirect
|
||||
github.com/klauspost/compress v1.18.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/klauspost/reedsolomon v1.12.5 // indirect
|
||||
github.com/klauspost/reedsolomon v1.12.6 // indirect
|
||||
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
|
||||
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
@@ -234,7 +234,7 @@ require (
|
||||
golang.org/x/image v0.33.0 // indirect
|
||||
golang.org/x/net v0.47.0 // indirect
|
||||
golang.org/x/oauth2 v0.32.0 // indirect
|
||||
golang.org/x/sync v0.18.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.38.0 // indirect
|
||||
golang.org/x/term v0.37.0 // indirect
|
||||
golang.org/x/text v0.31.0 // indirect
|
||||
|
||||
+4
-4
@@ -433,8 +433,8 @@ github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uq
|
||||
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/reedsolomon v1.12.5 h1:4cJuyH926If33BeDgiZpI5OU0pE+wUHZvMSyNGqN73Y=
|
||||
github.com/klauspost/reedsolomon v1.12.5/go.mod h1:LkXRjLYGM8K/iQfujYnaPeDmhZLqkrGUyG9p7zs5L68=
|
||||
github.com/klauspost/reedsolomon v1.12.6 h1:8pqE9aECQG/ZFitiUD1xK/E83zwosBAZtE3UbuZM8TQ=
|
||||
github.com/klauspost/reedsolomon v1.12.6/go.mod h1:ggJT9lc71Vu+cSOPBlxGvBN6TfAS77qB4fp8vJ05NSA=
|
||||
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 h1:CjEMN21Xkr9+zwPmZPaJJw+apzVbjGL5uK/6g9Q2jGU=
|
||||
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988/go.mod h1:/agobYum3uo/8V6yPVnq+R82pyVGCeuWW5arT4Txn8A=
|
||||
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 h1:FHVoZMOVRA+6/y4yRlbiR3WvsrOcKBd/f64H7YiWR2U=
|
||||
@@ -841,8 +841,8 @@ golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
|
||||
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
||||
@@ -113,7 +113,7 @@ services:
|
||||
- "18080:18080"
|
||||
command:
|
||||
- volume
|
||||
- -mserver=seaweedfs-master:9333
|
||||
- -master=seaweedfs-master:9333
|
||||
- -ip=seaweedfs-volume
|
||||
- -port=8080
|
||||
- -port.grpc=18080
|
||||
|
||||
@@ -64,7 +64,7 @@ services:
|
||||
- "8080:8080"
|
||||
command:
|
||||
- volume
|
||||
- -mserver=seaweedfs-master:9333
|
||||
- -master=seaweedfs-master:9333
|
||||
- -ip=seaweedfs-volume
|
||||
- -publicUrl=seaweedfs-volume:8080
|
||||
depends_on:
|
||||
|
||||
+1
-1
@@ -17,7 +17,7 @@ weed server -mq.broker -mq.agent -filer -volume -master.peers=none
|
||||
|
||||
# Or start components separately
|
||||
weed master -peers=none
|
||||
weed volume -mserver=localhost:9333
|
||||
weed volume -master=localhost:9333
|
||||
weed filer -master=localhost:9333
|
||||
weed mq.broker -filer=localhost:8888
|
||||
weed mq.agent -brokers=localhost:17777
|
||||
|
||||
@@ -74,7 +74,7 @@ start-seaweedfs: check-binary
|
||||
@sleep 3
|
||||
|
||||
# Start volume server
|
||||
@nohup $(SEAWEEDFS_BINARY) volume -port=$(VOLUME_PORT) -mserver=127.0.0.1:$(MASTER_PORT) -dir=/tmp/seaweedfs-test-copying-volume -ip=127.0.0.1 > /tmp/seaweedfs-volume.log 2>&1 &
|
||||
@nohup $(SEAWEEDFS_BINARY) volume -port=$(VOLUME_PORT) -master=127.0.0.1:$(MASTER_PORT) -dir=/tmp/seaweedfs-test-copying-volume -ip=127.0.0.1 > /tmp/seaweedfs-volume.log 2>&1 &
|
||||
@sleep 3
|
||||
|
||||
# Start filer server (using standard SeaweedFS gRPC port convention: HTTP port + 10000)
|
||||
|
||||
@@ -75,7 +75,7 @@ start-services: ## Start SeaweedFS services for testing
|
||||
-dataCenter=dc1 -rack=rack1 \
|
||||
-dir=test-volume-data \
|
||||
-max=100 \
|
||||
-mserver=localhost:$(MASTER_PORT) > weed-volume.log 2>&1 & \
|
||||
-master=localhost:$(MASTER_PORT) > weed-volume.log 2>&1 & \
|
||||
echo $$! > $(VOLUME_PID_FILE)
|
||||
|
||||
@echo "Waiting for volume server to be ready..."
|
||||
|
||||
@@ -21,7 +21,7 @@ services:
|
||||
seaweedfs-volume:
|
||||
image: chrislusf/seaweedfs:latest
|
||||
container_name: seaweedfs-volume-test
|
||||
command: volume -dir=/data -port=8083 -mserver=seaweedfs-master:9333
|
||||
command: volume -dir=/data -port=8083 -master=seaweedfs-master:9333
|
||||
ports:
|
||||
- "8083:8083"
|
||||
volumes:
|
||||
|
||||
@@ -51,7 +51,7 @@ services:
|
||||
ports:
|
||||
- "8083:8083"
|
||||
- "18083:18083"
|
||||
command: "volume -ip=weed-volume -port=8083 -dir=/data -mserver=weed-master:9333 -dataCenter=dc1 -rack=rack1"
|
||||
command: "volume -ip=weed-volume -port=8083 -dir=/data -master=weed-master:9333 -dataCenter=dc1 -rack=rack1"
|
||||
volumes:
|
||||
- volume-data:/data
|
||||
networks:
|
||||
|
||||
@@ -134,7 +134,7 @@ start-seaweedfs-ci: check-binary
|
||||
|
||||
# Start volume server with master HTTP port and increased capacity
|
||||
@echo "Starting volume server..."
|
||||
@nohup $(SEAWEEDFS_BINARY) volume -port=$(VOLUME_PORT) -mserver=127.0.0.1:$(MASTER_PORT) -dir=/tmp/seaweedfs-test-parquet-volume -max=$(VOLUME_MAX_COUNT) -ip=127.0.0.1 -preStopSeconds=1 > /tmp/seaweedfs-parquet-volume.log 2>&1 &
|
||||
@nohup $(SEAWEEDFS_BINARY) volume -port=$(VOLUME_PORT) -master=127.0.0.1:$(MASTER_PORT) -dir=/tmp/seaweedfs-test-parquet-volume -max=$(VOLUME_MAX_COUNT) -ip=127.0.0.1 -preStopSeconds=1 > /tmp/seaweedfs-parquet-volume.log 2>&1 &
|
||||
@sleep 5
|
||||
|
||||
# Start filer server with embedded S3
|
||||
|
||||
@@ -102,7 +102,7 @@ start-seaweedfs: check-binary
|
||||
@sleep 3
|
||||
|
||||
# Start volume server with master HTTP port and increased capacity
|
||||
@nohup $(SEAWEEDFS_BINARY) volume -port=$(VOLUME_PORT) -mserver=127.0.0.1:$(MASTER_PORT) -dir=/tmp/seaweedfs-test-sse-volume -max=$(VOLUME_MAX_COUNT) -ip=127.0.0.1 > /tmp/seaweedfs-sse-volume.log 2>&1 &
|
||||
@nohup $(SEAWEEDFS_BINARY) volume -port=$(VOLUME_PORT) -master=127.0.0.1:$(MASTER_PORT) -dir=/tmp/seaweedfs-test-sse-volume -max=$(VOLUME_MAX_COUNT) -ip=127.0.0.1 > /tmp/seaweedfs-sse-volume.log 2>&1 &
|
||||
@sleep 5
|
||||
|
||||
# Start filer server (using standard SeaweedFS gRPC port convention: HTTP port + 10000)
|
||||
@@ -359,7 +359,7 @@ start-seaweedfs-ci: check-binary
|
||||
|
||||
# Start volume server with master HTTP port and increased capacity
|
||||
@echo "Starting volume server..."
|
||||
@nohup $(SEAWEEDFS_BINARY) volume -port=$(VOLUME_PORT) -mserver=127.0.0.1:$(MASTER_PORT) -dir=/tmp/seaweedfs-test-sse-volume -max=$(VOLUME_MAX_COUNT) -ip=127.0.0.1 > /tmp/seaweedfs-sse-volume.log 2>&1 &
|
||||
@nohup $(SEAWEEDFS_BINARY) volume -port=$(VOLUME_PORT) -master=127.0.0.1:$(MASTER_PORT) -dir=/tmp/seaweedfs-test-sse-volume -max=$(VOLUME_MAX_COUNT) -ip=127.0.0.1 > /tmp/seaweedfs-sse-volume.log 2>&1 &
|
||||
@sleep 5
|
||||
|
||||
# Create S3 JSON configuration with KMS (Local provider) and basic identity for embedded S3
|
||||
|
||||
@@ -50,7 +50,7 @@ services:
|
||||
- "8080:8080"
|
||||
command:
|
||||
- volume
|
||||
- -mserver=seaweedfs-master:9333
|
||||
- -master=seaweedfs-master:9333
|
||||
- -port=8080
|
||||
- -ip=seaweedfs-volume
|
||||
- -publicUrl=seaweedfs-volume:8080
|
||||
|
||||
@@ -268,7 +268,7 @@ func (f *SftpTestFramework) startMaster(config *TestConfig) error {
|
||||
func (f *SftpTestFramework) startVolumeServer(config *TestConfig) error {
|
||||
args := []string{
|
||||
"volume",
|
||||
"-mserver=" + f.masterAddr,
|
||||
"-master=" + f.masterAddr,
|
||||
"-ip=127.0.0.1",
|
||||
"-port=18080",
|
||||
"-dir=" + filepath.Join(f.dataDir, "volume"),
|
||||
|
||||
@@ -52,3 +52,20 @@ enabled = false
|
||||
# create binding myexchange => myqueue
|
||||
topic_url = "rabbit://myexchange"
|
||||
sub_url = "rabbit://myqueue"
|
||||
|
||||
[notification.webhook]
|
||||
# Send file system events to HTTP webhook endpoints (push model)
|
||||
# BEST FOR: Low to moderate traffic (< 100 events/second sustained)
|
||||
# FOR HIGH TRAFFIC: Consider using Kafka, SQS, or pull-based event logs instead
|
||||
# Documentation: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Notification-Webhook
|
||||
enabled = false
|
||||
endpoint = "https://your-server.com/webhook" # required: HTTP endpoint URL
|
||||
bearer_token = "" # optional: bearer token for authentication
|
||||
timeout_seconds = 10 # optional: HTTP timeout (default: 10, range: 1-300)
|
||||
max_retries = 3 # optional: retry attempts (default: 3, range: 0-10)
|
||||
backoff_seconds = 3 # optional: initial backoff delay (default: 3, range: 1-60)
|
||||
max_backoff_seconds = 30 # optional: max backoff delay (default: 30, range: backoff_seconds-300)
|
||||
workers = 5 # optional: concurrent workers (default: 5, range: 1-100)
|
||||
buffer_size = 10000 # optional: event buffer size (default: 10000, range: 100-1000000)
|
||||
# event_types = ["create", "update", "delete", "rename"] # optional: filter by event types (default: all)
|
||||
# path_prefixes = ["/important", "/data"] # optional: filter by path prefixes (default: all)
|
||||
|
||||
@@ -186,14 +186,22 @@ func (p *OIDCProvider) Authenticate(ctx context.Context, token string) (*provide
|
||||
attributes["roles"] = strings.Join(roles, ",")
|
||||
}
|
||||
|
||||
return &providers.ExternalIdentity{
|
||||
identity := &providers.ExternalIdentity{
|
||||
UserID: claims.Subject,
|
||||
Email: email,
|
||||
DisplayName: displayName,
|
||||
Groups: groups,
|
||||
Attributes: attributes,
|
||||
Provider: p.name,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Pass the token expiration to limit session duration
|
||||
// This ensures the STS session doesn't exceed the source token's validity
|
||||
if !claims.ExpiresAt.IsZero() {
|
||||
identity.TokenExpiration = &claims.ExpiresAt
|
||||
}
|
||||
|
||||
return identity, nil
|
||||
}
|
||||
|
||||
// GetUserInfo retrieves user information from the UserInfo endpoint
|
||||
@@ -372,6 +380,24 @@ func (p *OIDCProvider) ValidateToken(ctx context.Context, token string) (*provid
|
||||
Claims: make(map[string]interface{}),
|
||||
}
|
||||
|
||||
// Extract time-based claims (exp, iat, nbf)
|
||||
for key, target := range map[string]*time.Time{
|
||||
"exp": &tokenClaims.ExpiresAt,
|
||||
"iat": &tokenClaims.IssuedAt,
|
||||
"nbf": &tokenClaims.NotBefore,
|
||||
} {
|
||||
if val, ok := claims[key]; ok {
|
||||
switch v := val.(type) {
|
||||
case float64:
|
||||
*target = time.Unix(int64(v), 0)
|
||||
case json.Number:
|
||||
if intVal, err := v.Int64(); err == nil {
|
||||
*target = time.Unix(intVal, 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Copy all claims
|
||||
for key, value := range claims {
|
||||
tokenClaims.Claims[key] = value
|
||||
|
||||
@@ -47,6 +47,10 @@ type ExternalIdentity struct {
|
||||
|
||||
// Provider is the name of the identity provider
|
||||
Provider string `json:"provider"`
|
||||
|
||||
// TokenExpiration is the expiration time of the source identity token
|
||||
// This is used to limit session duration to not exceed the token's exp claim
|
||||
TokenExpiration *time.Time `json:"tokenExpiration,omitempty"`
|
||||
}
|
||||
|
||||
// Validate validates the external identity structure
|
||||
|
||||
@@ -422,8 +422,9 @@ func (s *STSService) AssumeRoleWithWebIdentity(ctx context.Context, request *Ass
|
||||
return nil, fmt.Errorf("role assumption denied: %w", err)
|
||||
}
|
||||
|
||||
// 3. Calculate session duration
|
||||
sessionDuration := s.calculateSessionDuration(request.DurationSeconds)
|
||||
// 3. Calculate session duration, capping at the source token's expiration
|
||||
// This ensures sessions from short-lived tokens (e.g., GitLab CI job tokens) don't outlive their source
|
||||
sessionDuration := s.calculateSessionDuration(request.DurationSeconds, externalIdentity.TokenExpiration)
|
||||
expiresAt := time.Now().Add(sessionDuration)
|
||||
|
||||
// 4. Generate session ID and credentials
|
||||
@@ -502,7 +503,8 @@ func (s *STSService) AssumeRoleWithCredentials(ctx context.Context, request *Ass
|
||||
}
|
||||
|
||||
// 4. Calculate session duration
|
||||
sessionDuration := s.calculateSessionDuration(request.DurationSeconds)
|
||||
// For credential-based auth, there's no source token with expiration to cap against
|
||||
sessionDuration := s.calculateSessionDuration(request.DurationSeconds, nil)
|
||||
expiresAt := time.Now().Add(sessionDuration)
|
||||
|
||||
// 5. Generate session ID and temporary credentials
|
||||
@@ -745,14 +747,42 @@ func (s *STSService) validateRoleAssumptionForCredentials(ctx context.Context, r
|
||||
return nil
|
||||
}
|
||||
|
||||
// calculateSessionDuration calculates the session duration
|
||||
func (s *STSService) calculateSessionDuration(durationSeconds *int64) time.Duration {
|
||||
// calculateSessionDuration calculates the session duration, respecting the source token's expiration
|
||||
// If the incoming web identity token has an exp claim, the session duration is capped to not exceed it
|
||||
// This ensures that sessions from short-lived tokens (e.g., GitLab CI job tokens) don't outlive their source
|
||||
func (s *STSService) calculateSessionDuration(durationSeconds *int64, tokenExpiration *time.Time) time.Duration {
|
||||
var duration time.Duration
|
||||
if durationSeconds != nil {
|
||||
return time.Duration(*durationSeconds) * time.Second
|
||||
duration = time.Duration(*durationSeconds) * time.Second
|
||||
} else {
|
||||
// Use default from config
|
||||
duration = s.Config.TokenDuration.Duration
|
||||
}
|
||||
|
||||
// Use default from config
|
||||
return s.Config.TokenDuration.Duration
|
||||
// If the source token has an expiration, cap the session duration to not exceed it
|
||||
// This follows the principle: "if calculated exp > incoming exp claim, then limit outgoing exp to incoming exp"
|
||||
if tokenExpiration != nil && !tokenExpiration.IsZero() {
|
||||
timeUntilTokenExpiry := time.Until(*tokenExpiration)
|
||||
if timeUntilTokenExpiry <= 0 {
|
||||
// Token already expired - use minimal duration as defense-in-depth
|
||||
// The token should have been rejected during validation, but we handle this defensively
|
||||
glog.V(2).Infof("Source token already expired, using minimal session duration")
|
||||
duration = time.Minute
|
||||
} else if timeUntilTokenExpiry < duration {
|
||||
glog.V(2).Infof("Limiting session duration from %v to %v based on source token expiration",
|
||||
duration, timeUntilTokenExpiry)
|
||||
duration = timeUntilTokenExpiry
|
||||
}
|
||||
}
|
||||
|
||||
// Cap at MaxSessionLength if configured
|
||||
if s.Config.MaxSessionLength.Duration > 0 && duration > s.Config.MaxSessionLength.Duration {
|
||||
glog.V(2).Infof("Limiting session duration from %v to %v based on MaxSessionLength config",
|
||||
duration, s.Config.MaxSessionLength.Duration)
|
||||
duration = s.Config.MaxSessionLength.Duration
|
||||
}
|
||||
|
||||
return duration
|
||||
}
|
||||
|
||||
// extractSessionIdFromToken extracts session ID from JWT session token
|
||||
|
||||
@@ -451,3 +451,212 @@ func (m *MockIdentityProvider) ValidateToken(ctx context.Context, token string)
|
||||
}
|
||||
return nil, fmt.Errorf("invalid token")
|
||||
}
|
||||
|
||||
// TestSessionDurationCappedByTokenExpiration tests that session duration is capped by the source token's exp claim
|
||||
func TestSessionDurationCappedByTokenExpiration(t *testing.T) {
|
||||
service := NewSTSService()
|
||||
|
||||
config := &STSConfig{
|
||||
TokenDuration: FlexibleDuration{time.Hour}, // Default: 1 hour
|
||||
MaxSessionLength: FlexibleDuration{time.Hour * 12},
|
||||
Issuer: "test-sts",
|
||||
SigningKey: []byte("test-signing-key-32-characters-long"),
|
||||
}
|
||||
|
||||
err := service.Initialize(config)
|
||||
require.NoError(t, err)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
durationSeconds *int64
|
||||
tokenExpiration *time.Time
|
||||
expectedMaxSeconds int64
|
||||
description string
|
||||
}{
|
||||
{
|
||||
name: "no token expiration - use default duration",
|
||||
durationSeconds: nil,
|
||||
tokenExpiration: nil,
|
||||
expectedMaxSeconds: 3600, // 1 hour default
|
||||
description: "When no token expiration is set, use the configured default duration",
|
||||
},
|
||||
{
|
||||
name: "token expires before default duration",
|
||||
durationSeconds: nil,
|
||||
tokenExpiration: timePtr(time.Now().Add(30 * time.Minute)),
|
||||
expectedMaxSeconds: 30 * 60, // 30 minutes
|
||||
description: "When token expires in 30 min, session should be capped at 30 min",
|
||||
},
|
||||
{
|
||||
name: "token expires after default duration - use default",
|
||||
durationSeconds: nil,
|
||||
tokenExpiration: timePtr(time.Now().Add(2 * time.Hour)),
|
||||
expectedMaxSeconds: 3600, // 1 hour default, since it's less than 2 hour token expiry
|
||||
description: "When token expires after default duration, use the default duration",
|
||||
},
|
||||
{
|
||||
name: "requested duration shorter than token expiry",
|
||||
durationSeconds: int64Ptr(1800), // 30 min requested
|
||||
tokenExpiration: timePtr(time.Now().Add(time.Hour)),
|
||||
expectedMaxSeconds: 1800, // 30 minutes as requested
|
||||
description: "When requested duration is shorter than token expiry, use requested duration",
|
||||
},
|
||||
{
|
||||
name: "requested duration longer than token expiry - cap at token expiry",
|
||||
durationSeconds: int64Ptr(3600), // 1 hour requested
|
||||
tokenExpiration: timePtr(time.Now().Add(15 * time.Minute)),
|
||||
expectedMaxSeconds: 15 * 60, // Capped at 15 minutes
|
||||
description: "When requested duration exceeds token expiry, cap at token expiry",
|
||||
},
|
||||
{
|
||||
name: "GitLab CI short-lived token scenario",
|
||||
durationSeconds: nil,
|
||||
tokenExpiration: timePtr(time.Now().Add(5 * time.Minute)),
|
||||
expectedMaxSeconds: 5 * 60, // 5 minutes
|
||||
description: "GitLab CI job with 5 minute timeout should result in 5 minute session",
|
||||
},
|
||||
{
|
||||
name: "already expired token - defense in depth",
|
||||
durationSeconds: nil,
|
||||
tokenExpiration: timePtr(time.Now().Add(-5 * time.Minute)), // Expired 5 minutes ago
|
||||
expectedMaxSeconds: 60, // 1 minute minimum
|
||||
description: "Already expired token should result in minimal 1 minute session",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
duration := service.calculateSessionDuration(tt.durationSeconds, tt.tokenExpiration)
|
||||
|
||||
// Allow 5 second tolerance for time calculations
|
||||
maxExpected := time.Duration(tt.expectedMaxSeconds+5) * time.Second
|
||||
minExpected := time.Duration(tt.expectedMaxSeconds-5) * time.Second
|
||||
|
||||
assert.GreaterOrEqual(t, duration, minExpected,
|
||||
"%s: duration %v should be >= %v", tt.description, duration, minExpected)
|
||||
assert.LessOrEqual(t, duration, maxExpected,
|
||||
"%s: duration %v should be <= %v", tt.description, duration, maxExpected)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestAssumeRoleWithWebIdentityRespectsTokenExpiration tests end-to-end that session duration is capped
|
||||
func TestAssumeRoleWithWebIdentityRespectsTokenExpiration(t *testing.T) {
|
||||
service := NewSTSService()
|
||||
|
||||
config := &STSConfig{
|
||||
TokenDuration: FlexibleDuration{time.Hour},
|
||||
MaxSessionLength: FlexibleDuration{time.Hour * 12},
|
||||
Issuer: "test-sts",
|
||||
SigningKey: []byte("test-signing-key-32-characters-long"),
|
||||
}
|
||||
|
||||
err := service.Initialize(config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Set up mock trust policy validator
|
||||
mockValidator := &MockTrustPolicyValidator{}
|
||||
service.SetTrustPolicyValidator(mockValidator)
|
||||
|
||||
// Create a mock provider that returns tokens with short expiration
|
||||
shortLivedTokenExpiration := time.Now().Add(10 * time.Minute)
|
||||
mockProvider := &MockIdentityProviderWithExpiration{
|
||||
name: "short-lived-issuer",
|
||||
tokenExpiration: &shortLivedTokenExpiration,
|
||||
}
|
||||
service.RegisterProvider(mockProvider)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a JWT token with short expiration
|
||||
token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
|
||||
"iss": "short-lived-issuer",
|
||||
"sub": "test-user",
|
||||
"aud": "test-client",
|
||||
"exp": shortLivedTokenExpiration.Unix(),
|
||||
"iat": time.Now().Unix(),
|
||||
})
|
||||
tokenString, err := token.SignedString([]byte("test-signing-key"))
|
||||
require.NoError(t, err)
|
||||
|
||||
request := &AssumeRoleWithWebIdentityRequest{
|
||||
RoleArn: "arn:aws:iam::role/TestRole",
|
||||
WebIdentityToken: tokenString,
|
||||
RoleSessionName: "test-session",
|
||||
}
|
||||
|
||||
response, err := service.AssumeRoleWithWebIdentity(ctx, request)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, response)
|
||||
|
||||
// Verify the session expires at or before the token expiration
|
||||
// Allow 5 second tolerance
|
||||
assert.True(t, response.Credentials.Expiration.Before(shortLivedTokenExpiration.Add(5*time.Second)),
|
||||
"Session expiration (%v) should not exceed token expiration (%v)",
|
||||
response.Credentials.Expiration, shortLivedTokenExpiration)
|
||||
}
|
||||
|
||||
// MockIdentityProviderWithExpiration is a mock provider that returns tokens with configurable expiration
|
||||
type MockIdentityProviderWithExpiration struct {
|
||||
name string
|
||||
tokenExpiration *time.Time
|
||||
}
|
||||
|
||||
func (m *MockIdentityProviderWithExpiration) Name() string {
|
||||
return m.name
|
||||
}
|
||||
|
||||
func (m *MockIdentityProviderWithExpiration) GetIssuer() string {
|
||||
return m.name
|
||||
}
|
||||
|
||||
func (m *MockIdentityProviderWithExpiration) Initialize(config interface{}) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockIdentityProviderWithExpiration) Authenticate(ctx context.Context, token string) (*providers.ExternalIdentity, error) {
|
||||
// Parse the token to get subject
|
||||
parsedToken, _, err := new(jwt.Parser).ParseUnverified(token, jwt.MapClaims{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse token: %w", err)
|
||||
}
|
||||
|
||||
claims, ok := parsedToken.Claims.(jwt.MapClaims)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid claims")
|
||||
}
|
||||
|
||||
subject, _ := claims["sub"].(string)
|
||||
|
||||
identity := &providers.ExternalIdentity{
|
||||
UserID: subject,
|
||||
Email: subject + "@example.com",
|
||||
DisplayName: "Test User",
|
||||
Provider: m.name,
|
||||
TokenExpiration: m.tokenExpiration,
|
||||
}
|
||||
|
||||
return identity, nil
|
||||
}
|
||||
|
||||
func (m *MockIdentityProviderWithExpiration) GetUserInfo(ctx context.Context, userID string) (*providers.ExternalIdentity, error) {
|
||||
return &providers.ExternalIdentity{
|
||||
UserID: userID,
|
||||
Provider: m.name,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *MockIdentityProviderWithExpiration) ValidateToken(ctx context.Context, token string) (*providers.TokenClaims, error) {
|
||||
claims := &providers.TokenClaims{
|
||||
Subject: "test-user",
|
||||
Issuer: m.name,
|
||||
}
|
||||
if m.tokenExpiration != nil {
|
||||
claims.ExpiresAt = *m.tokenExpiration
|
||||
}
|
||||
return claims, nil
|
||||
}
|
||||
|
||||
func timePtr(t time.Time) *time.Time {
|
||||
return &t
|
||||
}
|
||||
|
||||
@@ -18,22 +18,28 @@ const (
|
||||
directoryStreamBaseOffset = 2 // . & ..
|
||||
)
|
||||
|
||||
// DirectoryHandle represents an open directory handle.
|
||||
// It maintains state for directory listing pagination and is protected by a mutex
|
||||
// to handle concurrent readdir operations from NFS-Ganesha and other multi-threaded clients.
|
||||
type DirectoryHandle struct {
|
||||
sync.Mutex
|
||||
isFinished bool
|
||||
entryStream []*filer.Entry
|
||||
entryStreamOffset uint64
|
||||
}
|
||||
|
||||
func (dh *DirectoryHandle) reset() {
|
||||
*dh = DirectoryHandle{
|
||||
isFinished: false,
|
||||
entryStream: []*filer.Entry{},
|
||||
entryStreamOffset: directoryStreamBaseOffset,
|
||||
dh.isFinished = false
|
||||
// Nil out pointers to allow garbage collection of old entries,
|
||||
// then reuse the slice's capacity to avoid re-allocations.
|
||||
for i := range dh.entryStream {
|
||||
dh.entryStream[i] = nil
|
||||
}
|
||||
dh.entryStream = dh.entryStream[:0]
|
||||
dh.entryStreamOffset = directoryStreamBaseOffset
|
||||
}
|
||||
|
||||
type DirectoryHandleToInode struct {
|
||||
// shares the file handle id sequencer with FileHandleToInode{nextFh}
|
||||
sync.Mutex
|
||||
dir2inode map[DirectoryHandleId]*DirectoryHandle
|
||||
}
|
||||
@@ -45,14 +51,14 @@ func NewDirectoryHandleToInode() *DirectoryHandleToInode {
|
||||
}
|
||||
|
||||
func (wfs *WFS) AcquireDirectoryHandle() (DirectoryHandleId, *DirectoryHandle) {
|
||||
fh := FileHandleId(util.RandomUint64())
|
||||
fh := DirectoryHandleId(util.RandomUint64())
|
||||
|
||||
wfs.dhMap.Lock()
|
||||
defer wfs.dhMap.Unlock()
|
||||
dh := new(DirectoryHandle)
|
||||
dh := &DirectoryHandle{}
|
||||
dh.reset()
|
||||
wfs.dhMap.dir2inode[DirectoryHandleId(fh)] = dh
|
||||
return DirectoryHandleId(fh), dh
|
||||
wfs.dhMap.dir2inode[fh] = dh
|
||||
return fh, dh
|
||||
}
|
||||
|
||||
func (wfs *WFS) GetDirectoryHandle(dhid DirectoryHandleId) *DirectoryHandle {
|
||||
@@ -61,7 +67,7 @@ func (wfs *WFS) GetDirectoryHandle(dhid DirectoryHandleId) *DirectoryHandle {
|
||||
if dh, found := wfs.dhMap.dir2inode[dhid]; found {
|
||||
return dh
|
||||
}
|
||||
dh := new(DirectoryHandle)
|
||||
dh := &DirectoryHandle{}
|
||||
dh.reset()
|
||||
wfs.dhMap.dir2inode[dhid] = dh
|
||||
return dh
|
||||
@@ -137,7 +143,13 @@ func (wfs *WFS) ReadDirPlus(cancel <-chan struct{}, input *fuse.ReadIn, out *fus
|
||||
}
|
||||
|
||||
func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPlusMode bool) fuse.Status {
|
||||
// Get the directory handle and lock it for the duration of this operation.
|
||||
// This serializes concurrent readdir calls on the same handle, fixing the
|
||||
// race condition that caused hangs with NFS-Ganesha.
|
||||
dh := wfs.GetDirectoryHandle(DirectoryHandleId(input.Fh))
|
||||
dh.Lock()
|
||||
defer dh.Unlock()
|
||||
|
||||
if input.Offset == 0 {
|
||||
dh.reset()
|
||||
} else if dh.isFinished && input.Offset >= dh.entryStreamOffset {
|
||||
@@ -154,7 +166,7 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPl
|
||||
}
|
||||
|
||||
var dirEntry fuse.DirEntry
|
||||
processEachEntryFn := func(entry *filer.Entry) (bool, error) {
|
||||
processEachEntryFn := func(entry *filer.Entry) bool {
|
||||
dirEntry.Name = entry.Name()
|
||||
dirEntry.Mode = toSyscallMode(entry.Mode)
|
||||
inode := wfs.inodeToPath.Lookup(dirPath.Child(dirEntry.Name), entry.Crtime.Unix(), entry.IsDirectory(), len(entry.HardLinkId) > 0, entry.Inode, isPlusMode)
|
||||
@@ -162,13 +174,13 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPl
|
||||
if !isPlusMode {
|
||||
if !out.AddDirEntry(dirEntry) {
|
||||
isEarlyTerminated = true
|
||||
return false, nil
|
||||
return false
|
||||
}
|
||||
} else {
|
||||
entryOut := out.AddDirLookupEntry(dirEntry)
|
||||
if entryOut == nil {
|
||||
isEarlyTerminated = true
|
||||
return false, nil
|
||||
return false
|
||||
}
|
||||
if fh, found := wfs.fhMap.FindFileHandle(inode); found {
|
||||
glog.V(4).Infof("readdir opened file %s", dirPath.Child(dirEntry.Name))
|
||||
@@ -176,7 +188,7 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPl
|
||||
}
|
||||
wfs.outputFilerEntry(entryOut, inode, entry)
|
||||
}
|
||||
return true, nil
|
||||
return true
|
||||
}
|
||||
|
||||
if input.Offset < directoryStreamBaseOffset {
|
||||
@@ -207,7 +219,7 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPl
|
||||
entryCurrentIndex := input.Offset - dh.entryStreamOffset
|
||||
for uint64(len(dh.entryStream)) > entryCurrentIndex {
|
||||
entry := dh.entryStream[entryCurrentIndex]
|
||||
if process, _ := processEachEntryFn(entry); process {
|
||||
if processEachEntryFn(entry) {
|
||||
lastEntryName = entry.Name()
|
||||
entryCurrentIndex++
|
||||
} else {
|
||||
@@ -217,14 +229,16 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPl
|
||||
}
|
||||
}
|
||||
|
||||
var err error
|
||||
if err = meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath); err != nil {
|
||||
if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath); err != nil {
|
||||
glog.Errorf("dir ReadDirAll %s: %v", dirPath, err)
|
||||
return fuse.EIO
|
||||
}
|
||||
listErr := wfs.metaCache.ListDirectoryEntries(context.Background(), dirPath, lastEntryName, false, int64(math.MaxInt32), func(entry *filer.Entry) (bool, error) {
|
||||
dh.entryStream = append(dh.entryStream, entry)
|
||||
return processEachEntryFn(entry)
|
||||
if !processEachEntryFn(entry) {
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
})
|
||||
if listErr != nil {
|
||||
glog.Errorf("list meta cache: %v", listErr)
|
||||
|
||||
@@ -8,16 +8,24 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||
)
|
||||
|
||||
const (
|
||||
maxWebhookRetryDepth = 2
|
||||
)
|
||||
|
||||
type httpClient struct {
|
||||
endpoint string
|
||||
token string
|
||||
timeout time.Duration
|
||||
endpoint string
|
||||
token string
|
||||
timeout time.Duration
|
||||
client *http.Client // Reused HTTP client with redirect prevention
|
||||
endpointMu sync.RWMutex
|
||||
finalURL string // Cached final URL after following redirects
|
||||
}
|
||||
|
||||
func newHTTPClient(cfg *config) (*httpClient, error) {
|
||||
@@ -25,10 +33,24 @@ func newHTTPClient(cfg *config) (*httpClient, error) {
|
||||
endpoint: cfg.endpoint,
|
||||
token: cfg.authBearerToken,
|
||||
timeout: time.Duration(cfg.timeoutSeconds) * time.Second,
|
||||
client: &http.Client{
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (h *httpClient) sendMessage(message *webhookMessage) error {
|
||||
return h.sendMessageWithRetry(message, 0)
|
||||
}
|
||||
|
||||
func (h *httpClient) sendMessageWithRetry(message *webhookMessage, depth int) error {
|
||||
// Prevent infinite recursion
|
||||
if depth > maxWebhookRetryDepth {
|
||||
return fmt.Errorf("webhook max retry depth exceeded")
|
||||
}
|
||||
|
||||
// Serialize the protobuf message to JSON for HTTP payload
|
||||
notificationData, err := json.Marshal(message.Notification)
|
||||
if err != nil {
|
||||
@@ -46,7 +68,16 @@ func (h *httpClient) sendMessage(message *webhookMessage) error {
|
||||
return fmt.Errorf("failed to marshal message: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, h.endpoint, bytes.NewBuffer(jsonData))
|
||||
// Use cached final URL if available, otherwise use original endpoint
|
||||
h.endpointMu.RLock()
|
||||
targetURL := h.finalURL
|
||||
usingCachedURL := targetURL != ""
|
||||
if targetURL == "" {
|
||||
targetURL = h.endpoint
|
||||
}
|
||||
h.endpointMu.RUnlock()
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, targetURL, bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
@@ -56,29 +87,88 @@ func (h *httpClient) sendMessage(message *webhookMessage) error {
|
||||
req.Header.Set("Authorization", "Bearer "+h.token)
|
||||
}
|
||||
|
||||
// Apply timeout via context (not on client) to avoid redundancy
|
||||
if h.timeout > 0 {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), h.timeout)
|
||||
defer cancel()
|
||||
req = req.WithContext(ctx)
|
||||
}
|
||||
|
||||
resp, err := util_http.Do(req)
|
||||
resp, err := h.client.Do(req)
|
||||
if err != nil {
|
||||
if err = drainResponse(resp); err != nil {
|
||||
glog.Errorf("failed to drain response: %v", err)
|
||||
if drainErr := drainResponse(resp); drainErr != nil {
|
||||
glog.Errorf("failed to drain response: %v", drainErr)
|
||||
}
|
||||
|
||||
// If using cached URL and request failed, clear cache and retry with original endpoint
|
||||
if usingCachedURL && depth == 0 {
|
||||
glog.V(1).Infof("Webhook request to cached URL %s failed, clearing cache and retrying with original endpoint", targetURL)
|
||||
h.setFinalURL("")
|
||||
return h.sendMessageWithRetry(message, depth+1)
|
||||
}
|
||||
|
||||
return fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Handle redirects by caching the final destination and recreating POST request
|
||||
if resp.StatusCode >= 300 && resp.StatusCode < 400 {
|
||||
// Drain and close response body to enable connection reuse
|
||||
util_http.CloseResponse(resp)
|
||||
|
||||
location := resp.Header.Get("Location")
|
||||
if location == "" {
|
||||
return fmt.Errorf("webhook returned redirect status %d without Location header", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Resolve relative URLs against the request URL
|
||||
reqURL := req.URL
|
||||
finalURL, err := reqURL.Parse(location)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse redirect location: %w", err)
|
||||
}
|
||||
|
||||
// Update finalURL to follow the redirect for this attempt
|
||||
finalURLStr := finalURL.String()
|
||||
h.setFinalURL(finalURLStr)
|
||||
|
||||
if depth == 0 {
|
||||
glog.V(1).Infof("Webhook endpoint redirected from %s to %s, caching final destination", targetURL, finalURLStr)
|
||||
} else {
|
||||
glog.V(1).Infof("Webhook endpoint redirected from %s to %s (following redirect on retry)", targetURL, finalURLStr)
|
||||
}
|
||||
|
||||
// Recreate the POST request to the final destination (increment depth to prevent infinite loops)
|
||||
return h.sendMessageWithRetry(message, depth+1)
|
||||
}
|
||||
|
||||
// If using cached URL and got an error response, clear cache and retry with original endpoint
|
||||
if resp.StatusCode >= 400 && usingCachedURL && depth == 0 {
|
||||
// Drain and close response body to enable connection reuse
|
||||
util_http.CloseResponse(resp)
|
||||
|
||||
glog.V(1).Infof("Webhook request to cached URL %s returned error %d, clearing cache and retrying with original endpoint", targetURL, resp.StatusCode)
|
||||
h.setFinalURL("")
|
||||
return h.sendMessageWithRetry(message, depth+1)
|
||||
}
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
// Drain and close response body before returning error to enable connection reuse
|
||||
util_http.CloseResponse(resp)
|
||||
return fmt.Errorf("webhook returned status code: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Drain and close response body on success to enable connection reuse
|
||||
util_http.CloseResponse(resp)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *httpClient) setFinalURL(url string) {
|
||||
h.endpointMu.Lock()
|
||||
h.finalURL = url
|
||||
h.endpointMu.Unlock()
|
||||
}
|
||||
|
||||
func drainResponse(resp *http.Response) error {
|
||||
if resp == nil || resp.Body == nil {
|
||||
return nil
|
||||
|
||||
@@ -148,3 +148,375 @@ func TestHttpClientSendMessageNetworkError(t *testing.T) {
|
||||
t.Error("Expected error for network failure")
|
||||
}
|
||||
}
|
||||
|
||||
// TestHttpClientFollowsRedirectAsPost verifies that redirects are followed with POST method preserved
|
||||
func TestHttpClientFollowsRedirectAsPost(t *testing.T) {
|
||||
redirectCalled := false
|
||||
finalCalled := false
|
||||
var finalMethod string
|
||||
var finalBody map[string]interface{}
|
||||
|
||||
// Create final destination server
|
||||
finalServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
finalCalled = true
|
||||
finalMethod = r.Method
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
json.Unmarshal(body, &finalBody)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer finalServer.Close()
|
||||
|
||||
// Create redirect server
|
||||
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
redirectCalled = true
|
||||
// Return 301 redirect to final server
|
||||
http.Redirect(w, r, finalServer.URL, http.StatusMovedPermanently)
|
||||
}))
|
||||
defer redirectServer.Close()
|
||||
|
||||
cfg := &config{
|
||||
endpoint: redirectServer.URL,
|
||||
authBearerToken: "test-token",
|
||||
timeoutSeconds: 5,
|
||||
}
|
||||
|
||||
client, err := newHTTPClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create HTTP client: %v", err)
|
||||
}
|
||||
|
||||
message := &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{
|
||||
Name: "test.txt",
|
||||
},
|
||||
}
|
||||
|
||||
// Send message - should follow redirect and recreate POST request
|
||||
err = client.sendMessage(newWebhookMessage("/test/path", message))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to send message: %v", err)
|
||||
}
|
||||
|
||||
if !redirectCalled {
|
||||
t.Error("Expected redirect server to be called")
|
||||
}
|
||||
|
||||
if !finalCalled {
|
||||
t.Error("Expected final server to be called after redirect")
|
||||
}
|
||||
|
||||
if finalMethod != "POST" {
|
||||
t.Errorf("Expected POST method at final destination, got %s", finalMethod)
|
||||
}
|
||||
|
||||
if finalBody["key"] != "/test/path" {
|
||||
t.Errorf("Expected key '/test/path' at final destination, got %v", finalBody["key"])
|
||||
}
|
||||
|
||||
// Verify the final URL is cached
|
||||
client.endpointMu.RLock()
|
||||
cachedURL := client.finalURL
|
||||
client.endpointMu.RUnlock()
|
||||
|
||||
if cachedURL != finalServer.URL {
|
||||
t.Errorf("Expected cached URL %s, got %s", finalServer.URL, cachedURL)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHttpClientUsesCachedRedirect verifies that subsequent requests use the cached redirect destination
|
||||
func TestHttpClientUsesCachedRedirect(t *testing.T) {
|
||||
redirectCount := 0
|
||||
finalCount := 0
|
||||
|
||||
// Create final destination server
|
||||
finalServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
finalCount++
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer finalServer.Close()
|
||||
|
||||
// Create redirect server
|
||||
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
redirectCount++
|
||||
http.Redirect(w, r, finalServer.URL, http.StatusMovedPermanently)
|
||||
}))
|
||||
defer redirectServer.Close()
|
||||
|
||||
cfg := &config{
|
||||
endpoint: redirectServer.URL,
|
||||
authBearerToken: "test-token",
|
||||
timeoutSeconds: 5,
|
||||
}
|
||||
|
||||
client, err := newHTTPClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create HTTP client: %v", err)
|
||||
}
|
||||
|
||||
message := &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{
|
||||
Name: "test.txt",
|
||||
},
|
||||
}
|
||||
|
||||
// First request - should hit redirect server
|
||||
err = client.sendMessage(newWebhookMessage("/test/path1", message))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to send first message: %v", err)
|
||||
}
|
||||
|
||||
if redirectCount != 1 {
|
||||
t.Errorf("Expected 1 redirect call, got %d", redirectCount)
|
||||
}
|
||||
if finalCount != 1 {
|
||||
t.Errorf("Expected 1 final call, got %d", finalCount)
|
||||
}
|
||||
|
||||
// Second request - should use cached URL and skip redirect server
|
||||
err = client.sendMessage(newWebhookMessage("/test/path2", message))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to send second message: %v", err)
|
||||
}
|
||||
|
||||
if redirectCount != 1 {
|
||||
t.Errorf("Expected redirect server to be called only once (cached), got %d calls", redirectCount)
|
||||
}
|
||||
if finalCount != 2 {
|
||||
t.Errorf("Expected 2 final calls, got %d", finalCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHttpClientPreservesPostMethod verifies POST method is preserved and not converted to GET
|
||||
func TestHttpClientPreservesPostMethod(t *testing.T) {
|
||||
var receivedMethod string
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
receivedMethod = r.Method
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
cfg := &config{
|
||||
endpoint: server.URL,
|
||||
authBearerToken: "test-token",
|
||||
timeoutSeconds: 5,
|
||||
}
|
||||
|
||||
client, err := newHTTPClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create HTTP client: %v", err)
|
||||
}
|
||||
|
||||
message := &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{
|
||||
Name: "test.txt",
|
||||
},
|
||||
}
|
||||
|
||||
err = client.sendMessage(newWebhookMessage("/test/path", message))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to send message: %v", err)
|
||||
}
|
||||
|
||||
if receivedMethod != "POST" {
|
||||
t.Errorf("Expected POST method, got %s", receivedMethod)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHttpClientInvalidatesCacheOnError verifies that cache is invalidated when cached URL fails
|
||||
func TestHttpClientInvalidatesCacheOnError(t *testing.T) {
|
||||
finalServerDown := false // Start with server UP
|
||||
originalCallCount := 0
|
||||
finalCallCount := 0
|
||||
|
||||
// Create final destination server that can be toggled
|
||||
finalServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
finalCallCount++
|
||||
if finalServerDown {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
} else {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
}))
|
||||
defer finalServer.Close()
|
||||
|
||||
// Create redirect server
|
||||
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
originalCallCount++
|
||||
http.Redirect(w, r, finalServer.URL, http.StatusMovedPermanently)
|
||||
}))
|
||||
defer redirectServer.Close()
|
||||
|
||||
cfg := &config{
|
||||
endpoint: redirectServer.URL,
|
||||
authBearerToken: "test-token",
|
||||
timeoutSeconds: 5,
|
||||
}
|
||||
|
||||
client, err := newHTTPClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create HTTP client: %v", err)
|
||||
}
|
||||
|
||||
message := &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{
|
||||
Name: "test.txt",
|
||||
},
|
||||
}
|
||||
|
||||
// First request - should follow redirect and cache the final URL
|
||||
err = client.sendMessage(newWebhookMessage("/test/path1", message))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to send first message: %v", err)
|
||||
}
|
||||
|
||||
if originalCallCount != 1 {
|
||||
t.Errorf("Expected 1 original call, got %d", originalCallCount)
|
||||
}
|
||||
if finalCallCount != 1 {
|
||||
t.Errorf("Expected 1 final call, got %d", finalCallCount)
|
||||
}
|
||||
|
||||
// Verify cache was set
|
||||
client.endpointMu.RLock()
|
||||
cachedURL := client.finalURL
|
||||
client.endpointMu.RUnlock()
|
||||
if cachedURL != finalServer.URL {
|
||||
t.Errorf("Expected cached URL %s, got %s", finalServer.URL, cachedURL)
|
||||
}
|
||||
|
||||
// Second request with cached URL working - should use cache
|
||||
err = client.sendMessage(newWebhookMessage("/test/path2", message))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to send second message: %v", err)
|
||||
}
|
||||
|
||||
if originalCallCount != 1 {
|
||||
t.Errorf("Expected still 1 original call (using cache), got %d", originalCallCount)
|
||||
}
|
||||
if finalCallCount != 2 {
|
||||
t.Errorf("Expected 2 final calls, got %d", finalCallCount)
|
||||
}
|
||||
|
||||
// Third request - bring final server DOWN, should invalidate cache and retry with original
|
||||
// Flow: cached URL (fail, depth=0) -> clear cache -> retry original (depth=1) -> redirect -> final (fail, depth=2)
|
||||
finalServerDown = true
|
||||
err = client.sendMessage(newWebhookMessage("/test/path3", message))
|
||||
if err == nil {
|
||||
t.Error("Expected error when cached URL fails and retry also fails")
|
||||
}
|
||||
|
||||
// originalCallCount: 1 (initial) + 1 (retry after cache invalidation) = 2
|
||||
if originalCallCount != 2 {
|
||||
t.Errorf("Expected 2 original calls, got %d", originalCallCount)
|
||||
}
|
||||
// finalCallCount: 2 (previous) + 1 (cached fail) + 1 (retry after redirect) = 4
|
||||
if finalCallCount != 4 {
|
||||
t.Errorf("Expected 4 final calls, got %d", finalCallCount)
|
||||
}
|
||||
|
||||
// Verify final URL is still set (to the failed destination from the redirect)
|
||||
client.endpointMu.RLock()
|
||||
finalURLAfterError := client.finalURL
|
||||
client.endpointMu.RUnlock()
|
||||
if finalURLAfterError != finalServer.URL {
|
||||
t.Errorf("Expected finalURL to be %s after error, got %s", finalServer.URL, finalURLAfterError)
|
||||
}
|
||||
|
||||
// Fourth request - bring final server back UP
|
||||
// Since cache still has the final URL, it should use it directly
|
||||
finalServerDown = false
|
||||
err = client.sendMessage(newWebhookMessage("/test/path4", message))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to send fourth message after recovery: %v", err)
|
||||
}
|
||||
|
||||
// Should have used the cached URL directly (no new original call)
|
||||
// originalCallCount: still 2
|
||||
if originalCallCount != 2 {
|
||||
t.Errorf("Expected 2 original calls (using cache), got %d", originalCallCount)
|
||||
}
|
||||
// finalCallCount: 4 + 1 = 5
|
||||
if finalCallCount != 5 {
|
||||
t.Errorf("Expected 5 final calls, got %d", finalCallCount)
|
||||
}
|
||||
|
||||
// Verify cache was re-established
|
||||
client.endpointMu.RLock()
|
||||
reestablishedCache := client.finalURL
|
||||
client.endpointMu.RUnlock()
|
||||
if reestablishedCache != finalServer.URL {
|
||||
t.Errorf("Expected cache to be re-established to %s, got %s", finalServer.URL, reestablishedCache)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHttpClientInvalidatesCacheOnNetworkError verifies cache invalidation on network errors
|
||||
func TestHttpClientInvalidatesCacheOnNetworkError(t *testing.T) {
|
||||
originalCallCount := 0
|
||||
var finalServer *httptest.Server
|
||||
// Create redirect server
|
||||
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
originalCallCount++
|
||||
if finalServer != nil {
|
||||
http.Redirect(w, r, finalServer.URL, http.StatusMovedPermanently)
|
||||
} else {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}
|
||||
}))
|
||||
defer redirectServer.Close()
|
||||
|
||||
// Create final destination server
|
||||
finalServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
|
||||
cfg := &config{
|
||||
endpoint: redirectServer.URL,
|
||||
authBearerToken: "test-token",
|
||||
timeoutSeconds: 5,
|
||||
}
|
||||
|
||||
client, err := newHTTPClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create HTTP client: %v", err)
|
||||
}
|
||||
|
||||
message := &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{
|
||||
Name: "test.txt",
|
||||
},
|
||||
}
|
||||
|
||||
// First request - establish cache
|
||||
err = client.sendMessage(newWebhookMessage("/test/path1", message))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to send first message: %v", err)
|
||||
}
|
||||
|
||||
if originalCallCount != 1 {
|
||||
t.Errorf("Expected 1 original call, got %d", originalCallCount)
|
||||
}
|
||||
|
||||
// Close final server to simulate network error
|
||||
cachedURL := finalServer.URL
|
||||
finalServer.Close()
|
||||
finalServer = nil
|
||||
|
||||
// Second request - cached URL is down, should invalidate and retry with original
|
||||
err = client.sendMessage(newWebhookMessage("/test/path2", message))
|
||||
if err == nil {
|
||||
t.Error("Expected error when network fails")
|
||||
}
|
||||
|
||||
if originalCallCount != 2 {
|
||||
t.Errorf("Expected 2 original calls (retry after cache invalidation), got %d", originalCallCount)
|
||||
}
|
||||
|
||||
// Verify cache was cleared
|
||||
client.endpointMu.RLock()
|
||||
clearedCache := client.finalURL
|
||||
client.endpointMu.RUnlock()
|
||||
if clearedCache == cachedURL {
|
||||
t.Errorf("Expected cache to be invalidated, but still has %s", clearedCache)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,6 +31,9 @@ type Queue struct {
|
||||
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
|
||||
// Semaphore for controlling concurrent webhook requests
|
||||
sem chan struct{}
|
||||
}
|
||||
|
||||
func (w *Queue) GetName() string {
|
||||
@@ -89,6 +92,9 @@ func (w *Queue) initialize(cfg *config) error {
|
||||
w.config = cfg
|
||||
w.filter = newFilter(cfg)
|
||||
|
||||
// Initialize semaphore for controlling concurrent webhook requests
|
||||
w.sem = make(chan struct{}, cfg.nWorkers)
|
||||
|
||||
hClient, err := newHTTPClient(cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create webhook http client: %w", err)
|
||||
@@ -141,14 +147,17 @@ func (w *Queue) setupWatermillQueue(cfg *config) error {
|
||||
router.AddPlugin(plugin.SignalsHandler)
|
||||
router.AddMiddleware(retryMiddleware, poisonQueue)
|
||||
|
||||
for i := 0; i < cfg.nWorkers; i++ {
|
||||
router.AddNoPublisherHandler(
|
||||
pubSubHandlerNameTemplate(i),
|
||||
pubSubTopicName,
|
||||
w.queueChannel,
|
||||
w.handleWebhook,
|
||||
)
|
||||
}
|
||||
// Add a single handler to avoid duplicate message delivery.
|
||||
// With gochannel's default behavior, each handler call creates
|
||||
// a separate subscription, and all subscriptions receive their own copy of each message.
|
||||
// Using a single handler ensures each webhook is sent only once.
|
||||
// Concurrency is controlled via semaphore in handleWebhook based on nWorkers config.
|
||||
router.AddConsumerHandler(
|
||||
"webhook_handler",
|
||||
pubSubTopicName,
|
||||
w.queueChannel,
|
||||
w.handleWebhook,
|
||||
)
|
||||
|
||||
go func() {
|
||||
// cancels the queue context so the dead letter logger exists in case context not canceled by the shutdown signal already
|
||||
@@ -165,6 +174,10 @@ func (w *Queue) setupWatermillQueue(cfg *config) error {
|
||||
}
|
||||
|
||||
func (w *Queue) handleWebhook(msg *message.Message) error {
|
||||
// Acquire semaphore slot (blocks if at capacity)
|
||||
w.sem <- struct{}{}
|
||||
defer func() { <-w.sem }()
|
||||
|
||||
var n filer_pb.EventNotification
|
||||
if err := proto.Unmarshal(msg.Payload, &n); err != nil {
|
||||
glog.Errorf("failed to unmarshal protobuf message: %v", err)
|
||||
|
||||
@@ -273,6 +273,7 @@ func TestQueueHandleWebhook(t *testing.T) {
|
||||
client, _ := newHTTPClient(cfg)
|
||||
q := &Queue{
|
||||
client: client,
|
||||
sem: make(chan struct{}, cfg.nWorkers),
|
||||
}
|
||||
|
||||
message := newWebhookMessage("/test/path", &filer_pb.EventNotification{
|
||||
@@ -386,6 +387,78 @@ func TestQueueRetryMechanism(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestQueueNoDuplicateWebhooks verifies that webhooks are sent only once regardless of worker count
|
||||
func TestQueueNoDuplicateWebhooks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
nWorkers int
|
||||
expected int
|
||||
}{
|
||||
{"1 worker", 1, 1},
|
||||
{"5 workers", 5, 1},
|
||||
{"10 workers", 10, 1},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
callCount := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
callCount++
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
cfg := &config{
|
||||
endpoint: server.URL,
|
||||
authBearerToken: "test-token",
|
||||
timeoutSeconds: 5,
|
||||
maxRetries: 0,
|
||||
backoffSeconds: 1,
|
||||
maxBackoffSeconds: 1,
|
||||
nWorkers: tt.nWorkers,
|
||||
bufferSize: 10,
|
||||
}
|
||||
|
||||
q := &Queue{}
|
||||
err := q.initialize(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to initialize queue: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if q.cancel != nil {
|
||||
q.cancel()
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if q.router != nil {
|
||||
q.router.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
// Wait for router and subscriber to be fully ready
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
msg := &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{
|
||||
Name: "test.txt",
|
||||
},
|
||||
}
|
||||
|
||||
err = q.SendMessage("/test/path", msg)
|
||||
if err != nil {
|
||||
t.Errorf("SendMessage() error = %v", err)
|
||||
}
|
||||
|
||||
// Wait for message processing
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
if callCount != tt.expected {
|
||||
t.Errorf("Expected %d webhook call(s), got %d with %d workers", tt.expected, callCount, tt.nWorkers)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueSendMessageWithFilter(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -582,8 +582,10 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action)
|
||||
// - No policy or indeterminate → fall through to IAM checks
|
||||
if iam.policyEngine != nil && bucket != "" {
|
||||
principal := buildPrincipalARN(identity)
|
||||
// Use context-aware policy evaluation to get the correct S3 action
|
||||
allowed, evaluated, err := iam.policyEngine.EvaluatePolicyWithContext(bucket, object, string(action), principal, r)
|
||||
// Phase 1: Evaluate bucket policy without object entry.
|
||||
// Tag-based conditions (s3:ExistingObjectTag) are re-checked by handlers
|
||||
// after fetching the entry, which is the Phase 2 check.
|
||||
allowed, evaluated, err := iam.policyEngine.EvaluatePolicy(bucket, object, string(action), principal, r, nil)
|
||||
|
||||
if err != nil {
|
||||
// SECURITY: Fail-close on policy evaluation errors
|
||||
|
||||
@@ -135,8 +135,70 @@ Standard AWS condition keys are supported:
|
||||
- `aws:UserAgent` - Client user agent
|
||||
- `s3:x-amz-acl` - Requested ACL
|
||||
- `s3:VersionId` - Object version ID
|
||||
- `s3:ExistingObjectTag/<tag-key>` - Value of an existing object tag (see example below)
|
||||
- And many more...
|
||||
|
||||
### 5. Object Tag-Based Access Control
|
||||
|
||||
You can control access based on object tags using `s3:ExistingObjectTag/<tag-key>`:
|
||||
|
||||
```json
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::my-bucket/*",
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:ExistingObjectTag/status": ["public"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
This allows anonymous access only to objects that have a tag `status=public`.
|
||||
|
||||
**Deny access to confidential objects:**
|
||||
|
||||
```json
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::my-bucket/*"
|
||||
},
|
||||
{
|
||||
"Effect": "Deny",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::my-bucket/*",
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:ExistingObjectTag/classification": ["confidential", "secret"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Supported Operations for Tag-Based Conditions:**
|
||||
|
||||
Tag-based conditions (`s3:ExistingObjectTag/<key>`) are evaluated for the following operations:
|
||||
- `s3:GetObject` (GET object)
|
||||
- `s3:GetObjectVersion` (GET object with versionId)
|
||||
- `s3:HeadObject` (HEAD object)
|
||||
|
||||
Note: For these conditions to be evaluated, the object must exist and the policy engine re-checks access after fetching the object metadata.
|
||||
|
||||
## Policy Evaluation
|
||||
|
||||
### Evaluation Order (AWS-Compatible)
|
||||
@@ -270,10 +332,39 @@ go test -v -run TestPolicyValidation
|
||||
|
||||
## Compatibility
|
||||
|
||||
- ✅ **Full backward compatibility** with existing `identities.json`
|
||||
- ✅ **AWS S3 API compatibility** for bucket policies
|
||||
- ✅ **Standard condition operators** and keys
|
||||
- ✅ **Proper evaluation precedence** (Deny > Allow > Default Deny)
|
||||
- ✅ **Performance optimized** with caching and compiled patterns
|
||||
- Full backward compatibility with existing `identities.json`
|
||||
- AWS S3 API compatibility for bucket policies
|
||||
- Standard condition operators and keys
|
||||
- Proper evaluation precedence (Deny > Allow > Default Deny)
|
||||
- Performance optimized with caching and compiled patterns
|
||||
|
||||
The policy engine provides a seamless upgrade path from SeaweedFS's existing simple IAM system to full AWS S3-compatible policies, giving you the best of both worlds: simplicity for basic use cases and power for complex enterprise scenarios.
|
||||
The policy engine provides a seamless upgrade path from SeaweedFS's existing simple IAM system to full AWS S3-compatible policies, giving you the best of both worlds: simplicity for basic use cases and power for complex enterprise scenarios.
|
||||
|
||||
## Feature Status
|
||||
|
||||
### Implemented
|
||||
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| Bucket Policies | Full AWS S3-compatible bucket policies |
|
||||
| Condition Operators | StringEquals, IpAddress, Bool, DateGreaterThan, etc. |
|
||||
| `aws:SourceIp` | IP-based access control with CIDR support |
|
||||
| `aws:SecureTransport` | Require HTTPS |
|
||||
| `aws:CurrentTime` | Time-based access control |
|
||||
| `s3:ExistingObjectTag/<key>` | Tag-based access control for existing objects |
|
||||
| Wildcard Patterns | Support for `*` and `?` in actions and resources |
|
||||
| Principal Matching | `*`, account IDs, and user ARNs |
|
||||
|
||||
### Planned
|
||||
|
||||
| Feature | GitHub Issue |
|
||||
|---------|--------------|
|
||||
| `s3:RequestObjectTag/<key>` | For tag conditions on PUT requests |
|
||||
| `s3:RequestObjectTagKeys` | Check which tag keys are in request |
|
||||
| `s3:x-amz-content-sha256` | Content hash condition |
|
||||
| `s3:x-amz-server-side-encryption` | SSE condition |
|
||||
| `s3:x-amz-storage-class` | Storage class condition |
|
||||
| Cross-account access | Access across different accounts |
|
||||
| VPC Endpoint policies | Network-level policies |
|
||||
|
||||
For feature requests or to track progress, see the [GitHub Issues](https://github.com/seaweedfs/seaweedfs/issues).
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
)
|
||||
|
||||
// LRUNode represents a node in the doubly-linked list for efficient LRU operations
|
||||
@@ -705,8 +706,33 @@ func GetConditionEvaluator(operator string) (ConditionEvaluator, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// ExistingObjectTagPrefix is the prefix for S3 policy condition keys
|
||||
const ExistingObjectTagPrefix = "s3:ExistingObjectTag/"
|
||||
|
||||
// getConditionContextValue resolves the value(s) for a condition key.
|
||||
// For s3:ExistingObjectTag/<key> conditions, it looks up the tag in objectEntry.
|
||||
// For other condition keys, it looks up the value in contextValues.
|
||||
func getConditionContextValue(key string, contextValues map[string][]string, objectEntry map[string][]byte) []string {
|
||||
if strings.HasPrefix(key, ExistingObjectTagPrefix) {
|
||||
tagKey := key[len(ExistingObjectTagPrefix):]
|
||||
metadataKey := s3_constants.AmzObjectTaggingPrefix + tagKey
|
||||
if objectEntry != nil {
|
||||
if tagValue, exists := objectEntry[metadataKey]; exists {
|
||||
return []string{string(tagValue)}
|
||||
}
|
||||
}
|
||||
return []string{}
|
||||
}
|
||||
|
||||
if vals, exists := contextValues[key]; exists {
|
||||
return vals
|
||||
}
|
||||
return []string{}
|
||||
}
|
||||
|
||||
// EvaluateConditions evaluates all conditions in a policy statement
|
||||
func EvaluateConditions(conditions PolicyConditions, contextValues map[string][]string) bool {
|
||||
// objectEntry is the object's metadata from entry.Extended (can be nil)
|
||||
func EvaluateConditions(conditions PolicyConditions, contextValues map[string][]string, objectEntry map[string][]byte) bool {
|
||||
if len(conditions) == 0 {
|
||||
return true // No conditions means always true
|
||||
}
|
||||
@@ -719,11 +745,7 @@ func EvaluateConditions(conditions PolicyConditions, contextValues map[string][]
|
||||
}
|
||||
|
||||
for key, value := range conditionMap {
|
||||
contextVals, exists := contextValues[key]
|
||||
if !exists {
|
||||
contextVals = []string{}
|
||||
}
|
||||
|
||||
contextVals := getConditionContextValue(key, contextValues, objectEntry)
|
||||
if !conditionEvaluator.Evaluate(value.Strings(), contextVals) {
|
||||
return false // If any condition fails, the whole condition block fails
|
||||
}
|
||||
@@ -734,7 +756,8 @@ func EvaluateConditions(conditions PolicyConditions, contextValues map[string][]
|
||||
}
|
||||
|
||||
// EvaluateConditionsLegacy evaluates conditions using the old interface{} format for backward compatibility
|
||||
func EvaluateConditionsLegacy(conditions map[string]interface{}, contextValues map[string][]string) bool {
|
||||
// objectEntry is the object's metadata from entry.Extended (can be nil)
|
||||
func EvaluateConditionsLegacy(conditions map[string]interface{}, contextValues map[string][]string, objectEntry map[string][]byte) bool {
|
||||
if len(conditions) == 0 {
|
||||
return true // No conditions means always true
|
||||
}
|
||||
@@ -753,11 +776,7 @@ func EvaluateConditionsLegacy(conditions map[string]interface{}, contextValues m
|
||||
}
|
||||
|
||||
for key, value := range conditionMapTyped {
|
||||
contextVals, exists := contextValues[key]
|
||||
if !exists {
|
||||
contextVals = []string{}
|
||||
}
|
||||
|
||||
contextVals := getConditionContextValue(key, contextValues, objectEntry)
|
||||
if !conditionEvaluator.Evaluate(value, contextVals) {
|
||||
return false // If any condition fails, the whole condition block fails
|
||||
}
|
||||
|
||||
@@ -91,6 +91,14 @@ func (engine *PolicyEngine) DeleteBucketPolicy(bucketName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// HasPolicyForBucket checks if a bucket has a policy configured
|
||||
func (engine *PolicyEngine) HasPolicyForBucket(bucketName string) bool {
|
||||
engine.mutex.RLock()
|
||||
defer engine.mutex.RUnlock()
|
||||
_, exists := engine.contexts[bucketName]
|
||||
return exists
|
||||
}
|
||||
|
||||
// EvaluatePolicy evaluates a policy for the given arguments
|
||||
func (engine *PolicyEngine) EvaluatePolicy(bucketName string, args *PolicyEvaluationArgs) PolicyEvaluationResult {
|
||||
engine.mutex.RLock()
|
||||
@@ -154,7 +162,7 @@ func (engine *PolicyEngine) evaluateStatement(stmt *CompiledStatement, args *Pol
|
||||
|
||||
// Check conditions
|
||||
if len(stmt.Statement.Condition) > 0 {
|
||||
if !EvaluateConditions(stmt.Statement.Condition, args.Conditions) {
|
||||
if !EvaluateConditions(stmt.Statement.Condition, args.Conditions, args.ObjectEntry) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -201,10 +209,8 @@ func ExtractConditionValuesFromRequest(r *http.Request) map[string][]string {
|
||||
values["aws:Referer"] = []string{referer}
|
||||
}
|
||||
|
||||
// S3 object-level conditions
|
||||
if r.Method == "GET" || r.Method == "HEAD" {
|
||||
values["s3:ExistingObjectTag"] = extractObjectTags(r)
|
||||
}
|
||||
// Note: s3:ExistingObjectTag/<key> conditions are evaluated using objectEntry
|
||||
// passed to EvaluatePolicy, not extracted from the request.
|
||||
|
||||
// S3 bucket-level conditions
|
||||
if delimiter := r.URL.Query().Get("delimiter"); delimiter != "" {
|
||||
@@ -243,13 +249,6 @@ func ExtractConditionValuesFromRequest(r *http.Request) map[string][]string {
|
||||
return values
|
||||
}
|
||||
|
||||
// extractObjectTags extracts object tags from request (placeholder implementation)
|
||||
func extractObjectTags(r *http.Request) []string {
|
||||
// This would need to be implemented based on how object tags are stored
|
||||
// For now, return empty slice
|
||||
return []string{}
|
||||
}
|
||||
|
||||
// BuildResourceArn builds an ARN for the given bucket and object
|
||||
func BuildResourceArn(bucketName, objectName string) string {
|
||||
if objectName == "" {
|
||||
@@ -352,15 +351,6 @@ func GetObjectNameFromArn(arn string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// HasPolicyForBucket checks if a bucket has a policy
|
||||
func (engine *PolicyEngine) HasPolicyForBucket(bucketName string) bool {
|
||||
engine.mutex.RLock()
|
||||
defer engine.mutex.RUnlock()
|
||||
|
||||
_, exists := engine.contexts[bucketName]
|
||||
return exists
|
||||
}
|
||||
|
||||
// GetPolicyStatements returns all policy statements for a bucket
|
||||
func (engine *PolicyEngine) GetPolicyStatements(bucketName string) []PolicyStatement {
|
||||
engine.mutex.RLock()
|
||||
|
||||
@@ -5,9 +5,23 @@ import (
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
)
|
||||
|
||||
// tagsToEntry converts a map of tag key-value pairs to the entry.Extended format
|
||||
// used for s3:ExistingObjectTag/<key> condition evaluation
|
||||
func tagsToEntry(tags map[string]string) map[string][]byte {
|
||||
if tags == nil {
|
||||
return nil
|
||||
}
|
||||
entry := make(map[string][]byte)
|
||||
for k, v := range tags {
|
||||
entry[s3_constants.AmzObjectTaggingPrefix+k] = []byte(v)
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
func TestPolicyEngine(t *testing.T) {
|
||||
engine := NewPolicyEngine()
|
||||
|
||||
@@ -714,3 +728,233 @@ type MockLegacyIAM struct{}
|
||||
func (m *MockLegacyIAM) authRequest(r *http.Request, action Action) (Identity, s3err.ErrorCode) {
|
||||
return nil, s3err.ErrNone
|
||||
}
|
||||
|
||||
// TestExistingObjectTagCondition tests s3:ExistingObjectTag/<tag-key> condition support
|
||||
func TestExistingObjectTagCondition(t *testing.T) {
|
||||
engine := NewPolicyEngine()
|
||||
|
||||
// Policy that allows GetObject only for objects with specific tag
|
||||
policyJSON := `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::test-bucket/*",
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:ExistingObjectTag/status": ["public"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}`
|
||||
|
||||
err := engine.SetBucketPolicy("test-bucket", policyJSON)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to set bucket policy: %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
objectTags map[string]string
|
||||
expected PolicyEvaluationResult
|
||||
}{
|
||||
{
|
||||
name: "Matching tag value - should allow",
|
||||
objectTags: map[string]string{"status": "public"},
|
||||
expected: PolicyResultAllow,
|
||||
},
|
||||
{
|
||||
name: "Non-matching tag value - should be indeterminate",
|
||||
objectTags: map[string]string{"status": "private"},
|
||||
expected: PolicyResultIndeterminate,
|
||||
},
|
||||
{
|
||||
name: "Missing tag - should be indeterminate",
|
||||
objectTags: map[string]string{"other": "value"},
|
||||
expected: PolicyResultIndeterminate,
|
||||
},
|
||||
{
|
||||
name: "No tags - should be indeterminate",
|
||||
objectTags: nil,
|
||||
expected: PolicyResultIndeterminate,
|
||||
},
|
||||
{
|
||||
name: "Empty tags - should be indeterminate",
|
||||
objectTags: map[string]string{},
|
||||
expected: PolicyResultIndeterminate,
|
||||
},
|
||||
{
|
||||
name: "Multiple tags with matching one - should allow",
|
||||
objectTags: map[string]string{"status": "public", "owner": "admin"},
|
||||
expected: PolicyResultAllow,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
args := &PolicyEvaluationArgs{
|
||||
Action: "s3:GetObject",
|
||||
Resource: "arn:aws:s3:::test-bucket/test-object",
|
||||
Principal: "*",
|
||||
ObjectEntry: tagsToEntry(tt.objectTags),
|
||||
}
|
||||
|
||||
result := engine.EvaluatePolicy("test-bucket", args)
|
||||
if result != tt.expected {
|
||||
t.Errorf("Expected %v, got %v", tt.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestExistingObjectTagConditionMultipleTags tests policies with multiple tag conditions
|
||||
func TestExistingObjectTagConditionMultipleTags(t *testing.T) {
|
||||
engine := NewPolicyEngine()
|
||||
|
||||
// Policy that requires multiple tag conditions
|
||||
policyJSON := `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::test-bucket/*",
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:ExistingObjectTag/status": ["public"],
|
||||
"s3:ExistingObjectTag/tier": ["free", "premium"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}`
|
||||
|
||||
err := engine.SetBucketPolicy("test-bucket", policyJSON)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to set bucket policy: %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
objectTags map[string]string
|
||||
expected PolicyEvaluationResult
|
||||
}{
|
||||
{
|
||||
name: "Both tags match - should allow",
|
||||
objectTags: map[string]string{"status": "public", "tier": "free"},
|
||||
expected: PolicyResultAllow,
|
||||
},
|
||||
{
|
||||
name: "Both tags match (premium tier) - should allow",
|
||||
objectTags: map[string]string{"status": "public", "tier": "premium"},
|
||||
expected: PolicyResultAllow,
|
||||
},
|
||||
{
|
||||
name: "Only status matches - should be indeterminate",
|
||||
objectTags: map[string]string{"status": "public"},
|
||||
expected: PolicyResultIndeterminate,
|
||||
},
|
||||
{
|
||||
name: "Only tier matches - should be indeterminate",
|
||||
objectTags: map[string]string{"tier": "free"},
|
||||
expected: PolicyResultIndeterminate,
|
||||
},
|
||||
{
|
||||
name: "Neither tag matches - should be indeterminate",
|
||||
objectTags: map[string]string{"status": "private", "tier": "basic"},
|
||||
expected: PolicyResultIndeterminate,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
args := &PolicyEvaluationArgs{
|
||||
Action: "s3:GetObject",
|
||||
Resource: "arn:aws:s3:::test-bucket/test-object",
|
||||
Principal: "*",
|
||||
ObjectEntry: tagsToEntry(tt.objectTags),
|
||||
}
|
||||
|
||||
result := engine.EvaluatePolicy("test-bucket", args)
|
||||
if result != tt.expected {
|
||||
t.Errorf("Expected %v, got %v", tt.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestExistingObjectTagDenyPolicy tests deny policies with tag conditions
|
||||
func TestExistingObjectTagDenyPolicy(t *testing.T) {
|
||||
engine := NewPolicyEngine()
|
||||
|
||||
// Policy that denies access to objects with confidential tag
|
||||
policyJSON := `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::test-bucket/*"
|
||||
},
|
||||
{
|
||||
"Effect": "Deny",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::test-bucket/*",
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:ExistingObjectTag/classification": ["confidential"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}`
|
||||
|
||||
err := engine.SetBucketPolicy("test-bucket", policyJSON)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to set bucket policy: %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
objectTags map[string]string
|
||||
expected PolicyEvaluationResult
|
||||
}{
|
||||
{
|
||||
name: "No tags - allow by default statement",
|
||||
objectTags: nil,
|
||||
expected: PolicyResultAllow,
|
||||
},
|
||||
{
|
||||
name: "Non-confidential tag - allow",
|
||||
objectTags: map[string]string{"classification": "public"},
|
||||
expected: PolicyResultAllow,
|
||||
},
|
||||
{
|
||||
name: "Confidential tag - deny",
|
||||
objectTags: map[string]string{"classification": "confidential"},
|
||||
expected: PolicyResultDeny,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
args := &PolicyEvaluationArgs{
|
||||
Action: "s3:GetObject",
|
||||
Resource: "arn:aws:s3:::test-bucket/test-object",
|
||||
Principal: "*",
|
||||
ObjectEntry: tagsToEntry(tt.objectTags),
|
||||
}
|
||||
|
||||
result := engine.EvaluatePolicy("test-bucket", args)
|
||||
if result != tt.expected {
|
||||
t.Errorf("Expected %v, got %v", tt.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,6 +106,11 @@ type PolicyEvaluationArgs struct {
|
||||
Resource string
|
||||
Principal string
|
||||
Conditions map[string][]string
|
||||
// ObjectEntry is the object's metadata from entry.Extended.
|
||||
// Used for evaluating conditions like s3:ExistingObjectTag/<tag-key>.
|
||||
// Tags are stored with s3_constants.AmzObjectTaggingPrefix (X-Amz-Tagging-) prefix.
|
||||
// Can be nil for bucket-level operations or when object doesn't exist.
|
||||
ObjectEntry map[string][]byte
|
||||
}
|
||||
|
||||
// PolicyCache for caching compiled policies
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestCheckPolicyWithEntry tests the checkPolicyWithEntry helper method
|
||||
// This verifies that tag-based conditions are properly evaluated when
|
||||
// the object entry is available (Phase 2 of policy evaluation)
|
||||
func TestCheckPolicyWithEntry(t *testing.T) {
|
||||
// Create a minimal S3ApiServer with policy engine
|
||||
s3a := &S3ApiServer{
|
||||
policyEngine: NewBucketPolicyEngine(),
|
||||
}
|
||||
|
||||
bucket := "test-bucket"
|
||||
|
||||
// Set up a policy that allows access only to objects with status=public tag
|
||||
policyJSON := `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::test-bucket/*",
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:ExistingObjectTag/status": ["public"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}`
|
||||
|
||||
err := s3a.policyEngine.engine.SetBucketPolicy(bucket, policyJSON)
|
||||
require.NoError(t, err)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
objectEntry map[string][]byte
|
||||
wantErrCode s3err.ErrorCode
|
||||
wantChecked bool
|
||||
description string
|
||||
}{
|
||||
{
|
||||
name: "object with public tag - allow",
|
||||
objectEntry: map[string][]byte{
|
||||
s3_constants.AmzObjectTaggingPrefix + "status": []byte("public"),
|
||||
},
|
||||
wantErrCode: s3err.ErrNone,
|
||||
wantChecked: true,
|
||||
description: "Object with status=public tag should be allowed",
|
||||
},
|
||||
{
|
||||
name: "object with private tag - indeterminate (no matching statement)",
|
||||
objectEntry: map[string][]byte{
|
||||
s3_constants.AmzObjectTaggingPrefix + "status": []byte("private"),
|
||||
},
|
||||
wantErrCode: s3err.ErrNone,
|
||||
wantChecked: false, // Indeterminate = no policy matched
|
||||
description: "Object with status=private should return indeterminate (falls back to IAM)",
|
||||
},
|
||||
{
|
||||
name: "object with no tags - indeterminate (no matching statement)",
|
||||
objectEntry: map[string][]byte{},
|
||||
wantErrCode: s3err.ErrNone,
|
||||
wantChecked: false, // Indeterminate = no policy matched
|
||||
description: "Object without tags should return indeterminate (falls back to IAM)",
|
||||
},
|
||||
{
|
||||
name: "nil entry - indeterminate (condition cannot be evaluated)",
|
||||
objectEntry: nil,
|
||||
wantErrCode: s3err.ErrNone,
|
||||
wantChecked: false, // Indeterminate = no policy matched
|
||||
description: "Nil entry should return indeterminate (falls back to IAM)",
|
||||
},
|
||||
{
|
||||
name: "object with multiple tags including public - allow",
|
||||
objectEntry: map[string][]byte{
|
||||
s3_constants.AmzObjectTaggingPrefix + "status": []byte("public"),
|
||||
s3_constants.AmzObjectTaggingPrefix + "category": []byte("documents"),
|
||||
},
|
||||
wantErrCode: s3err.ErrNone,
|
||||
wantChecked: true,
|
||||
description: "Object with status=public among other tags should be allowed",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/test-bucket/test-object.txt", nil)
|
||||
object := "/test-object.txt"
|
||||
principal := "*"
|
||||
action := string(s3_constants.ACTION_READ)
|
||||
|
||||
errCode, checked := s3a.checkPolicyWithEntry(req, bucket, object, action, principal, tt.objectEntry)
|
||||
|
||||
assert.Equal(t, tt.wantErrCode, errCode, tt.description)
|
||||
assert.Equal(t, tt.wantChecked, checked, "Policy should be evaluated")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestCheckPolicyWithEntryNoPolicyForBucket tests that checkPolicyWithEntry
|
||||
// returns early when there's no policy for the bucket
|
||||
func TestCheckPolicyWithEntryNoPolicyForBucket(t *testing.T) {
|
||||
s3a := &S3ApiServer{
|
||||
policyEngine: NewBucketPolicyEngine(),
|
||||
}
|
||||
|
||||
bucket := "bucket-without-policy"
|
||||
req := httptest.NewRequest(http.MethodGet, "/bucket-without-policy/test.txt", nil)
|
||||
|
||||
objectEntry := map[string][]byte{
|
||||
s3_constants.AmzObjectTaggingPrefix + "status": []byte("private"),
|
||||
}
|
||||
|
||||
errCode, checked := s3a.checkPolicyWithEntry(req, bucket, "/test.txt", string(s3_constants.ACTION_READ), "*", objectEntry)
|
||||
|
||||
assert.Equal(t, s3err.ErrNone, errCode, "No policy should mean no denial")
|
||||
assert.False(t, checked, "Policy should not be evaluated when bucket has no policy")
|
||||
}
|
||||
|
||||
// TestCheckPolicyWithEntryNilPolicyEngine tests that checkPolicyWithEntry
|
||||
// handles nil policy engine gracefully
|
||||
func TestCheckPolicyWithEntryNilPolicyEngine(t *testing.T) {
|
||||
s3a := &S3ApiServer{
|
||||
policyEngine: nil,
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/test-bucket/test.txt", nil)
|
||||
|
||||
errCode, checked := s3a.checkPolicyWithEntry(req, "test-bucket", "/test.txt", string(s3_constants.ACTION_READ), "*", nil)
|
||||
|
||||
assert.Equal(t, s3err.ErrNone, errCode, "Nil policy engine should allow access")
|
||||
assert.False(t, checked, "Policy should not be evaluated when engine is nil")
|
||||
}
|
||||
|
||||
// TestCheckPolicyWithEntryDenyPolicy tests deny policies with tag conditions
|
||||
func TestCheckPolicyWithEntryDenyPolicy(t *testing.T) {
|
||||
s3a := &S3ApiServer{
|
||||
policyEngine: NewBucketPolicyEngine(),
|
||||
}
|
||||
|
||||
bucket := "secure-bucket"
|
||||
|
||||
// Policy that allows all access but denies access to confidential objects
|
||||
policyJSON := `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::secure-bucket/*"
|
||||
},
|
||||
{
|
||||
"Effect": "Deny",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::secure-bucket/*",
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:ExistingObjectTag/classification": ["confidential"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}`
|
||||
|
||||
err := s3a.policyEngine.engine.SetBucketPolicy(bucket, policyJSON)
|
||||
require.NoError(t, err)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
objectEntry map[string][]byte
|
||||
wantErrCode s3err.ErrorCode
|
||||
description string
|
||||
}{
|
||||
{
|
||||
name: "public object - allow",
|
||||
objectEntry: map[string][]byte{
|
||||
s3_constants.AmzObjectTaggingPrefix + "classification": []byte("public"),
|
||||
},
|
||||
wantErrCode: s3err.ErrNone,
|
||||
description: "Object with classification=public should be allowed",
|
||||
},
|
||||
{
|
||||
name: "confidential object - deny",
|
||||
objectEntry: map[string][]byte{
|
||||
s3_constants.AmzObjectTaggingPrefix + "classification": []byte("confidential"),
|
||||
},
|
||||
wantErrCode: s3err.ErrAccessDenied,
|
||||
description: "Object with classification=confidential should be denied by explicit deny",
|
||||
},
|
||||
{
|
||||
name: "no classification tag - allow",
|
||||
objectEntry: map[string][]byte{},
|
||||
wantErrCode: s3err.ErrNone,
|
||||
description: "Object without classification tag should be allowed (deny condition not met)",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/secure-bucket/test.txt", nil)
|
||||
|
||||
errCode, checked := s3a.checkPolicyWithEntry(req, bucket, "/test.txt", string(s3_constants.ACTION_READ), "*", tt.objectEntry)
|
||||
|
||||
assert.Equal(t, tt.wantErrCode, errCode, tt.description)
|
||||
assert.True(t, checked, "Policy should be evaluated")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHasPolicyForBucket tests the HasPolicyForBucket method
|
||||
func TestHasPolicyForBucket(t *testing.T) {
|
||||
engine := policy_engine.NewPolicyEngine()
|
||||
|
||||
// Initially no policy
|
||||
assert.False(t, engine.HasPolicyForBucket("test-bucket"))
|
||||
|
||||
// Set a policy
|
||||
policyJSON := `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::test-bucket/*"
|
||||
}
|
||||
]
|
||||
}`
|
||||
err := engine.SetBucketPolicy("test-bucket", policyJSON)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Now has policy
|
||||
assert.True(t, engine.HasPolicyForBucket("test-bucket"))
|
||||
|
||||
// Other bucket still has no policy
|
||||
assert.False(t, engine.HasPolicyForBucket("other-bucket"))
|
||||
|
||||
// Delete the policy
|
||||
err = engine.DeleteBucketPolicy("test-bucket")
|
||||
require.NoError(t, err)
|
||||
|
||||
// No longer has policy
|
||||
assert.False(t, engine.HasPolicyForBucket("test-bucket"))
|
||||
}
|
||||
|
||||
@@ -765,8 +765,8 @@ func (s3a *S3ApiServer) AuthWithPublicRead(handler http.HandlerFunc, action Acti
|
||||
|
||||
// Check bucket policy for anonymous access using the policy engine
|
||||
principal := "*" // Anonymous principal
|
||||
// Use context-aware policy evaluation to get the correct S3 action
|
||||
allowed, evaluated, err := s3a.policyEngine.EvaluatePolicyWithContext(bucket, object, string(action), principal, r)
|
||||
// Evaluate bucket policy (objectEntry nil - not yet fetched)
|
||||
allowed, evaluated, err := s3a.policyEngine.EvaluatePolicy(bucket, object, string(action), principal, r, nil)
|
||||
if err != nil {
|
||||
// SECURITY: Fail-close on policy evaluation errors
|
||||
// If we can't evaluate the policy, deny access rather than falling through to IAM
|
||||
|
||||
@@ -87,12 +87,27 @@ func (bpe *BucketPolicyEngine) DeleteBucketPolicy(bucket string) error {
|
||||
return bpe.engine.DeleteBucketPolicy(bucket)
|
||||
}
|
||||
|
||||
// HasPolicyForBucket checks if a bucket has a policy configured
|
||||
func (bpe *BucketPolicyEngine) HasPolicyForBucket(bucket string) bool {
|
||||
return bpe.engine.HasPolicyForBucket(bucket)
|
||||
}
|
||||
|
||||
// EvaluatePolicy evaluates whether an action is allowed by bucket policy
|
||||
// Returns: (allowed bool, evaluated bool, error)
|
||||
// - allowed: whether the policy allows the action
|
||||
// - evaluated: whether a policy was found and evaluated (false = no policy exists)
|
||||
// - error: any error during evaluation
|
||||
func (bpe *BucketPolicyEngine) EvaluatePolicy(bucket, object, action, principal string) (allowed bool, evaluated bool, err error) {
|
||||
//
|
||||
// Parameters:
|
||||
// - bucket: the bucket name
|
||||
// - object: the object key (can be empty for bucket-level operations)
|
||||
// - action: the action being performed (e.g., "Read", "Write")
|
||||
// - principal: the principal ARN or identifier
|
||||
// - r: the HTTP request (optional, used for condition evaluation and action resolution)
|
||||
// - objectEntry: the object's metadata from entry.Extended (can be nil at auth time,
|
||||
// should be passed when available for tag-based conditions like s3:ExistingObjectTag)
|
||||
//
|
||||
// Returns:
|
||||
// - allowed: whether the policy allows the action
|
||||
// - evaluated: whether a policy was found and evaluated (false = no policy exists)
|
||||
// - error: any error during evaluation
|
||||
func (bpe *BucketPolicyEngine) EvaluatePolicy(bucket, object, action, principal string, r *http.Request, objectEntry map[string][]byte) (allowed bool, evaluated bool, err error) {
|
||||
// Validate required parameters
|
||||
if bucket == "" {
|
||||
return false, false, fmt.Errorf("bucket cannot be empty")
|
||||
@@ -101,19 +116,22 @@ func (bpe *BucketPolicyEngine) EvaluatePolicy(bucket, object, action, principal
|
||||
return false, false, fmt.Errorf("action cannot be empty")
|
||||
}
|
||||
|
||||
// Convert action to S3 action format using base mapping (no HTTP context available)
|
||||
s3Action := mapBaseActionToS3Format(action)
|
||||
// Convert action to S3 action format
|
||||
// ResolveS3Action handles nil request internally (falls back to mapBaseActionToS3Format)
|
||||
s3Action := ResolveS3Action(r, action, bucket, object)
|
||||
|
||||
// Build resource ARN
|
||||
resource := buildResourceARN(bucket, object)
|
||||
|
||||
glog.V(4).Infof("EvaluatePolicy: bucket=%s, resource=%s, action=%s, principal=%s", bucket, resource, s3Action, principal)
|
||||
glog.V(4).Infof("EvaluatePolicy: bucket=%s, resource=%s, action=%s, principal=%s",
|
||||
bucket, resource, s3Action, principal)
|
||||
|
||||
// Evaluate using the policy engine
|
||||
args := &policy_engine.PolicyEvaluationArgs{
|
||||
Action: s3Action,
|
||||
Resource: resource,
|
||||
Principal: principal,
|
||||
Action: s3Action,
|
||||
Resource: resource,
|
||||
Principal: principal,
|
||||
ObjectEntry: objectEntry,
|
||||
}
|
||||
|
||||
result := bpe.engine.EvaluatePolicy(bucket, args)
|
||||
@@ -133,53 +151,3 @@ func (bpe *BucketPolicyEngine) EvaluatePolicy(bucket, object, action, principal
|
||||
return false, false, fmt.Errorf("unknown policy result: %v", result)
|
||||
}
|
||||
}
|
||||
|
||||
// EvaluatePolicyWithContext evaluates whether an action is allowed by bucket policy using HTTP request context
|
||||
// This version uses the HTTP request to determine the actual S3 action more accurately
|
||||
func (bpe *BucketPolicyEngine) EvaluatePolicyWithContext(bucket, object, action, principal string, r *http.Request) (allowed bool, evaluated bool, err error) {
|
||||
// Validate required parameters
|
||||
if bucket == "" {
|
||||
return false, false, fmt.Errorf("bucket cannot be empty")
|
||||
}
|
||||
if action == "" {
|
||||
return false, false, fmt.Errorf("action cannot be empty")
|
||||
}
|
||||
|
||||
// Convert action to S3 action format using request context
|
||||
// ResolveS3Action handles nil request internally (falls back to mapBaseActionToS3Format)
|
||||
s3Action := ResolveS3Action(r, action, bucket, object)
|
||||
|
||||
// Build resource ARN
|
||||
resource := buildResourceARN(bucket, object)
|
||||
|
||||
glog.V(4).Infof("EvaluatePolicyWithContext: bucket=%s, resource=%s, action=%s (from %s), principal=%s",
|
||||
bucket, resource, s3Action, action, principal)
|
||||
|
||||
// Evaluate using the policy engine
|
||||
args := &policy_engine.PolicyEvaluationArgs{
|
||||
Action: s3Action,
|
||||
Resource: resource,
|
||||
Principal: principal,
|
||||
}
|
||||
|
||||
result := bpe.engine.EvaluatePolicy(bucket, args)
|
||||
|
||||
switch result {
|
||||
case policy_engine.PolicyResultAllow:
|
||||
glog.V(3).Infof("EvaluatePolicyWithContext: ALLOW - bucket=%s, action=%s, principal=%s", bucket, s3Action, principal)
|
||||
return true, true, nil
|
||||
case policy_engine.PolicyResultDeny:
|
||||
glog.V(3).Infof("EvaluatePolicyWithContext: DENY - bucket=%s, action=%s, principal=%s", bucket, s3Action, principal)
|
||||
return false, true, nil
|
||||
case policy_engine.PolicyResultIndeterminate:
|
||||
// No policy exists for this bucket
|
||||
glog.V(4).Infof("EvaluatePolicyWithContext: INDETERMINATE (no policy) - bucket=%s", bucket)
|
||||
return false, false, nil
|
||||
default:
|
||||
return false, false, fmt.Errorf("unknown policy result: %v", result)
|
||||
}
|
||||
}
|
||||
|
||||
// NOTE: The convertActionToS3Format wrapper has been removed for simplicity.
|
||||
// EvaluatePolicy and EvaluatePolicyWithContext now call ResolveS3Action or
|
||||
// mapBaseActionToS3Format directly, making the control flow more explicit.
|
||||
|
||||
@@ -634,6 +634,19 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
entryFetchTime = time.Since(tEntryFetch)
|
||||
|
||||
// Safety check: entry must be valid before tag-based policy evaluation
|
||||
if objectEntryForSSE == nil {
|
||||
glog.Errorf("GetObjectHandler: objectEntryForSSE is nil for %s/%s (should not happen)", bucket, object)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
|
||||
// Re-check bucket policy with object entry for tag-based conditions (e.g., s3:ExistingObjectTag)
|
||||
if errCode := s3a.recheckPolicyWithObjectEntry(r, bucket, object, string(s3_constants.ACTION_READ), objectEntryForSSE.Extended, "GetObjectHandler"); errCode != s3err.ErrNone {
|
||||
s3err.WriteErrorResponse(w, r, errCode)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if PartNumber query parameter is present (for multipart GET requests)
|
||||
partNumberStr := r.URL.Query().Get("partNumber")
|
||||
if partNumberStr == "" {
|
||||
@@ -2187,6 +2200,12 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
return
|
||||
}
|
||||
|
||||
// Re-check bucket policy with object entry for tag-based conditions (e.g., s3:ExistingObjectTag)
|
||||
if errCode := s3a.recheckPolicyWithObjectEntry(r, bucket, object, string(s3_constants.ACTION_READ), objectEntryForSSE.Extended, "HeadObjectHandler"); errCode != s3err.ErrNone {
|
||||
s3err.WriteErrorResponse(w, r, errCode)
|
||||
return
|
||||
}
|
||||
|
||||
// Implicit Directory Handling for s3fs Compatibility
|
||||
// ====================================================
|
||||
//
|
||||
|
||||
@@ -252,6 +252,62 @@ func (s3a *S3ApiServer) syncBucketPolicyToEngine(bucket string, policyDoc *polic
|
||||
}
|
||||
}
|
||||
|
||||
// checkPolicyWithEntry re-evaluates bucket policy with the object entry metadata.
|
||||
// This is used by handlers after fetching the entry to enforce tag-based conditions
|
||||
// like s3:ExistingObjectTag/<key>.
|
||||
//
|
||||
// Returns:
|
||||
// - s3err.ErrCode: ErrNone if allowed, ErrAccessDenied if denied
|
||||
// - bool: true if policy was evaluated (has policy for bucket), false if no policy
|
||||
func (s3a *S3ApiServer) checkPolicyWithEntry(r *http.Request, bucket, object, action, principal string, objectEntry map[string][]byte) (s3err.ErrorCode, bool) {
|
||||
if s3a.policyEngine == nil {
|
||||
return s3err.ErrNone, false
|
||||
}
|
||||
|
||||
// Skip if no policy for this bucket
|
||||
if !s3a.policyEngine.HasPolicyForBucket(bucket) {
|
||||
return s3err.ErrNone, false
|
||||
}
|
||||
|
||||
allowed, evaluated, err := s3a.policyEngine.EvaluatePolicy(bucket, object, action, principal, r, objectEntry)
|
||||
if err != nil {
|
||||
glog.Errorf("checkPolicyWithEntry: error evaluating policy for %s/%s: %v", bucket, object, err)
|
||||
return s3err.ErrInternalError, true
|
||||
}
|
||||
|
||||
if !evaluated {
|
||||
return s3err.ErrNone, false
|
||||
}
|
||||
|
||||
if !allowed {
|
||||
glog.V(3).Infof("checkPolicyWithEntry: policy denied access to %s/%s for principal %s", bucket, object, principal)
|
||||
return s3err.ErrAccessDenied, true
|
||||
}
|
||||
|
||||
return s3err.ErrNone, true
|
||||
}
|
||||
|
||||
// recheckPolicyWithObjectEntry performs the second phase of policy evaluation after
|
||||
// an object's entry is fetched. It extracts identity from context and checks for
|
||||
// tag-based conditions like s3:ExistingObjectTag/<key>.
|
||||
//
|
||||
// Returns s3err.ErrNone if allowed, or an error code if denied or on error.
|
||||
func (s3a *S3ApiServer) recheckPolicyWithObjectEntry(r *http.Request, bucket, object, action string, objectEntry map[string][]byte, handlerName string) s3err.ErrorCode {
|
||||
identityRaw := GetIdentityFromContext(r)
|
||||
var identity *Identity
|
||||
if identityRaw != nil {
|
||||
var ok bool
|
||||
identity, ok = identityRaw.(*Identity)
|
||||
if !ok {
|
||||
glog.Errorf("%s: unexpected identity type in context for %s/%s", handlerName, bucket, object)
|
||||
return s3err.ErrInternalError
|
||||
}
|
||||
}
|
||||
principal := buildPrincipalARN(identity)
|
||||
errCode, _ := s3a.checkPolicyWithEntry(r, bucket, object, action, principal, objectEntry)
|
||||
return errCode
|
||||
}
|
||||
|
||||
// classifyDomainNames classifies domains into path-style and virtual-host style domains.
|
||||
// A domain is considered path-style if:
|
||||
// 1. It contains a dot (has subdomains)
|
||||
|
||||
@@ -134,6 +134,11 @@ func (vs *VolumeServer) VolumeConfigure(ctx context.Context, req *volume_server_
|
||||
if err := vs.store.ConfigureVolume(needle.VolumeId(req.VolumeId), req.Replication); err != nil {
|
||||
glog.Errorf("volume configure %v: %v", req, err)
|
||||
resp.Error = fmt.Sprintf("volume configure %v: %v", req, err)
|
||||
// Try to re-mount to restore the volume state
|
||||
if mountErr := vs.store.MountVolume(needle.VolumeId(req.VolumeId)); mountErr != nil {
|
||||
glog.Errorf("volume configure failed to restore mount %v: %v", req, mountErr)
|
||||
resp.Error += fmt.Sprintf(". Also failed to restore mount: %v", mountErr)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -124,6 +124,10 @@ func (c *commandEcEncode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(volumeIds) == 0 {
|
||||
fmt.Println("No volumes, nothing to do.")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Collect volume locations BEFORE EC encoding starts to avoid race condition
|
||||
// where the master metadata is updated after EC encoding but before deletion
|
||||
|
||||
@@ -64,6 +64,9 @@ func (n *Needle) ReadNeedleMeta(r backend.BackendStorageFile, offset int64, size
|
||||
dataSize := GetActualSize(size, version)
|
||||
stopOffset := offset + dataSize
|
||||
metaSize := stopOffset - startOffset
|
||||
if metaSize < 0 || metaSize > 128*1024 {
|
||||
return fmt.Errorf("invalid needle meta size %d: DataSize=%d, size=%d, offset=%d", metaSize, n.DataSize, size, offset)
|
||||
}
|
||||
metaSlice := make([]byte, int(metaSize))
|
||||
|
||||
count, err = r.ReadAt(metaSlice, startOffset)
|
||||
|
||||
@@ -17,7 +17,7 @@ makediff test steps
|
||||
"garbageThreshold" for master and option "max" for volume should be set with specific value which would let
|
||||
preparing test prerequisite easier )
|
||||
a) ./weed master -garbageThreshold=0.99 -mdir=./m
|
||||
b) ./weed volume -dir=./data -max=1 -mserver=localhost:9333 -port=8080
|
||||
b) ./weed volume -dir=./data -max=1 -master=localhost:9333 -port=8080
|
||||
2. upload 4 different files, you could call dir/assign to get 4 different fids
|
||||
a) upload file A with fid a
|
||||
b) upload file B with fid b
|
||||
|
||||
@@ -2,6 +2,7 @@ package topology
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
@@ -289,6 +290,8 @@ func (dn *DataNode) GetVolumeIds() string {
|
||||
ids = append(ids, int(k))
|
||||
}
|
||||
|
||||
slices.Sort(ids)
|
||||
|
||||
return util.HumanReadableIntsMax(100, ids...)
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package topology
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
@@ -298,5 +299,7 @@ func (d *Disk) GetVolumeIds() string {
|
||||
ids = append(ids, int(k))
|
||||
}
|
||||
|
||||
slices.Sort(ids)
|
||||
|
||||
return util.HumanReadableIntsMax(100, ids...)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user